def __init__(self,stat,components,components_names,time_domain,time_domain_names,id=None):
# Set the id
if id is None:
self.id=""
else:
self.id=id
# Check components
if not isinstance(components,(list,tuple)):
raise StatisticError,"components argument must be a list"
else:
if isinstance(components_names[0],str):
self.components={}
for i in range(len(components)):
self.components[components[i]]=components_names[i]
elif isinstance(components_names,dict):
self.components=components
else:
raise StatisticError,"components_names argument must be a list"
# Check time_domain
if not isinstance(time_domain,(list,tuple)):
raise StatisticError,"time_domain argument must be a list"
else:
if isinstance(time_domain_names[0],str):
self.time_domain={}
for i in range(len(time_domain)):
self.time_domain[time_domain[i]]=time_domain_names[i]
elif isinstance(time_domain_names,dict):
self.time_domain=time_domain
else:
raise StatisticError,"time_domain_names argument must be a list"
# check stat array
if isinstance (stat,numpy.ndarray ) or numpy.ma.isMA(stat):
s=stat.shape
if len(s)!=2:
raise StatisticError,"stat argument must be 2D"
nc=len(components)
nt=len(time_domain)
if nc!=s[0]:
raise StatisticError,"You claim "+str(nc)+" components but your stat shows:"+str(s[0])
if nt!=s[1]:
raise StatisticError,"You claim "+str(nt)+" time_domain but your stat shows:"+str(s[1])
self.stat=cdms2.createVariable(stat,copy=0,id=str(self.id))
autobounds=cdms2.getAutoBounds()
cdms2.setAutoBounds('off')
compaxis=cdms2.createAxis(components)
compaxis.id='component'
timaxis=cdms2.createAxis(time_domain)
timaxis.id='time_domain'
self.stat.setAxis(0,compaxis)
self.stat.setAxis(1,timaxis)
self.stat.components=repr(self.components)
self.stat.time_domain=repr(self.time_domain)
cdms2.setAutoBounds(autobounds)
else:
raise StatisticError,"stat argument must be A numpy array a MA or a MV2"
def write(self,file,mode='w'):
autobounds=cdms2.getAutoBounds()
cdms2.setAutoBounds('off')
if isinstance(file,str):
f=cdms2.open(file,mode)
elif isinstance(file,cdms2.dataset.CdmsFile):
f=file
else:
raise StatisticError,"write arguments expects string or cdms2 file"
f.write(self.stat)
if not isinstance(file,cdms2.dataset.CdmsFile): f.close()
cdms2.setAutoBounds(autobounds)
def create_metrics(ref, test, ref_regrid, test_regrid, diff):
"""Creates the mean, max, min, rmse, corr in a dictionary"""
orig_bounds = cdms2.getAutoBounds()
cdms2.setAutoBounds(1)
lev = ref.getLevel()
if lev is not None:
lev.setBounds(None)
lev = test.getLevel()
if lev is not None:
lev.setBounds(None)
lev = test_regrid.getLevel()
if lev is not None:
lev.setBounds(None)
lev = ref_regrid.getLevel()
if lev is not None:
lev.setBounds(None)
lev = diff.getLevel()
if lev is not None:
lev.setBounds(None)
cdms2.setAutoBounds(orig_bounds)
metrics_dict = {}
metrics_dict['ref'] = {
'min': min_cdms(ref),
'max': max_cdms(ref),
#'mean': numpy.nan #mean(ref, axis='yz')
'mean': mean(ref, axis='yz')
}
metrics_dict['test'] = {
'min': min_cdms(test),
'max': max_cdms(test),
#'mean': numpy.nan #mean(test, axis='yz')
'mean': mean(test, axis='yz')
}
metrics_dict['diff'] = {
'min': min_cdms(diff),
'max': max_cdms(diff),
#'mean': numpy.nan #mean(diff, axis='yz')
'mean': mean(diff, axis='yz')
}
metrics_dict['misc'] = {
'rmse': rmse(test_regrid, ref_regrid, axis='yz'),
'corr': corr(test_regrid, ref_regrid, axis='yz')
}
return metrics_dict
def create_metrics(ref, test, ref_regrid, test_regrid, diff):
"""
Creates the mean, max, min, rmse, corr in a dictionary.
"""
orig_bounds = cdms2.getAutoBounds()
cdms2.setAutoBounds(1)
lev = ref.getLevel()
if lev is not None:
lev.setBounds(None)
lev = test.getLevel()
if lev is not None:
lev.setBounds(None)
lev = test_regrid.getLevel()
if lev is not None:
lev.setBounds(None)
lev = ref_regrid.getLevel()
if lev is not None:
lev.setBounds(None)
lev = diff.getLevel()
if lev is not None:
lev.setBounds(None)
cdms2.setAutoBounds(orig_bounds)
metrics_dict = {}
metrics_dict["ref"] = {
"min": min_cdms(ref),
"max": max_cdms(ref),
"mean": mean(ref, axis="xz"),
}
metrics_dict["test"] = {
"min": min_cdms(test),
"max": max_cdms(test),
"mean": mean(test, axis="xz"),
}
metrics_dict["diff"] = {
"min": min_cdms(diff),
"max": max_cdms(diff),
"mean": mean(diff, axis="xz"),
}
metrics_dict["misc"] = {
"rmse": rmse(test_regrid, ref_regrid, axis="xz"),
"corr": corr(test_regrid, ref_regrid, axis="xz"),
}
return metrics_dict
def getAxisList(self):
values = []
axes = []
for a in self.json_struct:
values.append(set())
self.get_axes_values_recursive(
0, len(self.json_struct) - 1, self.data, values)
autoBounds = cdms2.getAutoBounds()
cdms2.setAutoBounds("off")
for i, nm in enumerate(self.json_struct):
axes.append(cdms2.createAxis(sorted(list(values[i])), id=nm))
self.axes = axes
cdms2.setAutoBounds(autoBounds)
return self.axes
def compute(dm, do):
""" Computes ANNUAL MEAN"""
if dm is None and do is None: # just want the doc
return {
"Name": "Annual Mean",
"Abstract": "Compute Annual Mean",
"URI": "http://uvcdat.llnl.gov/documentation/" +
"utilities/utilities-2.html",
"Contact": "*****@*****.**",
"Comments": "Assumes input are 12 months climatology"
}
# Do we really want this? Wouldn't it better to let it fails
cdms2.setAutoBounds('on')
return cdutil.averager(dm, axis='t'), cdutil.averager(do, axis='t')
def compute(dm, do):
""" Computes ANNUAL MEAN"""
if dm is None and do is None: # just want the doc
return {
"Name": "Annual Mean",
"Abstract": "Compute Annual Mean",
"URI":
"http://uvcdat.llnl.gov/documentation/" +
"utilities/utilities-2.html",
"Contact": "Peter Gleckler <*****@*****.**>",
"Comments": "Assumes input are 12 months climatology"
}
# Do we really want this? Wouldn't it better to let it fails
cdms2.setAutoBounds('on')
return cdutil.averager(dm, axis='t'), cdutil.averager(do, axis='t')
def executeOperations(self, task, _inputs):
cdms2.setAutoBounds(2)
t0 = time.time()
self.logger.info( " Execute REGRID Task with metadata: " + str( task.metadata ) )
crsSpec = task.metadata.get("crs","")
if( len(crsSpec) and (crsSpec[0] == '~') ):
crsId = crsSpec[1:]
grid_input = _inputs.get( crsId, None )
if not grid_input: raise Exception( "Can't find grid variable uid: " + crsId + ", variable uids = " + str( _inputs.keys() ) )
toGrid = grid_input.getGrid()
else:
crsToks = crsSpec.split('~')
if( len(crsToks) > 1 ):
if crsToks[0] == "gaussian":
resolution = int(crsToks[1])
toGrid = cdms2.createGaussianGrid( resolution )
else: raise Exception( "Unrecognized grid type: " + crsToks[0])
else:
gridSpec = task.metadata.get("gridSpec","")
if not gridSpec: raise Exception( "Can't find crs spec in regrid kernel: " + str(crsToks))
toGrid = getGrid( gridSpec )
results = []
for input_id in task.inputs:
_input = _inputs.get( input_id.split('-')[0] )
variable = _input.getVariable()
ingrid = _input.getGrid()
inlatBounds, inlonBounds = ingrid.getBounds()
self.logger.info( " >> in LAT Bounds shape: " + str(inlatBounds.shape) )
self.logger.info( " >> in LON Bounds shape: " + str(inlonBounds.shape) )
outlatBounds, outlonBounds = toGrid.getBounds()
self.logger.info( " >> out LAT Bounds shape: " + str(outlatBounds.shape) )
self.logger.info( " >> out LON Bounds shape: " + str(outlonBounds.shape) )
if( not ingrid == toGrid ):
self.logger.info( " Regridding Variable {0} using grid {1} ".format( variable.id, str(toGrid) ) )
if self._debug:
self.logger.info( " >> Input Data Sample: [ {0} ]".format( ', '.join( [ str( variable.data.flat[i] ) for i in range(20,90) ] ) ) )
self.logger.info( " >> Input Variable Shape: {0}, Grid Shape: {1} ".format( str(variable.shape), str([len(ingrid.getLatitude()),len(ingrid.getLongitude())] )))
result_var = variable.regrid( toGrid, regridTool="esmf", regridMethod="linear" )
self.logger.info( " >> Gridded Data Sample: [ {0} ]".format( ', '.join( [ str( result_var.data.flat[i] ) for i in range(20,90) ] ) ) )
results.append( self.createResult( result_var, _input, task ) )
t1 = time.time()
self.logger.info(" @RRR@ Completed regrid operation for input variables: {0} in time {1}".format( str(_inputs.keys), (t1 - t0)))
return results
def testYxfilled(self):
f = cdms2.open(os.path.join(cdat_info.get_sampledata_path(), 'clt.nc'))
cdms2.setAutoBounds("on")
s = f("clt")
d1 = MV2.max(s, axis=0)
d2 = MV2.min(s, axis=0)
s1 = cdutil.averager(d1, axis='x')
s2 = cdutil.averager(d2, axis='x')
yf = vcsaddons.createyxvsxfill()
yf.linewidth = 5
yf.linecolor = "red"
yf.fillareacolor = "blue"
self.x.plot(s1, s2, yf)
self.checkImage("test_yxvsxfill.png")
def get_daily_ano_segment(d_seg):
"""
Note: 1. Get daily time series (3D: time and spatial 2D)
2. Meridionally average (2D: time and spatial, i.e., longitude)
3. Get anomaly by removing time mean of the segment
input
- d_seg: cdms2 data
output
- d_seg_x_ano: 2d array
"""
cdms2.setAutoBounds('on')
# sub region
d_seg = d_seg(latitude=(-10, 10))
# Get meridional average (3d (t, y, x) to 2d (t, y))
d_seg_x = cdutil.averager(d_seg, axis='y', weights='weighted')
# Get time-average in the segment on each longitude grid
d_seg_x_ave = cdutil.averager(d_seg_x, axis='t')
# Remove time mean for each segment
d_seg_x_ano = MV2.subtract(d_seg_x, d_seg_x_ave)
return d_seg_x_ano
def process_file(ifile,suffix,average=False,forcedaily=False,mask=True,xlist=[]):
try:
d = cdms2.open(ifile)
except:
print "Error opening file", ifile
usage()
sys.exit(1)
hcrit = 0.5 # Critical value of Heavyside function for inclusion.
ofilelist = []
for vn in d.variables:
var = d.variables[vn]
# Need to check whether it really has a stash_item to skip coordinate variables
# Note: need to match both item and section number
if not hasattr(var,'stash_item'):
continue
item_code = var.stash_section[0]*1000 + var.stash_item[0]
if item_code in xlist:
print "Skipping", item_code
continue
grid = var.getGrid()
time = var.getTime()
timevals = np.array(time[:])
if forcedaily:
# Work around cdms error in times
for k in range(len(time)):
timevals[k] = round(timevals[k],1)
umvar = stashvar.StashVar(item_code,var.stash_model[0])
vname = umvar.name
print vname, var[0,0,0,0]
# Create filename from variable name and cell_methods,
# checking for name collisions
if suffix:
ofile = "%s_%s.nc" % (umvar.uniquename, suffix)
else:
ofile = "%s.nc" % umvar.uniquename
if ofile in ofilelist:
raise Exception("Duplicate file name %s" % ofile)
ofilelist.append(ofile)
# If output file exists then append to it, otherwise create a new file
try:
file = cdms2.openDataset(ofile, 'r+')
newv = file.variables[vname]
newtime = newv.getTime()
except cdms2.error.CDMSError:
file = cdms2.createDataset(ofile)
# Stop it creating the bounds_latitude, bounds_longitude variables
cdms2.setAutoBounds("off")
# By default get names like latitude0, longitude1
# Need this awkwardness to get the variable/dimension name set correctly
# Is there a way to change the name cdms uses after
# newlat = newgrid.getLatitude() ????
newlat = file.createAxis('lat', grid.getLatitude()[:])
newlat.standard_name = "latitude"
newlat.axis = "Y"
newlat.units = 'degrees_north'
newlon = file.createAxis('lon', grid.getLongitude()[:])
newlon.standard_name = "longitude"
newlon.axis = "X"
newlon.units = 'degrees_east'
order = var.getOrder()
if order[1] == 'z':
lev = var.getLevel()
if len(lev) > 1:
newlev = file.createAxis('lev', lev[:])
for attr in ('standard_name', 'units', 'positive', 'axis'):
if hasattr(lev,attr):
setattr(newlev, attr, getattr(lev,attr))
else:
newlev = None
else:
# Pseudo-dimension
pdim = var.getAxis(1)
if len(pdim) > 1:
newlev = file.createAxis('pseudo', pdim[:])
else:
newlev = None
newtime = file.createAxis('time', None, cdms2.Unlimited)
newtime.standard_name = "time"
newtime.units = time.units # "days since " + `baseyear` + "-01-01 00:00"
newtime.setCalendar(time.getCalendar())
newtime.axis = "T"
if var.dtype == np.dtype('int32'):
vtype = cdms2.CdInt
missval = -2147483647
else:
vtype = cdms2.CdFloat
missval = 1.e20
if newlev:
newv = file.createVariable(vname, vtype, (newtime, newlev, newlat, newlon))
else:
newv = file.createVariable(vname, vtype, (newtime, newlat, newlon))
for attr in ("standard_name", "long_name", "units"):
if hasattr(umvar, attr):
newv.setattribute(attr, getattr(umvar,attr))
newv.missing_value = missval
newv.stash_section=var.stash_section[0]
newv.stash_item=var.stash_item[0]
newv._FillValue = missval
try:
newv.units = var.units
except AttributeError:
pass
# Get appropriate file position
# Uses 360 day calendar, all with same base time so must be 30 days on.
k = len(newtime)
# float needed here to get the later logical tests to work properly
avetime = float(MV.average(timevals[:])) # Works in either case
if k>0:
if average:
# if newtime[-1] != (avetime - 30):
# For Gregorian calendar relax this a bit
# Sometimes get differences slightly > 31
if not 28 <= avetime - newtime[-1] <= 31.5:
raise error, "Times not consecutive %f %f %f" % (newtime[-1], avetime, timevals[0])
else:
if k > 1:
# Need a better test that works when k = 1. This is just a
# temporary workaround
if not np.allclose( newtime[-1] + (newtime[-1]-newtime[-2]), timevals[0] ):
raise error, "Times not consecutive %f %f " % (newtime[-1], timevals[0])
if (30201 <= item_code <= 30303) and mask:
# P LEV/UV GRID with missing values treated as zero.
# Needs to be corrected by Heavyside fn
heavyside = d.variables['psag']
# Check variable code as well as the name.
if heavyside.stash_item[0] != 301 or heavyside.stash_section[0] != 30:
raise error, "Heavyside variable code mismatch"
if average:
newtime[k] = avetime
if var.shape[1] > 1:
# multiple levels
newv[k] = MV.average(var[:],axis=0).astype(np.float32)
else:
# single level
newv[k] = MV.average(var[:],axis=0)[0].astype(np.float32)
else:
for i in range(len(timevals)):
if var.shape[1] > 1:
# Multi-level
if (30201 <= item_code <= 30303) and mask:
newv[k+i] = np.where( np.greater(heavyside[i], hcrit), var[i]/heavyside[0], newv.getMissing())
else:
newv[k+i] = var[i]
else:
newv[k+i] = var[i,0]
newtime[k+i] = timevals[i]
file.close()
import os, cdms2, string, sys, pdb, MV2, numpy
from metrics.common import store_provenance
cdms2.setAutoBounds(0)
#example
#python makeSmallerData.py cam35_data T TAUX TAUY OCNFRAC LWCF SWCF FISCCP1 isccp_prs isccp_tau hyam hybm PS P0
old_dir = sys.argv[1]
vars = sys.argv[2:]
suffix = '_smaller'
print old_dir
print vars
new_dir = old_dir + suffix
print new_dir
os.makedirs(new_dir)
fns = os.listdir('./'+old_dir)
print fns
#pdb.set_trace()
for fn in fns:
f = cdms2.open(old_dir + '/' + fn)
g = cdms2.open(new_dir + '/' + fn, 'w')
store_provenance(g)
for key, value in f.attributes.iteritems():
setattr(g, key, value)
def linearInterpolation(A,I,levels=[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000], status=None):
"""
Linear interpolation
to interpolate a field from some levels to another set of levels
Value below "surface" are masked
Input
A : array to interpolate
I : interpolation field (usually Pressure or depth) from TOP (level 0) to BOTTOM (last level), i.e P value going up with each level
levels : levels to interplate to (same units as I), default levels are:[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000]
I and levels must have same units
Output
array on new levels (levels)
Examples:
A=interpolate(A,I,levels=[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000])
"""
try:
nlev=len(levels) # Number of pressure levels
except:
nlev=1 # if only one level len(levels) would breaks
levels=[levels,]
order=A.getOrder()
A=A(order='z...')
I=I(order='z...')
sh=list(I.shape)
nsigma=sh[0] #number of sigma levels
sh[0]=nlev
t=MV2.zeros(sh,typecode=MV2.float32)
sh2=I[0].shape
prev=-1
for ilev in range(nlev): # loop through pressure levels
if status is not None:
prev=genutil.statusbar(ilev,nlev-1.,prev)
lev=levels[ilev] # get value for the level
Iabv=MV2.ones(sh2,MV2.float)
Aabv=-1*Iabv # Array on sigma level Above
Abel=-1*Iabv # Array on sigma level Below
Ibel=-1*Iabv # Pressure on sigma level Below
Iabv=-1*Iabv # Pressure on sigma level Above
Ieq=MV2.masked_equal(Iabv,-1) # Area where Pressure == levels
for i in range(1,nsigma): # loop from second sigma level to last one
a = MV2.greater_equal(I[i], lev) # Where is the pressure greater than lev
b = MV2.less_equal(I[i-1],lev) # Where is the pressure less than lev
# Now looks if the pressure level is in between the 2 sigma levels
# If yes, sets Iabv, Ibel and Aabv, Abel
a=MV2.logical_and(a,b)
Iabv=MV2.where(a,I[i],Iabv) # Pressure on sigma level Above
Aabv=MV2.where(a,A[i],Aabv) # Array on sigma level Above
Ibel=MV2.where(a,I[i-1],Ibel) # Pressure on sigma level Below
Abel=MV2.where(a,A[i-1],Abel) # Array on sigma level Below
Ieq= MV2.where(MV2.equal(I[i],lev),A[i],Ieq)
val=MV2.masked_where(MV2.equal(Ibel,-1.),numpy.ones(Ibel.shape)*lev) # set to missing value if no data below lev if there is
tl=(val-Ibel)/(Iabv-Ibel)*(Aabv-Abel)+Abel # Interpolation
if ((Ieq.mask is None) or (Ieq.mask is MV22.nomask)):
tl=Ieq
else:
tl=MV2.where(1-Ieq.mask,Ieq,tl)
t[ilev]=tl.astype(MV2.float32)
ax=A.getAxisList()
autobnds=cdms2.getAutoBounds()
cdms2.setAutoBounds('off')
lvl=cdms2.createAxis(MV2.array(levels).filled())
cdms2.setAutoBounds(autobnds)
try:
lvl.units=I.units
except:
pass
lvl.id='plev'
try:
t.units=I.units
except:
pass
ax[0]=lvl
t.setAxisList(ax)
t.id=A.id
for att in A.listattributes():
setattr(t,att,getattr(A,att))
return t(order=order)
def executeOperations(self, task, _inputs):
"""
:type task: Task
:type _inputs: dict[str,npArray]
"""
cdms2.setAutoBounds(2)
start = time.time()
mdata = task.metadata
""":type : dict[str,str] """
self.logger.info(" Execute Ensemble Task with metadata: " +
str(task.metadata))
target = str(mdata.get("target", ""))
method = str(mdata.get("method", "linear")).lower()
res = sa2f(self.getListParm(mdata, "res"))
shape = sa2i(self.getListParm(mdata, "shape"))
plevs = sa2f(self.getListParm(mdata, "plev"))
resultDir = task.metadata.get("resultDir")
target_input = _inputs.get(target, None)
if (target_input is None):
raise Exception(
"Must set the 'target' parameter in EnsembleWorkflowKernel to the id of the input that determines the output grid"
)
outgrid = target_input.getGrid()
arrays = []
for (id, _input) in _inputs.iteritems():
dset_address = _input.metadata.get("uri",
_input.metadata.get("dataPath"))
vname = _input.metadata.get("name")
dset = cdms2.open(dset_address)
var = dset(vname, genutil.picker(
plev=plevs)) if (plevs) else dset(vname)
""":type : AbstractVariable """
if (id == target):
lat = var.getLatitude()
""":type : AbstractCoordinateAxis """
lon = var.getLongitude()
""":type : AbstractCoordinateAxis """
plev = var.getLevel()
""":type : AbstractCoordinateAxis """
t = var.getTime()
""":type : AbstractCoordinateAxis """
units = var.getattribute("units")
varname = vname
arrays.append(var)
else:
arrays.append(var.regrid(outgrid))
times = t.asComponentTime()
trange = str(times[0]) + "-" + str(times[-1])
concatenate = ma.masked_array(arrays)
average = cdms2.MV2.average(concatenate, axis=0)
ave = cdms2.createVariable(average, axes=(t, plev, lat, lon))
ave.units = units
ave.id = varname
ave.long_name = 'ensemble average'
outfile = cdms2.open(
resultDir + "/" + varname + '_6hr_reanalysis_ensemble_ave_' +
trange + '.nc', 'w')
outfile.write(ave)
outfile.close()
stdv = genutil.statistics.std(concatenate, axis=0)
stdvave = cdms2.createVariable(stdv, axes=(t, plev, lat, lon))
stdvave.id = varname + '_stdv'
stdvave.long_name = 'standard deviation'
stdvave.units = units
outfile_std = cdms2.open(
resultDir + "/" + varname + '6hr_reanalysis_ensemble_std_' +
trange + '.nc', 'w')
outfile_std.write(stdvave)
outfile_std.close()
#newgrid.shape
end = time.time()
print "Completed EnsembleWorkflowKernel in " + str(
(end - start) / 60.) + " min "
def surfTransf(fileFx,
fileTos,
fileSos,
fileHef,
fileWfo,
varNames,
outFile,
debug=True,
timeint='all',
noInterp=False,
domain='global'):
'''
The surfTransf() function takes files and variable arguments and creates
density bined surface transformation fields which are written to a specified outfile
Author: Eric Guilyardi : [email protected]
Co-author: Paul J. Durack : [email protected] : @durack1.
Created on Wed Oct 8 09:15:59 CEST 2014
Inputs:
------
- fileTos(time,lat,lon) - 3D SST array
- fileSos(time,lat,lon) - 3D SSS array
- fileHef(time,lat,lon) - 3D net surface heat flux array
- fileWfo(time,lat,lon) - 3D fresh water flux array
- fileFx(lat,lon) - 2D array containing the cell area values
- varNames[4] - 1D array containing the names of the variables
- outFile(str) - output file with full path specified.
- debug <optional> - boolean value
- timeint <optional> - specify temporal step for binning <init_idx>,<ncount>
- noInterp <optional> - if true no interpolation to target grid
- domain <optional> - specify domain for averaging when interpolated to WOA grid ('global','north',
'north40', 'south' for now)
Outputs:
--------
- netCDF file with monthly surface rhon, density fluxes, transformation (global and per basin)
- use cdo yearmean to compute annual mean
Usage:
------
'>>> from binDensity import surfTransf
'>>> surfTransf(file_fx, file_tos, file_sos, file_hef, file_wfo, [var1,var2,var3,var4]./output.nc, debug=True,timeint='all')
Notes:
-----
- EG 8 Oct 2014 - Initial function write and tests ok
- PJD 22 Nov 2014 - Code cleanup
- EG 4 Oct 2017 - code on ciclad, more cleanup and options
- EG 12 Sep 2018 - Add North vs. South calculation
'''
# Keep track of time (CPU and elapsed)
cpu0 = timc.clock()
#
# netCDF compression (use 0 for netCDF3)
comp = 1
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
#
# == Inits
#
npy.set_printoptions(precision=2)
# Determine file name from inputs
modeln = fileTos.split('/')[-1].split('.')[1]
#
if debug:
print ' Debug - File names:'
print ' ', fileTos
print ' ', fileSos
debugp = True
else:
debugp = False
#
# Open files
ftos = cdm.open(fileTos)
fsos = cdm.open(fileSos)
fhef = cdm.open(fileHef)
fwfo = cdm.open(fileWfo)
#timeax = ftos.getAxis('time')
timeax = ftos.getAxis('time_counter')
#print 'timeax'
#print timeax
#
# Dates to read
if timeint == 'all':
tmin = 0
tmax = timeax.shape[0]
timeaxis = timeax
else:
tmin = int(timeint.split(',')[0]) - 1
tmax = tmin + int(timeint.split(',')[1])
# update time axis
timeaxis = cdm.createAxis(timeax[tmin:tmax])
timeaxis.id = 'time'
timeaxis.units = timeax.units
timeaxis.designateTime()
#print timeaxis
if debugp:
print
print ' Debug mode'
# Read file attributes to carry on to output files
list_file = ftos.attributes.keys()
file_dic = {}
for i in range(0, len(list_file)):
file_dic[i] = list_file[i], ftos.attributes[list_file[i]]
#
# Read data
# varnames
tos_name = varNames[0]
sos_name = varNames[1]
hef_name = varNames[2]
wfo_name = varNames[3]
if debugp:
print ' Read ', tos_name, sos_name, tmin, tmax
tos = ftos(tos_name, time=slice(tmin, tmax))
sos = fsos(sos_name, time=slice(tmin, tmax))
if debugp:
print ' Read ', hef_name, wfo_name
qnet = fhef(hef_name, time=slice(tmin, tmax))
try:
emp = fwfo(wfo_name, time=slice(tmin, tmax))
empsw = 0
except Exception, err:
emp = fwfo('wfos', time=slice(tmin, tmax))
print ' Reading concentration dillution fresh water flux'
empsw = 0
def logLinearInterpolation(A,
P,
levels=[
100000, 92500, 85000, 70000, 60000, 50000,
40000, 30000, 25000, 20000, 15000, 10000, 7000,
5000, 3000, 2000, 1000
],
status=None):
"""
Log-linear interpolation
to convert a field from sigma levels to pressure levels
Value below surface are masked
Input
A : array on sigma levels
P : pressure field from TOP (level 0) to BOTTOM (last level)
levels : pressure levels to interplate to (same units as P), default levels are:[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000]
P and levels must have same units
Output
array on pressure levels (levels)
Examples:
A=logLinearInterpolation(A,P),levels=[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000])
"""
try:
nlev = len(levels) # Number of pressure levels
except:
nlev = 1 # if only one level len(levels) would breaks
levels = [
levels,
]
order = A.getOrder()
A = A(order='z...')
P = P(order='z...')
sh = list(P.shape)
nsigma = sh[0] #number of sigma levels
sh[0] = nlev
t = MV2.zeros(sh, typecode=MV2.float32)
sh2 = P[0].shape
prev = -1
for ilev in range(nlev): # loop through pressure levels
if status is not None:
prev = genutil.statusbar(ilev, nlev - 1., prev)
lev = levels[ilev] # get value for the level
Pabv = MV2.ones(sh2, MV2.float)
Aabv = -1 * Pabv # Array on sigma level Above
Abel = -1 * Pabv # Array on sigma level Below
Pbel = -1 * Pabv # Pressure on sigma level Below
Pabv = -1 * Pabv # Pressure on sigma level Above
Peq = MV2.masked_equal(Pabv, -1) # Area where Pressure == levels
for i in range(1, nsigma): # loop from second sigma level to last one
a = MV2.greater_equal(
P[i], lev) # Where is the pressure greater than lev
b = MV2.less_equal(P[i - 1],
lev) # Where is the pressure less than lev
# Now looks if the pressure level is in between the 2 sigma levels
# If yes, sets Pabv, Pbel and Aabv, Abel
a = MV2.logical_and(a, b)
Pabv = MV2.where(a, P[i], Pabv) # Pressure on sigma level Above
Aabv = MV2.where(a, A[i], Aabv) # Array on sigma level Above
Pbel = MV2.where(a, P[i - 1],
Pbel) # Pressure on sigma level Below
Abel = MV2.where(a, A[i - 1], Abel) # Array on sigma level Below
Peq = MV2.where(MV2.equal(P[i], lev), A[i], Peq)
val = MV2.masked_where(
MV2.equal(Pbel, -1),
numpy.ones(Pbel.shape) *
lev) # set to missing value if no data below lev if there is
tl = MV2.log(val / Pbel) / MV2.log(
Pabv / Pbel) * (Aabv - Abel) + Abel # Interpolation
if ((Peq.mask is None) or (Peq.mask is MV2.nomask)):
tl = Peq
else:
tl = MV2.where(1 - Peq.mask, Peq, tl)
t[ilev] = tl.astype(MV2.float32)
ax = A.getAxisList()
autobnds = cdms2.getAutoBounds()
cdms2.setAutoBounds('off')
lvl = cdms2.createAxis(MV2.array(levels).filled())
cdms2.setAutoBounds(autobnds)
try:
lvl.units = P.units
except:
pass
lvl.id = 'plev'
try:
t.units = P.units
except:
pass
ax[0] = lvl
t.setAxisList(ax)
t.id = A.id
for att in A.listattributes():
setattr(t, att, getattr(A, att))
return t(order=order)
def compute_metrics(Var, dm, do):
# Var is sometimes sent with level associated
var = Var.split("_")[0]
# Did we send data? Or do we just want the info?
if dm is None and do is None:
metrics_defs = collections.OrderedDict()
metrics_defs["rms_xyt"] = pcmdi_metrics.pcmdi.rms_xyt.compute(
None,
None)
metrics_defs["rms_xy"] = pcmdi_metrics.pcmdi.rms_xy.compute(None, None)
metrics_defs["bias_xy"] = pcmdi_metrics.pcmdi.bias_xy.compute(
None, None)
metrics_defs["mae_xy"] = pcmdi_metrics.pcmdi.meanabs_xy.compute(
None,
None)
# metrics_defs["cor_xyt"] = pcmdi_metrics.pcmdi.cor_xyt.compute(
# None,
# None)
metrics_defs["cor_xy"] = pcmdi_metrics.pcmdi.cor_xy.compute(None, None)
metrics_defs["std_xy"] = pcmdi_metrics.pcmdi.std_xy.compute(None)
metrics_defs["std_xyt"] = pcmdi_metrics.pcmdi.std_xyt.compute(None)
metrics_defs["seasonal_mean"] = \
pcmdi_metrics.pcmdi.seasonal_mean.compute(
None,
None)
metrics_defs["annual_mean"] = pcmdi_metrics.pcmdi.annual_mean.compute(
None,
None)
metrics_defs["zonal_mean"] = pcmdi_metrics.pcmdi.zonal_mean.compute(
None,
None)
return metrics_defs
cdms.setAutoBounds('on')
metrics_dictionary = {}
# SET CONDITIONAL ON INPUT VARIABLE
if var == 'pr':
conv = 86400.
else:
conv = 1.
if var in ['hus']:
sig_digits = '.5f'
else:
sig_digits = '.3f'
# CALCULATE ANNUAL CYCLE SPACE-TIME RMS, CORRELATIONS and STD
rms_xyt = pcmdi_metrics.pcmdi.rms_xyt.compute(dm, do)
# cor_xyt = pcmdi_metrics.pcmdi.cor_xyt.compute(dm, do)
stdObs_xyt = pcmdi_metrics.pcmdi.std_xyt.compute(do)
std_xyt = pcmdi_metrics.pcmdi.std_xyt.compute(dm)
# CALCULATE ANNUAL MEANS
dm_am, do_am = pcmdi_metrics.pcmdi.annual_mean.compute(dm, do)
# CALCULATE ANNUAL MEAN BIAS
bias_xy = pcmdi_metrics.pcmdi.bias_xy.compute(dm_am, do_am)
# CALCULATE MEAN ABSOLUTE ERROR
mae_xy = pcmdi_metrics.pcmdi.meanabs_xy.compute(dm_am, do_am)
# CALCULATE ANNUAL MEAN RMS
rms_xy = pcmdi_metrics.pcmdi.rms_xy.compute(dm_am, do_am)
# CALCULATE ANNUAL MEAN CORRELATION
cor_xy = pcmdi_metrics.pcmdi.cor_xy.compute(dm_am, do_am)
# CALCULATE ANNUAL OBS and MOD STD
stdObs_xy = pcmdi_metrics.pcmdi.std_xy.compute(do_am)
std_xy = pcmdi_metrics.pcmdi.std_xy.compute(dm_am)
# ZONAL MEANS ######
# CALCULATE ANNUAL MEANS
dm_amzm, do_amzm = pcmdi_metrics.pcmdi.zonal_mean.compute(dm_am, do_am)
# CALCULATE ANNUAL AND ZONAL MEAN RMS
rms_y = pcmdi_metrics.pcmdi.rms_0.compute(dm_amzm, do_amzm)
# CALCULATE ANNUAL MEAN DEVIATION FROM ZONAL MEAN RMS
dm_amzm_grown, dummy = grower(dm_amzm, dm_am)
dm_am_devzm = MV2.subtract(dm_am, dm_amzm_grown)
do_amzm_grown, dummy = grower(do_amzm, do_am)
do_am_devzm = MV2.subtract(do_am, do_amzm_grown)
rms_xy_devzm = pcmdi_metrics.pcmdi.rms_xy.compute(
dm_am_devzm, do_am_devzm)
# CALCULATE ANNUAL AND ZONAL MEAN STD
# CALCULATE ANNUAL MEAN DEVIATION FROM ZONAL MEAN STD
stdObs_xy_devzm = pcmdi_metrics.pcmdi.std_xy.compute(do_am_devzm)
std_xy_devzm = pcmdi_metrics.pcmdi.std_xy.compute(dm_am_devzm)
for stat in ["std-obs_xy", "std_xy", "std-obs_xyt",
"std_xyt", "std-obs_xy_devzm", "std_xy_devzm",
"rms_xyt", "rms_xy", "cor_xy", "bias_xy",
"mae_xy", "rms_y", "rms_devzm"]:
metrics_dictionary[stat] = {}
metrics_dictionary[
'std-obs_xy']['ann'] = format(
stdObs_xy *
conv,
sig_digits)
metrics_dictionary[
'std_xy']['ann'] = format(
std_xy *
conv,
sig_digits)
metrics_dictionary[
'std-obs_xyt']['ann'] = format(
stdObs_xyt *
conv,
sig_digits)
metrics_dictionary[
'std_xyt']['ann'] = format(
std_xyt *
conv,
sig_digits)
metrics_dictionary[
'std-obs_xy_devzm']['ann'] = format(
stdObs_xy_devzm *
conv,
sig_digits)
metrics_dictionary[
'std_xy_devzm']['ann'] = format(
std_xy_devzm *
conv,
sig_digits)
metrics_dictionary[
'rms_xyt']['ann'] = format(
rms_xyt *
conv,
sig_digits)
metrics_dictionary[
'rms_xy']['ann'] = format(
rms_xy *
conv,
sig_digits)
metrics_dictionary[
'cor_xy']['ann'] = format(
cor_xy,
sig_digits)
metrics_dictionary[
'bias_xy']['ann'] = format(
bias_xy *
conv,
sig_digits)
metrics_dictionary[
'mae_xy']['ann'] = format(
mae_xy *
conv,
sig_digits)
# ZONAL MEAN CONTRIBUTIONS
metrics_dictionary[
'rms_y']['ann'] = format(
rms_y *
conv,
sig_digits)
metrics_dictionary[
'rms_devzm']['ann'] = format(
rms_xy_devzm *
conv,
sig_digits)
# CALCULATE SEASONAL MEANS
for sea in ['djf', 'mam', 'jja', 'son']:
dm_sea = pcmdi_metrics.pcmdi.seasonal_mean.compute(dm, sea)
do_sea = pcmdi_metrics.pcmdi.seasonal_mean.compute(do, sea)
# CALCULATE SEASONAL RMS AND CORRELATION
rms_sea = pcmdi_metrics.pcmdi.rms_xy.compute(dm_sea, do_sea)
cor_sea = pcmdi_metrics.pcmdi.cor_xy.compute(dm_sea, do_sea)
mae_sea = pcmdi_metrics.pcmdi.meanabs_xy.compute(dm_sea, do_sea)
bias_sea = pcmdi_metrics.pcmdi.bias_xy.compute(dm_sea, do_sea)
# CALCULATE ANNUAL OBS and MOD STD
stdObs_xy_sea = pcmdi_metrics.pcmdi.std_xy.compute(do_sea)
std_xy_sea = pcmdi_metrics.pcmdi.std_xy.compute(dm_sea)
# ZONAL MEANS ######
# CALCULATE SEASONAL MEANS
# dm_smzm, do_smzm = pcmdi_metrics.pcmdi.zonal_mean.compute(dm_sea,
# do_sea)
# CALCULATE SEASONAL AND ZONAL MEAN RMS
# rms_y = pcmdi_metrics.pcmdi.rms_y.compute(dm_smzm, do_smzm)
# CALCULATE SEASONAL MEAN DEVIATION FROM ZONAL MEAN RMS
# dm_smzm_grown,dummy = grower(dm_smzm,dm_sea)
# dm_sea_devzm = MV.subtract(dm_sea,dm_smzm_grown)
# do_smzm_grown,dummy = grower(do_smzm,do_sea)
# do_sm_devzm = MV.subtract(do_sea,do_smzm_grown)
# rms_xy_devzm = pcmdi_metrics.pcmdi.rms_xy.compute(dm_sm_devzm,
# do_sm_devzm)
# print 'SEASONAL ZM HERE>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>'
metrics_dictionary['bias_xy'][sea] = format(
bias_sea *
conv,
sig_digits)
metrics_dictionary['rms_xy'][sea] = format(
rms_sea *
conv,
sig_digits)
metrics_dictionary['cor_xy'][sea] = format(
cor_sea,
'.2f')
metrics_dictionary['mae_xy'][sea] = format(
mae_sea *
conv,
sig_digits)
metrics_dictionary['std-obs_xy'][sea] = format(
stdObs_xy_sea *
conv,
sig_digits)
metrics_dictionary['std_xy'][sea] = format(
std_xy_sea *
conv,
sig_digits)
# ZONAL AND SEASONAL MEAN CONTRIBUTIONS
# metrics_dictionary[ 'rms_y'][ sea] = format(
# rms_y *
# conv,
# sig_digits)
# metrics_dictionary[ 'rms_devzm'][ sea] = format(
# rms_xy_devzm *
# conv,
# sig_digits)
return metrics_dictionary
def run(self):
if self.netcdf3: io.netcdf3()
else: io.netcdf4(level=self.netcdf4level)
cdms2.setAutoBounds(0)
self.notice('Masking %s to %s', self.inpfile, self.outfile)
inpfile = cdms2.open(self.inpfile)
if os.path.isfile(self.outfile):
if os.path.samefile(self.inpfile, self.outfile):
raise Exception('Cannot use same input and output file')
if self.overwrite:
os.remove(self.outfile)
else:
raise Exception('Output file already exists and overwriting is not requested')
outfile = cdms2.open(self.outfile, 'w')
# Copy global attributes
for a,v in inpfile.attributes.items():
setattr(outfile, a, v)
# Keep grid masks in memory to improve performances (TODO: use a file cache as for basemaps ?)
mask_cache = dict()
stats = True # self.is_verbose()
# Iterate over input variables
for varid,filevar in inpfile.variables.items():
# Process only gridded/specified variables
grid = filevar.getGrid()
maskit = grid is not None
if maskit and self.included_variables and varid not in self.included_variables: maskit = False
if maskit and self.excluded_variables and varid in self.excluded_variables: maskit = False
if not maskit and self.masked_only:
self.verbose('Ignoring %s: variable has no grid and masked_only was specified')
continue
self.logger.info(bases.psinfo())
self.notice('Processing variable: %s, grid: %s', varid, grid.shape if grid else None)
# NOTE: With scalar variables, memvar could be numpy.<type> instead of cdms2.tvariable.TransientVariable
memvar = filevar()
self.info(bases.describe(memvar, stats=stats))
self.logger.info(bases.psinfo())
if maskit:
# Build the mask, check if it is already cached ?
cache_id = id(grid)
self.info('Get mask for grid %s', cache_id)
mask = mask_cache.get(cache_id, None)
if mask is None:
self.notice('Loading mask: %s', dict(resolution=self.resolution, mode=self.mode, thresholds=self.thresholds, reverse=self.reverse))
mask = masking.polygon_mask(grid, self.resolution, mode=self.mode, thresholds=self.thresholds)
if self.reverse: mask = ~mask
mask_cache[cache_id] = mask
self.info(bases.describe(mask, stats=stats))
self.logger.info(bases.psinfo())
# Mask variable
# TODO: check/handle dimensions count and order
self.notice('Masking variable: %s, mask: %s', memvar.shape, mask.shape)
mask = MV2.resize(mask, filevar.shape)
memvar[:] = MV2.masked_where(mask, memvar)
self.info(bases.describe(memvar, stats=stats))
# Special scalar case which could fail if directly written (because fill_value is None)
if not filevar.shape:
fill_value = filevar.getMissing()
if fill_value is None:
fill_value = -memvar
memvar = cdms2.createVariable(
memvar, id=filevar.id, shape=(), typecode=filevar.typecode(),
fill_value=fill_value, attributes=filevar.attributes)
# Write masked variable to output file
self.notice('Writing variable to file')
outfile.write(memvar)
self.logger.info(bases.psinfo())
outfile.close()
inpfile.close()
@author: durack1
"""
import datetime,gc,os,sys
import cdms2 as cdm
import cdutil as cdu
import numpy as np
import MV2 as mv
sys.path.append('/export/durack1/git/durolib/durolib')
from durolib import globalAttWrite
# netCDF compression (use 0 for netCDF3)
cdm.setNetcdfShuffleFlag(1)
cdm.setNetcdfDeflateFlag(1)
cdm.setNetcdfDeflateLevelFlag(9) ; # 9(shuf=1) 466.6KB; 9(shuf=0) 504.1KB; 4(shuf=0) 822.93KB;
cdm.setAutoBounds(1)
#%% Change directory
os.chdir('/work/durack1/Shared/obs_data/WOD18/')
sourceDir = '190312'
#%%
#del(asc,count,depths,e,lat_ind,latitude,line,lon_ind,longitude,pi,tmp)
#del(asc,count,depths,lat_ind,latitude,line,lon_ind,longitude,tmp)
#%% Declare two grids
grids = {'1deg':'01','0p25deg':'04'}
for count,grid in enumerate(grids):
gridId = grid
fileId = grids[gridId]
def compute_metrics(Var, dm_glb, do_glb):
# Var is sometimes sent with level associated
var = Var.split("_")[0]
# Did we send data? Or do we just want the info?
if dm_glb is None and do_glb is None:
metrics_defs = collections.OrderedDict()
metrics_defs["rms_xyt"] = pcmdi_metrics.pcmdi.rms_xyt.compute(
None,
None)
metrics_defs["rms_xy"] = pcmdi_metrics.pcmdi.rms_xy.compute(None, None)
metrics_defs["bias_xy"] = pcmdi_metrics.pcmdi.bias.compute(None, None)
metrics_defs["mae_xy"] = pcmdi_metrics.pcmdi.meanabs_xy.compute(
None,
None)
metrics_defs["cor_xyt"] = pcmdi_metrics.pcmdi.cor_xyt.compute(
None,
None)
metrics_defs["cor_xy"] = pcmdi_metrics.pcmdi.cor_xy.compute(None, None)
metrics_defs["seasonal_mean"] = \
pcmdi_metrics.pcmdi.seasonal_mean.compute(
None,
None)
metrics_defs["annual_mean"] = pcmdi_metrics.pcmdi.annual_mean.compute(
None,
None)
metrics_defs["zonal_mean"] = pcmdi_metrics.pcmdi.zonal_mean.compute(
None,
None)
return metrics_defs
cdms.setAutoBounds('on')
metrics_dictionary = {}
# SET CONDITIONAL ON INPUT VARIABLE
if var == 'pr':
conv = 1.e5
else:
conv = 1.
if var in ['hus']:
sig_digits = '.5f'
else:
sig_digits = '.3f'
domains = ['GLB', 'NHEX', 'TROPICS', 'SHEX']
for dom in domains:
dm = dm_glb
do = do_glb
if dom == 'NHEX':
dm = dm_glb(latitude=(30., 90))
do = do_glb(latitude=(30., 90))
if dom == 'SHEX':
dm = dm_glb(latitude=(-90., -30))
do = do_glb(latitude=(-90., -30))
if dom == 'TROPICS':
dm = dm_glb(latitude=(-30., 30))
do = do_glb(latitude=(-30., 30))
# CALCULATE ANNUAL CYCLE SPACE-TIME RMS AND CORRELATIONS
rms_xyt = pcmdi_metrics.pcmdi.rms_xyt.compute(dm, do)
cor_xyt = pcmdi_metrics.pcmdi.cor_xyt.compute(dm, do)
# CALCULATE ANNUAL MEANS
dm_am, do_am = pcmdi_metrics.pcmdi.annual_mean.compute(dm, do)
# CALCULATE ANNUAL MEAN BIAS
bias_xy = pcmdi_metrics.pcmdi.bias.compute(dm_am, do_am)
# CALCULATE MEAN ABSOLUTE ERROR
mae_xy = pcmdi_metrics.pcmdi.meanabs_xy.compute(dm_am, do_am)
# CALCULATE ANNUAL MEAN RMS
rms_xy = pcmdi_metrics.pcmdi.rms_xy.compute(dm_am, do_am)
# ZONAL MEANS ######
# CALCULATE ANNUAL MEANS
dm_amzm, do_amzm = pcmdi_metrics.pcmdi.zonal_mean.compute(dm_am, do_am)
# CALCULATE ANNUAL AND ZONAL MEAN RMS
rms_y = pcmdi_metrics.pcmdi.rms_0.compute(dm_amzm, do_amzm)
# CALCULATE ANNUAL MEAN DEVIATION FROM ZONAL MEAN RMS
dm_amzm_grown, dummy = grower(dm_amzm, dm_am)
dm_am_devzm = MV.subtract(dm_am, dm_amzm_grown)
do_amzm_grown, dummy = grower(do_amzm, do_am)
do_am_devzm = MV.subtract(do_am, do_amzm_grown)
rms_xy_devzm = pcmdi_metrics.pcmdi.rms_xy.compute(
dm_am_devzm, do_am_devzm)
metrics_dictionary[
'rms_xyt_ann_' +
dom] = format(
rms_xyt *
conv,
sig_digits)
metrics_dictionary[
'rms_xy_ann_' +
dom] = format(
rms_xy *
conv,
sig_digits)
metrics_dictionary[
'bias_xy_ann_' +
dom] = format(
bias_xy *
conv,
sig_digits)
metrics_dictionary[
'cor_xyt_ann_' +
dom] = format(
cor_xyt *
conv,
'.2f')
metrics_dictionary[
'mae_xy_ann_' +
dom] = format(
mae_xy *
conv,
sig_digits)
# ZONAL MEAN CONTRIBUTIONS
metrics_dictionary[
'rms_y_ann_' +
dom] = format(
rms_y *
conv,
sig_digits)
metrics_dictionary[
'rms_devzm_ann_' +
dom] = format(
rms_xy_devzm *
conv,
sig_digits)
# CALCULATE SEASONAL MEANS
for sea in ['djf', 'mam', 'jja', 'son']:
dm_sea = pcmdi_metrics.pcmdi.seasonal_mean.compute(dm, sea)
do_sea = pcmdi_metrics.pcmdi.seasonal_mean.compute(do, sea)
# CALCULATE SEASONAL RMS AND CORRELATION
rms_sea = pcmdi_metrics.pcmdi.rms_xy.compute(dm_sea, do_sea)
cor_sea = pcmdi_metrics.pcmdi.cor_xy.compute(dm_sea, do_sea)
mae_sea = pcmdi_metrics.pcmdi.meanabs_xy.compute(dm_sea, do_sea)
bias_sea = pcmdi_metrics.pcmdi.bias.compute(dm_sea, do_sea)
# ZONAL MEANS ######
# CALCULATE SEASONAL MEANS
# dm_smzm, do_smzm = pcmdi_metrics.pcmdi.zonal_mean.compute(dm_sea,
# do_sea)
# CALCULATE SEASONAL AND ZONAL MEAN RMS
# rms_y = pcmdi_metrics.pcmdi.rms_y.compute(dm_smzm, do_smzm)
# CALCULATE SEASONAL MEAN DEVIATION FROM ZONAL MEAN RMS
# dm_smzm_grown,dummy = grower(dm_smzm,dm_sea)
# dm_sea_devzm = MV.subtract(dm_sea,dm_smzm_grown)
# do_smzm_grown,dummy = grower(do_smzm,do_sea)
# do_sm_devzm = MV.subtract(do_sea,do_smzm_grown)
# rms_xy_devzm = pcmdi_metrics.pcmdi.rms_xy.compute(dm_sm_devzm,
# do_sm_devzm)
# print 'SEASONAL ZM HERE>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>'
metrics_dictionary[
'bias_xy_' +
sea +
'_' +
dom] = format(
bias_sea *
conv,
sig_digits)
metrics_dictionary[
'rms_xy_' +
sea +
'_' +
dom] = format(
rms_sea *
conv,
sig_digits)
metrics_dictionary[
'cor_xy_' +
sea +
'_' +
dom] = format(
cor_sea,
'.2f')
metrics_dictionary[
'mae_xy_' +
sea +
'_' +
dom] = format(
mae_sea *
conv,
sig_digits)
# ZONAL AND SEASONAL MEAN CONTRIBUTIONS
# metrics_dictionary[
# 'rms_y_' + sea + '_' +
# dom] = format(
# rms_y *
# conv,
# sig_digits)
# metrics_dictionary[
# 'rms_devzm_' + sea + '_' +
# dom] = format(
# rms_xy_devzm *
# conv,
# sig_digits)
return metrics_dictionary
def correctFile(idxcorr, ncorr, inFile, inDir, outFile, outDir):
'''
Correct density binned files (undefined ptop & long 0 issue)
idxcorr = [idx_i,idx_i1,jmax] indices for longitude correction - if [0,0,0] ignore
ncorr = number of corrections: 1 or 2
'''
# CDMS initialisation - netCDF compression
comp = 1 # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
varList3D = ['isondepthg','isonthickg', 'sog','thetaog']
varList2D = ['ptopsoxy','ptopdepthxy','ptopsigmaxy','ptopthetaoxy','persistmxy']
# First test, read level by level and write level by level (memory management)
# use ncpdq -a time,lev,lat,lon to recover the dimension order
fi = cdm.open(inDir+'/'+inFile)
fo = cdm.open(outDir+'/'+outFile,'w')
isondg = fi['isondepthg'] ; # Create variable handle
# Get grid objects
#axesList = isondg.getAxisList()
#sigmaGrd = isondg.getLevel()
lonN = isondg.shape[3]
latN = isondg.shape[2]
levN = isondg.shape[1]
timN = isondg.shape[0]
#valmask = isondg.missing_value
if ncorr == 2:
ic1 = idxcorr[0][0]
ic2 = idxcorr[0][1]
jcmax = idxcorr[0][2]
ic12 = idxcorr[1][0]
ic22 = idxcorr[1][1]
jcmax2 = idxcorr[1][2]
if ic2 >= lonN-1:
ic2 = 0
if ic22 >= lonN-1:
ic22 = 0
elif ncorr == 1:
ic1 = idxcorr[0]
ic2 = idxcorr[1]
jcmax = idxcorr[2]
if ic2 >= lonN-1:
ic2 = 0
#print ic1,ic2,jcmax
corr_long = True
if ic1 == 0 and ic2 == 0 and jcmax == 0:
corr_long = False
#testp = 10
for it in range(timN):
#if it/testp*testp == it:
# print ' year =',it
# test
#i = 90
#j = 90
#i2d = 6
#j2d = 12
#ij = j*lonN+i
#ij2d = j2d*lonN+i2d
#print 'ij=',ij
# 3D variables
for iv in varList3D:
#print iv
outVar = fi(iv,time = slice(it,it+1))
# Correct for longitude interpolation issue
if corr_long:
for jt in range(jcmax):
outVar[:,:,jt,ic1] = (outVar[:,:,jt,ic1-1]+outVar[:,:,jt,ic2+1])/2
outVar[:,:,jt,ic2] = outVar[:,:,jt,ic1]
if ncorr == 2:
for jt in range(jcmax2):
outVar[:,:,jt,ic12] = (outVar[:,:,jt,ic12-1]+outVar[:,:,jt,ic22+1])/2
outVar[:,:,jt,ic22] = outVar[:,:,jt,ic12]
# Correct Bowl properties
if iv =='isondepthg':
vardepth = npy.reshape(outVar,(levN,latN*lonN))
#print 'test'
#print outVar[:,:,j2d,i2d]
#print vardepth[:,ij2d]
# find values of surface points
vardepthBowl = npy.min(npy.reshape(outVar,(levN,latN*lonN)),axis=0)
vardepthBowlTile = npy.repeat(vardepthBowl,levN,axis=0).reshape((latN*lonN,levN)).transpose()
#print vardepthBowlTile.shape
#print vardepthBowl[ij2d], vardepthBowlTile[:,ij2d]
levs = outVar.getAxisList()[1][:]
#print 'levs',levs
levs3d = mv.reshape(npy.tile(levs,latN*lonN),(latN*lonN,levN)).transpose()
varsigmaBowl = npy.max(npy.where(vardepth == vardepthBowlTile,levs3d,0),axis=0)
#print varsigmaBowl[ij2d],levs3d[:,ij2d]
elif iv == 'sog':
varsog = npy.reshape(outVar,(levN,latN*lonN))
varsoBowl = npy.max(npy.where(vardepth == vardepthBowlTile,varsog,0),axis=0)
#print varsoBowl[ij2d], varsog[:,ij2d]
#print vardepth[:,ij2d],vardepthBowlTile[:,ij2d]
del (varsog); gc.collect()
elif iv =='thetaog':
varthetao = npy.reshape(outVar,(levN,latN*lonN))
varthetaoBowl = npy.max(npy.where(vardepth == vardepthBowlTile,varthetao,-1000),axis=0)
#print varthetaoBowl[ij2d],varthetao[:,ij2d]
del (varthetao); gc.collect()
# Write
fo.write(outVar.astype('float32'), extend = 1, index = it)
fo.sync()
del (vardepth); gc.collect()
# 2D variables and correct isondepthg = 0
for iv in varList2D:
outVar = fi(iv,time = slice(it,it+1))
# Correct for longitude interpolation issue
if corr_long:
for jt in range(jcmax):
outVar[:,jt,ic1] = (outVar[:,jt,ic1-1]+outVar[:,jt,ic2+1])/2
outVar[:,jt,ic2] = outVar[:,jt,ic1]
if ncorr == 2:
for jt in range(jcmax2):
outVar[:,jt,ic12] = (outVar[:,jt,ic12-1]+outVar[:,jt,ic22+1])/2
outVar[:,jt,ic22] = outVar[:,jt,ic12]
# Correct for ptopsoxy < 30
#print 'before',outVar[:,j2d,i2d]
if iv == 'ptopsoxy':
testso = npy.reshape(outVar,(latN*lonN)) < 30.
#print 'testdepth', testdepth[ij2d]
#print npy.argwhere(testdepth)[0:10]/lonN, npy.argwhere(testdepth)[0:10]-npy.argwhere(testdepth)[0:10]/lonN*lonN
outVar.data[...] = npy.where(testso,varsoBowl,npy.reshape(outVar,(latN*lonN))).reshape(outVar.shape)[...]
elif iv == 'ptopdepthxy':
outVar.data[...] = npy.where(testso,vardepthBowl,npy.reshape(outVar,(latN*lonN))).reshape(outVar.shape)[...]
elif iv == 'ptopthetaoxy':
outVar.data[...] = npy.where(testso,varthetaoBowl,npy.reshape(outVar,(latN*lonN))).reshape(outVar.shape)[...]
elif iv == 'ptopsigmaxy':
outVar.data[...] = npy.where(testso,varsigmaBowl,npy.reshape(outVar,(latN*lonN))).reshape(outVar.shape)[...]
#print 'after',outVar[:,j2d,i2d]
# Write
fo.write(outVar.astype('float32'), extend = 1, index = it)
fo.sync()
fi.close()
fo.close()
# testing
#model = 'CCSM4'
#idxcorr=[139,140,145]
#ncorr = 1
#inFile = 'cmip5.CCSM4.historical24.r1i1p1.an.ocn.Omon.density.ver-v20121128.nc'
#inDir = '/Users/ericg/Projets/Density_bining/Raw_testing'
#outFile = 'cmip5.CCSM4.historical24.outtest.nc'
#model = 'CanESM2'
#idxcorr=[179,180,180]
#ncorr = 1
#inFile = 'cmip5.CanESM2.historical24.r1i1p1.an.ocn.Omon.density.ver-1.nc'
#inDir = '/Users/ericg/Projets/Density_bining/Raw_testing'
#outFile = 'cmip5.CanESM2.historical24.outtest.nc'
#model = 'IPSL-CM5A-LR'
#idxcorr=[0,0,0]
#ncorr=1
#inFile = 'cmip5.IPSL-CM5A-LR.historical24.r1i1p1.an.ocn.Omon.density.ver-v20111119.nc'
#inDir = '/Users/ericg/Projets/Density_bining/Raw_testing'
#outFile = 'cmip5.IPSL-CM5A-LR.historical24.outtest.nc'
#model = 'Ishii'
#idxcorr=[[359,359,39],[180,180,180]]
#idxcorr=[359,359,39]
#ncorr = 1
#inFile = 'obs.Ishii.historical.r0i0p0.an.ocn.Omon.density.ver-1.latestX.nc'
#inDir='/Volumes/hciclad/data/Density_binning/Prod_density_obs_april16'
#outFile = 'obs.Ishii.historical.r0i0p0.an.ocn.Omon.density.ver-1.latestXCorr.nc'
#model = 'EN4'
#idxcorr=[[359,359,39],[180,180,180]]
#idxcorr=[359,359,39]
#ncorr = 2
#inFile = 'obs.EN4.historical.r0i0p0.mo.ocn.Omon.density.ver-1.latestX.nc'
#inDir='/Volumes/hciclad/data/Density_binning/Prod_density_obs_april16'
#outFile = 'obs.EN4.historical.r0i0p0.mo.ocn.Omon.density.ver-1.latestXCorr.nc'
#outDir = inDir
#correctFile(idxcorr, ncorr, inFile, inDir, outFile, outDir)
def plot(self,data,data2,template = None, bg=0, x=None):
if x is None:
x = self.x
if template is None:
template = self.template
elif isinstance(template,str):
template = x.gettemplate(template)
elif not vcs.istemplate(template):
raise "Error did not know what to do with template: %s" % template
if not isinstance(data,cdms2.tvariable.TransientVariable):
mode= cdms2.getAutoBounds()
cdms2.setAutoBounds("on")
data = MV2.array(data)
data.getAxis(-1).getBounds()
cdms2.setAutoBounds(mode)
while data.rank()>1:
data = data[0]
while data2.rank()>1:
data2 = data2[0]
# ok now we have a good x and a good data
npts1 = len(data)
npts2 = len(data2)
# create the primitive
fill = x.createfillarea()
line = x.createline()
fill.viewport = [template.data.x1,template.data.x2,template.data.y1,template.data.y2]
line.viewport = [template.data.x1,template.data.x2,template.data.y1,template.data.y2]
ax = data.getAxis(0)[:]
ax2 = data.getAxis(0)[:]
xmn,xmx = vcs.minmax(ax,ax2)
ymn,ymx = vcs.minmax(data,data2)
xmn,xmx,ymn,ymx = self.prep_plot(xmn,xmx,ymn,ymx)
fill.worldcoordinate=[xmn,xmx,ymn,ymx]
line.worldcoordinate=[xmn,xmx,ymn,ymx]
fill.style = [self.fillareastyle,]
fill.color = [self.fillareacolor,]
fill.index = [self.fillareaindex,]
line.type = [self.line,]
line.width = [self.linewidth,]
line.color = [self.linecolor,]
xs = []
ys = []
xs = numpy.concatenate((ax[:],ax2[::-1])).tolist()
ys = numpy.concatenate((data[:],data2[::-1])).tolist()
xs.append(xs[0])
ys.append(ys[0])
fill.x = xs
fill.y = ys
line.x = xs
line.y = ys
displays = []
displays.append(x.plot(fill,bg=bg))
displays.append(x.plot(line,bg=bg))
x.worldcoordinate = fill.worldcoordinate
dsp = template.plot(data,self,bg=bg)
for d in dsp:
displays.append(d)
self.restore()
return displays
timeComp = timeComp[1:-5]
print 'varComp shape:',varComp.shape
print 'timeComp len:',len(timeComp)
#%% Get variable into memory
count = 0
for filePath in newList:
obsVsBC = filePath.split('/')[-1].split('_')[0]
varName = filePath.split('/')[-1].split('_')[1]
climCheck = filePath.split('/')[-1].split('_')[-1]
if obsVsBC == 'bcinfo' or obsVsBC == 'spinup' or obsVsBC == '.out' or climCheck == 'clim.nc':
print 'Invalid file, skipping..'
continue
if 'bc' in obsVsBC:
BC = 'bcs'
cdm.setAutoBounds(2) ; # Turn off time_bounds - only latitude/longitude written
else:
BC = ''
cdm.setAutoBounds(1) ; # Turn on time_bounds - time/latitude/longitude written
if varName == 'sst':
varLoad = ''.join(['tos',BC])
varPath = 'tos'
else:
varLoad = ''.join([varName,BC])
varPath = varName
print filePath
fH = cdm.open(filePath)
if (last_month == 6 and last_year in filePath):
var = fH(varLoad,time=slice(0,6))
else:
var = fH(varLoad)
def compute_metrics(Var,dm_glb,do_glb):
# Var is sometimes sent with level associated
var = Var.split("_")[0]
cdms.setAutoBounds('on')
metrics_dictionary = {}
domains = ['GLB','NHEX','TROPICS','SHEX']
for dom in domains:
dm = dm_glb
do = do_glb
if dom == 'NHEX':
dm = dm_glb(latitude = (30.,90))
do = do_glb(latitude = (30.,90))
if dom == 'SHEX':
dm = dm_glb(latitude = (-90.,-30))
do = do_glb(latitude = (-90.,-30))
if dom == 'TROPICS':
dm = dm_glb(latitude = (-30.,30))
do = do_glb(latitude = (-30.,30))
### CALCULATE ANNUAL CYCLE SPACE-TIME RMS AND CORRELATIONS
rms_xyt = pcmdi_metrics.pcmdi.rms_xyt.compute(dm,do)
cor_xyt = pcmdi_metrics.pcmdi.cor_xyt.compute(dm,do)
### CALCULATE ANNUAL MEANS
dm_am, do_am = pcmdi_metrics.pcmdi.annual_mean.compute(dm,do)
### CALCULATE ANNUAL MEAN BIAS
bias_xy = pcmdi_metrics.pcmdi.bias.compute(dm_am,do_am)
### CALCULATE MEAN ABSOLUTE ERROR
mae_xy = pcmdi_metrics.pcmdi.meanabs_xy.compute(dm_am,do_am)
### CALCULATE ANNUAL MEAN RMS
rms_xy = pcmdi_metrics.pcmdi.rms_xy.compute(dm_am,do_am)
# SET CONDITIONAL ON INPUT VARIABLE
if var == 'pr':
conv = 1.e5
else:
conv = 1.
sig_digits = '.2f'
if var in ['hus']: sig_digits = '.5f'
for m in ['rms_xyt','rms_xy','bias_xy','cor_xyt','mae_xy']:
if m == 'rms_xyt': metrics_dictionary[m + '_ann_' + dom] = format(rms_xyt*conv,sig_digits)
if m == 'rms_xy': metrics_dictionary[m + '_ann_' + dom] = format(rms_xy*conv,sig_digits)
if m == 'bias_xy': metrics_dictionary[m + '_ann_' + dom] = format(bias_xy*conv,sig_digits)
if m == 'mae_xy': metrics_dictionary[m + '_ann_' + dom] = format(mae_xy*conv,sig_digits)
if m == 'cor_xyt': metrics_dictionary[m + '_ann_' + dom] = format(cor_xyt,'.2f')
### CALCULATE SEASONAL MEANS
for sea in ['djf','mam','jja','son']:
dm_sea = pcmdi_metrics.pcmdi.seasonal_mean.compute(dm,sea)
do_sea = pcmdi_metrics.pcmdi.seasonal_mean.compute(do,sea)
### CALCULATE SEASONAL RMS AND CORRELATION
rms_sea = pcmdi_metrics.pcmdi.rms_xy.compute(dm_sea,do_sea)
cor_sea = pcmdi_metrics.pcmdi.cor_xy.compute(dm_sea,do_sea)
mae_sea = pcmdi_metrics.pcmdi.meanabs_xy.compute(dm_sea,do_sea)
bias_sea = pcmdi_metrics.pcmdi.bias.compute(dm_sea,do_sea)
metrics_dictionary['bias_xy_' + sea + '_' + dom] = format(bias_sea*conv,sig_digits)
metrics_dictionary['rms_xy_' + sea + '_' + dom] = format(rms_sea*conv,sig_digits)
metrics_dictionary['cor_xy_' + sea + '_' + dom] = format(cor_sea,'.2f')
metrics_dictionary['mae_xy_' + sea + '_' + dom] = format(mae_sea*conv,sig_digits)
return metrics_dictionary
import cdms2 as cdm
import cdtime
import cdutil as cdu
import MV2 as MV # functions for dealing with masked values.
import numpy
import pytz
# Set nc classic as outputs
cdm.setCompressionWarnings(0)
# Suppress warnings
cdm.setNetcdfShuffleFlag(0)
cdm.setNetcdfDeflateFlag(1)
# was 0 130717
cdm.setNetcdfDeflateLevelFlag(9)
# was 0 130717
cdm.setAutoBounds(1)
# Ensure bounds on time and depth axes are generated
#^^^^^^^^^^^^^^^ MODIFY FOLLOWING THINGS DEPENDING ON THE DATASET ^^^^^^^^^^^^^^^
#userinitials="CRT"
userinitials = "CZ"
#input_hostpath="/work/terai1/ACME/OAFLUX/" #location of input data
#output_hostpath="/work/terai1/ACME/OAFLUX/" #location to save climatology files
data_name = 'GPCP'
input_hostpath = "/p/user_pub/e3sm/zhang40/analysis_data_e3sm_diags/" + data_name + "/time_series/" #location of input data
output_hostpath = "/p/user_pub/e3sm/zhang40/analysis_data_e3sm_diags/" + data_name + "/climatology/" #location to save climatology files
if data_name == 'GPCP':
rootname = 'GPCP_v2.3' #name of the climo files
startyear = '1979' #start year of the dataset
startmonth = '01' #start month of the dataset, typically 01
# Adapted for numpy/ma/cdms2 by convertcdms.py
import cdms2,sys,ZonalMeans
cdms2.setAutoBounds("on")
cdms2.axis.level_aliases.append('depth')
import vcs
import vcs.test.support
bg= vcs.test.support.bg
x=vcs.init()
# Open data file
f=cdms2.open('test_data.nc')
O2=f("O2",level=slice(0,5))
print O2.shape,O2.mask
print 'Level:',O2.getLevel()
f.close()
# Open grid file
f=cdms2.open('test_grid.nc')
# 1st with just bounds
blon = f('bounds_lon')
blat = f('bounds_lat')
mask = f('mask')
g=mask.getGrid()
O2.setAxis(-1,mask.getAxis(-1))
O2.setAxis(-2,mask.getAxis(-2))
O2.setGrid(g)
#sys.exit()
area = f('area')
res=ZonalMeans.compute(O2,area=area,delta_band=5)
def coloc_mod_on_pro(self, model, profiles, varnames, select=None, method='nearest'):
'''Colocalize model on profile data.
Load model data corresponding to the selected profiles positions and time.
Returns loaded model longitudes, latitudes, depths and requested variable(s)
:Params:
- **model**: model data :class:`~vacumm.data.misc.dataset.Dataset`
- **profiles**: profile data :class:`~vacumm.data.misc.profile.ProfilesDataset`
- **varnames**: variables to load (ex: ('temp','sal') or (('temp','temperature'),('sal','salinity'))
- **select**: selector
- **method**: coloc method (**nearest** or **interp**)
:Return:
- **lons_mod**: model longitude coordinates, shape: (profile)
- **lats_mod**: model latitude coordinates, shape: (profile)
- **deps_mod**: model depth coordinates, shape: (level,profile)
- **var1**: requested variables, shape: (level,profile)
- ...
- **varN**
.. todo::
- also load and return profile data here
- exclude coords where profile data is masked (no data for specified depth)
- return time coordinates
- return depth and vars with shape (profile,level)
'''
self.verbose('Colocalizing %s on %s\nvarnames: %s\nselect: %s\n method: %s', model.__class__.__name__, profiles.__class__.__name__, varnames, select, method)
prof_pro = profiles.get_axis('profile', select=select)
if prof_pro is None or not len(prof_pro):
raise Exception('No profiles found, aborting')
lev_pro = profiles.get_axis('level', select=select)
time_pro = profiles.get_variable('time', select=select)
lons_pro = profiles.get_variable('longitude', select=select)
lats_pro = profiles.get_variable('latitude', select=select)
dates = create_time(time_pro).asComponentTime()
self.info('Number of profiles: %s', len(dates))
self.info('Profiles time coverage: %s to %s', dates[0], dates[-1])
# Init model
td = model.get_time_res()
dtmax = (td.days*86400+td.seconds, 'seconds')
self.info('Detected model time step: %s', td)
grid_mod = model.get_grid()
xres, yres = resol(grid_mod)
time_mod = model.get_time()
ctime_mod = time_mod.asComponentTime()
self.info('Model time coverage: %s to %s', ctime_mod[0], ctime_mod[-1])
level_mod = model.get_level(select=select)
lons_mod = MV2.zeros((len(prof_pro),))+MV2.masked
lats_mod = lons_mod.clone()
deps_mod = MV2.zeros((len(level_mod), len(prof_pro)))+MV2.masked
deps_mod.setAxis(1, prof_pro)
lons_mod.id, lats_mod.id, deps_mod.id = 'longitude', 'latitude', 'depth'
# Creation des variables demandees
variables = []
for n in varnames:
v = MV2.zeros((len(level_mod), len(prof_pro)))+MV2.masked
v.setAxis(1, prof_pro)
v.id = is_iterable(n) and n[0] or n
variables.append(v)
cdms2.setAutoBounds(1) # ???
# Boucle temporelle
for ip, date in enumerate(dates):
try:
# Limites spatiales
lon = lons_pro[ip]
lat = lats_pro[ip]
lon_min = lon-2*xres
lon_max = lon+2*xres
lat_min = lat-2*yres
lat_max = lat+2*yres
date_interval = (add_time(date, - dtmax[0], dtmax[1]), add_time(date, dtmax[0], dtmax[1]), 'ccb')
self.info('Colocalizing data for date %s, lon: %s, lat: %s', date, lon, lat)
# Methode 1 : donnees les plus proches
if method == 'nearest':
sel = dict(time=(date, date, 'ccb'), longitude=(lon, lon, 'ccb'), latitude=(lat, lat, 'ccb'))
# Verifier la disponibilite des donnees
if time_mod.mapIntervalExt(sel['time']) is None:
self.warning('Time interval %s not found', sel['time'])
continue
if grid_mod.getLatitude().mapInterval(sel['latitude']) is None:
self.warning('Latitude coordinate %s not found', sel['latitude'])
continue
if grid_mod.getLongitude().mapInterval(sel['longitude']) is None:
self.warning('Longitude coordinate %s not found', sel['longitude'])
continue
# Load tmp depth to get lon & lat coordinates
#tmp = model.get_depth(select=sel, squeeze=False) # tmp squeezed !!! see sigma ?
tmp = model.get_variable(varnames[0], select=sel, squeeze=False)
lons_mod[ip] = tmp.getLongitude()[0]
lats_mod[ip] = tmp.getLatitude()[0]
deps_mod[:, ip] = model.get_depth(select=sel, squeeze=True)
for iv,vn in enumerate(varnames):
variables[iv][:,ip] = model.get_variable(vn, select=sel, squeeze=True)
# Methode 2 : interpolation
elif method == 'interp':
sel = dict(time=date_interval, longitude=(lon_min, lon_max), latitude=(lat_min, lat_max))
if time_mod.mapIntervalExt(sel['time']) is None:
self.warning('Time interval %s not found', sel['time'])
continue
if grid_mod.getLatitude().mapInterval(sel['latitude']) is None:
self.warning('Latitude coordinate %s not found', sel['latitude'])
continue
if grid_mod.getLongitude().mapInterval(sel['longitude']) is None:
self.warning('Longitude coordinate %s not found', sel['longitude'])
continue
# Lectures
order = 'tzyx'
# lon & lat du profile car interp sur cette position
lons_mod[ip], lats_mod[ip] = lon, lat
deps_mod_tzyx = model.get_depth(select=sel, order=order, squeeze=True)
tmp_tzyx = []
for iv,vn in enumerate(varnames):
tmp_tzyx.append(model.get_variable(vn, select=sel, order=order, squeeze=True))
# Interpolations temporelles
mctime = tmp_tzyx[0].getTime()
mrtime = mctime.asRelativeTime()
d0 = date.torel(mctime.units).value - mrtime[0].value
d1 = mrtime[1].value - date.torel(mctime.units).value
f0 = d0 / (d0 + d1)
f1 = d1 / (d0 + d1)
deps_mod_zyx = f0 * deps_mod_tzyx[0] + f1 * deps_mod_tzyx[1]
tmp_zyx = []
for iv,vn in enumerate(varnames):
tmp_zyx.append(f0 * tmp_tzyx[iv][0] + f1 * tmp_tzyx[iv][1])
del tmp_tzyx
# Interpolations spatiales
deps_mod[:,ip] = numpy.squeeze(grid2xy(deps_mod_zyx, numpy.array([lon]), numpy.array([lat]), method='nat'))
for iv,vn in enumerate(varnames):
variables[iv][:,ip] = numpy.squeeze(grid2xy(tmp_zyx[iv], numpy.array([lon]), numpy.array([lat]), method='nat'))
del tmp_zyx
else:
raise ValueError('Invalid colocation method: %s'%(method))
except:
self.exception('Failed to colocalize data for date %s', date)
for v in [deps_mod] + variables:
v.getAxis(0).id = 'level'
v.getAxis(0).designateLevel()
data = tuple([lons_mod, lats_mod, deps_mod] + variables)
self.verbose('Colocalized data:\n %s', '\n '.join(self.describe(o) for o in data))
return data
import string
import numpy as npy
import numpy.ma as ma
import cdutil as cdu
from genutil import statistics
import support_density as sd
import time as timc
import timeit
#import matplotlib.pyplot as plt
#
# netCDF compression (use 0 for netCDF3)
comp = 0
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
#
# == Arguments
#
#
# == Inits
#
home = os.getcwd()
outdir = os.getcwd()
hist_file_dir=home
#
# == Arguments
#
# == get command line options
parser = argparse.ArgumentParser(description='Script to perform density bining analysis')
parser.add_argument('-d', help='toggle debug mode', action='count', default=0)
def mmeAveMsk1D(listFiles, sw2d, years, inDir, outDir, outFile, timeInt, mme, ToeType, fullTS, debug=True):
'''
The mmeAveMsk1D() function averages rhon or scalar density bined files with differing masks
It ouputs the MME and a percentage of non-masked bins
Created on Tue Nov 25 13:56:20 CET 2014
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- sw2d - dimension of fields to consider (1 or 2)
- years(t1,t2) - years for slice read
- inDir(str) - input directory where files are stored
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- FfllTS - 0/1: if 1, uses full time serie (ignores years(t1,t2))
- debug <optional> - boolean value
Notes:
-----
- EG 25 Nov 2014 - Initial function write
- EG 9 Dec 2014 - Add agreement on difference with init period - save as <var>Agree
- EG 04 Oct 2016 - Add 3D files support
TODO:
------
'''
# CDMS initialisation - netCDF compression
comp = 1 ; # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
if t2 <= 0:
useLastYears = True
t2 = -t2
else:
useLastYears = False
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
# Find dimension
runN = len(listFiles)
try:
fi = cdm.open(inDir[0]+'/'+listFiles[0])
except:
print ' *** file not found ',inDir[0]+'/'+listFiles[0]
sys.exit(' Abort')
if sw2d == 1:
ptopd0 = fi['ptopdepth'] ; # Create variable handle
latN = ptopd0.shape[2]
basN = ptopd0.shape[1]
elif sw2d == 2:
ptopd0 = fi['ptopdepthxy'] ; # Create variable handle
lonN = ptopd0.shape[2]
latN = ptopd0.shape[1]
#timN = ptopd0.shape[0]
timN = t2-t1
if fullTS:
print ' !!! Working on full Time Serie (fullTS = True)'
timN = ptopd0.shape[0]
t1=0
t2=timN
t10 = t1
t20 = t2
# Get grid objects
axesList = ptopd0.getAxisList()
# Declare and open files for writing
if os.path.isfile(outDir+'/'+outFile):
os.remove(outDir+'/'+outFile)
outFile_f = cdm.open(outDir+'/'+outFile,'w')
print ' Number of members:',len(listFiles)
valmask = ptopd0.missing_value
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
time.designateTime()
# loop on variables
# init percent array
if sw2d == 1:
varList = ['ptopdepth','ptopsigma','ptopso','ptopthetao','volpers','salpers','tempers']
#varList = ['ptopdepth']
varDim = [1,1,1,1,0,0,0]
percent = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*0.
elif sw2d == 2:
varList = ['ptopdepthxy','ptopsigmaxy','ptopsoxy','ptopthetaoxy']
#varList = ['ptopdepthxy']
varDim = [2,2,2,2]
percent = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*0.
varFill = [valmask,valmask,valmask,valmask,valmask,valmask,valmask,valmask,valmask]
axis1D = [time,axesList[1],axesList[2]]
axis0D = [time,axesList[1]]
print ' timN = ',timN
# loop on 1D variables
for iv,var in enumerate(varList):
ti0 = timc.clock()
# Array inits
if varDim[iv] == 2:
isonvar = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask
vardiff = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask
varones = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*1.
axisVar = axis1D
elif varDim[iv] == 1:
isonvar = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*valmask
vardiff = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*valmask
varones = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*1.
axisVar = axis1D
else:
isonvar = npy.ma.ones([runN,timN,basN], dtype='float32')*valmask
vardiff = npy.ma.ones([runN,timN,basN], dtype='float32')*valmask
varones = npy.ma.ones([runN,timN,basN], dtype='float32')*1.
axisVar = axis0D
print ' Variable ',iv, var, varDim[iv]
# loop over files to fill up array
for ic,file in enumerate(listFiles):
ft = cdm.open(inDir[0]+'/'+file)
timeax = ft.getAxis('time')
try:
tmax = timeax.shape[0]
except:
print ic,file, timeax
if ic == 0:
tmax0 = tmax
#print ic,file, tmax
#adapt [t1,t2] time bounds to piControl last NN years
if useLastYears:
t1 = tmax-t20
t2 = tmax
else:
if tmax != tmax0:
print 'tmax <> tmax0',tmax,tmax0
print 'wrong time axis: exiting...'
return
#print 'Time dims:',ic, t1,t2,tmax
# read array
computeVar = True
allVars = ft.variables.keys()
if 'ptopsigmaxy' in allVars:
computeVar = False
if (var == 'ptopsigmaxy') & computeVar:
#print ' ic = ',ic
# reconstruct from isondepthg and ptopdepthxy
isond = ft('isondepthg',time = slice(t1,t2))
#print isond.data.shape, timN*latN*lonN
itest = 94*360+150
axesList = isond.getAxisList()
levs = axesList[1][:]
levN = len(levs)
#ti02 = timc.clock()
levs3d0 = mv.reshape(npy.tile(levs,latN*lonN),(latN*lonN,levN))
#ti05 = timc.clock()
isonRead = npy.ma.ones([timN,latN,lonN], dtype='float32')*valmask
for it in range(timN): # loop on time to limit memory usage
levs3d = levs3d0*1.
depthlo = mv.reshape(vardepth[ic,it,...],latN*lonN)
depth3d = npy.reshape(npy.repeat(depthlo,levN),(latN*lonN,levN))
isond3d = mv.reshape(npy.transpose(isond.data[it,...],(1,2,0)),(latN*lonN,levN))
#print isond3d[itest,:]
isond3d[isond3d > valmask/10] = 0.
#print isond3d[itest,:]
isond3dp1 = npy.roll(isond3d,-1,axis=1)
isond3dp1[:,-1] = isond3d[:,-1]
#print isond3dp1[itest,:]
#levs3d[levs3d > 30. ] = 0. # to distinguish bottom masked points from surface masked points
#print levs3d[itest,:]
levs3d[(depth3d <= isond3d)] = 0.
#print levs3d[itest,:]
levs3d[(depth3d > isond3dp1)] = 0.
#print levs3d[itest,:]
#isonwrk = npy.sum(levs3d,axis=1)
isonwrk = npy.max(levs3d,axis=1)
if it < 0:
print ic,it
print depthlo[itest]
print isond3d[itest,:]
print isonwrk[itest]
print
isonRead[it,...] = mv.reshape(isonwrk,(latN,lonN))
# <-- end of loop on time
del (isond3d,isond3dp1); gc.collect()
# mask with depthxy and where sigmaxy = 0
isonRead.mask = vardepth.mask[ic,...]
isonRead = mv.masked_where(isonRead == 0, isonRead)
isonRead.long_name = var
isonRead.units = 'sigma_n'
isonRead.id = var
del (isond,depth3d,levs3d,levs3d0,isonwrk); gc.collect()
#ti3 = timc.clock()
#print ti02-ti0,ti05-ti02, ti1-ti05,ti12-ti1,ti15-ti12,ti2-ti15,ti3-ti2
#print ti3-ti0
# write ptopsigmaxy
if os.path.isfile(inDir[0]+'/work_ptopsigmaxy/'+file):
os.remove(inDir[0]+'/work_ptopsigmaxy/'+file)
fiout = cdm.open(inDir[0]+'/work_ptopsigmaxy/'+file,'w')
if ic == 0:
print ' Creating ',inDir[0]+'/work_ptopsigmaxy/'+file
isonsigxy = cdm.createVariable(isonRead, axes = axis1D, id = 'ptopsigmaxy')
isonsigxy.long_name = 'Density of shallowest persistent ocean on ison'
isonsigxy.units = 'sigma_n'
fiout.write(isonsigxy.astype('float32'))
fiout.close()
else:
# Direct read of variable
isonRead = ft(var,time = slice(t1,t2))
#print isonRead.shape, timN
if varFill[iv] != valmask:
isonvar[ic,...] = isonRead.filled(varFill[iv])
else:
isonvar[ic,...] = isonRead
#print isonvar[ic,:,40,100]
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data,valmask).mask
percent[ic,...] = npy.float32(npy.equal(maskvar,0))
if mme:
# if mme then just average Bowl and Agree fields
varst = var+'Agree'
vardiff[ic,...] = ft(varst,time = slice(t1,t2))
else:
# Compute difference with average of first initN years, use mask of last month
varinit = cdu.averager(isonvar[ic,peri1:peri2,...],axis=0)
for tr in range(timN):
vardiff[ic,tr,...] = isonvar[ic,tr,...] - varinit
vardiff[ic,...].mask = isonvar[ic,...].mask
ft.close()
# <-- end of loop on files
# TODO remove masked points at longitudes 0 or 180deg for some models
# if ptopdepthxy, keep for ptopsigmaxy computation (reconstruct from isondepthg and ptopdepthxy)
if var =='ptopdepthxy':
vardepth = isonvar
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent,axis=0))*100.
percenta = mv.masked_less(percenta, 50)
percentw = cdm.createVariable(percenta, axes = axis1D, id = 'ptoppercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'))
# Sign of difference
if mme:
vardiffsgSum = cdu.averager(vardiff, axis=0)
vardiffsgSum = cdm.createVariable(vardiffsgSum , axes = axisVar , id = 'foo')
vardiffsgSum = maskVal(vardiffsgSum, valmask)
vardiffsgSum.mask = percentw.mask
else:
vardiffsg = npy.copysign(varones,vardiff)
# average signs
vardiffsgSum = cdu.averager(vardiffsg, axis=0)
vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
vardiffsgSum.mask = percentw.mask
vardiffsgSum._FillValue = valmask
# average accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = cdm.createVariable(isonVarAve , axes = axisVar , id = 'foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
# Write
isonave = cdm.createVariable(isonVarAve, axes = axisVar, id = isonRead.id)
isonave.long_name = isonRead.long_name
isonave.units = isonRead.units
isonavediff = cdm.createVariable(vardiffsgSum, axes = axisVar, id = isonRead.id+'Agree')
isonavediff.long_name = isonRead.long_name
isonavediff.units = isonRead.units
outFile_f.write(isonave.astype('float32'))
outFile_f.write(isonavediff.astype('float32'))
tf = timc.clock()
#print ' time var',tf-ti0
# <--- end of loop on variables
outFile_f.close()
fi.close()
def linearInterpolation(A,
I,
levels=[
100000, 92500, 85000, 70000, 60000, 50000, 40000,
30000, 25000, 20000, 15000, 10000, 7000, 5000,
3000, 2000, 1000
],
status=None):
"""
Linear interpolation
to interpolate a field from some levels to another set of levels
Value below "surface" are masked
Input
A : array to interpolate
I : interpolation field (usually Pressure or depth) from TOP (level 0) to BOTTOM (last level), i.e P value going up with each level
levels : levels to interplate to (same units as I), default levels are:[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000]
I and levels must have same units
Output
array on new levels (levels)
Examples:
A=interpolate(A,I,levels=[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000])
"""
try:
nlev = len(levels) # Number of pressure levels
except:
nlev = 1 # if only one level len(levels) would breaks
levels = [
levels,
]
order = A.getOrder()
A = A(order='z...')
I = I(order='z...')
sh = list(I.shape)
nsigma = sh[0] #number of sigma levels
sh[0] = nlev
t = MV2.zeros(sh, typecode=MV2.float32)
sh2 = I[0].shape
prev = -1
for ilev in range(nlev): # loop through pressure levels
if status is not None:
prev = genutil.statusbar(ilev, nlev - 1., prev)
lev = levels[ilev] # get value for the level
Iabv = MV2.ones(sh2, MV2.float)
Aabv = -1 * Iabv # Array on sigma level Above
Abel = -1 * Iabv # Array on sigma level Below
Ibel = -1 * Iabv # Pressure on sigma level Below
Iabv = -1 * Iabv # Pressure on sigma level Above
Ieq = MV2.masked_equal(Iabv, -1) # Area where Pressure == levels
for i in range(1, nsigma): # loop from second sigma level to last one
a = MV2.greater_equal(
I[i], lev) # Where is the pressure greater than lev
b = MV2.less_equal(I[i - 1],
lev) # Where is the pressure less than lev
# Now looks if the pressure level is in between the 2 sigma levels
# If yes, sets Iabv, Ibel and Aabv, Abel
a = MV2.logical_and(a, b)
Iabv = MV2.where(a, I[i], Iabv) # Pressure on sigma level Above
Aabv = MV2.where(a, A[i], Aabv) # Array on sigma level Above
Ibel = MV2.where(a, I[i - 1],
Ibel) # Pressure on sigma level Below
Abel = MV2.where(a, A[i - 1], Abel) # Array on sigma level Below
Ieq = MV2.where(MV2.equal(I[i], lev), A[i], Ieq)
val = MV2.masked_where(
MV2.equal(Ibel, -1.),
numpy.ones(Ibel.shape) *
lev) # set to missing value if no data below lev if there is
tl = (val - Ibel) / (Iabv - Ibel) * (Aabv -
Abel) + Abel # Interpolation
if ((Ieq.mask is None) or (Ieq.mask is MV22.nomask)):
tl = Ieq
else:
tl = MV2.where(1 - Ieq.mask, Ieq, tl)
t[ilev] = tl.astype(MV2.float32)
ax = A.getAxisList()
autobnds = cdms2.getAutoBounds()
cdms2.setAutoBounds('off')
lvl = cdms2.createAxis(MV2.array(levels).filled())
cdms2.setAutoBounds(autobnds)
try:
lvl.units = I.units
except:
pass
lvl.id = 'plev'
try:
t.units = I.units
except:
pass
ax[0] = lvl
t.setAxisList(ax)
t.id = A.id
for att in A.listattributes():
setattr(t, att, getattr(A, att))
return t(order=order)
def __call__(self, merge=[], **kargs):
""" Returns the array of values"""
# First clean up kargs
if "merge" in kargs:
merge = kargs["merge"]
del(kargs["merge"])
order = None
axes_ids = self.getAxisIds()
if "order" in kargs:
# If it's an actual axis assume that it's what user wants
# Otherwise it's an out order keyword
if "order" not in axes_ids:
order = kargs["order"]
del(kargs["order"])
ab = cdms2.getAutoBounds()
cdms2.setAutoBounds("off")
axes = self.getAxisList()
if merge != []:
if isinstance(merge[0], str):
merge = [merge, ]
if merge != []:
for merger in merge:
for merge_axis_id in merger:
if merge_axis_id not in axes_ids:
raise RuntimeError(
"You requested to merge axis is '{}' which is not valid. Axes: {}".format(
merge_axis_id, axes_ids))
sh = []
ids = []
used_ids = []
for a in axes:
# Regular axis not a merged one
sh.append(len(a)) # store length to construct array shape
ids.append(a.id) # store ids
used_ids.append(a.id)
# first let's see which vars are actually asked for
# for now assume all keys means restriction on dims
if not isinstance(merge, (list, tuple)):
raise RuntimeError(
"merge keyword must be a list of dimensions to merge together")
if len(merge) > 0 and not isinstance(merge[0], (list, tuple)):
merge = [merge, ]
for axis_id in kargs:
if axis_id not in ids:
raise ValueError("Invalid axis '%s'" % axis_id)
index = ids.index(axis_id)
value = kargs[axis_id]
if isinstance(value, basestring):
value = [value]
if not isinstance(value, (list, tuple, slice)):
raise TypeError(
"Invalid subsetting type for axis '%s', axes can only be subsetted by string,list or slice" %
axis_id)
if isinstance(value, slice):
axes[index] = axes[index].subAxis(
value.start, value.stop, value.step)
sh[index] = len(axes[index])
else: # ok it's a list
for v in value:
if v not in axes[index][:]:
raise ValueError(
"Unkwown value '%s' for axis '%s'" %
(v, axis_id))
axis = cdms2.createAxis(value, id=axes[index].id)
axes[index] = axis
sh[index] = len(axis)
array = numpy.ma.ones(sh, dtype=numpy.float)
# Now let's fill this array
self.get_array_values_from_dict_recursive(array, [], [], [], axes)
# Ok at this point we need to take care of merged axes
# First let's create the merged axes
axes_to_group = []
for merger in merge:
merged_axes = []
for axid in merger:
for ax in axes:
if ax.id == axid:
merged_axes.append(ax)
axes_to_group.append(merged_axes)
new_axes = [groupAxes(grp_axes) for grp_axes in axes_to_group]
sh2 = list(sh)
for merger in merge:
for merger in merge: # loop through all possible merging
merged_indices = []
for id in merger:
merged_indices.append(axes_ids.index(id))
for indx in merged_indices:
sh2[indx] = 1
smallest = min(merged_indices)
for indx in merged_indices:
sh2[smallest] *= sh[indx]
myorder = []
for index in range(len(sh)):
if index in myorder:
continue
for merger in merge:
merger = [axes_ids.index(x) for x in merger]
if index in merger and index not in myorder:
for indx in merger:
myorder.append(indx)
if index not in myorder: # ok did not find this one anywhere
myorder.append(index)
outData = numpy.transpose(array, myorder)
outData = numpy.reshape(outData, sh2)
yank = []
for merger in merge:
merger = [axes_ids.index(x) for x in merger]
mn = min(merger)
merger.remove(mn)
yank += merger
yank = sorted(yank, reverse=True)
for yk in yank:
extract = (slice(0, None),) * yk
extract += (0,)
outData = outData[extract]
# Ok now let's apply the newaxes
sub = 0
outData = MV2.array(outData)
merged_axis_done = []
for index in range(len(array.shape)):
foundInMerge = False
for imerge, merger in enumerate(merge):
merger = [axes_ids.index(x) for x in merger]
if index in merger:
foundInMerge = True
if imerge not in merged_axis_done:
merged_axis_done.append(imerge)
setMergedAxis = imerge
else:
setMergedAxis = -1
if not foundInMerge:
outData.setAxis(index - sub, axes[index])
else:
if setMergedAxis == -1:
sub += 1
else:
outData.setAxis(index - sub, new_axes[setMergedAxis])
outData = MV2.masked_greater(outData, 9.98e20)
outData.id = "pmp"
if order is not None:
myorder = "".join(["({})".format(nm) for nm in order])
outData = outData(order=myorder)
# Merge needs cleaning for extra dims crated
if merge != []:
for i in range(outData.ndim):
outData = scrap(outData, axis=i)
outData = MV2.masked_greater(outData, 9.9e19)
cdms2.setAutoBounds(ab)
return outData
continue
print var
#d = f(var.strip()) ; # [] creates a file variable, () loads the variable into memory
d = f(
var.strip(),
squeeze=1) # squeeze gets rid of all singleton dimensions)
print d.shape
lat = d.getLatitude()
lon = d.getLongitude()
time = d.getTime()
#print time.getBounds()
if time.getBounds() == None:
cdm.setAutoBounds('on')
print 'Time bounds could not be found, set AutoBounds on'
BoundsIssue = 'Had to set AutoBounds = ON, b/c time bounds could not be found'
print time
try:
lev = d.getLevel()
print lev
except AttributeError:
print '3D variable, no levels'
# Force local file attribute as history
cmor.set_cur_dataset_attribute('history', f.history)
#if 'sic' in varName:
# table = 'CMIP6_Amon.json'
def compute(dm, do):
""" Computes bias"""
cdms2.setAutoBounds(
'on') # Do we really want this? Wouldn't it better to let it fails
return cdutil.averager(dm, axis='t'), cdutil.averager(do, axis='t')
def logLinearInterpolation(A,P,levels=[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000],status=None):
"""
Log-linear interpolation
to convert a field from sigma levels to pressure levels
Value below surface are masked
Input
A : array on sigma levels
P : pressure field from TOP (level 0) to BOTTOM (last level)
levels : pressure levels to interplate to (same units as P), default levels are:[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000]
P and levels must have same units
Output
array on pressure levels (levels)
Examples:
A=logLinearInterpolation(A,P),levels=[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000])
"""
try:
nlev=len(levels) # Number of pressure levels
except:
nlev=1 # if only one level len(levels) would breaks
levels=[levels,]
order=A.getOrder()
A=A(order='z...')
P=P(order='z...')
sh=list(P.shape)
nsigma=sh[0] #number of sigma levels
sh[0]=nlev
t=MV2.zeros(sh,typecode=MV2.float32)
sh2=P[0].shape
prev=-1
for ilev in range(nlev): # loop through pressure levels
if status is not None:
prev=genutil.statusbar(ilev,nlev-1.,prev)
lev=levels[ilev] # get value for the level
Pabv=MV2.ones(sh2,MV2.float)
Aabv=-1*Pabv # Array on sigma level Above
Abel=-1*Pabv # Array on sigma level Below
Pbel=-1*Pabv # Pressure on sigma level Below
Pabv=-1*Pabv # Pressure on sigma level Above
Peq=MV2.masked_equal(Pabv,-1) # Area where Pressure == levels
for i in range(1,nsigma): # loop from second sigma level to last one
a=MV2.greater_equal(P[i], lev) # Where is the pressure greater than lev
b= MV2.less_equal(P[i-1],lev) # Where is the pressure less than lev
# Now looks if the pressure level is in between the 2 sigma levels
# If yes, sets Pabv, Pbel and Aabv, Abel
a=MV2.logical_and(a,b)
Pabv=MV2.where(a,P[i],Pabv) # Pressure on sigma level Above
Aabv=MV2.where(a,A[i],Aabv) # Array on sigma level Above
Pbel=MV2.where(a,P[i-1],Pbel) # Pressure on sigma level Below
Abel=MV2.where(a,A[i-1],Abel) # Array on sigma level Below
Peq= MV2.where(MV2.equal(P[i],lev),A[i],Peq)
val=MV2.masked_where(MV2.equal(Pbel,-1),numpy.ones(Pbel.shape)*lev) # set to missing value if no data below lev if there is
tl=MV2.log(val/Pbel)/MV2.log(Pabv/Pbel)*(Aabv-Abel)+Abel # Interpolation
if ((Peq.mask is None) or (Peq.mask is MV2.nomask)):
tl=Peq
else:
tl=MV2.where(1-Peq.mask,Peq,tl)
t[ilev]=tl.astype(MV2.float32)
ax=A.getAxisList()
autobnds=cdms2.getAutoBounds()
cdms2.setAutoBounds('off')
lvl=cdms2.createAxis(MV2.array(levels).filled())
cdms2.setAutoBounds(autobnds)
try:
lvl.units=P.units
except:
pass
lvl.id='plev'
try:
t.units=P.units
except:
pass
ax[0]=lvl
t.setAxisList(ax)
t.id=A.id
for att in A.listattributes():
setattr(t,att,getattr(A,att))
return t(order=order)
def testFullAveraging(self):
cdms2.setAutoBounds("on")
a = MV2.masked_greater(MV2.array([1, 4, 5, 6, 7, 8, 9.]), .5)
self.assertTrue(numpy.ma.is_masked(cdutil.averager(a)))
def mmeAveMsk3D(listFiles, years, inDir, outDir, outFile, timeInt, mme, ToeType, debug=True):
'''
The mmeAveMsk3D() function averages rhon/lat density bined files with differing masks
It ouputs
- the MME
- a percentage of non-masked bins
- the sign agreement of period2-period1 differences
- ToE per run and for MME
Author: Eric Guilyardi : [email protected]
Created on Tue Nov 21 2016
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- years(t1,t2) - years for slice read
- inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE)
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- ToeType(str) - ToE type ('F': none, 'histnat')
-> requires running first mm+mme without ToE to compute Stddev
- debug <optional> - boolean value
Notes:
-----
- EG 21 Nov 2016 - Initial function write
- TODO :
- add computation of ToE per model (toe 1 and toe 2) see ticket #50
- add isonhtc (see ticket #48)
'''
# CDMS initialisation - netCDF compression
comp = 1 # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
fi = cdm.open(inDir[0]+'/'+listFiles[0])
# Switch if only variables below the bowl are present/treated
nobowl = True
if nobowl:
isond0 = fi['isondepthgBowl'] ; # Create variable handle
else:
isond0 = fi['isondepthg'] ; # Create variable handle
# Get grid objects
axesList = isond0.getAxisList()
sigmaGrd = isond0.getLevel()
#time = isond0.getTime()
lonN = isond0.shape[3]
latN = isond0.shape[2]
levN = isond0.shape[1]
varsig='ptopsigmaxy'
# Limit number of models to 3 for testing of mme
#if mme:
# listFiles = listFiles[0:2]
# print ' !!! ### Testing 3 models ###', listFiles
# Declare and open files for writing
if os.path.isfile(outDir+'/'+outFile):
os.remove(outDir+'/'+outFile)
outFile_f = cdm.open(outDir+'/'+outFile,'w')
#timN = isond0.shape[0]
timN = t2-t1
runN = len(listFiles)
print ' Number of members:',len(listFiles)
valmask = isond0.missing_value
varList = ['isondepthg','persistmxy','sog','thetaog','isonthickg']
varFill = [valmask,valmask,valmask,valmask,valmask]
percent = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*0.
varbowl = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*1.
#varList = ['isondepthg']
#print ' !!! ### Testing one variable ###', varList
# init sigma axis
sigma = cdm.createAxis(npy.float32(range(1)))
sigma.id = axesList[1].id
sigma.units = axesList[1].units
sigma.designateTime()
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
# init ensemble axis
ensembleAxis = cdm.createAxis(npy.float32(range(runN)))
ensembleAxis.id = 'members'
ensembleAxis.units = 'N'
# Output axis
sigmaList = [sigma,axesList[2],axesList[3]] ; # sigma, lat, lon
sigmaTimeList = [sigma,time,axesList[2],axesList[3]] ; # sigma, time, lat, lon
# init arrays
isonvar = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask
varbowl2D = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask
varstd,varToE1,varToE2 = [npy.ma.ones([runN,latN,lonN], dtype='float32')*valmask for _ in range(3)]
# Loop on density levels (for memory management, becomes UNLIMITED axis and requires a ncpq to reorder dimensions)
delta_ib = 1
print ' Sigma index:'
for ib in range(levN):
ib1 = ib + delta_ib
print ib,
tim0 = timc.clock()
# loop on variables
for iv,var in enumerate(varList):
if nobowl:
varb = var+'Bowl'
else:
varb = var
if ib == 0:
print ' Variable ',iv, varb
# loop over files to fill up array
for i,file in enumerate(listFiles):
tim01 = timc.clock()
ft = cdm.open(inDir[0]+'/'+file)
model = file.split('.')[1]
timeax = ft.getAxis('time')
if i == 0:
tmax0 = timeax.shape[0]
tmax = timeax.shape[0]
if tmax != tmax0:
print 'wrong time axis: exiting...'
return
# read array
isonRead = ft(varb,time = slice(t1,t2), lev = slice(ib,ib1)).squeeze()
if varFill[iv] != valmask:
isonvar[i,...] = isonRead.filled(varFill[iv])
else:
isonvar[i,...] = isonRead
tim02 = timc.clock()
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data,valmask).mask
percent[i,...] = npy.float32(npy.equal(maskvar,0))
tim03 = timc.clock()
if mme:
# if mme then just accumulate Bowl, Agree and Std fields
#varst = var+'Agree'
#vardiff[i,...] = ft(varst,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze()
isonRead = ft(varb,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze()
varbowl2D[i,...] = isonRead
else:
# Compute difference with average of first initN years
#varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0)
#for t in range(timN):
# vardiff[i,t,...] = isonvar[i,t,...] - varinit
#vardiff[i,...].mask = isonvar[i,...].mask
# Read bowl to truncate field above bowl
if ib == 0 and iv == 0:
varbowl[i,...] = ft(varsig,time = slice(t1,t2))
#varbowl[i,...] = bowlRead
# Compute Stddev
varstd[i,...] = npy.ma.std(isonvar[i,...], axis=0)
# Compute ToE
if ToeType == 'histnat':
toto=1
# TODO
# Read mean and Std dev from histnat
# if i == 0:
# filehn = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0]
# #filehn = replace(outFile,'historical','historicalNat')
# fthn = cdm.open(filehn)
# varmeanhn = fthn(var)
# varst = var+'Std'
# varmaxstd = fthn(varst)
# toemult = 1.
# signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN))
# noise = npy.reshape(varmaxstd,(basN*levN*latN))
# varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
# toemult = 2.
# varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
tim04 = timc.clock()
ft.close()
#print 'ib, section 1 timing',ib, tim02-tim01,tim03-tim02,tim04-tim03
# <-- end of loop on files (i)
tim1 = timc.clock()
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent,axis=0))*100.
percenta = mv.masked_less(percenta, 50)
percenta = npy.reshape(percenta,[delta_ib,timN,latN,lonN])
percentw = cdm.createVariable(percenta, axes = sigmaTimeList, id = 'isonpercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'), extend = 1, index = ib)
# Sign of difference
#if mme:
# vardiffsgSum = cdu.averager(vardiff, axis=0)
# vardiffsgSum = cdm.createVariable(vardiffsgSum , axes = sigmaTimeList , id = 'foo')
# vardiffsgSum = maskVal(vardiffsgSum, valmask)
# vardiffsgSum.mask = percentw.mask
#else:
# vardiffsg = npy.copysign(varones,vardiff)
# # average signs
# vardiffsgSum = cdu.averager(vardiffsg, axis=0)
# vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
# vardiffsgSum.mask = percentw.mask
# vardiffsgSum._FillValue = valmask
# average variable accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = npy.reshape(isonVarAve,[delta_ib,timN,latN,lonN])
isonVarAve = cdm.createVariable(isonVarAve , axes = sigmaTimeList , id = 'foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
tim2 = timc.clock()
# Only keep points with rhon > bowl-delta_rho
delta_rho = 0.
# mme case
if mme: # start from average of <var>Agree
isonVarBowl = cdu.averager(varbowl2D, axis=0)
isonVarBowl = npy.reshape(isonVarBowl,[delta_ib,timN,latN,lonN])
isonVarBowl = cdm.createVariable(isonVarBowl , axes = sigmaTimeList , id = 'foo')
isonVarBowl = maskVal(isonVarBowl, valmask)
isonVarBowl.mask = percentw.mask
# Compute intermodel stddev
isonVarStd = statistics.std(varbowl2D, axis=0)
isonVarStd = npy.reshape(isonVarStd,[delta_ib,timN,latN,lonN])
isonVarStd = cdm.createVariable(isonVarStd , axes = sigmaTimeList , id = 'foo')
isonVarStd = maskVal(isonVarStd, valmask)
isonVarStd.mask = percentw.mask
# Write
isonvarbowlw = cdm.createVariable(isonVarBowl , axes = sigmaTimeList , id = isonRead.id)
isonvarbowlw.long_name = isonRead.long_name
isonvarbowlw.units = isonRead.units
isonvarstdw = cdm.createVariable(isonVarStd , axes = sigmaTimeList , id = isonRead.id+'Std')
isonvarstdw.long_name = isonRead.long_name+' intermodel std'
isonvarstdw.units = isonRead.units
outFile_f.write(isonvarbowlw.astype('float32'), extend = 1, index = ib)
outFile_f.write(isonvarstdw.astype('float32'), extend = 1, index = ib)
#if ib == 0 and iv == 0:
# # TODO review
# # Read multimodel sigma on bowl and average in time
# file1d = replace(outDir+'/'+outFile,'2D','1D')
# if os.path.isfile(file1d):
# f1d = cdm.open(file1d)
# else:
# print 'ERROR:',file1d,'missing (if mme, run 2D first)'
# sys.exit(1)
# bowlRead = f1d(varsig,time = slice(t1,t2),lev = slice(ib,ib1))
# f1d.close()
# siglimit = cdu.averager(bowlRead, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
#if sw2d == 1:
# for il in range(latN):
# for ib in range(basN):
# #if ib == 2:
# # print il, siglimit[ib,il]
# if siglimit[ib,il] < valmask/1000.:
# # if mme bowl density defined, mask above bowl
# index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
# isonVarBowl [:,ib,0:index[0],il].mask = True
# isonVarStd [:,ib,0:index[0],il].mask = True
# vardiffsgSum[:,ib,0:index[0],il].mask = True
# else:
# # mask all points
# isonVarBowl [:,ib,:,il].mask = True
# isonVarStd [:,ib,:,il].mask = True
# vardiffsgSum[:,ib,:,il].mask = True
# mm case
else:
isonVarBowl = isonVarAve*1. # start from variable
#isonVarStd = isonVarAve*1. # start from variable
if ib == 0 and iv == 0:
# build bowl position
siglimit = cdu.averager(varbowl, axis=0) # average accross members
siglimit = npy.reshape(siglimit,[timN*latN*lonN]) - delta_rho
if iv == 0:
sigarr = siglimit*1.
sigarr[:] = sigmaGrd[ib]
# test
i = 60
j = 60
ij = j*lonN+i
isonVarBowl = npy.reshape(isonVarBowl,[timN*latN*lonN])
#vardiffsgSum = npy.reshape(vardiffsgSum,[timN*latN*lonN])
isonVarBowl.mask = npy.where(sigarr < siglimit, True, isonVarBowl.mask)
#vardiffsgSum.mask = npy.where(sigarr < siglimit, True, vardiffsgSum.mask)
isonVarBowl = npy.reshape(isonVarBowl,[timN,latN,lonN])
#vardiffsgSum = npy.reshape(vardiffsgSum,[timN,latN,lonN])
isonVarBowl = maskVal(isonVarBowl, valmask)
#vardiffsgSum = maskVal(vardiffsgSum, valmask)
# Find max of Std dev of all members
isonVarStd = npy.ma.max(varstd, axis=0)
# mask
isonVarStd = maskVal(isonVarStd, valmask)
# Write
#isonave = cdm.createVariable(isonVarAve, axes = sigmaTimeList, id = isonRead.id)
#isonave.long_name = isonRead.long_name
#isonave.units = isonRead.units
#vardiffsgSum = npy.reshape(vardiffsgSum,[delta_ib,timN,latN,lonN])
#isonavediff = cdm.createVariable(vardiffsgSum, axes = sigmaTimeList, id = isonRead.id+'Agree')
#isonavediff.long_name = isonRead.long_name
#isonavediff.units = isonRead.units
isonVarBowl = npy.reshape(isonVarBowl,[delta_ib,timN,latN,lonN])
isonavebowl = cdm.createVariable(isonVarBowl, axes = sigmaTimeList, id = isonRead.id+'Bowl')
isonavebowl.long_name = isonRead.long_name
isonavebowl.units = isonRead.units
isonVarStd = npy.reshape(isonVarStd,[delta_ib,latN,lonN])
isonmaxstd = cdm.createVariable(isonVarStd, axes = sigmaList, id = isonRead.id+'Std')
isonmaxstd.long_name = isonRead.long_name
isonmaxstd.units = isonRead.units
#outFile_f.write( isonave.astype('float32'), extend = 1, index = ib)
#outFile_f.write(isonavediff.astype('float32'), extend = 1, index = ib)
outFile_f.write(isonavebowl.astype('float32'), extend = 1, index = ib)
outFile_f.write(isonmaxstd.astype('float32'), extend = 1, index = ib)
tim3 = timc.clock()
if ToeType == 'histnat':
isontoe1 = cdm.createVariable(varToE1, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE1')
isontoe1.long_name = 'ToE 1 for '+isonRead.long_name
isontoe1.units = 'Year'
isontoe2 = cdm.createVariable(varToE2, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE2')
isontoe2.long_name = 'ToE 2 for '+isonRead.long_name
isontoe2.units = 'Year'
outFile_f.write(isontoe1.astype('float32'), extend = 1, index = ib)
outFile_f.write(isontoe2.astype('float32'), extend = 1, index = ib)
tim4 = timc.clock()
# <--- end of loop on variables
#print 'ib, timing',ib, tim01-tim0,tim1-tim01,tim2-tim1,tim3-tim2,tim4-tim3
# <--- end of loop on density
print ' '
outFile_f.close()
fi.close()
import nappy.utils
import nappy.nc_interface.nc_to_na
# Import external packages (if available)
if sys.platform.find("win") > -1:
raise na_error.NAPlatformError("Windows does not support CDMS. CDMS is required to convert to CDMS objects and NetCDF.")
try:
import cdms2 as cdms
except:
try:
import cdms
except:
raise Exception("Could not import third-party software. Nappy requires the CDMS and Numeric packages to be installed to convert to CDMS and NetCDF.")
cdms.setAutoBounds("off")
# Define global variables
DEBUG = nappy.utils.getDebug()
default_delimiter = nappy.utils.getDefault("default_delimiter")
default_float_format = nappy.utils.getDefault("default_float_format")
# Define logger
logging.basicConfig()
log = logging.getLogger(__name__)
class CDMSObjectsToNAFile(nappy.nc_interface.nc_to_na.NCToNA):
"""
Converts a set of CDMS Objects to one or more NASA Ames files.
"""
inputFilePathend = '/HadISSTv1.1/' #xmls/'
#inputFilePathend = '/TMP/'
inputFileName = ['HadISST_sst.nc'] #['HadISST.xml']
inputVarName = ['sst']
outputVarName = ['ts']
outputUnits = ['K']
### BETTER IF THE USER DOES NOT CHANGE ANYTHING BELOW THIS LINE...
#%% Process variable (with time axis)
# Open and read input netcdf file
for fi in range(len(inputVarName)):
inputFilePath = inputFilePathbgn + inputFilePathend
f = cdm.open(inputFilePath + inputFileName[fi])
cdm.setAutoBounds('on')
d = f(inputVarName[fi])
d = d + 273.15
cdm.setAutoBounds('off')
#cdutil.times.setTimeBoundsMonthly(d)
lat = d.getLatitude()
lon = d.getLongitude()
#time = d.getTime() ; # Assumes variable is named 'time', for the demo file this is named 'months'
time = d.getAxis(0)
# Rather use a file dimension-based load statement
# Deal with problematic "months since" calendar/time axis
time_bounds = time.getBounds()
#time_bounds[:,0] = time[:]
#time_bounds[:-1,1] = time[1:]
#time_bounds[-1,1] = time_bounds[-1,0]+1
#!/usr/bin/env python
# Adapted for numpy/ma/cdms2 by convertcdms.py
var = 'tas'
import cdtime, cdms2, os, sys, vcs
from cdutil import times
import MV2
cdms2.setAutoBounds('on')
f = cdms2.open(os.path.join(vcs.prefix, 'sample_data', 'tas_mo.nc'))
fsc = cdms2.open(os.path.join(vcs.prefix, 'sample_data', 'tas_mo_clim.nc'))
print "Step #0 : Reading data"
s = f(var, longitude=(0, 360, 'co'))
acok = fsc('climseas', longitude=(0, 360, 'co'))
print 'Test #1 : Test result'
ac = times.JAN.climatology(s)
assert (MV2.allclose(ac[0], acok[0]))
f.close()
fsc.close()
a = cdtime.comptime(1980)
b = cdtime.comptime(1980, 5)
f = cdms2.open(os.path.join(vcs.prefix, 'sample_data', 'tas_6h.nc'))
def plot(self,data,template = None, bg=0, x=None):
if x is None:
x = self.x
if template is None:
template = self.template
elif isinstance(template,str):
template = x.gettemplate(template)
elif not vcs.istemplate(template):
raise "Error did not know what to do with template: %s" % template
if not isinstance(data,cdms2.tvariable.TransientVariable):
mode= cdms2.getAutoBounds()
cdms2.setAutoBounds("on")
data = MV2.array(data)
data.getAxis(-1).getBounds()
cdms2.setAutoBounds(mode)
while data.rank()>1:
data = data[0]
# ok now we have a good x and a good data
nbars = len(data)
# create the primitive
fill = x.createfillarea()
line = x.createline()
fill.viewport = [template.data.x1,template.data.x2,template.data.y1,template.data.y2]
line.viewport = [template.data.x1,template.data.x2,template.data.y1,template.data.y2]
axb = data.getAxis(0).getBounds()
xmn,xmx = vcs.minmax(axb)
ymn,ymx = vcs.minmax(data)
xmn,xmx,ymn,ymx = self.prep_plot(xmn,xmx,ymn,ymx)
fill.worldcoordinate=[xmn,xmx,ymn,ymx]
line.worldcoordinate=[xmn,xmx,ymn,ymx]
styles =[]
cols = []
indices = []
lt = []
lw =[]
lc = []
xs = []
ys = []
for i in range(nbars):
if i < len(self.fillareastyles):
styles.append(self.fillareastyles[i])
else:
styles.append(self.fillareastyles[-1])
if i < len(self.fillareacolors):
cols.append(self.fillareacolors[i])
else:
cols.append(self.fillareacolors[-1])
if i < len(self.fillareaindices):
indices.append(self.fillareaindices[i])
else:
indices.append(self.fillareaindices[-1])
if i < len(self.line):
lt.append( self.line[i])
else:
lt.append(self.line[-1])
if i < len(self.linewidth):
lw.append( self.linewidth[i])
else:
lw.append(self.linewidth[-1])
if i < len(self.line):
lc.append( self.linecolors[i])
else:
lc.append(self.linecolors[-1])
xs.append( [axb[i][0],axb[i][1],axb[i][1],axb[i][0],axb[i][0]])
ys.append( [0,0,data[i],data[i],0])
fill.style = styles
fill.x = xs
fill.y = ys
fill.style
fill.index = indices
fill.color = cols
line.x = xs
line.y = ys
line.type = lt
line.width = lw
line.color = lc
displays = []
displays.append(x.plot(fill,bg=bg))
displays.append(x.plot(line,bg=bg))
x.worldcoordinate = fill.worldcoordinate
dsp = template.plot(data,self,bg=bg)
for d in dsp:
displays.append(d)
self.restore()
return displays
version = nappy.utils.getVersion()
# Import external packages (if available)
if sys.platform.find("win") > -1:
raise na_error.NAPlatformError("Windows does not support CDMS. CDMS is required to convert to CDMS objects and NetCDF.")
try:
import cdms2 as cdms
import numpy as N
except:
try:
import cdms
import Numeric as N
except:
raise Exception("Could not import third-party software. Nappy requires the CDMS and Numeric packages to be installed to convert to CDMS and NetCDF.")
cdms.setAutoBounds("off")
DEBUG = nappy.utils.getDebug()
logging.basicConfig()
log = logging.getLogger(__name__)
class NAContentCollector(nappy.na_file.na_core.NACore):
"""
Class to build a NASA Ames File object from a set of
CDMS variables and global attributes (optional).
"""
def __init__(self, variables, global_attributes=[], requested_ffi=None):
"""
Sets up instance variables and calls appropriate methods to
generate sections of NASA Ames file object.
def correctFile(idxcorr, ncorr, inFile, inDir, outFile, outDir):
'''
Correct density binned files (undefined ptop & long 0 issue)
idxcorr = [idx_i,idx_i1,jmax] indices for longitude correction - if [0,0,0] ignore
ncorr = number of corrections: 1 or 2
'''
# CDMS initialisation - netCDF compression
comp = 1 # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
varList3D = ['isondepthg', 'isonthickg', 'sog', 'thetaog']
varList2D = [
'ptopsoxy', 'ptopdepthxy', 'ptopsigmaxy', 'ptopthetaoxy', 'persistmxy'
]
# First test, read level by level and write level by level (memory management)
# use ncpdq -a time,lev,lat,lon to recover the dimension order
fi = cdm.open(inDir + '/' + inFile)
fo = cdm.open(outDir + '/' + outFile, 'w')
isondg = fi['isondepthg']
# Create variable handle
# Get grid objects
#axesList = isondg.getAxisList()
#sigmaGrd = isondg.getLevel()
lonN = isondg.shape[3]
latN = isondg.shape[2]
levN = isondg.shape[1]
timN = isondg.shape[0]
#valmask = isondg.missing_value
if ncorr == 2:
ic1 = idxcorr[0][0]
ic2 = idxcorr[0][1]
jcmax = idxcorr[0][2]
ic12 = idxcorr[1][0]
ic22 = idxcorr[1][1]
jcmax2 = idxcorr[1][2]
if ic2 >= lonN - 1:
ic2 = 0
if ic22 >= lonN - 1:
ic22 = 0
elif ncorr == 1:
ic1 = idxcorr[0]
ic2 = idxcorr[1]
jcmax = idxcorr[2]
if ic2 >= lonN - 1:
ic2 = 0
#print ic1,ic2,jcmax
corr_long = True
if ic1 == 0 and ic2 == 0 and jcmax == 0:
corr_long = False
#testp = 10
for it in range(timN):
#if it/testp*testp == it:
# print ' year =',it
# test
#i = 90
#j = 90
#i2d = 6
#j2d = 12
#ij = j*lonN+i
#ij2d = j2d*lonN+i2d
#print 'ij=',ij
# 3D variables
for iv in varList3D:
#print iv
outVar = fi(iv, time=slice(it, it + 1))
# Correct for longitude interpolation issue
if corr_long:
for jt in range(jcmax):
outVar[:, :, jt, ic1] = (outVar[:, :, jt, ic1 - 1] +
outVar[:, :, jt, ic2 + 1]) / 2
outVar[:, :, jt, ic2] = outVar[:, :, jt, ic1]
if ncorr == 2:
for jt in range(jcmax2):
outVar[:, :, jt,
ic12] = (outVar[:, :, jt, ic12 - 1] +
outVar[:, :, jt, ic22 + 1]) / 2
outVar[:, :, jt, ic22] = outVar[:, :, jt, ic12]
# # Correct Bowl properties
# if iv =='isondepthg':
# vardepth = npy.reshape(outVar,(levN,latN*lonN))
# #print 'test'
# #print outVar[:,:,j2d,i2d]
# #print vardepth[:,ij2d]
# # find values of surface points
# vardepthBowl = npy.min(npy.reshape(outVar,(levN,latN*lonN)),axis=0)
# vardepthBowlTile = npy.repeat(vardepthBowl,levN,axis=0).reshape((latN*lonN,levN)).transpose()
# #print vardepthBowlTile.shape
# #print vardepthBowl[ij2d], vardepthBowlTile[:,ij2d]
# levs = outVar.getAxisList()[1][:]
# #print 'levs',levs
# levs3d = mv.reshape(npy.tile(levs,latN*lonN),(latN*lonN,levN)).transpose()
# varsigmaBowl = npy.max(npy.where(vardepth == vardepthBowlTile,levs3d,0),axis=0)
# #print varsigmaBowl[ij2d],levs3d[:,ij2d]
# elif iv == 'sog':
# varsog = npy.reshape(outVar,(levN,latN*lonN))
# varsoBowl = npy.max(npy.where(vardepth == vardepthBowlTile,varsog,0),axis=0)
# #print varsoBowl[ij2d], varsog[:,ij2d]
# #print vardepth[:,ij2d],vardepthBowlTile[:,ij2d]
# del (varsog); gc.collect()
# elif iv =='thetaog':
# varthetao = npy.reshape(outVar,(levN,latN*lonN))
# varthetaoBowl = npy.max(npy.where(vardepth == vardepthBowlTile,varthetao,-1000),axis=0)
# #print varthetaoBowl[ij2d],varthetao[:,ij2d]
# del (varthetao); gc.collect()
# Write
fo.write(outVar.astype('float32'), extend=1, index=it)
fo.sync()
del (vardepth)
gc.collect()
# 2D variables and correct isondepthg = 0
for iv in varList2D:
outVar = fi(iv, time=slice(it, it + 1))
# Correct for longitude interpolation issue
if corr_long:
for jt in range(jcmax):
outVar[:, jt, ic1] = (outVar[:, jt, ic1 - 1] +
outVar[:, jt, ic2 + 1]) / 2
outVar[:, jt, ic2] = outVar[:, jt, ic1]
if ncorr == 2:
for jt in range(jcmax2):
outVar[:, jt, ic12] = (outVar[:, jt, ic12 - 1] +
outVar[:, jt, ic22 + 1]) / 2
outVar[:, jt, ic22] = outVar[:, jt, ic12]
# Correct for ptopsoxy < 30
#print 'before',outVar[:,j2d,i2d]
# if iv == 'ptopsoxy':
# testso = npy.reshape(outVar,(latN*lonN)) < 30.
# #print 'testdepth', testdepth[ij2d]
# #print npy.argwhere(testdepth)[0:10]/lonN, npy.argwhere(testdepth)[0:10]-npy.argwhere(testdepth)[0:10]/lonN*lonN
# outVar.data[...] = npy.where(testso,varsoBowl,npy.reshape(outVar,(latN*lonN))).reshape(outVar.shape)[...]
# elif iv == 'ptopdepthxy':
# outVar.data[...] = npy.where(testso,vardepthBowl,npy.reshape(outVar,(latN*lonN))).reshape(outVar.shape)[...]
# elif iv == 'ptopthetaoxy':
# outVar.data[...] = npy.where(testso,varthetaoBowl,npy.reshape(outVar,(latN*lonN))).reshape(outVar.shape)[...]
# elif iv == 'ptopsigmaxy':
# outVar.data[...] = npy.where(testso,varsigmaBowl,npy.reshape(outVar,(latN*lonN))).reshape(outVar.shape)[...]
#print 'after',outVar[:,j2d,i2d]
# Write
fo.write(outVar.astype('float32'), extend=1, index=it)
fo.sync()
fi.close()
fo.close()
# testing
#model = 'CCSM4'
#idxcorr=[139,140,145]
#ncorr = 1
#inFile = 'cmip5.CCSM4.historical24.r1i1p1.an.ocn.Omon.density.ver-v20121128.nc'
#inDir = '/Users/ericg/Projets/Density_bining/Raw_testing'
#outFile = 'cmip5.CCSM4.historical24.outtest.nc'
#model = 'CanESM2'
#idxcorr=[179,180,180]
#ncorr = 1
#inFile = 'cmip5.CanESM2.historical24.r1i1p1.an.ocn.Omon.density.ver-1.nc'
#inDir = '/Users/ericg/Projets/Density_bining/Raw_testing'
#outFile = 'cmip5.CanESM2.historical24.outtest.nc'
#model = 'IPSL-CM5A-LR'
#idxcorr=[0,0,0]
#ncorr=1
#inFile = 'cmip5.IPSL-CM5A-LR.historical24.r1i1p1.an.ocn.Omon.density.ver-v20111119.nc'
#inDir = '/Users/ericg/Projets/Density_bining/Raw_testing'
#outFile = 'cmip5.IPSL-CM5A-LR.historical24.outtest.nc'
#model = 'Ishii'
#idxcorr=[[359,359,39],[180,180,180]]
#idxcorr=[359,359,39]
#ncorr = 1
#inFile = 'obs.Ishii.historical.r0i0p0.an.ocn.Omon.density.ver-1.latestX.nc'
#inDir='/Volumes/hciclad/data/Density_binning/Prod_density_obs_april16'
#outFile = 'obs.Ishii.historical.r0i0p0.an.ocn.Omon.density.ver-1.latestXCorr.nc'
#model = 'EN4'
#idxcorr=[[359,359,39],[180,180,180]]
#idxcorr=[359,359,39]
#ncorr = 2
#inFile = 'obs.EN4.historical.r0i0p0.mo.ocn.Omon.density.ver-1.latestX.nc'
#inDir='/Volumes/hciclad/data/Density_binning/Prod_density_obs_april16'
#outFile = 'obs.EN4.historical.r0i0p0.mo.ocn.Omon.density.ver-1.latestXCorr.nc'
#outDir = inDir
#correctFile(idxcorr, ncorr, inFile, inDir, outFile, outDir)
def mmeAveMsk2D(listFiles, years, inDir, outDir, outFile, timeInt, mme, timeBowl, ToeType, debug=True):
'''
The mmeAveMsk2D() function averages rhon/lat density bined files with differing masks
It ouputs
- the MME
- a percentage of non-masked bins
- the sign agreement of period2-period1 differences
- ToE per run and for MME
Author: Eric Guilyardi : [email protected]
Created on Tue Nov 25 13:56:20 CET 2014
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- years(t1,t2) - years for slice read
- inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE)
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- timeBowl - either time 'mean' or time 'max' bowl used to mask out bowl
- ToeType(str) - ToE type ('F': none, 'histnat')
-> requires running first mm+mme without ToE to compute Stddev
- debug <optional> - boolean value
Notes:
-----
- EG 25 Nov 2014 - Initial function write
- EG 27 Nov 2014 - Rewrite with loop on variables
- EG 06 Dec 2014 - Added agreement on difference with init period - save as <var>Agree
- EG 07 Dec 2014 - Read bowl to remove points above bowl - save as <var>Bowl
- EG 19 Apr 2016 - ToE computation (just for 2D files)
- EG 07 Oct 2016 - add 3D file support
- EG 21 Nov 2016 - move 3D support to new function
- EG 10 jan 2017 - added timeBowl option
- TODO :
- remove loops
- add computation of ToE per model (toe 1 and toe 2) see ticket #50
- add isonhtc (see ticket #48)
'''
# CDMS initialisation - netCDF compression
comp = 1 # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
if t2 <= 0:
useLastYears = True
t2 = -t2
else:
useLastYears = False
t10 = t1
t20 = t2
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
fi = cdm.open(inDir[0]+'/'+listFiles[0])
isond0 = fi['isondepth'] ; # Create variable handle
# Get grid objects
axesList = isond0.getAxisList()
sigmaGrd = isond0.getLevel()
latN = isond0.shape[3]
levN = isond0.shape[2]
basN = isond0.shape[1]
varsig='ptopsigma'
# Declare and open files for writing
if os.path.isfile(outDir+'/'+outFile):
os.remove(outDir+'/'+outFile)
outFile_f = cdm.open(outDir+'/'+outFile,'w')
# Testing mme with less models
#listFiles=listFiles[0:4]
#timN = isond0.shape[0]
timN = t2-t1
runN = len(listFiles)
print ' Number of members:',len(listFiles)
valmask = isond0.missing_value[0]
varList = ['isondepth','isonpers','isonso','isonthetao','isonthick','isonvol']
varFill = [0.,0.,valmask,valmask,0.,0.]
# init arrays (2D rho/lat)
percent = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*0.
#minbowl = npy.ma.ones([basN,latN], dtype='float32')*1000.
varbowl = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*1.
#varList = ['isondepth']
#print ' !!! ### Testing one variable ###'
#varList = ['isonthetao']
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
time.designateTime()
# init ensemble axis
ensembleAxis = cdm.createAxis(npy.float32(range(runN)))
ensembleAxis.id = 'members'
ensembleAxis.units = 'N'
# loop on variables
for iv,var in enumerate(varList):
# Array inits (2D rho/lat 3D rho/lat/lon)
#shapeR = [basN,levN,latN]
isonvar = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*valmask
vardiff,varbowl2D = [npy.ma.ones(npy.ma.shape(isonvar)) for _ in range(2)]
varstd,varToE1,varToE2 = [npy.ma.ones([runN,basN,levN,latN], dtype='float32')*valmask for _ in range(3)]
varones = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*1.
print ' Variable ',iv, var
# loop over files to fill up array
for i,file in enumerate(listFiles):
ft = cdm.open(inDir[0]+'/'+file)
model = file.split('.')[1]
timeax = ft.getAxis('time')
file1d = replace(inDir[0]+'/'+file,'2D','1D')
if os.path.isfile(file1d):
f1d = cdm.open(file1d)
else:
print 'ERROR:',file1d,'missing (if mme, run 1D first)'
sys.exit(1)
tmax = timeax.shape[0]
if i == 0:
tmax0 = tmax
#adapt [t1,t2] time bounds to piControl last NN years
if useLastYears:
t1 = tmax-t20
t2 = tmax
else:
if tmax != tmax0:
print 'wrong time axis: exiting...'
return
# read array
# loop over time/density for memory management
for it in range(timN):
t1r = t1 + it
t2r = t1r + 1
isonRead = ft(var,time = slice(t1r,t2r))
if varFill[iv] != valmask:
isonvar[i,it,...] = isonRead.filled(varFill[iv])
else:
isonvar[i,it,...] = isonRead
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data,valmask).mask
percent[i,...] = npy.float32(npy.equal(maskvar,0))
if mme:
# if mme then just accumulate Bowl, Agree fields
varst = var+'Agree'
vardiff[i,...] = ft(varst,time = slice(t1,t2))
varb = var+'Bowl'
varbowl2D[i,...] = ft(varb,time = slice(t1,t2))
else:
# Compute difference with average of first initN years
varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0)
for t in range(timN):
vardiff[i,t,...] = isonvar[i,t,...] - varinit
vardiff[i,...].mask = isonvar[i,...].mask
# Read bowl and truncate 2D field above bowl
if iv == 0:
bowlRead = f1d(varsig,time = slice(t1,t2))
varbowl[i,...] = bowlRead
# Compute Stddev
varstd[i,...] = npy.ma.std(isonvar[i,...], axis=0)
# Compute ToE
if ToeType == 'histnat':
# Read mean and Std dev from histnat
if i == 0:
filehn = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0]
#filehn = replace(outFile,'historical','historicalNat')
fthn = cdm.open(filehn)
varmeanhn = fthn(var)
varst = var+'Std'
varmaxstd = fthn(varst)
toemult = 1.
signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN))
noise = npy.reshape(varmaxstd,(basN*levN*latN))
varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
toemult = 2.
varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
ft.close()
f1d.close()
# <-- end of loop on files
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent,axis=0))*100.
percenta = mv.masked_less(percenta, 50)
percentw = cdm.createVariable(percenta, axes = [time,axesList[1],axesList[2],axesList[3]], id = 'isonpercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'))
# Sign of difference
if mme:
vardiffsgSum = cdu.averager(vardiff, axis=0)
vardiffsgSum = cdm.createVariable(vardiffsgSum , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
vardiffsgSum = maskVal(vardiffsgSum, valmask)
vardiffsgSum.mask = percentw.mask
else:
vardiffsg = npy.copysign(varones,vardiff)
# average signs
vardiffsgSum = cdu.averager(vardiffsg, axis=0)
vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
vardiffsgSum.mask = percentw.mask
vardiffsgSum._FillValue = valmask
# average variable accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = cdm.createVariable(isonVarAve , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
# Only keep points with rhon > bowl-delta_rho
delta_rho = 0.
if mme: # start from average of <var>Agree
isonVarBowl = cdu.averager(varbowl2D, axis=0)
isonVarBowl = cdm.createVariable(isonVarBowl , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
isonVarBowl = maskVal(isonVarBowl, valmask)
isonVarBowl.mask = percentw.mask
# Compute intermodel stddev
isonVarStd = statistics.std(varbowl2D, axis=0)
isonVarStd = cdm.createVariable(isonVarStd , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
isonVarStd = maskVal(isonVarStd, valmask)
isonVarStd.mask = percentw.mask
if iv == 0:
# Read mulitmodel sigma on bowl and average in time
file1d = replace(outDir+'/'+outFile,'2D','1D')
if os.path.isfile(file1d):
f1d = cdm.open(file1d)
else:
print 'ERROR:',file1d,'missing (if mme, run 1D first)'
sys.exit(1)
bowlRead = f1d(varsig,time = slice(t1,t2))
f1d.close()
siglimit = cdu.averager(bowlRead, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
for il in range(latN):
for ib in range(basN):
#if ib == 2:
# print il, siglimit[ib,il]
if siglimit[ib,il] < valmask/1000.:
# if mme bowl density defined, mask above bowl
index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
isonVarBowl [:,ib,0:index[0],il].mask = True
isonVarStd [:,ib,0:index[0],il].mask = True
vardiffsgSum[:,ib,0:index[0],il].mask = True
else:
# mask all points
isonVarBowl [:,ib,:,il].mask = True
isonVarStd [:,ib,:,il].mask = True
vardiffsgSum[:,ib,:,il].mask = True
else:
isonVarBowl = isonVarAve*1. # start from variable
isonVarStd = isonVarAve*1. # start from variable
if iv == 0:
siglimit = cdu.averager(varbowl, axis=0) # average accross members
# Average bowl in time
if timeBowl == 'mean':
siglimit = cdu.averager(siglimit, axis=0) - delta_rho
# or take largest sigma over time
else:
siglimit = npy.ma.max(siglimit, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
for il in range(latN):
for ib in range(basN):
if siglimit[ib,il] < valmask/1000.:
# if bowl density defined, mask above bowl
index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
isonVarBowl[:,ib,0:index[0],il].mask = True
vardiffsgSum[:,ib,0:index[0],il].mask = True
else:
# mask all points
vardiffsgSum[:,ib,:,il].mask = True
isonVarBowl = maskVal(isonVarBowl, valmask)
# Find max of Std dev of all members
isonVarStd = npy.ma.max(varstd, axis=0)
# mask
if varFill[iv] == valmask:
isonVarStd = maskVal(isonVarStd, valmask)
# Write
isonave = cdm.createVariable(isonVarAve, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id)
isonave.long_name = isonRead.long_name
isonave.units = isonRead.units
isonavediff = cdm.createVariable(vardiffsgSum, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Agree')
isonavediff.long_name = isonRead.long_name
isonavediff.units = isonRead.units
isonavebowl = cdm.createVariable(isonVarBowl, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Bowl')
isonavebowl.long_name = isonRead.long_name
isonavebowl.units = isonRead.units
if not mme:
isonmaxstd = cdm.createVariable(isonVarStd, axes = [axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Std')
isonmaxstd.long_name = isonRead.long_name
isonmaxstd.units = isonRead.units
outFile_f.write( isonave.astype('float32'))
outFile_f.write(isonavediff.astype('float32'))
outFile_f.write(isonavebowl.astype('float32'))
if not mme:
outFile_f.write( isonmaxstd.astype('float32'))
if ToeType == 'histnat':
isontoe1 = cdm.createVariable(varToE1, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE1')
isontoe1.long_name = 'ToE 1 for '+isonRead.long_name
isontoe1.units = 'Year'
isontoe2 = cdm.createVariable(varToE2, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE2')
isontoe2.long_name = 'ToE 2 for '+isonRead.long_name
isontoe2.units = 'Year'
outFile_f.write(isontoe1.astype('float32'))
outFile_f.write(isontoe2.astype('float32'))
if mme:
isonvarstd = cdm.createVariable(isonVarStd , axes =[time,axesList[1],axesList[2],axesList[3]] , id = isonRead.id+'ModStd')
isonvarstd.long_name = isonRead.long_name+' intermodel std'
isonvarstd.units = isonRead.units
outFile_f.write(isonvarstd.astype('float32'))
# <--- end of loop on variables
outFile_f.close()
fi.close()
def plot(self, data, data2, template=None, bg=0, x=None):
if x is None:
x = self.x
if template is None:
template = self.template
elif isinstance(template, str):
template = x.gettemplate(template)
elif not vcs.istemplate(template):
raise "Error did not know what to do with template: %s" % template
if not isinstance(data, cdms2.tvariable.TransientVariable):
mode = cdms2.getAutoBounds()
cdms2.setAutoBounds("on")
data = MV2.array(data)
data.getAxis(-1).getBounds()
cdms2.setAutoBounds(mode)
while data.ndim > 1:
data = data[0]
while data2.ndim > 1:
data2 = data2[0]
# ok now we have a good x and a good data
# create the primitive
fill = x.createfillarea()
line = x.createline()
fill.viewport = [
template.data.x1, template.data.x2, template.data.y1,
template.data.y2
]
line.viewport = [
template.data.x1, template.data.x2, template.data.y1,
template.data.y2
]
ax = data.getAxis(0)[:]
ax2 = data.getAxis(0)[:]
xmn, xmx = vcs.minmax(ax, ax2)
ymn, ymx = vcs.minmax(data, data2)
xmn, xmx, ymn, ymx = self.prep_plot(xmn, xmx, ymn, ymx)
fill.worldcoordinate = [xmn, xmx, ymn, ymx]
line.worldcoordinate = [xmn, xmx, ymn, ymx]
fill.style = [
self.fillareastyle,
]
fill.color = [
self.fillareacolor,
]
fill.index = [
self.fillareaindex,
]
line.type = [
self.linetype,
]
line.width = [
self.linewidth,
]
line.color = [
self.linecolor,
]
xs = []
ys = []
xs = numpy.concatenate((ax[:], ax2[::-1])).tolist()
ys = numpy.concatenate((data[:], data2[::-1])).tolist()
xs.append(xs[0])
ys.append(ys[0])
fill.x = xs
fill.y = ys
line.x = xs
line.y = ys
displays = []
displays.append(x.plot(fill, bg=bg))
displays.append(x.plot(line, bg=bg))
x.worldcoordinate = fill.worldcoordinate
dsp = template.plot(x, data, self, bg=bg)
for d in dsp:
displays.append(d)
self.restore()
return displays
def mmeAveMsk1D(listFiles,
sw2d,
years,
inDir,
outDir,
outFile,
timeInt,
mme,
ToeType,
fullTS,
debug=True):
'''
The mmeAveMsk1D() function averages rhon or scalar density bined files with differing masks
It ouputs the MME and a percentage of non-masked bins
Created on Tue Nov 25 13:56:20 CET 2014
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- sw2d - dimension of fields to consider (1 or 2)
- years(t1,t2) - years for slice read
- inDir(str) - input directory where files are stored
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- FfllTS - 0/1: if 1, uses full time serie (ignores years(t1,t2))
- debug <optional> - boolean value
Notes:
-----
- EG 25 Nov 2014 - Initial function write
- EG 9 Dec 2014 - Add agreement on difference with init period - save as <var>Agree
- EG 04 Oct 2016 - Add 3D files support
TODO:
------
'''
# CDMS initialisation - netCDF compression
comp = 1
# 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
if t2 <= 0:
useLastYears = True
t2 = -t2
else:
useLastYears = False
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
# Find dimension
runN = len(listFiles)
try:
fi = cdm.open(inDir[0] + '/' + listFiles[0])
except:
print ' *** file not found ', inDir[0] + '/' + listFiles[0]
sys.exit(' Abort')
if sw2d == 1:
ptopd0 = fi['ptopdepth']
# Create variable handle
latN = ptopd0.shape[2]
basN = ptopd0.shape[1]
elif sw2d == 2:
ptopd0 = fi['ptopdepthxy']
# Create variable handle
lonN = ptopd0.shape[2]
latN = ptopd0.shape[1]
#timN = ptopd0.shape[0]
timN = t2 - t1
if fullTS:
print ' !!! Working on full Time Serie (fullTS = True)'
timN = ptopd0.shape[0]
t1 = 0
t2 = timN
t10 = t1
t20 = t2
# Get grid objects
axesList = ptopd0.getAxisList()
# Declare and open files for writing
if os.path.isfile(outDir + '/' + outFile):
os.remove(outDir + '/' + outFile)
outFile_f = cdm.open(outDir + '/' + outFile, 'w')
print ' Number of members:', len(listFiles)
valmask = ptopd0.missing_value
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
time.designateTime()
# loop on variables
# init percent array
if sw2d == 1:
varList = [
'ptopdepth', 'ptopsigma', 'ptopso', 'ptopthetao', 'volpers',
'salpers', 'tempers'
]
#varList = ['ptopdepth']
varDim = [1, 1, 1, 1, 0, 0, 0]
percent = npy.ma.ones([runN, timN, basN, latN], dtype='float32') * 0.
elif sw2d == 2:
varList = ['ptopdepthxy', 'ptopsigmaxy', 'ptopsoxy', 'ptopthetaoxy']
#varList = ['ptopdepthxy']
varDim = [2, 2, 2, 2]
percent = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * 0.
varFill = [
valmask, valmask, valmask, valmask, valmask, valmask, valmask, valmask,
valmask
]
axis1D = [time, axesList[1], axesList[2]]
axis0D = [time, axesList[1]]
print ' timN = ', timN
# loop on 1D variables
for iv, var in enumerate(varList):
ti0 = timc.clock()
# Array inits
if varDim[iv] == 2:
isonvar = npy.ma.ones([runN, timN, latN, lonN],
dtype='float32') * valmask
vardiff = npy.ma.ones([runN, timN, latN, lonN],
dtype='float32') * valmask
varones = npy.ma.ones([runN, timN, latN, lonN],
dtype='float32') * 1.
axisVar = axis1D
elif varDim[iv] == 1:
isonvar = npy.ma.ones([runN, timN, basN, latN],
dtype='float32') * valmask
vardiff = npy.ma.ones([runN, timN, basN, latN],
dtype='float32') * valmask
varones = npy.ma.ones([runN, timN, basN, latN],
dtype='float32') * 1.
axisVar = axis1D
else:
isonvar = npy.ma.ones([runN, timN, basN],
dtype='float32') * valmask
vardiff = npy.ma.ones([runN, timN, basN],
dtype='float32') * valmask
varones = npy.ma.ones([runN, timN, basN], dtype='float32') * 1.
axisVar = axis0D
print ' Variable ', iv, var, varDim[iv]
# loop over files to fill up array
for ic, file in enumerate(listFiles):
ft = cdm.open(inDir[0] + '/' + file)
timeax = ft.getAxis('time')
try:
tmax = timeax.shape[0]
except:
print ic, file, timeax
if ic == 0:
tmax0 = tmax
#print ic,file, tmax
#adapt [t1,t2] time bounds to piControl last NN years
if useLastYears:
t1 = tmax - t20
t2 = tmax
else:
if tmax != tmax0:
print 'tmax <> tmax0', tmax, tmax0
print 'wrong time axis: exiting...'
return
#print 'Time dims:',ic, t1,t2,tmax
# read array
computeVar = True
allVars = ft.variables.keys()
if 'ptopsigmaxy' in allVars:
computeVar = False
if (var == 'ptopsigmaxy') & computeVar:
#print ' ic = ',ic
# reconstruct from isondepthg and ptopdepthxy
isond = ft('isondepthg', time=slice(t1, t2))
#print isond.data.shape, timN*latN*lonN
itest = 94 * 360 + 150
axesList = isond.getAxisList()
levs = axesList[1][:]
levN = len(levs)
#ti02 = timc.clock()
levs3d0 = mv.reshape(npy.tile(levs, latN * lonN),
(latN * lonN, levN))
#ti05 = timc.clock()
isonRead = npy.ma.ones([timN, latN, lonN],
dtype='float32') * valmask
for it in range(timN): # loop on time to limit memory usage
levs3d = levs3d0 * 1.
depthlo = mv.reshape(vardepth[ic, it, ...], latN * lonN)
depth3d = npy.reshape(npy.repeat(depthlo, levN),
(latN * lonN, levN))
isond3d = mv.reshape(
npy.transpose(isond.data[it, ...], (1, 2, 0)),
(latN * lonN, levN))
#print isond3d[itest,:]
isond3d[isond3d > valmask / 10] = 0.
#print isond3d[itest,:]
isond3dp1 = npy.roll(isond3d, -1, axis=1)
isond3dp1[:, -1] = isond3d[:, -1]
#print isond3dp1[itest,:]
#levs3d[levs3d > 30. ] = 0. # to distinguish bottom masked points from surface masked points
#print levs3d[itest,:]
levs3d[(depth3d <= isond3d)] = 0.
#print levs3d[itest,:]
levs3d[(depth3d > isond3dp1)] = 0.
#print levs3d[itest,:]
#isonwrk = npy.sum(levs3d,axis=1)
isonwrk = npy.max(levs3d, axis=1)
if it < 0:
print ic, it
print depthlo[itest]
print isond3d[itest, :]
print isonwrk[itest]
print
isonRead[it, ...] = mv.reshape(isonwrk, (latN, lonN))
# <-- end of loop on time
del (isond3d, isond3dp1)
gc.collect()
# mask with depthxy and where sigmaxy = 0
isonRead.mask = vardepth.mask[ic, ...]
isonRead = mv.masked_where(isonRead == 0, isonRead)
isonRead.long_name = var
isonRead.units = 'sigma_n'
isonRead.id = var
del (isond, depth3d, levs3d, levs3d0, isonwrk)
gc.collect()
#ti3 = timc.clock()
#print ti02-ti0,ti05-ti02, ti1-ti05,ti12-ti1,ti15-ti12,ti2-ti15,ti3-ti2
#print ti3-ti0
# write ptopsigmaxy
if os.path.isfile(inDir[0] + '/work_ptopsigmaxy/' + file):
os.remove(inDir[0] + '/work_ptopsigmaxy/' + file)
fiout = cdm.open(inDir[0] + '/work_ptopsigmaxy/' + file, 'w')
if ic == 0:
print ' Creating ', inDir[0] + '/work_ptopsigmaxy/' + file
isonsigxy = cdm.createVariable(isonRead,
axes=axis1D,
id='ptopsigmaxy')
isonsigxy.long_name = 'Density of shallowest persistent ocean on ison'
isonsigxy.units = 'sigma_n'
fiout.write(isonsigxy.astype('float32'))
fiout.close()
else:
# Direct read of variable
isonRead = ft(var, time=slice(t1, t2))
#print isonRead.shape, timN
if varFill[iv] != valmask:
isonvar[ic, ...] = isonRead.filled(varFill[iv])
else:
isonvar[ic, ...] = isonRead
#print isonvar[ic,:,40,100]
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data, valmask).mask
percent[ic, ...] = npy.float32(npy.equal(maskvar, 0))
if mme:
# if mme then just average Bowl and Agree fields
varst = var + 'Agree'
vardiff[ic, ...] = ft(varst, time=slice(t1, t2))
else:
# Compute difference with average of first initN years, use mask of last month
varinit = cdu.averager(isonvar[ic, peri1:peri2, ...], axis=0)
for tr in range(timN):
vardiff[ic, tr, ...] = isonvar[ic, tr, ...] - varinit
vardiff[ic, ...].mask = isonvar[ic, ...].mask
ft.close()
# <-- end of loop on files
# TODO remove masked points at longitudes 0 or 180deg for some models
# if ptopdepthxy, keep for ptopsigmaxy computation (reconstruct from isondepthg and ptopdepthxy)
if var == 'ptopdepthxy':
vardepth = isonvar
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent, axis=0)) * 100.
percenta = mv.masked_less(percenta, 50)
percentw = cdm.createVariable(percenta,
axes=axis1D,
id='ptoppercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'))
# Sign of difference
if mme:
vardiffsgSum = cdu.averager(vardiff, axis=0)
vardiffsgSum = cdm.createVariable(vardiffsgSum,
axes=axisVar,
id='foo')
vardiffsgSum = maskVal(vardiffsgSum, valmask)
vardiffsgSum.mask = percentw.mask
else:
vardiffsg = npy.copysign(varones, vardiff)
# average signs
vardiffsgSum = cdu.averager(vardiffsg, axis=0)
vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
vardiffsgSum.mask = percentw.mask
vardiffsgSum._FillValue = valmask
# average accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = cdm.createVariable(isonVarAve, axes=axisVar, id='foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
# Write
isonave = cdm.createVariable(isonVarAve, axes=axisVar, id=isonRead.id)
isonave.long_name = isonRead.long_name
isonave.units = isonRead.units
isonavediff = cdm.createVariable(vardiffsgSum,
axes=axisVar,
id=isonRead.id + 'Agree')
isonavediff.long_name = isonRead.long_name
isonavediff.units = isonRead.units
outFile_f.write(isonave.astype('float32'))
outFile_f.write(isonavediff.astype('float32'))
tf = timc.clock()
#print ' time var',tf-ti0
# <--- end of loop on variables
outFile_f.close()
fi.close()
def compute(dm,do):
""" Computes bias"""
cdms2.setAutoBounds('on') # Do we really want this? Wouldn't it better to let it fails
return cdutil.averager(dm,axis='t'),cdutil.averager(do,axis='t')
def surfTransf(fileFx, fileTos, fileSos, fileHef, fileWfo, varNames, outFile, debug=True, timeint='all',noInterp=False, domain='global'):
'''
The surfTransf() function takes files and variable arguments and creates
density bined surface transformation fields which are written to a specified outfile
Author: Eric Guilyardi : [email protected]
Co-author: Paul J. Durack : [email protected] : @durack1.
Created on Wed Oct 8 09:15:59 CEST 2014
Inputs:
------
- fileTos(time,lat,lon) - 3D SST array
- fileSos(time,lat,lon) - 3D SSS array
- fileHef(time,lat,lon) - 3D net surface heat flux array
- fileWfo(time,lat,lon) - 3D fresh water flux array
- fileFx(lat,lon) - 2D array containing the cell area values
- varNames[4] - 1D array containing the names of the variables
- outFile(str) - output file with full path specified.
- debug <optional> - boolean value
- timeint <optional> - specify temporal step for binning <init_idx>,<ncount>
- noInterp <optional> - if true no interpolation to target grid
- domain <optional> - specify domain for averaging when interpolated to WOA grid ('global','north',
'north40', 'south' for now)
Outputs:
--------
- netCDF file with monthly surface rhon, density fluxes, transformation (global and per basin)
- use cdo yearmean to compute annual mean
Usage:
------
'>>> from binDensity import surfTransf
'>>> surfTransf(file_fx, file_tos, file_sos, file_hef, file_wfo, [var1,var2,var3,var4]./output.nc, debug=True,timeint='all')
Notes:
-----
- EG 8 Oct 2014 - Initial function write and tests ok
- PJD 22 Nov 2014 - Code cleanup
- EG 4 Oct 2017 - code on ciclad, more cleanup and options
- EG 12 Sep 2018 - Add North vs. South calculation
'''
# Keep track of time (CPU and elapsed)
cpu0 = timc.clock()
#
# netCDF compression (use 0 for netCDF3)
comp = 1
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
#
# == Inits
#
npy.set_printoptions(precision = 2)
# Determine file name from inputs
modeln = fileTos.split('/')[-1].split('.')[1]
#
if debug:
print ' Debug - File names:'
print ' ', fileTos
print ' ', fileSos
debugp = True
else:
debugp = False
#
# Open files
ftos = cdm.open(fileTos)
fsos = cdm.open(fileSos)
fhef = cdm.open(fileHef)
fwfo = cdm.open(fileWfo)
#timeax = ftos.getAxis('time')
timeax = ftos.getAxis('time_counter')
#print 'timeax'
#print timeax
#
# Dates to read
if timeint == 'all':
tmin = 0
tmax = timeax.shape[0]
timeaxis = timeax
else:
tmin = int(timeint.split(',')[0]) - 1
tmax = tmin + int(timeint.split(',')[1])
# update time axis
timeaxis = cdm.createAxis(timeax[tmin:tmax])
timeaxis.id = 'time'
timeaxis.units = timeax.units
timeaxis.designateTime()
#print timeaxis
if debugp:
print; print ' Debug mode'
# Read file attributes to carry on to output files
list_file = ftos.attributes.keys()
file_dic = {}
for i in range(0,len(list_file)):
file_dic[i]=list_file[i],ftos.attributes[list_file[i] ]
#
# Read data
# varnames
tos_name = varNames[0]
sos_name = varNames[1]
hef_name = varNames[2]
wfo_name = varNames[3]
if debugp:
print' Read ',tos_name, sos_name,tmin,tmax
tos = ftos(tos_name , time = slice(tmin,tmax))
sos = fsos(sos_name , time = slice(tmin,tmax))
if debugp:
print' Read ',hef_name, wfo_name
qnet = fhef(hef_name, time = slice(tmin,tmax))
try:
emp = fwfo(wfo_name , time = slice(tmin,tmax))
empsw = 0
except Exception,err:
emp = fwfo('wfos' , time = slice(tmin,tmax))
print ' Reading concentration dillution fresh water flux'
empsw = 0
#!/usr/bin/env python
# Adapted for numpy/ma/cdms2 by convertcdms.py
import cdms2,cdutil,sys,os,numpy
cdms2.setAutoBounds('on')
f = cdms2.open(os.path.join(sys.prefix,'sample_data','th_yr.nc'))
th=f('th',time=slice(-3,None,1))
t=th.getTime()
cdutil.setTimeBoundsYearly(t)
assert(th.shape==(3,64,128))
assert(numpy.equal(th.getTime().getBounds()[0],[1997.,1998.]).all())
dep=cdutil.YEAR.departures(th,statusbar=None)
assert(dep.shape==(3, 64, 128))
from durolib import fixVarUnits,globalAttWrite,writeToLog
from numpy import mod
from numpy.core import shape
if 'e' in locals():
del(e,pi,sctypeNA,typeNA)
gc.collect()
# Set netcdf file criterion - turned on from default 0s
cdm.setCompressionWarnings(0) ; # Suppress warnings
cdm.setNetcdfShuffleFlag(0)
cdm.setNetcdfDeflateFlag(1)
cdm.setNetcdfDeflateLevelFlag(9)
# Hi compression: 1.4Gb file ; # Single salt variable
# No compression: 5.6Gb ; Standard (compression/shuffling): 1.5Gb ; Hi compression w/ shuffling: 1.5Gb
cdm.setAutoBounds(1) ; # Ensure bounds on time and depth axes are generated
start_time = time.time() ; # Set time counter
# Set conditional whether files are created or just numbers are calculated
parser = argparse.ArgumentParser()
parser.add_argument('model_suite',metavar='str',type=str,help='include \'cmip3/5\' as a command line argument')
parser.add_argument('experiment',metavar='str',type=str,help='including \'experiment\' will select one experiment to process')
parser.add_argument('realm',nargs='?',default='',metavar='str',type=str,help='including optional argument \'realm\' will subprocess variables')
args = parser.parse_args()
# Get realm and experiment
all_experiments = False ; all_realms = False ; # Preset variables before testing
if (args.model_suite not in ['cmip3','cmip5']):
print "** No valid model suite specified - no *.nc files will be written **"
sys.exit()
import cmor
import cdms2 as cdm
import numpy as np
import MV2 as mv
import cdutil
import cdtime
cdm.setAutoBounds(
'on'
) # Caution, this attempts to automatically set coordinate bounds - please check outputs using this option
#import pdb ; # Debug statement - import if enabling below
#%% User provided input
cmorTable = '../Tables/PMPObs_Amon.json'
# Aday,Amon,Lmon,Omon,SImon,fx,monNobs,monStderr - Load target table, axis info (coordinates, grid*) and CVs
inputJson = 'ERA5-MARS-input.json'
# Update contents of this file to set your global_attributes
inputFilePathbgn = '/p/user_pub/pmp/pmp_obs_preparation/orig/data/ERA5-CREATEIP/'
inputFilePathend = ['']
#inputFileName = ['ua_ERA5.xml','va_ERA5.xml','ta_ERA5.xml','zg_ERA5.xml']
inputFileName = ['ta_ERA5.xml', 'zg_ERA5.xml']
inputFileName = [
'CREATE-IP.reanalysis.ECMWF.IFS-Cy41r2.ERA5.atmos.mon.v20200608.psl_Amon_reanalysis_ERA5_197901-201912.nc',
'CREATE-IP.reanalysis.ECMWF.IFS-Cy41r2.ERA5.atmos.mon.v20200608.tas_Amon_reanalysis_ERA5_197901-201912.nc',
'CREATE-IP.reanalysis.ECMWF.IFS-Cy41r2.ERA5.atmos.mon.v20200608.ts_Amon_reanalysis_ERA5_197901-201912.nc'
]
#inputVarName = ['ua','va'] #,'ta','zg']
inputVarName = ['psl', 'tas', 'ts']
import cdms2
import cdutil
import MV2
#Data downloaded from here https://www.metoffice.gov.uk/hadobs/hadisst/data/download.html
#This script is to fix overlapping timebounds from original data
#also add ice mask
# Set nc classic as outputs
cdms2.setCompressionWarnings(0)
# Suppress warnings
cdms2.setNetcdfShuffleFlag(0)
cdms2.setNetcdfDeflateFlag(1)
# was 0 130717
cdms2.setNetcdfDeflateLevelFlag(9)
# was 0 130717
cdms2.setAutoBounds(1)
# Ensure bounds on time and depth axes are generated
filepath = '/p/user_pub/e3sm/zhang40/analysis_data_e3sm_diags/HadISST/original_data/'
filename1 = 'HadISST_ice.nc'
filename2 = 'HadISST_sst.nc'
fin1 = cdms2.open(filepath + filename1)
fin2 = cdms2.open(filepath + filename2)
ice = fin1('sic')
sst = fin2('sst')
fout = cdms2.open(filepath + 'HadISST_sst_ice_masked.nc', 'w')
sst_masked = MV2.masked_where(ice > 0, sst, copy=True)
sst_masked.id = 'sst'
cdutil.setTimeBoundsMonthly(sst_masked)
fout.write(sst_masked)
def __call__(self, merge=[], **kargs):
""" Returns the array of values"""
# First clean up kargs
if "merge" in kargs:
merge = kargs["merge"]
del (kargs["merge"])
order = None
axes_ids = self.getAxisIds()
if "order" in kargs:
# If it's an actual axis assume that it's what user wants
# Otherwise it's an out order keyword
if "order" not in axes_ids:
order = kargs["order"]
del (kargs["order"])
ab = cdms2.getAutoBounds()
cdms2.setAutoBounds("off")
axes = self.getAxisList()
if merge != []:
if isinstance(merge[0], str):
merge = [
merge,
]
if merge != []:
for merger in merge:
for merge_axis_id in merger:
if merge_axis_id not in axes_ids:
raise RuntimeError(
"You requested to merge axis is '{}' which is not valid. Axes: {}"
.format(merge_axis_id, axes_ids))
sh = []
ids = []
used_ids = []
for a in axes:
# Regular axis not a merged one
sh.append(len(a)) # store length to construct array shape
ids.append(a.id) # store ids
used_ids.append(a.id)
# first let's see which vars are actually asked for
# for now assume all keys means restriction on dims
if not isinstance(merge, (list, tuple)):
raise RuntimeError(
"merge keyword must be a list of dimensions to merge together")
if len(merge) > 0 and not isinstance(merge[0], (list, tuple)):
merge = [
merge,
]
for axis_id in kargs:
if axis_id not in ids:
raise ValueError("Invalid axis '%s'" % axis_id)
index = ids.index(axis_id)
value = kargs[axis_id]
if isinstance(value, basestring):
value = [value]
if not isinstance(value, (list, tuple, slice)):
raise TypeError(
"Invalid subsetting type for axis '%s', axes can only be subsetted by string,list or slice"
% axis_id)
if isinstance(value, slice):
axes[index] = axes[index].subAxis(value.start, value.stop,
value.step)
sh[index] = len(axes[index])
else: # ok it's a list
for v in value:
if v not in axes[index][:]:
raise ValueError("Unkwown value '%s' for axis '%s'" %
(v, axis_id))
axis = cdms2.createAxis(value, id=axes[index].id)
axes[index] = axis
sh[index] = len(axis)
array = numpy.ma.ones(sh, dtype=numpy.float)
# Now let's fill this array
self.get_array_values_from_dict_recursive(array, [], [], [], axes)
# Ok at this point we need to take care of merged axes
# First let's create the merged axes
axes_to_group = []
for merger in merge:
merged_axes = []
for axid in merger:
for ax in axes:
if ax.id == axid:
merged_axes.append(ax)
axes_to_group.append(merged_axes)
new_axes = [groupAxes(grp_axes) for grp_axes in axes_to_group]
sh2 = list(sh)
for merger in merge:
for merger in merge: # loop through all possible merging
merged_indices = []
for id in merger:
merged_indices.append(axes_ids.index(id))
for indx in merged_indices:
sh2[indx] = 1
smallest = min(merged_indices)
for indx in merged_indices:
sh2[smallest] *= sh[indx]
myorder = []
for index in range(len(sh)):
if index in myorder:
continue
for merger in merge:
merger = [axes_ids.index(x) for x in merger]
if index in merger and index not in myorder:
for indx in merger:
myorder.append(indx)
if index not in myorder: # ok did not find this one anywhere
myorder.append(index)
outData = numpy.transpose(array, myorder)
outData = numpy.reshape(outData, sh2)
yank = []
for merger in merge:
merger = [axes_ids.index(x) for x in merger]
mn = min(merger)
merger.remove(mn)
yank += merger
yank = sorted(yank, reverse=True)
for yk in yank:
extract = (slice(0, None), ) * yk
extract += (0, )
outData = outData[extract]
# Ok now let's apply the newaxes
sub = 0
outData = MV2.array(outData)
merged_axis_done = []
for index in range(len(array.shape)):
foundInMerge = False
for imerge, merger in enumerate(merge):
merger = [axes_ids.index(x) for x in merger]
if index in merger:
foundInMerge = True
if imerge not in merged_axis_done:
merged_axis_done.append(imerge)
setMergedAxis = imerge
else:
setMergedAxis = -1
if not foundInMerge:
outData.setAxis(index - sub, axes[index])
else:
if setMergedAxis == -1:
sub += 1
else:
outData.setAxis(index - sub, new_axes[setMergedAxis])
outData = MV2.masked_greater(outData, 9.98e20)
outData.id = "pmp"
if order is not None:
myorder = "".join(["({})".format(nm) for nm in order])
outData = outData(order=myorder)
# Merge needs cleaning for extra dims crated
if merge != []:
for i in range(outData.ndim):
outData = scrap(outData, axis=i)
outData = MV2.masked_greater(outData, 9.9e19)
cdms2.setAutoBounds(ab)
return outData
def mmeAveMsk2D(listFiles,
years,
inDir,
outDir,
outFile,
timeInt,
mme,
timeBowl,
ToeType,
debug=True):
'''
The mmeAveMsk2D() function averages rhon/lat density bined files with differing masks
It ouputs
- the MME
- a percentage of non-masked bins
- the sign agreement of period2-period1 differences
- ToE per run and for MME
Author: Eric Guilyardi : [email protected]
Created on Tue Nov 25 13:56:20 CET 2014
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- years(t1,t2) - years for slice read
- inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE)
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- timeBowl - either time 'mean' or time 'max' bowl used to mask out bowl
- ToeType(str) - ToE type ('F': none, 'histnat')
-> requires running first mm+mme without ToE to compute Stddev
- debug <optional> - boolean value
Notes:
-----
- EG 25 Nov 2014 - Initial function write
- EG 27 Nov 2014 - Rewrite with loop on variables
- EG 06 Dec 2014 - Added agreement on difference with init period - save as <var>Agree
- EG 07 Dec 2014 - Read bowl to remove points above bowl - save as <var>Bowl
- EG 19 Apr 2016 - ToE computation (just for 2D files)
- EG 07 Oct 2016 - add 3D file support
- EG 21 Nov 2016 - move 3D support to new function
- EG 10 jan 2017 - added timeBowl option
- TODO :
- remove loops
- add computation of ToE per model (toe 1 and toe 2) see ticket #50
- add isonhtc (see ticket #48)
'''
# CDMS initialisation - netCDF compression
comp = 1 # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
if t2 <= 0:
useLastYears = True
t2 = -t2
else:
useLastYears = False
t10 = t1
t20 = t2
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
fi = cdm.open(inDir[0] + '/' + listFiles[0])
isond0 = fi['isondepth']
# Create variable handle
# Get grid objects
axesList = isond0.getAxisList()
sigmaGrd = isond0.getLevel()
latN = isond0.shape[3]
levN = isond0.shape[2]
basN = isond0.shape[1]
varsig = 'ptopsigma'
# Declare and open files for writing
if os.path.isfile(outDir + '/' + outFile):
os.remove(outDir + '/' + outFile)
outFile_f = cdm.open(outDir + '/' + outFile, 'w')
# Testing mme with less models
#listFiles=listFiles[0:4]
#timN = isond0.shape[0]
timN = t2 - t1
runN = len(listFiles)
print ' Number of members:', len(listFiles)
valmask = isond0.missing_value[0]
varList = [
'isondepth', 'isonpers', 'isonso', 'isonthetao', 'isonthick', 'isonvol'
]
varFill = [0., 0., valmask, valmask, 0., 0.]
# init arrays (2D rho/lat)
percent = npy.ma.ones([runN, timN, basN, levN, latN], dtype='float32') * 0.
#minbowl = npy.ma.ones([basN,latN], dtype='float32')*1000.
varbowl = npy.ma.ones([runN, timN, basN, latN], dtype='float32') * 1.
#varList = ['isondepth']
#print ' !!! ### Testing one variable ###'
#varList = ['isonthetao']
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
time.designateTime()
# init ensemble axis
ensembleAxis = cdm.createAxis(npy.float32(range(runN)))
ensembleAxis.id = 'members'
ensembleAxis.units = 'N'
# loop on variables
for iv, var in enumerate(varList):
# Array inits (2D rho/lat 3D rho/lat/lon)
#shapeR = [basN,levN,latN]
isonvar = npy.ma.ones([runN, timN, basN, levN, latN],
dtype='float32') * valmask
print('isonvar shape: ', isonvar.shape)
vardiff, varbowl2D = [
npy.ma.ones([runN, timN, basN, levN, latN], dtype='float32')
for _ in range(2)
]
varstd, varToE1, varToE2 = [
npy.ma.ones([runN, basN, levN, latN], dtype='float32') * valmask
for _ in range(3)
]
varones = npy.ma.ones([runN, timN, basN, levN, latN],
dtype='float32') * 1.
print ' Variable ', iv, var
# loop over files to fill up array
for i, file in enumerate(listFiles):
ft = cdm.open(inDir[0] + '/' + file)
model = file.split('.')[1]
timeax = ft.getAxis('time')
file1d = replace(inDir[0] + '/' + file, '2D', '1D')
if os.path.isfile(file1d):
f1d = cdm.open(file1d)
else:
print 'ERROR:', file1d, 'missing (if mme, run 1D first)'
sys.exit(1)
tmax = timeax.shape[0]
if i == 0:
tmax0 = tmax
#adapt [t1,t2] time bounds to piControl last NN years
if useLastYears:
t1 = tmax - t20
t2 = tmax
else:
if tmax != tmax0:
print 'wrong time axis: exiting...'
return
# read array
# loop over time/density for memory management
for it in range(timN):
t1r = t1 + it
t2r = t1r + 1
isonRead = ft(var, time=slice(t1r, t2r))
if varFill[iv] != valmask:
isonvar[i, it, ...] = isonRead.filled(varFill[iv])
else:
isonvar[i, it, ...] = isonRead
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data, valmask).mask
percent[i, ...] = npy.float32(npy.equal(maskvar, 0))
if mme:
# if mme then just accumulate Bowl, Agree fields
varst = var + 'Agree'
vardiff[i, ...] = ft(varst, time=slice(t1, t2))
varb = var + 'Bowl'
varbowl2D[i, ...] = ft(varb, time=slice(t1, t2))
else:
# Compute difference with average of first initN years
varinit = cdu.averager(isonvar[i, peri1:peri2, ...], axis=0)
for t in range(timN):
vardiff[i, t, ...] = isonvar[i, t, ...] - varinit
vardiff[i, ...].mask = isonvar[i, ...].mask
# Read bowl and truncate 2D field above bowl
if iv == 0:
bowlRead = f1d(varsig, time=slice(t1, t2))
varbowl[i, ...] = bowlRead
# Compute Stddev
varstd[i, ...] = npy.ma.std(isonvar[i, ...], axis=0)
# Compute ToE
if ToeType == 'histnat':
# Read mean and Std dev from histnat
if i == 0:
filehn = glob.glob(inDir[1] + '/cmip5.' + model +
'.*zon2D*')[0]
#filehn = replace(outFile,'historical','historicalNat')
fthn = cdm.open(filehn)
varmeanhn = fthn(var)
varst = var + 'Std'
varmaxstd = fthn(varst)
toemult = 1.
signal = npy.reshape(isonvar[i, ...] - varmeanhn,
(timN, basN * levN * latN))
noise = npy.reshape(varmaxstd, (basN * levN * latN))
varToE1[i,
...] = npy.reshape(findToE(signal, noise, toemult),
(basN, levN, latN))
toemult = 2.
varToE2[i,
...] = npy.reshape(findToE(signal, noise, toemult),
(basN, levN, latN))
ft.close()
f1d.close()
# <-- end of loop on files
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent, axis=0)) * 100.
percenta = mv.masked_less(percenta, 50)
percentw = cdm.createVariable(
percenta,
axes=[time, axesList[1], axesList[2], axesList[3]],
id='isonpercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'))
# Sign of difference
if mme:
vardiffsgSum = cdu.averager(vardiff, axis=0)
vardiffsgSum = cdm.createVariable(
vardiffsgSum,
axes=[time, axesList[1], axesList[2], axesList[3]],
id='foo')
vardiffsgSum = maskVal(vardiffsgSum, valmask)
vardiffsgSum.mask = percentw.mask
else:
vardiffsg = npy.copysign(varones, vardiff)
# average signs
vardiffsgSum = cdu.averager(vardiffsg, axis=0)
vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
vardiffsgSum.mask = percentw.mask
vardiffsgSum._FillValue = valmask
# average variable accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = cdm.createVariable(
isonVarAve,
axes=[time, axesList[1], axesList[2], axesList[3]],
id='foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
# Only keep points with rhon > bowl-delta_rho
delta_rho = 0.
if mme: # start from average of <var>Agree
isonVarBowl = cdu.averager(varbowl2D, axis=0)
isonVarBowl = cdm.createVariable(
isonVarBowl,
axes=[time, axesList[1], axesList[2], axesList[3]],
id='foo')
isonVarBowl = maskVal(isonVarBowl, valmask)
isonVarBowl.mask = percentw.mask
# Compute intermodel stddev
isonVarStd = statistics.std(varbowl2D, axis=0)
isonVarStd = cdm.createVariable(
isonVarStd,
axes=[time, axesList[1], axesList[2], axesList[3]],
id='foo')
isonVarStd = maskVal(isonVarStd, valmask)
isonVarStd.mask = percentw.mask
if iv == 0:
# Read mulitmodel sigma on bowl and average in time
file1d = replace(outDir + '/' + outFile, '2D', '1D')
if os.path.isfile(file1d):
f1d = cdm.open(file1d)
else:
print 'ERROR:', file1d, 'missing (if mme, run 1D first)'
sys.exit(1)
bowlRead = f1d(varsig, time=slice(t1, t2))
f1d.close()
siglimit = cdu.averager(bowlRead, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
for il in range(latN):
for ib in range(basN):
#if ib == 2:
# print il, siglimit[ib,il]
if siglimit[ib, il] < valmask / 1000.:
# if mme bowl density defined, mask above bowl
index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib, il]))
isonVarBowl[:, ib, 0:index[0], il].mask = True
isonVarStd[:, ib, 0:index[0], il].mask = True
vardiffsgSum[:, ib, 0:index[0], il].mask = True
else:
# mask all points
isonVarBowl[:, ib, :, il].mask = True
isonVarStd[:, ib, :, il].mask = True
vardiffsgSum[:, ib, :, il].mask = True
else:
isonVarBowl = isonVarAve * 1. # start from variable
isonVarStd = isonVarAve * 1. # start from variable
if iv == 0:
siglimit = cdu.averager(varbowl,
axis=0) # average accross members
# Average bowl in time
if timeBowl == 'mean':
siglimit = cdu.averager(siglimit, axis=0) - delta_rho
# or take largest sigma over time
else:
siglimit = npy.ma.max(siglimit, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
for il in range(latN):
for ib in range(basN):
if siglimit[ib, il] < valmask / 1000.:
# if bowl density defined, mask above bowl
index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib, il])
)[:, 0] #Add [:,0] for python Yona
#import code
#code.interact(banner='index', local=dict(locals(), **globals()))
isonVarBowl[:, ib, 0:index[0], il].mask = True
vardiffsgSum[:, ib, 0:index[0], il].mask = True
else:
# mask all points
vardiffsgSum[:, ib, :, il].mask = True
isonVarBowl = maskVal(isonVarBowl, valmask)
# Find max of Std dev of all members
isonVarStd = npy.ma.max(varstd, axis=0)
# mask
if varFill[iv] == valmask:
isonVarStd = maskVal(isonVarStd, valmask)
# Write
isonave = cdm.createVariable(
isonVarAve,
axes=[time, axesList[1], axesList[2], axesList[3]],
id=isonRead.id)
isonave.long_name = isonRead.long_name
isonave.units = isonRead.units
isonavediff = cdm.createVariable(
vardiffsgSum,
axes=[time, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'Agree')
isonavediff.long_name = isonRead.long_name
isonavediff.units = isonRead.units
isonavebowl = cdm.createVariable(
isonVarBowl,
axes=[time, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'Bowl')
isonavebowl.long_name = isonRead.long_name
isonavebowl.units = isonRead.units
if not mme:
isonmaxstd = cdm.createVariable(
isonVarStd,
axes=[axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'Std')
isonmaxstd.long_name = isonRead.long_name
isonmaxstd.units = isonRead.units
outFile_f.write(isonave.astype('float32'))
outFile_f.write(isonavediff.astype('float32'))
outFile_f.write(isonavebowl.astype('float32'))
if not mme:
outFile_f.write(isonmaxstd.astype('float32'))
if ToeType == 'histnat':
isontoe1 = cdm.createVariable(
varToE1,
axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'ToE1')
isontoe1.long_name = 'ToE 1 for ' + isonRead.long_name
isontoe1.units = 'Year'
isontoe2 = cdm.createVariable(
varToE2,
axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'ToE2')
isontoe2.long_name = 'ToE 2 for ' + isonRead.long_name
isontoe2.units = 'Year'
outFile_f.write(isontoe1.astype('float32'))
outFile_f.write(isontoe2.astype('float32'))
if mme:
isonvarstd = cdm.createVariable(
isonVarStd,
axes=[time, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'ModStd')
isonvarstd.long_name = isonRead.long_name + ' intermodel std'
isonvarstd.units = isonRead.units
outFile_f.write(isonvarstd.astype('float32'))
# <--- end of loop on variables
outFile_f.close()
fi.close()
# flake8:noqa
import cdms2
import cdutil
import MV2
# Data downloaded from here https://www.metoffice.gov.uk/hadobs/hadisst/data/download.html
# This script is to fix overlapping timebounds from original data
# also add ice mask
# Set nc classic as outputs
cdms2.setCompressionWarnings(0) # Suppress warnings
cdms2.setNetcdfShuffleFlag(0)
cdms2.setNetcdfDeflateFlag(1) # was 0 130717
cdms2.setNetcdfDeflateLevelFlag(9) # was 0 130717
cdms2.setAutoBounds(1) # Ensure bounds on time and depth axes are generated
filepath = "/p/user_pub/e3sm/zhang40/analysis_data_e3sm_diags/HadISST/original_data/"
filename1 = "HadISST_ice.nc"
filename2 = "HadISST_sst.nc"
fin1 = cdms2.open(filepath + filename1)
fin2 = cdms2.open(filepath + filename2)
ice = fin1("sic")
sst = fin2("sst")
fout = cdms2.open(filepath + "HadISST_sst_ice_masked.nc", "w")
sst_masked = MV2.masked_where(ice > 0, sst, copy=True)
sst_masked.id = "sst"
cdutil.setTimeBoundsMonthly(sst_masked)
# reverse latitude so that latitude in ascending
sst_masked = sst_masked[:, ::-1, :]
fout.write(sst_masked)
def mmeAveMsk3D(listFiles,
years,
inDir,
outDir,
outFile,
timeInt,
mme,
ToeType,
debug=True):
'''
The mmeAveMsk3D() function averages rhon/lat density bined files with differing masks
It ouputs
- the MME
- a percentage of non-masked bins
- the sign agreement of period2-period1 differences
- ToE per run and for MME
Author: Eric Guilyardi : [email protected]
Created on Tue Nov 21 2016
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- years(t1,t2) - years for slice read
- inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE)
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- ToeType(str) - ToE type ('F': none, 'histnat')
-> requires running first mm+mme without ToE to compute Stddev
- debug <optional> - boolean value
Notes:
-----
- EG 21 Nov 2016 - Initial function write
- TODO :
- add computation of ToE per model (toe 1 and toe 2) see ticket #50
- add isonhtc (see ticket #48)
'''
# CDMS initialisation - netCDF compression
comp = 1 # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
fi = cdm.open(inDir[0] + '/' + listFiles[0])
# Switch if only variables below the bowl are present/treated
nobowl = True
if nobowl:
isond0 = fi['isondepthgBowl']
# Create variable handle
else:
isond0 = fi['isondepthg']
# Create variable handle
# Get grid objects
axesList = isond0.getAxisList()
sigmaGrd = isond0.getLevel()
#time = isond0.getTime()
lonN = isond0.shape[3]
latN = isond0.shape[2]
levN = isond0.shape[1]
varsig = 'ptopsigmaxy'
# Limit number of models to 3 for testing of mme
#if mme:
# listFiles = listFiles[0:2]
# print ' !!! ### Testing 3 models ###', listFiles
# Declare and open files for writing
if os.path.isfile(outDir + '/' + outFile):
os.remove(outDir + '/' + outFile)
outFile_f = cdm.open(outDir + '/' + outFile, 'w')
#timN = isond0.shape[0]
timN = t2 - t1
runN = len(listFiles)
print ' Number of members:', len(listFiles)
valmask = isond0.missing_value
varList = ['isondepthg', 'persistmxy', 'sog', 'thetaog', 'isonthickg']
varFill = [valmask, valmask, valmask, valmask, valmask]
percent = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * 0.
varbowl = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * 1.
#varList = ['isondepthg']
#print ' !!! ### Testing one variable ###', varList
# init sigma axis
sigma = cdm.createAxis(npy.float32(range(1)))
sigma.id = axesList[1].id
sigma.units = axesList[1].units
sigma.designateTime()
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
# init ensemble axis
ensembleAxis = cdm.createAxis(npy.float32(range(runN)))
ensembleAxis.id = 'members'
ensembleAxis.units = 'N'
# Output axis
sigmaList = [sigma, axesList[2], axesList[3]]
# sigma, lat, lon
sigmaTimeList = [sigma, time, axesList[2], axesList[3]]
# sigma, time, lat, lon
# init arrays
isonvar = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * valmask
varbowl2D = npy.ma.ones([runN, timN, latN, lonN],
dtype='float32') * valmask
varstd, varToE1, varToE2 = [
npy.ma.ones([runN, latN, lonN], dtype='float32') * valmask
for _ in range(3)
]
# Loop on density levels (for memory management, becomes UNLIMITED axis and requires a ncpq to reorder dimensions)
delta_ib = 1
print ' Sigma index:'
for ib in range(levN):
ib1 = ib + delta_ib
print ib,
tim0 = timc.clock()
# loop on variables
for iv, var in enumerate(varList):
if nobowl:
varb = var + 'Bowl'
else:
varb = var
if ib == 0:
print ' Variable ', iv, varb
# loop over files to fill up array
for i, file in enumerate(listFiles):
tim01 = timc.clock()
ft = cdm.open(inDir[0] + '/' + file)
model = file.split('.')[1]
timeax = ft.getAxis('time')
if i == 0:
tmax0 = timeax.shape[0]
tmax = timeax.shape[0]
if tmax != tmax0:
print 'wrong time axis: exiting...'
return
# read array
isonRead = ft(varb, time=slice(t1, t2),
lev=slice(ib, ib1)).squeeze()
if varFill[iv] != valmask:
isonvar[i, ...] = isonRead.filled(varFill[iv])
else:
isonvar[i, ...] = isonRead
tim02 = timc.clock()
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data, valmask).mask
percent[i, ...] = npy.float32(npy.equal(maskvar, 0))
tim03 = timc.clock()
if mme:
# if mme then just accumulate Bowl, Agree and Std fields
#varst = var+'Agree'
#vardiff[i,...] = ft(varst,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze()
isonRead = ft(varb, time=slice(t1, t2),
lev=slice(ib, ib1)).squeeze()
varbowl2D[i, ...] = isonRead
else:
# Compute difference with average of first initN years
#varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0)
#for t in range(timN):
# vardiff[i,t,...] = isonvar[i,t,...] - varinit
#vardiff[i,...].mask = isonvar[i,...].mask
# Read bowl to truncate field above bowl
if ib == 0 and iv == 0:
varbowl[i, ...] = ft(varsig, time=slice(t1, t2))
#varbowl[i,...] = bowlRead
# Compute Stddev
varstd[i, ...] = npy.ma.std(isonvar[i, ...], axis=0)
# Compute ToE
if ToeType == 'histnat':
toto = 1
# TODO
# Read mean and Std dev from histnat
# if i == 0:
# filehn = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0]
# #filehn = replace(outFile,'historical','historicalNat')
# fthn = cdm.open(filehn)
# varmeanhn = fthn(var)
# varst = var+'Std'
# varmaxstd = fthn(varst)
# toemult = 1.
# signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN))
# noise = npy.reshape(varmaxstd,(basN*levN*latN))
# varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
# toemult = 2.
# varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
tim04 = timc.clock()
ft.close()
#print 'ib, section 1 timing',ib, tim02-tim01,tim03-tim02,tim04-tim03
# <-- end of loop on files (i)
tim1 = timc.clock()
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent, axis=0)) * 100.
percenta = mv.masked_less(percenta, 50)
percenta = npy.reshape(percenta, [delta_ib, timN, latN, lonN])
percentw = cdm.createVariable(percenta,
axes=sigmaTimeList,
id='isonpercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'), extend=1, index=ib)
# Sign of difference
#if mme:
# vardiffsgSum = cdu.averager(vardiff, axis=0)
# vardiffsgSum = cdm.createVariable(vardiffsgSum , axes = sigmaTimeList , id = 'foo')
# vardiffsgSum = maskVal(vardiffsgSum, valmask)
# vardiffsgSum.mask = percentw.mask
#else:
# vardiffsg = npy.copysign(varones,vardiff)
# # average signs
# vardiffsgSum = cdu.averager(vardiffsg, axis=0)
# vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
# vardiffsgSum.mask = percentw.mask
# vardiffsgSum._FillValue = valmask
# average variable accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = npy.reshape(isonVarAve, [delta_ib, timN, latN, lonN])
isonVarAve = cdm.createVariable(isonVarAve,
axes=sigmaTimeList,
id='foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
tim2 = timc.clock()
# Only keep points with rhon > bowl-delta_rho
delta_rho = 0.
# mme case
if mme: # start from average of <var>Agree
isonVarBowl = cdu.averager(varbowl2D, axis=0)
isonVarBowl = npy.reshape(isonVarBowl,
[delta_ib, timN, latN, lonN])
isonVarBowl = cdm.createVariable(isonVarBowl,
axes=sigmaTimeList,
id='foo')
isonVarBowl = maskVal(isonVarBowl, valmask)
isonVarBowl.mask = percentw.mask
# Compute intermodel stddev
isonVarStd = statistics.std(varbowl2D, axis=0)
isonVarStd = npy.reshape(isonVarStd,
[delta_ib, timN, latN, lonN])
isonVarStd = cdm.createVariable(isonVarStd,
axes=sigmaTimeList,
id='foo')
isonVarStd = maskVal(isonVarStd, valmask)
isonVarStd.mask = percentw.mask
# Write
isonvarbowlw = cdm.createVariable(isonVarBowl,
axes=sigmaTimeList,
id=isonRead.id)
isonvarbowlw.long_name = isonRead.long_name
isonvarbowlw.units = isonRead.units
isonvarstdw = cdm.createVariable(isonVarStd,
axes=sigmaTimeList,
id=isonRead.id + 'Std')
isonvarstdw.long_name = isonRead.long_name + ' intermodel std'
isonvarstdw.units = isonRead.units
outFile_f.write(isonvarbowlw.astype('float32'),
extend=1,
index=ib)
outFile_f.write(isonvarstdw.astype('float32'),
extend=1,
index=ib)
#if ib == 0 and iv == 0:
# # TODO review
# # Read multimodel sigma on bowl and average in time
# file1d = replace(outDir+'/'+outFile,'2D','1D')
# if os.path.isfile(file1d):
# f1d = cdm.open(file1d)
# else:
# print 'ERROR:',file1d,'missing (if mme, run 2D first)'
# sys.exit(1)
# bowlRead = f1d(varsig,time = slice(t1,t2),lev = slice(ib,ib1))
# f1d.close()
# siglimit = cdu.averager(bowlRead, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
#if sw2d == 1:
# for il in range(latN):
# for ib in range(basN):
# #if ib == 2:
# # print il, siglimit[ib,il]
# if siglimit[ib,il] < valmask/1000.:
# # if mme bowl density defined, mask above bowl
# index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
# isonVarBowl [:,ib,0:index[0],il].mask = True
# isonVarStd [:,ib,0:index[0],il].mask = True
# vardiffsgSum[:,ib,0:index[0],il].mask = True
# else:
# # mask all points
# isonVarBowl [:,ib,:,il].mask = True
# isonVarStd [:,ib,:,il].mask = True
# vardiffsgSum[:,ib,:,il].mask = True
# mm case
else:
isonVarBowl = isonVarAve * 1. # start from variable
#isonVarStd = isonVarAve*1. # start from variable
if ib == 0 and iv == 0:
# build bowl position
siglimit = cdu.averager(varbowl,
axis=0) # average accross members
siglimit = npy.reshape(siglimit,
[timN * latN * lonN]) - delta_rho
if iv == 0:
sigarr = siglimit * 1.
sigarr[:] = sigmaGrd[ib]
# test
i = 60
j = 60
ij = j * lonN + i
isonVarBowl = npy.reshape(isonVarBowl, [timN * latN * lonN])
#vardiffsgSum = npy.reshape(vardiffsgSum,[timN*latN*lonN])
isonVarBowl.mask = npy.where(sigarr < siglimit, True,
isonVarBowl.mask)
#vardiffsgSum.mask = npy.where(sigarr < siglimit, True, vardiffsgSum.mask)
isonVarBowl = npy.reshape(isonVarBowl, [timN, latN, lonN])
#vardiffsgSum = npy.reshape(vardiffsgSum,[timN,latN,lonN])
isonVarBowl = maskVal(isonVarBowl, valmask)
#vardiffsgSum = maskVal(vardiffsgSum, valmask)
# Find max of Std dev of all members
isonVarStd = npy.ma.max(varstd, axis=0)
# mask
isonVarStd = maskVal(isonVarStd, valmask)
# Write
#isonave = cdm.createVariable(isonVarAve, axes = sigmaTimeList, id = isonRead.id)
#isonave.long_name = isonRead.long_name
#isonave.units = isonRead.units
#vardiffsgSum = npy.reshape(vardiffsgSum,[delta_ib,timN,latN,lonN])
#isonavediff = cdm.createVariable(vardiffsgSum, axes = sigmaTimeList, id = isonRead.id+'Agree')
#isonavediff.long_name = isonRead.long_name
#isonavediff.units = isonRead.units
isonVarBowl = npy.reshape(isonVarBowl,
[delta_ib, timN, latN, lonN])
isonavebowl = cdm.createVariable(isonVarBowl,
axes=sigmaTimeList,
id=isonRead.id + 'Bowl')
isonavebowl.long_name = isonRead.long_name
isonavebowl.units = isonRead.units
isonVarStd = npy.reshape(isonVarStd, [delta_ib, latN, lonN])
isonmaxstd = cdm.createVariable(isonVarStd,
axes=sigmaList,
id=isonRead.id + 'Std')
isonmaxstd.long_name = isonRead.long_name
isonmaxstd.units = isonRead.units
#outFile_f.write( isonave.astype('float32'), extend = 1, index = ib)
#outFile_f.write(isonavediff.astype('float32'), extend = 1, index = ib)
outFile_f.write(isonavebowl.astype('float32'),
extend=1,
index=ib)
outFile_f.write(isonmaxstd.astype('float32'),
extend=1,
index=ib)
tim3 = timc.clock()
if ToeType == 'histnat':
isontoe1 = cdm.createVariable(
varToE1,
axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'ToE1')
isontoe1.long_name = 'ToE 1 for ' + isonRead.long_name
isontoe1.units = 'Year'
isontoe2 = cdm.createVariable(
varToE2,
axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'ToE2')
isontoe2.long_name = 'ToE 2 for ' + isonRead.long_name
isontoe2.units = 'Year'
outFile_f.write(isontoe1.astype('float32'), extend=1, index=ib)
outFile_f.write(isontoe2.astype('float32'), extend=1, index=ib)
tim4 = timc.clock()
# <--- end of loop on variables
#print 'ib, timing',ib, tim01-tim0,tim1-tim01,tim2-tim1,tim3-tim2,tim4-tim3
# <--- end of loop on density
print ' '
outFile_f.close()
fi.close()