def spatial_std(self):
""" Ecart-type pour chaque pas de temps """
from genutil import statistics
import cdms2
# centered and biased std (cf. http://www2-pcmdi.llnl.gov/cdat/manuals/cdutil/cdat_utilities-2.html)
self.model.spa_std = list() #Initialise une liste
self.obs.spa_std = list() #Initialise une liste
# Estimation de la std a chaque pas de temps
tps = self.obs.getTime()
for i, t in enumerate(tps):
self.model.spa_std.append(
statistics.std(self.model[i, :, :], axis='yx'))
self.obs.spa_std.append(
statistics.std(self.obs[i, :, :], axis='yx'))
# Transformation en variable cdms2
self.model.spa_std = cdms2.createVariable(
self.model.spa_std,
typecode='f',
id='model_spa_std',
attributes=dict(units=self.model.units))
self.obs.spa_std = cdms2.createVariable(
self.obs.spa_std,
typecode='f',
id='obs_spa_std',
attributes=dict(units=self.obs.units))
# Ajout de l'axe des temps correspondant a self...
self.model.spa_std.setAxis(0, tps)
self.obs.spa_std.setAxis(0, tps)
def temporal_std(self):
""" Ecart-type en chaque point geographique """
from genutil import statistics
from vacumm.misc.grid import get_grid, set_grid
# centered and biased std (cf. http://www2-pcmdi.llnl.gov/cdat/manuals/cdutil/cdat_utilities-2.html)
self.model.temp_std = () #Initialise un tuple
self.obs.temp_std = () #Initialise un tuple
self.model.temp_std = statistics.std(self.model, axis=0)
self.obs.temp_std = statistics.std(self.obs, axis=0)
ggm = get_grid(self.model)
set_grid(self.model.temp_std, ggm, axes=True)
ggo = get_grid(self.obs)
set_grid(self.obs.temp_std, ggm, axes=True)
def temporal_std(self):
""" Ecart-type en chaque point geographique """
from genutil import statistics
from vacumm.misc.grid import get_grid, set_grid
# centered and biased std (cf. http://www2-pcmdi.llnl.gov/cdat/manuals/cdutil/cdat_utilities-2.html)
self.model.temp_std = () #Initialise un tuple
self.obs.temp_std = () #Initialise un tuple
self.model.temp_std = statistics.std(self.model, axis = 0)
self.obs.temp_std = statistics.std(self.obs, axis = 0)
ggm = get_grid(self.model)
set_grid(self.model.temp_std, ggm, axes=True)
ggo = get_grid(self.obs)
set_grid(self.obs.temp_std, ggm, axes=True)
def spatial_std(self):
""" Ecart-type pour chaque pas de temps """
from genutil import statistics
import cdms2
# centered and biased std (cf. http://www2-pcmdi.llnl.gov/cdat/manuals/cdutil/cdat_utilities-2.html)
self.model.spa_std = list() #Initialise une liste
self.obs.spa_std = list() #Initialise une liste
# Estimation de la std a chaque pas de temps
tps = self.obs.getTime()
for i, t in enumerate(tps):
self.model.spa_std.append(statistics.std(self.model[i, :, :],axis='yx'))
self.obs.spa_std.append(statistics.std(self.obs[i, :, :],axis='yx'))
# Transformation en variable cdms2
self.model.spa_std = cdms2.createVariable(self.model.spa_std,typecode='f',id='model_spa_std', attributes=dict(units=self.model.units))
self.obs.spa_std = cdms2.createVariable(self.obs.spa_std,typecode='f',id='obs_spa_std', attributes=dict(units=self.obs.units))
# Ajout de l'axe des temps correspondant a self...
self.model.spa_std.setAxis(0, tps)
self.obs.spa_std.setAxis(0, tps)
gui_control.record_command( parent, s, 1 )
elif (t[ this_dim[i] ] == 'awt'): # return the altered weighted averaged dimension
weight_slab = parent.panelDM.dim[i_index].new_weight
tempslab = cdutil.averager(slab, axis= "(" + slab.getAxis(i_index).id + ")",
weight=weight_slab.filled() )
# record sum command
s = "\n# Currently, VCDAT cannot record the altered average weight command."
gui_control.record_command( parent, s, 1 )
record_okay = 0
elif (t[ this_dim[i] ] == 'gtm'): # return the geometrical mean
tempslab = geometricmean(slab, axis = i_index )
# record geometric mean command
s = "\n%s=genutil.statistics.geometricmean( %s, axis=%d )" % (slab.id, var_name, i_index)
gui_control.record_command( parent, s, 1 )
elif (t[ this_dim[i] ] == 'std'): # return the standard deviation
tempslab = std(slab, axis= i_index)
# record geometric mean command
s = "\n%s=genutil.statistics.std( %s, axis=%d )" % (slab.id, var_name, i_index)
gui_control.record_command( parent, s, 1 )
# copy the original's slab's id back to the averaged slab
if record_okay == 1:
tempslab.id = slab.id
gui_control.record_command( parent, ("%s.id = '%s'" % (slab.id, slab.id)), 1 )
slab = tempslab
else:
i_index += 1
if (parent.menu.squeeze_dim_flg == 1) and (len(tempslab.shape) > 0):
if record_okay == 1:
axis="(" +
slab.getAxis(i_index).id + ")",
weight=weight_slab.filled())
# record sum command
s = "\n# Currently, VCDAT cannot record the altered average weight command."
gui_control.record_command(parent, s, 1)
record_okay = 0
elif (t[this_dim[i]] == 'gtm'): # return the geometrical mean
tempslab = geometricmean(slab, axis=i_index)
# record geometric mean command
s = "\n%s=genutil.statistics.geometricmean( %s, axis=%d )" % (
slab.id, var_name, i_index)
gui_control.record_command(parent, s, 1)
elif (t[this_dim[i]] == 'std'
): # return the standard deviation
tempslab = std(slab, axis=i_index)
# record geometric mean command
s = "\n%s=genutil.statistics.std( %s, axis=%d )" % (
slab.id, var_name, i_index)
gui_control.record_command(parent, s, 1)
# copy the original's slab's id back to the averaged slab
if record_okay == 1:
tempslab.id = slab.id
gui_control.record_command(parent, ("%s.id = '%s'" %
(slab.id, slab.id)), 1)
slab = tempslab
else:
i_index += 1
plt.bar(np.r_[0, 4, 8], np.r_[cor11, cor12, cor13], width=1)
plt.bar(np.r_[1, 5, 9], np.r_[cor21, cor22, cor23], width=1)
plt.ylim(0, 1)
plt.show()
plt.clf()
# Test taylor diagrom here:
TSstd1, TSstd2 = [], []
TSrms1, TSrms2 = [], []
TScor1, TScor2 = [], []
flag = 1
for idx in range(OBSsp.shape[0]):
aaa = ERAsp[idx]
bbb = NOAsp[idx]
ccc = OBSsp[idx]
std0 = stat.std(ccc, axis='yx')
std1 = stat.std(aaa, axis='yx') / std0
std2 = stat.std(bbb, axis='yx') / std0
cor1 = stat.correlation(aaa, ccc, axis='yx')
cor2 = stat.correlation(bbb, ccc, axis='yx')
rms1 = stat.rms(aaa, ccc, axis='yx', centered=0)
rms2 = stat.rms(bbb, ccc, axis='yx', centered=0)
TSstd1.append(std1)
TSstd2.append(std2)
TSrms1.append(rms1)
TSrms2.append(rms2)
TScor1.append(cor1)
TScor2.append(cor2)
if flag == 1:
sm.taylor_diagram(np.array([1.0, np.mean(TSstd1)]),
np.array([0.0, np.mean(TSrms1)]),
def mmeAveMsk3D(listFiles,
years,
inDir,
outDir,
outFile,
timeInt,
mme,
ToeType,
debug=True):
'''
The mmeAveMsk3D() function averages rhon/lat density bined files with differing masks
It ouputs
- the MME
- a percentage of non-masked bins
- the sign agreement of period2-period1 differences
- ToE per run and for MME
Author: Eric Guilyardi : [email protected]
Created on Tue Nov 21 2016
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- years(t1,t2) - years for slice read
- inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE)
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- ToeType(str) - ToE type ('F': none, 'histnat')
-> requires running first mm+mme without ToE to compute Stddev
- debug <optional> - boolean value
Notes:
-----
- EG 21 Nov 2016 - Initial function write
- TODO :
- add computation of ToE per model (toe 1 and toe 2) see ticket #50
- add isonhtc (see ticket #48)
'''
# CDMS initialisation - netCDF compression
comp = 1 # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
fi = cdm.open(inDir[0] + '/' + listFiles[0])
# Switch if only variables below the bowl are present/treated
nobowl = True
if nobowl:
isond0 = fi['isondepthgBowl']
# Create variable handle
else:
isond0 = fi['isondepthg']
# Create variable handle
# Get grid objects
axesList = isond0.getAxisList()
sigmaGrd = isond0.getLevel()
#time = isond0.getTime()
lonN = isond0.shape[3]
latN = isond0.shape[2]
levN = isond0.shape[1]
varsig = 'ptopsigmaxy'
# Limit number of models to 3 for testing of mme
#if mme:
# listFiles = listFiles[0:2]
# print ' !!! ### Testing 3 models ###', listFiles
# Declare and open files for writing
if os.path.isfile(outDir + '/' + outFile):
os.remove(outDir + '/' + outFile)
outFile_f = cdm.open(outDir + '/' + outFile, 'w')
#timN = isond0.shape[0]
timN = t2 - t1
runN = len(listFiles)
print ' Number of members:', len(listFiles)
valmask = isond0.missing_value
varList = ['isondepthg', 'persistmxy', 'sog', 'thetaog', 'isonthickg']
varFill = [valmask, valmask, valmask, valmask, valmask]
percent = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * 0.
varbowl = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * 1.
#varList = ['isondepthg']
#print ' !!! ### Testing one variable ###', varList
# init sigma axis
sigma = cdm.createAxis(npy.float32(range(1)))
sigma.id = axesList[1].id
sigma.units = axesList[1].units
sigma.designateTime()
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
# init ensemble axis
ensembleAxis = cdm.createAxis(npy.float32(range(runN)))
ensembleAxis.id = 'members'
ensembleAxis.units = 'N'
# Output axis
sigmaList = [sigma, axesList[2], axesList[3]]
# sigma, lat, lon
sigmaTimeList = [sigma, time, axesList[2], axesList[3]]
# sigma, time, lat, lon
# init arrays
isonvar = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * valmask
varbowl2D = npy.ma.ones([runN, timN, latN, lonN],
dtype='float32') * valmask
varstd, varToE1, varToE2 = [
npy.ma.ones([runN, latN, lonN], dtype='float32') * valmask
for _ in range(3)
]
# Loop on density levels (for memory management, becomes UNLIMITED axis and requires a ncpq to reorder dimensions)
delta_ib = 1
print ' Sigma index:'
for ib in range(levN):
ib1 = ib + delta_ib
print ib,
tim0 = timc.clock()
# loop on variables
for iv, var in enumerate(varList):
if nobowl:
varb = var + 'Bowl'
else:
varb = var
if ib == 0:
print ' Variable ', iv, varb
# loop over files to fill up array
for i, file in enumerate(listFiles):
tim01 = timc.clock()
ft = cdm.open(inDir[0] + '/' + file)
model = file.split('.')[1]
timeax = ft.getAxis('time')
if i == 0:
tmax0 = timeax.shape[0]
tmax = timeax.shape[0]
if tmax != tmax0:
print 'wrong time axis: exiting...'
return
# read array
isonRead = ft(varb, time=slice(t1, t2),
lev=slice(ib, ib1)).squeeze()
if varFill[iv] != valmask:
isonvar[i, ...] = isonRead.filled(varFill[iv])
else:
isonvar[i, ...] = isonRead
tim02 = timc.clock()
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data, valmask).mask
percent[i, ...] = npy.float32(npy.equal(maskvar, 0))
tim03 = timc.clock()
if mme:
# if mme then just accumulate Bowl, Agree and Std fields
#varst = var+'Agree'
#vardiff[i,...] = ft(varst,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze()
isonRead = ft(varb, time=slice(t1, t2),
lev=slice(ib, ib1)).squeeze()
varbowl2D[i, ...] = isonRead
else:
# Compute difference with average of first initN years
#varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0)
#for t in range(timN):
# vardiff[i,t,...] = isonvar[i,t,...] - varinit
#vardiff[i,...].mask = isonvar[i,...].mask
# Read bowl to truncate field above bowl
if ib == 0 and iv == 0:
varbowl[i, ...] = ft(varsig, time=slice(t1, t2))
#varbowl[i,...] = bowlRead
# Compute Stddev
varstd[i, ...] = npy.ma.std(isonvar[i, ...], axis=0)
# Compute ToE
if ToeType == 'histnat':
toto = 1
# TODO
# Read mean and Std dev from histnat
# if i == 0:
# filehn = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0]
# #filehn = replace(outFile,'historical','historicalNat')
# fthn = cdm.open(filehn)
# varmeanhn = fthn(var)
# varst = var+'Std'
# varmaxstd = fthn(varst)
# toemult = 1.
# signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN))
# noise = npy.reshape(varmaxstd,(basN*levN*latN))
# varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
# toemult = 2.
# varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
tim04 = timc.clock()
ft.close()
#print 'ib, section 1 timing',ib, tim02-tim01,tim03-tim02,tim04-tim03
# <-- end of loop on files (i)
tim1 = timc.clock()
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent, axis=0)) * 100.
percenta = mv.masked_less(percenta, 50)
percenta = npy.reshape(percenta, [delta_ib, timN, latN, lonN])
percentw = cdm.createVariable(percenta,
axes=sigmaTimeList,
id='isonpercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'), extend=1, index=ib)
# Sign of difference
#if mme:
# vardiffsgSum = cdu.averager(vardiff, axis=0)
# vardiffsgSum = cdm.createVariable(vardiffsgSum , axes = sigmaTimeList , id = 'foo')
# vardiffsgSum = maskVal(vardiffsgSum, valmask)
# vardiffsgSum.mask = percentw.mask
#else:
# vardiffsg = npy.copysign(varones,vardiff)
# # average signs
# vardiffsgSum = cdu.averager(vardiffsg, axis=0)
# vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
# vardiffsgSum.mask = percentw.mask
# vardiffsgSum._FillValue = valmask
# average variable accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = npy.reshape(isonVarAve, [delta_ib, timN, latN, lonN])
isonVarAve = cdm.createVariable(isonVarAve,
axes=sigmaTimeList,
id='foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
tim2 = timc.clock()
# Only keep points with rhon > bowl-delta_rho
delta_rho = 0.
# mme case
if mme: # start from average of <var>Agree
isonVarBowl = cdu.averager(varbowl2D, axis=0)
isonVarBowl = npy.reshape(isonVarBowl,
[delta_ib, timN, latN, lonN])
isonVarBowl = cdm.createVariable(isonVarBowl,
axes=sigmaTimeList,
id='foo')
isonVarBowl = maskVal(isonVarBowl, valmask)
isonVarBowl.mask = percentw.mask
# Compute intermodel stddev
isonVarStd = statistics.std(varbowl2D, axis=0)
isonVarStd = npy.reshape(isonVarStd,
[delta_ib, timN, latN, lonN])
isonVarStd = cdm.createVariable(isonVarStd,
axes=sigmaTimeList,
id='foo')
isonVarStd = maskVal(isonVarStd, valmask)
isonVarStd.mask = percentw.mask
# Write
isonvarbowlw = cdm.createVariable(isonVarBowl,
axes=sigmaTimeList,
id=isonRead.id)
isonvarbowlw.long_name = isonRead.long_name
isonvarbowlw.units = isonRead.units
isonvarstdw = cdm.createVariable(isonVarStd,
axes=sigmaTimeList,
id=isonRead.id + 'Std')
isonvarstdw.long_name = isonRead.long_name + ' intermodel std'
isonvarstdw.units = isonRead.units
outFile_f.write(isonvarbowlw.astype('float32'),
extend=1,
index=ib)
outFile_f.write(isonvarstdw.astype('float32'),
extend=1,
index=ib)
#if ib == 0 and iv == 0:
# # TODO review
# # Read multimodel sigma on bowl and average in time
# file1d = replace(outDir+'/'+outFile,'2D','1D')
# if os.path.isfile(file1d):
# f1d = cdm.open(file1d)
# else:
# print 'ERROR:',file1d,'missing (if mme, run 2D first)'
# sys.exit(1)
# bowlRead = f1d(varsig,time = slice(t1,t2),lev = slice(ib,ib1))
# f1d.close()
# siglimit = cdu.averager(bowlRead, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
#if sw2d == 1:
# for il in range(latN):
# for ib in range(basN):
# #if ib == 2:
# # print il, siglimit[ib,il]
# if siglimit[ib,il] < valmask/1000.:
# # if mme bowl density defined, mask above bowl
# index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
# isonVarBowl [:,ib,0:index[0],il].mask = True
# isonVarStd [:,ib,0:index[0],il].mask = True
# vardiffsgSum[:,ib,0:index[0],il].mask = True
# else:
# # mask all points
# isonVarBowl [:,ib,:,il].mask = True
# isonVarStd [:,ib,:,il].mask = True
# vardiffsgSum[:,ib,:,il].mask = True
# mm case
else:
isonVarBowl = isonVarAve * 1. # start from variable
#isonVarStd = isonVarAve*1. # start from variable
if ib == 0 and iv == 0:
# build bowl position
siglimit = cdu.averager(varbowl,
axis=0) # average accross members
siglimit = npy.reshape(siglimit,
[timN * latN * lonN]) - delta_rho
if iv == 0:
sigarr = siglimit * 1.
sigarr[:] = sigmaGrd[ib]
# test
i = 60
j = 60
ij = j * lonN + i
isonVarBowl = npy.reshape(isonVarBowl, [timN * latN * lonN])
#vardiffsgSum = npy.reshape(vardiffsgSum,[timN*latN*lonN])
isonVarBowl.mask = npy.where(sigarr < siglimit, True,
isonVarBowl.mask)
#vardiffsgSum.mask = npy.where(sigarr < siglimit, True, vardiffsgSum.mask)
isonVarBowl = npy.reshape(isonVarBowl, [timN, latN, lonN])
#vardiffsgSum = npy.reshape(vardiffsgSum,[timN,latN,lonN])
isonVarBowl = maskVal(isonVarBowl, valmask)
#vardiffsgSum = maskVal(vardiffsgSum, valmask)
# Find max of Std dev of all members
isonVarStd = npy.ma.max(varstd, axis=0)
# mask
isonVarStd = maskVal(isonVarStd, valmask)
# Write
#isonave = cdm.createVariable(isonVarAve, axes = sigmaTimeList, id = isonRead.id)
#isonave.long_name = isonRead.long_name
#isonave.units = isonRead.units
#vardiffsgSum = npy.reshape(vardiffsgSum,[delta_ib,timN,latN,lonN])
#isonavediff = cdm.createVariable(vardiffsgSum, axes = sigmaTimeList, id = isonRead.id+'Agree')
#isonavediff.long_name = isonRead.long_name
#isonavediff.units = isonRead.units
isonVarBowl = npy.reshape(isonVarBowl,
[delta_ib, timN, latN, lonN])
isonavebowl = cdm.createVariable(isonVarBowl,
axes=sigmaTimeList,
id=isonRead.id + 'Bowl')
isonavebowl.long_name = isonRead.long_name
isonavebowl.units = isonRead.units
isonVarStd = npy.reshape(isonVarStd, [delta_ib, latN, lonN])
isonmaxstd = cdm.createVariable(isonVarStd,
axes=sigmaList,
id=isonRead.id + 'Std')
isonmaxstd.long_name = isonRead.long_name
isonmaxstd.units = isonRead.units
#outFile_f.write( isonave.astype('float32'), extend = 1, index = ib)
#outFile_f.write(isonavediff.astype('float32'), extend = 1, index = ib)
outFile_f.write(isonavebowl.astype('float32'),
extend=1,
index=ib)
outFile_f.write(isonmaxstd.astype('float32'),
extend=1,
index=ib)
tim3 = timc.clock()
if ToeType == 'histnat':
isontoe1 = cdm.createVariable(
varToE1,
axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'ToE1')
isontoe1.long_name = 'ToE 1 for ' + isonRead.long_name
isontoe1.units = 'Year'
isontoe2 = cdm.createVariable(
varToE2,
axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'ToE2')
isontoe2.long_name = 'ToE 2 for ' + isonRead.long_name
isontoe2.units = 'Year'
outFile_f.write(isontoe1.astype('float32'), extend=1, index=ib)
outFile_f.write(isontoe2.astype('float32'), extend=1, index=ib)
tim4 = timc.clock()
# <--- end of loop on variables
#print 'ib, timing',ib, tim01-tim0,tim1-tim01,tim2-tim1,tim3-tim2,tim4-tim3
# <--- end of loop on density
print ' '
outFile_f.close()
fi.close()
def mmeAveMsk2D(listFiles,
years,
inDir,
outDir,
outFile,
timeInt,
mme,
timeBowl,
ToeType,
debug=True):
'''
The mmeAveMsk2D() function averages rhon/lat density bined files with differing masks
It ouputs
- the MME
- a percentage of non-masked bins
- the sign agreement of period2-period1 differences
- ToE per run and for MME
Author: Eric Guilyardi : [email protected]
Created on Tue Nov 25 13:56:20 CET 2014
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- years(t1,t2) - years for slice read
- inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE)
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- timeBowl - either time 'mean' or time 'max' bowl used to mask out bowl
- ToeType(str) - ToE type ('F': none, 'histnat')
-> requires running first mm+mme without ToE to compute Stddev
- debug <optional> - boolean value
Notes:
-----
- EG 25 Nov 2014 - Initial function write
- EG 27 Nov 2014 - Rewrite with loop on variables
- EG 06 Dec 2014 - Added agreement on difference with init period - save as <var>Agree
- EG 07 Dec 2014 - Read bowl to remove points above bowl - save as <var>Bowl
- EG 19 Apr 2016 - ToE computation (just for 2D files)
- EG 07 Oct 2016 - add 3D file support
- EG 21 Nov 2016 - move 3D support to new function
- EG 10 jan 2017 - added timeBowl option
- TODO :
- remove loops
- add computation of ToE per model (toe 1 and toe 2) see ticket #50
- add isonhtc (see ticket #48)
'''
# CDMS initialisation - netCDF compression
comp = 1 # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
if t2 <= 0:
useLastYears = True
t2 = -t2
else:
useLastYears = False
t10 = t1
t20 = t2
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
fi = cdm.open(inDir[0] + '/' + listFiles[0])
isond0 = fi['isondepth']
# Create variable handle
# Get grid objects
axesList = isond0.getAxisList()
sigmaGrd = isond0.getLevel()
latN = isond0.shape[3]
levN = isond0.shape[2]
basN = isond0.shape[1]
varsig = 'ptopsigma'
# Declare and open files for writing
if os.path.isfile(outDir + '/' + outFile):
os.remove(outDir + '/' + outFile)
outFile_f = cdm.open(outDir + '/' + outFile, 'w')
# Testing mme with less models
#listFiles=listFiles[0:4]
#timN = isond0.shape[0]
timN = t2 - t1
runN = len(listFiles)
print ' Number of members:', len(listFiles)
valmask = isond0.missing_value[0]
varList = [
'isondepth', 'isonpers', 'isonso', 'isonthetao', 'isonthick', 'isonvol'
]
varFill = [0., 0., valmask, valmask, 0., 0.]
# init arrays (2D rho/lat)
percent = npy.ma.ones([runN, timN, basN, levN, latN], dtype='float32') * 0.
#minbowl = npy.ma.ones([basN,latN], dtype='float32')*1000.
varbowl = npy.ma.ones([runN, timN, basN, latN], dtype='float32') * 1.
#varList = ['isondepth']
#print ' !!! ### Testing one variable ###'
#varList = ['isonthetao']
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
time.designateTime()
# init ensemble axis
ensembleAxis = cdm.createAxis(npy.float32(range(runN)))
ensembleAxis.id = 'members'
ensembleAxis.units = 'N'
# loop on variables
for iv, var in enumerate(varList):
# Array inits (2D rho/lat 3D rho/lat/lon)
#shapeR = [basN,levN,latN]
isonvar = npy.ma.ones([runN, timN, basN, levN, latN],
dtype='float32') * valmask
print('isonvar shape: ', isonvar.shape)
vardiff, varbowl2D = [
npy.ma.ones([runN, timN, basN, levN, latN], dtype='float32')
for _ in range(2)
]
varstd, varToE1, varToE2 = [
npy.ma.ones([runN, basN, levN, latN], dtype='float32') * valmask
for _ in range(3)
]
varones = npy.ma.ones([runN, timN, basN, levN, latN],
dtype='float32') * 1.
print ' Variable ', iv, var
# loop over files to fill up array
for i, file in enumerate(listFiles):
ft = cdm.open(inDir[0] + '/' + file)
model = file.split('.')[1]
timeax = ft.getAxis('time')
file1d = replace(inDir[0] + '/' + file, '2D', '1D')
if os.path.isfile(file1d):
f1d = cdm.open(file1d)
else:
print 'ERROR:', file1d, 'missing (if mme, run 1D first)'
sys.exit(1)
tmax = timeax.shape[0]
if i == 0:
tmax0 = tmax
#adapt [t1,t2] time bounds to piControl last NN years
if useLastYears:
t1 = tmax - t20
t2 = tmax
else:
if tmax != tmax0:
print 'wrong time axis: exiting...'
return
# read array
# loop over time/density for memory management
for it in range(timN):
t1r = t1 + it
t2r = t1r + 1
isonRead = ft(var, time=slice(t1r, t2r))
if varFill[iv] != valmask:
isonvar[i, it, ...] = isonRead.filled(varFill[iv])
else:
isonvar[i, it, ...] = isonRead
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data, valmask).mask
percent[i, ...] = npy.float32(npy.equal(maskvar, 0))
if mme:
# if mme then just accumulate Bowl, Agree fields
varst = var + 'Agree'
vardiff[i, ...] = ft(varst, time=slice(t1, t2))
varb = var + 'Bowl'
varbowl2D[i, ...] = ft(varb, time=slice(t1, t2))
else:
# Compute difference with average of first initN years
varinit = cdu.averager(isonvar[i, peri1:peri2, ...], axis=0)
for t in range(timN):
vardiff[i, t, ...] = isonvar[i, t, ...] - varinit
vardiff[i, ...].mask = isonvar[i, ...].mask
# Read bowl and truncate 2D field above bowl
if iv == 0:
bowlRead = f1d(varsig, time=slice(t1, t2))
varbowl[i, ...] = bowlRead
# Compute Stddev
varstd[i, ...] = npy.ma.std(isonvar[i, ...], axis=0)
# Compute ToE
if ToeType == 'histnat':
# Read mean and Std dev from histnat
if i == 0:
filehn = glob.glob(inDir[1] + '/cmip5.' + model +
'.*zon2D*')[0]
#filehn = replace(outFile,'historical','historicalNat')
fthn = cdm.open(filehn)
varmeanhn = fthn(var)
varst = var + 'Std'
varmaxstd = fthn(varst)
toemult = 1.
signal = npy.reshape(isonvar[i, ...] - varmeanhn,
(timN, basN * levN * latN))
noise = npy.reshape(varmaxstd, (basN * levN * latN))
varToE1[i,
...] = npy.reshape(findToE(signal, noise, toemult),
(basN, levN, latN))
toemult = 2.
varToE2[i,
...] = npy.reshape(findToE(signal, noise, toemult),
(basN, levN, latN))
ft.close()
f1d.close()
# <-- end of loop on files
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent, axis=0)) * 100.
percenta = mv.masked_less(percenta, 50)
percentw = cdm.createVariable(
percenta,
axes=[time, axesList[1], axesList[2], axesList[3]],
id='isonpercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'))
# Sign of difference
if mme:
vardiffsgSum = cdu.averager(vardiff, axis=0)
vardiffsgSum = cdm.createVariable(
vardiffsgSum,
axes=[time, axesList[1], axesList[2], axesList[3]],
id='foo')
vardiffsgSum = maskVal(vardiffsgSum, valmask)
vardiffsgSum.mask = percentw.mask
else:
vardiffsg = npy.copysign(varones, vardiff)
# average signs
vardiffsgSum = cdu.averager(vardiffsg, axis=0)
vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
vardiffsgSum.mask = percentw.mask
vardiffsgSum._FillValue = valmask
# average variable accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = cdm.createVariable(
isonVarAve,
axes=[time, axesList[1], axesList[2], axesList[3]],
id='foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
# Only keep points with rhon > bowl-delta_rho
delta_rho = 0.
if mme: # start from average of <var>Agree
isonVarBowl = cdu.averager(varbowl2D, axis=0)
isonVarBowl = cdm.createVariable(
isonVarBowl,
axes=[time, axesList[1], axesList[2], axesList[3]],
id='foo')
isonVarBowl = maskVal(isonVarBowl, valmask)
isonVarBowl.mask = percentw.mask
# Compute intermodel stddev
isonVarStd = statistics.std(varbowl2D, axis=0)
isonVarStd = cdm.createVariable(
isonVarStd,
axes=[time, axesList[1], axesList[2], axesList[3]],
id='foo')
isonVarStd = maskVal(isonVarStd, valmask)
isonVarStd.mask = percentw.mask
if iv == 0:
# Read mulitmodel sigma on bowl and average in time
file1d = replace(outDir + '/' + outFile, '2D', '1D')
if os.path.isfile(file1d):
f1d = cdm.open(file1d)
else:
print 'ERROR:', file1d, 'missing (if mme, run 1D first)'
sys.exit(1)
bowlRead = f1d(varsig, time=slice(t1, t2))
f1d.close()
siglimit = cdu.averager(bowlRead, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
for il in range(latN):
for ib in range(basN):
#if ib == 2:
# print il, siglimit[ib,il]
if siglimit[ib, il] < valmask / 1000.:
# if mme bowl density defined, mask above bowl
index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib, il]))
isonVarBowl[:, ib, 0:index[0], il].mask = True
isonVarStd[:, ib, 0:index[0], il].mask = True
vardiffsgSum[:, ib, 0:index[0], il].mask = True
else:
# mask all points
isonVarBowl[:, ib, :, il].mask = True
isonVarStd[:, ib, :, il].mask = True
vardiffsgSum[:, ib, :, il].mask = True
else:
isonVarBowl = isonVarAve * 1. # start from variable
isonVarStd = isonVarAve * 1. # start from variable
if iv == 0:
siglimit = cdu.averager(varbowl,
axis=0) # average accross members
# Average bowl in time
if timeBowl == 'mean':
siglimit = cdu.averager(siglimit, axis=0) - delta_rho
# or take largest sigma over time
else:
siglimit = npy.ma.max(siglimit, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
for il in range(latN):
for ib in range(basN):
if siglimit[ib, il] < valmask / 1000.:
# if bowl density defined, mask above bowl
index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib, il])
)[:, 0] #Add [:,0] for python Yona
#import code
#code.interact(banner='index', local=dict(locals(), **globals()))
isonVarBowl[:, ib, 0:index[0], il].mask = True
vardiffsgSum[:, ib, 0:index[0], il].mask = True
else:
# mask all points
vardiffsgSum[:, ib, :, il].mask = True
isonVarBowl = maskVal(isonVarBowl, valmask)
# Find max of Std dev of all members
isonVarStd = npy.ma.max(varstd, axis=0)
# mask
if varFill[iv] == valmask:
isonVarStd = maskVal(isonVarStd, valmask)
# Write
isonave = cdm.createVariable(
isonVarAve,
axes=[time, axesList[1], axesList[2], axesList[3]],
id=isonRead.id)
isonave.long_name = isonRead.long_name
isonave.units = isonRead.units
isonavediff = cdm.createVariable(
vardiffsgSum,
axes=[time, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'Agree')
isonavediff.long_name = isonRead.long_name
isonavediff.units = isonRead.units
isonavebowl = cdm.createVariable(
isonVarBowl,
axes=[time, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'Bowl')
isonavebowl.long_name = isonRead.long_name
isonavebowl.units = isonRead.units
if not mme:
isonmaxstd = cdm.createVariable(
isonVarStd,
axes=[axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'Std')
isonmaxstd.long_name = isonRead.long_name
isonmaxstd.units = isonRead.units
outFile_f.write(isonave.astype('float32'))
outFile_f.write(isonavediff.astype('float32'))
outFile_f.write(isonavebowl.astype('float32'))
if not mme:
outFile_f.write(isonmaxstd.astype('float32'))
if ToeType == 'histnat':
isontoe1 = cdm.createVariable(
varToE1,
axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'ToE1')
isontoe1.long_name = 'ToE 1 for ' + isonRead.long_name
isontoe1.units = 'Year'
isontoe2 = cdm.createVariable(
varToE2,
axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'ToE2')
isontoe2.long_name = 'ToE 2 for ' + isonRead.long_name
isontoe2.units = 'Year'
outFile_f.write(isontoe1.astype('float32'))
outFile_f.write(isontoe2.astype('float32'))
if mme:
isonvarstd = cdm.createVariable(
isonVarStd,
axes=[time, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'ModStd')
isonvarstd.long_name = isonRead.long_name + ' intermodel std'
isonvarstd.units = isonRead.units
outFile_f.write(isonvarstd.astype('float32'))
# <--- end of loop on variables
outFile_f.close()
fi.close()
def monsoon_wang_runner(args):
# args = P.parse_args(sys.argv[1:])
modpath = genutil.StringConstructor(args.test_data_path)
modpath.variable = args.modvar
outpathdata = args.results_dir
if isinstance(args.modnames, str):
mods = eval(args.modnames)
else:
mods = args.modnames
json_filename = args.outnamejson
if json_filename == "CMIP_MME":
json_filename = "/MPI_" + args.mip + "_" + args.experiment
# VAR IS FIXED TO BE PRECIP FOR CALCULATING MONSOON PRECIPITATION INDICES
var = args.modvar
thr = args.threshold
sig_digits = ".3f"
# Get flag for CMEC output
cmec = args.cmec
#########################################
# PMP monthly default PR obs
cdms2.axis.longitude_aliases.append("longitude_prclim_mpd")
cdms2.axis.latitude_aliases.append("latitude_prclim_mpd")
fobs = cdms2.open(args.reference_data_path)
dobs_orig = fobs(args.obsvar)
fobs.close()
obsgrid = dobs_orig.getGrid()
########################################
# FCN TO COMPUTE GLOBAL ANNUAL RANGE AND MONSOON PRECIP INDEX
annrange_obs, mpi_obs = mpd(dobs_orig)
#########################################
# SETUP WHERE TO OUTPUT RESULTING DATA (netcdf)
nout = os.path.join(outpathdata,
"_".join([args.experiment, args.mip, "wang-monsoon"]))
try:
os.makedirs(nout)
except BaseException:
pass
# SETUP WHERE TO OUTPUT RESULTS (json)
jout = outpathdata
try:
os.makedirs(nout)
except BaseException:
pass
gmods = [] # "Got" these MODS
for i, mod in enumerate(mods):
modpath.model = mod
for k in modpath.keys():
try:
val = getattr(args, k)
except Exception:
continue
if not isinstance(val, (list, tuple)):
setattr(modpath, k, val)
else:
setattr(modpath, k, val[i])
l1 = modpath()
if os.path.isfile(l1) is True:
gmods.append(mod)
if len(gmods) == 0:
raise RuntimeError("No model file found!")
#########################################
egg_pth = resources.resource_path()
globals = {}
locals = {}
exec(
compile(
open(os.path.join(egg_pth, "default_regions.py")).read(),
os.path.join(egg_pth, "default_regions.py"),
"exec",
),
globals,
locals,
)
regions_specs = locals["regions_specs"]
doms = ["AllMW", "AllM", "NAMM", "SAMM", "NAFM", "SAFM", "ASM", "AUSM"]
mpi_stats_dic = {}
for i, mod in enumerate(gmods):
modpath.model = mod
for k in modpath.keys():
try:
val = getattr(args, k)
except Exception:
continue
if not isinstance(val, (list, tuple)):
setattr(modpath, k, val)
else:
setattr(modpath, k, val[i])
modelFile = modpath()
mpi_stats_dic[mod] = {}
print(
"******************************************************************************************"
)
print(modelFile)
f = cdms2.open(modelFile)
d_orig = f(var)
annrange_mod, mpi_mod = mpd(d_orig)
annrange_mod = annrange_mod.regrid(obsgrid,
regridTool="regrid2",
regridMethod="conserve",
mkCyclic=True)
mpi_mod = mpi_mod.regrid(obsgrid,
regridTool="regrid2",
regridMethod="conserve",
mkCyclic=True)
for dom in doms:
mpi_stats_dic[mod][dom] = {}
reg_sel = regions_specs[dom]["domain"]
mpi_obs_reg = mpi_obs(reg_sel)
mpi_obs_reg_sd = float(statistics.std(mpi_obs_reg, axis="xy"))
mpi_mod_reg = mpi_mod(reg_sel)
cor = float(
statistics.correlation(mpi_mod_reg, mpi_obs_reg, axis="xy"))
rms = float(statistics.rms(mpi_mod_reg, mpi_obs_reg, axis="xy"))
rmsn = rms / mpi_obs_reg_sd
# DOMAIN SELECTED FROM GLOBAL ANNUAL RANGE FOR MODS AND OBS
annrange_mod_dom = annrange_mod(reg_sel)
annrange_obs_dom = annrange_obs(reg_sel)
# SKILL SCORES
# HIT/(HIT + MISSED + FALSE ALARMS)
hit, missed, falarm, score, hitmap, missmap, falarmmap = mpi_skill_scores(
annrange_mod_dom, annrange_obs_dom, thr)
# POPULATE DICTIONARY FOR JSON FILES
mpi_stats_dic[mod][dom] = {}
mpi_stats_dic[mod][dom]["cor"] = format(cor, sig_digits)
mpi_stats_dic[mod][dom]["rmsn"] = format(rmsn, sig_digits)
mpi_stats_dic[mod][dom]["threat_score"] = format(score, sig_digits)
# SAVE ANNRANGE AND HIT MISS AND FALSE ALARM FOR EACH MOD DOM
fm = os.path.join(nout, "_".join([mod, dom, "wang-monsoon.nc"]))
g = cdms2.open(fm, "w")
g.write(annrange_mod_dom)
g.write(hitmap, dtype=numpy.int32)
g.write(missmap, dtype=numpy.int32)
g.write(falarmmap, dtype=numpy.int32)
g.close()
f.close()
# OUTPUT METRICS TO JSON FILE
OUT = pcmdi_metrics.io.base.Base(os.path.abspath(jout), json_filename)
disclaimer = open(os.path.join(egg_pth, "disclaimer.txt")).read()
metrics_dictionary = collections.OrderedDict()
metrics_dictionary["DISCLAIMER"] = disclaimer
metrics_dictionary["REFERENCE"] = (
"The statistics in this file are based on" +
" Wang, B., Kim, HJ., Kikuchi, K. et al. " +
"Clim Dyn (2011) 37: 941. doi:10.1007/s00382-010-0877-0")
metrics_dictionary["RESULTS"] = mpi_stats_dic # collections.OrderedDict()
OUT.var = var
OUT.write(
metrics_dictionary,
json_structure=["model", "domain", "statistic"],
indent=4,
separators=(",", ": "),
)
if cmec:
print("Writing cmec file")
OUT.write_cmec(indent=4, separators=(",", ": "))
def monsoon_wang_runner(args):
# args = P.parse_args(sys.argv[1:])
modpath = args.modpath
outpathjsons = args.outpathjsons
outpathdata = args.results_dir
if isinstance(args.modnames, str):
mods = eval(args.modnames)
else:
mods = args.modnames
json_filename = args.jsonname
if json_filename == 'CMIP_MME':
json_filename = '/MPI_' + args.mip + '_' + args.experiment
if args.mip == 'CMIP5' and args.experiment == 'historical' and mods is None:
mods = [
'ACCESS1-0',
'ACCESS1-3',
'bcc-csm1-1',
'bcc-csm1-1-m',
'BNU-ESM',
'CanCM4',
'CanESM2',
'CCSM4',
'CESM1-BGC',
'CESM1-CAM5',
'CESM1-FASTCHEM',
'CESM1-WACCM',
'CMCC-CESM',
'CMCC-CM',
'CMCC-CMS',
'CNRM-CM5-2',
'CNRM-CM5',
'CSIRO-Mk3-6-0',
'FGOALS-g2',
'FIO-ESM',
'GFDL-CM2p1',
'GFDL-CM3',
'GFDL-ESM2G',
'GFDL-ESM2M',
'GISS-E2-H',
'GISS-E2-H-CC',
'GISS-E2-R',
'GISS-E2-R-CC',
'HadCM3',
'HadGEM2-AO',
'HadGEM2-CC',
'HadGEM2-ES',
'inmcm4',
'IPSL-CM5A-LR',
'IPSL-CM5A-MR',
'IPSL-CM5B-LR',
'MIROC4h',
'MIROC5',
'MIROC-ESM',
'MIROC-ESM-CHEM',
'MPI-ESM-LR',
'MPI-ESM-MR',
'MPI-ESM-P',
'MRI-CGCM3',
'MRI-ESM1',
'NorESM1-M',
'NorESM1-ME']
# VAR IS FIXED TO BE PRECIP FOR CALCULATING MONSOON PRECIPITATION INDICES
var = args.modvar
thr = args.threshold
sig_digits = '.3f'
#########################################
# PMP monthly default PR obs
cdms2.axis.longitude_aliases.append("longitude_prclim_mpd")
cdms2.axis.latitude_aliases.append("latitude_prclim_mpd")
fobs = cdms2.open(args.reference_data_path)
dobs_orig = fobs(args.obsvar)
fobs.close()
obsgrid = dobs_orig.getGrid()
########################################
# FCN TO COMPUTE GLOBAL ANNUAL RANGE AND MONSOON PRECIP INDEX
annrange_obs, mpi_obs = mpd(dobs_orig)
#########################################
# SETUP WHERE TO OUTPUT RESULTING DATA (netcdf)
nout = os.path.join(outpathdata, "_".join(
[args.experiment, args.mip, 'wang-monsoon']))
try:
os.makedirs(nout)
except BaseException:
pass
# SETUP WHERE TO OUTPUT RESULTS (json)
jout = outpathjsons
try:
os.makedirs(nout)
except BaseException:
pass
modpathall = modpath.replace('MODS', '*')
lst = glob.glob(modpathall)
# CONFIRM DATA FOR MODS IS AVAIL AND REMOVE THOSE IT IS NOT
gmods = [] # "Got" these MODS
for mod in mods:
for l in lst:
l1 = modpath.replace('MODS', mod)
if os.path.isfile(l1) is True:
if mod not in gmods:
gmods.append(mod)
if args.experiment == 'historical' and mods is None:
gmods = [
'ACCESS1-0',
'ACCESS1-3',
'bcc-csm1-1',
'bcc-csm1-1-m',
'BNU-ESM',
'CanCM4',
'CanESM2',
'CCSM4',
'CESM1-BGC',
'CESM1-CAM5',
'CESM1-FASTCHEM',
'CESM1-WACCM',
'CMCC-CESM',
'CMCC-CM',
'CMCC-CMS',
'CNRM-CM5-2',
'CNRM-CM5',
'CSIRO-Mk3-6-0',
'FGOALS-g2',
'FIO-ESM',
'GFDL-CM2p1',
'GFDL-CM3',
'GFDL-ESM2G',
'GFDL-ESM2M',
'GISS-E2-H',
'GISS-E2-H-CC',
'GISS-E2-R',
'GISS-E2-R-CC',
'HadCM3',
'HadGEM2-AO',
'HadGEM2-CC',
'HadGEM2-ES',
'inmcm4',
'IPSL-CM5A-LR',
'IPSL-CM5A-MR',
'IPSL-CM5B-LR',
'MIROC4h',
'MIROC5',
'MIROC-ESM',
'MIROC-ESM-CHEM',
'MPI-ESM-LR',
'MPI-ESM-MR',
'MPI-ESM-P',
'MRI-CGCM3',
'MRI-ESM1',
'NorESM1-M',
'NorESM1-ME']
#########################################
try:
egg_pth = pkg_resources.resource_filename(
pkg_resources.Requirement.parse("pcmdi_metrics"), "share/pmp")
except Exception:
# python 2 seems to fail when ran in home directory of source?
egg_pth = os.path.join(os.getcwd(), "share", "pmp")
globals = {}
locals = {}
exec(compile(open(os.path.join(egg_pth, "default_regions.py")).read(),
os.path.join(egg_pth, "default_regions.py"), 'exec'), globals, locals)
regions_specs = locals["regions_specs"]
doms = ['AllMW', 'AllM', 'NAMM', 'SAMM', 'NAFM', 'SAFM', 'ASM', 'AUSM']
mpi_stats_dic = {}
for mod in gmods:
modelFile = modpath.replace('MODS', mod)
mpi_stats_dic[mod] = {}
print("******************************************************************************************")
print(modelFile)
f = cdms2.open(modelFile)
d_orig = f(var)
annrange_mod, mpi_mod = mpd(d_orig)
annrange_mod = annrange_mod.regrid(obsgrid)
mpi_mod = mpi_mod.regrid(obsgrid)
for dom in doms:
mpi_stats_dic[mod][dom] = {}
reg_sel = regions_specs[dom]['domain']
mpi_obs_reg = mpi_obs(reg_sel)
mpi_obs_reg_sd = float(statistics.std(mpi_obs_reg, axis='xy'))
mpi_mod_reg = mpi_mod(reg_sel)
cor = float(
statistics.correlation(
mpi_mod_reg,
mpi_obs_reg,
axis='xy'))
rms = float(statistics.rms(mpi_mod_reg, mpi_obs_reg, axis='xy'))
rmsn = rms / mpi_obs_reg_sd
# DOMAIN SELECTED FROM GLOBAL ANNUAL RANGE FOR MODS AND OBS
annrange_mod_dom = annrange_mod(reg_sel)
annrange_obs_dom = annrange_obs(reg_sel)
# SKILL SCORES
# HIT/(HIT + MISSED + FALSE ALARMS)
hit, missed, falarm, score, hitmap, missmap, falarmmap = mpi_skill_scores(
annrange_mod_dom, annrange_obs_dom, thr)
# POPULATE DICTIONARY FOR JSON FILES
mpi_stats_dic[mod][dom] = {}
mpi_stats_dic[mod][dom]['cor'] = format(cor, sig_digits)
mpi_stats_dic[mod][dom]['rmsn'] = format(rmsn, sig_digits)
mpi_stats_dic[mod][dom]['threat_score'] = format(score, sig_digits)
# SAVE ANNRANGE AND HIT MISS AND FALSE ALARM FOR EACH MOD DOM
fm = os.path.join(nout, '_'.join([mod, dom, 'wang-monsoon.nc']))
g = cdms2.open(fm, 'w')
g.write(annrange_mod_dom)
g.write(hitmap, dtype=numpy.int32)
g.write(missmap, dtype=numpy.int32)
g.write(falarmmap, dtype=numpy.int32)
g.close()
f.close()
# OUTPUT METRICS TO JSON FILE
OUT = pcmdi_metrics.io.base.Base(os.path.abspath(jout), json_filename)
disclaimer = open(
os.path.join(
egg_pth,
"disclaimer.txt")).read()
metrics_dictionary = collections.OrderedDict()
metrics_dictionary["DISCLAIMER"] = disclaimer
metrics_dictionary["REFERENCE"] = "The statistics in this file are based on" +\
" Wang, B., Kim, HJ., Kikuchi, K. et al. " +\
"Clim Dyn (2011) 37: 941. doi:10.1007/s00382-010-0877-0"
metrics_dictionary["RESULTS"] = mpi_stats_dic # collections.OrderedDict()
OUT.var = var
OUT.write(
metrics_dictionary,
json_structure=["model", "domain", "statistic"],
indent=4,
separators=(
',',
': '))
print(modelFile)
f = cdms2.open(modelFile)
d_orig = f(var)
annrange_mod, mpi_mod = mpd(d_orig)
annrange_mod = annrange_mod.regrid(obsgrid)
mpi_mod = mpi_mod.regrid(obsgrid)
for dom in doms:
mpi_stats_dic[mod][dom] = {}
reg_sel = regions_specs[dom]['domain']
mpi_obs_reg = mpi_obs(reg_sel)
mpi_obs_reg_sd = float(statistics.std(mpi_obs_reg, axis='xy'))
mpi_mod_reg = mpi_mod(reg_sel)
cor = float(statistics.correlation(mpi_mod_reg, mpi_obs_reg,
axis='xy'))
rms = float(statistics.rms(mpi_mod_reg, mpi_obs_reg, axis='xy'))
rmsn = rms / mpi_obs_reg_sd
# DOMAIN SELECTED FROM GLOBAL ANNUAL RANGE FOR MODS AND OBS
annrange_mod_dom = annrange_mod(reg_sel)
annrange_obs_dom = annrange_obs(reg_sel)
# SKILL SCORES
# HIT/(HIT + MISSED + FALSE ALARMS)
hit, missed, falarm, score, hitmap, missmap, falarmmap = mpi_skill_scores(
annrange_mod_dom, annrange_obs_dom, thr)
def monsoon_wang_runner(args):
# args = P.parse_args(sys.argv[1:])
modpath = args.modpath
outpathjsons = args.outpathjsons
outpathdata = args.results_dir
if isinstance(args.modnames, str):
mods = eval(args.modnames)
else:
mods = args.modnames
json_filename = args.jsonname
if json_filename == 'CMIP_MME':
json_filename = '/MPI_' + args.mip + '_' + args.experiment
if args.mip == 'CMIP5' and args.experiment == 'historical' and mods is None:
mods = [
'ACCESS1-0', 'ACCESS1-3', 'bcc-csm1-1', 'bcc-csm1-1-m', 'BNU-ESM',
'CanCM4', 'CanESM2', 'CCSM4', 'CESM1-BGC', 'CESM1-CAM5',
'CESM1-FASTCHEM', 'CESM1-WACCM', 'CMCC-CESM', 'CMCC-CM',
'CMCC-CMS', 'CNRM-CM5-2', 'CNRM-CM5', 'CSIRO-Mk3-6-0', 'FGOALS-g2',
'FIO-ESM', 'GFDL-CM2p1', 'GFDL-CM3', 'GFDL-ESM2G', 'GFDL-ESM2M',
'GISS-E2-H', 'GISS-E2-H-CC', 'GISS-E2-R', 'GISS-E2-R-CC', 'HadCM3',
'HadGEM2-AO', 'HadGEM2-CC', 'HadGEM2-ES', 'inmcm4', 'IPSL-CM5A-LR',
'IPSL-CM5A-MR', 'IPSL-CM5B-LR', 'MIROC4h', 'MIROC5', 'MIROC-ESM',
'MIROC-ESM-CHEM', 'MPI-ESM-LR', 'MPI-ESM-MR', 'MPI-ESM-P',
'MRI-CGCM3', 'MRI-ESM1', 'NorESM1-M', 'NorESM1-ME'
]
# VAR IS FIXED TO BE PRECIP FOR CALCULATING MONSOON PRECIPITATION INDICES
var = args.modvar
thr = args.threshold
sig_digits = '.3f'
#########################################
# PMP monthly default PR obs
cdms2.axis.longitude_aliases.append("longitude_prclim_mpd")
cdms2.axis.latitude_aliases.append("latitude_prclim_mpd")
fobs = cdms2.open(args.reference_data_path)
dobs_orig = fobs(args.obsvar)
fobs.close()
obsgrid = dobs_orig.getGrid()
########################################
# FCN TO COMPUTE GLOBAL ANNUAL RANGE AND MONSOON PRECIP INDEX
annrange_obs, mpi_obs = mpd(dobs_orig)
#########################################
# SETUP WHERE TO OUTPUT RESULTING DATA (netcdf)
nout = os.path.join(outpathdata,
"_".join([args.experiment, args.mip, 'wang-monsoon']))
try:
os.makedirs(nout)
except BaseException:
pass
# SETUP WHERE TO OUTPUT RESULTS (json)
jout = outpathjsons
try:
os.makedirs(nout)
except BaseException:
pass
modpathall = modpath.replace('MODS', '*')
lst = glob.glob(modpathall)
# CONFIRM DATA FOR MODS IS AVAIL AND REMOVE THOSE IT IS NOT
gmods = [] # "Got" these MODS
for mod in mods:
for l in lst:
l1 = modpath.replace('MODS', mod)
if os.path.isfile(l1) is True:
if mod not in gmods:
gmods.append(mod)
if args.experiment == 'historical' and mods is None:
gmods = [
'ACCESS1-0', 'ACCESS1-3', 'bcc-csm1-1', 'bcc-csm1-1-m', 'BNU-ESM',
'CanCM4', 'CanESM2', 'CCSM4', 'CESM1-BGC', 'CESM1-CAM5',
'CESM1-FASTCHEM', 'CESM1-WACCM', 'CMCC-CESM', 'CMCC-CM',
'CMCC-CMS', 'CNRM-CM5-2', 'CNRM-CM5', 'CSIRO-Mk3-6-0', 'FGOALS-g2',
'FIO-ESM', 'GFDL-CM2p1', 'GFDL-CM3', 'GFDL-ESM2G', 'GFDL-ESM2M',
'GISS-E2-H', 'GISS-E2-H-CC', 'GISS-E2-R', 'GISS-E2-R-CC', 'HadCM3',
'HadGEM2-AO', 'HadGEM2-CC', 'HadGEM2-ES', 'inmcm4', 'IPSL-CM5A-LR',
'IPSL-CM5A-MR', 'IPSL-CM5B-LR', 'MIROC4h', 'MIROC5', 'MIROC-ESM',
'MIROC-ESM-CHEM', 'MPI-ESM-LR', 'MPI-ESM-MR', 'MPI-ESM-P',
'MRI-CGCM3', 'MRI-ESM1', 'NorESM1-M', 'NorESM1-ME'
]
#########################################
try:
egg_pth = pkg_resources.resource_filename(
pkg_resources.Requirement.parse("pcmdi_metrics"), "share/pmp")
except Exception:
# python 2 seems to fail when ran in home directory of source?
egg_pth = os.path.join(os.getcwd(), "share", "pmp")
globals = {}
locals = {}
exec(
compile(
open(os.path.join(egg_pth, "default_regions.py")).read(),
os.path.join(egg_pth, "default_regions.py"), 'exec'), globals,
locals)
regions_specs = locals["regions_specs"]
doms = ['AllMW', 'AllM', 'NAMM', 'SAMM', 'NAFM', 'SAFM', 'ASM', 'AUSM']
mpi_stats_dic = {}
for mod in gmods:
modelFile = modpath.replace('MODS', mod)
mpi_stats_dic[mod] = {}
print(
"******************************************************************************************"
)
print(modelFile)
f = cdms2.open(modelFile)
d_orig = f(var)
annrange_mod, mpi_mod = mpd(d_orig)
annrange_mod = annrange_mod.regrid(obsgrid)
mpi_mod = mpi_mod.regrid(obsgrid)
for dom in doms:
mpi_stats_dic[mod][dom] = {}
reg_sel = regions_specs[dom]['domain']
mpi_obs_reg = mpi_obs(reg_sel)
mpi_obs_reg_sd = float(statistics.std(mpi_obs_reg, axis='xy'))
mpi_mod_reg = mpi_mod(reg_sel)
cor = float(
statistics.correlation(mpi_mod_reg, mpi_obs_reg, axis='xy'))
rms = float(statistics.rms(mpi_mod_reg, mpi_obs_reg, axis='xy'))
rmsn = rms / mpi_obs_reg_sd
# DOMAIN SELECTED FROM GLOBAL ANNUAL RANGE FOR MODS AND OBS
annrange_mod_dom = annrange_mod(reg_sel)
annrange_obs_dom = annrange_obs(reg_sel)
# SKILL SCORES
# HIT/(HIT + MISSED + FALSE ALARMS)
hit, missed, falarm, score, hitmap, missmap, falarmmap = mpi_skill_scores(
annrange_mod_dom, annrange_obs_dom, thr)
# POPULATE DICTIONARY FOR JSON FILES
mpi_stats_dic[mod][dom] = {}
mpi_stats_dic[mod][dom]['cor'] = format(cor, sig_digits)
mpi_stats_dic[mod][dom]['rmsn'] = format(rmsn, sig_digits)
mpi_stats_dic[mod][dom]['threat_score'] = format(score, sig_digits)
# SAVE ANNRANGE AND HIT MISS AND FALSE ALARM FOR EACH MOD DOM
fm = os.path.join(nout, '_'.join([mod, dom, 'wang-monsoon.nc']))
g = cdms2.open(fm, 'w')
g.write(annrange_mod_dom)
g.write(hitmap, dtype=numpy.int32)
g.write(missmap, dtype=numpy.int32)
g.write(falarmmap, dtype=numpy.int32)
g.close()
f.close()
# OUTPUT METRICS TO JSON FILE
OUT = pcmdi_metrics.io.base.Base(os.path.abspath(jout), json_filename)
disclaimer = open(os.path.join(egg_pth, "disclaimer.txt")).read()
metrics_dictionary = collections.OrderedDict()
metrics_dictionary["DISCLAIMER"] = disclaimer
metrics_dictionary["REFERENCE"] = "The statistics in this file are based on" +\
" Wang, B., Kim, HJ., Kikuchi, K. et al. " +\
"Clim Dyn (2011) 37: 941. doi:10.1007/s00382-010-0877-0"
metrics_dictionary["RESULTS"] = mpi_stats_dic # collections.OrderedDict()
OUT.var = var
OUT.write(metrics_dictionary,
json_structure=["model", "domain", "statistic"],
indent=4,
separators=(',', ': '))
def getARData(slab, quslab, qvslab, anoslab, quano, qvano, areas, mask_list,
axis_list, timestr, param_dict, shift_lon, isplot, outputdir):
'''Fetch AR related data
Args:
slab (cdms.TransientVariable): (n * m) 2D array of IVT, in kg/m/s.
quslab (cdms.TransientVariable): (n * m) 2D array of u-flux, in kg/m/s.
qvslab (cdms.TransientVariable): (n * m) 2D array of v-flux, in kg/m/s.
anoslab (cdms.TransientVariable): (n * m) 2D array of IVT anomalies,
in kg/m/s.
quano (cdms.TransientVariable): (n * m) 2D array of u-flux anomalies,
in kg/m/s.
qvano (cdms.TransientVariable): (n * m) 2D array of v-flux anomalies,
in kg/m/s.
areas (cdms.TransientVariable): (n * m) 2D grid cell area slab, in km^2.
mask_list (list): list of 2D binary masks, each with the same shape as
<anoslab> etc., and with 1s denoting the location of a
found AR.
axis_list (list): list of AR axis coordinates. Each coordinate is defined
as a Nx2 ndarray storing (y, x) indices of the axis
(indices defined in the matrix of corresponding mask
in <masks>.)
timestr (str): string of time snap.
param_dict (dict): parameter dict defined in Global preamble.
shift_lon (float): starting longitude of data domain, defined in Global
preamble.
isplot (bool): if True, create plot of AR axis, flux orientation and
cross-sectional flux for each AR.
outputdir (str): folder to save plots. If None, don't save. If <isplot>
is False, not relevant.
Returns:
labels (cdms.TransientVariable): (n * m) 2D int map showing all ARs
at current time. Each AR is labeled by
an int label, starting from 1. Background
is filled with 0s.
angles (cdms.TransientVariable): (n * m) 2D map showing orientation
differences between AR axes and fluxes,
for all ARs. In degrees.
crossfluxes (cdms.TransientVariable): (n * m) 2D map showing cross-
sectional fluxes in all ARs.
In kg/m/s.
anocrossflux (cdms.TransientVariable): similar as <crossfluxes> but for
anomalous fluxes (corresponding
to <anoslab>).
df (pandas.DataFrame): AR record table. Each row is an AR, see code
below for columns.
'''
max_isoq = param_dict['max_isoq']
min_length = param_dict['min_length']
min_length_hard = param_dict['min_length_hard']
rdp_thres = param_dict['rdp_thres']
min_area = param_dict['min_area']
lonax = slab.getLongitude() # NOTE: max > 360
latax = slab.getLatitude()
# prepare outputs
labels = MV.zeros(slab.shape)
angles = MV.zeros(slab.shape)
crossfluxes = MV.zeros(slab.shape)
results = {}
#-----------------Loop through ARs-----------------
for ii in range(len(mask_list)):
maskii = mask_list[ii]
# region properties, in pixel units
rpii = measure.regionprops(maskii, intensity_image=np.array(slab))[0]
# get centroid
centroidy, centroidx = rpii.weighted_centroid
centroidy = latax[int(centroidy)]
centroidx = lonax[int(centroidx)]
# get axis coordinate array
skelii = axis_list[ii]
latsii = latax[skelii[:, 0]]
lonsii = lonax[skelii[:, 1]]
axisii = np.c_[latsii, lonsii]
# segment axis using rdp
axis_rdpii = np.array(rdp.rdpGC(axisii.tolist(), rdp_thres)) # lat,lon
# area
areaii = (maskii * areas).sum() # km^2
# compute length
lens = funcs.greatCircle(axis_rdpii[:-1, 0], axis_rdpii[:-1, 1],
axis_rdpii[1:, 0], axis_rdpii[1:, 1]) / 1e3
lenii = lens.sum() #km
# skip if too small and too short
if areaii < min_area or lenii < min_length_hard:
continue
# mean width
widthii = areaii / lenii # km
# mask contour
contii = funcs.getBinContour(maskii, lonax, latax)
# isoperimetric quotient
isoquoii = 4 * np.pi * rpii.area / rpii.perimeter**2
# length/width ratio
ratioii = lenii / widthii
# mean strength
slabii = MV.masked_where(maskii == 0, slab)
strengthii = cdutil.averager(slabii,
axis='xy',
weights=['generate', 'generate'])
# strength std
strengthstdii = float(stats.std(slabii, axis='xy'))
# anomaly strength
anoslabii = MV.masked_where(maskii == 0, anoslab)
anostrengthii = cdutil.averager(anoslabii,
axis='xy',
weights=['generate', 'generate'])
# max strength
max_strengthii = float(MV.max(slabii))
# compute angles and cross-section flux of total flux
cropmask, cropidx = cropMask(maskii)
cropskelii = skelii - np.array([cropidx[0].min(), cropidx[1].min()])
cropu = applyCropIdx(quslab, cropidx)
cropv = applyCropIdx(qvslab, cropidx)
anglesii, anglesmeanii, crossfluxii, seg_thetasii = crossSectionFlux(
cropmask, cropu, cropv, axis_rdpii)
# create plots
if isplot:
pass
#plotARCrosssectionFlux(cropmask, cropu, cropv, cropskelii, axis_rdpii,
#'%s AR-%d' %(timestr, ii+1), shift_lon, anglesii, anglesmeanii,
#crossfluxii, seg_thetasii, outputdir)
# insert crop back to the big map
anglesii = insertCropSlab(maskii.shape, anglesii, cropidx,
slab.getAxisList())
anglesii = MV.where(maskii == 1, anglesii, 0)
crossfluxii = insertCropSlab(maskii.shape, crossfluxii, cropidx,
slab.getAxisList())
crossfluxii = MV.where(maskii == 1, crossfluxii, 0)
# mean meridional flux
cropv = applyCropIdx(qvslab, cropidx)
cropv = MV.masked_where(cropmask == 0, cropv)
qvmeanii=cdutil.averager(cropv,axis='xy',weights=['generate',\
'generate'])
# is candidate a strict AR
is_relaxedii = False
if isoquoii > max_isoq or ratioii < 2:
is_relaxedii = True
if lenii < min_length:
is_relaxedii = True
if qvmeanii <= 0:
is_relaxedii = True
labels = labels + maskii * (ii + 1)
angles = angles + anglesii
crossfluxes = crossfluxes + crossfluxii
results[ii + 1] = {
'id': ii + 1,
'time': timestr,
'contour_y': contii.vertices[:, 1],
'contour_x': contii.vertices[:, 0],
'centroid_y': centroidy,
'centroid_x': centroidx,
'axis_y': axisii[:, 0],
'axis_x': axisii[:, 1],
'axis_rdp_y': axis_rdpii[:, 0],
'axis_rdp_x': axis_rdpii[:, 1],
'area': areaii,
'length': lenii,
'width': widthii,
'iso_quotient': isoquoii,
'LW_ratio': ratioii,
'strength': strengthii,
'strength_ano': anostrengthii,
'strength_std': strengthstdii,
'max_strength': max_strengthii,
'mean_angle': float(anglesmeanii),
'is_relaxed': is_relaxedii,
'qv_mean': qvmeanii
}
labels.setAxisList(slab.getAxisList())
angles.setAxisList(slab.getAxisList())
crossfluxes.setAxisList(slab.getAxisList())
labels.id = 'labels'
labels.long_name = 'AR labels'
labels.standard_name = labels.long_name
labels.title = labels.long_name
labels.units = ''
angles.id = 'angles'
angles.long_name = 'AR moisture flux orientation difference'
angles.standard_name = angles.long_name
angles.title = angles.long_name
angles.units = 'degree'
crossfluxes.id = 'ivt_cross'
crossfluxes.long_name = 'AR total cross sectional moisture flux'
crossfluxes.standard_name = crossfluxes.long_name
crossfluxes.title = crossfluxes.long_name
crossfluxes.units = getattr(slab, 'units', '')
keys = [
'id', 'time', 'contour_y', 'contour_x', 'centroid_y', 'centroid_x',
'axis_y', 'axis_x', 'axis_rdp_y', 'axis_rdp_x', 'area', 'length',
'width', 'iso_quotient', 'LW_ratio', 'strength', 'strength_ano',
'strength_std', 'max_strength', 'mean_angle', 'is_relaxed', 'qv_mean'
]
df = pd.DataFrame(results).T
if len(df) > 0:
df = df[keys]
return labels, angles, crossfluxes, df
def mmeAveMsk2D(listFiles, years, inDir, outDir, outFile, timeInt, mme, timeBowl, ToeType, debug=True):
'''
The mmeAveMsk2D() function averages rhon/lat density bined files with differing masks
It ouputs
- the MME
- a percentage of non-masked bins
- the sign agreement of period2-period1 differences
- ToE per run and for MME
Author: Eric Guilyardi : [email protected]
Created on Tue Nov 25 13:56:20 CET 2014
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- years(t1,t2) - years for slice read
- inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE)
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- timeBowl - either time 'mean' or time 'max' bowl used to mask out bowl
- ToeType(str) - ToE type ('F': none, 'histnat')
-> requires running first mm+mme without ToE to compute Stddev
- debug <optional> - boolean value
Notes:
-----
- EG 25 Nov 2014 - Initial function write
- EG 27 Nov 2014 - Rewrite with loop on variables
- EG 06 Dec 2014 - Added agreement on difference with init period - save as <var>Agree
- EG 07 Dec 2014 - Read bowl to remove points above bowl - save as <var>Bowl
- EG 19 Apr 2016 - ToE computation (just for 2D files)
- EG 07 Oct 2016 - add 3D file support
- EG 21 Nov 2016 - move 3D support to new function
- EG 10 jan 2017 - added timeBowl option
- TODO :
- remove loops
- add computation of ToE per model (toe 1 and toe 2) see ticket #50
- add isonhtc (see ticket #48)
'''
# CDMS initialisation - netCDF compression
comp = 1 # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
if t2 <= 0:
useLastYears = True
t2 = -t2
else:
useLastYears = False
t10 = t1
t20 = t2
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
fi = cdm.open(inDir[0]+'/'+listFiles[0])
isond0 = fi['isondepth'] ; # Create variable handle
# Get grid objects
axesList = isond0.getAxisList()
sigmaGrd = isond0.getLevel()
latN = isond0.shape[3]
levN = isond0.shape[2]
basN = isond0.shape[1]
varsig='ptopsigma'
# Declare and open files for writing
if os.path.isfile(outDir+'/'+outFile):
os.remove(outDir+'/'+outFile)
outFile_f = cdm.open(outDir+'/'+outFile,'w')
# Testing mme with less models
#listFiles=listFiles[0:4]
#timN = isond0.shape[0]
timN = t2-t1
runN = len(listFiles)
print ' Number of members:',len(listFiles)
valmask = isond0.missing_value[0]
varList = ['isondepth','isonpers','isonso','isonthetao','isonthick','isonvol']
varFill = [0.,0.,valmask,valmask,0.,0.]
# init arrays (2D rho/lat)
percent = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*0.
#minbowl = npy.ma.ones([basN,latN], dtype='float32')*1000.
varbowl = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*1.
#varList = ['isondepth']
#print ' !!! ### Testing one variable ###'
#varList = ['isonthetao']
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
time.designateTime()
# init ensemble axis
ensembleAxis = cdm.createAxis(npy.float32(range(runN)))
ensembleAxis.id = 'members'
ensembleAxis.units = 'N'
# loop on variables
for iv,var in enumerate(varList):
# Array inits (2D rho/lat 3D rho/lat/lon)
#shapeR = [basN,levN,latN]
isonvar = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*valmask
vardiff,varbowl2D = [npy.ma.ones(npy.ma.shape(isonvar)) for _ in range(2)]
varstd,varToE1,varToE2 = [npy.ma.ones([runN,basN,levN,latN], dtype='float32')*valmask for _ in range(3)]
varones = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*1.
print ' Variable ',iv, var
# loop over files to fill up array
for i,file in enumerate(listFiles):
ft = cdm.open(inDir[0]+'/'+file)
model = file.split('.')[1]
timeax = ft.getAxis('time')
file1d = replace(inDir[0]+'/'+file,'2D','1D')
if os.path.isfile(file1d):
f1d = cdm.open(file1d)
else:
print 'ERROR:',file1d,'missing (if mme, run 1D first)'
sys.exit(1)
tmax = timeax.shape[0]
if i == 0:
tmax0 = tmax
#adapt [t1,t2] time bounds to piControl last NN years
if useLastYears:
t1 = tmax-t20
t2 = tmax
else:
if tmax != tmax0:
print 'wrong time axis: exiting...'
return
# read array
# loop over time/density for memory management
for it in range(timN):
t1r = t1 + it
t2r = t1r + 1
isonRead = ft(var,time = slice(t1r,t2r))
if varFill[iv] != valmask:
isonvar[i,it,...] = isonRead.filled(varFill[iv])
else:
isonvar[i,it,...] = isonRead
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data,valmask).mask
percent[i,...] = npy.float32(npy.equal(maskvar,0))
if mme:
# if mme then just accumulate Bowl, Agree fields
varst = var+'Agree'
vardiff[i,...] = ft(varst,time = slice(t1,t2))
varb = var+'Bowl'
varbowl2D[i,...] = ft(varb,time = slice(t1,t2))
else:
# Compute difference with average of first initN years
varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0)
for t in range(timN):
vardiff[i,t,...] = isonvar[i,t,...] - varinit
vardiff[i,...].mask = isonvar[i,...].mask
# Read bowl and truncate 2D field above bowl
if iv == 0:
bowlRead = f1d(varsig,time = slice(t1,t2))
varbowl[i,...] = bowlRead
# Compute Stddev
varstd[i,...] = npy.ma.std(isonvar[i,...], axis=0)
# Compute ToE
if ToeType == 'histnat':
# Read mean and Std dev from histnat
if i == 0:
filehn = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0]
#filehn = replace(outFile,'historical','historicalNat')
fthn = cdm.open(filehn)
varmeanhn = fthn(var)
varst = var+'Std'
varmaxstd = fthn(varst)
toemult = 1.
signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN))
noise = npy.reshape(varmaxstd,(basN*levN*latN))
varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
toemult = 2.
varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
ft.close()
f1d.close()
# <-- end of loop on files
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent,axis=0))*100.
percenta = mv.masked_less(percenta, 50)
percentw = cdm.createVariable(percenta, axes = [time,axesList[1],axesList[2],axesList[3]], id = 'isonpercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'))
# Sign of difference
if mme:
vardiffsgSum = cdu.averager(vardiff, axis=0)
vardiffsgSum = cdm.createVariable(vardiffsgSum , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
vardiffsgSum = maskVal(vardiffsgSum, valmask)
vardiffsgSum.mask = percentw.mask
else:
vardiffsg = npy.copysign(varones,vardiff)
# average signs
vardiffsgSum = cdu.averager(vardiffsg, axis=0)
vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
vardiffsgSum.mask = percentw.mask
vardiffsgSum._FillValue = valmask
# average variable accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = cdm.createVariable(isonVarAve , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
# Only keep points with rhon > bowl-delta_rho
delta_rho = 0.
if mme: # start from average of <var>Agree
isonVarBowl = cdu.averager(varbowl2D, axis=0)
isonVarBowl = cdm.createVariable(isonVarBowl , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
isonVarBowl = maskVal(isonVarBowl, valmask)
isonVarBowl.mask = percentw.mask
# Compute intermodel stddev
isonVarStd = statistics.std(varbowl2D, axis=0)
isonVarStd = cdm.createVariable(isonVarStd , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
isonVarStd = maskVal(isonVarStd, valmask)
isonVarStd.mask = percentw.mask
if iv == 0:
# Read mulitmodel sigma on bowl and average in time
file1d = replace(outDir+'/'+outFile,'2D','1D')
if os.path.isfile(file1d):
f1d = cdm.open(file1d)
else:
print 'ERROR:',file1d,'missing (if mme, run 1D first)'
sys.exit(1)
bowlRead = f1d(varsig,time = slice(t1,t2))
f1d.close()
siglimit = cdu.averager(bowlRead, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
for il in range(latN):
for ib in range(basN):
#if ib == 2:
# print il, siglimit[ib,il]
if siglimit[ib,il] < valmask/1000.:
# if mme bowl density defined, mask above bowl
index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
isonVarBowl [:,ib,0:index[0],il].mask = True
isonVarStd [:,ib,0:index[0],il].mask = True
vardiffsgSum[:,ib,0:index[0],il].mask = True
else:
# mask all points
isonVarBowl [:,ib,:,il].mask = True
isonVarStd [:,ib,:,il].mask = True
vardiffsgSum[:,ib,:,il].mask = True
else:
isonVarBowl = isonVarAve*1. # start from variable
isonVarStd = isonVarAve*1. # start from variable
if iv == 0:
siglimit = cdu.averager(varbowl, axis=0) # average accross members
# Average bowl in time
if timeBowl == 'mean':
siglimit = cdu.averager(siglimit, axis=0) - delta_rho
# or take largest sigma over time
else:
siglimit = npy.ma.max(siglimit, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
for il in range(latN):
for ib in range(basN):
if siglimit[ib,il] < valmask/1000.:
# if bowl density defined, mask above bowl
index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
isonVarBowl[:,ib,0:index[0],il].mask = True
vardiffsgSum[:,ib,0:index[0],il].mask = True
else:
# mask all points
vardiffsgSum[:,ib,:,il].mask = True
isonVarBowl = maskVal(isonVarBowl, valmask)
# Find max of Std dev of all members
isonVarStd = npy.ma.max(varstd, axis=0)
# mask
if varFill[iv] == valmask:
isonVarStd = maskVal(isonVarStd, valmask)
# Write
isonave = cdm.createVariable(isonVarAve, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id)
isonave.long_name = isonRead.long_name
isonave.units = isonRead.units
isonavediff = cdm.createVariable(vardiffsgSum, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Agree')
isonavediff.long_name = isonRead.long_name
isonavediff.units = isonRead.units
isonavebowl = cdm.createVariable(isonVarBowl, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Bowl')
isonavebowl.long_name = isonRead.long_name
isonavebowl.units = isonRead.units
if not mme:
isonmaxstd = cdm.createVariable(isonVarStd, axes = [axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Std')
isonmaxstd.long_name = isonRead.long_name
isonmaxstd.units = isonRead.units
outFile_f.write( isonave.astype('float32'))
outFile_f.write(isonavediff.astype('float32'))
outFile_f.write(isonavebowl.astype('float32'))
if not mme:
outFile_f.write( isonmaxstd.astype('float32'))
if ToeType == 'histnat':
isontoe1 = cdm.createVariable(varToE1, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE1')
isontoe1.long_name = 'ToE 1 for '+isonRead.long_name
isontoe1.units = 'Year'
isontoe2 = cdm.createVariable(varToE2, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE2')
isontoe2.long_name = 'ToE 2 for '+isonRead.long_name
isontoe2.units = 'Year'
outFile_f.write(isontoe1.astype('float32'))
outFile_f.write(isontoe2.astype('float32'))
if mme:
isonvarstd = cdm.createVariable(isonVarStd , axes =[time,axesList[1],axesList[2],axesList[3]] , id = isonRead.id+'ModStd')
isonvarstd.long_name = isonRead.long_name+' intermodel std'
isonvarstd.units = isonRead.units
outFile_f.write(isonvarstd.astype('float32'))
# <--- end of loop on variables
outFile_f.close()
fi.close()
def mmeAveMsk3D(listFiles, years, inDir, outDir, outFile, timeInt, mme, ToeType, debug=True):
'''
The mmeAveMsk3D() function averages rhon/lat density bined files with differing masks
It ouputs
- the MME
- a percentage of non-masked bins
- the sign agreement of period2-period1 differences
- ToE per run and for MME
Author: Eric Guilyardi : [email protected]
Created on Tue Nov 21 2016
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- years(t1,t2) - years for slice read
- inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE)
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- ToeType(str) - ToE type ('F': none, 'histnat')
-> requires running first mm+mme without ToE to compute Stddev
- debug <optional> - boolean value
Notes:
-----
- EG 21 Nov 2016 - Initial function write
- TODO :
- add computation of ToE per model (toe 1 and toe 2) see ticket #50
- add isonhtc (see ticket #48)
'''
# CDMS initialisation - netCDF compression
comp = 1 # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
fi = cdm.open(inDir[0]+'/'+listFiles[0])
# Switch if only variables below the bowl are present/treated
nobowl = True
if nobowl:
isond0 = fi['isondepthgBowl'] ; # Create variable handle
else:
isond0 = fi['isondepthg'] ; # Create variable handle
# Get grid objects
axesList = isond0.getAxisList()
sigmaGrd = isond0.getLevel()
#time = isond0.getTime()
lonN = isond0.shape[3]
latN = isond0.shape[2]
levN = isond0.shape[1]
varsig='ptopsigmaxy'
# Limit number of models to 3 for testing of mme
#if mme:
# listFiles = listFiles[0:2]
# print ' !!! ### Testing 3 models ###', listFiles
# Declare and open files for writing
if os.path.isfile(outDir+'/'+outFile):
os.remove(outDir+'/'+outFile)
outFile_f = cdm.open(outDir+'/'+outFile,'w')
#timN = isond0.shape[0]
timN = t2-t1
runN = len(listFiles)
print ' Number of members:',len(listFiles)
valmask = isond0.missing_value
varList = ['isondepthg','persistmxy','sog','thetaog','isonthickg']
varFill = [valmask,valmask,valmask,valmask,valmask]
percent = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*0.
varbowl = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*1.
#varList = ['isondepthg']
#print ' !!! ### Testing one variable ###', varList
# init sigma axis
sigma = cdm.createAxis(npy.float32(range(1)))
sigma.id = axesList[1].id
sigma.units = axesList[1].units
sigma.designateTime()
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
# init ensemble axis
ensembleAxis = cdm.createAxis(npy.float32(range(runN)))
ensembleAxis.id = 'members'
ensembleAxis.units = 'N'
# Output axis
sigmaList = [sigma,axesList[2],axesList[3]] ; # sigma, lat, lon
sigmaTimeList = [sigma,time,axesList[2],axesList[3]] ; # sigma, time, lat, lon
# init arrays
isonvar = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask
varbowl2D = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask
varstd,varToE1,varToE2 = [npy.ma.ones([runN,latN,lonN], dtype='float32')*valmask for _ in range(3)]
# Loop on density levels (for memory management, becomes UNLIMITED axis and requires a ncpq to reorder dimensions)
delta_ib = 1
print ' Sigma index:'
for ib in range(levN):
ib1 = ib + delta_ib
print ib,
tim0 = timc.clock()
# loop on variables
for iv,var in enumerate(varList):
if nobowl:
varb = var+'Bowl'
else:
varb = var
if ib == 0:
print ' Variable ',iv, varb
# loop over files to fill up array
for i,file in enumerate(listFiles):
tim01 = timc.clock()
ft = cdm.open(inDir[0]+'/'+file)
model = file.split('.')[1]
timeax = ft.getAxis('time')
if i == 0:
tmax0 = timeax.shape[0]
tmax = timeax.shape[0]
if tmax != tmax0:
print 'wrong time axis: exiting...'
return
# read array
isonRead = ft(varb,time = slice(t1,t2), lev = slice(ib,ib1)).squeeze()
if varFill[iv] != valmask:
isonvar[i,...] = isonRead.filled(varFill[iv])
else:
isonvar[i,...] = isonRead
tim02 = timc.clock()
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data,valmask).mask
percent[i,...] = npy.float32(npy.equal(maskvar,0))
tim03 = timc.clock()
if mme:
# if mme then just accumulate Bowl, Agree and Std fields
#varst = var+'Agree'
#vardiff[i,...] = ft(varst,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze()
isonRead = ft(varb,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze()
varbowl2D[i,...] = isonRead
else:
# Compute difference with average of first initN years
#varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0)
#for t in range(timN):
# vardiff[i,t,...] = isonvar[i,t,...] - varinit
#vardiff[i,...].mask = isonvar[i,...].mask
# Read bowl to truncate field above bowl
if ib == 0 and iv == 0:
varbowl[i,...] = ft(varsig,time = slice(t1,t2))
#varbowl[i,...] = bowlRead
# Compute Stddev
varstd[i,...] = npy.ma.std(isonvar[i,...], axis=0)
# Compute ToE
if ToeType == 'histnat':
toto=1
# TODO
# Read mean and Std dev from histnat
# if i == 0:
# filehn = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0]
# #filehn = replace(outFile,'historical','historicalNat')
# fthn = cdm.open(filehn)
# varmeanhn = fthn(var)
# varst = var+'Std'
# varmaxstd = fthn(varst)
# toemult = 1.
# signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN))
# noise = npy.reshape(varmaxstd,(basN*levN*latN))
# varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
# toemult = 2.
# varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
tim04 = timc.clock()
ft.close()
#print 'ib, section 1 timing',ib, tim02-tim01,tim03-tim02,tim04-tim03
# <-- end of loop on files (i)
tim1 = timc.clock()
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent,axis=0))*100.
percenta = mv.masked_less(percenta, 50)
percenta = npy.reshape(percenta,[delta_ib,timN,latN,lonN])
percentw = cdm.createVariable(percenta, axes = sigmaTimeList, id = 'isonpercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'), extend = 1, index = ib)
# Sign of difference
#if mme:
# vardiffsgSum = cdu.averager(vardiff, axis=0)
# vardiffsgSum = cdm.createVariable(vardiffsgSum , axes = sigmaTimeList , id = 'foo')
# vardiffsgSum = maskVal(vardiffsgSum, valmask)
# vardiffsgSum.mask = percentw.mask
#else:
# vardiffsg = npy.copysign(varones,vardiff)
# # average signs
# vardiffsgSum = cdu.averager(vardiffsg, axis=0)
# vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
# vardiffsgSum.mask = percentw.mask
# vardiffsgSum._FillValue = valmask
# average variable accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = npy.reshape(isonVarAve,[delta_ib,timN,latN,lonN])
isonVarAve = cdm.createVariable(isonVarAve , axes = sigmaTimeList , id = 'foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
tim2 = timc.clock()
# Only keep points with rhon > bowl-delta_rho
delta_rho = 0.
# mme case
if mme: # start from average of <var>Agree
isonVarBowl = cdu.averager(varbowl2D, axis=0)
isonVarBowl = npy.reshape(isonVarBowl,[delta_ib,timN,latN,lonN])
isonVarBowl = cdm.createVariable(isonVarBowl , axes = sigmaTimeList , id = 'foo')
isonVarBowl = maskVal(isonVarBowl, valmask)
isonVarBowl.mask = percentw.mask
# Compute intermodel stddev
isonVarStd = statistics.std(varbowl2D, axis=0)
isonVarStd = npy.reshape(isonVarStd,[delta_ib,timN,latN,lonN])
isonVarStd = cdm.createVariable(isonVarStd , axes = sigmaTimeList , id = 'foo')
isonVarStd = maskVal(isonVarStd, valmask)
isonVarStd.mask = percentw.mask
# Write
isonvarbowlw = cdm.createVariable(isonVarBowl , axes = sigmaTimeList , id = isonRead.id)
isonvarbowlw.long_name = isonRead.long_name
isonvarbowlw.units = isonRead.units
isonvarstdw = cdm.createVariable(isonVarStd , axes = sigmaTimeList , id = isonRead.id+'Std')
isonvarstdw.long_name = isonRead.long_name+' intermodel std'
isonvarstdw.units = isonRead.units
outFile_f.write(isonvarbowlw.astype('float32'), extend = 1, index = ib)
outFile_f.write(isonvarstdw.astype('float32'), extend = 1, index = ib)
#if ib == 0 and iv == 0:
# # TODO review
# # Read multimodel sigma on bowl and average in time
# file1d = replace(outDir+'/'+outFile,'2D','1D')
# if os.path.isfile(file1d):
# f1d = cdm.open(file1d)
# else:
# print 'ERROR:',file1d,'missing (if mme, run 2D first)'
# sys.exit(1)
# bowlRead = f1d(varsig,time = slice(t1,t2),lev = slice(ib,ib1))
# f1d.close()
# siglimit = cdu.averager(bowlRead, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
#if sw2d == 1:
# for il in range(latN):
# for ib in range(basN):
# #if ib == 2:
# # print il, siglimit[ib,il]
# if siglimit[ib,il] < valmask/1000.:
# # if mme bowl density defined, mask above bowl
# index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
# isonVarBowl [:,ib,0:index[0],il].mask = True
# isonVarStd [:,ib,0:index[0],il].mask = True
# vardiffsgSum[:,ib,0:index[0],il].mask = True
# else:
# # mask all points
# isonVarBowl [:,ib,:,il].mask = True
# isonVarStd [:,ib,:,il].mask = True
# vardiffsgSum[:,ib,:,il].mask = True
# mm case
else:
isonVarBowl = isonVarAve*1. # start from variable
#isonVarStd = isonVarAve*1. # start from variable
if ib == 0 and iv == 0:
# build bowl position
siglimit = cdu.averager(varbowl, axis=0) # average accross members
siglimit = npy.reshape(siglimit,[timN*latN*lonN]) - delta_rho
if iv == 0:
sigarr = siglimit*1.
sigarr[:] = sigmaGrd[ib]
# test
i = 60
j = 60
ij = j*lonN+i
isonVarBowl = npy.reshape(isonVarBowl,[timN*latN*lonN])
#vardiffsgSum = npy.reshape(vardiffsgSum,[timN*latN*lonN])
isonVarBowl.mask = npy.where(sigarr < siglimit, True, isonVarBowl.mask)
#vardiffsgSum.mask = npy.where(sigarr < siglimit, True, vardiffsgSum.mask)
isonVarBowl = npy.reshape(isonVarBowl,[timN,latN,lonN])
#vardiffsgSum = npy.reshape(vardiffsgSum,[timN,latN,lonN])
isonVarBowl = maskVal(isonVarBowl, valmask)
#vardiffsgSum = maskVal(vardiffsgSum, valmask)
# Find max of Std dev of all members
isonVarStd = npy.ma.max(varstd, axis=0)
# mask
isonVarStd = maskVal(isonVarStd, valmask)
# Write
#isonave = cdm.createVariable(isonVarAve, axes = sigmaTimeList, id = isonRead.id)
#isonave.long_name = isonRead.long_name
#isonave.units = isonRead.units
#vardiffsgSum = npy.reshape(vardiffsgSum,[delta_ib,timN,latN,lonN])
#isonavediff = cdm.createVariable(vardiffsgSum, axes = sigmaTimeList, id = isonRead.id+'Agree')
#isonavediff.long_name = isonRead.long_name
#isonavediff.units = isonRead.units
isonVarBowl = npy.reshape(isonVarBowl,[delta_ib,timN,latN,lonN])
isonavebowl = cdm.createVariable(isonVarBowl, axes = sigmaTimeList, id = isonRead.id+'Bowl')
isonavebowl.long_name = isonRead.long_name
isonavebowl.units = isonRead.units
isonVarStd = npy.reshape(isonVarStd,[delta_ib,latN,lonN])
isonmaxstd = cdm.createVariable(isonVarStd, axes = sigmaList, id = isonRead.id+'Std')
isonmaxstd.long_name = isonRead.long_name
isonmaxstd.units = isonRead.units
#outFile_f.write( isonave.astype('float32'), extend = 1, index = ib)
#outFile_f.write(isonavediff.astype('float32'), extend = 1, index = ib)
outFile_f.write(isonavebowl.astype('float32'), extend = 1, index = ib)
outFile_f.write(isonmaxstd.astype('float32'), extend = 1, index = ib)
tim3 = timc.clock()
if ToeType == 'histnat':
isontoe1 = cdm.createVariable(varToE1, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE1')
isontoe1.long_name = 'ToE 1 for '+isonRead.long_name
isontoe1.units = 'Year'
isontoe2 = cdm.createVariable(varToE2, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE2')
isontoe2.long_name = 'ToE 2 for '+isonRead.long_name
isontoe2.units = 'Year'
outFile_f.write(isontoe1.astype('float32'), extend = 1, index = ib)
outFile_f.write(isontoe2.astype('float32'), extend = 1, index = ib)
tim4 = timc.clock()
# <--- end of loop on variables
#print 'ib, timing',ib, tim01-tim0,tim1-tim01,tim2-tim1,tim3-tim2,tim4-tim3
# <--- end of loop on density
print ' '
outFile_f.close()
fi.close()
