def select_CF(sCF, wCF, idx, *i):
if idx == 1.:
masked_sCF = np.ceil(sCF)
masked_wCF = np.ceil(wCF)
s_avg_tmp = cdutil.averager(sCF, axis='yx')
w_avg_tmp = cdutil.averager(wCF, axis='yx')
elif idx == 2.:
# set threshold;
# you can change this values based on needs;
s_thresholds = 0.26
w_thresholds = 0.26
# set grids with values lower than the threshold to 0;
sCF[sCF < s_thresholds] = 0.
wCF[wCF < w_thresholds] = 0.
# mask grids with a value of 0 (grids that are not needed);
masked_sCF = set_axes(MV.masked_equal(sCF, 0.) * 0. + 1)
masked_wCF = set_axes(MV.masked_equal(wCF, 0.) * 0. + 1)
s_avg_tmp = cdutil.averager(masked_sCF * sCF, axis='yx')
w_avg_tmp = cdutil.averager(masked_wCF * wCF, axis='yx')
elif idx == 3.:
# set threshold, top X%;
# you can change this values based on needs;
p_threshold_s = 0.25
p_threshold_w = 0.25
# sort all grids, values for grids that are
# outside the interested region are set to -1;
s_reshape = np.sort(np.array(MV.filled(sCF, -1)).ravel())[::-1]
w_reshape = np.sort(np.array(MV.filled(wCF, -1)).ravel())[::-1]
# remove grids with values of -1;
s_no_minus1 = s_reshape[s_reshape != -1]
w_no_minus1 = w_reshape[w_reshape != -1]
# find the threshold for the top X%
num_s = int(len(s_no_minus1) * p_threshold_s)
num_w = int(len(w_no_minus1) * p_threshold_w)
s_thresholds = s_no_minus1[num_s - 1]
w_thresholds = w_no_minus1[num_w - 1]
# mask grids with a value lower than the derived thresholds (grids that are not needed);
masked_sCF = set_axes(MV.masked_less(sCF, s_thresholds) * 0. + 1)
masked_wCF = set_axes(MV.masked_less(wCF, w_thresholds) * 0. + 1)
s_avg_tmp = cdutil.averager(masked_sCF * sCF, axis='yx')
w_avg_tmp = cdutil.averager(masked_wCF * wCF, axis='yx')
print(
f"For method {idx}, averaged solar capacity factor for the filtered grids is: {s_avg_tmp}"
)
print(
f"For method {idx}, averaged wind capacity factor for the filtered grids is: {w_avg_tmp}"
)
print(
f"Selected grid cells solar {np.ceil(masked_sCF).sum()}\nSelected grid cells wind {np.ceil(masked_wCF).sum()}"
)
return masked_sCF, masked_wCF
def testContiguousRegridNANIssue(self):
a = MV2.reshape(MV2.sin(MV2.arange(20000)), (2, 1, 100, 100))
lon = cdms2.createAxis(MV2.arange(100) * 3.6)
lon.designateLongitude()
lon.units = "degrees_east"
lon.id = "longitude"
lat = cdms2.createAxis(MV2.arange(100) * 1.8 - 90.)
lat.id = "latitude"
lat.designateLatitude()
lat.units = "degrees_north"
lev = cdms2.createAxis([1000.])
lev.id = "plev"
lev.designateLevel()
lev.units = "hPa"
t = cdms2.createAxis([0, 31.])
t.id = "time"
t.designateTime()
t.units = "days since 2014"
cdutil.setTimeBoundsMonthly(t)
a.setAxisList((t, lev, lat, lon))
a = MV2.masked_less(a, .5)
grd = cdms2.createGaussianGrid(64)
a = a.ascontiguous()
a = a.regrid(grd, regridTool="regrid2")
a = cdutil.averager(a, axis='txy')
self.assertEqual(a[0], 0.7921019540305255)
def testAxismissing(self):
data = """
-999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999.
0.059503571833625334
0.059503571833625334 0.05664014775641405 0.05193557222118004
0.04777129850801233 0.0407139313814465 0.029382624830271705
0.018469399844287374 0.0162382275289592 0.02646680241827459
0.04792041732949079 0.0689138797030203 0.08167038620212037
0.09273558459066569 0.11266293431057901 0.13663018925347364
0.15229174546388072 0.15284435880966177 0.13423845476113883
0.09945904378274077 0.07032267160267985 0.05551039827020481
0.045537187647785464 0.040532491867244946 0.03577527125478327
-999. -999. -999.
-0.058062458673116 -0.08764922509099882 -0.11697036914487152
-0.14836133615864944 -0.17956528904564023 -0.21109198032585794
-0.23846429237248942 -0.2598536549218765 -0.27795672866320387
-0.2939939095159731 -0.30541031366330024 -0.307643559333884
-0.30078421139811795 -0.2841339526883441 -0.26485737397202497
-0.24287299694779327 -0.22379014890999907 -0.20121548204699846
-0.1746486732156772 -0.14585019344118372 -0.12070675757803526
-0.0997891159111037 -0.08229393660994214 -0.06779720501287469
-0.057213385470859794 -0.04875768191096844 -0.0402377347189964
-0.030169328367807245 -0.017560662894847895 -0.006968922654137132
0.0009773980274431048 0.007054306637034288 0.010472286514133042
0.010702384151997032 0.009231553701801242 0.007544033101056543
0.004639797857203645 -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999.
-999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999.
-999. -999. -999.
""".split()
data = numpy.array(data, dtype=numpy.float)
data = MV2.masked_less(data, -900)
d2 = cdms2.createAxis(data)
self.assertTrue(numpy.ma.allclose(data, d2[:]))
def testMaskingFunctions(self):
xouter = MV2.outerproduct(MV2.arange(5.), [1] * 10)
masked = MV2.masked_greater(xouter, 1)
self.assertTrue(MV2.allequal(masked.mask[2:], True))
self.assertTrue(MV2.allequal(masked.mask[:2], False))
masked = MV2.masked_greater_equal(xouter, 1)
self.assertTrue(MV2.allequal(masked.mask[1:], True))
self.assertTrue(MV2.allequal(masked.mask[:1], False))
masked = MV2.masked_less(xouter, 1)
self.assertTrue(MV2.allequal(masked.mask[:1], True))
self.assertTrue(MV2.allequal(masked.mask[1:], False))
masked = MV2.masked_less_equal(xouter, 1)
self.assertTrue(MV2.allequal(masked.mask[:2], True))
self.assertTrue(MV2.allequal(masked.mask[2:], False))
masked = MV2.masked_not_equal(xouter, 1)
self.assertTrue(MV2.allequal(masked.mask[1], False))
self.assertTrue(MV2.allequal(masked.mask[0], True))
self.assertTrue(MV2.allequal(masked.mask[2:], True))
masked = MV2.masked_equal(xouter, 1)
self.assertTrue(MV2.allequal(masked.mask[1], True))
self.assertTrue(MV2.allequal(masked.mask[0], False))
self.assertTrue(MV2.allequal(masked.mask[2:], False))
masked = MV2.masked_outside(xouter, 1, 3)
self.assertTrue(MV2.allequal(masked.mask[0:1], True))
self.assertTrue(MV2.allequal(masked.mask[1:4], False))
self.assertTrue(MV2.allequal(masked.mask[4:], True))
masked = MV2.masked_where(
MV2.logical_or(MV2.greater(xouter, 3), MV2.less(xouter, 2)),
xouter)
self.assertTrue(MV2.allequal(masked.mask[0:2], True))
self.assertTrue(MV2.allequal(masked.mask[2:4], False))
self.assertTrue(MV2.allequal(masked.mask[4:], True))
def bony_sorting_part1(w500, binedges):
A, B, C = w500.shape
dx = np.diff(binedges)[0]
# Compute composite:
OKwaps = nanarray(
(A, B, C, 2 + len(binedges))) # add 2 for the exceedances
xx = 0
for x in binedges:
xx += 1
w500_bin = MV.masked_less(w500, x)
OKwaps[..., xx] = MV.masked_greater_equal(w500_bin, x + dx)
# do the first wap bin:
OKwaps[..., 0] = MV.masked_greater_equal(w500, binedges[0])
# do the last wap bin:
OKwaps[..., -1] = MV.masked_less(w500, binedges[-1] + dx)
return OKwaps # [month,lat,lon,wapbin]
def map_SWkern_to_lon(Ksw, albcsmap):
"""
Map each location's clear-sky surface albedo to the correct albedo bin
"""
albcsmap = MV.masked_greater(albcsmap, 1.0)
albcsmap = MV.masked_less(albcsmap, 0.0)
from scipy.interpolate import interp1d
# Ksw is size 12,7,7,lats,3
# albcsmap is size A,lats,lons
albcs = np.arange(0.0, 1.5, 0.5)
A = albcsmap.shape[0]
TT = Ksw.shape[1]
PP = Ksw.shape[2]
lenlat = Ksw.shape[3]
lenlon = albcsmap.shape[2]
SWkernel_map = MV.zeros((A, TT, PP, lenlat, lenlon))
SWkernel_map = MV.masked_where(SWkernel_map == 0, SWkernel_map)
for M in range(A):
MM = M
while MM > 11:
MM = MM - 12
for LA in range(lenlat):
alon = albcsmap[M, LA, :]
# interp1d can't handle mask but it can deal with NaN (?)
try:
alon2 = MV.where(alon.mask, np.nan, alon)
except:
alon2 = alon
if np.ma.count(alon2) > 1: # at least 1 unmasked value
if len(np.where(Ksw[MM, :, :, LA, :] > 0)) == 0:
SWkernel_map[M, :, :, LA, :] = 0
else:
f = interp1d(albcs, Ksw[MM, :, :, LA, :], axis=2)
ynew = f(alon2.data)
ynew = MV.masked_where(alon2.mask, ynew)
SWkernel_map[M, :, :, LA, :] = ynew
else:
continue
return SWkernel_map
def mmeAveMsk1D(listFiles,
sw2d,
years,
inDir,
outDir,
outFile,
timeInt,
mme,
ToeType,
fullTS,
debug=True):
'''
The mmeAveMsk1D() function averages rhon or scalar density bined files with differing masks
It ouputs the MME and a percentage of non-masked bins
Created on Tue Nov 25 13:56:20 CET 2014
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- sw2d - dimension of fields to consider (1 or 2)
- years(t1,t2) - years for slice read
- inDir(str) - input directory where files are stored
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- FfllTS - 0/1: if 1, uses full time serie (ignores years(t1,t2))
- debug <optional> - boolean value
Notes:
-----
- EG 25 Nov 2014 - Initial function write
- EG 9 Dec 2014 - Add agreement on difference with init period - save as <var>Agree
- EG 04 Oct 2016 - Add 3D files support
TODO:
------
'''
# CDMS initialisation - netCDF compression
comp = 1
# 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
if t2 <= 0:
useLastYears = True
t2 = -t2
else:
useLastYears = False
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
# Find dimension
runN = len(listFiles)
try:
fi = cdm.open(inDir[0] + '/' + listFiles[0])
except:
print ' *** file not found ', inDir[0] + '/' + listFiles[0]
sys.exit(' Abort')
if sw2d == 1:
ptopd0 = fi['ptopdepth']
# Create variable handle
latN = ptopd0.shape[2]
basN = ptopd0.shape[1]
elif sw2d == 2:
ptopd0 = fi['ptopdepthxy']
# Create variable handle
lonN = ptopd0.shape[2]
latN = ptopd0.shape[1]
#timN = ptopd0.shape[0]
timN = t2 - t1
if fullTS:
print ' !!! Working on full Time Serie (fullTS = True)'
timN = ptopd0.shape[0]
t1 = 0
t2 = timN
t10 = t1
t20 = t2
# Get grid objects
axesList = ptopd0.getAxisList()
# Declare and open files for writing
if os.path.isfile(outDir + '/' + outFile):
os.remove(outDir + '/' + outFile)
outFile_f = cdm.open(outDir + '/' + outFile, 'w')
print ' Number of members:', len(listFiles)
valmask = ptopd0.missing_value
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
time.designateTime()
# loop on variables
# init percent array
if sw2d == 1:
varList = [
'ptopdepth', 'ptopsigma', 'ptopso', 'ptopthetao', 'volpers',
'salpers', 'tempers'
]
#varList = ['ptopdepth']
varDim = [1, 1, 1, 1, 0, 0, 0]
percent = npy.ma.ones([runN, timN, basN, latN], dtype='float32') * 0.
elif sw2d == 2:
varList = ['ptopdepthxy', 'ptopsigmaxy', 'ptopsoxy', 'ptopthetaoxy']
#varList = ['ptopdepthxy']
varDim = [2, 2, 2, 2]
percent = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * 0.
varFill = [
valmask, valmask, valmask, valmask, valmask, valmask, valmask, valmask,
valmask
]
axis1D = [time, axesList[1], axesList[2]]
axis0D = [time, axesList[1]]
print ' timN = ', timN
# loop on 1D variables
for iv, var in enumerate(varList):
ti0 = timc.clock()
# Array inits
if varDim[iv] == 2:
isonvar = npy.ma.ones([runN, timN, latN, lonN],
dtype='float32') * valmask
vardiff = npy.ma.ones([runN, timN, latN, lonN],
dtype='float32') * valmask
varones = npy.ma.ones([runN, timN, latN, lonN],
dtype='float32') * 1.
axisVar = axis1D
elif varDim[iv] == 1:
isonvar = npy.ma.ones([runN, timN, basN, latN],
dtype='float32') * valmask
vardiff = npy.ma.ones([runN, timN, basN, latN],
dtype='float32') * valmask
varones = npy.ma.ones([runN, timN, basN, latN],
dtype='float32') * 1.
axisVar = axis1D
else:
isonvar = npy.ma.ones([runN, timN, basN],
dtype='float32') * valmask
vardiff = npy.ma.ones([runN, timN, basN],
dtype='float32') * valmask
varones = npy.ma.ones([runN, timN, basN], dtype='float32') * 1.
axisVar = axis0D
print ' Variable ', iv, var, varDim[iv]
# loop over files to fill up array
for ic, file in enumerate(listFiles):
ft = cdm.open(inDir[0] + '/' + file)
timeax = ft.getAxis('time')
try:
tmax = timeax.shape[0]
except:
print ic, file, timeax
if ic == 0:
tmax0 = tmax
#print ic,file, tmax
#adapt [t1,t2] time bounds to piControl last NN years
if useLastYears:
t1 = tmax - t20
t2 = tmax
else:
if tmax != tmax0:
print 'tmax <> tmax0', tmax, tmax0
print 'wrong time axis: exiting...'
return
#print 'Time dims:',ic, t1,t2,tmax
# read array
computeVar = True
allVars = ft.variables.keys()
if 'ptopsigmaxy' in allVars:
computeVar = False
if (var == 'ptopsigmaxy') & computeVar:
#print ' ic = ',ic
# reconstruct from isondepthg and ptopdepthxy
isond = ft('isondepthg', time=slice(t1, t2))
#print isond.data.shape, timN*latN*lonN
itest = 94 * 360 + 150
axesList = isond.getAxisList()
levs = axesList[1][:]
levN = len(levs)
#ti02 = timc.clock()
levs3d0 = mv.reshape(npy.tile(levs, latN * lonN),
(latN * lonN, levN))
#ti05 = timc.clock()
isonRead = npy.ma.ones([timN, latN, lonN],
dtype='float32') * valmask
for it in range(timN): # loop on time to limit memory usage
levs3d = levs3d0 * 1.
depthlo = mv.reshape(vardepth[ic, it, ...], latN * lonN)
depth3d = npy.reshape(npy.repeat(depthlo, levN),
(latN * lonN, levN))
isond3d = mv.reshape(
npy.transpose(isond.data[it, ...], (1, 2, 0)),
(latN * lonN, levN))
#print isond3d[itest,:]
isond3d[isond3d > valmask / 10] = 0.
#print isond3d[itest,:]
isond3dp1 = npy.roll(isond3d, -1, axis=1)
isond3dp1[:, -1] = isond3d[:, -1]
#print isond3dp1[itest,:]
#levs3d[levs3d > 30. ] = 0. # to distinguish bottom masked points from surface masked points
#print levs3d[itest,:]
levs3d[(depth3d <= isond3d)] = 0.
#print levs3d[itest,:]
levs3d[(depth3d > isond3dp1)] = 0.
#print levs3d[itest,:]
#isonwrk = npy.sum(levs3d,axis=1)
isonwrk = npy.max(levs3d, axis=1)
if it < 0:
print ic, it
print depthlo[itest]
print isond3d[itest, :]
print isonwrk[itest]
print
isonRead[it, ...] = mv.reshape(isonwrk, (latN, lonN))
# <-- end of loop on time
del (isond3d, isond3dp1)
gc.collect()
# mask with depthxy and where sigmaxy = 0
isonRead.mask = vardepth.mask[ic, ...]
isonRead = mv.masked_where(isonRead == 0, isonRead)
isonRead.long_name = var
isonRead.units = 'sigma_n'
isonRead.id = var
del (isond, depth3d, levs3d, levs3d0, isonwrk)
gc.collect()
#ti3 = timc.clock()
#print ti02-ti0,ti05-ti02, ti1-ti05,ti12-ti1,ti15-ti12,ti2-ti15,ti3-ti2
#print ti3-ti0
# write ptopsigmaxy
if os.path.isfile(inDir[0] + '/work_ptopsigmaxy/' + file):
os.remove(inDir[0] + '/work_ptopsigmaxy/' + file)
fiout = cdm.open(inDir[0] + '/work_ptopsigmaxy/' + file, 'w')
if ic == 0:
print ' Creating ', inDir[0] + '/work_ptopsigmaxy/' + file
isonsigxy = cdm.createVariable(isonRead,
axes=axis1D,
id='ptopsigmaxy')
isonsigxy.long_name = 'Density of shallowest persistent ocean on ison'
isonsigxy.units = 'sigma_n'
fiout.write(isonsigxy.astype('float32'))
fiout.close()
else:
# Direct read of variable
isonRead = ft(var, time=slice(t1, t2))
#print isonRead.shape, timN
if varFill[iv] != valmask:
isonvar[ic, ...] = isonRead.filled(varFill[iv])
else:
isonvar[ic, ...] = isonRead
#print isonvar[ic,:,40,100]
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data, valmask).mask
percent[ic, ...] = npy.float32(npy.equal(maskvar, 0))
if mme:
# if mme then just average Bowl and Agree fields
varst = var + 'Agree'
vardiff[ic, ...] = ft(varst, time=slice(t1, t2))
else:
# Compute difference with average of first initN years, use mask of last month
varinit = cdu.averager(isonvar[ic, peri1:peri2, ...], axis=0)
for tr in range(timN):
vardiff[ic, tr, ...] = isonvar[ic, tr, ...] - varinit
vardiff[ic, ...].mask = isonvar[ic, ...].mask
ft.close()
# <-- end of loop on files
# TODO remove masked points at longitudes 0 or 180deg for some models
# if ptopdepthxy, keep for ptopsigmaxy computation (reconstruct from isondepthg and ptopdepthxy)
if var == 'ptopdepthxy':
vardepth = isonvar
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent, axis=0)) * 100.
percenta = mv.masked_less(percenta, 50)
percentw = cdm.createVariable(percenta,
axes=axis1D,
id='ptoppercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'))
# Sign of difference
if mme:
vardiffsgSum = cdu.averager(vardiff, axis=0)
vardiffsgSum = cdm.createVariable(vardiffsgSum,
axes=axisVar,
id='foo')
vardiffsgSum = maskVal(vardiffsgSum, valmask)
vardiffsgSum.mask = percentw.mask
else:
vardiffsg = npy.copysign(varones, vardiff)
# average signs
vardiffsgSum = cdu.averager(vardiffsg, axis=0)
vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
vardiffsgSum.mask = percentw.mask
vardiffsgSum._FillValue = valmask
# average accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = cdm.createVariable(isonVarAve, axes=axisVar, id='foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
# Write
isonave = cdm.createVariable(isonVarAve, axes=axisVar, id=isonRead.id)
isonave.long_name = isonRead.long_name
isonave.units = isonRead.units
isonavediff = cdm.createVariable(vardiffsgSum,
axes=axisVar,
id=isonRead.id + 'Agree')
isonavediff.long_name = isonRead.long_name
isonavediff.units = isonRead.units
outFile_f.write(isonave.astype('float32'))
outFile_f.write(isonavediff.astype('float32'))
tf = timc.clock()
#print ' time var',tf-ti0
# <--- end of loop on variables
outFile_f.close()
fi.close()
def mmeAveMsk3D(listFiles,
years,
inDir,
outDir,
outFile,
timeInt,
mme,
ToeType,
debug=True):
'''
The mmeAveMsk3D() function averages rhon/lat density bined files with differing masks
It ouputs
- the MME
- a percentage of non-masked bins
- the sign agreement of period2-period1 differences
- ToE per run and for MME
Author: Eric Guilyardi : [email protected]
Created on Tue Nov 21 2016
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- years(t1,t2) - years for slice read
- inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE)
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- ToeType(str) - ToE type ('F': none, 'histnat')
-> requires running first mm+mme without ToE to compute Stddev
- debug <optional> - boolean value
Notes:
-----
- EG 21 Nov 2016 - Initial function write
- TODO :
- add computation of ToE per model (toe 1 and toe 2) see ticket #50
- add isonhtc (see ticket #48)
'''
# CDMS initialisation - netCDF compression
comp = 1 # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
fi = cdm.open(inDir[0] + '/' + listFiles[0])
# Switch if only variables below the bowl are present/treated
nobowl = True
if nobowl:
isond0 = fi['isondepthgBowl']
# Create variable handle
else:
isond0 = fi['isondepthg']
# Create variable handle
# Get grid objects
axesList = isond0.getAxisList()
sigmaGrd = isond0.getLevel()
#time = isond0.getTime()
lonN = isond0.shape[3]
latN = isond0.shape[2]
levN = isond0.shape[1]
varsig = 'ptopsigmaxy'
# Limit number of models to 3 for testing of mme
#if mme:
# listFiles = listFiles[0:2]
# print ' !!! ### Testing 3 models ###', listFiles
# Declare and open files for writing
if os.path.isfile(outDir + '/' + outFile):
os.remove(outDir + '/' + outFile)
outFile_f = cdm.open(outDir + '/' + outFile, 'w')
#timN = isond0.shape[0]
timN = t2 - t1
runN = len(listFiles)
print ' Number of members:', len(listFiles)
valmask = isond0.missing_value
varList = ['isondepthg', 'persistmxy', 'sog', 'thetaog', 'isonthickg']
varFill = [valmask, valmask, valmask, valmask, valmask]
percent = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * 0.
varbowl = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * 1.
#varList = ['isondepthg']
#print ' !!! ### Testing one variable ###', varList
# init sigma axis
sigma = cdm.createAxis(npy.float32(range(1)))
sigma.id = axesList[1].id
sigma.units = axesList[1].units
sigma.designateTime()
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
# init ensemble axis
ensembleAxis = cdm.createAxis(npy.float32(range(runN)))
ensembleAxis.id = 'members'
ensembleAxis.units = 'N'
# Output axis
sigmaList = [sigma, axesList[2], axesList[3]]
# sigma, lat, lon
sigmaTimeList = [sigma, time, axesList[2], axesList[3]]
# sigma, time, lat, lon
# init arrays
isonvar = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * valmask
varbowl2D = npy.ma.ones([runN, timN, latN, lonN],
dtype='float32') * valmask
varstd, varToE1, varToE2 = [
npy.ma.ones([runN, latN, lonN], dtype='float32') * valmask
for _ in range(3)
]
# Loop on density levels (for memory management, becomes UNLIMITED axis and requires a ncpq to reorder dimensions)
delta_ib = 1
print ' Sigma index:'
for ib in range(levN):
ib1 = ib + delta_ib
print ib,
tim0 = timc.clock()
# loop on variables
for iv, var in enumerate(varList):
if nobowl:
varb = var + 'Bowl'
else:
varb = var
if ib == 0:
print ' Variable ', iv, varb
# loop over files to fill up array
for i, file in enumerate(listFiles):
tim01 = timc.clock()
ft = cdm.open(inDir[0] + '/' + file)
model = file.split('.')[1]
timeax = ft.getAxis('time')
if i == 0:
tmax0 = timeax.shape[0]
tmax = timeax.shape[0]
if tmax != tmax0:
print 'wrong time axis: exiting...'
return
# read array
isonRead = ft(varb, time=slice(t1, t2),
lev=slice(ib, ib1)).squeeze()
if varFill[iv] != valmask:
isonvar[i, ...] = isonRead.filled(varFill[iv])
else:
isonvar[i, ...] = isonRead
tim02 = timc.clock()
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data, valmask).mask
percent[i, ...] = npy.float32(npy.equal(maskvar, 0))
tim03 = timc.clock()
if mme:
# if mme then just accumulate Bowl, Agree and Std fields
#varst = var+'Agree'
#vardiff[i,...] = ft(varst,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze()
isonRead = ft(varb, time=slice(t1, t2),
lev=slice(ib, ib1)).squeeze()
varbowl2D[i, ...] = isonRead
else:
# Compute difference with average of first initN years
#varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0)
#for t in range(timN):
# vardiff[i,t,...] = isonvar[i,t,...] - varinit
#vardiff[i,...].mask = isonvar[i,...].mask
# Read bowl to truncate field above bowl
if ib == 0 and iv == 0:
varbowl[i, ...] = ft(varsig, time=slice(t1, t2))
#varbowl[i,...] = bowlRead
# Compute Stddev
varstd[i, ...] = npy.ma.std(isonvar[i, ...], axis=0)
# Compute ToE
if ToeType == 'histnat':
toto = 1
# TODO
# Read mean and Std dev from histnat
# if i == 0:
# filehn = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0]
# #filehn = replace(outFile,'historical','historicalNat')
# fthn = cdm.open(filehn)
# varmeanhn = fthn(var)
# varst = var+'Std'
# varmaxstd = fthn(varst)
# toemult = 1.
# signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN))
# noise = npy.reshape(varmaxstd,(basN*levN*latN))
# varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
# toemult = 2.
# varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
tim04 = timc.clock()
ft.close()
#print 'ib, section 1 timing',ib, tim02-tim01,tim03-tim02,tim04-tim03
# <-- end of loop on files (i)
tim1 = timc.clock()
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent, axis=0)) * 100.
percenta = mv.masked_less(percenta, 50)
percenta = npy.reshape(percenta, [delta_ib, timN, latN, lonN])
percentw = cdm.createVariable(percenta,
axes=sigmaTimeList,
id='isonpercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'), extend=1, index=ib)
# Sign of difference
#if mme:
# vardiffsgSum = cdu.averager(vardiff, axis=0)
# vardiffsgSum = cdm.createVariable(vardiffsgSum , axes = sigmaTimeList , id = 'foo')
# vardiffsgSum = maskVal(vardiffsgSum, valmask)
# vardiffsgSum.mask = percentw.mask
#else:
# vardiffsg = npy.copysign(varones,vardiff)
# # average signs
# vardiffsgSum = cdu.averager(vardiffsg, axis=0)
# vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
# vardiffsgSum.mask = percentw.mask
# vardiffsgSum._FillValue = valmask
# average variable accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = npy.reshape(isonVarAve, [delta_ib, timN, latN, lonN])
isonVarAve = cdm.createVariable(isonVarAve,
axes=sigmaTimeList,
id='foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
tim2 = timc.clock()
# Only keep points with rhon > bowl-delta_rho
delta_rho = 0.
# mme case
if mme: # start from average of <var>Agree
isonVarBowl = cdu.averager(varbowl2D, axis=0)
isonVarBowl = npy.reshape(isonVarBowl,
[delta_ib, timN, latN, lonN])
isonVarBowl = cdm.createVariable(isonVarBowl,
axes=sigmaTimeList,
id='foo')
isonVarBowl = maskVal(isonVarBowl, valmask)
isonVarBowl.mask = percentw.mask
# Compute intermodel stddev
isonVarStd = statistics.std(varbowl2D, axis=0)
isonVarStd = npy.reshape(isonVarStd,
[delta_ib, timN, latN, lonN])
isonVarStd = cdm.createVariable(isonVarStd,
axes=sigmaTimeList,
id='foo')
isonVarStd = maskVal(isonVarStd, valmask)
isonVarStd.mask = percentw.mask
# Write
isonvarbowlw = cdm.createVariable(isonVarBowl,
axes=sigmaTimeList,
id=isonRead.id)
isonvarbowlw.long_name = isonRead.long_name
isonvarbowlw.units = isonRead.units
isonvarstdw = cdm.createVariable(isonVarStd,
axes=sigmaTimeList,
id=isonRead.id + 'Std')
isonvarstdw.long_name = isonRead.long_name + ' intermodel std'
isonvarstdw.units = isonRead.units
outFile_f.write(isonvarbowlw.astype('float32'),
extend=1,
index=ib)
outFile_f.write(isonvarstdw.astype('float32'),
extend=1,
index=ib)
#if ib == 0 and iv == 0:
# # TODO review
# # Read multimodel sigma on bowl and average in time
# file1d = replace(outDir+'/'+outFile,'2D','1D')
# if os.path.isfile(file1d):
# f1d = cdm.open(file1d)
# else:
# print 'ERROR:',file1d,'missing (if mme, run 2D first)'
# sys.exit(1)
# bowlRead = f1d(varsig,time = slice(t1,t2),lev = slice(ib,ib1))
# f1d.close()
# siglimit = cdu.averager(bowlRead, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
#if sw2d == 1:
# for il in range(latN):
# for ib in range(basN):
# #if ib == 2:
# # print il, siglimit[ib,il]
# if siglimit[ib,il] < valmask/1000.:
# # if mme bowl density defined, mask above bowl
# index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
# isonVarBowl [:,ib,0:index[0],il].mask = True
# isonVarStd [:,ib,0:index[0],il].mask = True
# vardiffsgSum[:,ib,0:index[0],il].mask = True
# else:
# # mask all points
# isonVarBowl [:,ib,:,il].mask = True
# isonVarStd [:,ib,:,il].mask = True
# vardiffsgSum[:,ib,:,il].mask = True
# mm case
else:
isonVarBowl = isonVarAve * 1. # start from variable
#isonVarStd = isonVarAve*1. # start from variable
if ib == 0 and iv == 0:
# build bowl position
siglimit = cdu.averager(varbowl,
axis=0) # average accross members
siglimit = npy.reshape(siglimit,
[timN * latN * lonN]) - delta_rho
if iv == 0:
sigarr = siglimit * 1.
sigarr[:] = sigmaGrd[ib]
# test
i = 60
j = 60
ij = j * lonN + i
isonVarBowl = npy.reshape(isonVarBowl, [timN * latN * lonN])
#vardiffsgSum = npy.reshape(vardiffsgSum,[timN*latN*lonN])
isonVarBowl.mask = npy.where(sigarr < siglimit, True,
isonVarBowl.mask)
#vardiffsgSum.mask = npy.where(sigarr < siglimit, True, vardiffsgSum.mask)
isonVarBowl = npy.reshape(isonVarBowl, [timN, latN, lonN])
#vardiffsgSum = npy.reshape(vardiffsgSum,[timN,latN,lonN])
isonVarBowl = maskVal(isonVarBowl, valmask)
#vardiffsgSum = maskVal(vardiffsgSum, valmask)
# Find max of Std dev of all members
isonVarStd = npy.ma.max(varstd, axis=0)
# mask
isonVarStd = maskVal(isonVarStd, valmask)
# Write
#isonave = cdm.createVariable(isonVarAve, axes = sigmaTimeList, id = isonRead.id)
#isonave.long_name = isonRead.long_name
#isonave.units = isonRead.units
#vardiffsgSum = npy.reshape(vardiffsgSum,[delta_ib,timN,latN,lonN])
#isonavediff = cdm.createVariable(vardiffsgSum, axes = sigmaTimeList, id = isonRead.id+'Agree')
#isonavediff.long_name = isonRead.long_name
#isonavediff.units = isonRead.units
isonVarBowl = npy.reshape(isonVarBowl,
[delta_ib, timN, latN, lonN])
isonavebowl = cdm.createVariable(isonVarBowl,
axes=sigmaTimeList,
id=isonRead.id + 'Bowl')
isonavebowl.long_name = isonRead.long_name
isonavebowl.units = isonRead.units
isonVarStd = npy.reshape(isonVarStd, [delta_ib, latN, lonN])
isonmaxstd = cdm.createVariable(isonVarStd,
axes=sigmaList,
id=isonRead.id + 'Std')
isonmaxstd.long_name = isonRead.long_name
isonmaxstd.units = isonRead.units
#outFile_f.write( isonave.astype('float32'), extend = 1, index = ib)
#outFile_f.write(isonavediff.astype('float32'), extend = 1, index = ib)
outFile_f.write(isonavebowl.astype('float32'),
extend=1,
index=ib)
outFile_f.write(isonmaxstd.astype('float32'),
extend=1,
index=ib)
tim3 = timc.clock()
if ToeType == 'histnat':
isontoe1 = cdm.createVariable(
varToE1,
axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'ToE1')
isontoe1.long_name = 'ToE 1 for ' + isonRead.long_name
isontoe1.units = 'Year'
isontoe2 = cdm.createVariable(
varToE2,
axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'ToE2')
isontoe2.long_name = 'ToE 2 for ' + isonRead.long_name
isontoe2.units = 'Year'
outFile_f.write(isontoe1.astype('float32'), extend=1, index=ib)
outFile_f.write(isontoe2.astype('float32'), extend=1, index=ib)
tim4 = timc.clock()
# <--- end of loop on variables
#print 'ib, timing',ib, tim01-tim0,tim1-tim01,tim2-tim1,tim3-tim2,tim4-tim3
# <--- end of loop on density
print ' '
outFile_f.close()
fi.close()
def mmeAveMsk2D(listFiles,
years,
inDir,
outDir,
outFile,
timeInt,
mme,
timeBowl,
ToeType,
debug=True):
'''
The mmeAveMsk2D() function averages rhon/lat density bined files with differing masks
It ouputs
- the MME
- a percentage of non-masked bins
- the sign agreement of period2-period1 differences
- ToE per run and for MME
Author: Eric Guilyardi : [email protected]
Created on Tue Nov 25 13:56:20 CET 2014
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- years(t1,t2) - years for slice read
- inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE)
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- timeBowl - either time 'mean' or time 'max' bowl used to mask out bowl
- ToeType(str) - ToE type ('F': none, 'histnat')
-> requires running first mm+mme without ToE to compute Stddev
- debug <optional> - boolean value
Notes:
-----
- EG 25 Nov 2014 - Initial function write
- EG 27 Nov 2014 - Rewrite with loop on variables
- EG 06 Dec 2014 - Added agreement on difference with init period - save as <var>Agree
- EG 07 Dec 2014 - Read bowl to remove points above bowl - save as <var>Bowl
- EG 19 Apr 2016 - ToE computation (just for 2D files)
- EG 07 Oct 2016 - add 3D file support
- EG 21 Nov 2016 - move 3D support to new function
- EG 10 jan 2017 - added timeBowl option
- TODO :
- remove loops
- add computation of ToE per model (toe 1 and toe 2) see ticket #50
- add isonhtc (see ticket #48)
'''
# CDMS initialisation - netCDF compression
comp = 1 # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
if t2 <= 0:
useLastYears = True
t2 = -t2
else:
useLastYears = False
t10 = t1
t20 = t2
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
fi = cdm.open(inDir[0] + '/' + listFiles[0])
isond0 = fi['isondepth']
# Create variable handle
# Get grid objects
axesList = isond0.getAxisList()
sigmaGrd = isond0.getLevel()
latN = isond0.shape[3]
levN = isond0.shape[2]
basN = isond0.shape[1]
varsig = 'ptopsigma'
# Declare and open files for writing
if os.path.isfile(outDir + '/' + outFile):
os.remove(outDir + '/' + outFile)
outFile_f = cdm.open(outDir + '/' + outFile, 'w')
# Testing mme with less models
#listFiles=listFiles[0:4]
#timN = isond0.shape[0]
timN = t2 - t1
runN = len(listFiles)
print ' Number of members:', len(listFiles)
valmask = isond0.missing_value[0]
varList = [
'isondepth', 'isonpers', 'isonso', 'isonthetao', 'isonthick', 'isonvol'
]
varFill = [0., 0., valmask, valmask, 0., 0.]
# init arrays (2D rho/lat)
percent = npy.ma.ones([runN, timN, basN, levN, latN], dtype='float32') * 0.
#minbowl = npy.ma.ones([basN,latN], dtype='float32')*1000.
varbowl = npy.ma.ones([runN, timN, basN, latN], dtype='float32') * 1.
#varList = ['isondepth']
#print ' !!! ### Testing one variable ###'
#varList = ['isonthetao']
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
time.designateTime()
# init ensemble axis
ensembleAxis = cdm.createAxis(npy.float32(range(runN)))
ensembleAxis.id = 'members'
ensembleAxis.units = 'N'
# loop on variables
for iv, var in enumerate(varList):
# Array inits (2D rho/lat 3D rho/lat/lon)
#shapeR = [basN,levN,latN]
isonvar = npy.ma.ones([runN, timN, basN, levN, latN],
dtype='float32') * valmask
print('isonvar shape: ', isonvar.shape)
vardiff, varbowl2D = [
npy.ma.ones([runN, timN, basN, levN, latN], dtype='float32')
for _ in range(2)
]
varstd, varToE1, varToE2 = [
npy.ma.ones([runN, basN, levN, latN], dtype='float32') * valmask
for _ in range(3)
]
varones = npy.ma.ones([runN, timN, basN, levN, latN],
dtype='float32') * 1.
print ' Variable ', iv, var
# loop over files to fill up array
for i, file in enumerate(listFiles):
ft = cdm.open(inDir[0] + '/' + file)
model = file.split('.')[1]
timeax = ft.getAxis('time')
file1d = replace(inDir[0] + '/' + file, '2D', '1D')
if os.path.isfile(file1d):
f1d = cdm.open(file1d)
else:
print 'ERROR:', file1d, 'missing (if mme, run 1D first)'
sys.exit(1)
tmax = timeax.shape[0]
if i == 0:
tmax0 = tmax
#adapt [t1,t2] time bounds to piControl last NN years
if useLastYears:
t1 = tmax - t20
t2 = tmax
else:
if tmax != tmax0:
print 'wrong time axis: exiting...'
return
# read array
# loop over time/density for memory management
for it in range(timN):
t1r = t1 + it
t2r = t1r + 1
isonRead = ft(var, time=slice(t1r, t2r))
if varFill[iv] != valmask:
isonvar[i, it, ...] = isonRead.filled(varFill[iv])
else:
isonvar[i, it, ...] = isonRead
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data, valmask).mask
percent[i, ...] = npy.float32(npy.equal(maskvar, 0))
if mme:
# if mme then just accumulate Bowl, Agree fields
varst = var + 'Agree'
vardiff[i, ...] = ft(varst, time=slice(t1, t2))
varb = var + 'Bowl'
varbowl2D[i, ...] = ft(varb, time=slice(t1, t2))
else:
# Compute difference with average of first initN years
varinit = cdu.averager(isonvar[i, peri1:peri2, ...], axis=0)
for t in range(timN):
vardiff[i, t, ...] = isonvar[i, t, ...] - varinit
vardiff[i, ...].mask = isonvar[i, ...].mask
# Read bowl and truncate 2D field above bowl
if iv == 0:
bowlRead = f1d(varsig, time=slice(t1, t2))
varbowl[i, ...] = bowlRead
# Compute Stddev
varstd[i, ...] = npy.ma.std(isonvar[i, ...], axis=0)
# Compute ToE
if ToeType == 'histnat':
# Read mean and Std dev from histnat
if i == 0:
filehn = glob.glob(inDir[1] + '/cmip5.' + model +
'.*zon2D*')[0]
#filehn = replace(outFile,'historical','historicalNat')
fthn = cdm.open(filehn)
varmeanhn = fthn(var)
varst = var + 'Std'
varmaxstd = fthn(varst)
toemult = 1.
signal = npy.reshape(isonvar[i, ...] - varmeanhn,
(timN, basN * levN * latN))
noise = npy.reshape(varmaxstd, (basN * levN * latN))
varToE1[i,
...] = npy.reshape(findToE(signal, noise, toemult),
(basN, levN, latN))
toemult = 2.
varToE2[i,
...] = npy.reshape(findToE(signal, noise, toemult),
(basN, levN, latN))
ft.close()
f1d.close()
# <-- end of loop on files
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent, axis=0)) * 100.
percenta = mv.masked_less(percenta, 50)
percentw = cdm.createVariable(
percenta,
axes=[time, axesList[1], axesList[2], axesList[3]],
id='isonpercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'))
# Sign of difference
if mme:
vardiffsgSum = cdu.averager(vardiff, axis=0)
vardiffsgSum = cdm.createVariable(
vardiffsgSum,
axes=[time, axesList[1], axesList[2], axesList[3]],
id='foo')
vardiffsgSum = maskVal(vardiffsgSum, valmask)
vardiffsgSum.mask = percentw.mask
else:
vardiffsg = npy.copysign(varones, vardiff)
# average signs
vardiffsgSum = cdu.averager(vardiffsg, axis=0)
vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
vardiffsgSum.mask = percentw.mask
vardiffsgSum._FillValue = valmask
# average variable accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = cdm.createVariable(
isonVarAve,
axes=[time, axesList[1], axesList[2], axesList[3]],
id='foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
# Only keep points with rhon > bowl-delta_rho
delta_rho = 0.
if mme: # start from average of <var>Agree
isonVarBowl = cdu.averager(varbowl2D, axis=0)
isonVarBowl = cdm.createVariable(
isonVarBowl,
axes=[time, axesList[1], axesList[2], axesList[3]],
id='foo')
isonVarBowl = maskVal(isonVarBowl, valmask)
isonVarBowl.mask = percentw.mask
# Compute intermodel stddev
isonVarStd = statistics.std(varbowl2D, axis=0)
isonVarStd = cdm.createVariable(
isonVarStd,
axes=[time, axesList[1], axesList[2], axesList[3]],
id='foo')
isonVarStd = maskVal(isonVarStd, valmask)
isonVarStd.mask = percentw.mask
if iv == 0:
# Read mulitmodel sigma on bowl and average in time
file1d = replace(outDir + '/' + outFile, '2D', '1D')
if os.path.isfile(file1d):
f1d = cdm.open(file1d)
else:
print 'ERROR:', file1d, 'missing (if mme, run 1D first)'
sys.exit(1)
bowlRead = f1d(varsig, time=slice(t1, t2))
f1d.close()
siglimit = cdu.averager(bowlRead, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
for il in range(latN):
for ib in range(basN):
#if ib == 2:
# print il, siglimit[ib,il]
if siglimit[ib, il] < valmask / 1000.:
# if mme bowl density defined, mask above bowl
index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib, il]))
isonVarBowl[:, ib, 0:index[0], il].mask = True
isonVarStd[:, ib, 0:index[0], il].mask = True
vardiffsgSum[:, ib, 0:index[0], il].mask = True
else:
# mask all points
isonVarBowl[:, ib, :, il].mask = True
isonVarStd[:, ib, :, il].mask = True
vardiffsgSum[:, ib, :, il].mask = True
else:
isonVarBowl = isonVarAve * 1. # start from variable
isonVarStd = isonVarAve * 1. # start from variable
if iv == 0:
siglimit = cdu.averager(varbowl,
axis=0) # average accross members
# Average bowl in time
if timeBowl == 'mean':
siglimit = cdu.averager(siglimit, axis=0) - delta_rho
# or take largest sigma over time
else:
siglimit = npy.ma.max(siglimit, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
for il in range(latN):
for ib in range(basN):
if siglimit[ib, il] < valmask / 1000.:
# if bowl density defined, mask above bowl
index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib, il])
)[:, 0] #Add [:,0] for python Yona
#import code
#code.interact(banner='index', local=dict(locals(), **globals()))
isonVarBowl[:, ib, 0:index[0], il].mask = True
vardiffsgSum[:, ib, 0:index[0], il].mask = True
else:
# mask all points
vardiffsgSum[:, ib, :, il].mask = True
isonVarBowl = maskVal(isonVarBowl, valmask)
# Find max of Std dev of all members
isonVarStd = npy.ma.max(varstd, axis=0)
# mask
if varFill[iv] == valmask:
isonVarStd = maskVal(isonVarStd, valmask)
# Write
isonave = cdm.createVariable(
isonVarAve,
axes=[time, axesList[1], axesList[2], axesList[3]],
id=isonRead.id)
isonave.long_name = isonRead.long_name
isonave.units = isonRead.units
isonavediff = cdm.createVariable(
vardiffsgSum,
axes=[time, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'Agree')
isonavediff.long_name = isonRead.long_name
isonavediff.units = isonRead.units
isonavebowl = cdm.createVariable(
isonVarBowl,
axes=[time, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'Bowl')
isonavebowl.long_name = isonRead.long_name
isonavebowl.units = isonRead.units
if not mme:
isonmaxstd = cdm.createVariable(
isonVarStd,
axes=[axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'Std')
isonmaxstd.long_name = isonRead.long_name
isonmaxstd.units = isonRead.units
outFile_f.write(isonave.astype('float32'))
outFile_f.write(isonavediff.astype('float32'))
outFile_f.write(isonavebowl.astype('float32'))
if not mme:
outFile_f.write(isonmaxstd.astype('float32'))
if ToeType == 'histnat':
isontoe1 = cdm.createVariable(
varToE1,
axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'ToE1')
isontoe1.long_name = 'ToE 1 for ' + isonRead.long_name
isontoe1.units = 'Year'
isontoe2 = cdm.createVariable(
varToE2,
axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'ToE2')
isontoe2.long_name = 'ToE 2 for ' + isonRead.long_name
isontoe2.units = 'Year'
outFile_f.write(isontoe1.astype('float32'))
outFile_f.write(isontoe2.astype('float32'))
if mme:
isonvarstd = cdm.createVariable(
isonVarStd,
axes=[time, axesList[1], axesList[2], axesList[3]],
id=isonRead.id + 'ModStd')
isonvarstd.long_name = isonRead.long_name + ' intermodel std'
isonvarstd.units = isonRead.units
outFile_f.write(isonvarstd.astype('float32'))
# <--- end of loop on variables
outFile_f.close()
fi.close()
def basicGm(self,
gm_type,
projtype="default",
lat1=0,
lat2=0,
lon1=0,
lon2=0,
rg=False,
flip=False,
zero=False,
transparent=False,
mask=False,
bigvalues=False):
self.x.clear()
self.x.setcolormap(None)
cdms2.tvariable.TransientVariable.variable_count = 1
loc = locals()
exec("gm=vcs.create%s()" % gm_type)
gm = loc["gm"]
if projtype != "default":
p = vcs.createprojection()
try:
ptype = int(projtype)
except BaseException:
ptype = projtype
p.type = ptype
gm.projection = p
nm_xtra = ""
xtra = {}
if lat1 != lat2:
if rg:
if flip:
gm.datawc_y1 = lat2
gm.datawc_y2 = lat1
nm_xtra += "_gmflip"
else:
gm.datawc_y1 = lat1
gm.datawc_y2 = lat2
xtra["latitude"] = (lat1, lat2)
if lat1 < 0:
nm_xtra += "_SH"
else:
nm_xtra += "_NH"
if lon1 != lon2:
if rg:
gm.datawc_x1 = lon1
gm.datawc_x2 = lon2
xtra["longitude"] = (lon1, lon2)
nm_xtra += "_%i_%i" % (lon1, lon2)
if rg:
nm_xtra += "_via_gm"
if gm_type == "meshfill":
f = cdms2.open(os.path.join(vcs.sample_data,
'sampleCurveGrid4.nc'))
else:
f = self.clt
if gm_type == "vector":
u = f("u", **xtra)
v = f("v", **xtra)
if mask:
u = MV2.masked_greater(u, 58.)
if zero:
u -= u
v -= v
elif gm_type == "meshfill":
s = f("sample", **xtra)
if mask:
s = MV2.masked_less(s, 1150.)
elif bigvalues:
s[s < 1150] = 1e40
if zero:
s -= s
else:
s = f("clt", **xtra)
if mask:
s = MV2.masked_greater(s, 78.)
elif bigvalues:
s[s > 78] = 1e40
if gm_type in ["1d", "yxvsx", "xyvsy", "xvsy", "scatter"]:
s = s(latitude=(20, 20, "cob"),
longitude=(112, 112, "cob"),
squeeze=1)
s2 = MV2.sin(s)
if zero:
s2 -= s2
if zero:
s -= s
if bigvalues:
gm.levels = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 1.e36]
if transparent:
cmap = self.x.createcolormap()
for i in range(256): # tweaks all colors
cmap.setcolorcell(i, 100., 0, 0, i / 2.55)
self.x.setcolormap(cmap)
if gm_type == "vector":
gm.linecolor = [100, 0, 0, 50.]
elif gm_type in ["yxvsx", "xyvsy", "yvsx", "scatter", "1d"]:
gm.linecolor = [100, 0, 0, 50.]
gm.markercolor = [100, 0, 0, 50.]
if gm_type == "vector":
gm.scale = 4.
self.x.plot(u, v, gm, bg=self.bg)
elif gm_type in ["scatter", "xvsy"]:
self.x.plot(s, s2, gm, bg=self.bg)
else:
self.x.plot(s, gm, bg=self.bg)
fnm = "test_vcs_basic_%s" % gm_type.lower()
if mask:
fnm += "_masked"
elif bigvalues:
fnm += "_bigvalues"
if projtype != "default":
fnm += "_%s_proj" % projtype
if zero:
fnm += "_zero"
if transparent:
fnm += "_transparent"
fnm += nm_xtra
self.checkImage(fnm + '.png', threshold=20)
-0.14836133615864944 -0.17956528904564023 -0.21109198032585794 -0.23846429237248942 -0.2598536549218765 -0.27795672866320387 -0.2939939095159731 -0.30541031366330024 -0.307643559333884 -0.30078421139811795 -0.2841339526883441 -0.26485737397202497 -0.24287299694779327 -0.22379014890999907 -0.20121548204699846 -0.1746486732156772 -0.14585019344118372 -0.12070675757803526 -0.0997891159111037 -0.08229393660994214 -0.06779720501287469 -0.057213385470859794 -0.04875768191096844 -0.0402377347189964 -0.030169328367807245 -0.017560662894847895 -0.006968922654137132 0.0009773980274431048 0.007054306637034288 0.010472286514133042 0.010702384151997032 0.009231553701801242 0.007544033101056543 0.004639797857203645 -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. """.split() data = numpy.array(data,dtype=numpy.float) data = MV2.masked_less(data,-900) #yx.datawc_x1 = 0 #yx.datawc_x2 = 80 ##yx.datawc_y1 =-12 #yx.datawc_y2 = 12 x.plot(data,yx,bg=1) fnm = "test_vcs_1D_datawc_missing.png" x.png(fnm) print "fnm:",fnm print "src:",src ret = checkimage.check_result_image(fnm,src,0.05) sys.exit(ret)
import sys, os
import vcs
import sys
import cdms2
import vtk
import os
import MV2
bg = not False
x = vcs.init()
x.setcolormap("rainbow")
gm = vcs.createmeshfill()
p = vcs.createprojection()
ptype = int('0')
p.type = ptype
gm.projection = p
xtra = {}
f = cdms2.open(os.path.join(vcs.prefix, 'sample_data', 'sampleCurveGrid4.nc'))
s = f("sample", **xtra)
gm.mesh = True
s = MV2.masked_less(s, 1150.)
x.plot(s, gm, bg=bg)
x.png('test_vcs_basic_meshfill_masked_0_proj.png')
'''
import os, sys
sys.path.append(os.path.abspath('../../'))
import cdms2, MV2, vcs
from region_utils import getAreaOfAllClosedDomains
f = cdms2.open('data/snc.nc')
data = f('snc', time=slice(1), squeeze=1)
print "Contions \n 1. Mask less than 100 \n 2. Mask less than 30 and greater than 70"
con = input('Enter Condion Option 1 or 2 : ')
if con == 1:
mask_condition = MV2.masked_less(data, 100.)
value = 'value equal to 100'
dirname = 'outplots_eq_100'
elif con == 2:
c1 = MV2.masked_greater(data, 70.)
c2 = MV2.masked_less(data, 30.)
mask_condition = MV2.logical_and(c1, c2)
value = 'value equal to within 30 to 70'
dirname = 'outplots_eq_30_to_70'
# end of if con == 1:
if not os.path.isdir(dirname): os.mkdir(dirname)
print "Output plots will be saved in '%s' directory" % dirname
v = vcs.init()
# Assign the variable "cf_asd" to the persistent 'ASD' isofill graphics methods.
import vcs
import cdms2
import sys
import os
import MV2
f = cdms2.open(os.path.join(vcs.prefix, "sample_data", "clt.nc"))
s = f("clt", time=slice(0, 1), squeeze=1)
s = MV2.masked_less(s, 65.)
x = vcs.init()
gm = x.createisofill()
#gm=x.createboxfill()
gm.missing = 252
#gm.levels = [65,70,75,80,85,90,95,100]
x.plot(s, gm)
x.png(gm.g_name)
x.interact()
def obscuration_terms3(c1, c2):
"""
USE THIS VERSION FOR DIFFERENCES OF 2 CLIMATOLOGIES (E.G. AMIP4K, 2xCO2 SLAB RUNS)
Compute the components required for the obscuration-affected low cloud feedback
These are the terms shown in Eq 4 of Scott et al (2020) DOI: 10.1175/JCLI-D-19-1028.1
L_prime = dunobsc + dobsc + dobsc_cov, where
dunobsc = L_R_prime * F_bar (delta unobscured low clouds, i.e., true low cloud feedback)
dobsc = L_R_bar * F_prime (delta obscuration by upper level clouds)
dobsc_cov = (L_R_prime * F_prime) - climo(L_R_prime * F_prime) (covariance term)
"""
# c is [mo,tau,ctp,lat,lon]
# c is in percent
AX = c2.getAxisList()
c1 = MV.masked_where(c2.mask, c1)
c2 = MV.masked_where(c1.mask, c2)
# SPLICE c1 and c2:
# MAKE SURE c1 and c2 are the same size!!!
if c1.shape != c2.shape:
raise RuntimeError('c1 and c2 are NOT the same size!!!')
c12 = np.ma.append(c1, c2, axis=0)
midpt = len(c1)
U12 = MV.sum(MV.sum(c12[:, :, 2:, :], 1), 1) / 100.
L12 = c12[:, :, :2, :] / 100.
F12 = 1. - U12
F12 = MV.masked_less(F12, 0)
F12b = MV.array(np.expand_dims(np.expand_dims(F12, axis=1), axis=1))
F12b = MV.masked_where(L12[:, :1, :1, :].mask, F12b)
L_R12 = L12 / F12b
sum_L_R12 = MV.sum(MV.sum(L_R12, 1), 1)
sum_L_R12b = MV.array(
np.expand_dims(np.expand_dims(sum_L_R12, axis=1), axis=1))
sum_L_R12c = np.broadcast_to(sum_L_R12b, L_R12.shape)
this = MV.masked_outside(sum_L_R12c, 0, 1)
L_R12 = MV.masked_where(this.mask, L_R12)
L_R12 = MV.masked_where(sum_L_R12c > 1, L_R12)
L_R_prime, L_R_bar = monthly_anomalies(L_R12)
F_prime, F_bar = monthly_anomalies(F12b)
L_prime, L_bar = monthly_anomalies(L12)
# Cannot have negative cloud fractions:
L_R_bar[L_R_bar < 0] = 0
F_bar[F_bar < 0] = 0
rep_L_bar = tile_uneven(L_bar, L12)
rep_L_R_bar = tile_uneven(L_R_bar, L_R12)
rep_F_bar = tile_uneven(F_bar, F12b)
# Cannot have negative cloud fractions:
L_R_bar[L_R_bar < 0] = 0
F_bar[F_bar < 0] = 0
dobsc = rep_L_R_bar * F_prime
dunobsc = L_R_prime * rep_F_bar
prime_prime = (L_R_prime * F_prime)
dobsc_cov, climo_prime_prime = monthly_anomalies(prime_prime)
# Re-scale these anomalies by 2, since we have computed all anomalies w.r.t.
# the ctl+pert average rather than w.r.t. the ctl average
dobsc *= 2
dunobsc *= 2
dobsc_cov *= 2
return (rep_L_R_bar[midpt:], dobsc[midpt:], dunobsc[midpt:],
dobsc_cov[midpt:])
xouter.setAxisList([lat,lon]) # Equivalent ## masked_equal(x, value) ## masked_equal(x, value) = x masked where x == value ## For floating point consider masked_values(x, value) instead. ## masked_greater(x, value) ## masked_greater(x, value) = x masked where x > value ## masked_greater_equal(x, value) ## masked_greater_equal(x, value) = x masked where x >= value xge = MV2.masked_greater_equal(xouter, 120) ## masked_less(x, value) ## masked_less(x, value) = x masked where x < value xl = MV2.masked_less(xouter, 160) ## masked_less_equal(x, value) ## masked_less_equal(x, value) = x masked where x <= value ## masked_not_equal(x, value) ## masked_not_equal(x, value) = x masked where x != value ## masked_outside(x, v1, v2) ## x with mask of all values of x that are outside [v1,v2] xmo = MV2.masked_outside(xouter, 120, 160) ## count(a, axis=None) ## Count of the non-masked elements in a, or along a certain axis. xcount = MV2.count(xmo,0) xcount2 = MV2.count(xmo,1)
print 'Sum of all PFT is', pft_sum2010.shape
#x.plot(pft_sum2010[::-1,:],gm,bg=bg)
#x.png('pft_sum2010.png')
#x.clear()
#In case if sum of all PFTs is greater than 1, reduce all PFTs
re_pft2010 = MV2.masked_greater(pft_sum2010 - lndf_hurt, 0.).mask
for npft in range(19):
#print 'npft',npft
pft_tmp1 = pft2010[npft, :, :]
pft_tmp1 = MV2.choose(re_pft2010,
(pft_tmp1, pft_tmp1 * lndf_hurt / pft_sum2010))
pft2010[npft, :, :] = pft_tmp1
#In case if sum of all PFTs is less than 1, increase all PFTs
inc_pft2010 = MV2.masked_less(pft_sum2010 - lndf_hurt, 0.).mask
for npft in range(19):
#print 'npft',npft
pft_tmp2 = pft2010[npft, :, :]
pft_tmp2 = MV2.choose(inc_pft2010,
(pft_tmp2, pft_tmp2 * lndf_hurt / pft_sum2010))
pft2010[npft, :, :] = pft_tmp2
newpftsum = MV2.sum(pft2010, axis=0) + icwtr
sys.stdout.flush()
pft_back[[nyears - 1], :, :, :] = pft2010
#x.plot(newpftsum[::-1,:],gm,bg=bg)
#x.png('newpftsum.png')
#x.clear()
#if (pft_sum2010.all()==1):
# print 'Sum of all PFTs is equal to 1'
lon=cdms2.createAxis(MV2.arange(100)*3.6) lon.designateLongitude() lon.units="degrees_east" lon.id="longitude" lat = cdms2.createAxis(MV2.arange(100)*1.8-90.) lat.id="latitude" lat.designateLatitude() lat.units="degrees_north" lev = cdms2.createAxis([1000.]) lev.id="plev" lev.designateLevel() lev.units="hPa" t=cdms2.createAxis([0,31.]) t.id="time" t.designateTime() t.units="days since 2014" cdutil.setTimeBoundsMonthly(t) a.setAxisList((t,lev,lat,lon)) a=MV2.masked_less(a,.5) grd=cdms2.createGaussianGrid(64) a=a.ascontiguous() a=a.regrid(grd,regridTool="regrid2") a=cdutil.averager(a,axis='txy') assert a[0]==0.7921019540305255
xouter.setAxisList([lat, lon]) # Equivalent ## masked_equal(x, value) ## masked_equal(x, value) = x masked where x == value ## For floating point consider masked_values(x, value) instead. ## masked_greater(x, value) ## masked_greater(x, value) = x masked where x > value ## masked_greater_equal(x, value) ## masked_greater_equal(x, value) = x masked where x >= value xge = MV2.masked_greater_equal(xouter, 120) ## masked_less(x, value) ## masked_less(x, value) = x masked where x < value xl = MV2.masked_less(xouter, 160) ## masked_less_equal(x, value) ## masked_less_equal(x, value) = x masked where x <= value ## masked_not_equal(x, value) ## masked_not_equal(x, value) = x masked where x != value ## masked_outside(x, v1, v2) ## x with mask of all values of x that are outside [v1,v2] xmo = MV2.masked_outside(xouter, 120, 160) ## count(a, axis=None) ## Count of the non-masked elements in a, or along a certain axis. xcount = MV2.count(xmo, 0) xcount2 = MV2.count(xmo, 1)
select = dict(lat=slice(4, -4), lon=slice(4, -4), squeeze=1)
slp = f('pslvl', **select)
rain = f('rain', **select)
u = f('u10m', **select)
v = f('v10m', **select)
f.close()
# Pression de surface
slp[:] = generic2d(slp*0.01, 5)
map2(slp, fill=False, contour=True, show=False, figsize=(6, 5.5),
title='WRF Bretagne',
fillcontinents=True, zorder=5, projection='merc', right=1, bottom=.07,
lowhighs=True, lowhighs_smooth=9, lowhighs_zorder=5, fillcontinents_color='.95',
lowhighs_color=(0, .5, 0), contour_colors=[(0, .5, 0)], drawcoastlines_linewidth=.6)
# Pluie
cmap_rain = cmap_custom([('0.9', 0), ('b', .8), ('r', 1.)])
rain[:] = MV2.masked_less(rain, 0.1)
map2(rain, cmap=cmap_rain, vmin=0.,
fill='pcolor', fillcontinents=False, show=False,
shadow_xoffset=4, shadow_yoffset=-4,shadow_width=4, colorbar_shrink=.7,
alpha=.7, shadow=True, shadow_alpha=.3, contour=False, zorder=15)
# Vent
u[:] = ms2kt(u*5)
v[:] = ms2kt(v*5)
m = map2((u[::3, ::3], v[::3, ::3]), fill=False, contour=False, barbs=True, projection='merc',
quiver_sizes=dict(height=.2, spacing=.15), quiver_linewidths=.8, zorder=10,
shadow=True, quiver_alpha=.5, savefigs=__file__, show=False, savefigs_pdf=True,
fillcontinents=False)
def mmeAveMsk2D(listFiles, years, inDir, outDir, outFile, timeInt, mme, timeBowl, ToeType, debug=True):
'''
The mmeAveMsk2D() function averages rhon/lat density bined files with differing masks
It ouputs
- the MME
- a percentage of non-masked bins
- the sign agreement of period2-period1 differences
- ToE per run and for MME
Author: Eric Guilyardi : [email protected]
Created on Tue Nov 25 13:56:20 CET 2014
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- years(t1,t2) - years for slice read
- inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE)
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- timeBowl - either time 'mean' or time 'max' bowl used to mask out bowl
- ToeType(str) - ToE type ('F': none, 'histnat')
-> requires running first mm+mme without ToE to compute Stddev
- debug <optional> - boolean value
Notes:
-----
- EG 25 Nov 2014 - Initial function write
- EG 27 Nov 2014 - Rewrite with loop on variables
- EG 06 Dec 2014 - Added agreement on difference with init period - save as <var>Agree
- EG 07 Dec 2014 - Read bowl to remove points above bowl - save as <var>Bowl
- EG 19 Apr 2016 - ToE computation (just for 2D files)
- EG 07 Oct 2016 - add 3D file support
- EG 21 Nov 2016 - move 3D support to new function
- EG 10 jan 2017 - added timeBowl option
- TODO :
- remove loops
- add computation of ToE per model (toe 1 and toe 2) see ticket #50
- add isonhtc (see ticket #48)
'''
# CDMS initialisation - netCDF compression
comp = 1 # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
if t2 <= 0:
useLastYears = True
t2 = -t2
else:
useLastYears = False
t10 = t1
t20 = t2
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
fi = cdm.open(inDir[0]+'/'+listFiles[0])
isond0 = fi['isondepth'] ; # Create variable handle
# Get grid objects
axesList = isond0.getAxisList()
sigmaGrd = isond0.getLevel()
latN = isond0.shape[3]
levN = isond0.shape[2]
basN = isond0.shape[1]
varsig='ptopsigma'
# Declare and open files for writing
if os.path.isfile(outDir+'/'+outFile):
os.remove(outDir+'/'+outFile)
outFile_f = cdm.open(outDir+'/'+outFile,'w')
# Testing mme with less models
#listFiles=listFiles[0:4]
#timN = isond0.shape[0]
timN = t2-t1
runN = len(listFiles)
print ' Number of members:',len(listFiles)
valmask = isond0.missing_value[0]
varList = ['isondepth','isonpers','isonso','isonthetao','isonthick','isonvol']
varFill = [0.,0.,valmask,valmask,0.,0.]
# init arrays (2D rho/lat)
percent = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*0.
#minbowl = npy.ma.ones([basN,latN], dtype='float32')*1000.
varbowl = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*1.
#varList = ['isondepth']
#print ' !!! ### Testing one variable ###'
#varList = ['isonthetao']
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
time.designateTime()
# init ensemble axis
ensembleAxis = cdm.createAxis(npy.float32(range(runN)))
ensembleAxis.id = 'members'
ensembleAxis.units = 'N'
# loop on variables
for iv,var in enumerate(varList):
# Array inits (2D rho/lat 3D rho/lat/lon)
#shapeR = [basN,levN,latN]
isonvar = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*valmask
vardiff,varbowl2D = [npy.ma.ones(npy.ma.shape(isonvar)) for _ in range(2)]
varstd,varToE1,varToE2 = [npy.ma.ones([runN,basN,levN,latN], dtype='float32')*valmask for _ in range(3)]
varones = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*1.
print ' Variable ',iv, var
# loop over files to fill up array
for i,file in enumerate(listFiles):
ft = cdm.open(inDir[0]+'/'+file)
model = file.split('.')[1]
timeax = ft.getAxis('time')
file1d = replace(inDir[0]+'/'+file,'2D','1D')
if os.path.isfile(file1d):
f1d = cdm.open(file1d)
else:
print 'ERROR:',file1d,'missing (if mme, run 1D first)'
sys.exit(1)
tmax = timeax.shape[0]
if i == 0:
tmax0 = tmax
#adapt [t1,t2] time bounds to piControl last NN years
if useLastYears:
t1 = tmax-t20
t2 = tmax
else:
if tmax != tmax0:
print 'wrong time axis: exiting...'
return
# read array
# loop over time/density for memory management
for it in range(timN):
t1r = t1 + it
t2r = t1r + 1
isonRead = ft(var,time = slice(t1r,t2r))
if varFill[iv] != valmask:
isonvar[i,it,...] = isonRead.filled(varFill[iv])
else:
isonvar[i,it,...] = isonRead
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data,valmask).mask
percent[i,...] = npy.float32(npy.equal(maskvar,0))
if mme:
# if mme then just accumulate Bowl, Agree fields
varst = var+'Agree'
vardiff[i,...] = ft(varst,time = slice(t1,t2))
varb = var+'Bowl'
varbowl2D[i,...] = ft(varb,time = slice(t1,t2))
else:
# Compute difference with average of first initN years
varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0)
for t in range(timN):
vardiff[i,t,...] = isonvar[i,t,...] - varinit
vardiff[i,...].mask = isonvar[i,...].mask
# Read bowl and truncate 2D field above bowl
if iv == 0:
bowlRead = f1d(varsig,time = slice(t1,t2))
varbowl[i,...] = bowlRead
# Compute Stddev
varstd[i,...] = npy.ma.std(isonvar[i,...], axis=0)
# Compute ToE
if ToeType == 'histnat':
# Read mean and Std dev from histnat
if i == 0:
filehn = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0]
#filehn = replace(outFile,'historical','historicalNat')
fthn = cdm.open(filehn)
varmeanhn = fthn(var)
varst = var+'Std'
varmaxstd = fthn(varst)
toemult = 1.
signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN))
noise = npy.reshape(varmaxstd,(basN*levN*latN))
varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
toemult = 2.
varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
ft.close()
f1d.close()
# <-- end of loop on files
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent,axis=0))*100.
percenta = mv.masked_less(percenta, 50)
percentw = cdm.createVariable(percenta, axes = [time,axesList[1],axesList[2],axesList[3]], id = 'isonpercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'))
# Sign of difference
if mme:
vardiffsgSum = cdu.averager(vardiff, axis=0)
vardiffsgSum = cdm.createVariable(vardiffsgSum , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
vardiffsgSum = maskVal(vardiffsgSum, valmask)
vardiffsgSum.mask = percentw.mask
else:
vardiffsg = npy.copysign(varones,vardiff)
# average signs
vardiffsgSum = cdu.averager(vardiffsg, axis=0)
vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
vardiffsgSum.mask = percentw.mask
vardiffsgSum._FillValue = valmask
# average variable accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = cdm.createVariable(isonVarAve , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
# Only keep points with rhon > bowl-delta_rho
delta_rho = 0.
if mme: # start from average of <var>Agree
isonVarBowl = cdu.averager(varbowl2D, axis=0)
isonVarBowl = cdm.createVariable(isonVarBowl , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
isonVarBowl = maskVal(isonVarBowl, valmask)
isonVarBowl.mask = percentw.mask
# Compute intermodel stddev
isonVarStd = statistics.std(varbowl2D, axis=0)
isonVarStd = cdm.createVariable(isonVarStd , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
isonVarStd = maskVal(isonVarStd, valmask)
isonVarStd.mask = percentw.mask
if iv == 0:
# Read mulitmodel sigma on bowl and average in time
file1d = replace(outDir+'/'+outFile,'2D','1D')
if os.path.isfile(file1d):
f1d = cdm.open(file1d)
else:
print 'ERROR:',file1d,'missing (if mme, run 1D first)'
sys.exit(1)
bowlRead = f1d(varsig,time = slice(t1,t2))
f1d.close()
siglimit = cdu.averager(bowlRead, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
for il in range(latN):
for ib in range(basN):
#if ib == 2:
# print il, siglimit[ib,il]
if siglimit[ib,il] < valmask/1000.:
# if mme bowl density defined, mask above bowl
index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
isonVarBowl [:,ib,0:index[0],il].mask = True
isonVarStd [:,ib,0:index[0],il].mask = True
vardiffsgSum[:,ib,0:index[0],il].mask = True
else:
# mask all points
isonVarBowl [:,ib,:,il].mask = True
isonVarStd [:,ib,:,il].mask = True
vardiffsgSum[:,ib,:,il].mask = True
else:
isonVarBowl = isonVarAve*1. # start from variable
isonVarStd = isonVarAve*1. # start from variable
if iv == 0:
siglimit = cdu.averager(varbowl, axis=0) # average accross members
# Average bowl in time
if timeBowl == 'mean':
siglimit = cdu.averager(siglimit, axis=0) - delta_rho
# or take largest sigma over time
else:
siglimit = npy.ma.max(siglimit, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
for il in range(latN):
for ib in range(basN):
if siglimit[ib,il] < valmask/1000.:
# if bowl density defined, mask above bowl
index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
isonVarBowl[:,ib,0:index[0],il].mask = True
vardiffsgSum[:,ib,0:index[0],il].mask = True
else:
# mask all points
vardiffsgSum[:,ib,:,il].mask = True
isonVarBowl = maskVal(isonVarBowl, valmask)
# Find max of Std dev of all members
isonVarStd = npy.ma.max(varstd, axis=0)
# mask
if varFill[iv] == valmask:
isonVarStd = maskVal(isonVarStd, valmask)
# Write
isonave = cdm.createVariable(isonVarAve, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id)
isonave.long_name = isonRead.long_name
isonave.units = isonRead.units
isonavediff = cdm.createVariable(vardiffsgSum, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Agree')
isonavediff.long_name = isonRead.long_name
isonavediff.units = isonRead.units
isonavebowl = cdm.createVariable(isonVarBowl, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Bowl')
isonavebowl.long_name = isonRead.long_name
isonavebowl.units = isonRead.units
if not mme:
isonmaxstd = cdm.createVariable(isonVarStd, axes = [axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Std')
isonmaxstd.long_name = isonRead.long_name
isonmaxstd.units = isonRead.units
outFile_f.write( isonave.astype('float32'))
outFile_f.write(isonavediff.astype('float32'))
outFile_f.write(isonavebowl.astype('float32'))
if not mme:
outFile_f.write( isonmaxstd.astype('float32'))
if ToeType == 'histnat':
isontoe1 = cdm.createVariable(varToE1, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE1')
isontoe1.long_name = 'ToE 1 for '+isonRead.long_name
isontoe1.units = 'Year'
isontoe2 = cdm.createVariable(varToE2, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE2')
isontoe2.long_name = 'ToE 2 for '+isonRead.long_name
isontoe2.units = 'Year'
outFile_f.write(isontoe1.astype('float32'))
outFile_f.write(isontoe2.astype('float32'))
if mme:
isonvarstd = cdm.createVariable(isonVarStd , axes =[time,axesList[1],axesList[2],axesList[3]] , id = isonRead.id+'ModStd')
isonvarstd.long_name = isonRead.long_name+' intermodel std'
isonvarstd.units = isonRead.units
outFile_f.write(isonvarstd.astype('float32'))
# <--- end of loop on variables
outFile_f.close()
fi.close()
def plot_bathy(bathy, shadow=True, contour=True, shadow_stretch=1., shadow_shapiro=False,
show=True, shadow_alpha=1., shadow_black=.3, white_deep=False, nmax=30,m=None, alpha=1.,
zmin=None, zmax=None, **kwargs):
"""Plot a bathymetry
- *lon*: Longitude range.
- *lat*: Latitude range.
- *show*:Display the figure [default: True]
- *pcolor*: Use pcolor instead of contour [default: False]
- *contour*: Add line contours [default: True]
- *shadow*:Plot south-west shadows instead of filled contours.
- *nmax*: Max number of levels for contours [default: 30]
- *white_deep*: Deep contours are white [default: False]
- All other keyword are passed to :func:`~vacumm.misc.plot.map2`
"""
# Input
bb = bathy
if isinstance(bathy, GriddedBathy):
bathy = bathy.bathy()
if shadow:
xxs = getattr(bb, '_xxs', None)
yys = getattr(bb, '_yys', None)
if xxs is None:
lon2d = bb._lon2d
lat2d = bb._lat2d
elif shadow:
xxs = yys = None
lon2d,lat2d = meshgrid(get_axis(bathy, -1).getValue(),get_axis(bathy, -2).getValue())
# Masking
if 'maxdep' in kwargs:
zmin = -maxdep
if 'maxalt' in kwargs:
zmax = maxalt
if zmin is not None:
bathy[:] = MV2.masked_less(bathy, zmin)
if zmax is not None:
bathy[:] = MV2.masked_greater(bathy, zmax)
# Default arguments for map
if hasattr(bathy, 'long_name'):
kwargs.setdefault('title',bathy.long_name)
if 'cmap' not in kwargs:
vmin, vmax = minmax(bathy)
# print 'cmap topo', vmin, vmax
kwargs['cmap'] = auto_cmap_topo((kwargs.get('vmin', vmin), kwargs.get('vmax', vmax)))
# kwargs.setdefault('ticklabel_size','smaller')
kwargs.setdefault('clabel_fontsize', 8)
kwargs.setdefault('clabel_alpha',.7*alpha)
kwargs.setdefault('clabel_glow_alpha', kwargs['clabel_alpha'])
kwargs.setdefault('fill', 'contourf')
kwargs['nmax'] = nmax
kwargs['show'] = False
kwargs['contour'] = contour
if shadow: kwargs.setdefault('alpha',.5*alpha)
kwargs.setdefault('projection', 'merc')
kwargs.setdefault('fmt', BathyFormatter())
kwargs.setdefault('colorbar_format', BathyFormatter())
kwargs.setdefault('units', False)
kwargs.setdefault('levels_mode','normal')
kwargs.setdefault('bgcolor', '0.8')
kwargs.setdefault('contour_linestyle', '-')
savefig = kwargs.pop('savefig', None)
kwsavefig = kwfilter(kwargs, 'savefig_')
# White contour when dark
if contour and white_deep:
levels = auto_scale(bathy,nmax=nmax)
colors = []
nlevel = len(levels)
for i in range(nlevel):
if i < old_div(nlevel,2):
colors.append('w')
else:
colors.append('k')
kwargs.setdefault('contour_colors',tuple(colors))
# Call to map
m = map2(bathy, m=m, **kwargs)
# Add shadow
if shadow:
# Filter
data = MV.array(bathy,'f',fill_value=0.)
if shadow_shapiro:
data = shapiro2d(data,fast=True).shape
# Gradient
grd = deriv2d(data,direction=45.,fast=True,fill_value=0.).filled(0.)
grdn = refine(grd, 3)
grdn = norm_atan(grdn,stretch=shadow_stretch).clip(0,1.) ; del grd
# Grid
# im = m.map.imshow(grdn,cmap=P.get_cmap('gist_yarg'),alpha=1) # gist_yarg , YlGnBu
if xxs is None or yys is None:
xx, yy = m(lon2d,lat2d)
xxr = refine(xx, 3)
yyr = refine(yy, 3)
xxs, yys = meshbounds(xxr, yyr)
if isinstance(bb, GriddedBathy):
bb._xxs = xxs
bb._yys = yys
del xx, yy, xxr, yyr
# Cmap
cmap = cmap_custom(( ((1, )*3, 0), ((shadow_black, )*3, 1) ))
# Plot
pp = m.map.pcolormesh(xxs, yys, grdn,cmap=cmap)#P.get_cmap('gist_yarg'))
pp.set_zorder(.9)
pp.set_linewidth(0)
pp.set_alpha(shadow_alpha*N.clip(alpha*2, 0, 1))
del grdn
# Show it?
if savefig:
m.savefig(savefig, **kwsavefig)
if show:
P.show()
return m
def mmeAveMsk1D(listFiles, sw2d, years, inDir, outDir, outFile, timeInt, mme, ToeType, fullTS, debug=True):
'''
The mmeAveMsk1D() function averages rhon or scalar density bined files with differing masks
It ouputs the MME and a percentage of non-masked bins
Created on Tue Nov 25 13:56:20 CET 2014
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- sw2d - dimension of fields to consider (1 or 2)
- years(t1,t2) - years for slice read
- inDir(str) - input directory where files are stored
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- FfllTS - 0/1: if 1, uses full time serie (ignores years(t1,t2))
- debug <optional> - boolean value
Notes:
-----
- EG 25 Nov 2014 - Initial function write
- EG 9 Dec 2014 - Add agreement on difference with init period - save as <var>Agree
- EG 04 Oct 2016 - Add 3D files support
TODO:
------
'''
# CDMS initialisation - netCDF compression
comp = 1 ; # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
if t2 <= 0:
useLastYears = True
t2 = -t2
else:
useLastYears = False
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
# Find dimension
runN = len(listFiles)
try:
fi = cdm.open(inDir[0]+'/'+listFiles[0])
except:
print ' *** file not found ',inDir[0]+'/'+listFiles[0]
sys.exit(' Abort')
if sw2d == 1:
ptopd0 = fi['ptopdepth'] ; # Create variable handle
latN = ptopd0.shape[2]
basN = ptopd0.shape[1]
elif sw2d == 2:
ptopd0 = fi['ptopdepthxy'] ; # Create variable handle
lonN = ptopd0.shape[2]
latN = ptopd0.shape[1]
#timN = ptopd0.shape[0]
timN = t2-t1
if fullTS:
print ' !!! Working on full Time Serie (fullTS = True)'
timN = ptopd0.shape[0]
t1=0
t2=timN
t10 = t1
t20 = t2
# Get grid objects
axesList = ptopd0.getAxisList()
# Declare and open files for writing
if os.path.isfile(outDir+'/'+outFile):
os.remove(outDir+'/'+outFile)
outFile_f = cdm.open(outDir+'/'+outFile,'w')
print ' Number of members:',len(listFiles)
valmask = ptopd0.missing_value
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
time.designateTime()
# loop on variables
# init percent array
if sw2d == 1:
varList = ['ptopdepth','ptopsigma','ptopso','ptopthetao','volpers','salpers','tempers']
#varList = ['ptopdepth']
varDim = [1,1,1,1,0,0,0]
percent = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*0.
elif sw2d == 2:
varList = ['ptopdepthxy','ptopsigmaxy','ptopsoxy','ptopthetaoxy']
#varList = ['ptopdepthxy']
varDim = [2,2,2,2]
percent = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*0.
varFill = [valmask,valmask,valmask,valmask,valmask,valmask,valmask,valmask,valmask]
axis1D = [time,axesList[1],axesList[2]]
axis0D = [time,axesList[1]]
print ' timN = ',timN
# loop on 1D variables
for iv,var in enumerate(varList):
ti0 = timc.clock()
# Array inits
if varDim[iv] == 2:
isonvar = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask
vardiff = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask
varones = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*1.
axisVar = axis1D
elif varDim[iv] == 1:
isonvar = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*valmask
vardiff = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*valmask
varones = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*1.
axisVar = axis1D
else:
isonvar = npy.ma.ones([runN,timN,basN], dtype='float32')*valmask
vardiff = npy.ma.ones([runN,timN,basN], dtype='float32')*valmask
varones = npy.ma.ones([runN,timN,basN], dtype='float32')*1.
axisVar = axis0D
print ' Variable ',iv, var, varDim[iv]
# loop over files to fill up array
for ic,file in enumerate(listFiles):
ft = cdm.open(inDir[0]+'/'+file)
timeax = ft.getAxis('time')
try:
tmax = timeax.shape[0]
except:
print ic,file, timeax
if ic == 0:
tmax0 = tmax
#print ic,file, tmax
#adapt [t1,t2] time bounds to piControl last NN years
if useLastYears:
t1 = tmax-t20
t2 = tmax
else:
if tmax != tmax0:
print 'tmax <> tmax0',tmax,tmax0
print 'wrong time axis: exiting...'
return
#print 'Time dims:',ic, t1,t2,tmax
# read array
computeVar = True
allVars = ft.variables.keys()
if 'ptopsigmaxy' in allVars:
computeVar = False
if (var == 'ptopsigmaxy') & computeVar:
#print ' ic = ',ic
# reconstruct from isondepthg and ptopdepthxy
isond = ft('isondepthg',time = slice(t1,t2))
#print isond.data.shape, timN*latN*lonN
itest = 94*360+150
axesList = isond.getAxisList()
levs = axesList[1][:]
levN = len(levs)
#ti02 = timc.clock()
levs3d0 = mv.reshape(npy.tile(levs,latN*lonN),(latN*lonN,levN))
#ti05 = timc.clock()
isonRead = npy.ma.ones([timN,latN,lonN], dtype='float32')*valmask
for it in range(timN): # loop on time to limit memory usage
levs3d = levs3d0*1.
depthlo = mv.reshape(vardepth[ic,it,...],latN*lonN)
depth3d = npy.reshape(npy.repeat(depthlo,levN),(latN*lonN,levN))
isond3d = mv.reshape(npy.transpose(isond.data[it,...],(1,2,0)),(latN*lonN,levN))
#print isond3d[itest,:]
isond3d[isond3d > valmask/10] = 0.
#print isond3d[itest,:]
isond3dp1 = npy.roll(isond3d,-1,axis=1)
isond3dp1[:,-1] = isond3d[:,-1]
#print isond3dp1[itest,:]
#levs3d[levs3d > 30. ] = 0. # to distinguish bottom masked points from surface masked points
#print levs3d[itest,:]
levs3d[(depth3d <= isond3d)] = 0.
#print levs3d[itest,:]
levs3d[(depth3d > isond3dp1)] = 0.
#print levs3d[itest,:]
#isonwrk = npy.sum(levs3d,axis=1)
isonwrk = npy.max(levs3d,axis=1)
if it < 0:
print ic,it
print depthlo[itest]
print isond3d[itest,:]
print isonwrk[itest]
print
isonRead[it,...] = mv.reshape(isonwrk,(latN,lonN))
# <-- end of loop on time
del (isond3d,isond3dp1); gc.collect()
# mask with depthxy and where sigmaxy = 0
isonRead.mask = vardepth.mask[ic,...]
isonRead = mv.masked_where(isonRead == 0, isonRead)
isonRead.long_name = var
isonRead.units = 'sigma_n'
isonRead.id = var
del (isond,depth3d,levs3d,levs3d0,isonwrk); gc.collect()
#ti3 = timc.clock()
#print ti02-ti0,ti05-ti02, ti1-ti05,ti12-ti1,ti15-ti12,ti2-ti15,ti3-ti2
#print ti3-ti0
# write ptopsigmaxy
if os.path.isfile(inDir[0]+'/work_ptopsigmaxy/'+file):
os.remove(inDir[0]+'/work_ptopsigmaxy/'+file)
fiout = cdm.open(inDir[0]+'/work_ptopsigmaxy/'+file,'w')
if ic == 0:
print ' Creating ',inDir[0]+'/work_ptopsigmaxy/'+file
isonsigxy = cdm.createVariable(isonRead, axes = axis1D, id = 'ptopsigmaxy')
isonsigxy.long_name = 'Density of shallowest persistent ocean on ison'
isonsigxy.units = 'sigma_n'
fiout.write(isonsigxy.astype('float32'))
fiout.close()
else:
# Direct read of variable
isonRead = ft(var,time = slice(t1,t2))
#print isonRead.shape, timN
if varFill[iv] != valmask:
isonvar[ic,...] = isonRead.filled(varFill[iv])
else:
isonvar[ic,...] = isonRead
#print isonvar[ic,:,40,100]
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data,valmask).mask
percent[ic,...] = npy.float32(npy.equal(maskvar,0))
if mme:
# if mme then just average Bowl and Agree fields
varst = var+'Agree'
vardiff[ic,...] = ft(varst,time = slice(t1,t2))
else:
# Compute difference with average of first initN years, use mask of last month
varinit = cdu.averager(isonvar[ic,peri1:peri2,...],axis=0)
for tr in range(timN):
vardiff[ic,tr,...] = isonvar[ic,tr,...] - varinit
vardiff[ic,...].mask = isonvar[ic,...].mask
ft.close()
# <-- end of loop on files
# TODO remove masked points at longitudes 0 or 180deg for some models
# if ptopdepthxy, keep for ptopsigmaxy computation (reconstruct from isondepthg and ptopdepthxy)
if var =='ptopdepthxy':
vardepth = isonvar
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent,axis=0))*100.
percenta = mv.masked_less(percenta, 50)
percentw = cdm.createVariable(percenta, axes = axis1D, id = 'ptoppercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'))
# Sign of difference
if mme:
vardiffsgSum = cdu.averager(vardiff, axis=0)
vardiffsgSum = cdm.createVariable(vardiffsgSum , axes = axisVar , id = 'foo')
vardiffsgSum = maskVal(vardiffsgSum, valmask)
vardiffsgSum.mask = percentw.mask
else:
vardiffsg = npy.copysign(varones,vardiff)
# average signs
vardiffsgSum = cdu.averager(vardiffsg, axis=0)
vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
vardiffsgSum.mask = percentw.mask
vardiffsgSum._FillValue = valmask
# average accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = cdm.createVariable(isonVarAve , axes = axisVar , id = 'foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
# Write
isonave = cdm.createVariable(isonVarAve, axes = axisVar, id = isonRead.id)
isonave.long_name = isonRead.long_name
isonave.units = isonRead.units
isonavediff = cdm.createVariable(vardiffsgSum, axes = axisVar, id = isonRead.id+'Agree')
isonavediff.long_name = isonRead.long_name
isonavediff.units = isonRead.units
outFile_f.write(isonave.astype('float32'))
outFile_f.write(isonavediff.astype('float32'))
tf = timc.clock()
#print ' time var',tf-ti0
# <--- end of loop on variables
outFile_f.close()
fi.close()
0.059503571833625334 0.059503571833625334 0.05664014775641405 0.05193557222118004 0.04777129850801233 0.0407139313814465 0.029382624830271705 0.018469399844287374 0.0162382275289592 0.02646680241827459 0.04792041732949079 0.0689138797030203 0.08167038620212037 0.09273558459066569 0.11266293431057901 0.13663018925347364 0.15229174546388072 0.15284435880966177 0.13423845476113883 0.09945904378274077 0.07032267160267985 0.05551039827020481 0.045537187647785464 0.040532491867244946 0.03577527125478327 -999. -999. -999. -0.058062458673116 -0.08764922509099882 -0.11697036914487152 -0.14836133615864944 -0.17956528904564023 -0.21109198032585794 -0.23846429237248942 -0.2598536549218765 -0.27795672866320387 -0.2939939095159731 -0.30541031366330024 -0.307643559333884 -0.30078421139811795 -0.2841339526883441 -0.26485737397202497 -0.24287299694779327 -0.22379014890999907 -0.20121548204699846 -0.1746486732156772 -0.14585019344118372 -0.12070675757803526 -0.0997891159111037 -0.08229393660994214 -0.06779720501287469 -0.057213385470859794 -0.04875768191096844 -0.0402377347189964 -0.030169328367807245 -0.017560662894847895 -0.006968922654137132 0.0009773980274431048 0.007054306637034288 0.010472286514133042 0.010702384151997032 0.009231553701801242 0.007544033101056543 0.004639797857203645 -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. """.split() data = numpy.array(data, dtype=numpy.float) data = MV2.masked_less(data, -900) x.plot(data, yx, bg=1) regression.run(x, "test_vcs_1D_datawc_missing.png")
import vcs
import cdms2
import sys
import os
import MV2
f=cdms2.open(os.path.join(sys.prefix,"sample_data","clt.nc"))
s=f("clt",time=slice(0,1),squeeze=1)
s=MV2.masked_less(s,65.)
x=vcs.init()
gm=x.createisofill()
#gm=x.createboxfill()
gm.missing = 252
#gm.levels = [65,70,75,80,85,90,95,100]
x.plot(s,gm)
x.png(gm.g_name)
x.interact()
if gm_type=="meshfill":
f=cdms2.open(os.path.join(vcs.sample_data,'sampleCurveGrid4.nc'))
else:
f=cdms2.open(os.path.join(vcs.sample_data,'clt.nc'))
if gm_type=="vector":
u=f("u",**xtra)
v=f("v",**xtra)
if args.mask:
u=MV2.masked_greater(u,58.)
if args.zero:
u-=u
v-=v
elif gm_type=="meshfill":
s=f("sample",**xtra)
if args.mask:
s=MV2.masked_less(s,1150.)
elif args.bigvalues:
s[s < 1150] = 1e40
if args.zero:
s-=s
else:
s=f("clt",**xtra)
if args.mask:
s=MV2.masked_greater(s,78.)
elif args.bigvalues:
s[s > 78] = 1e40
if gm_type in ["1d","yxvsx","xyvsy","xvsy","scatter"]:
s = s(latitude=(20,20,"cob"),longitude=(112,112,"cob"),squeeze=1)
s2=MV2.sin(s)
if args.zero:
s2-=s2
def mmeAveMsk3D(listFiles, years, inDir, outDir, outFile, timeInt, mme, ToeType, debug=True):
'''
The mmeAveMsk3D() function averages rhon/lat density bined files with differing masks
It ouputs
- the MME
- a percentage of non-masked bins
- the sign agreement of period2-period1 differences
- ToE per run and for MME
Author: Eric Guilyardi : [email protected]
Created on Tue Nov 21 2016
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- years(t1,t2) - years for slice read
- inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE)
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- ToeType(str) - ToE type ('F': none, 'histnat')
-> requires running first mm+mme without ToE to compute Stddev
- debug <optional> - boolean value
Notes:
-----
- EG 21 Nov 2016 - Initial function write
- TODO :
- add computation of ToE per model (toe 1 and toe 2) see ticket #50
- add isonhtc (see ticket #48)
'''
# CDMS initialisation - netCDF compression
comp = 1 # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
fi = cdm.open(inDir[0]+'/'+listFiles[0])
# Switch if only variables below the bowl are present/treated
nobowl = True
if nobowl:
isond0 = fi['isondepthgBowl'] ; # Create variable handle
else:
isond0 = fi['isondepthg'] ; # Create variable handle
# Get grid objects
axesList = isond0.getAxisList()
sigmaGrd = isond0.getLevel()
#time = isond0.getTime()
lonN = isond0.shape[3]
latN = isond0.shape[2]
levN = isond0.shape[1]
varsig='ptopsigmaxy'
# Limit number of models to 3 for testing of mme
#if mme:
# listFiles = listFiles[0:2]
# print ' !!! ### Testing 3 models ###', listFiles
# Declare and open files for writing
if os.path.isfile(outDir+'/'+outFile):
os.remove(outDir+'/'+outFile)
outFile_f = cdm.open(outDir+'/'+outFile,'w')
#timN = isond0.shape[0]
timN = t2-t1
runN = len(listFiles)
print ' Number of members:',len(listFiles)
valmask = isond0.missing_value
varList = ['isondepthg','persistmxy','sog','thetaog','isonthickg']
varFill = [valmask,valmask,valmask,valmask,valmask]
percent = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*0.
varbowl = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*1.
#varList = ['isondepthg']
#print ' !!! ### Testing one variable ###', varList
# init sigma axis
sigma = cdm.createAxis(npy.float32(range(1)))
sigma.id = axesList[1].id
sigma.units = axesList[1].units
sigma.designateTime()
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
# init ensemble axis
ensembleAxis = cdm.createAxis(npy.float32(range(runN)))
ensembleAxis.id = 'members'
ensembleAxis.units = 'N'
# Output axis
sigmaList = [sigma,axesList[2],axesList[3]] ; # sigma, lat, lon
sigmaTimeList = [sigma,time,axesList[2],axesList[3]] ; # sigma, time, lat, lon
# init arrays
isonvar = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask
varbowl2D = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask
varstd,varToE1,varToE2 = [npy.ma.ones([runN,latN,lonN], dtype='float32')*valmask for _ in range(3)]
# Loop on density levels (for memory management, becomes UNLIMITED axis and requires a ncpq to reorder dimensions)
delta_ib = 1
print ' Sigma index:'
for ib in range(levN):
ib1 = ib + delta_ib
print ib,
tim0 = timc.clock()
# loop on variables
for iv,var in enumerate(varList):
if nobowl:
varb = var+'Bowl'
else:
varb = var
if ib == 0:
print ' Variable ',iv, varb
# loop over files to fill up array
for i,file in enumerate(listFiles):
tim01 = timc.clock()
ft = cdm.open(inDir[0]+'/'+file)
model = file.split('.')[1]
timeax = ft.getAxis('time')
if i == 0:
tmax0 = timeax.shape[0]
tmax = timeax.shape[0]
if tmax != tmax0:
print 'wrong time axis: exiting...'
return
# read array
isonRead = ft(varb,time = slice(t1,t2), lev = slice(ib,ib1)).squeeze()
if varFill[iv] != valmask:
isonvar[i,...] = isonRead.filled(varFill[iv])
else:
isonvar[i,...] = isonRead
tim02 = timc.clock()
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data,valmask).mask
percent[i,...] = npy.float32(npy.equal(maskvar,0))
tim03 = timc.clock()
if mme:
# if mme then just accumulate Bowl, Agree and Std fields
#varst = var+'Agree'
#vardiff[i,...] = ft(varst,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze()
isonRead = ft(varb,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze()
varbowl2D[i,...] = isonRead
else:
# Compute difference with average of first initN years
#varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0)
#for t in range(timN):
# vardiff[i,t,...] = isonvar[i,t,...] - varinit
#vardiff[i,...].mask = isonvar[i,...].mask
# Read bowl to truncate field above bowl
if ib == 0 and iv == 0:
varbowl[i,...] = ft(varsig,time = slice(t1,t2))
#varbowl[i,...] = bowlRead
# Compute Stddev
varstd[i,...] = npy.ma.std(isonvar[i,...], axis=0)
# Compute ToE
if ToeType == 'histnat':
toto=1
# TODO
# Read mean and Std dev from histnat
# if i == 0:
# filehn = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0]
# #filehn = replace(outFile,'historical','historicalNat')
# fthn = cdm.open(filehn)
# varmeanhn = fthn(var)
# varst = var+'Std'
# varmaxstd = fthn(varst)
# toemult = 1.
# signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN))
# noise = npy.reshape(varmaxstd,(basN*levN*latN))
# varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
# toemult = 2.
# varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
tim04 = timc.clock()
ft.close()
#print 'ib, section 1 timing',ib, tim02-tim01,tim03-tim02,tim04-tim03
# <-- end of loop on files (i)
tim1 = timc.clock()
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent,axis=0))*100.
percenta = mv.masked_less(percenta, 50)
percenta = npy.reshape(percenta,[delta_ib,timN,latN,lonN])
percentw = cdm.createVariable(percenta, axes = sigmaTimeList, id = 'isonpercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'), extend = 1, index = ib)
# Sign of difference
#if mme:
# vardiffsgSum = cdu.averager(vardiff, axis=0)
# vardiffsgSum = cdm.createVariable(vardiffsgSum , axes = sigmaTimeList , id = 'foo')
# vardiffsgSum = maskVal(vardiffsgSum, valmask)
# vardiffsgSum.mask = percentw.mask
#else:
# vardiffsg = npy.copysign(varones,vardiff)
# # average signs
# vardiffsgSum = cdu.averager(vardiffsg, axis=0)
# vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
# vardiffsgSum.mask = percentw.mask
# vardiffsgSum._FillValue = valmask
# average variable accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = npy.reshape(isonVarAve,[delta_ib,timN,latN,lonN])
isonVarAve = cdm.createVariable(isonVarAve , axes = sigmaTimeList , id = 'foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
tim2 = timc.clock()
# Only keep points with rhon > bowl-delta_rho
delta_rho = 0.
# mme case
if mme: # start from average of <var>Agree
isonVarBowl = cdu.averager(varbowl2D, axis=0)
isonVarBowl = npy.reshape(isonVarBowl,[delta_ib,timN,latN,lonN])
isonVarBowl = cdm.createVariable(isonVarBowl , axes = sigmaTimeList , id = 'foo')
isonVarBowl = maskVal(isonVarBowl, valmask)
isonVarBowl.mask = percentw.mask
# Compute intermodel stddev
isonVarStd = statistics.std(varbowl2D, axis=0)
isonVarStd = npy.reshape(isonVarStd,[delta_ib,timN,latN,lonN])
isonVarStd = cdm.createVariable(isonVarStd , axes = sigmaTimeList , id = 'foo')
isonVarStd = maskVal(isonVarStd, valmask)
isonVarStd.mask = percentw.mask
# Write
isonvarbowlw = cdm.createVariable(isonVarBowl , axes = sigmaTimeList , id = isonRead.id)
isonvarbowlw.long_name = isonRead.long_name
isonvarbowlw.units = isonRead.units
isonvarstdw = cdm.createVariable(isonVarStd , axes = sigmaTimeList , id = isonRead.id+'Std')
isonvarstdw.long_name = isonRead.long_name+' intermodel std'
isonvarstdw.units = isonRead.units
outFile_f.write(isonvarbowlw.astype('float32'), extend = 1, index = ib)
outFile_f.write(isonvarstdw.astype('float32'), extend = 1, index = ib)
#if ib == 0 and iv == 0:
# # TODO review
# # Read multimodel sigma on bowl and average in time
# file1d = replace(outDir+'/'+outFile,'2D','1D')
# if os.path.isfile(file1d):
# f1d = cdm.open(file1d)
# else:
# print 'ERROR:',file1d,'missing (if mme, run 2D first)'
# sys.exit(1)
# bowlRead = f1d(varsig,time = slice(t1,t2),lev = slice(ib,ib1))
# f1d.close()
# siglimit = cdu.averager(bowlRead, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
#if sw2d == 1:
# for il in range(latN):
# for ib in range(basN):
# #if ib == 2:
# # print il, siglimit[ib,il]
# if siglimit[ib,il] < valmask/1000.:
# # if mme bowl density defined, mask above bowl
# index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
# isonVarBowl [:,ib,0:index[0],il].mask = True
# isonVarStd [:,ib,0:index[0],il].mask = True
# vardiffsgSum[:,ib,0:index[0],il].mask = True
# else:
# # mask all points
# isonVarBowl [:,ib,:,il].mask = True
# isonVarStd [:,ib,:,il].mask = True
# vardiffsgSum[:,ib,:,il].mask = True
# mm case
else:
isonVarBowl = isonVarAve*1. # start from variable
#isonVarStd = isonVarAve*1. # start from variable
if ib == 0 and iv == 0:
# build bowl position
siglimit = cdu.averager(varbowl, axis=0) # average accross members
siglimit = npy.reshape(siglimit,[timN*latN*lonN]) - delta_rho
if iv == 0:
sigarr = siglimit*1.
sigarr[:] = sigmaGrd[ib]
# test
i = 60
j = 60
ij = j*lonN+i
isonVarBowl = npy.reshape(isonVarBowl,[timN*latN*lonN])
#vardiffsgSum = npy.reshape(vardiffsgSum,[timN*latN*lonN])
isonVarBowl.mask = npy.where(sigarr < siglimit, True, isonVarBowl.mask)
#vardiffsgSum.mask = npy.where(sigarr < siglimit, True, vardiffsgSum.mask)
isonVarBowl = npy.reshape(isonVarBowl,[timN,latN,lonN])
#vardiffsgSum = npy.reshape(vardiffsgSum,[timN,latN,lonN])
isonVarBowl = maskVal(isonVarBowl, valmask)
#vardiffsgSum = maskVal(vardiffsgSum, valmask)
# Find max of Std dev of all members
isonVarStd = npy.ma.max(varstd, axis=0)
# mask
isonVarStd = maskVal(isonVarStd, valmask)
# Write
#isonave = cdm.createVariable(isonVarAve, axes = sigmaTimeList, id = isonRead.id)
#isonave.long_name = isonRead.long_name
#isonave.units = isonRead.units
#vardiffsgSum = npy.reshape(vardiffsgSum,[delta_ib,timN,latN,lonN])
#isonavediff = cdm.createVariable(vardiffsgSum, axes = sigmaTimeList, id = isonRead.id+'Agree')
#isonavediff.long_name = isonRead.long_name
#isonavediff.units = isonRead.units
isonVarBowl = npy.reshape(isonVarBowl,[delta_ib,timN,latN,lonN])
isonavebowl = cdm.createVariable(isonVarBowl, axes = sigmaTimeList, id = isonRead.id+'Bowl')
isonavebowl.long_name = isonRead.long_name
isonavebowl.units = isonRead.units
isonVarStd = npy.reshape(isonVarStd,[delta_ib,latN,lonN])
isonmaxstd = cdm.createVariable(isonVarStd, axes = sigmaList, id = isonRead.id+'Std')
isonmaxstd.long_name = isonRead.long_name
isonmaxstd.units = isonRead.units
#outFile_f.write( isonave.astype('float32'), extend = 1, index = ib)
#outFile_f.write(isonavediff.astype('float32'), extend = 1, index = ib)
outFile_f.write(isonavebowl.astype('float32'), extend = 1, index = ib)
outFile_f.write(isonmaxstd.astype('float32'), extend = 1, index = ib)
tim3 = timc.clock()
if ToeType == 'histnat':
isontoe1 = cdm.createVariable(varToE1, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE1')
isontoe1.long_name = 'ToE 1 for '+isonRead.long_name
isontoe1.units = 'Year'
isontoe2 = cdm.createVariable(varToE2, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE2')
isontoe2.long_name = 'ToE 2 for '+isonRead.long_name
isontoe2.units = 'Year'
outFile_f.write(isontoe1.astype('float32'), extend = 1, index = ib)
outFile_f.write(isontoe2.astype('float32'), extend = 1, index = ib)
tim4 = timc.clock()
# <--- end of loop on variables
#print 'ib, timing',ib, tim01-tim0,tim1-tim01,tim2-tim1,tim3-tim2,tim4-tim3
# <--- end of loop on density
print ' '
outFile_f.close()
fi.close()
