def __call__(self, **kargs):
""" Returns the array of values"""
axes = self.getAxisList()
sh = []
ids = []
for a in axes:
sh.append(len(a)) # store length to construct array shape
ids.append(a.id) # store ids
# first let's see which vars are actually asked for
# for now assume all keys means restriction on dims
for axis_id in kargs:
if axis_id not in ids:
raise ValueError("Invalid axis '%s'" % axis_id)
index = ids.index(axis_id)
value = kargs[axis_id]
if isinstance(value, basestring):
value = [value]
if not isinstance(value, (list, tuple, slice)):
raise TypeError(
"Invalid subsetting type for axis '%s', axes can only be subsetted by string,list or slice" %
axis_id)
if isinstance(value, slice):
axes[index] = axes[index].subAxis(
value.start, value.stop, value.step)
sh[index] = len(axes[index])
else: # ok it's a list
for v in value:
if v not in axes[index][:]:
raise ValueError(
"Unkwown value '%s' for axis '%s'" %
(v, axis_id))
axis = cdms2.createAxis(value, id=axes[index].id)
axes[index] = axis
sh[index] = len(axis)
array = numpy.ma.ones(sh, dtype=numpy.float)
# Now let's fill this array
self.get_array_values_from_dict_recursive(array, [], [], [], axes)
array = MV2.masked_greater(array, 9.e19)
array.id = "pmp"
array.setAxisList(axes)
return array
def __call__(self, merge=[], **kargs):
""" Returns the array of values"""
# First clean up kargs
if "merge" in kargs:
merge = kargs["merge"]
del (kargs["merge"])
order = None
axes_ids = self.getAxisIds()
if "order" in kargs:
# If it's an actual axis assume that it's what user wants
# Otherwise it's an out order keyword
if "order" not in axes_ids:
order = kargs["order"]
del (kargs["order"])
ab = cdms2.getAutoBounds()
cdms2.setAutoBounds("off")
axes = self.getAxisList()
if merge != []:
if isinstance(merge[0], str):
merge = [
merge,
]
if merge != []:
for merger in merge:
for merge_axis_id in merger:
if merge_axis_id not in axes_ids:
raise RuntimeError(
"You requested to merge axis is '{}' which is not valid. Axes: {}"
.format(merge_axis_id, axes_ids))
sh = []
ids = []
used_ids = []
for a in axes:
# Regular axis not a merged one
sh.append(len(a)) # store length to construct array shape
ids.append(a.id) # store ids
used_ids.append(a.id)
# first let's see which vars are actually asked for
# for now assume all keys means restriction on dims
if not isinstance(merge, (list, tuple)):
raise RuntimeError(
"merge keyword must be a list of dimensions to merge together")
if len(merge) > 0 and not isinstance(merge[0], (list, tuple)):
merge = [
merge,
]
for axis_id in kargs:
if axis_id not in ids:
raise ValueError("Invalid axis '%s'" % axis_id)
index = ids.index(axis_id)
value = kargs[axis_id]
if isinstance(value, basestring):
value = [value]
if not isinstance(value, (list, tuple, slice)):
raise TypeError(
"Invalid subsetting type for axis '%s', axes can only be subsetted by string,list or slice"
% axis_id)
if isinstance(value, slice):
axes[index] = axes[index].subAxis(value.start, value.stop,
value.step)
sh[index] = len(axes[index])
else: # ok it's a list
for v in value:
if v not in axes[index][:]:
raise ValueError("Unkwown value '%s' for axis '%s'" %
(v, axis_id))
axis = cdms2.createAxis(value, id=axes[index].id)
axes[index] = axis
sh[index] = len(axis)
array = numpy.ma.ones(sh, dtype=numpy.float)
# Now let's fill this array
self.get_array_values_from_dict_recursive(array, [], [], [], axes)
# Ok at this point we need to take care of merged axes
# First let's create the merged axes
axes_to_group = []
for merger in merge:
merged_axes = []
for axid in merger:
for ax in axes:
if ax.id == axid:
merged_axes.append(ax)
axes_to_group.append(merged_axes)
new_axes = [groupAxes(grp_axes) for grp_axes in axes_to_group]
sh2 = list(sh)
for merger in merge:
for merger in merge: # loop through all possible merging
merged_indices = []
for id in merger:
merged_indices.append(axes_ids.index(id))
for indx in merged_indices:
sh2[indx] = 1
smallest = min(merged_indices)
for indx in merged_indices:
sh2[smallest] *= sh[indx]
myorder = []
for index in range(len(sh)):
if index in myorder:
continue
for merger in merge:
merger = [axes_ids.index(x) for x in merger]
if index in merger and index not in myorder:
for indx in merger:
myorder.append(indx)
if index not in myorder: # ok did not find this one anywhere
myorder.append(index)
outData = numpy.transpose(array, myorder)
outData = numpy.reshape(outData, sh2)
yank = []
for merger in merge:
merger = [axes_ids.index(x) for x in merger]
mn = min(merger)
merger.remove(mn)
yank += merger
yank = sorted(yank, reverse=True)
for yk in yank:
extract = (slice(0, None), ) * yk
extract += (0, )
outData = outData[extract]
# Ok now let's apply the newaxes
sub = 0
outData = MV2.array(outData)
merged_axis_done = []
for index in range(len(array.shape)):
foundInMerge = False
for imerge, merger in enumerate(merge):
merger = [axes_ids.index(x) for x in merger]
if index in merger:
foundInMerge = True
if imerge not in merged_axis_done:
merged_axis_done.append(imerge)
setMergedAxis = imerge
else:
setMergedAxis = -1
if not foundInMerge:
outData.setAxis(index - sub, axes[index])
else:
if setMergedAxis == -1:
sub += 1
else:
outData.setAxis(index - sub, new_axes[setMergedAxis])
outData = MV2.masked_greater(outData, 9.98e20)
outData.id = "pmp"
if order is not None:
myorder = "".join(["({})".format(nm) for nm in order])
outData = outData(order=myorder)
# Merge needs cleaning for extra dims crated
if merge != []:
for i in range(outData.ndim):
outData = scrap(outData, axis=i)
outData = MV2.masked_greater(outData, 9.9e19)
cdms2.setAutoBounds(ab)
return outData
import cdms2, MV2, vcs
from region_utils import getAreaOfAllClosedDomains
f = cdms2.open('data/snc.nc')
data = f('snc', time=slice(1), squeeze=1)
print "Contions \n 1. Mask less than 100 \n 2. Mask less than 30 and greater than 70"
con = input('Enter Condion Option 1 or 2 : ')
if con == 1:
mask_condition = MV2.masked_less(data, 100.)
value = 'value equal to 100'
dirname = 'outplots_eq_100'
elif con == 2:
c1 = MV2.masked_greater(data, 70.)
c2 = MV2.masked_less(data, 30.)
mask_condition = MV2.logical_and(c1, c2)
value = 'value equal to within 30 to 70'
dirname = 'outplots_eq_30_to_70'
# end of if con == 1:
if not os.path.isdir(dirname): os.mkdir(dirname)
print "Output plots will be saved in '%s' directory" % dirname
v = vcs.init()
# Assign the variable "cf_asd" to the persistent 'ASD' isofill graphics methods.
cf_asd = v.createboxfill()
cf_asd.level_1 = data.min() # change to default minimum level
cf_asd.level_2 = data.max() # change to default maximum level
import os, sys, MV2, cdms2, vcs, testing.regression as regression
x = regression.init()
f = cdms2.open(os.path.join(vcs.sample_data,"clt.nc"))
s = f("clt",slice(0,1),latitude=(30, 70),longitude=(-130, -60))
s2 = MV2.masked_greater(s, 65.)
x.plot(s2,"default","isofill",bg=1)
regression.run(x, "test_vcs_isofill_mask_cell_shift.png")
x = vcs.init()
x.setantialiasing(0)
x.drawlogooff()
if bg:
x.setbgoutputdimensions(1200,1091,units="pixels")
x.setcolormap("rainbow")
gm = vcs.createvector()
gm.scale = args.scale
nm_xtra = ""
xtra = {}
import cdms2
import os
f=cdms2.open(os.path.join(vcs.sample_data,"clt.nc"))
u=f("u")
v=f("v")
u=MV2.masked_greater(u,35.)
v=MV2.masked_greater(v,888.)
print u.max(),v.max(),u.min(),v.min()
#x.plot(U)
x.plot(u,v,gm,bg=bg)
ret=0
if args.show:
pass
#x.interact()
else:
fnm = "test_vcs_vectors_missing"
if args.scale!=1.:
fnm+="_%.1g" % args.scale
fnm+=nm_xtra
x.png(fnm)
ret = regression.check_result_image(fnm+'.png', src, regression.defaultThreshold, cleanup=not args.keep)
import os,sys
import MV2
import vcs
import cdms2
src=sys.argv[1]
pth = os.path.join(os.path.dirname(__file__),"..")
sys.path.append(pth)
import checkimage
x=vcs.init()
x.setantialiasing(0)
x.drawlogooff()
x.setbgoutputdimensions(1200,1091,units="pixels")
f=cdms2.open(os.path.join(vcs.sample_data,"clt.nc"))
s=f("clt",slice(0,1),latitude=(30,70),longitude=(-130,-60))
s2=MV2.masked_greater(s,65.)
x.plot(s2,"default","isofill",bg=1)
fnm = "test_vcs_isofill_mask_cell_shift.png"
x.png(fnm)
print "fnm:",fnm
print "src:",src
ret = checkimage.check_result_image(fnm,src,checkimage.defaultThreshold)
sys.exit(ret)
import cdutil,MV2
import cdms2,numpy
cdms2.setAutoBounds("on")
a = MV2.masked_greater(MV2.array([1,4,5,6,7,8,9.]),.5)
assert numpy.ma.is_masked(cdutil.averager(a))
x = vcs.init()
x.drawlogooff()
if bg:
x.setbgoutputdimensions(1200, 1091, units="pixels")
x.setcolormap("rainbow")
gm = vcs.createvector()
gm.scale = args.scale
nm_xtra = ""
xtra = {}
import cdms2
import os
f = cdms2.open(os.path.join(vcs.prefix, "sample_data", "clt.nc"))
u = f("u")
v = f("v")
u = MV2.masked_greater(u, 35.)
v = MV2.masked_greater(v, 888.)
print u.max(), v.max(), u.min(), v.min()
#x.plot(U)
x.plot(u, v, gm, bg=bg)
ret = 0
if args.show:
pass
#x.interact()
else:
fnm = "test_vcs_vectors_missing"
if args.scale != 1.:
fnm += "_%.1g" % args.scale
fnm += nm_xtra
x.png(fnm)
print "fnm:", fnm
def mmeAveMsk2D(listFiles, years, inDir, outDir, outFile, timeInt, mme, timeBowl, ToeType, debug=True):
'''
The mmeAveMsk2D() function averages rhon/lat density bined files with differing masks
It ouputs
- the MME
- a percentage of non-masked bins
- the sign agreement of period2-period1 differences
- ToE per run and for MME
Author: Eric Guilyardi : [email protected]
Created on Tue Nov 25 13:56:20 CET 2014
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- years(t1,t2) - years for slice read
- inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE)
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- timeBowl - either time 'mean' or time 'max' bowl used to mask out bowl
- ToeType(str) - ToE type ('F': none, 'histnat')
-> requires running first mm+mme without ToE to compute Stddev
- debug <optional> - boolean value
Notes:
-----
- EG 25 Nov 2014 - Initial function write
- EG 27 Nov 2014 - Rewrite with loop on variables
- EG 06 Dec 2014 - Added agreement on difference with init period - save as <var>Agree
- EG 07 Dec 2014 - Read bowl to remove points above bowl - save as <var>Bowl
- EG 19 Apr 2016 - ToE computation (just for 2D files)
- EG 07 Oct 2016 - add 3D file support
- EG 21 Nov 2016 - move 3D support to new function
- EG 10 jan 2017 - added timeBowl option
- TODO :
- remove loops
- add computation of ToE per model (toe 1 and toe 2) see ticket #50
- add isonhtc (see ticket #48)
'''
# CDMS initialisation - netCDF compression
comp = 1 # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
if t2 <= 0:
useLastYears = True
t2 = -t2
else:
useLastYears = False
t10 = t1
t20 = t2
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
fi = cdm.open(inDir[0]+'/'+listFiles[0])
isond0 = fi['isondepth'] ; # Create variable handle
# Get grid objects
axesList = isond0.getAxisList()
sigmaGrd = isond0.getLevel()
latN = isond0.shape[3]
levN = isond0.shape[2]
basN = isond0.shape[1]
varsig='ptopsigma'
# Declare and open files for writing
if os.path.isfile(outDir+'/'+outFile):
os.remove(outDir+'/'+outFile)
outFile_f = cdm.open(outDir+'/'+outFile,'w')
# Testing mme with less models
#listFiles=listFiles[0:4]
#timN = isond0.shape[0]
timN = t2-t1
runN = len(listFiles)
print ' Number of members:',len(listFiles)
valmask = isond0.missing_value[0]
varList = ['isondepth','isonpers','isonso','isonthetao','isonthick','isonvol']
varFill = [0.,0.,valmask,valmask,0.,0.]
# init arrays (2D rho/lat)
percent = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*0.
#minbowl = npy.ma.ones([basN,latN], dtype='float32')*1000.
varbowl = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*1.
#varList = ['isondepth']
#print ' !!! ### Testing one variable ###'
#varList = ['isonthetao']
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
time.designateTime()
# init ensemble axis
ensembleAxis = cdm.createAxis(npy.float32(range(runN)))
ensembleAxis.id = 'members'
ensembleAxis.units = 'N'
# loop on variables
for iv,var in enumerate(varList):
# Array inits (2D rho/lat 3D rho/lat/lon)
#shapeR = [basN,levN,latN]
isonvar = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*valmask
vardiff,varbowl2D = [npy.ma.ones(npy.ma.shape(isonvar)) for _ in range(2)]
varstd,varToE1,varToE2 = [npy.ma.ones([runN,basN,levN,latN], dtype='float32')*valmask for _ in range(3)]
varones = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*1.
print ' Variable ',iv, var
# loop over files to fill up array
for i,file in enumerate(listFiles):
ft = cdm.open(inDir[0]+'/'+file)
model = file.split('.')[1]
timeax = ft.getAxis('time')
file1d = replace(inDir[0]+'/'+file,'2D','1D')
if os.path.isfile(file1d):
f1d = cdm.open(file1d)
else:
print 'ERROR:',file1d,'missing (if mme, run 1D first)'
sys.exit(1)
tmax = timeax.shape[0]
if i == 0:
tmax0 = tmax
#adapt [t1,t2] time bounds to piControl last NN years
if useLastYears:
t1 = tmax-t20
t2 = tmax
else:
if tmax != tmax0:
print 'wrong time axis: exiting...'
return
# read array
# loop over time/density for memory management
for it in range(timN):
t1r = t1 + it
t2r = t1r + 1
isonRead = ft(var,time = slice(t1r,t2r))
if varFill[iv] != valmask:
isonvar[i,it,...] = isonRead.filled(varFill[iv])
else:
isonvar[i,it,...] = isonRead
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data,valmask).mask
percent[i,...] = npy.float32(npy.equal(maskvar,0))
if mme:
# if mme then just accumulate Bowl, Agree fields
varst = var+'Agree'
vardiff[i,...] = ft(varst,time = slice(t1,t2))
varb = var+'Bowl'
varbowl2D[i,...] = ft(varb,time = slice(t1,t2))
else:
# Compute difference with average of first initN years
varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0)
for t in range(timN):
vardiff[i,t,...] = isonvar[i,t,...] - varinit
vardiff[i,...].mask = isonvar[i,...].mask
# Read bowl and truncate 2D field above bowl
if iv == 0:
bowlRead = f1d(varsig,time = slice(t1,t2))
varbowl[i,...] = bowlRead
# Compute Stddev
varstd[i,...] = npy.ma.std(isonvar[i,...], axis=0)
# Compute ToE
if ToeType == 'histnat':
# Read mean and Std dev from histnat
if i == 0:
filehn = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0]
#filehn = replace(outFile,'historical','historicalNat')
fthn = cdm.open(filehn)
varmeanhn = fthn(var)
varst = var+'Std'
varmaxstd = fthn(varst)
toemult = 1.
signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN))
noise = npy.reshape(varmaxstd,(basN*levN*latN))
varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
toemult = 2.
varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
ft.close()
f1d.close()
# <-- end of loop on files
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent,axis=0))*100.
percenta = mv.masked_less(percenta, 50)
percentw = cdm.createVariable(percenta, axes = [time,axesList[1],axesList[2],axesList[3]], id = 'isonpercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'))
# Sign of difference
if mme:
vardiffsgSum = cdu.averager(vardiff, axis=0)
vardiffsgSum = cdm.createVariable(vardiffsgSum , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
vardiffsgSum = maskVal(vardiffsgSum, valmask)
vardiffsgSum.mask = percentw.mask
else:
vardiffsg = npy.copysign(varones,vardiff)
# average signs
vardiffsgSum = cdu.averager(vardiffsg, axis=0)
vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
vardiffsgSum.mask = percentw.mask
vardiffsgSum._FillValue = valmask
# average variable accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = cdm.createVariable(isonVarAve , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
# Only keep points with rhon > bowl-delta_rho
delta_rho = 0.
if mme: # start from average of <var>Agree
isonVarBowl = cdu.averager(varbowl2D, axis=0)
isonVarBowl = cdm.createVariable(isonVarBowl , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
isonVarBowl = maskVal(isonVarBowl, valmask)
isonVarBowl.mask = percentw.mask
# Compute intermodel stddev
isonVarStd = statistics.std(varbowl2D, axis=0)
isonVarStd = cdm.createVariable(isonVarStd , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
isonVarStd = maskVal(isonVarStd, valmask)
isonVarStd.mask = percentw.mask
if iv == 0:
# Read mulitmodel sigma on bowl and average in time
file1d = replace(outDir+'/'+outFile,'2D','1D')
if os.path.isfile(file1d):
f1d = cdm.open(file1d)
else:
print 'ERROR:',file1d,'missing (if mme, run 1D first)'
sys.exit(1)
bowlRead = f1d(varsig,time = slice(t1,t2))
f1d.close()
siglimit = cdu.averager(bowlRead, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
for il in range(latN):
for ib in range(basN):
#if ib == 2:
# print il, siglimit[ib,il]
if siglimit[ib,il] < valmask/1000.:
# if mme bowl density defined, mask above bowl
index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
isonVarBowl [:,ib,0:index[0],il].mask = True
isonVarStd [:,ib,0:index[0],il].mask = True
vardiffsgSum[:,ib,0:index[0],il].mask = True
else:
# mask all points
isonVarBowl [:,ib,:,il].mask = True
isonVarStd [:,ib,:,il].mask = True
vardiffsgSum[:,ib,:,il].mask = True
else:
isonVarBowl = isonVarAve*1. # start from variable
isonVarStd = isonVarAve*1. # start from variable
if iv == 0:
siglimit = cdu.averager(varbowl, axis=0) # average accross members
# Average bowl in time
if timeBowl == 'mean':
siglimit = cdu.averager(siglimit, axis=0) - delta_rho
# or take largest sigma over time
else:
siglimit = npy.ma.max(siglimit, axis=0) - delta_rho
# TODO: remove loop by building global array with 1/0
for il in range(latN):
for ib in range(basN):
if siglimit[ib,il] < valmask/1000.:
# if bowl density defined, mask above bowl
index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
isonVarBowl[:,ib,0:index[0],il].mask = True
vardiffsgSum[:,ib,0:index[0],il].mask = True
else:
# mask all points
vardiffsgSum[:,ib,:,il].mask = True
isonVarBowl = maskVal(isonVarBowl, valmask)
# Find max of Std dev of all members
isonVarStd = npy.ma.max(varstd, axis=0)
# mask
if varFill[iv] == valmask:
isonVarStd = maskVal(isonVarStd, valmask)
# Write
isonave = cdm.createVariable(isonVarAve, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id)
isonave.long_name = isonRead.long_name
isonave.units = isonRead.units
isonavediff = cdm.createVariable(vardiffsgSum, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Agree')
isonavediff.long_name = isonRead.long_name
isonavediff.units = isonRead.units
isonavebowl = cdm.createVariable(isonVarBowl, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Bowl')
isonavebowl.long_name = isonRead.long_name
isonavebowl.units = isonRead.units
if not mme:
isonmaxstd = cdm.createVariable(isonVarStd, axes = [axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Std')
isonmaxstd.long_name = isonRead.long_name
isonmaxstd.units = isonRead.units
outFile_f.write( isonave.astype('float32'))
outFile_f.write(isonavediff.astype('float32'))
outFile_f.write(isonavebowl.astype('float32'))
if not mme:
outFile_f.write( isonmaxstd.astype('float32'))
if ToeType == 'histnat':
isontoe1 = cdm.createVariable(varToE1, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE1')
isontoe1.long_name = 'ToE 1 for '+isonRead.long_name
isontoe1.units = 'Year'
isontoe2 = cdm.createVariable(varToE2, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE2')
isontoe2.long_name = 'ToE 2 for '+isonRead.long_name
isontoe2.units = 'Year'
outFile_f.write(isontoe1.astype('float32'))
outFile_f.write(isontoe2.astype('float32'))
if mme:
isonvarstd = cdm.createVariable(isonVarStd , axes =[time,axesList[1],axesList[2],axesList[3]] , id = isonRead.id+'ModStd')
isonvarstd.long_name = isonRead.long_name+' intermodel std'
isonvarstd.units = isonRead.units
outFile_f.write(isonvarstd.astype('float32'))
# <--- end of loop on variables
outFile_f.close()
fi.close()
def mmeAveMsk1D(listFiles, sw2d, years, inDir, outDir, outFile, timeInt, mme, ToeType, fullTS, debug=True):
'''
The mmeAveMsk1D() function averages rhon or scalar density bined files with differing masks
It ouputs the MME and a percentage of non-masked bins
Created on Tue Nov 25 13:56:20 CET 2014
Inputs:
-------
- listFiles(str) - the list of files to be averaged
- sw2d - dimension of fields to consider (1 or 2)
- years(t1,t2) - years for slice read
- inDir(str) - input directory where files are stored
- outDir(str) - output directory
- outFile(str) - output file
- timeInt(2xindices) - indices of init period to compare with (e.g. [1,20])
- mme(bool) - multi-model mean (will read in single model ensemble stats)
- FfllTS - 0/1: if 1, uses full time serie (ignores years(t1,t2))
- debug <optional> - boolean value
Notes:
-----
- EG 25 Nov 2014 - Initial function write
- EG 9 Dec 2014 - Add agreement on difference with init period - save as <var>Agree
- EG 04 Oct 2016 - Add 3D files support
TODO:
------
'''
# CDMS initialisation - netCDF compression
comp = 1 ; # 0 for no compression
cdm.setNetcdfShuffleFlag(comp)
cdm.setNetcdfDeflateFlag(comp)
cdm.setNetcdfDeflateLevelFlag(comp)
cdm.setAutoBounds('on')
# Numpy initialisation
npy.set_printoptions(precision=2)
if debug:
debug = True
else:
debug = False
# File dim and grid inits
t1 = years[0]
t2 = years[1]
if t2 <= 0:
useLastYears = True
t2 = -t2
else:
useLastYears = False
# Bound of period average to remove
peri1 = timeInt[0]
peri2 = timeInt[1]
# Find dimension
runN = len(listFiles)
try:
fi = cdm.open(inDir[0]+'/'+listFiles[0])
except:
print ' *** file not found ',inDir[0]+'/'+listFiles[0]
sys.exit(' Abort')
if sw2d == 1:
ptopd0 = fi['ptopdepth'] ; # Create variable handle
latN = ptopd0.shape[2]
basN = ptopd0.shape[1]
elif sw2d == 2:
ptopd0 = fi['ptopdepthxy'] ; # Create variable handle
lonN = ptopd0.shape[2]
latN = ptopd0.shape[1]
#timN = ptopd0.shape[0]
timN = t2-t1
if fullTS:
print ' !!! Working on full Time Serie (fullTS = True)'
timN = ptopd0.shape[0]
t1=0
t2=timN
t10 = t1
t20 = t2
# Get grid objects
axesList = ptopd0.getAxisList()
# Declare and open files for writing
if os.path.isfile(outDir+'/'+outFile):
os.remove(outDir+'/'+outFile)
outFile_f = cdm.open(outDir+'/'+outFile,'w')
print ' Number of members:',len(listFiles)
valmask = ptopd0.missing_value
# init time axis
time = cdm.createAxis(npy.float32(range(timN)))
time.id = 'time'
time.units = 'years since 1861'
time.designateTime()
# loop on variables
# init percent array
if sw2d == 1:
varList = ['ptopdepth','ptopsigma','ptopso','ptopthetao','volpers','salpers','tempers']
#varList = ['ptopdepth']
varDim = [1,1,1,1,0,0,0]
percent = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*0.
elif sw2d == 2:
varList = ['ptopdepthxy','ptopsigmaxy','ptopsoxy','ptopthetaoxy']
#varList = ['ptopdepthxy']
varDim = [2,2,2,2]
percent = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*0.
varFill = [valmask,valmask,valmask,valmask,valmask,valmask,valmask,valmask,valmask]
axis1D = [time,axesList[1],axesList[2]]
axis0D = [time,axesList[1]]
print ' timN = ',timN
# loop on 1D variables
for iv,var in enumerate(varList):
ti0 = timc.clock()
# Array inits
if varDim[iv] == 2:
isonvar = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask
vardiff = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask
varones = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*1.
axisVar = axis1D
elif varDim[iv] == 1:
isonvar = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*valmask
vardiff = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*valmask
varones = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*1.
axisVar = axis1D
else:
isonvar = npy.ma.ones([runN,timN,basN], dtype='float32')*valmask
vardiff = npy.ma.ones([runN,timN,basN], dtype='float32')*valmask
varones = npy.ma.ones([runN,timN,basN], dtype='float32')*1.
axisVar = axis0D
print ' Variable ',iv, var, varDim[iv]
# loop over files to fill up array
for ic,file in enumerate(listFiles):
ft = cdm.open(inDir[0]+'/'+file)
timeax = ft.getAxis('time')
try:
tmax = timeax.shape[0]
except:
print ic,file, timeax
if ic == 0:
tmax0 = tmax
#print ic,file, tmax
#adapt [t1,t2] time bounds to piControl last NN years
if useLastYears:
t1 = tmax-t20
t2 = tmax
else:
if tmax != tmax0:
print 'tmax <> tmax0',tmax,tmax0
print 'wrong time axis: exiting...'
return
#print 'Time dims:',ic, t1,t2,tmax
# read array
computeVar = True
allVars = ft.variables.keys()
if 'ptopsigmaxy' in allVars:
computeVar = False
if (var == 'ptopsigmaxy') & computeVar:
#print ' ic = ',ic
# reconstruct from isondepthg and ptopdepthxy
isond = ft('isondepthg',time = slice(t1,t2))
#print isond.data.shape, timN*latN*lonN
itest = 94*360+150
axesList = isond.getAxisList()
levs = axesList[1][:]
levN = len(levs)
#ti02 = timc.clock()
levs3d0 = mv.reshape(npy.tile(levs,latN*lonN),(latN*lonN,levN))
#ti05 = timc.clock()
isonRead = npy.ma.ones([timN,latN,lonN], dtype='float32')*valmask
for it in range(timN): # loop on time to limit memory usage
levs3d = levs3d0*1.
depthlo = mv.reshape(vardepth[ic,it,...],latN*lonN)
depth3d = npy.reshape(npy.repeat(depthlo,levN),(latN*lonN,levN))
isond3d = mv.reshape(npy.transpose(isond.data[it,...],(1,2,0)),(latN*lonN,levN))
#print isond3d[itest,:]
isond3d[isond3d > valmask/10] = 0.
#print isond3d[itest,:]
isond3dp1 = npy.roll(isond3d,-1,axis=1)
isond3dp1[:,-1] = isond3d[:,-1]
#print isond3dp1[itest,:]
#levs3d[levs3d > 30. ] = 0. # to distinguish bottom masked points from surface masked points
#print levs3d[itest,:]
levs3d[(depth3d <= isond3d)] = 0.
#print levs3d[itest,:]
levs3d[(depth3d > isond3dp1)] = 0.
#print levs3d[itest,:]
#isonwrk = npy.sum(levs3d,axis=1)
isonwrk = npy.max(levs3d,axis=1)
if it < 0:
print ic,it
print depthlo[itest]
print isond3d[itest,:]
print isonwrk[itest]
print
isonRead[it,...] = mv.reshape(isonwrk,(latN,lonN))
# <-- end of loop on time
del (isond3d,isond3dp1); gc.collect()
# mask with depthxy and where sigmaxy = 0
isonRead.mask = vardepth.mask[ic,...]
isonRead = mv.masked_where(isonRead == 0, isonRead)
isonRead.long_name = var
isonRead.units = 'sigma_n'
isonRead.id = var
del (isond,depth3d,levs3d,levs3d0,isonwrk); gc.collect()
#ti3 = timc.clock()
#print ti02-ti0,ti05-ti02, ti1-ti05,ti12-ti1,ti15-ti12,ti2-ti15,ti3-ti2
#print ti3-ti0
# write ptopsigmaxy
if os.path.isfile(inDir[0]+'/work_ptopsigmaxy/'+file):
os.remove(inDir[0]+'/work_ptopsigmaxy/'+file)
fiout = cdm.open(inDir[0]+'/work_ptopsigmaxy/'+file,'w')
if ic == 0:
print ' Creating ',inDir[0]+'/work_ptopsigmaxy/'+file
isonsigxy = cdm.createVariable(isonRead, axes = axis1D, id = 'ptopsigmaxy')
isonsigxy.long_name = 'Density of shallowest persistent ocean on ison'
isonsigxy.units = 'sigma_n'
fiout.write(isonsigxy.astype('float32'))
fiout.close()
else:
# Direct read of variable
isonRead = ft(var,time = slice(t1,t2))
#print isonRead.shape, timN
if varFill[iv] != valmask:
isonvar[ic,...] = isonRead.filled(varFill[iv])
else:
isonvar[ic,...] = isonRead
#print isonvar[ic,:,40,100]
# compute percentage of non-masked points accros MME
if iv == 0:
maskvar = mv.masked_values(isonRead.data,valmask).mask
percent[ic,...] = npy.float32(npy.equal(maskvar,0))
if mme:
# if mme then just average Bowl and Agree fields
varst = var+'Agree'
vardiff[ic,...] = ft(varst,time = slice(t1,t2))
else:
# Compute difference with average of first initN years, use mask of last month
varinit = cdu.averager(isonvar[ic,peri1:peri2,...],axis=0)
for tr in range(timN):
vardiff[ic,tr,...] = isonvar[ic,tr,...] - varinit
vardiff[ic,...].mask = isonvar[ic,...].mask
ft.close()
# <-- end of loop on files
# TODO remove masked points at longitudes 0 or 180deg for some models
# if ptopdepthxy, keep for ptopsigmaxy computation (reconstruct from isondepthg and ptopdepthxy)
if var =='ptopdepthxy':
vardepth = isonvar
# Compute percentage of bin presence
# Only keep points where percent > 50%
if iv == 0:
percenta = (cdu.averager(percent,axis=0))*100.
percenta = mv.masked_less(percenta, 50)
percentw = cdm.createVariable(percenta, axes = axis1D, id = 'ptoppercent')
percentw._FillValue = valmask
percentw.long_name = 'percentage of MME bin'
percentw.units = '%'
outFile_f.write(percentw.astype('float32'))
# Sign of difference
if mme:
vardiffsgSum = cdu.averager(vardiff, axis=0)
vardiffsgSum = cdm.createVariable(vardiffsgSum , axes = axisVar , id = 'foo')
vardiffsgSum = maskVal(vardiffsgSum, valmask)
vardiffsgSum.mask = percentw.mask
else:
vardiffsg = npy.copysign(varones,vardiff)
# average signs
vardiffsgSum = cdu.averager(vardiffsg, axis=0)
vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
vardiffsgSum.mask = percentw.mask
vardiffsgSum._FillValue = valmask
# average accross members
isonVarAve = cdu.averager(isonvar, axis=0)
isonVarAve = cdm.createVariable(isonVarAve , axes = axisVar , id = 'foo')
# mask
if varFill[iv] == valmask:
isonVarAve = maskVal(isonVarAve, valmask)
isonVarAve.mask = percentw.mask
# Write
isonave = cdm.createVariable(isonVarAve, axes = axisVar, id = isonRead.id)
isonave.long_name = isonRead.long_name
isonave.units = isonRead.units
isonavediff = cdm.createVariable(vardiffsgSum, axes = axisVar, id = isonRead.id+'Agree')
isonavediff.long_name = isonRead.long_name
isonavediff.units = isonRead.units
outFile_f.write(isonave.astype('float32'))
outFile_f.write(isonavediff.astype('float32'))
tf = timc.clock()
#print ' time var',tf-ti0
# <--- end of loop on variables
outFile_f.close()
fi.close()
def plot_bathy(bathy,
shadow=True,
contour=True,
shadow_stretch=1.,
shadow_shapiro=False,
show=True,
shadow_alpha=1.,
shadow_black=.3,
white_deep=False,
nmax=30,
m=None,
alpha=1.,
zmin=None,
zmax=None,
**kwargs):
"""Plot a bathymetry
- *lon*: Longitude range.
- *lat*: Latitude range.
- *show*:Display the figure [default: True]
- *pcolor*: Use pcolor instead of contour [default: False]
- *contour*: Add line contours [default: True]
- *shadow*:Plot south-west shadows instead of filled contours.
- *nmax*: Max number of levels for contours [default: 30]
- *white_deep*: Deep contours are white [default: False]
- All other keyword are passed to :func:`~vacumm.misc.plot.map2`
"""
# Input
bb = bathy
if isinstance(bathy, GriddedBathy):
bathy = bathy.bathy()
if shadow:
xxs = getattr(bb, '_xxs', None)
yys = getattr(bb, '_yys', None)
if xxs is None:
lon2d = bb._lon2d
lat2d = bb._lat2d
elif shadow:
xxs = yys = None
lon2d, lat2d = meshgrid(
get_axis(bathy, -1).getValue(),
get_axis(bathy, -2).getValue())
# Masking
if 'maxdep' in kwargs:
zmin = -maxdep
if 'maxalt' in kwargs:
zmax = maxalt
if zmin is not None:
bathy[:] = MV2.masked_less(bathy, zmin)
if zmax is not None:
bathy[:] = MV2.masked_greater(bathy, zmax)
# Default arguments for map
if hasattr(bathy, 'long_name'):
kwargs.setdefault('title', bathy.long_name)
if not kwargs.has_key('cmap'):
vmin, vmax = minmax(bathy)
# print 'cmap topo', vmin, vmax
kwargs['cmap'] = auto_cmap_topo(
(kwargs.get('vmin', vmin), kwargs.get('vmax', vmax)))
# kwargs.setdefault('ticklabel_size','smaller')
kwargs.setdefault('clabel_fontsize', 8)
kwargs.setdefault('clabel_alpha', .7 * alpha)
kwargs.setdefault('clabel_glow_alpha', kwargs['clabel_alpha'])
kwargs.setdefault('fill', 'contourf')
kwargs['nmax'] = nmax
kwargs['show'] = False
kwargs['contour'] = contour
if shadow: kwargs.setdefault('alpha', .5 * alpha)
kwargs.setdefault('projection', 'merc')
kwargs.setdefault('fmt', BathyFormatter())
kwargs.setdefault('colorbar_format', BathyFormatter())
kwargs.setdefault('units', False)
kwargs.setdefault('levels_mode', 'normal')
kwargs.setdefault('bgcolor', '0.8')
kwargs.setdefault('contour_linestyle', '-')
savefig = kwargs.pop('savefig', None)
kwsavefig = kwfilter(kwargs, 'savefig_')
# White contour when dark
if contour and white_deep:
levels = auto_scale(bathy, nmax=nmax)
colors = []
nlevel = len(levels)
for i in xrange(nlevel):
if i < nlevel / 2:
colors.append('w')
else:
colors.append('k')
kwargs.setdefault('contour_colors', tuple(colors))
# Call to map
m = map2(bathy, m=m, **kwargs)
# Add shadow
if shadow:
# Filter
data = MV.array(bathy, 'f', fill_value=0.)
if shadow_shapiro:
data = shapiro2d(data, fast=True).shape
# Gradient
grd = deriv2d(data, direction=45., fast=True, fill_value=0.).filled(0.)
grdn = refine(grd, 3)
grdn = norm_atan(grdn, stretch=shadow_stretch).clip(0, 1.)
del grd
# Grid
# im = m.map.imshow(grdn,cmap=P.get_cmap('gist_yarg'),alpha=1) # gist_yarg , YlGnBu
if xxs is None or yys is None:
xx, yy = m(lon2d, lat2d)
xxr = refine(xx, 3)
yyr = refine(yy, 3)
xxs, yys = meshbounds(xxr, yyr)
if isinstance(bb, GriddedBathy):
bb._xxs = xxs
bb._yys = yys
del xx, yy, xxr, yyr
# Cmap
cmap = cmap_custom((((1, ) * 3, 0), ((shadow_black, ) * 3, 1)))
# Plot
pp = m.map.pcolormesh(xxs, yys, grdn,
cmap=cmap) #P.get_cmap('gist_yarg'))
pp.set_zorder(.9)
pp.set_linewidth(0)
pp.set_alpha(shadow_alpha * N.clip(alpha * 2, 0, 1))
del grdn
# Show it?
if savefig:
m.savefig(savefig, **kwsavefig)
if show:
P.show()
return m
from vacumm.misc.plot import savefigs, add_key
import MV2, numpy as N, matplotlib.pyplot as P
# Champs initial
xx, yy = N.indices((50, 100), 'f')
x0 = y0 = 25
dxy = 30
var = MV2.exp(-((xx-x0)**2+(yy-y0)**2)/dxy**2)
# Masques
# - reference
mask = var.filled()>.9
mask[:, 50:] = True
mask[15:35, 65:85] = False
# - variable
var[:] = MV2.masked_greater(var, .8)
var[:, 50:] = MV2.masked
# Erode
vare = erode_coast(var, mask)
# Plots
P.figure(figsize=(6, 9))
P.subplots_adjust(hspace=.2)
P.subplot(311)
P.pcolormesh(var.asma())
P.title('Original variable') ; add_key(1, color='w')
P.subplot(312)
P.pcolormesh(mask)
P.title('Reference mask') ; add_key(2, color='w')
P.subplot(313)
import cdms2, cdat_info
import MV2
import sys
f = cdms2.open(cdat_info.get_prefix() + "/sample_data/clt.nc")
s = f('clt')
S2 = MV2.masked_greater(s, 87)
G = cdms2.createGaussianGrid(22)
S3 = S2.regrid(G)
assert (S3.max() < 87.1)
gm.datawc_x1 = args.lon1
gm.datawc_x2 = args.lon2
xtra["longitude"] = (args.lon1, args.lon2)
nm_xtra += "_%i_%i" % (args.lon1, args.lon2)
if args.rg:
nm_xtra += "_via_gm"
if gm_type == "meshfill":
f = cdms2.open(
os.path.join(vcs.prefix, 'sample_data', 'sampleCurveGrid4.nc'))
else:
f = cdms2.open(os.path.join(vcs.prefix, 'sample_data', 'clt.nc'))
if gm_type == "vector":
u = f("u", **xtra)
v = f("v", **xtra)
if args.mask:
u = MV2.masked_greater(u, 58.)
if args.zero:
u -= u
v -= v
elif gm_type == "meshfill":
s = f("sample", **xtra)
if args.mask:
s = MV2.masked_less(s, 1150.)
elif args.bigvalues:
s[s < 1150] = 1e40
if args.zero:
s -= s
else:
s = f("clt", **xtra)
if args.mask:
s = MV2.masked_greater(s, 78.)
c4_range2010 = c4gr2010 * 0.0
c3_range2010 = MV2.where(((rangel2010 != 0) & (pft_biome == 3) |
(pft_biome == 4) | (pft_biome == 5)), rangel2010, 0)
c4_range2010 = MV2.where(
((rangel2010 != 0) & (pft_biome == 1) | (pft_biome == 2)), rangel2010, 0)
pft2010[9, :, :] = pft2010[9, :, :] + c3_range2010
pft2010[10, :, :] = pft2010[10, :, :] + c4_range2010
pft_sum2010 = MV2.sum(pft2010, axis=0) + icwtr
print 'Sum of all PFT is', pft_sum2010.shape
#x.plot(pft_sum2010[::-1,:],gm,bg=bg)
#x.png('pft_sum2010.png')
#x.clear()
#In case if sum of all PFTs is greater than 1, reduce all PFTs
re_pft2010 = MV2.masked_greater(pft_sum2010 - lndf_hurt, 0.).mask
for npft in range(19):
#print 'npft',npft
pft_tmp1 = pft2010[npft, :, :]
pft_tmp1 = MV2.choose(re_pft2010,
(pft_tmp1, pft_tmp1 * lndf_hurt / pft_sum2010))
pft2010[npft, :, :] = pft_tmp1
#In case if sum of all PFTs is less than 1, increase all PFTs
inc_pft2010 = MV2.masked_less(pft_sum2010 - lndf_hurt, 0.).mask
for npft in range(19):
#print 'npft',npft
pft_tmp2 = pft2010[npft, :, :]
pft_tmp2 = MV2.choose(inc_pft2010,
(pft_tmp2, pft_tmp2 * lndf_hurt / pft_sum2010))
pft2010[npft, :, :] = pft_tmp2
import sys
import cdms2
import MV2
import cdutil
f = cdms2.open(sys.prefix + "/sample_data/clt.nc")
s = f("clt")
print s.missing_value
s.set_fill_value(1.e36)
s2 = MV2.masked_greater(s, 79.)
print s2.missing_value
cdutil.times.setTimeBoundsMonthly(s2)
s3 = cdutil.ANNUALCYCLE(s2)
print s3.missing_value
s3 = cdutil.ANNUALCYCLE.departures(s2)
print s3.missing_value
s3 = cdutil.ANNUALCYCLE.climatology(s2)
print s3.missing_value
x = vcs.init()
x.setantialiasing(0)
x.drawlogooff()
if bg:
x.setbgoutputdimensions(1200, 1091, units="pixels")
x.setcolormap("rainbow")
gm = vcs.createvector()
gm.scale = args.scale
nm_xtra = ""
xtra = {}
import cdms2
import os
f = cdms2.open(os.path.join(vcs.sample_data, "clt.nc"))
u = f("u")
v = f("v")
u = MV2.masked_greater(u, 35.)[..., ::2, ::2]
v = MV2.masked_greater(v, 888.)[..., ::2, ::2]
print u.max(), v.max(), u.min(), v.min()
#x.plot(U)
x.plot(u, v, gm, bg=bg)
ret = 0
if args.show:
pass
#x.interact()
else:
fnm = "test_vcs_vectors_missing"
if args.scale != 5.:
fnm += "_%.1g" % args.scale
fnm += nm_xtra
x.png(fnm)
ret = regression.check_result_image(fnm + '.png',
import sys, os
import vcs
import sys
import cdms2
import vtk
import os
import MV2
bg = not False
x = vcs.init()
x.setcolormap("rainbow")
gm = vcs.createboxfill()
p = vcs.createprojection()
ptype = int('0')
p.type = ptype
gm.projection = p
xtra = {}
f = cdms2.open(os.path.join(vcs.prefix, 'sample_data', 'clt.nc'))
s = f("clt", **xtra)
s = MV2.masked_greater(s, 78.)
x.plot(s, gm, bg=bg)
x.png('test_vcs_basic_boxfill_masked_0_proj.png')
from vacumm.misc.plot import savefigs, add_key
import MV2, numpy as N, matplotlib.pyplot as P
# Champs initial
xx, yy = N.indices((50, 100), 'f')
x0 = y0 = 25
dxy = 30
var = MV2.exp(-((xx - x0)**2 + (yy - y0)**2) / dxy**2)
# Masques
# - reference
mask = var.filled() > .9
mask[:, 50:] = True
mask[15:35, 65:85] = False
# - variable
var[:] = MV2.masked_greater(var, .8)
var[:, 50:] = MV2.masked
# Erode
vare = erode_coast(var, mask)
# Plots
P.figure(figsize=(6, 9))
P.subplots_adjust(hspace=.2)
P.subplot(311)
P.pcolormesh(var.asma())
P.title('Original variable')
add_key(1, color='w')
P.subplot(312)
P.pcolormesh(mask)
P.title('Reference mask')
def __call__(self, merge=[], **kargs):
""" Returns the array of values"""
# First clean up kargs
if "merge" in kargs:
merge = kargs["merge"]
del(kargs["merge"])
order = None
axes_ids = self.getAxisIds()
if "order" in kargs:
# If it's an actual axis assume that it's what user wants
# Otherwise it's an out order keyword
if "order" not in axes_ids:
order = kargs["order"]
del(kargs["order"])
ab = cdms2.getAutoBounds()
cdms2.setAutoBounds("off")
axes = self.getAxisList()
if merge != []:
if isinstance(merge[0], str):
merge = [merge, ]
if merge != []:
for merger in merge:
for merge_axis_id in merger:
if merge_axis_id not in axes_ids:
raise RuntimeError(
"You requested to merge axis is '{}' which is not valid. Axes: {}".format(
merge_axis_id, axes_ids))
sh = []
ids = []
used_ids = []
for a in axes:
# Regular axis not a merged one
sh.append(len(a)) # store length to construct array shape
ids.append(a.id) # store ids
used_ids.append(a.id)
# first let's see which vars are actually asked for
# for now assume all keys means restriction on dims
if not isinstance(merge, (list, tuple)):
raise RuntimeError(
"merge keyword must be a list of dimensions to merge together")
if len(merge) > 0 and not isinstance(merge[0], (list, tuple)):
merge = [merge, ]
for axis_id in kargs:
if axis_id not in ids:
raise ValueError("Invalid axis '%s'" % axis_id)
index = ids.index(axis_id)
value = kargs[axis_id]
if isinstance(value, basestring):
value = [value]
if not isinstance(value, (list, tuple, slice)):
raise TypeError(
"Invalid subsetting type for axis '%s', axes can only be subsetted by string,list or slice" %
axis_id)
if isinstance(value, slice):
axes[index] = axes[index].subAxis(
value.start, value.stop, value.step)
sh[index] = len(axes[index])
else: # ok it's a list
for v in value:
if v not in axes[index][:]:
raise ValueError(
"Unkwown value '%s' for axis '%s'" %
(v, axis_id))
axis = cdms2.createAxis(value, id=axes[index].id)
axes[index] = axis
sh[index] = len(axis)
array = numpy.ma.ones(sh, dtype=numpy.float)
# Now let's fill this array
self.get_array_values_from_dict_recursive(array, [], [], [], axes)
# Ok at this point we need to take care of merged axes
# First let's create the merged axes
axes_to_group = []
for merger in merge:
merged_axes = []
for axid in merger:
for ax in axes:
if ax.id == axid:
merged_axes.append(ax)
axes_to_group.append(merged_axes)
new_axes = [groupAxes(grp_axes) for grp_axes in axes_to_group]
sh2 = list(sh)
for merger in merge:
for merger in merge: # loop through all possible merging
merged_indices = []
for id in merger:
merged_indices.append(axes_ids.index(id))
for indx in merged_indices:
sh2[indx] = 1
smallest = min(merged_indices)
for indx in merged_indices:
sh2[smallest] *= sh[indx]
myorder = []
for index in range(len(sh)):
if index in myorder:
continue
for merger in merge:
merger = [axes_ids.index(x) for x in merger]
if index in merger and index not in myorder:
for indx in merger:
myorder.append(indx)
if index not in myorder: # ok did not find this one anywhere
myorder.append(index)
outData = numpy.transpose(array, myorder)
outData = numpy.reshape(outData, sh2)
yank = []
for merger in merge:
merger = [axes_ids.index(x) for x in merger]
mn = min(merger)
merger.remove(mn)
yank += merger
yank = sorted(yank, reverse=True)
for yk in yank:
extract = (slice(0, None),) * yk
extract += (0,)
outData = outData[extract]
# Ok now let's apply the newaxes
sub = 0
outData = MV2.array(outData)
merged_axis_done = []
for index in range(len(array.shape)):
foundInMerge = False
for imerge, merger in enumerate(merge):
merger = [axes_ids.index(x) for x in merger]
if index in merger:
foundInMerge = True
if imerge not in merged_axis_done:
merged_axis_done.append(imerge)
setMergedAxis = imerge
else:
setMergedAxis = -1
if not foundInMerge:
outData.setAxis(index - sub, axes[index])
else:
if setMergedAxis == -1:
sub += 1
else:
outData.setAxis(index - sub, new_axes[setMergedAxis])
outData = MV2.masked_greater(outData, 9.98e20)
outData.id = "pmp"
if order is not None:
myorder = "".join(["({})".format(nm) for nm in order])
outData = outData(order=myorder)
# Merge needs cleaning for extra dims crated
if merge != []:
for i in range(outData.ndim):
outData = scrap(outData, axis=i)
outData = MV2.masked_greater(outData, 9.9e19)
cdms2.setAutoBounds(ab)
return outData
import MV2
from markError import clearError,markError,reportError
print 'Test 15: reshape and mask and average ...',
a=MV2.arange(100)
try:
failed = False
a.shape=(10,10)
except:
failed = True
a = MV2.reshape(a,(10,10))
if failed is True: markError('shape should not have worked (protected attribute)')
if len(a.getAxisList())!=2: markError('reshape did not produce 2 axes')
a=MV2.masked_greater(a,23)
b=MV2.average(a,axis=0)
c=a-b
import cdms2,cdat_info
import MV2
import sys
f = cdms2.open(cdat_info.get_prefix() + "/sample_data/clt.nc")
s = f('clt')
S2 = MV2.masked_greater(s, 87)
G = cdms2.createGaussianGrid(22)
S3 = S2.regrid(G)
assert(S3.max() < 87.1)
import sys
import cdms2
import MV2
import cdutil
f = cdms2.open(sys.prefix + "/sample_data/clt.nc")
s = f("clt")
print s.missing_value
s.set_fill_value(1.0e36)
s2 = MV2.masked_greater(s, 79.0)
print s2.missing_value
cdutil.times.setTimeBoundsMonthly(s2)
s3 = cdutil.ANNUALCYCLE(s2)
print s3.missing_value
s3 = cdutil.ANNUALCYCLE.departures(s2)
print s3.missing_value
s3 = cdutil.ANNUALCYCLE.climatology(s2)
print s3.missing_value
import sys,os
import vcs
import sys
import cdms2
import vtk
import os
import MV2
bg = not False
x=vcs.init()
x.setcolormap("rainbow")
gm = vcs.createisoline()
p = vcs.createprojection()
ptype = 'aeqd'
p.type = ptype
gm.projection = p
xtra = {}
f=cdms2.open(os.path.join(vcs.prefix,'sample_data','clt.nc'))
s=f("clt",**xtra)
s=MV2.masked_greater(s,78.)
x.plot(s,gm,bg=bg)
x.png('test_vcs_basic_isoline_masked_aeqd_proj.png')
def basicGm(self,
gm_type,
projtype="default",
lat1=0,
lat2=0,
lon1=0,
lon2=0,
rg=False,
flip=False,
zero=False,
transparent=False,
mask=False,
bigvalues=False,
color=False,
evenlySpaced=False):
self.x.clear()
self.x.setcolormap(None)
cdms2.tvariable.TransientVariable.variable_count = 1
loc = locals()
exec("gm=vcs.create%s()" % gm_type)
gm = loc["gm"]
if projtype != "default":
p = vcs.createprojection()
try:
ptype = int(projtype)
except BaseException:
ptype = projtype
p.type = ptype
gm.projection = p
nm_xtra = ""
xtra = {}
if lat1 != lat2:
if rg:
if flip:
gm.datawc_y1 = lat2
gm.datawc_y2 = lat1
nm_xtra += "_gmflip"
else:
gm.datawc_y1 = lat1
gm.datawc_y2 = lat2
xtra["latitude"] = (lat1, lat2)
if lat1 < 0:
nm_xtra += "_SH"
else:
nm_xtra += "_NH"
if lon1 != lon2:
if rg:
gm.datawc_x1 = lon1
gm.datawc_x2 = lon2
xtra["longitude"] = (lon1, lon2)
nm_xtra += "_%i_%i" % (lon1, lon2)
if rg:
nm_xtra += "_via_gm"
if gm_type == "meshfill":
f = cdms2.open(os.path.join(vcs.sample_data,
'sampleCurveGrid4.nc'))
else:
f = self.clt
if gm_type in ["vector", "streamline"]:
u = f("u", **xtra)
v = f("v", **xtra)
if mask:
u = MV2.masked_greater(u, 58.)
if zero:
u -= u
v -= v
elif gm_type == "meshfill":
s = f("sample", **xtra)
if mask:
s = MV2.masked_less(s, 1150.)
elif bigvalues:
s[s < 1150] = 1e40
if zero:
s -= s
else:
s = f("clt", **xtra)
if mask:
s = MV2.masked_greater(s, 78.)
elif bigvalues:
s[s > 78] = 1e40
if gm_type in ["1d", "yxvsx", "xyvsy", "xvsy", "scatter"]:
s = s(latitude=(20, 20, "cob"),
longitude=(112, 112, "cob"),
squeeze=1)
s2 = MV2.sin(s)
if zero:
s2 -= s2
if zero:
s -= s
if bigvalues:
gm.levels = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 1.e36]
if transparent:
cmap = self.x.createcolormap()
for i in range(256): # tweaks all colors
cmap.setcolorcell(i, 100., 0, 0, i / 2.55)
self.x.setcolormap(cmap)
if gm_type in ["vector", "streamline"]:
gm.linecolor = [100, 0, 0, 50.]
elif gm_type in ["yxvsx", "xyvsy", "yvsx", "scatter", "1d"]:
gm.linecolor = [100, 0, 0, 50.]
gm.markercolor = [100, 0, 0, 50.]
if gm_type in ["vector", "streamline"]:
if gm_type == "vector":
gm.scale = 4.
elif gm_type == "streamline":
gm.coloredbyvector = color
gm.evenlyspaced = evenlySpaced
if (not evenlySpaced):
gm.integratortype = vtk.vtkStreamTracer.RUNGE_KUTTA4
self.x.plot(u, v, gm, bg=self.bg)
elif gm_type in ["scatter", "xvsy"]:
self.x.plot(s, s2, gm, bg=self.bg)
else:
self.x.plot(s, gm, bg=self.bg)
fnm = "test_vcs_basic_%s" % gm_type.lower()
if gm_type == 'streamline':
if (color):
fnm += "_colored"
if (evenlySpaced):
fnm += "_evenlyspaced"
if mask:
fnm += "_masked"
elif bigvalues:
fnm += "_bigvalues"
if projtype != "default":
fnm += "_%s_proj" % projtype
if zero:
fnm += "_zero"
if transparent:
fnm += "_transparent"
fnm += nm_xtra
self.checkImage(fnm + '.png', threshold=20)
x.setantialiasing(0)
x.drawlogooff()
if bg:
x.setbgoutputdimensions(1200, 1091, units="pixels")
x.setcolormap("rainbow")
gm = vcs.createvector()
gm.scale = args.scale
nm_xtra = ""
xtra = {}
import cdms2
import os
f = cdms2.open(os.path.join(vcs.sample_data, "clt.nc"))
u = f("u")
v = f("v")
u = MV2.masked_greater(u, 35.0)[..., ::2, ::2]
v = MV2.masked_greater(v, 888.0)[..., ::2, ::2]
print u.max(), v.max(), u.min(), v.min()
# x.plot(U)
x.plot(u, v, gm, bg=bg)
ret = 0
if args.show:
pass
# x.interact()
else:
fnm = "test_vcs_vectors_missing"
if args.scale != 5.0:
fnm += "_%.1g" % args.scale
fnm += nm_xtra
x.png(fnm)
ret = regression.check_result_image(fnm + ".png", src, regression.defaultThreshold, cleanup=not args.keep)
import vcs
import sys
import cdms2
import vtk
import os
import MV2
x=vcs.init()
x.setcolormap("rainbow")
interact = True
f=cdms2.open(os.path.join(sys.prefix,'sample_data','clt.nc'))
s=f("clt")
s=MV2.masked_greater(s,67.)
#f=cdms2.open(os.path.join(sys.prefix,"sample_data","sampleCurveGrid4.nc"))
#s=f("sample")
tmpl = vcs.createtemplate()
#tmpl.data.x1=.001
#tmpl.data.x2=.5
#tmpl.data.y1=.0001
#tmpl.data.y2=.25
gm=vcs.createisoline()
gm=vcs.createboxfill()
gm=vcs.createisofill()
#gm.datawc_x1=-180
#gm.datawc_x2=0
#gm.datawc_y1=60
#gm.datawc_y2=65
gm.datawc_x1=-180.
gm.datawc_x2=180.
gm.datawc_y1=-90
if args.rg:
gm.datawc_x1=args.lon1
gm.datawc_x2=args.lon2
xtra["longitude"] = (args.lon1,args.lon2)
nm_xtra+="_%i_%i" % (args.lon1,args.lon2)
if args.rg:
nm_xtra+="_via_gm"
if gm_type=="meshfill":
f=cdms2.open(os.path.join(vcs.sample_data,'sampleCurveGrid4.nc'))
else:
f=cdms2.open(os.path.join(vcs.sample_data,'clt.nc'))
if gm_type=="vector":
u=f("u",**xtra)
v=f("v",**xtra)
if args.mask:
u=MV2.masked_greater(u,58.)
if args.zero:
u-=u
v-=v
elif gm_type=="meshfill":
s=f("sample",**xtra)
if args.mask:
s=MV2.masked_less(s,1150.)
elif args.bigvalues:
s[s < 1150] = 1e40
if args.zero:
s-=s
else:
s=f("clt",**xtra)
if args.mask:
s=MV2.masked_greater(s,78.)
import sys,os
import vcs
import sys
import cdms2
import vtk
import os
import MV2
bg = not False
x=vcs.init()
x.setcolormap("rainbow")
gm = vcs.createvector()
xtra = {}
f=cdms2.open(os.path.join(vcs.prefix,'sample_data','clt.nc'))
u=f("u",**xtra)
v=f("v",**xtra)
u=MV2.masked_greater(u,58.)
x.plot(u,v,gm,bg=bg)
x.png('test_vcs_basic_vector_masked.png')
