def regrid_5by5(rawOrAnomaly='Anomaly', filteredOrNot='Unfiltered'):
"""
Written By : Arulalan.T
Date : 22.07.2013
"""
inpath = os.path.join(processfilesPath, rawOrAnomaly, filteredOrNot)
for subName in os.listdir(inpath):
anopath = os.path.join(inpath, subName)
for anofile in os.listdir(anopath):
anofilename = anofile.split('.')[0]
varName = anofilename.split('_')[0]
if not anofilename.endswith('_5x5'):
anofilename += '_5x5'
# end of if not anofilename.endswith('_5x5'):
anoFilePath = os.path.join(anopath, anofile)
outfile = anofilename + '.nc'
outpath = os.path.join(anopath, outfile)
if os.path.isfile(outpath):
print "5x5 regridded file already exists for", outpath
continue
# end of if os.path.isfile(outpath):
grid5x5file = os.path.join(__curDir__, _5x5_ncfilename)
varName5x5 = _5x5_ncfilename.split('.')[0]
f = cdms2.open(grid5x5file)
grid5x5 = f[varName5x5].getGrid()
inf = cdms2.open(anoFilePath)
grid_infile = inf[varName].getGrid()
# Regridding the anomaly data
# Creating the horizontal lat,lon regrid
# Note that 'grid_infile' is the source and 'grid5x5' is the target
regridfunc = Horizontal(grid_infile, grid5x5)
anomalytime = inf[varName].getTime().asComponentTime()
print "The out path is ", outpath
loopCount = len(anomalytime)
preview = 0
for day in anomalytime:
if __Debug__:
print "The Date : ", day
data_5x5_regridded = regridfunc(inf(varName, time=day))
outf = cdms2.open(outpath, 'a')
outf.write(data_5x5_regridded)
outf.close()
# make memory free
del data_5x5_regridded
if __showStatusBar:
preview = statusbar(anomalytime.index(day), total=loopCount,
title='Regridding', prev=preview)
# end of if __showStatusBar:
# end of for day in anomalytime:
print
print "The 5x5 regrid for the variable '%s' is stored %s" % (varName, outpath)
inf.close()
f.close()
def linearInterpolation(A,I,levels=[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000], status=None):
"""
Linear interpolation
to interpolate a field from some levels to another set of levels
Value below "surface" are masked
Input
A : array to interpolate
I : interpolation field (usually Pressure or depth) from TOP (level 0) to BOTTOM (last level), i.e P value going up with each level
levels : levels to interplate to (same units as I), default levels are:[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000]
I and levels must have same units
Output
array on new levels (levels)
Examples:
A=interpolate(A,I,levels=[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000])
"""
try:
nlev=len(levels) # Number of pressure levels
except:
nlev=1 # if only one level len(levels) would breaks
levels=[levels,]
order=A.getOrder()
A=A(order='z...')
I=I(order='z...')
sh=list(I.shape)
nsigma=sh[0] #number of sigma levels
sh[0]=nlev
t=MV2.zeros(sh,typecode=MV2.float32)
sh2=I[0].shape
prev=-1
for ilev in range(nlev): # loop through pressure levels
if status is not None:
prev=genutil.statusbar(ilev,nlev-1.,prev)
lev=levels[ilev] # get value for the level
Iabv=MV2.ones(sh2,MV2.float)
Aabv=-1*Iabv # Array on sigma level Above
Abel=-1*Iabv # Array on sigma level Below
Ibel=-1*Iabv # Pressure on sigma level Below
Iabv=-1*Iabv # Pressure on sigma level Above
Ieq=MV2.masked_equal(Iabv,-1) # Area where Pressure == levels
for i in range(1,nsigma): # loop from second sigma level to last one
a = MV2.greater_equal(I[i], lev) # Where is the pressure greater than lev
b = MV2.less_equal(I[i-1],lev) # Where is the pressure less than lev
# Now looks if the pressure level is in between the 2 sigma levels
# If yes, sets Iabv, Ibel and Aabv, Abel
a=MV2.logical_and(a,b)
Iabv=MV2.where(a,I[i],Iabv) # Pressure on sigma level Above
Aabv=MV2.where(a,A[i],Aabv) # Array on sigma level Above
Ibel=MV2.where(a,I[i-1],Ibel) # Pressure on sigma level Below
Abel=MV2.where(a,A[i-1],Abel) # Array on sigma level Below
Ieq= MV2.where(MV2.equal(I[i],lev),A[i],Ieq)
val=MV2.masked_where(MV2.equal(Ibel,-1.),numpy.ones(Ibel.shape)*lev) # set to missing value if no data below lev if there is
tl=(val-Ibel)/(Iabv-Ibel)*(Aabv-Abel)+Abel # Interpolation
if ((Ieq.mask is None) or (Ieq.mask is MV22.nomask)):
tl=Ieq
else:
tl=MV2.where(1-Ieq.mask,Ieq,tl)
t[ilev]=tl.astype(MV2.float32)
ax=A.getAxisList()
autobnds=cdms2.getAutoBounds()
cdms2.setAutoBounds('off')
lvl=cdms2.createAxis(MV2.array(levels).filled())
cdms2.setAutoBounds(autobnds)
try:
lvl.units=I.units
except:
pass
lvl.id='plev'
try:
t.units=I.units
except:
pass
ax[0]=lvl
t.setAxisList(ax)
t.id=A.id
for att in A.listattributes():
setattr(t,att,getattr(A,att))
return t(order=order)
def logLinearInterpolation(A,P,levels=[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000],status=None):
"""
Log-linear interpolation
to convert a field from sigma levels to pressure levels
Value below surface are masked
Input
A : array on sigma levels
P : pressure field from TOP (level 0) to BOTTOM (last level)
levels : pressure levels to interplate to (same units as P), default levels are:[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000]
P and levels must have same units
Output
array on pressure levels (levels)
Examples:
A=logLinearInterpolation(A,P),levels=[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000])
"""
try:
nlev=len(levels) # Number of pressure levels
except:
nlev=1 # if only one level len(levels) would breaks
levels=[levels,]
order=A.getOrder()
A=A(order='z...')
P=P(order='z...')
sh=list(P.shape)
nsigma=sh[0] #number of sigma levels
sh[0]=nlev
t=MV2.zeros(sh,typecode=MV2.float32)
sh2=P[0].shape
prev=-1
for ilev in range(nlev): # loop through pressure levels
if status is not None:
prev=genutil.statusbar(ilev,nlev-1.,prev)
lev=levels[ilev] # get value for the level
Pabv=MV2.ones(sh2,MV2.float)
Aabv=-1*Pabv # Array on sigma level Above
Abel=-1*Pabv # Array on sigma level Below
Pbel=-1*Pabv # Pressure on sigma level Below
Pabv=-1*Pabv # Pressure on sigma level Above
Peq=MV2.masked_equal(Pabv,-1) # Area where Pressure == levels
for i in range(1,nsigma): # loop from second sigma level to last one
a=MV2.greater_equal(P[i], lev) # Where is the pressure greater than lev
b= MV2.less_equal(P[i-1],lev) # Where is the pressure less than lev
# Now looks if the pressure level is in between the 2 sigma levels
# If yes, sets Pabv, Pbel and Aabv, Abel
a=MV2.logical_and(a,b)
Pabv=MV2.where(a,P[i],Pabv) # Pressure on sigma level Above
Aabv=MV2.where(a,A[i],Aabv) # Array on sigma level Above
Pbel=MV2.where(a,P[i-1],Pbel) # Pressure on sigma level Below
Abel=MV2.where(a,A[i-1],Abel) # Array on sigma level Below
Peq= MV2.where(MV2.equal(P[i],lev),A[i],Peq)
val=MV2.masked_where(MV2.equal(Pbel,-1),numpy.ones(Pbel.shape)*lev) # set to missing value if no data below lev if there is
tl=MV2.log(val/Pbel)/MV2.log(Pabv/Pbel)*(Aabv-Abel)+Abel # Interpolation
if ((Peq.mask is None) or (Peq.mask is MV2.nomask)):
tl=Peq
else:
tl=MV2.where(1-Peq.mask,Peq,tl)
t[ilev]=tl.astype(MV2.float32)
ax=A.getAxisList()
autobnds=cdms2.getAutoBounds()
cdms2.setAutoBounds('off')
lvl=cdms2.createAxis(MV2.array(levels).filled())
cdms2.setAutoBounds(autobnds)
try:
lvl.units=P.units
except:
pass
lvl.id='plev'
try:
t.units=P.units
except:
pass
ax[0]=lvl
t.setAxisList(ax)
t.id=A.id
for att in A.listattributes():
setattr(t,att,getattr(A,att))
return t(order=order)
def linearInterpolation(A,
I,
levels=[
100000, 92500, 85000, 70000, 60000, 50000, 40000,
30000, 25000, 20000, 15000, 10000, 7000, 5000,
3000, 2000, 1000
],
status=None):
"""
Linear interpolation
to interpolate a field from some levels to another set of levels
Value below "surface" are masked
Input
A : array to interpolate
I : interpolation field (usually Pressure or depth) from TOP (level 0) to BOTTOM (last level), i.e P value going up with each level
levels : levels to interplate to (same units as I), default levels are:[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000]
I and levels must have same units
Output
array on new levels (levels)
Examples:
A=interpolate(A,I,levels=[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000])
"""
try:
nlev = len(levels) # Number of pressure levels
except:
nlev = 1 # if only one level len(levels) would breaks
levels = [
levels,
]
order = A.getOrder()
A = A(order='z...')
I = I(order='z...')
sh = list(I.shape)
nsigma = sh[0] #number of sigma levels
sh[0] = nlev
t = MV2.zeros(sh, typecode=MV2.float32)
sh2 = I[0].shape
prev = -1
for ilev in range(nlev): # loop through pressure levels
if status is not None:
prev = genutil.statusbar(ilev, nlev - 1., prev)
lev = levels[ilev] # get value for the level
Iabv = MV2.ones(sh2, MV2.float)
Aabv = -1 * Iabv # Array on sigma level Above
Abel = -1 * Iabv # Array on sigma level Below
Ibel = -1 * Iabv # Pressure on sigma level Below
Iabv = -1 * Iabv # Pressure on sigma level Above
Ieq = MV2.masked_equal(Iabv, -1) # Area where Pressure == levels
for i in range(1, nsigma): # loop from second sigma level to last one
a = MV2.greater_equal(
I[i], lev) # Where is the pressure greater than lev
b = MV2.less_equal(I[i - 1],
lev) # Where is the pressure less than lev
# Now looks if the pressure level is in between the 2 sigma levels
# If yes, sets Iabv, Ibel and Aabv, Abel
a = MV2.logical_and(a, b)
Iabv = MV2.where(a, I[i], Iabv) # Pressure on sigma level Above
Aabv = MV2.where(a, A[i], Aabv) # Array on sigma level Above
Ibel = MV2.where(a, I[i - 1],
Ibel) # Pressure on sigma level Below
Abel = MV2.where(a, A[i - 1], Abel) # Array on sigma level Below
Ieq = MV2.where(MV2.equal(I[i], lev), A[i], Ieq)
val = MV2.masked_where(
MV2.equal(Ibel, -1.),
numpy.ones(Ibel.shape) *
lev) # set to missing value if no data below lev if there is
tl = (val - Ibel) / (Iabv - Ibel) * (Aabv -
Abel) + Abel # Interpolation
if ((Ieq.mask is None) or (Ieq.mask is MV22.nomask)):
tl = Ieq
else:
tl = MV2.where(1 - Ieq.mask, Ieq, tl)
t[ilev] = tl.astype(MV2.float32)
ax = A.getAxisList()
autobnds = cdms2.getAutoBounds()
cdms2.setAutoBounds('off')
lvl = cdms2.createAxis(MV2.array(levels).filled())
cdms2.setAutoBounds(autobnds)
try:
lvl.units = I.units
except:
pass
lvl.id = 'plev'
try:
t.units = I.units
except:
pass
ax[0] = lvl
t.setAxisList(ax)
t.id = A.id
for att in A.listattributes():
setattr(t, att, getattr(A, att))
return t(order=order)
def logLinearInterpolation(A,
P,
levels=[
100000, 92500, 85000, 70000, 60000, 50000,
40000, 30000, 25000, 20000, 15000, 10000, 7000,
5000, 3000, 2000, 1000
],
status=None):
"""
Log-linear interpolation
to convert a field from sigma levels to pressure levels
Value below surface are masked
Input
A : array on sigma levels
P : pressure field from TOP (level 0) to BOTTOM (last level)
levels : pressure levels to interplate to (same units as P), default levels are:[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000]
P and levels must have same units
Output
array on pressure levels (levels)
Examples:
A=logLinearInterpolation(A,P),levels=[100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000])
"""
try:
nlev = len(levels) # Number of pressure levels
except:
nlev = 1 # if only one level len(levels) would breaks
levels = [
levels,
]
order = A.getOrder()
A = A(order='z...')
P = P(order='z...')
sh = list(P.shape)
nsigma = sh[0] #number of sigma levels
sh[0] = nlev
t = MV2.zeros(sh, typecode=MV2.float32)
sh2 = P[0].shape
prev = -1
for ilev in range(nlev): # loop through pressure levels
if status is not None:
prev = genutil.statusbar(ilev, nlev - 1., prev)
lev = levels[ilev] # get value for the level
Pabv = MV2.ones(sh2, MV2.float)
Aabv = -1 * Pabv # Array on sigma level Above
Abel = -1 * Pabv # Array on sigma level Below
Pbel = -1 * Pabv # Pressure on sigma level Below
Pabv = -1 * Pabv # Pressure on sigma level Above
Peq = MV2.masked_equal(Pabv, -1) # Area where Pressure == levels
for i in range(1, nsigma): # loop from second sigma level to last one
a = MV2.greater_equal(
P[i], lev) # Where is the pressure greater than lev
b = MV2.less_equal(P[i - 1],
lev) # Where is the pressure less than lev
# Now looks if the pressure level is in between the 2 sigma levels
# If yes, sets Pabv, Pbel and Aabv, Abel
a = MV2.logical_and(a, b)
Pabv = MV2.where(a, P[i], Pabv) # Pressure on sigma level Above
Aabv = MV2.where(a, A[i], Aabv) # Array on sigma level Above
Pbel = MV2.where(a, P[i - 1],
Pbel) # Pressure on sigma level Below
Abel = MV2.where(a, A[i - 1], Abel) # Array on sigma level Below
Peq = MV2.where(MV2.equal(P[i], lev), A[i], Peq)
val = MV2.masked_where(
MV2.equal(Pbel, -1),
numpy.ones(Pbel.shape) *
lev) # set to missing value if no data below lev if there is
tl = MV2.log(val / Pbel) / MV2.log(
Pabv / Pbel) * (Aabv - Abel) + Abel # Interpolation
if ((Peq.mask is None) or (Peq.mask is MV2.nomask)):
tl = Peq
else:
tl = MV2.where(1 - Peq.mask, Peq, tl)
t[ilev] = tl.astype(MV2.float32)
ax = A.getAxisList()
autobnds = cdms2.getAutoBounds()
cdms2.setAutoBounds('off')
lvl = cdms2.createAxis(MV2.array(levels).filled())
cdms2.setAutoBounds(autobnds)
try:
lvl.units = P.units
except:
pass
lvl.id = 'plev'
try:
t.units = P.units
except:
pass
ax[0] = lvl
t.setAxisList(ax)
t.id = A.id
for att in A.listattributes():
setattr(t, att, getattr(A, att))
return t(order=order)
def generateSurfaceTypeByRegionMask(mask,sftbyrgn=None,sftbyrgnmask=215,regions=range(201,223),maximum_regions_per_cell=4,extend_up_to=3,verbose=True):
"""
Maps a "regions" dataset onto a user provided land/sea mask or grid
Usage:
-----
mapped,found = generateSurfaceTypeByRegionMask(mask,sftbyrgn=None,sftbyrgnmask=None,regions=None,maximum_regions_per_cell=4,extend_up_to=3,verbose=True)
Input:
-----
mask User provided land/sea mask (100/0) or grid (the land/sea mask will be generated automagically) which will be mapped using the "sftbyrgn" internal dataset (will generate a land/sea mask for you)
sftbyrgn Mask you wish to map onto your grid (if None uses internal "sftbyrgn" dataset (old ezget type))
sftbyrgnmask Land/sea mask for sftbyrgn (or a number specifying value limits for sftbyrgn which indicates land/sea threshold (greater values are land) - see URL below for integer region map)
regions Numbers from sftbyrgn array that you want to map onto mask (integers from 201-222)
maximum_regions_per_cell Maximum number of regions considered for a single cell
extend_up_to How many grid cells around a cell can we extend to identify a guess
verbose Prints to the screen what's going on (default is True)
Output:
-----
mapped Mapped input grid/mask using provided (or default) regions - sftbyrgn -> user provided grid/mask
found Matrix containing number of regions matched for each output cell
Notes:
-----
- More detailed information, including a region map and tabulated region numbers are available from http://www-pcmdi.llnl.gov/publications/pdf/34.pdf
"""
cdat_info.pingPCMDIdb("cdat","cdutil.generateSurfaceTypeByRegionMask")
## OK first determine which regions are available
## Must be integer values
if isinstance(mask,cdms2.grid.TransientRectGrid):
mask = cdutil.generateLandSeaMask(mask)*100.
if sftbyrgn is None:
sftbyrgn = cdms2.open(os.path.join(cdat_info.get_prefix(),'share','cdutil','sftbyrgn.nc'))('sftbyrgn')
if regions is None:
if verbose: print 'Preparing regions'
#regions = range(201,223)
regions = []
for i in range(0,10000):
genutil.statusbar(i,9999)
c = float(MV2.sum(MV2.ravel(MV2.equal(sftbyrgn,i)),0))
if c != 0: regions.append(i)
if verbose: print 'Regions:',regions
## If no mask passed fr sftbyrgn, assumes everything greater 5000 is land)
if isinstance(sftbyrgnmask,int):
split = sftbyrgnmask
n = MV2.maximum(mask)
sftbyrgnmask = MV2.greater_equal(sftbyrgn,sftbyrgnmask)*n
else:
split = MV2.maximum(sftbyrgnmask)/2.
## Now guess the type for each regions
keys = {}
## ## Nice way to do it
## for r in regions:
## c=MV2.not_equal(sftbyrgn,r)
## c=MV2.masked_where(c,sftbyrgnmask)
## n=MV2.count(c)
## c=float(MV2.sum(MV2.ravel(c),0)/n)
## print r,c,n
## keys[r]=c
## Fast but not so "general" way to do it
for r in regions:
if r< split:
keys[r] = 0.
else:
keys[r] = 100.
sh = list(mask.shape)
sh.insert(0,maximum_regions_per_cell)
potential = MV2.ones(sh,dtype='d')*-999
potential_reg = MV2.ones(sh,dtype='d')*-999
g1 = sftbyrgn.getGrid()
g2 = mask.getGrid()
r1 = regrid2.Horizontal(g1,g2)
w = cdutil.area_weights(sftbyrgn)
if verbose: print 'First pass'
itmp = 0.
for ireg in keys.keys():
genutil.statusbar(itmp,len(keys.keys())-1)
itmp += 1.
c = MV2.equal(sftbyrgn,ireg)
w2 = 1.-c*w
s2,w3 = r1(sftbyrgn,mask=w2.filled(),returnTuple=1)
c2 = MV2.equal(mask,keys[ireg])
loop(potential,potential_reg,c2,w3,ireg)
found = MV2.zeros(sh[1:],typecode='f')
for i in range(maximum_regions_per_cell):
found = found+MV2.not_equal(potential[i],-999)
sh2 = list(sh)
for k in range(extend_up_to):
sh2[1] = sh[1]+2*(k+1)
sh2[2] = sh[2]+2*(k+1)
## Form the possible i/j couples !
s = MV2.sum(MV2.ravel(MV2.equal(potential[0],-999)),0)
if verbose: print 'Expanding up to',k+1,'cells while trying to fix',s,'cells'
#if dump:
#f=cdms2.open('tmp_'+str(k)+'.nc','w')
#f.write(sumregions(potential_reg,potential).astype('f'),id='sftbyrgn',axes=mask.getAxisList())
#f.close()
#g=sumregions(potential_reg,potential).astype('d')
#g=MV2.masked_equal(g,-999)
#g=MV2.greater(g,4999)*100.
#g=MV2.absolute(mask-g)
#g=MV2.masked_equal(g,0.)
#print 'Number of differences:',MV2.count(g)
if float(s) != 0:
c0 = MV2.equal(potential[0],-999)
couples = []
sft2 = MV2.zeros(sh2[1:],dtype='d')-888.
sft2[k+1:-k-1,k+1:-k-1] = mask
for i in range(-k-1,k+2):
for j in range(-k-1,k+2):
if abs(i)>k or abs(j)>k: couples.append([i,j])
ntot = len(keys.keys())*len(couples)-1
itmp = 0
for ireg in keys.keys():
c = MV2.equal(sftbyrgn,ireg)
w2 = 1.-c*w
s2,w3 = r1(sftbyrgn,mask=w2.filled(),returnTuple=1)
w4 = MV2.zeros(sh2[1:],typecode='d')
w4[k+1:-k-1,k+1:-k-1] = w3
for i,j in couples:
if verbose: genutil.statusbar(itmp,ntot)
itmp += 1.
c2 = MV2.equal(sft2[j+k+1:j+k+1+sh[1],i+k+1:i+k+1+sh[2]],keys[ireg])
c3 = MV2.equal(sft2[j+k+1:j+k+1+sh[1],i+k+1:i+k+1+sh[2]],mask)
c2 = MV2.logical_and(c2,c3)
c2 = MV2.logical_and(c2,c0)
loop(potential,potential_reg,c2,w4[j+k+1:j+k+1+sh[1],i+k+1:i+k+1+sh[2]],ireg)
found = MV2.where(MV2.equal(potential[0],-999),found-1,found)
out = sumregions(potential_reg,potential)
out.setAxisList(mask.getAxisList())
out.id = 'sftbyrgn'
out = out.astype('i')
out.missing_value = -999
found.setAxisList(mask.getAxisList())
found.id = 'found'
found = found.astype('i')
found.missing_value = -999
del(out.name)
del(found.name)
return out,found
def dailyClimatology(varName, infile, outfile, leapday=False, **kwarg):
"""
dailyClimatology : It will create the daily climatolgy and stored
in the outfile.
Inputs:
varName : variable name to extract from the input file
infile : Input file absolute path
outfile : outfile absolute path (will be created in write mode)
leapday : False | True
If it is True, then it will create 366 days climatolgy
(include 29th feb)
If it is False, then it will create 365 days climatolgy
KWargs:
ovar : out varName. If it is passed then the climatology variable
name will be set as ovar. Otherwise the input varName will
be set to it.
squeeze : 1 (it will squeeze single dimension in the climatolgy)
todo : need to set year 1 for 366 days climatology.
Written By : Arulalan.T
Date : 13.08.2013
"""
ovar = kwarg.get('ovar', None)
squeeze = kwarg.get('squeeze', 1)
timobj = TimeUtility()
if leapday:
tlen = 366
year = 4
cunits = 'days since 4-1-1'
# change the above units into 1-1-1 if cdtime.ClimLeapcaleder Bug fixed
else:
tlen = 365
year = 1
cunits = 'days since 1-1-1'
# end of if leapday:
f = cdms2.open(infile)
latAxis = f[varName].getLatitude()
lonAxis = f[varName].getLongitude()
levAxis = f[varName].getLevel()
clim = numpy.array([])
ctimeAxisVal = []
preview = 0
for day in range(tlen):
cdate = timobj._dayCount2Comp(day, year)
data = timobj.getSameDayData(varName, infile, day=cdate.day,
mon=cdate.month, squeeze=squeeze)
avg = cdutil.averager(data, axis='t', weights='weighted')
### the above avg works fine.
### need to test the above method with missing values.
### If it fails, the below commented lines should works correctly.
# dataSum = cdutil.averager(data, axis='t', action='sum') # weights=data.mask) #'weighted')
# ### This count will counts the no of False in the masked array
# ### with same shape. i.e. returns total no of elements
# count = data.count(axis=0)
# avg = dataSum / count
# make memory free
del data
fillvalue = avg.fill_value
if clim.shape == (0,):
clim = avg.filled()
else:
clim = numpy.concatenate((clim, avg.filled()))
# end of if clim.shape == (1,):
ctimeAxisVal.append(day)
# make memory free
del avg
if __showStatusBar:
preview = statusbar(day, total=tlen,
title='Climatology', prev=preview)
# Either averager function or setSlabTimeBoundsDaily
# fucnction will print 'convention.getDsetnodeAuxAxisIds'
# msg. So status bar unable to retain the same line.
# To avoid that I added the below line.
# The character '\x1b' is the escape character.
# The character [1A moves the cursor to 1 previous line.
# (i.e. at the end of the previous line \n char).
# The character [80D moves cursor to 80 positions to the left.
# The character [K clears the line.
# Refer ANSCII escape sequence for more details.
sys.stdout.write("\x1b[1A" + "\x1b[80D" + "\x1b[K")
# end of if __showStatusBar:
# end of for day in range(tlen):
print
climMask = (clim == fillvalue)
clim = cdms2.createVariable(clim)
if climMask.any():
clim.mask = climMask
# end of if climMask.any():
if ovar:
clim.id = ovar
else:
clim.id = varName
# end of if ovar:
# create climatolgy timeAxis
ctimeAxis = cdms2.createAxis(ctimeAxisVal, id='time')
ctimeAxis.units = cunits
ctimeAxis.designateTime()
#ctimeAxis.calendar = cdtime.ClimLeapCalendar ????
axislist = [axis for axis in [ctimeAxis, levAxis, latAxis, lonAxis] if axis]
if squeeze:
# squeezing the axis lists
axislist = [axis for axis in axislist if len(axis[:]) > 1]
# end of if squeeze:
# set the axis information to the clim data
clim.setAxisList(axislist)
cdutil.setSlabTimeBoundsDaily(clim)
# save/write the climatolgy data
outf = cdms2.open(outfile, 'w')
outf.write(clim)
outf.close()
f.close()
print "The Climatology data year is",
if leapday:
print 4
else:
print 1
print "The climatolgy data has written into", outfile
def process(self, data):
## if self.symetric:
## data = symetric(data)
# Make sure we have an even number of time steps
t = data.getTime()
# length of time axis
nt = len(t)
if nt % 2 != 0:
print "Warning time wasn't even, removed last time step"
data = data[:-1]
t = data.getTime() ## get the new time axis
nt = len(t)
if len(t) < self._NTSub:
raise Exception, "Error your data must have at least %i time steps, adjust frequency (currently: %i/day) or number_of_days (currently: %i processed at once) to reach that limit, or get more data" % (
self._NTSub, self.frequency, self.number_of_days)
## Computes PP, number of sub-domain
PP = float(nt - self._NTSub) / self._NShift + 1
PP = int(PP)
## Number of longitudes
lons = data.getLongitude()
NL = len(lons)
tt = cdms2.createAxis(numpy.arange(self._NTSub), id='sub_time')
## Should redo that with just an arange (eventually...)!!!
## Frequencies in cycles/day
ff = numpy.arange(0, self._NTSub + 1, 1, numpy.float)
for i in range(1, self._NTSub + 2):
ff[i -
1] = float(i - 1 - self._NTSub / 2.) * self.frequency / float(
self._NTSub)
## Should redo that with just an arange (eventually...)!!!
## Wave numbers
ss = numpy.arange(0, NL + 1, 1, numpy.float)
for i in range(1, NL + 2):
ss[i - 1] = float(i - 1 - NL / 2.)
## print 'Frequencies:',ff
## print 'Wave numbers:',ss
## Ok, we now do the real stuff
## Creates the array of powers (Number of subtimes,fqcy,wave_numbers,latitudes)
lats = data.getLatitude()
Power = numpy.zeros((PP, self._NTSub + 1, NL + 1, len(lats)),
numpy.float)
## LOOP through time sub domains
prev = 0 # initialize the scrolling bar
for Pcount in range(PP):
if PP > 1:
prev = genutil.statusbar(float(Pcount),
PP - 1,
prev=prev,
tk=self.tkbar)
## Get the time subdomain
EEo = data[Pcount * self._NShift:Pcount * self._NShift +
self._NTSub](order='tx...')
## First does the symetric/antisymetric thing if needed
if self.symetric: EEo = symetrick(EEo)
## Now detrending
## Step 1- Get the slope and intercept
slope, intercept = genutil.statistics.linearregression(
EEo, nointercept=0)
## Step 2- remove the trend
## Step 2a: Create an array with the time values
a = EEo.getTime()
A = MV2.array(a[:], typecode='d')
A.setAxis(0, a)
## Step 2b: "Grows" it so it has the same shape than data
A, EEo = genutil.grower(A, EEo)
## Step 2c: Actually remove the trend
EE = EEo - A * slope - intercept
## we don't need A,EEo,slope,intercept anymore
del (EEo)
del (slope)
del (intercept)
del (A)
## Remove the time mean
mean = MV2.average(EE, 0)
EE = EE - mean
del (mean) # could be big in memory
## Tapering time...
tapertozero(EE, 1, len(EE) - 1, 5 * self.frequency)
## OK here Wheeler has some windowing on longitude, but it's commented out
## I'll pass it for now
## Ok the actuall FFT work
EE = numpy.fft.fft2(EE, axes=(1, 0)) / NL / self._NTSub
## OK NOW THE LITTLE MAGIC WITH REORDERING !
A = numpy.absolute(EE[0:self._NTSub / 2 + 1, 1:NL / 2 + 1])**2
B = numpy.absolute(EE[self._NTSub / 2:self._NTSub,
1:NL / 2 + 1])**2
C = numpy.absolute(EE[self._NTSub / 2:self._NTSub,
0:NL / 2 + 1])**2
D = numpy.absolute(EE[0:self._NTSub / 2 + 1, 0:NL / 2 + 1])**2
Power[Pcount, self._NTSub / 2:, :NL / 2] = A[:, ::-1]
Power[Pcount, :self._NTSub / 2, :NL / 2] = B[:, ::-1]
Power[Pcount, self._NTSub / 2 + 1:, NL / 2:] = C[::-1, :]
Power[Pcount, :self._NTSub / 2 + 1, NL / 2:] = D[::-1, :]
## End of Pcount loop
if self.tkbar and PP > 1:
prev[1].destroy()
prev[0].destroy()
## Now generates the decorations
## first the time axis (subdomains)
vals = []
bounds = []
pp = 0
for i in range(0, len(t) - self._NShift, self._NShift):
st = t.subAxis(i, i + self._NTSub)
if len(st[:]) == self._NTSub:
pp += 1
vals.append((st[0] + st[-1]) / 2.)
bds = st.getBounds()
#print 'Bounds:',bds
if bds is None:
raise ValueError, "Data need to have bounds on time dimension"
else:
bounds.append([bds[0][0], bds[-1][1]])
## Convert lists to arrays
vals = numpy.array(vals)
bounds = numpy.array(bounds)
## Creates the time axis
dumt = cdms2.createAxis(vals, bounds=bounds)
dumt.id = 'time'
dumt.units = t.units
dumt.designateTime()
dumt.setCalendar(t.getCalendar())
## Create the frequencies axis
T = cdms2.createAxis(ff)
T.id = 'frequency'
T.units = 'cycles per day'
## Create the wave numbers axis
S = cdms2.createAxis(ss)
S.id = 'planetaryzonalwavenumber'
S.units = '-'
## Makes it an MV2 with axis and id (id come sfrom orignal data id)
Power = MV2.array(Power,
axes=(dumt, T, S, lats),
id=data.id + '_' + 'power')
## Adds a long name attribute
Power.longname = 'Real power spectrum for the many different parts (i.e. over separate time divisions)'
## And return the whole thing ordered 'time', 'latitude', 'frequencies','wavenumbers'
return Power(order='ty...')
def linearInterpolation(A,
Idx,
levels=[
100000, 92500, 85000, 70000, 60000, 50000, 40000,
30000, 25000, 20000, 15000, 10000, 7000, 5000,
3000, 2000, 1000
],
status=None,
axis='z'):
"""
Linear interpolation to interpolate a field from some levels to another set of levels
Values below "surface" are masked.
:param A: array to interpolate
:type A:
:param I: interpolation field (usually Pressure or depth) from TOP (level 0) to BOTTOM (last level)
i.e P value going up with each level.
:type I:
:param levels: levels to interpolate to (same units as I).
Default levels:[100000, 92500, 85000, 70000, 60000, 50000, 40000,
30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000]
:type levels:
:param axis: Axis over which to do the linear interpolation.
Can provide either an int representing axis index, or the axis name.
Default: 'z'.
:type axis: str or int
.. note::
I and levels must have same units
:returns: array on new levels (levels)
:Examples:
.. doctest:: vertical_linearInterpolation
>>> A=interpolate(A,I) # interpolates A over default levels
"""
try:
nlev = len(levels) # Number of pressure levels
except BaseException:
nlev = 1 # if only one level len(levels) would breaks
levels = [
levels,
]
order = A.getOrder()
A = A(order='%s...' % axis)
Idx = Idx(order='%s...' % axis)
sh = list(Idx.shape)
nsigma = sh[0] # number of sigma levels
sh[0] = nlev
t = MV2.zeros(sh, typecode=MV2.float32)
sh2 = Idx[0].shape
prev = -1
for ilev in range(nlev): # loop through pressure levels
if status is not None:
prev = genutil.statusbar(ilev, nlev - 1., prev)
lev = levels[ilev] # get value for the level
Iabv = MV2.ones(sh2, MV2.float)
Aabv = -1 * Iabv # Array on sigma level Above
Abel = -1 * Iabv # Array on sigma level Below
Ibel = -1 * Iabv # Pressure on sigma level Below
Iabv = -1 * Iabv # Pressure on sigma level Above
Ieq = MV2.masked_equal(Iabv, -1) # Area where Pressure == levels
for i in range(1, nsigma): # loop from second sigma level to last one
a = MV2.greater_equal(
Idx[i], lev) # Where is the pressure greater than lev
b = MV2.less_equal(Idx[i - 1],
lev) # Where is the pressure less than lev
# Now looks if the pressure level is in between the 2 sigma levels
# If yes, sets Iabv, Ibel and Aabv, Abel
a = MV2.logical_and(a, b)
Iabv = MV2.where(a, Idx[i], Iabv) # Pressure on sigma level Above
Aabv = MV2.where(a, A[i], Aabv) # Array on sigma level Above
Ibel = MV2.where(a, Idx[i - 1],
Ibel) # Pressure on sigma level Below
Abel = MV2.where(a, A[i - 1], Abel) # Array on sigma level Below
Ieq = MV2.where(MV2.equal(Idx[i], lev), A[i], Ieq)
val = MV2.masked_where(MV2.equal(Ibel, -1.),
numpy.ones(Ibel.shape) * lev)
# set to missing value if no data below lev if
# there is
tl = (val - Ibel) / (Iabv - Ibel) * \
(Aabv - Abel) + Abel # Interpolation
if ((Ieq.mask is None) or (Ieq.mask is MV2.nomask)):
tl = Ieq
else:
tl = MV2.where(1 - Ieq.mask, Ieq, tl)
t[ilev] = tl.astype(MV2.float32)
ax = A.getAxisList()
autobnds = cdms2.getAutoBounds()
cdms2.setAutoBounds('off')
lvl = cdms2.createAxis(MV2.array(levels).filled())
cdms2.setAutoBounds(autobnds)
try:
lvl.units = Idx.units
except BaseException:
pass
lvl.id = 'plev'
try:
t.units = Idx.units
except BaseException:
pass
ax[0] = lvl
t.setAxisList(ax)
t.id = A.id
for att in A.listattributes():
setattr(t, att, getattr(A, att))
return t(order=order)
# Adapted for numpy/ma/cdms2 by convertcdms.py
from genutil import statusbar
import os,sys,cdutil
n=100
try:
prev=0
for i in range(n):
prev=statusbar(i,n-1,prev=prev)
except:
raise "Error simple statusbar died...."
print
try:
prev=0
for i in range(n):
prev=statusbar(i,float(n-1),prev=prev,tk=1)
except:
raise "Error simple tk statusbar died...."
import cdutil,cdms2 as cdms
f=cdms.open(os.path.join(cdms.__path__[0],'..','..','..','..','sample_data','clt.nc'))
s=f('clt')
cdutil.setTimeBoundsMonthly(s)
try:
ac=cdutil.ANNUALCYCLE( s , statusbar=1 )
except:
raise "Error statusbar of extractions died (not tk)...."
try:
ac=cdutil.ANNUALCYCLE.climatology( s , statusbar=1 )
def generateSurfaceTypeByRegionMask(mask,
sftbyrgn=None,
sftbyrgnmask=215,
regions=range(201, 223),
maximum_regions_per_cell=4,
extend_up_to=3,
verbose=True):
""" Maps a "types" dataset onto a landsea mask
Usage:
mapped,found = generateSurfaceTypeByRegionMask(mask,sftbyrgn,sftbyrgnmask=None,regions=None,maximum_regions_per_cell=4,extend_up_to=3,verbode=True)
Input:
mask : land/sea mask (100/0) onto you wish to map our grid (will generate a ld/sea mask for you)
sftbyrgn: mask you wish to map
if None then uses our own "sftbyrgn" dataset (old ezget type)
sftbyrgnmask: land/sea mask for sftbyrgn
or a number specifying limit in values of sftbygrn
which indicate the threshold land/sea (greater values are land)
regions: Numbers from sftbyrgn array that you want to map onto mask
maximum_regions_per_cell: maximum number f regions concidered in a cell
extend_up_to : how many grid cells away around a cell can we extent to identify a guess
verbose: prints to the screen what's going on (default is True)
Output:
mapped : mapped input mask
found : ???
"""
## OK first determine which regions are available
## Must be integer values
if isinstance(mask, cdms2.grid.TransientRectGrid):
mask = cdutil.generateLandSeaMask(mask) * 100.
if sftbyrgn is None:
sftbyrgn = cdms2.open(
os.path.join(sys.prefix, 'sample_data', 'sftbyrgn.nc'))('sftbyrgn')
if regions is None:
if verbose: print 'Preparing regions'
## regions = range(201,223)
regions = []
for i in range(0, 10000):
genutil.statusbar(i, 9999)
c = float(MV2.sum(MV2.ravel(MV2.equal(sftbyrgn, i)), 0))
if c != 0: regions.append(i)
if verbose: print 'Regions:', regions
## If no mask passed fr sftbyrgn, assumes everything greater 5000 is land)
if isinstance(sftbyrgnmask, int):
split = sftbyrgnmask
n = MV2.maximum(mask)
sftbyrgnmask = MV2.greater_equal(sftbyrgn, sftbyrgnmask) * n
else:
split = MV2.maximum(sftbyrgnmask) / 2.
## Now guess the type for each regions
keys = {}
## ## Nice way to do it
## for r in regions:
## c=MV2.not_equal(sftbyrgn,r)
## c=MV2.masked_where(c,sftbyrgnmask)
## n=MV2.count(c)
## c=float(MV2.sum(MV2.ravel(c),0)/n)
## print r,c,n
## keys[r]=c
## Fast but not so "general" way to do it
for r in regions:
if r < split:
keys[r] = 0.
else:
keys[r] = 100.
sh = list(mask.shape)
sh.insert(0, maximum_regions_per_cell)
potential = MV2.ones(sh, dtype='d') * -999
potential_reg = MV2.ones(sh, dtype='d') * -999
g1 = sftbyrgn.getGrid()
g2 = mask.getGrid()
r1 = regrid2.Regridder(g1, g2)
w = cdutil.area_weights(sftbyrgn)
if verbose: print 'First pass'
itmp = 0.
for ireg in keys.keys():
genutil.statusbar(itmp, len(keys.keys()) - 1)
itmp += 1.
c = MV2.equal(sftbyrgn, ireg)
w2 = 1. - c * w
s2, w3 = r1(sftbyrgn, mask=w2.filled(), returnTuple=1)
c2 = MV2.equal(mask, keys[ireg])
loop(potential, potential_reg, c2, w3, ireg)
found = MV2.zeros(sh[1:], typecode='f')
for i in range(maximum_regions_per_cell):
found = found + MV2.not_equal(potential[i], -999)
sh2 = list(sh)
for k in range(extend_up_to):
sh2[1] = sh[1] + 2 * (k + 1)
sh2[2] = sh[2] + 2 * (k + 1)
## Form the possible i/j couples !
s = MV2.sum(MV2.ravel(MV2.equal(potential[0], -999)), 0)
if verbose:
print 'Expanding up to', k + 1, 'cells while trying to fix', s, 'cells'
## if dump:
## f=cdms2.open('tmp_'+str(k)+'.nc','w')
## f.write(sumregions(potential_reg,potential).astype('f'),id='sftbyrgn',axes=mask.getAxisList())
## f.close()
## g=sumregions(potential_reg,potential).astype('d')
## g=MV2.masked_equal(g,-999)
## g=MV2.greater(g,4999)*100.
## g=MV2.absolute(mask-g)
## g=MV2.masked_equal(g,0.)
## print 'Number of differences:',MV2.count(g)
if float(s) != 0:
c0 = MV2.equal(potential[0], -999)
couples = []
sft2 = MV2.zeros(sh2[1:], dtype='d') - 888.
sft2[k + 1:-k - 1, k + 1:-k - 1] = mask
for i in range(-k - 1, k + 2):
for j in range(-k - 1, k + 2):
if abs(i) > k or abs(j) > k: couples.append([i, j])
ntot = len(keys.keys()) * len(couples) - 1
itmp = 0
for ireg in keys.keys():
c = MV2.equal(sftbyrgn, ireg)
w2 = 1. - c * w
s2, w3 = r1(sftbyrgn, mask=w2.filled(), returnTuple=1)
w4 = MV2.zeros(sh2[1:], typecode='d')
w4[k + 1:-k - 1, k + 1:-k - 1] = w3
for i, j in couples:
if verbose: genutil.statusbar(itmp, ntot)
itmp += 1.
c2 = MV2.equal(
sft2[j + k + 1:j + k + 1 + sh[1],
i + k + 1:i + k + 1 + sh[2]], keys[ireg])
c3 = MV2.equal(
sft2[j + k + 1:j + k + 1 + sh[1],
i + k + 1:i + k + 1 + sh[2]], mask)
c2 = MV2.logical_and(c2, c3)
c2 = MV2.logical_and(c2, c0)
loop(
potential, potential_reg, c2,
w4[j + k + 1:j + k + 1 + sh[1],
i + k + 1:i + k + 1 + sh[2]], ireg)
found = MV2.where(MV2.equal(potential[0], -999), found - 1, found)
out = sumregions(potential_reg, potential)
out.setAxisList(mask.getAxisList())
found.setAxisList(mask.getAxisList())
found = found.astype('i')
found.missing_value = -999
found.id = 'found'
out.id = 'sftbyrgn'
out = out.astype('i')
out.missing_value = -999
del (out.name)
del (found.name)
return out, found
def generateSurfaceTypeByRegionMask(mask,sftbyrgn=None,sftbyrgnmask=215,regions = range(201,223), maximum_regions_per_cell=4,extend_up_to=3,verbose=True):
""" Maps a "types" dataset onto a landsea mask
Usage:
mapped,found = generateSurfaceTypeByRegionMask(mask,sftbyrgn,sftbyrgnmask=None,regions=None,maximum_regions_per_cell=4,extend_up_to=3,verbode=True)
Input:
mask : land/sea mask (100/0) onto you wish to map our grid (will generate a ld/sea mask for you)
sftbyrgn: mask you wish to map
if None then uses our own "sftbyrgn" dataset (old ezget type)
sftbyrgnmask: land/sea mask for sftbyrgn
or a number specifying limit in values of sftbygrn
which indicate the threshold land/sea (greater values are land)
regions: Numbers from sftbyrgn array that you want to map onto mask
maximum_regions_per_cell: maximum number f regions concidered in a cell
extend_up_to : how many grid cells away around a cell can we extent to identify a guess
verbose: prints to the screen what's going on (default is True)
Output:
mapped : mapped input mask
found : ???
"""
## OK first determine which regions are available
## Must be integer values
if isinstance(mask, cdms2.grid.TransientRectGrid):
mask = cdutil.generateLandSeaMask(mask)*100.
if sftbyrgn is None:
sftbyrgn = cdms2.open(os.path.join(sys.prefix,'sample_data','sftbyrgn.nc'))('sftbyrgn')
if regions is None:
if verbose: print 'Preparing regions'
## regions = range(201,223)
regions=[]
for i in range(0,10000):
genutil.statusbar(i,9999)
c=float(MV2.sum(MV2.ravel(MV2.equal(sftbyrgn,i)),0))
if c!=0: regions.append(i)
if verbose: print 'Regions:',regions
## If no mask passed fr sftbyrgn, assumes everything greater 5000 is land)
if isinstance(sftbyrgnmask,int):
split = sftbyrgnmask
n=MV2.maximum(mask)
sftbyrgnmask=MV2.greater_equal(sftbyrgn,sftbyrgnmask)*n
else:
split = MV2.maximum(sftbyrgnmask)/2.
## Now guess the type for each regions
keys={}
## ## Nice way to do it
## for r in regions:
## c=MV2.not_equal(sftbyrgn,r)
## c=MV2.masked_where(c,sftbyrgnmask)
## n=MV2.count(c)
## c=float(MV2.sum(MV2.ravel(c),0)/n)
## print r,c,n
## keys[r]=c
## Fast but not so "general" way to do it
for r in regions:
if r< split:
keys[r]=0.
else:
keys[r]=100.
sh=list(mask.shape)
sh.insert(0,maximum_regions_per_cell)
potential=MV2.ones(sh,dtype='d')*-999
potential_reg=MV2.ones(sh,dtype='d')*-999
g1=sftbyrgn.getGrid()
g2=mask.getGrid()
r1=regrid2.Regridder(g1,g2)
w=cdutil.area_weights(sftbyrgn)
if verbose: print 'First pass'
itmp=0.
for ireg in keys.keys():
genutil.statusbar(itmp,len(keys.keys())-1)
itmp+=1.
c=MV2.equal(sftbyrgn,ireg)
w2=1.-c*w
s2,w3=r1(sftbyrgn,mask=w2.filled(),returnTuple=1)
c2=MV2.equal(mask,keys[ireg])
loop(potential,potential_reg,c2,w3,ireg)
found=MV2.zeros(sh[1:],typecode='f')
for i in range(maximum_regions_per_cell):
found=found+MV2.not_equal(potential[i],-999)
sh2=list(sh)
for k in range(extend_up_to):
sh2[1]=sh[1]+2*(k+1)
sh2[2]=sh[2]+2*(k+1)
## Form the possible i/j couples !
s=MV2.sum(MV2.ravel(MV2.equal(potential[0],-999)),0)
if verbose: print 'Expanding up to',k+1,'cells while trying to fix',s,'cells'
## if dump:
## f=cdms2.open('tmp_'+str(k)+'.nc','w')
## f.write(sumregions(potential_reg,potential).astype('f'),id='sftbyrgn',axes=mask.getAxisList())
## f.close()
## g=sumregions(potential_reg,potential).astype('d')
## g=MV2.masked_equal(g,-999)
## g=MV2.greater(g,4999)*100.
## g=MV2.absolute(mask-g)
## g=MV2.masked_equal(g,0.)
## print 'Number of differences:',MV2.count(g)
if float(s)!=0:
c0=MV2.equal(potential[0],-999)
couples=[]
sft2=MV2.zeros(sh2[1:],dtype='d')-888.
sft2[k+1:-k-1,k+1:-k-1]=mask
for i in range(-k-1,k+2):
for j in range(-k-1,k+2):
if abs(i)>k or abs(j)>k: couples.append([i,j])
ntot=len(keys.keys())*len(couples)-1
itmp=0
for ireg in keys.keys():
c=MV2.equal(sftbyrgn,ireg)
w2=1.-c*w
s2,w3=r1(sftbyrgn,mask=w2.filled(),returnTuple=1)
w4=MV2.zeros(sh2[1:],typecode='d')
w4[k+1:-k-1,k+1:-k-1]=w3
for i,j in couples:
if verbose: genutil.statusbar(itmp,ntot)
itmp+=1.
c2=MV2.equal(sft2[j+k+1:j+k+1+sh[1],i+k+1:i+k+1+sh[2]],keys[ireg])
c3=MV2.equal(sft2[j+k+1:j+k+1+sh[1],i+k+1:i+k+1+sh[2]],mask)
c2=MV2.logical_and(c2,c3)
c2=MV2.logical_and(c2,c0)
loop(potential,potential_reg,c2,w4[j+k+1:j+k+1+sh[1],i+k+1:i+k+1+sh[2]],ireg)
found=MV2.where(MV2.equal(potential[0],-999),found-1,found)
out=sumregions(potential_reg,potential)
out.setAxisList(mask.getAxisList())
found.setAxisList(mask.getAxisList())
found=found.astype('i')
found.missing_value=-999
found.id='found'
out.id='sftbyrgn'
out=out.astype('i')
out.missing_value=-999
del(out.name)
del(found.name)
return out,found
def genDailyAnomalyFiles(dailyAnomalyPath, modelType, modelHour, year,
climRegridPath, climPFileName, climatologyYear, **kwarg):
"""
:func:`genDailyAnomalyFiles` : It should calculate daily anomaly
from the daily analysis and daily climatology for the current day.
i.e. daily model analysis - daily climatology
Finally stores it as nc files in corresponding directory path
which are passed in this function args.
Do the same procedure for the model forecast hours anomaly.
i.e. daily model forecast - daily climatology
Inputs : dailyAnomalyPath is the absolute path where the processed mean
anomaly nc files are going to store.
modelType is either 'a' for 'analysis' or 'f' for 'forecast'.
modelHour is the forecast hour.
climRegridPath is the absolute path #[where the regridded monthly
mean climatologies (w.r.t the model vertical resolution) optional]#
nc files were already stored.
climPFileName is the partial nc filename of the climatology.
climatologyYear is the year of the climatology to access it.
KWargs: modelName, modelXmlPath, modelXmlObj, cregrid, dregrid,
modelName - is the model data name which will become part of the
process nc files name.
modelPath - is the absolute path of data where the model xml files
are located.
modelXmlObj - is an instance of the GribXmlAccess class instance.
If we are passing modelXmlObj means, it will be optimized one
when we calls this same function for same model for different
months.
We can pass either modelXmlPath or modelXmlObj KWarg is enough.
ovar - output anomaly variable. By default it will be same as
model variable name. If user need to change it, then
they can pass new variable name for anomaly data.
convertTI2N - It takes either True or False. If it is True, then the
model data will be converted from Time Integrated to Normal form.
i.e. Units will be converted from Wsm^-2 to Wm^2.
sign - change sign of the model data (will be used in convertTI2N fn)
When modeldata and climatology shapes are mis-match, then
cregrid : if True, then climatology data will be regridded w.r.t
model/obs/data and then anomaly will be calculated.
dregrid : if True, then model/obs/data will be regridded w.r.t
climatology data and then anomaly will be calculated.
..note:: We can not enable both cregrid and dregrid at the same time.
CAUTION: If "from diag_setup.varsdict import variables" contains 'anl_hour' arg
in any of the variable statement, then instead of 'analysis' modelType,
it will take input data from the 'fcst' modelType and its hour will be
its corresponding anl_hour. It will be useful when 'analysis' type
'olr' variable is not available, then we can use '00'th fcst hour's olr.
But the output filename reamains ends with same as analysis
Outputs : It should create daily anomaly for the particular variables which
are all set the clim_var option in the vars.txt file. Finally
store it as nc file formate in the proper directories structure
(modelname, 'Anomaly', year, daily and then 'analysis' or
fcst hours hierarchy).
Written By : Arulalan.T
Date : 30.12.2011
Updated Date : 02.07.2013
"""
modelXmlObj, modelPath = None, None
if 'modelName' in kwarg:
modelName = kwarg.get('modelName')
else:
raise RuntimeError("KWarg modelName must be passed")
if 'modelXmlObj' in kwarg:
modelXmlObj = kwarg.get('modelXmlObj')
elif 'modelXmlPath' in kwarg:
modelPath = kwarg.get('modelXmlPath')
else:
raise RuntimeError("you must pass atleast one KWargs of modelXmlPath \
or modelXmlPath ")
if not modelXmlObj:
xmlobj = xml_data_access.GribXmlAccess(modelPath)
else:
if isinstance(modelXmlObj, xml_data_access.GribXmlAccess):
xmlobj = modelXmlObj
else:
raise ValueError("Passed modelXmlObj instance %s is not an \
instance of GribXmlAccess class " % type(modelXmlObj))
ovar = kwarg.get('ovar', '')
cregrid = kwarg.get('cregrid', False)
dregrid = kwarg.get('dregrid', False)
convertTI2N = kwarg.get('convertTI2N', False)
sign = kwarg.get('sign', 1)
arglog = kwarg.get('log', None)
if arglog is not None:
log = arglog
# end of if arglog is not None:
# get the nc files name of mean anomaly
ncfiles = [f for f in os.listdir(dailyAnomalyPath) if f.endswith('.nc')]
# make ncfiles as dictionary with key as var name
ncfiledic = {}
for ncfile in ncfiles:
vari = ncfile.split('_')
if vari[1].isdigit():
# for mjo works, variable_level_foo.nc has mentioned.
vari = vari[:2]
else:
vari = [vari[0]]
# end of if vari[1].isdigit():
var = '_'.join(vari)
ncfiledic[var] = ncfile
# make memory free
del ncfiles
# get the namedtuple object from the global 'vars.txt' file
totalvars = variables.get(modelName)
for globalvar in totalvars.itervalues():
if modelType in ['a', 'anl', 'f', 'fcst']:
# get the model var name
mvar = globalvar.model_var
elif modelType in ['o', 'obs']:
# get the observation var name
mvar = globalvar.obs_var
# get the climatology var name
cvar = globalvar.clim_var
# get the model variable level
mlevel = globalvar.model_level
if globalvar.anl_hour is not None and modelType in ['a', 'anl', 'analysis']:
# overwrite the mtype and its hour to extract this variable data
# alone from fcst and globalvar.anl_hour (for eg fcst 00 th hour).
# But the output filename reamains ends with same as analysis
mtype = 'f' # fcst type
mhour = globalvar.anl_hour
else:
mtype = modelType
mhour = modelHour
# end of if globalvar.anl_hour is not None ... :
if not cvar:
print "Climatology var name is empty string. So skipping anomaly \
process for %s model var name " % mvar
continue
# end of if not cvar:
modelvariables = xmlobj.listvariable(Type=mtype, hour=mhour)
if not mvar in modelvariables:
print "The variable %s is not available in the xml anl/obs file object" % mvar
print "So skipping the anomaly and mean analysis processes \
for this variable %s which is one of the keys of the \
variables dictionary" % mvar
continue
# end of if not mvar in allvariables:
# partial nc file name
pFileName = [mvar, modelName, year, 'daily_']
# variable name key generating to access ncfiledic
ncvar = mvar
if mlevel:
strlevel = str(int(mlevel))
pFileName.insert(1, strlevel)
ncvar += '_' + strlevel
# end of if mlevel:
pFileName = '_'.join(pFileName)
if modelType in ['a', 'anl']:
pFileName += 'anl'
elif modelType in ['o', 'obs']:
pFileName += 'obs'
elif modelType in ['f', 'fcst']:
pFileName += 'f' + modelHour + 'hr'
# end of if modelType in ['a', 'anl']:
anomalyLatestDate = None
# store anomaly into proper nc file
if ncvar in ncfiledic:
anomalyFileName = ncfiledic.get(ncvar)
anomalyFilePath = os.path.join(dailyAnomalyPath, anomalyFileName)
anomalyLatestDate = None
try:
# open nc file in append mode
anomalyFile = cdms2.open(anomalyFilePath, 'a') # append mode.
# get the ncfile timeAxis
xmlFileLatestDate = xmlobj[mvar, mtype, mhour].getTime().asComponentTime()[-1]
anomalyLatestDate = anomalyFile[mvar].getTime().asComponentTime()[-1]
# Do check either this month timeAxis is already exists in
# the nc file's timeAxis or not. If exists means skip it.
if __timeCheck__:
print anomalyFilePath, xmlFileLatestDate, anomalyLatestDate
if anomalyLatestDate == xmlFileLatestDate:
log.info("The daily anomaly is already exists in the \
file %s for '%s'. So skipping var '%s' ",
anomalyFileName, mvar, str(anomalyLatestDate))
anomalyFile.close()
continue
# end of if __timeCheck__:
except (AttributeError, cdms2.error.CDMSError):
# if it getting this error means, it may not written
# properly. So open nc file in write mode freshly.
log.warning("Got Problem. nc file is correpted at last time. \
May be lost the previous days data.\
Now creating same nc file '%s' freshly & fully ",
anomalyFileName)
# remove it
os.remove(anomalyFilePath)
# create it freshly
anomalyFile = cdms2.open(anomalyFilePath, 'w')
except Exception, e:
print e
else:
# generate the nc filename
anomalyFileName = pFileName + '_anomaly.nc'
anomalyFilePath = dailyAnomalyPath + '/' + anomalyFileName
# open new nc file in write mode
anomalyFile = cdms2.open(anomalyFilePath, 'w')
log.info("opening anomaly file in write mode '%s'", anomalyFilePath)
# end of if ncvar in ncfiledic:
# generate the climatology file name
climatologyFile = cvar + climPFileName
cfile = cdms2.open(climRegridPath + '/' + climatologyFile, 'r')
# get the model data
modelObj = xmlobj[mvar, mtype, mhour]
modelTime = modelObj.getTime().asComponentTime()
modelFirstYear = modelTime[0].year
levAxis = modelObj.getLevel()
latAxis = modelObj.getLatitude()
lonAxis = modelObj.getLongitude()
regridfunc = None
if anomalyLatestDate:
nextIndex = modelTime.index(anomalyLatestDate) + 1
modelTime = modelTime[nextIndex:]
log.info("The last successfull anomaly date is '%s'. \
So continuing anomaly from the next day onwards", anomalyLatestDate)
# end of if anomalyLatestDate:
log.info("Writing daily anomaly into %s file", anomalyFileName)
preview = 0
for mdate in modelTime:
try:
# get the model data for this day alone
modelData = xmlobj.getData(mvar, Type=mtype,
hour=mhour, date=mdate, level=mlevel, squeeze=1)
except:
print "Coundn't get the analysis data of %s var %s time" \
% (mvar, mdate)
print "So skipping the anomaly for the '%s' variable" % mvar
continue
# end of try:
climDataTime = cdtime.comptime(climatologyYear, mdate.month, mdate.day)
try:
# get the climatology data for this month alone
climatologyData = cfile(cvar, time=climDataTime, squeeze=1)
except:
print "Coundn't get the climatology data for the variable %s and \
time %s " % (cvar, climDataTime)
print "So skipping anomaly for the variable %s" % mvar
continue
# end of try:
if not __showStatusBar:
log.info("Calculating Daily Anomaly on %s", str(mdate))
# end of if not __showStatusBar:
if convertTI2N:
# Time integrated data unit Wsm^-2. Converted to Wm^2.
modelData = convertTimeIntegratedToNormal(modelData, modelHour)
# end of if convertTI2N:
if sign == -1:
# Changing Sign and making OLR data Positive
modelData = modelData * (-1)
# end of if sign == -1:
# anomaly
if modelData.shape == climatologyData.shape:
# calculate anomaly
anomaly = modelData - climatologyData
else:
clim_grid = climatologyData.getGrid()
data_grid = modelData.getGrid()
if cregrid and not dregrid:
# Regridding the climatology data
# Creating the horizontal lat,lon regrid
# Here 'clim_grid' is the source and 'data_grid' is the target
if not regridfunc:
regridfunc = Horizontal(clim_grid, data_grid)
# end of if not regridfunc:
climatologyData = regridfunc(climatologyData)
elif dregrid and not cregrid:
# Regridding the model/obs data
# Creating the horizontal lat,lon regrid
# Here 'data_grid' is the source and 'clim_grid' is the target
if not regridfunc:
regridfunc = Horizontal(data_grid, clim_grid)
# update the regridded data axis
latAxis = climatologyData.getLatitude()
lonAxis = climatologyData.getLongitude()
levAxis = climatologyData.getLevel()
# end of if not regridfunc:
modelData = regridfunc(modelData)
elif dregrid and cregrid:
raise ValueError("Can not do both 'cregrid' and 'dregrid'. \
Make one option as False.")
elif not dregrid and not cregrid:
print "model data shape ", modelData.shape
print "climatology data shape ", climatologyData.shape
raise ValueError("model data and climatology data shapes are mis-match.")
# end of if cregrid and not dregrid:
# calculate anomaly for regridded data sets.
anomaly = modelData - climatologyData
# end of if data.shape == climatology.shape:
# make free memory
del modelData, climatologyData
mstart = timobj._getDayCountOfYear(mdate, year=modelFirstYear)
startDayOfYear = '%s-1-1' % str(modelFirstYear)
dailyAnomalyTime = timobj._generateTimeAxis([mstart], startDayOfYear)
# setting model time axis to the anomaly
# reshape the anomaly to set timeAxis
anoshape = list(anomaly.shape)
anoshape.insert(0, 1)
anomaly = anomaly.reshape(anoshape)
# get the needed axis list
if len(anoshape) == 3:
axislist = (dailyAnomalyTime, latAxis, lonAxis)
elif len(anoshape) == 4:
axislist = (dailyAnomalyTime, levAxis, latAxis, lonAxis)
# end of if len(anoshape) == 3:
axislist = [axis for axis in axislist if axis]
# set the axis information to the anomaly
anomaly.setAxisList(axislist)
if ovar:
anomaly.id = ovar
else:
anomaly.id = mvar
#anomaly.comments = 'monthly mean anomaly of %s model data of %s' % (modelName, year)
# check the file status is closed or not. This is cdms2 fileobj.
if anomalyFile._status_ == 'closed':
# file is closed, so reopen it in append mode.
#
anomalyFile = cdms2.open(anomalyFilePath, 'a')
# end of if anomalyFile.closed:
anomalyFile.write(anomaly)
anomalyFile.close()
### file has to close inside the modelTime loop itself.
# Otherwise if we run a big year data in loop,
# it may get failure to write when system crashes.
# If we close the file obj inside loop, the previous loop anomaly work
# has been saved/written into the fileobj. So be cool.
###
# make memory free
del anomaly
if __showStatusBar:
preview = statusbar(modelTime.index(mdate), total=len(modelTime),
title='Anomaly', prev=preview)
# end of if __showStatusBar:
# end of for mdate in modelTime:
print
def lfilter(data, weights, cyclic=True, **kwarg):
"""
Lanczos Filtered
Todo : Write doc of the work flow
KWarg:
returntlen : return time axis length. By default it takes None.
It can take positive or negative integer as value.
If it is +ve no, then it will do filter only to
the first no days rather than doing filter to the
whole available days of the data.
If it is -ve no, then it will do filter only to
the last no days.
Timeaxis also will set properly w.r.t return data.
if cyclic is enabled or disabled, then returntlen
index will change according to cyclic flag.
Lets assume weights length is 101.
Eg1: Lets consider data length is 365.
If cyclic is False, then filteredData length is
365-(100*2)=165 when returntlen is None.
If returntlen = 10, so filteredData length is 10
and its index is range(100, 110).
If returntlen = -10, so filteredData length is 10
and its index is range(255, 265).
Eg2: If cyclic is True, then filteredData length is
365 when returntlen is None.
If returntlen = 10, so filteredData length is 10
and its index is range(0, 10).
If returntlen = -10, so filteredData length is 10
and its index is range(355, 365).
So returntlen option is useful when we need to filter
only to the certain days. But regardless of returntlen
option, we need to pass the full data to apply Lanczos
Filter to either returntlen days or full days.
This option will save a lot of time when returntlen is
very less compare to the total length of the data.
Written By : Arulalan.T
Date : 15.09.2012
Updated : 22.09.2013
"""
returntlen = kwarg.get('returntlen', None)
if returntlen:
if not isinstance(returntlen, int):
raise ValueError("Keyword arg returntlen must be an integer")
# end of if returntlen:
timeAxis = data.getTime()
latAxis = data.getLatitude()
lonAxis = data.getLongitude()
levAxis = data.getLevel()
dataid = data.id
# get the total length of the timeAxis of the original data
tlen = len(timeAxis)
# get the units of timeAxis of the original data
tunits = timeAxis.units
# we can use the below option if we dont want to initialize it with
# size/dimension. But we need to add append option instead of assign
# resultant filteredData.
#filteredData = numpy.array([0])
zweight = weights[0]
weights = numpy.array(weights[1:])
wlen = len(weights)
# get the axis length shape
axshape = [len(axis) for axis in [levAxis, latAxis, lonAxis] if axis]
# reshape the weights according to the passed data shape
# (just to shape match to multiply with data)
wshape = [1 for axis in [levAxis, latAxis, lonAxis] if axis]
wshape.insert(0, wlen)
weights = weights.reshape(wshape)
# set flag if returntlen is -ve integer
reverse = True if returntlen < 0 else False
if cyclic:
# for cyclic, the filteredData timeAxis should be same as passed
# data's timeAxis. i.e. tlen length.
# get full tlen or returntlen as length of the return filteredData
loopCount = abs(returntlen) if returntlen else tlen
if reverse:
# return the last loopCount days
loopdays = xrange(tlen-loopCount, tlen)
timeAxisList = timeAxis[tlen-loopCount: tlen]
else:
# return loopCount days from the begining of the data
loopdays = xrange(loopCount)
timeAxisList = timeAxis[:loopCount]
# end of if reverse:
else:
# if it non-cyclic, then we will lost the timeAxis length of wlen*2
# days. i.e tlen-(wlen*2) length.
# get tlen-(wlen*2) or returntlen as length of the return filteredData
loopCount = abs(returntlen) if returntlen else tlen-(wlen*2)
if reverse:
# return the last loopCount days
loopdays = xrange(tlen-wlen-loopCount, tlen-wlen)
timeAxisList = timeAxis[tlen-wlen-loopCount: tlen-wlen]
else:
# return loopCount days from the begining of the data
loopdays = xrange(wlen, wlen+loopCount)
timeAxisList = timeAxis[wlen: wlen+loopCount]
# end of if reverse:
# end of if cyclic:
# insert the tlength at first in the axis length shape
axshape.insert(0, loopCount)
# filteredDate dummy initialization shape
filteredData = numpy.zeros(axshape)
# overwrite the timeAxis
timeAxis = cdms2.createAxis(timeAxisList)
timeAxis.id = 'time'
timeAxis.units = tunits
timeAxis.designateTime()
preview = 0
for idx, cday in enumerate(loopdays):
# loop through all the days (check above)
# apply zeroth weight to the current day data
cfilter = data[cday] * zweight
ps = (tlen - wlen) + cday
#pe = (tlen + cday) % tlen
if ps <= tlen:
preData = numpy.concatenate((data[ps:], data[:cday]))
elif ps > tlen:
ps = ps % tlen
preData = data[ps:cday]
# end of if ps <= tlen:
# reverse the previous data of length wlen
preData = preData[::-1]
ns = cday + 1
ne = ns + wlen
if ne <= tlen:
nextData = data[ns:ne]
elif ne > tlen:
ne = ne % tlen
nextData = numpy.concatenate((data[ns:], data[:ne])) #data[ns:ne]
# end of if ne <= tlen:
# add together the wlen days previous (reversed) data & wlen days next
# data (with out reversed) to the current day data.
# And then multiplied with wlen size of weights
# (multiplied in all dimensions).
# now sum together, the time dimension length becomes one.
wlenfilter = sum((preData + nextData) * weights)
# finally again sum together the above wlenfilter (wlength filtered
# data) with current day filtered (zeroth day filtered) data.
if not cyclic:
# get the proper index to store into the filteredData array
cday = cday % wlen
# end of if not cyclic:
# store to the filteredData array
filteredData[idx] = cfilter + wlenfilter
# make memory free
del preData, nextData, wlenfilter
if __showStatusBar:
preview = statusbar(idx, total=loopCount,
title='Filtering', prev=preview)
# end of if __showStatusBar:
# end of for idx, cday in enumerate(loopdays):
print
# make memory free
del data
# get the needed axis list
axislist = [axis for axis in (timeAxis, levAxis, latAxis, lonAxis) if axis]
# set the axis information to the filteredData variable
filteredData = cdms2.createVariable(filteredData)
filteredData.id = dataid
filteredData.setAxisList(axislist)
return filteredData
from genutil import statusbar
prevall=0
previ=0
prevj=0
prevtxt=0
ni=100
nj=20
# status in tk gui
for i in range(ni):
previ=statusbar(float(i)/ni,prev=previ,tk=1)
for j in range(nj):
prevj=statusbar(float(j)/nj,prev=prevj,tk=1)
prevall=statusbar([float(i)/ni,float(j)/nj],prev=prevall,tk=1,title='Test')
prevtxt=statusbar(float(j)/nj,prev=prevtxt,tk=1,title='Test')
#### Status in terminal
prev=0
for I in range(1000):
prev=statusbar(I,total=1000, title='Status:',prev=prev)
print
# The final above empty print outside the loop is necessary
# to endup with proper terminal '$' at its original position
# Adapted for numpy/ma/cdms2 by convertcdms.py
from genutil import statusbar
import os, sys, cdutil
n = 100
try:
prev = 0
for i in range(n):
prev = statusbar(i, n - 1, prev=prev)
except:
raise "Error simple statusbar died...."
print
try:
prev = 0
for i in range(n):
prev = statusbar(i, float(n - 1), prev=prev, tk=1)
except:
raise "Error simple tk statusbar died...."
import cdutil, cdms2 as cdms
f = cdms.open(os.path.join(vcs.sample_data, 'clt.nc'))
s = f('clt')
cdutil.setTimeBoundsMonthly(s)
try:
ac = cdutil.ANNUALCYCLE(s, statusbar=1)
except:
raise "Error statusbar of extractions died (not tk)...."
try:
ac = cdutil.ANNUALCYCLE.climatology(s, statusbar=1)
except:
def background(self,
S,
A,
wavenumber_smoothing_windows=((
1000,
10,
), ),
frequencies_smoothing_windows=((.1, 5), (.2, 10), (.3, 20),
(1000, 40))):
""" return the background power
Input:
Power, with frequencies as dimension 0 and wave numbers as dim 1
frenquencies_smoothing_windows: list of tuples of frequency/number of 121 smoothing, frequencies must be increasing
wavenumber_smoothing_windows: list of tuples of wavenumber/number of 121 smoothing, wavenumber must be increasing
Output:
background_power
Usage:
bg=background(power,wavenumber_smoothing=((1000,10)),frenquencies_smoothing_windows=((.1,5),(.2,10),(.3,20),(1000,40)))
"""
id = S.id.split("_")[0]
power = (S + A) / 2.
trans = range(power.rank())
trans[0] = 1
trans[1] = 0
## Puts wave number first
power = MV2.transpose(power, trans)
istart = 0
fqcies = power.getAxis(1)[:]
## Little bit to do the bar
total = -1
for fqcy, n in frequencies_smoothing_windows:
total += n
for wn, n in wavenumber_smoothing_windows:
total += n
counter = 0
prev = 0
for fqcy, n in frequencies_smoothing_windows:
## determine which one is the limit
iend = 0
for i in range(len(fqcies)):
f = fqcies[i]
if f <= fqcy: iend = i
## print istart,iend+1
for i in range(n):
tmp = MWsmooth121(power[:, istart:iend + 1])
power[:, istart:iend + 1] = tmp
prev = genutil.statusbar(float(counter),
total,
prev=prev,
tk=self.tkbar)
counter += 1
istart = iend + 1
## Puts fqcies first
power = MV2.transpose(power, trans)
istart = 0
wns = power.getAxis(1)[:]
for wn, n in wavenumber_smoothing_windows:
## determine which one is the limit
iend = 0
for i in range(len(wns)):
w = wns[i]
if w <= wn: iend = i
## print istart,iend+1
for i in range(n):
tmp = MWsmooth121(power[:, istart:iend + 1])
power[:, istart:iend + 1] = tmp
prev = genutil.statusbar(float(counter),
total,
prev=prev,
tk=self.tkbar)
counter += 1
power.id = id + '_background'
if self.tkbar:
prev[1].destroy()
prev[0].destroy()
return power
def background(self,S,A,wavenumber_smoothing_windows=((1000,10,),),frequencies_smoothing_windows=((.1,5),(.2,10),(.3,20),(1000,40))):
""" return the background power
Input:
Power, with frequencies as dimension 0 and wave numbers as dim 1
frenquencies_smoothing_windows: list of tuples of frequency/number of 121 smoothing, frequencies must be increasing
wavenumber_smoothing_windows: list of tuples of wavenumber/number of 121 smoothing, wavenumber must be increasing
Output:
background_power
Usage:
bg=background(power,wavenumber_smoothing=((1000,10)),frenquencies_smoothing_windows=((.1,5),(.2,10),(.3,20),(1000,40)))
"""
id = S.id.split("_")[0]
power=(S+A)/2.
trans=range(power.rank())
trans[0]=1
trans[1]=0
## Puts wave number first
power=MV2.transpose(power,trans)
istart=0
fqcies=power.getAxis(1)[:]
## Little bit to do the bar
total = -1
for fqcy,n in frequencies_smoothing_windows:
total+=n
for wn,n in wavenumber_smoothing_windows:
total+=n
counter = 0
prev = 0
for fqcy,n in frequencies_smoothing_windows:
## determine which one is the limit
iend=0
for i in range(len(fqcies)):
f=fqcies[i]
if f<=fqcy: iend=i
for i in range(n):
tmp=MWsmooth121(power[:,istart:iend+1])
power[:,istart:iend+1]=tmp
prev=genutil.statusbar(float(counter),total,prev=prev,tk=self.tkbar)
counter+=1
istart=iend+1
## Puts fqcies first
power=MV2.transpose(power,trans)
istart=0
wns=power.getAxis(1)[:]
for wn,n in wavenumber_smoothing_windows:
## determine which one is the limit
iend=0
for i in range(len(wns)):
w=wns[i]
if w<=wn: iend=i
for i in range(n):
tmp=MWsmooth121(power[:,istart:iend+1])
power[:,istart:iend+1]=tmp
prev=genutil.statusbar(float(counter),total,prev=prev,tk=self.tkbar)
counter+=1
power.id = id+'_background'
if self.tkbar:
prev[1].destroy()
prev[0].destroy()
return power
def process(self,data):
## if self.symetric:
## data = symetric(data)
# Make sure we have an even number of time steps
t=data.getTime()
# length of time axis
nt=len(t)
if nt%2!=0:
print "Warning time wasn't even, removed last time step"
data=data[:-1]
t=data.getTime() ## get the new time axis
nt=len(t)
if len(t)<self._NTSub:
raise Exception,"Error your data must have at least %i time steps, adjust frequency (currently: %i/day) or number_of_days (currently: %i processed at once) to reach that limit, or get more data" % (self._NTSub,self.frequency,self.number_of_days)
## Computes PP, number of sub-domain
PP=float(nt-self._NTSub)/self._NShift+1
PP=int(PP)
## Number of longitudes
lons=data.getLongitude()
NL=len(lons)
tt=cdms2.createAxis(numpy.arange(self._NTSub),id='sub_time')
## Should redo that with just an arange (eventually...)!!!
## Frequencies in cycles/day
ff=numpy.arange(0,self._NTSub+1,1,numpy.float)
for i in range(1,self._NTSub+2):
ff[i-1]=float(i-1-self._NTSub/2.)*self.frequency/float(self._NTSub)
## Should redo that with just an arange (eventually...)!!!
## Wave numbers
ss=numpy.arange(0,NL+1,1,numpy.float)
for i in range(1,NL+2):
ss[i-1]=float(i-1-NL/2.)
## print 'Frequencies:',ff
## print 'Wave numbers:',ss
## Ok, we now do the real stuff
## Creates the array of powers (Number of subtimes,fqcy,wave_numbers,latitudes)
lats=data.getLatitude()
Power=numpy.zeros((PP,self._NTSub+1,NL+1,len(lats)),numpy.float)
## LOOP through time sub domains
prev=0 # initialize the scrolling bar
for Pcount in range(PP):
if PP>1:
prev=genutil.statusbar(float(Pcount),PP-1,prev=prev,tk=self.tkbar)
## Get the time subdomain
EEo=data[Pcount*self._NShift:Pcount*self._NShift+self._NTSub](order='tx...')
## First does the symetric/antisymetric thing if needed
if self.symetric: EEo=symetrick(EEo)
## Now detrending
## Step 1- Get the slope and intercept
slope,intercept=genutil.statistics.linearregression(EEo,nointercept=0)
## Step 2- remove the trend
## Step 2a: Create an array with the time values
a=EEo.getTime()
A=MV2.array(a[:],typecode='d')
A.setAxis(0,a)
## Step 2b: "Grows" it so it has the same shape than data
A,EEo=genutil.grower(A,EEo)
## Step 2c: Actually remove the trend
EE=EEo-A*slope-intercept
## we don't need A,EEo,slope,intercept anymore
del(EEo)
del(slope)
del(intercept)
del(A)
## Remove the time mean
mean=MV2.average(EE,0)
EE=EE-mean
del(mean) # could be big in memory
## Tapering time...
tapertozero(EE,1,len(EE)-1,5*self.frequency)
## OK here Wheeler has some windowing on longitude, but it's commented out
## I'll pass it for now
## Ok the actuall FFT work
EE=numpy.fft.fft2(EE,axes=(1,0))/NL/self._NTSub
## OK NOW THE LITTLE MAGIC WITH REORDERING !
A=numpy.absolute(EE[0:self._NTSub/2+1,1:NL/2+1])**2
B=numpy.absolute(EE[self._NTSub/2:self._NTSub,1:NL/2+1])**2
C=numpy.absolute(EE[self._NTSub/2:self._NTSub,0:NL/2+1])**2
D=numpy.absolute(EE[0:self._NTSub/2+1,0:NL/2+1])**2
Power[Pcount,self._NTSub/2:,:NL/2]=A[:,::-1]
Power[Pcount,:self._NTSub/2,:NL/2]=B[:,::-1]
Power[Pcount,self._NTSub/2+1:,NL/2:]=C[::-1,:]
Power[Pcount,:self._NTSub/2+1,NL/2:]=D[::-1,:]
## End of Pcount loop
if self.tkbar and PP>1:
prev[1].destroy()
prev[0].destroy()
## Now generates the decorations
## first the time axis (subdomains)
vals=[]
bounds=[]
pp=0
for i in range(0,len(t)-self._NShift,self._NShift):
st=t.subAxis(i,i+self._NTSub)
if len(st[:])==self._NTSub:
pp+=1
vals.append((st[0]+st[-1])/2.)
bds=st.getBounds()
#print 'Bounds:',bds
if bds is None:
raise ValueError, "Data need to have bounds on time dimension"
else:
bounds.append([bds[0][0],bds[-1][1]])
## Convert lists to arrays
vals=numpy.array(vals)
bounds=numpy.array(bounds)
## Creates the time axis
dumt=cdms2.createAxis(vals,bounds=bounds)
dumt.id='time'
dumt.units=t.units
dumt.designateTime()
dumt.setCalendar(t.getCalendar())
## Create the frequencies axis
T=cdms2.createAxis(ff)
T.id='frequency'
T.units='cycles per day'
## Create the wave numbers axis
S=cdms2.createAxis(ss)
S.id='planetaryzonalwavenumber'
S.units='-'
## Makes it an MV2 with axis and id (id come sfrom orignal data id)
Power=MV2.array(Power,axes=(dumt,T,S,lats),id=data.id+'_'+'power')
## Adds a long name attribute
Power.longname='Real power spectrum for the many different parts (i.e. over separate time divisions)'
## And return the whole thing ordered 'time', 'latitude', 'frequencies','wavenumbers'
return Power(order='ty...')
def monthlyClimatology(varName, infile, outfile, memory='low', **kwarg):
"""
monthlyClimatology : It will create the monthly climatolgy.
Its timeaxis dimension length is 12.
memory : 'low'/'high'.
If it is low, then it compute climatology in optimized
manner by extracting full timeseries data of particular
latitude, longitude & level points by loop throughing
each latitude, longitude & level axis. It needs low RAM memory.
If it is 'high', then it load the whole data from the input
file and compute climatology. It needs high RAM memory.
KWargs:
ovar : out varName. If it is passed then the climatology variable
name will be set as ovar. Otherwise the input varName will
be set to it.
squeeze : 1 (it will squeeze single dimension in the climatolgy)
todo : need to give option to create 366 days climatology.
Written By : Arulalan.T
Date : 13.08.2013
"""
ovar = kwarg.get('ovar', None)
squeeze = kwarg.get('squeeze', 1)
inf = cdms2.open(infile)
if memory in [1, 'high']:
data = inf(varName)
# calculate climatology over timeAxis for global data
clim = cdutil.ANNUALCYCLE.climatology(data)
if ovar: clim.id = ovar
# write the climatology data into nc file
outf = cdms2.open(outfile, 'w')
outf.write(clim)
outf.close()
elif memory in [0, 'low']:
ctlen = 12
latitudes = inf[varName].getLatitude()
longitudes = inf[varName].getLongitude()
levels = inf[varName].getLevel()
# create climatology time axis
ctimeAxis = cdms2.createAxis(range(ctlen), id='time')
ctimeAxis.units = 'months since 1-1-1'
ctimeAxis.designateTime()
# create dummy array to stroe the climatology data
if levels:
clim = numpy.zeros((ctlen, len(levels), len(latitudes), len(longitudes)))
else:
clim = numpy.zeros((ctlen, len(latitudes), len(longitudes)))
preview = 0
for latidx, lat in enumerate(latitudes):
if levels:
for levidx, lev in enumerate(levels):
# calculate climatology over timeAxis for particular
# lat, lon & level point data in loop
data = inf(varName, level=(lev, lev), latitude=(lat, lat))
# store the climatology data into dummy array
cdata = cdutil.ANNUALCYCLE.climatology(data)(squeeze=1)
clim[:, levidx, latidx, :] = cdata
else:
# calculate climatology over timeAxis for particular
# lat, lon point data in loop
data = inf(varName, latitude=(lat, lat))
# store the climatology data into dummy array
clim[:, latidx, :] = cdutil.ANNUALCYCLE.climatology(data)
# end of if levels:
if __showStatusBar:
preview = statusbar(latidx, total=len(latitudes),
title='Climatology', prev=preview)
# Either averager function or setSlabTimeBoundsDaily
# fucnction will print 'convention.getDsetnodeAuxAxisIds'
# msg. So status bar unable to retain the same line.
# To avoid that I added the below line.
# The character '\x1b' is the escape character.
# The character [1A moves the cursor to 1 previous line.
# (i.e. at the end of the previous line \n char).
# The character [80D moves cursor to 80 positions to the left.
# The character [K clears the line.
# Refer ANSCII escape sequence for more details.
sys.stdout.write("\x1b[1A" + "\x1b[80D" + "\x1b[K")
# end of if __showStatusBar:
# end of for latidx, lat in enumerate(latitudes):
print
axislist = [axis for axis in [ctimeAxis, levels, latitudes, longitudes] if axis]
if squeeze:
clim = clim.squeeze()
# squeezing the axis lists
axislist = [axis for axis in axislist if len(axis[:]) > 1]
# end of if squeeze:
# create the climatology cdms2 variable with its axis information
clim = cdms2.createVariable(clim)
if ovar:
clim.id = ovar
else:
clim.id = varName
clim.setAxisList(axislist)
cdutil.setSlabTimeBoundsMonthly(clim)
# write the climatology data into nc file
outf = cdms2.open(outfile, 'w')
outf.write(clim)
outf.close()
# end of if memory in [1, 'high']:
inf.close()
print "The Climatology data year is", 1
print "The climatolgy data has written into", outfile
