def test_001_dataIsEqual(slef):
arr = N.arange(10.0, 20.0, 0.25)
arr = arr.reshape(8, 5)
v1 = cdms.createVariable(arr, id='v1')
v2 = cdms.createVariable(arr, id='v2')
nt.assert_true(var_utils.isVariableDataEqual(v1, v2))
def BLH(P,T,Z3):
# Calculates the boundary layer height (BLH), which is defined as the level of maximum dTheta/dZ
mv_time=Z3.getTime()
mv_lat=Z3.getAxis(2)
mv_lon=Z3.getAxis(3)
z3_units=Z3.units
mv_typecode=Z3.typecode()
t_units=T.units
T=np.array(T)
Theta=T*(100000./P)**(2./7.)
Z3=np.array(Z3)
dThetadZ3=(Theta[:,1:,:,:]-Theta[:,:-1,:,:])/(Z3[:,1:,:,:]-Z3[:,:-1,:,:])
Z3_avheight=(Z3[:,1:,:,:]+Z3[:,:-1,:,:])/2.
BLH_array=np.zeros((Z3.shape[0],Z3.shape[2],Z3.shape[3]))
dThetadZ3_array=np.zeros((Z3.shape[0],Z3.shape[2],Z3.shape[3]))
id_max_dThetadZ3=np.argmax(dThetadZ3[:,13:,:,:],axis=1)
max_dThetadZ3=np.amax(dThetadZ3[:,13:,:,:],axis=1)
dThetadZ3_array=max_dThetadZ3
Z3_avheight_subset=Z3_avheight[:,13:,:,:]
for x in np.arange(Z3.shape[2]):
for y in np.arange(Z3.shape[3]):
for t in np.arange(Z3.shape[0]):
BLH_array[t,x,y]=Z3_avheight_subset[t,id_max_dThetadZ3[t,x,y],x,y]
BLH_var=cdm.createVariable(BLH_array,typecode=mv_typecode,axes=[mv_time,mv_lat,mv_lon],id='BLH')
BLH_var.units=z3_units
BLH.long_name='Boundary Layer Height based on height with maximum dTheta/dZ below 10000 m'
dThetadZ3_var=cdm.createVariable(dThetadZ3_array,typecode=mv_typecode,axes=[mv_time,mv_lat,mv_lon],id='dThetadZ')
dThetadZ3_var.long_name='Maximum dTheta/dZ below 10000 m'
dThetadZ3_var.units='K m-1'
return BLH_var,dThetadZ3_var
def reference_solution(container_type):
"""Generate a reference field and a corresponding EOF solution.
**Argument:**
*container_type*
The type of the solution containers. Either 'numpy' for
:py:mod:`numpy` arrays or 'cdms2' for :py:mod:`cdms2`.
"""
sf, eofs, pcs = _construct_reference()
if container_type.lower() == 'numpy':
# Return the solution as-is for numpy containers.
return sf, eofs, pcs
# Create meta-data for cdms2 containers.
try:
time = cdms2.createAxis(np.arange(100), id='time')
time.designateTime()
time.units = 'days since 2011-01-01 00:00:0.0'
longitude = cdms2.createAxis(np.arange(0., 360., 360. / 225.),
id='longitude')
longitude.designateLongitude()
eof = cdms2.createAxis(range(2), id='eof')
eof.long_name = 'eof number'
sf = cdms2.createVariable(sf, axes=[time, longitude], id='sf')
eofs = cdms2.createVariable(eofs, axes=[eof, longitude], id='eofs')
pcs = cdms2.createVariable(pcs, axes=[time, eof], id='pcs')
except NameError:
raise ValueError("can't create cdms2 containers without cdms2")
return sf, eofs, pcs
def reference_solution(container_type):
"""Generate a reference field and a corresponding EOF solution.
**Argument:**
*container_type*
The type of the solution containers. Either 'numpy' for
:py:mod:`numpy` arrays or 'cdms2' for :py:mod:`cdms2`.
"""
sf, eofs, pcs = _construct_reference()
if container_type.lower() == 'numpy':
# Return the solution as-is for numpy containers.
return sf, eofs, pcs
# Create meta-data for cdms2 containers.
try:
time = cdms2.createAxis(np.arange(100), id='time')
time.designateTime()
time.units = 'days since 2011-01-01 00:00:0.0'
longitude = cdms2.createAxis(np.arange(0., 360., 360./225.),
id='longitude')
longitude.designateLongitude()
eof = cdms2.createAxis(range(2), id='eof')
eof.long_name = 'eof number'
sf = cdms2.createVariable(sf, axes=[time,longitude], id='sf')
eofs = cdms2.createVariable(eofs, axes=[eof,longitude], id='eofs')
pcs = cdms2.createVariable(pcs, axes=[time,eof], id='pcs')
except NameError:
raise ValueError("can't create cdms2 containers without cdms2")
return sf, eofs, pcs
def systematic_bias(self):
""" Biais systematique (+1: sous-estimation du modele, -1: sur-estimation du modele) """
import numpy as np
import cdms2
from vacumm.misc.grid import get_grid, set_grid
map_bias = list() #Initialise une liste
# Estimation de la moyenne a chaque pas de temps
tps = self.obs.getTime()
for i, t in enumerate(tps):
map_bias.append(np.sign(self.obs[i, :, :]-self.model[i, :, :] ))
map_bias = cdms2.createVariable(map_bias,typecode='f',id='computation')
res = np.sum(map_bias, axis=0)
res
# Trouve valeurs egale a la longueur de la serie temporelle (<=> uniquement valeurs positives)
one = (res==len(map_bias)).nonzero()
# Trouve valeurs egales a moins la longueur de la serie temporelle (<=> uniquement valeurs negatives)
mone = (res==-len(map_bias)).nonzero()
self.biassyst = np.zeros(res.shape)
self.biassyst[one]=1.
self.biassyst[mone]=-1.
self.biassyst = cdms2.createVariable(self.biassyst, typecode='f',id='syst_bias')
ggm = get_grid(self.model)
set_grid(self.biassyst, ggm, axes=True)
def test_constructLevelMetadata():
a = N.array([[29, 30, 31, 32], [29, 30, 31, 32]])
temp = cdms.createVariable(a, id='temp')
temp2 = cdms.createVariable(a, id='temp')
temp3 = cdms.createVariable(a, id='temp')
temp2.coordinates = "exists"
nose.tools.assert_raises(Exception, axis_utils.constructLevelMetadata,
temp2, 1, 'km')
(result_var,
result_singletonVar) = axis_utils.constructLevelMetadata(temp, 112, 'km')
#the coorinates will always have id height
assert (temp.coordinates == 'height')
singleton_atts = {
'units': 'km',
'id': 'height',
'standard_name': 'height',
'long_name': 'height',
'positive': 'up',
'axis': 'Z'
}
for key in singleton_atts.keys():
assert (getattr(result_singletonVar, key) == singleton_atts[key])
assert (result_singletonVar.getValue() == 112)
#check that any level axis is removed
temp3.getAxisList()[0].axis = 'Z'
def select_lev( mv, slev ):
"""Input is a level-dependent variable mv and a level slev to select.
slev is an instance of udunits - thus it has a value and a units attribute.
This function will create and return a new variable mvs without a level axis.
The values of mvs correspond to the values of mv with level set to slev.
Interpolation isn't done yet, but is planned !"""
levax = levAxis(mv)
# Get ig, the first index for which levax[ig]>slev
# Assume that levax values are monotonic.
dummy,slev = reconcile_units( levax, slev ) # new slev has same units as levax
if levax[0]<=levax[-1]:
ids = numpy.where( levax[:]>=slev.value ) # assumes levax values are monotonic increasing
else:
ids = numpy.where( levax[:]<=slev.value ) # assumes levax values are monotonic decreasing
if ids is None or len(ids)==0:
ig = len(levax)-1
else:
ig = ids[0][0]
# Crude fist cut: don't interpolate, just return a value
if levax == mv.getAxisList()[0]:
mvs = cdms2.createVariable( mv[ig:ig+1,...], copy=1 ) # why ig:ig+1 rather than ig? bug workaround.
elif levax == mv.getAxisList()[1]:
mvs = cdms2.createVariable( mv[:,ig:ig+1,...], copy=1 )
else:
print "ERROR, select_lev() does not support level axis except as first or second dimentions"
return None
return mvs
def systematic_bias(self):
""" Biais systematique (+1: sous-estimation du modele, -1: sur-estimation du modele) """
import numpy as np
import cdms2
from vacumm.misc.grid import get_grid, set_grid
map_bias = list() #Initialise une liste
# Estimation de la moyenne a chaque pas de temps
tps = self.obs.getTime()
for i, t in enumerate(tps):
map_bias.append(np.sign(self.obs[i, :, :] - self.model[i, :, :]))
map_bias = cdms2.createVariable(map_bias,
typecode='f',
id='computation')
res = np.sum(map_bias, axis=0)
res
# Trouve valeurs egale a la longueur de la serie temporelle (<=> uniquement valeurs positives)
one = (res == len(map_bias)).nonzero()
# Trouve valeurs egales a moins la longueur de la serie temporelle (<=> uniquement valeurs negatives)
mone = (res == -len(map_bias)).nonzero()
self.biassyst = np.zeros(res.shape)
self.biassyst[one] = 1.
self.biassyst[mone] = -1.
self.biassyst = cdms2.createVariable(self.biassyst,
typecode='f',
id='syst_bias')
ggm = get_grid(self.model)
set_grid(self.biassyst, ggm, axes=True)
def spatial_std(self):
""" Ecart-type pour chaque pas de temps """
from genutil import statistics
import cdms2
# centered and biased std (cf. http://www2-pcmdi.llnl.gov/cdat/manuals/cdutil/cdat_utilities-2.html)
self.model.spa_std = list() #Initialise une liste
self.obs.spa_std = list() #Initialise une liste
# Estimation de la std a chaque pas de temps
tps = self.obs.getTime()
for i, t in enumerate(tps):
self.model.spa_std.append(
statistics.std(self.model[i, :, :], axis='yx'))
self.obs.spa_std.append(
statistics.std(self.obs[i, :, :], axis='yx'))
# Transformation en variable cdms2
self.model.spa_std = cdms2.createVariable(
self.model.spa_std,
typecode='f',
id='model_spa_std',
attributes=dict(units=self.model.units))
self.obs.spa_std = cdms2.createVariable(
self.obs.spa_std,
typecode='f',
id='obs_spa_std',
attributes=dict(units=self.obs.units))
# Ajout de l'axe des temps correspondant a self...
self.model.spa_std.setAxis(0, tps)
self.obs.spa_std.setAxis(0, tps)
def createTransientVariableFromIndices(self, fileIndices, timeIndices):
"""
Aggregate a time file variable. Start and End Indices use slice notation.
@param fileIndices the file indices to aggregate across
@param timeIndices which time steps with in each file
@return aggregated time dep. variable. Has shape of full grid.
Subset the grid after exiting.
"""
from numpy import reshape
firsttime = True
nTSF = self.nTimeStepsPerFile
if type(fileIndices) is not int:
for files, times in zip(fileIndices, timeIndices):
for indx, file in enumerate(files):
# Should make these slices.
cvar = self.fvs[file][times[indx]]
grid = self.fvs[file].getGrid()
atts = cvar.attributes
# Insert the new time axis.
axisTime = self.fvs[file].getTime()
timeAxis = TransientAxis([file * nTSF + times[indx]],
attributes = axisTime.attributes,
id = axisTime.id)
axes = self.buildAxes(timeAxis, self.fvs[file].getAxisList())
# shape --> tm1.shape = (1, :, :)
tm1 = reshape(cvar, tuple([1] + list(cvar.shape)))
# Attach needed items
var = cdms2.createVariable(tm1,
axes = axes,
grid = grid,
attributes = atts,
id = cvar.standard_name)
# Create cdms2 transient variable
if firsttime:
new = var
firsttime = False
else:
# insert the new time axis.
taA = new.getTime()
newTime = axisConcatenate((taA, timeAxis),
attributes = axisTime.attributes,
id = axisTime.id)
axes = self.buildAxes(newTime, self.fvs[file].getAxisList())
tmp = MV2concatenate((new, var))
new = cdms2.createVariable(tmp,
axes = axes,
grid = grid,
attributes = atts,
id = cvar.standard_name)
else:
new = self.fvs[fileIndices][timeIndices]
return new
def test_005_gridsAreTheSame(self):
grid1 = cdms.createRectGrid(self.y, self.x, 'yx')
v1 = cdms.createVariable(self.arr, axes=[self.x, self.y], grid=grid1)
v2 = cdms.createVariable(self.arr, axes=[self.x, self.y], grid=grid1)
nt.assert_true(var_utils.areGridsEqual(v1, v2))
nt.assert_true(var_utils.areGridsEqual(v2, v1))
def test_004_oneGridMissing(self):
grid1 = cdms.createRectGrid(self.y, self.x, 'yx')
v1 = cdms.createVariable(self.arr, axes=[self.x, self.y], grid=grid1)
v2 = cdms.createVariable(self.arr)
nt.assert_false(var_utils.areGridsEqual(v1, v2))
nt.assert_false(var_utils.areGridsEqual(v2, v1))
def regrid_to_lower_res_1d(mv1, mv2):
"""Regrid 1-D transient variable toward lower resolution of two variables."""
if mv1 is None or mv2 is None:
return None
# missing = mv1.get_fill_value()
# latitude needs to be in ascending
axis1 = mv1.getAxisList()[0]
axis2 = mv2.getAxisList()[0]
if axis1[-1] < axis1[0]:
mv1 = mv1[::-1]
if axis2[-1] < axis2[0]:
mv2 = mv2[::-1]
axis1 = mv1.getAxisList()[0]
axis2 = mv2.getAxisList()[0]
if len(axis1) <= len(axis2):
missing = mv2.get_fill_value()
mv1_reg = mv1
b0 = numpy.interp(axis1[:],
axis2[:],
mv2[:],
left=missing,
right=missing)
b0_mask = numpy.interp(axis1[:],
axis2[:],
mv2.mask[:],
left=missing,
right=missing)
mv2_reg = cdms2.createVariable(
b0,
mask=[
True if bb == missing or bb_mask != 0.0 else False
for (bb, bb_mask) in zip(b0[:], b0_mask[:])
],
axes=[axis1])
else:
missing = mv1.get_fill_value()
a0 = numpy.interp(axis2[:],
axis1[:],
mv1[:],
left=missing,
right=missing)
a0_mask = numpy.interp(axis2[:],
axis1[:],
mv1.mask[:],
left=missing,
right=missing)
mv1_reg = cdms2.createVariable(
a0,
mask=[
True if aa == missing or aa_mask != 0.0 else False
for (aa, aa_mask) in zip(a0[:], a0_mask[:])
],
axes=[axis2])
mv2_reg = mv2
return mv1_reg, mv2_reg
def test_tidyAxisNames():
a = N.array([[[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12],
[13, 14, 15, 16]],
[[17, 18, 19, 20], [21, 22, 23, 24], [25, 26, 27, 28],
[29, 30, 31, 32]]]])
temp = cdms.createVariable(a, id='temp')
axisX = cdms.createAxis(N.array([10, 20, 30, 40]))
axisX.id = 'x'
axisX.axis = 'X' # make the axis a longitude axis
axisY = cdms.createAxis(N.array([45, 55, 65, 75]))
axisY.id = 'lat_y' # make the axis a latitude axis
axisZ = cdms.createAxis(N.array([1, 2]))
axisZ.id = 'z'
axisZ.axis = 'Z' # make a Level axis
axisT = cdms.createAxis(N.array([12]))
axisT.axis = 'T' # make a time axis
axisT.id = 't'
temp.setAxisList([axisT, axisZ, axisY, axisX])
axis_utils.tidyAxisNames(temp)
assert(temp.getLongitude().id == 'longitude' and \
temp.getLongitude().long_name =='Longitude')
assert(temp.getLatitude().id == 'latitude' and \
temp.getLatitude().long_name =='Latitude')
assert(temp.getLevel().id == 'level' and \
temp.getLevel().long_name =='Vertical Level')
assert(temp.getTime().id == 'time' and \
temp.getTime().long_name =='Time')
assert (hasattr(temp, "cell_methods") == False)
assert (hasattr(temp, "long_name") == False)
temp2 = cdms.createVariable(a, id='temp')
axisX = cdms.createAxis(N.array([10, 20, 30, 40]))
axisX.id = 'x'
axisX.axis = 'X' # make the axis a longitude axis
axisY = cdms.createAxis(N.array([45, 55, 65, 75]))
axisY.id = 'lat_y' # make the axis a latitude axis
axisZ = cdms.createAxis(N.array([1, 2]))
axisZ.id = 'z'
axisZ.axis = 'Z' # make a Level axis
axisT = cdms.createAxis(N.array([12]))
axisT.axis = 'T' # make a time axis
axisT.id = 't'
temp2.setAxisList([axisT, axisZ, axisY, axisX])
temp2.cell_methods = 'x:lat_y:z:something else,t:'
temp2.long_name = 'some data about x:lat_y:z: at level t:'
axis_utils.tidyAxisNames(temp2)
assert (
temp2.cell_methods == 'longitude:latitude:level:something else,time:')
assert (temp2.long_name ==
'some data about longitude:latitude:level: at level time:')
def test_003_gridLatitudeDifferent(self):
grid1 = cdms.createRectGrid(self.y, self.x, 'yx')
y2 = cdms.createAxis([8.0, 10.0], id='y')
grid2 = cdms.createRectGrid(y2, self.x, 'yx')
v1 = cdms.createVariable(self.arr, axes=[self.x, self.y], grid=grid1)
v2 = cdms.createVariable(self.arr, axes=[self.x, y2], grid=grid2)
nt.assert_false(var_utils.areGridsEqual(v1, v2))
def test_tidyAxisNames():
a = N.array([[[[ 1, 2, 3, 4], [ 5, 6, 7, 8], [ 9, 10, 11, 12], [13, 14, 15, 16]],
[[17, 18, 19, 20], [21, 22, 23, 24], [25, 26, 27, 28], [29, 30, 31, 32]]]])
temp = cdms.createVariable(a,id='temp')
axisX = cdms.createAxis(N.array([10, 20, 30, 40]))
axisX.id = 'x'
axisX.axis = 'X' # make the axis a longitude axis
axisY = cdms.createAxis(N.array([45, 55, 65, 75]))
axisY.id = 'lat_y' # make the axis a latitude axis
axisZ = cdms.createAxis(N.array([1, 2]))
axisZ.id = 'z'
axisZ.axis = 'Z' # make a Level axis
axisT = cdms.createAxis(N.array([12]))
axisT.axis = 'T' # make a time axis
axisT.id = 't'
temp.setAxisList([axisT, axisZ, axisY, axisX])
axis_utils.tidyAxisNames(temp)
assert(temp.getLongitude().id == 'longitude' and \
temp.getLongitude().long_name =='Longitude')
assert(temp.getLatitude().id == 'latitude' and \
temp.getLatitude().long_name =='Latitude')
assert(temp.getLevel().id == 'level' and \
temp.getLevel().long_name =='Vertical Level')
assert(temp.getTime().id == 'time' and \
temp.getTime().long_name =='Time')
assert(hasattr(temp, "cell_methods") == False)
assert(hasattr(temp, "long_name") == False)
temp2 = cdms.createVariable(a,id='temp')
axisX = cdms.createAxis(N.array([10, 20, 30, 40]))
axisX.id = 'x'
axisX.axis = 'X' # make the axis a longitude axis
axisY = cdms.createAxis(N.array([45, 55, 65, 75]))
axisY.id = 'lat_y' # make the axis a latitude axis
axisZ = cdms.createAxis(N.array([1, 2]))
axisZ.id = 'z'
axisZ.axis = 'Z' # make a Level axis
axisT = cdms.createAxis(N.array([12]))
axisT.axis = 'T' # make a time axis
axisT.id = 't'
temp2.setAxisList([axisT, axisZ, axisY, axisX])
temp2.cell_methods = 'x:lat_y:z:something else,t:'
temp2.long_name = 'some data about x:lat_y:z: at level t:'
axis_utils.tidyAxisNames(temp2)
assert(temp2.cell_methods == 'longitude:latitude:level:something else,time:')
assert(temp2.long_name == 'some data about longitude:latitude:level: at level time:')
def _wrap_cdms(solution, neofs, time_units):
try:
import cdms2
except ImportError:
raise ValueError("cannot use container 'cdms' without "
"the cdms2 module")
time_dim = cdms2.createAxis(solution['time'], id='time')
time_dim.designateTime()
time_dim.units = time_units
lat_dim = cdms2.createAxis(solution['latitude'], id='latitude')
lat_dim.designateLatitude()
lon_dim = cdms2.createAxis(solution['longitude'], id='longitude')
lon_dim.designateLongitude()
eof_dim = cdms2.createAxis(np.arange(1, neofs+1), id='eof')
eof_dim.long_name = 'eof_number'
solution['sst'] = cdms2.createVariable(
solution['sst'],
axes=[time_dim, lat_dim, lon_dim],
id='sst')
solution['eigenvalues'] = cdms2.createVariable(
solution['eigenvalues'],
axes=[eof_dim],
id='eigenvalues')
solution['eofs'] = cdms2.createVariable(
solution['eofs'],
axes=[eof_dim, lat_dim, lon_dim],
id='eofs')
solution['pcs'] = cdms2.createVariable(
solution['pcs'],
axes=[time_dim, eof_dim],
id='pcs')
solution['variance'] = cdms2.createVariable(
solution['variance'],
axes=[eof_dim],
id='variance')
solution['eofscor'] = cdms2.createVariable(
solution['eofscor'],
axes=[eof_dim, lat_dim, lon_dim],
id='eofscor')
solution['eofscov'] = cdms2.createVariable(
solution['eofscov'],
axes=[eof_dim, lat_dim, lon_dim],
id='eofscov')
solution['errors'] = cdms2.createVariable(
solution['errors'],
axes=[eof_dim],
id='errors')
solution['scaled_errors'] = cdms2.createVariable(
solution['scaled_errors'],
axes=[eof_dim],
id='scaled_errors')
solution['rcon'] = cdms2.createVariable(
solution['rcon'],
axes=[time_dim, lat_dim, lon_dim],
id='reconstructed_sst')
def test_002_dataIsNotEqual(self):
arr = N.arange(10.0, 20.0, 0.25)
arr = arr.reshape(8, 5)
v1 = cdms.createVariable(arr, id='v1')
arr = N.arange(10.0, 20.0, 0.25)
arr = arr.reshape(8, 5)
arr[5, 2] = 18.25
v2 = cdms.createVariable(arr, id='v2')
nt.assert_false(var_utils.isVariableDataEqual(v1, v2))
def test_005_areTheSame(self):
grid = cdms.createRectGrid(self.y, self.x, 'yx')
v1 = cdms.createVariable(self.arr,
axes=[self.x, self.y],
grid=grid,
attributes={'name': 'v1'})
v2 = cdms.createVariable(self.arr,
axes=[self.x, self.y],
grid=grid,
attributes={'name': 'v1'})
nt.assert_true(var_utils.areVariablesEqual(v1, v2))
def makeGrid(nx,ny, order):
dims = (nx, ny)
xbot, xtop, ybot, ytop = 0, 359, -89, 89
x = numpy.linspace(xbot, xtop, nx)
y = numpy.linspace(ybot, ytop, ny)
theGrid = cdms2.grid.createUniformGrid(ybot, ny, (2*ytop)/float(ny-1),
xbot, nx, xtop/float(nx-1),
order = order)
if order == 'xy':
xx = numpy.outer(x, numpy.ones(ny))
yy = numpy.outer(numpy.ones(nx), y)
axisList = [theGrid.getLongitude(), theGrid.getLatitude()]
else:
xx = numpy.outer(numpy.ones(ny), x)
yy = numpy.outer(y, numpy.ones(nx))
axisList = [theGrid.getLatitude(), theGrid.getLongitude()]
theData = xx + yy
# Make a cdms2 variable
cdmsVar = cdms2.createVariable(theData, axes = axisList,
grid = theGrid,
id = 'TestOrderInterp')
return cdmsVar
def _readCDMSFile(self, var_ids=None, exclude_vars=[]):
"""
Reads the file and returns all the CDMS variables in a list as well
as the global attributes: (cdms_variable_list, global_atts_list)
If var_ids is defined then only get those.
"""
fin = cdms.open(self.nc_file)
cdms_variables = []
# Make sure var_ids is a list
if type(var_ids) == type("string"):
var_ids = [var_ids]
for var_id in fin.listvariables():
if var_ids == None or var_id in var_ids:
if var_id not in exclude_vars:
# Check whether singleton variable, if so create variable
vm = fin[var_id]
var = fin(var_id)
if hasattr(vm, "rank") and vm.rank() == 0:
var = cdms.createVariable(numpy.array(float(fin(var_id))), id=vm.id, attributes=vm.attributes)
cdms_variables.append(var)
globals = fin.attributes.items()
return (cdms_variables, globals)
def spatial_rmsc(self):
""" RMS centree pour chaque pas de temps """
from genutil import statistics
import cdms2
import numpy as np
# centered and biased RMS
self.spa_rmsc = list() #Initialise une liste
# Estimation de la std a chaque pas de temps
tps = self.obs.getTime()
for i, t in enumerate(tps):
minterm = np.reshape(self.model[i, :, :],
(self.model.shape[-1] * self.model.shape[-2]))
ointerm = np.reshape(self.obs[i, :, :],
(self.obs.shape[-1] * self.obs.shape[-2]))
#modif J.Gatti - pour MFS
if not minterm._grid_ is None:
minterm._grid_ = None
if not ointerm._grid_ is None:
ointerm._grid_ = None
#fin modif J.Gatti
self.spa_rmsc.append(statistics.rms(minterm, ointerm, centered=1))
# Transformation en variable cdms2
self.spa_rmsc = cdms2.createVariable(
self.spa_rmsc,
typecode='f',
id='spa_rmsc',
attributes=dict(units=self.model.units))
# Ajout de l'axe des temps correspondant a self...
self.spa_rmsc.setAxis(0, tps)
def test3DRect():
"""
Test data on 3d rectilinear grid
"""
import cdms2
from numpy import pi, cos, sin, exp
nelv, nlat, nlon = 3, 4, 5
delv, dlon, dlat = 90000./float(nelv-1), \
60.0/float(nlon-1), 30.0/float(nlat-1)
elvs1D = numpy.array([100000 - i*delv for i in range(nelv)])
lons1D = numpy.array([0.0 + i*dlon for i in range(nlon)])
lats1D = numpy.array([0.0 + i*dlat for i in range(nlat)])
# any order should work
lons = numpy.zeros( (nlon, nlat, nelv), numpy.float32 )
lats = numpy.zeros( (nlon, nlat, nelv), numpy.float32 )
elvs = numpy.zeros( (nlon, nlat, nelv), numpy.float32 )
data = numpy.zeros( (nlon, nlat, nelv), numpy.float32 )
for i in range(nlon):
for j in range(nlat):
for k in range(nelv):
elvs[i, j, k] = elvs1D[k]
lats[i, j, k] = lats1D[j]
lons[i, j, k] = lons1D[i]
data[i, j, k] = cos(3*pi*lats[i, j, k]/180.) * \
sin(5*pi*lons[i, j, k]/180.) * exp(-elvs[i, j, k])
var = cdms2.createVariable(data, id='fake_data_3d_rect',
axes=(elvs, lats, lons))
sphere_mesh = SphereMesh(var)
print sphereMesh.getXYZCoords()
def varianceFraction(self, neigs=None):
"""Fractional EOF variances.
The fraction of the total variance explained by each EOF, a
value between 0 and 1 inclusive, in a :py:mod:`cdms2` variable.
**Optional argument:**
*neigs*
Number of eigenvalues to return the fractional variance for.
Defaults to all eigenvalues.
**Examples:**
The fractional variance represented by each eigenvalue:
>>> varfrac = eofobj.varianceFraction()
The fractional variance represented by the first 3 eigenvalues:
>>> varfrac = eofobj.VarianceFraction(neigs=3)
"""
vfrac = self._solver.varianceFraction(neigs=neigs)
eofax = cdms2.createAxis(range(len(vfrac)), id="eigenvalue")
axlist = [eofax]
vfrac = cdms2.createVariable(vfrac, id="variance", axes=axlist)
vfrac.name = "variance_fraction"
vfrac.long_name = "variance fraction"
return vfrac
def test3DposDown():
"""
Test 3d data with elev positive down. Need to work with 1D axes.
"""
print 'test positive down'
import cdms2
import numpy
nlev, nlat, nlon = 4, 5, 6
dlev, dlat, dlon = 5000./float(nlev-1), 180./float(nlat-1), 360./float(nlon-1)
levs1d = numpy.arange(0., 5001., dlev)
lats1d = numpy.array([0. - i*dlat for i in range(nlat)])
lons1d = numpy.array([0. - i*dlon for i in range(nlon)])
data = numpy.zeros((nlev, nlat, nlon), numpy.float32)
for k in range(nlev):
for j in range(nlat):
for i in range(nlon):
data[k, j, i] = numpy.cos(3*numpy.pi*lats1d[j]/180.) * \
numpy.sin(5*numpy.pi*lons1d[i]/180.) * \
numpy.exp(-levs1d[k])
a1 = cdms2.axis.TransientAxis(levs1d, id = 'levels',
attributes = {'positive':'down'})
a2 = cdms2.axis.TransientAxis(lats1d, id = 'latitude')
a3 = cdms2.axis.TransientAxis(lons1d, id = 'longitude')
var = cdms2.createVariable(data, id = 'pos_down_3d_data',
axes = (a1, a2, a3))
sphereMesh = SphereMesh(var)
aa = sphereMesh.getXYZCoords()
bb = aa.reshape((4, 5, 6, 3))
for i in range(nlev): print levs1d[i], bb[i, 0, 0, :]
def _convertSingletonVars(self, variables):
"""
Loops through variables to convert singleton variables (i.e. Masked Arrays/Numeric Arrays)
to proper CDMS variables. Then code won't break when asking for rank attribute later.
Returns a list of CDMS variable objects
"""
vars = []
for variable in variables:
var_obj = variable
# If singleton variable then convert into proper CDMS variables so code doesn't break later
if not hasattr(var_obj, "rank"):
var_metadata = var_obj.attributes
var_value = var_obj
var_obj = cdms.createVariable(
N.array(var_obj),
id=cdms_utils.var_utils.getBestName(var_metadata).replace(
" ", "_"),
attributes=var_metadata)
var_obj.value = var_obj._data[0]
vars.append(var_obj)
return vars
def testReadCurvGrid(self):
f = self.getDataFile('sampleCurveGrid4.nc')
data = f("sample")
attrs = copy.copy(data.attributes)
axes = list([x[0].clone() for x in data.getDomain()])
data2 = cdms2.createVariable(data, copy=0, attributes=attrs, axes=axes)
data3=data2(*(),squeeze=1)
def do_avg(infile, inpath, variable, nodata, outfile):
# for netcdf3: set flags to 0
cdms2.setNetcdfShuffleFlag(1)
cdms2.setNetcdfDeflateFlag(1)
cdms2.setNetcdfDeflateLevelFlag(3)
# note that this version will erase data whereever a nodata is found in the series
avg=None
nodatamask = None
for ifile in infile:
fname = os.path.join(inpath, ifile)
if not os.path.exists(fname): messageOnExit('file {0} not found on path {1}. Exit(100).'.format(ifile, path), 100)
thisfile = cdms2.open(fname, 'r')
if avg is None:
avg = numpy.array(thisfile[variable][:])
nodatamask = avg >= nodata
else:
avg = avg + numpy.array(thisfile[variable][:])
thisfile.close()
avg = avg/len(infile)
if nodatamask.any():
avg[nodatamask] = nodata
if os.path.exists(outfile): os.remove(outfile)
outfh = cdms2.open(outfile, 'w')
outvar=cdms2.createVariable(avg, typecode='f', id=variable, fill_value=nodata )
outfh.write(outvar)
outfh.close()
def testVarID(self):
NLAT = 16
NLON = 32
latarr = numpy.ma.arange(NLAT) * (180. / (NLAT - 1)) - 90.
lonarr = numpy.ma.arange(NLON) * (360.0 / NLON)
timearr = numpy.ma.array([0.0, 366.0, 731.0])
u = self.test_arr[0]
tobj = cdms2.createAxis(id='time', data=timearr)
tobj.units = 'days since 2000-1-1'
latobj = cdms2.createAxis(id='latitude', data=latarr)
latobj.units = 'degrees_north'
lonobj = cdms2.createAxis(id='longitude', data=lonarr)
lonobj.units = 'degrees_east'
varid = "1234567890" * 26
var = cdms2.createVariable(u,
id=varid,
typecode=numpy.float,
axes=(tobj, latobj, lonobj))
var.units = 'm/s'
with cdms2.open("bad.nc", "w") as f:
f.write(var)
f.close()
f = cdms2.open("bad.nc")
listvar = f.listvariables()
self.assertEqual(listvar[2], varid[:127])
f.close()
def test3DposDown():
"""
Test 3d data with elev positive down. Need to work with 1D axes.
"""
print('test positive down')
import cdms2
import numpy
nlev, nlat, nlon = 4, 5, 6
dlev, dlat, dlon = 5000. / \
float(nlev - 1), 180. / float(nlat - 1), 360. / float(nlon - 1)
levs1d = numpy.arange(0., 5001., dlev)
lats1d = numpy.array([0. - i * dlat for i in range(nlat)])
lons1d = numpy.array([0. - i * dlon for i in range(nlon)])
data = numpy.zeros((nlev, nlat, nlon), numpy.float32)
for k in range(nlev):
for j in range(nlat):
for i in range(nlon):
data[k, j, i] = numpy.cos(3 * numpy.pi * lats1d[j] / 180.) * \
numpy.sin(5 * numpy.pi * lons1d[i] / 180.) * \
numpy.exp(-levs1d[k])
a1 = cdms2.axis.TransientAxis(levs1d, id='levels',
attributes={'positive': 'down'})
a2 = cdms2.axis.TransientAxis(lats1d, id='latitude')
a3 = cdms2.axis.TransientAxis(lons1d, id='longitude')
var = cdms2.createVariable(data, id='pos_down_3d_data',
axes=(a1, a2, a3))
sphereMesh = SphereMesh(var)
aa = sphereMesh.getXYZCoords()
bb = aa.reshape((4, 5, 6, 3))
for i in range(nlev):
print(levs1d[i], bb[i, 0, 0, :])
def makeDummyNC(filename):
''' This makes some dummy data for use in test routines '''
f=cdms2.open(filename,'w')
nx,ny,nz,nt=30,12,4,12
lat=np.linspace(-30,30,ny)
lon=np.linspace(0,330,nx)
z=np.linspace(1,4,nz)
t=np.linspace(1,14,nt)
tb=np.array([[ti,ti+1] for ti in t])
tax=cdms2.createAxis(t)
tax.id='time'
tax.units='days since 2013-1-1'
tax.setBounds(tb)
xax=cdms2.createAxis(lon)
xax.id='longitude'
xax.units='degrees_east'
yax=cdms2.createAxis(lat)
yax.id='latitude'
yax.units='degrees_north'
zax=cdms2.createAxis(z)
zax.id='levels'
d=np.outer(np.sin(lon),np.cos(lat))
data=np.resize(d,(nx,ny,nz,nt))
dv=cdms2.createVariable(data,axes=[xax,yax,zax,tax],fill_value=-999.)
dv.long_name='Dummy Data'
dv.units='Kelvin'
shape=dv.shape
f.write(dv)
f.close()
return shape
def reduce2lat_old( mv, vid=None ):
"""returns the mean of the variable over all axes but latitude, as a cdms2 variable, i.e. a MV.
The input mv is a also cdms2 variable, assumed to be indexed as is usual for CF-compliant
variables, i.e. mv(time,lat,lon). At present, no other axes (e.g. level) are supported.
At present mv must depend on all three axes.
"""
# >>> For now, I'm assuming that the only axes are time,lat,lon
# And I'm assuming equal spacing in lon (so all longitudes contribute equally to the average)
# If they aren't, it's best to use area from cell_measures attribute if available; otherwise
# compute it with lat_bnds, lon_bnds etc.
# If I base another reduction function on this one, it's important to note that an average
# in the lat direction will unavoidably need weights, because of the geometry.
if vid==None:
vid = 'reduced_'+mv.id
time_axis, lat_axis, lon_axis = tllAxes( mv )
mvta = timeave_old( mv )
zm = numpy.zeros( mvta.shape[0] )
for i in range(len(lat_axis)):
for j in range(len(lon_axis)):
zm[i] += mvta[i,j]
zm[i] /= len(lon_axis)
zmv = cdms2.createVariable( zm, axes=[lat_axis], id=vid )
return zmv
def reconstructedField(self, neofs):
"""Reconstructed data field based on a subset of EOFs.
If weights were passed to the :py:class:`~eof2.MultipleEof`
instance then the returned reconstructed fields will be
automatically un-weighted. Otherwise the returned reconstructed
field will be weighted in the same manner as the input to the
:py:class:`~eof2.MultipleEof` instance.
**Argument:**
*neofs*
Number of EOFs to use for the reconstruction.
**Examples:**
Reconstruct the input fields using 3 EOFs.
>>> rfield_a, rfield_b = eofobj.reconstructedField(neofs=3)
"""
rfset = self._solver.reconstructedField(neofs)
for iset in xrange(self._numdsets):
axlist = [self._timeax] + self._multichannels[iset]
rfset[iset].fill_value = self._multimissing[iset]
rfset[iset] = cdms2.createVariable(
rfset[iset],
id="rcon",
axes=axlist,
fill_value=self._multimissing[iset])
rfset[iset].long_name = "reconstructed_field"
return rfset
def reference_solutions(container_type, gridtype):
"""Generate reference solutions in the required container."""
container_type = container_type.lower()
if container_type not in ("standard", "iris", "cdms"):
raise ValueError("unknown container type: " "'{!s}'".format(container_type))
reference = __read_reference_solutions()
if container_type == "standard":
# Reference solution already in numpy arrays.
return reference
# Generate coordinate dimensions for meta-data interfaces.
lons = np.arange(0, 360, 2.5)
lats = np.linspace(90, -90, 73)
if container_type == "cdms":
# Solution in cdms2 variables.
try:
londim = cdms2.createAxis(lons, id="longitude")
londim.designateLongitude()
latdim = cdms2.createAxis(lats, id="latitude")
latdim.designateLatitude()
for name in reference.keys():
reference[name] = cdms2.createVariable(reference[name], axes=[latdim, londim], id=name)
except NameError:
raise ValueError("cannot use container 'cdms' without cdms2")
elif container_type == "iris":
# Solution in iris cubes.
try:
londim = DimCoord(lons, standard_name="longitude", units="degrees_east")
latdim = DimCoord(lats, standard_name="latitude", units="degrees_north")
coords = zip((latdim, londim), (0, 1))
for name in reference.keys():
reference[name] = Cube(reference[name], dim_coords_and_dims=coords, long_name=name)
except NameError:
raise ValueError("cannot use container 'iris' without iris")
return reference
def eigenvalues(self, neigs=None):
"""Eigenvalues (decreasing variances) associated with each EOF.
**Optional argument:**
*neigs*
Number of eigenvalues to return. Defaults to all
eigenvalues.If the number of eigenvalues requested is more
than the number that are available, then all available
eigenvalues will be returned.
**Returns:**
*eigenvalues*
A `cdms2` variable containing the eigenvalues arranged
largest to smallest.
**Examples:**
All eigenvalues::
eigenvalues = solver.eigenvalues()
The first eigenvalue::
eigenvalue1 = solver.eigenvalues(neigs=1)
"""
lambdas = self._solver.eigenvalues(neigs=neigs)
eofax = cdms2.createAxis(range(len(lambdas)), id='eigenvalue')
eofax.long_name = 'eigenvalue_number'
axlist = [eofax]
lambdas = cdms2.createVariable(lambdas, id='eigenvalues', axes=axlist)
lambdas.long_name = 'eigenvalues'
return lambdas
def _readCDMSFile(self, var_ids=None, exclude_vars=[]):
"""
Reads the file and returns all the CDMS variables in a list as well
as the global attributes: (cdms_variable_list, global_atts_list)
If var_ids is defined then only get those.
"""
fin = cdms.open(self.nc_file)
cdms_variables = []
# Make sure var_ids is a list
if type(var_ids) == type("string"):
var_ids = [var_ids]
for var_id in fin.listvariables():
if var_ids == None or var_id in var_ids:
if var_id not in exclude_vars:
# Check whether singleton variable, if so create variable
vm = fin[var_id]
var = fin(var_id)
if hasattr(vm, "rank") and vm.rank() == 0:
var = cdms.createVariable(numpy.array(
float(fin(var_id))),
id=vm.id,
attributes=vm.attributes)
cdms_variables.append(var)
globals = fin.attributes.items()
return (cdms_variables, globals)
def test2D():
"""
Test data on 2D curvilinear grid
"""
import cdms2
from cdms2.coord import TransientAxis2D, TransientVirtualAxis
from cdms2.hgrid import TransientCurveGrid
from numpy import pi, cos, sin
nlat, nlon = 3, 4
dlon, dlat = 60.0/float(nlon - 1), 30.0/float(nlat - 1)
lons1D = numpy.array([0.0 + i*dlon for i in range(nlon)])
lats1D = numpy.array([0.0 + j*dlat for j in range(nlat)])
lons = numpy.outer(numpy.ones((nlat,)), lons1D)
lats = numpy.outer(lats1D, numpy.ones((nlon,)))
data = cos(3*pi*lats/180.0) * sin(5*pi*lons/180.0)
# create grid
iaxis = TransientVirtualAxis("i", nlon)
jaxis = TransientVirtualAxis("j", nlat)
lataxis = TransientAxis2D(lats,
axes=(jaxis, iaxis),
attributes={'units': 'degree_north'},
id='lats')
lonaxis = TransientAxis2D(lons,
axes=(jaxis, iaxis),
attributes={'units': 'degree_east'},
id='lons')
grid = TransientCurveGrid(lataxis, lonaxis, id='lats_lons')
var = cdms2.createVariable(data, id='fake_data_2d',
axes = grid.getAxisList(),
grid = grid,
attributes = {'coordinates': 'lats lons'},
)
sphere_mesh = SphereMesh(var)
print sphere_mesh.getXYZCoords()
def _convertNAToCdmsVariable(self, var_number, attributes={}):
"""
Creates a single cdms variable from the variable number provided in the list.
"""
(var_name, units, miss, scal) = self.na_file_obj.getVariable(var_number)
msg = "\nAdding variable: %s" % self.na_file_obj.VNAME[var_number]
log.debug(msg)
self.output_message.append(msg)
array = N.array(self.na_file_obj.V[var_number])
array = array * scal
# Set up axes
if not hasattr(self, 'cdms_axes'):
self._convertCdmsAxes()
# Set up variable
var=cdms.createVariable(array, axes=self.cdms_axes, fill_value=miss, attributes=attributes)
# Sort units etc
if units: var.units=units
# Add the best variable name
if len(var_name) < max_id_length:
var.id=safe_nc_id.sub("_", var_name).lower()
else:
var.id="naVariable_%s" % (var_number)
# Check if mapping provided for renaming this variable
if var_name in self.rename_variables.keys():
var_name = self.rename_variables[var_name]
var.long_name = var.name = var.title = var_name
# Add a NASA Ames variable number (for mapping correctly back to NASA Ames)
var.nasa_ames_var_number = var_number
return var
def to_cdms2(dataarray, copy=True):
"""Convert a DataArray into a cdms2 variable
"""
# we don't want cdms2 to be a hard dependency
import cdms2
def set_cdms2_attrs(var, attrs):
for k, v in attrs.items():
setattr(var, k, v)
# 1D axes
axes = []
for dim in dataarray.dims:
coord = encode(dataarray.coords[dim])
axis = cdms2.createAxis(coord.values, id=dim)
set_cdms2_attrs(axis, coord.attrs)
axes.append(axis)
# Data
var = encode(dataarray)
cdms2_var = cdms2.createVariable(
var.values, axes=axes, id=dataarray.name, mask=pd.isnull(var.values), copy=copy
)
# Attributes
set_cdms2_attrs(cdms2_var, var.attrs)
# Curvilinear and unstructured grids
if dataarray.name not in dataarray.coords:
cdms2_axes = {}
for coord_name in set(dataarray.coords.keys()) - set(dataarray.dims):
coord_array = dataarray.coords[coord_name].to_cdms2()
cdms2_axis_cls = (
cdms2.coord.TransientAxis2D
if coord_array.ndim
else cdms2.auxcoord.TransientAuxAxis1D
)
cdms2_axis = cdms2_axis_cls(coord_array)
if cdms2_axis.isLongitude():
cdms2_axes["lon"] = cdms2_axis
elif cdms2_axis.isLatitude():
cdms2_axes["lat"] = cdms2_axis
if "lon" in cdms2_axes and "lat" in cdms2_axes:
if len(cdms2_axes["lon"].shape) == 2:
cdms2_grid = cdms2.hgrid.TransientCurveGrid(
cdms2_axes["lat"], cdms2_axes["lon"]
)
else:
cdms2_grid = cdms2.gengrid.AbstractGenericGrid(
cdms2_axes["lat"], cdms2_axes["lon"]
)
for axis in cdms2_grid.getAxisList():
cdms2_var.setAxis(cdms2_var.getAxisIds().index(axis.id), axis)
cdms2_var.setGrid(cdms2_grid)
return cdms2_var
def reconstructedField(self, neofs):
"""Reconstructed data field based on a subset of EOFs.
If weights were passed to the :py:class:`~eof2.MultipleEof`
instance then the returned reconstructed fields will be
automatically un-weighted. Otherwise the returned reconstructed
field will be weighted in the same manner as the input to the
:py:class:`~eof2.MultipleEof` instance.
**Argument:**
*neofs*
Number of EOFs to use for the reconstruction.
**Examples:**
Reconstruct the input fields using 3 EOFs.
>>> rfield_a, rfield_b = eofobj.reconstructedField(neofs=3)
"""
rfset = self._solver.reconstructedField(neofs)
for iset in xrange(self._numdsets):
axlist = [self._timeax] + self._multichannels[iset]
rfset[iset].fill_value = self._multimissing[iset]
rfset[iset] = cdms2.createVariable(rfset[iset], id="rcon", axes=axlist,
fill_value=self._multimissing[iset])
rfset[iset].long_name = "reconstructed_field"
return rfset
def test_projectfield_dimensions_invalid(self):
"""Projection onto EOFs of field with incorrect dimensionality."""
levax = cdms2.createAxis([0], id='level')
var = cdms2.createVariable(
cdms2.MV.reshape(self.sf, self.sf.shape+(1,)),
axes=self.sf.getAxisList()+[levax], id=self.sf.id)
pf = self.eofobj.projectField(var)
def eofsAsCovariance(self, neofs=None, pcscaling=1):
"""
Empirical orthogonal functions (EOFs) expressed as the
covariance between the principal component time series (PCs)
and the time series of the `Eof` input *dataset* at each grid
point.
**Optional arguments:**
*neofs*
Number of EOFs to return. Defaults to all EOFs. If the
number of EOFs requested is more than the number that are
available, then all available EOFs will be returned.
*pcscaling*
Set the scaling of the PCs used to compute covariance. The
following values are accepted:
* *0* : Un-scaled PCs.
* *1* : PCs are scaled to unit variance (divided by the
square-root of their eigenvalue) (default).
* *2* : PCs are multiplied by the square-root of their
eigenvalue.
The default is to divide PCs by the square-root of their
eigenvalue so that the PCs are scaled to unit variance
(option 1).
**Returns:**
*eofs*
A `cdms2` variable containing the ordered EOFs. The EOFs are
numbered from 0 to *neofs* - 1.
**Examples:**
All EOFs::
eofs = solver.eofsAsCovariance()
The leading EOF::
eof1 = solver.eofsAsCovariance(neofs=1)
The leading EOF using un-scaled PCs::
eof1 = solver.eofsAsCovariance(neofs=1, pcscaling=0)
"""
eofs = self._solver.eofsAsCovariance(neofs, pcscaling)
eofs.fill_value = self._missing_value
eofax = cdms2.createAxis(range(len(eofs)), id='eof')
axlist = [eofax] + self._channels
eofs = cdms2.createVariable(eofs,
id='eofs_cov',
axes=axlist,
fill_value=self._missing_value)
eofs.long_name = 'covariance_between_pcs_and_{:s}'.format(
self._dataset_name)
return eofs
def eigenvalues(self, neigs=None):
"""Eigenvalues (decreasing variances) associated with each EOF.
Returns the ordered eigenvalues in a :py:mod:`cdms2` variable.
**Optional argument:**
*neigs*
Number of eigenvalues to return. Defaults to all
eigenvalues.
**Examples:**
All eigenvalues:
>>> lambdas = eofobj.eigenvalues()
The first eigenvalue:
>>> lambda1 = eofobj.eigenvalues(neigs=1)
"""
lambdas = self._solver.eigenvalues(neigs=neigs)
eofax = cdms2.createAxis(range(len(lambdas)), id="eigenvalue")
axlist = [eofax]
lambdas = cdms2.createVariable(lambdas, id="eigenvalues", axes=axlist)
lambdas.name = "eigenvalues"
lambdas.long_name = "eigenvalues"
return lambdas
def spatial_rmsc(self):
""" RMS centree pour chaque pas de temps """
from genutil import statistics
import cdms2
import numpy as np
# centered and biased RMS
self.spa_rmsc = list() #Initialise une liste
# Estimation de la std a chaque pas de temps
tps = self.obs.getTime()
for i, t in enumerate(tps):
minterm = np.reshape(self.model[i, :, :], (self.model.shape[-1]*self.model.shape[-2]))
ointerm = np.reshape(self.obs[i, :, :], (self.obs.shape[-1]*self.obs.shape[-2]))
#modif J.Gatti - pour MFS
if not minterm._grid_ is None:
minterm._grid_=None
if not ointerm._grid_ is None:
ointerm._grid_=None
#fin modif J.Gatti
self.spa_rmsc.append(statistics.rms(minterm, ointerm, centered=1))
# Transformation en variable cdms2
self.spa_rmsc = cdms2.createVariable(self.spa_rmsc,typecode='f',id='spa_rmsc', attributes=dict(units=self.model.units))
# Ajout de l'axe des temps correspondant a self...
self.spa_rmsc.setAxis(0, tps)
def minmin_maxmax( *args ):
"""returns a TransientVariable containing the minimum and maximum values of all the variables
provided as arguments"""
rmin = min( [ mv.min() for mv in args ] )
rmax = max( [ mv.max() for mv in args ] )
rmv = cdms2.createVariable( [rmin,rmax] )
return rmv
def reconstructedField(self, neofs):
"""Reconstructed data field based on a subset of EOFs.
If weights were passed to the :py:class:`~eof2.Eof` instance
then the returned reconstructed field will be automatically
un-weighted. Otherwise the returned reconstructed field will
be weighted in the same manner as the input to the
:py:class:`~eof2.Eof` instance.
**Argument:**
*neofs*
Number of EOFs to use for the reconstruction.
**Examples:**
Reconstruct the input field using 3 EOFs.
>>> rfield = eofobj.reconstructedField(neofs=3)
"""
rfield = self._solver.reconstructedField(neofs)
rfield.fill_value = self._missing_value
axlist = [self._timeax] + self._channels
rfield = cdms2.createVariable(rfield, id="rcon", axes=axlist,
fill_value=self._missing_value)
rfield.long_name = "reconstructed_field"
return rfield
def varianceFraction(self, neigs=None):
"""Fractional EOF variances.
The fraction of the total variance explained by each EOF, a
value between 0 and 1 inclusive, in a :py:mod:`cdms2` variable.
**Optional argument:**
*neigs*
Number of eigenvalues to return the fractional variance for.
Defaults to all eigenvalues.
**Examples:**
The fractional variance represented by each eigenvalue:
>>> varfrac = eofobj.varianceFraction()
The fractional variance represented by the first 3 eigenvalues:
>>> varfrac = eofobj.VarianceFraction(neigs=3)
"""
vfrac = self._solver.varianceFraction(neigs=neigs)
eofax = cdms2.createAxis(range(len(vfrac)), id="eigenvalue")
axlist = [eofax]
vfrac = cdms2.createVariable(vfrac, id="variance", axes=axlist)
vfrac.name = "variance_fraction"
vfrac.long_name = "variance fraction"
return vfrac
def setup():
global arr, arrLowerBound1, arrUpperBound10, arrLowerBound1AndUpperBound10, tempVar
global arrMaskBelow5, arrMaskAbove12, arrMaskBelow5Above12, missing
arr = N.array([
[ 1.2, 1.0, 8.3, 4.6, 0.3],
[53.2, 1.2, 1.2, 4.5, 12.4]])
arrLowerBound1 = [
[ 1.2, 1.0, 8.3, 4.6, 1.0],
[53.2, 1.2, 1.2, 4.5, 12.4]]
arrUpperBound10 = [
[ 1.2, 1.0, 8.3, 4.6, 0.3],
[10.0, 1.2, 1.2, 4.5, 10.0]]
arrLowerBound1AndUpperBound10 = [
[ 1.2, 1.0, 8.3, 4.6, 1.0],
[10.0, 1.2, 1.2, 4.5, 10.0]]
m = missing
arrMaskBelow5 =[
[m, m, 8.3, m, m],
[53.2, m, m, m, 12.4]]
arrMaskAbove12 = [
[ 1.2, 1.0, 8.3, 4.6, 0.3],
[m, 1.2, 1.2, 4.5, m]]
arrMaskBelow5Above12 = [
[m, m, 8.3, m, m],
[m, m, m, m, m]]
tempVar = cdms.createVariable(arr,id='temp')
tempVar.setMissing(missing)
def CurveGrid(v, lat, lon):
ni, nj = lat.shape
idi = "i"
idj = "j"
lat_units = 'degrees_north'
lon_units = 'degrees_east'
iaxis = TransientVirtualAxis(idi, ni)
jaxis = TransientVirtualAxis(idj, nj)
lataxis = TransientAxis2D(lat,
axes=(iaxis, jaxis),
attributes={'units': lat_units},
id="latitude")
lonaxis = TransientAxis2D(lon,
axes=(iaxis, jaxis),
attributes={'units': lon_units},
id="longitude")
curvegrid = TransientGenericGrid(lataxis, lonaxis, tempmask=None)
attributs = None
vid = None
if hasattr(v, 'attributes'): attributs = v.attributes
if hasattr(v, 'id'): vid = v.id
axis0 = v.getAxis(0)
return cdms2.createVariable(v,
axes=[axis0, iaxis, jaxis],
grid=curvegrid,
attributes=attributs,
id=v.id)
def eigenvalues(self, neigs=None):
"""Eigenvalues (decreasing variances) associated with each EOF.
Returns the ordered eigenvalues in a :py:mod:`cdms2` variable.
**Optional argument:**
*neigs*
Number of eigenvalues to return. Defaults to all
eigenvalues.
**Examples:**
All eigenvalues:
>>> lambdas = eofobj.eigenvalues()
The first eigenvalue:
>>> lambda1 = eofobj.eigenvalues(neigs=1)
"""
lambdas = self._solver.eigenvalues(neigs=neigs)
eofax = cdms2.createAxis(range(len(lambdas)), id="eigenvalue")
axlist = [eofax]
lambdas = cdms2.createVariable(lambdas, id="eigenvalues", axes=axlist)
lambdas.name = "eigenvalues"
lambdas.long_name = "eigenvalues"
return lambdas
def do_transform(infile, outfile, template):
# for netcdf3:
cdms2.setNetcdfShuffleFlag(0)
cdms2.setNetcdfDeflateFlag(0)
cdms2.setNetcdfDeflateLevelFlag(0)
with open(infile, mode='rb') as file:
fileContent = file.read()
(referenceGrid, latAxis, lonAxis, latBounds, lonBounds)=makeGrid()
if os.path.exists(outfile):os.remove(outfile)
fout = cdms2.open(outfile, "w")
#thisData = struct.unpack(template['read_type'] * ((template['read_nl'] * template['read_ns']) // template['read_type_size']) , fileContent[template['skip_byte']:template['skip_byte']+(template['read_nl']*template['read_ns'])*template['read_type_size']] )
skip=4*8
thisData = numpy.array(struct.unpack('>64800f', fileContent[skip:skip + (180*360)*4]))
thisVar = cdms2.createVariable(thisData.reshape( (template['read_ns'], template['read_nl']) ), typecode=template['read_type'], id=template['id'], \
fill_value=template['nodata'], grid=referenceGrid, copaxes=1 )
fout.write(thisVar)
# thisData2 = numpy.array(struct.unpack('64800B', fileContent[skip + (180*360)*4 : skip + (180*360)*4 + 180*360]))
# thisVar2 = cdms2.createVariable(thisData2.reshape( (template['read_ns'], template['read_nl']) ), typecode='B', id='ice', \
# fill_value=0, grid=referenceGrid, copaxes=1 )
# fout.write(thisVar2)
fout.close()
def __call__(self, _var, region):
var = _var.clone()
if not region in self.regions.keys():
return None
val = self.regions[region]
# reading mask data
regions_data = self.file['mask']
regions_var = cdms2.createVariable(
ones(regions_data.shape),
grid = cdms2.createUniformGrid(89.75, 360, -0.5, -180, 720, 0.5),
mask = where(equal(regions_data, val), 0, 1))
lats = cdms2.createUniformLatitudeAxis(89.75, 360, -0.5)
lons = cdms2.createUniformLongitudeAxis(-180, 720, 0.5)
regions_var.setAxisList((lats,lons))
new_mask_var = regions_var.regrid(var.getGrid(), regridTool='regrid2', regridMethod='linear')
new_mask = getmask(new_mask_var)
if var.mask <> None:
var.mask = logical_or(var.mask, new_mask)
else:
var.mask = new_mask;
return var
def _convertNAToCdmsVariable(self, var_number, attributes={}):
"""
Creates a single cdms variable from the variable number provided in the list.
"""
(var_name, units, miss, scal) = self.na_file_obj.getVariable(var_number)
msg = "\nAdding variable: %s" % self.na_file_obj.VNAME[var_number]
log.debug(msg)
self.output_message.append(msg)
array = N.array(self.na_file_obj.V[var_number])
array = array * scal
# Set up axes
if not hasattr(self, 'cdms_axes'):
self._convertCdmsAxes()
# Set up variable
var=cdms.createVariable(array, axes=self.cdms_axes, fill_value=miss, attributes=attributes)
# Sort units etc
if units: var.units=units
# Add the best variable name
if len(var_name) < max_id_length:
var.id=safe_nc_id.sub("_", var_name).lower()
else:
var.id="naVariable_%s" % (var_number)
# Check if mapping provided for renaming this variable
if var_name in self.rename_variables.keys():
var_name = self.rename_variables[var_name]
var.long_name = var.name = var.title = var_name
# Add a NASA Ames variable number (for mapping correctly back to NASA Ames)
var.nasa_ames_var_number = var_number
return var
def eigenvalues(self, neigs=None):
"""Eigenvalues (decreasing variances) associated with each EOF.
**Optional argument:**
*neigs*
Number of eigenvalues to return. Defaults to all
eigenvalues.If the number of eigenvalues requested is more
than the number that are available, then all available
eigenvalues will be returned.
**Returns:**
*eigenvalues*
A `cdms2` variable containing the eigenvalues arranged
largest to smallest.
**Examples:**
All eigenvalues::
eigenvalues = solver.eigenvalues()
The first eigenvalue::
eigenvalue1 = solver.eigenvalues(neigs=1)
"""
lambdas = self._solver.eigenvalues(neigs=neigs)
eofax = cdms2.createAxis(list(range(len(lambdas))), id='eigenvalue')
eofax.long_name = 'eigenvalue_number'
axlist = [eofax]
lambdas = cdms2.createVariable(lambdas, id='eigenvalues', axes=axlist)
lambdas.long_name = 'eigenvalues'
return lambdas
def eofsAsCovariance(self, neofs=None, pcscaling=1):
"""
Empirical orthogonal functions (EOFs) expressed as the
covariance between the principal component time series (PCs)
and the time series of the `Eof` input *dataset* at each grid
point.
**Optional arguments:**
*neofs*
Number of EOFs to return. Defaults to all EOFs. If the
number of EOFs requested is more than the number that are
available, then all available EOFs will be returned.
*pcscaling*
Set the scaling of the PCs used to compute covariance. The
following values are accepted:
* *0* : Un-scaled PCs.
* *1* : PCs are scaled to unit variance (divided by the
square-root of their eigenvalue) (default).
* *2* : PCs are multiplied by the square-root of their
eigenvalue.
The default is to divide PCs by the square-root of their
eigenvalue so that the PCs are scaled to unit variance
(option 1).
**Returns:**
*eofs*
A `cdms2` variable containing the ordered EOFs. The EOFs are
numbered from 0 to *neofs* - 1.
**Examples:**
All EOFs::
eofs = solver.eofsAsCovariance()
The leading EOF::
eof1 = solver.eofsAsCovariance(neofs=1)
The leading EOF using un-scaled PCs::
eof1 = solver.eofsAsCovariance(neofs=1, pcscaling=0)
"""
eofs = self._solver.eofsAsCovariance(neofs, pcscaling)
eofs.fill_value = self._missing_value
eofax = cdms2.createAxis(list(range(len(eofs))), id='eof')
axlist = [eofax] + self._channels
eofs = cdms2.createVariable(eofs,
id='eofs_cov',
axes=axlist,
fill_value=self._missing_value)
eofs.long_name = 'covariance_between_pcs_and_{:s}'.format(
self._dataset_name)
return eofs
def post(self,fetched,slab,axes,specifications,confined_by,aux,axismap):
''' Post processing retouches the bounds and later will deal with the mask'''
import cdms2 as cdms
fetched=cdms.createVariable(fetched,copy=1)
faxes=fetched.getAxisList()
a=None
for i in range(len(faxes)):
if confined_by[i] is self:
newaxvals=[]
bounds=[]
a=None
sh=list(fetched.shape)
sh[i]=1
for l in self.aux[i]:
try:
tmp=fetched(**{faxes[i].id:(l,l)})
ax=tmp.getAxis(i)
#print ax
newaxvals.append(ax[0])
if ax.getBounds()!=None:
bounds.append(ax.getBounds()[0])
else:
bounds=None
except Exception,err:
#print 'err:',err,'match:',self.match
if self.match==1:
raise Exception,'Error axis value :'+str(l)+' was requested but is not present in slab\n(more missing might exists)'
elif self.match==0:
tmp=MV2.ones(sh,typecode=MV2.float)
tmp=MV2.masked_equal(tmp,1)
if type(l)==type(cdtime.comptime(1999)) or type(l)==type(cdtime.reltime(0,'days since 1999')) or type(l)==type(''):
if type(l)!=type(''):
newaxvals.append(l.torel(faxes[i].units).value)
else:
newaxvals.append(cdtime.s2r(l,faxes[i].units).value)
else:
newaxvals.append(l)
if bounds is not None:
bounds.append([ax[-1]-1.,ax[-1]+1])
else:
tmp=None
if not tmp is None:
if a is None:
a=tmp
elif not tmp is None:
a=MV2.concatenate((a,tmp),i)
if bounds is not None:
newax=cdms.createAxis(numpy.array(newaxvals),bounds=numpy.array(bounds),id=ax.id)
else:
newax=cdms.createAxis(numpy.array(newaxvals),id=ax.id)
for att in faxes[i].attributes.keys():
setattr(newax,att,faxes[i].attributes.get(att))
for j in range(len(fetched.shape)):
if j==i:
a.setAxis(i,newax)
else:
a.setAxis(j,faxes[j])
fetched=a.astype(fetched.dtype.char)
faxes=fetched.getAxisList()
def eofsAsCovariance(self, neofs=None, pcscaling=1):
"""
Empirical orthogonal functions (EOFs) expressed as the
covariance between the principal component time series (PCs)
and the each data set in the `MultivariateEof` input *datasets*.
**Optional argument:**
*neofs*
Number of EOFs to return. Defaults to all EOFs. If the
number of EOFs requested is more than the number that are
available, then all available EOFs will be returned.
*pcscaling*
Set the scaling of the PCs used to compute covariance. The
following values are accepted:
* *0* : Un-scaled PCs.
* *1* : PCs are scaled to unit variance (divided by the
square-root of their eigenvalue) (default).
* *2* : PCs are multiplied by the square-root of their
eigenvalue.
The default is to divide PCs by the square-root of their
eigenvalue so that the PCs are scaled to unit variance
(option 1).
**Returns:**
*eofs_list*
A list of `cdms2` variables containing the ordered EOFs for
each variable.
**Examples:**
All EOFs of each data set::
eofs_list = solver.eofsAsCovariance()
The leading EOF of each data set::
eof1_list = solver.eofsAsCovariance(neofs=1)
"""
eofset = self._solver.eofsAsCovariance(neofs)
neofs = eofset[0].shape[0]
eofax = cdms2.createAxis(list(range(neofs)), id='eof')
eofax.long_name = 'eof_number'
for iset in range(self._ndata):
axlist = [eofax] + self._multichannels[iset]
eofset[iset].fill_value = self._multimissing[iset]
eofset[iset] = cdms2.createVariable(
eofset[iset],
id='eofs',
axes=axlist,
fill_value=self._multimissing[iset])
eofset[iset].long_name = 'covariance_between_pcs_and_{:s}'.format(
self._dataset_names[iset])
return eofset
