def ensrank(obs, ens, gethist=False, getnrz=False, centered=False):
"""Compute the rank of a reference (observations) in an ensemble
If ``nens`` is the size of the ensemble,
the rank may go from ``0`` to ``nens``.
- **obs**, (...): Observation array
- **ens**, (nens,...): Ensemble array
- **getrank**, optional: If True, return also the rank histogram
- **getnrz**, optional: If True, return also non recovery zones
- **centered**, optional: Center the ensemble data on the observations
before computing the rank
"""
# Check
ens = MV2.asarray(ens)
obs = MV2.asarray(obs)
nens = ens.shape[0]
assert nens>3, 'Your ensemble is too small (nens=%i)'%nens
# Inits
rank = obs.clone()
rank.id = 'rank'
rank.long_name = 'Rank'
if hasattr(rank, 'units'): del rank.units
nrz = obs.clone()
nrz.id = 'nrz'
nrz.long_name = 'Non recovery zones'
nrz[:] = MV2.masked
# Sort ensemble at each pixel and compute the min biais
bias = MV2.sort(ens, axis=0)-obs
if centered:
bias -= bias.mean(axis=0)
nrz[:] = MV2.where((bias[-1]<0.).astype('b'), bias[-1], nrz)
nrz[:] = MV2.where((bias[0]>0.).astype('b'), bias[0], nrz)
# Index of member = rank
rank[:] = (bias<0).astype('i').sum(axis=0)
del bias
## Apply observation mask
#nrz[:] = MV2.masked_where(MV2.getmask(obs), nrz, copy=0)
#rank[:] = MV2.masked_where(MV2.getmask(obs), rank, copy=0)
# Rank histogram
if gethist:
if N.ma.isMA(rank): rank = rank.compressed()
hist = N.histogram(rank, N.arange(0,nens+1))
# Out
ret = rank,
if gethist:
ret += hist,
if getnrz:
ret += nrz,
return ret[0] if len(ret)==1 else ret
def ensrank(obs, ens, gethist=False, getnrz=False, centered=False):
"""Compute the rank of a reference (observations) in an ensemble
If ``nens`` is the size of the ensemble,
the rank may go from ``0`` to ``nens``.
- **obs**, (...): Observation array
- **ens**, (nens,...): Ensemble array
- **getrank**, optional: If True, return also the rank histogram
- **getnrz**, optional: If True, return also non recovery zones
- **centered**, optional: Center the ensemble data on the observations
before computing the rank
"""
# Check
ens = MV2.asarray(ens)
obs = MV2.asarray(obs)
nens = ens.shape[0]
assert nens>3, 'Your ensemble is too small (nens=%i)'%nens
# Inits
rank = obs.clone()
rank.id = 'rank'
rank.long_name = 'Rank'
if hasattr(rank, 'units'): del rank.units
nrz = obs.clone()
nrz.id = 'nrz'
nrz.long_name = 'Non recovery zones'
nrz[:] = MV2.masked
# Sort ensemble at each pixel and compute the min biais
bias = MV2.sort(ens, axis=0)-obs
if centered:
bias -= bias.mean(axis=0)
nrz[:] = MV2.where((bias[-1]<0.).astype('b'), bias[-1], nrz)
nrz[:] = MV2.where((bias[0]>0.).astype('b'), bias[0], nrz)
# Index of member = rank
rank[:] = (bias<0).astype('i').sum(axis=0)
del bias
## Apply observation mask
#nrz[:] = MV2.masked_where(MV2.getmask(obs), nrz, copy=0)
#rank[:] = MV2.masked_where(MV2.getmask(obs), rank, copy=0)
# Rank histogram
if gethist:
if N.ma.isMA(rank): rank = rank.compressed()
hist = N.histogram(rank, N.arange(0,nens+1))
# Out
ret = rank,
if gethist:
ret += hist,
if getnrz:
ret += nrz,
return ret[0] if len(ret)==1 else ret
def ws2w(us, vs, rhoa=1.25, cd=0.016, format_axes=False, alongxy=None):
"""Convert from wind stress to 10m wind components
This function is the reverse one of :func:`wind_stress`.
Output variables are formatted using :func:`~vacumm.data.cf.format_var`.
:Formula:
.. math::
U_{10m} = (\\rho_a C_d)^{-\\frac{1}{2}}
(\\tau_x^2+\\tau_y^2)^{-\\frac{1}{4}} \\tau_x
V_{10m} = (\\rho_a C_d)^{-\\frac{1}{2}}
(\\tau_x^2+\\tau_y^2)^{-\\frac{1}{4}} \\tau_y
:Params:
- **us/vs**: Wind stress components.
- **rhoa**, optional: Air density (in kg.m-3).
- **cd**, optional: Drag coefficient.
- **format_axes**, optional: Also format axes using
:func:`~vacumm.data.cf.format_axis`.
- **alongxy**, optional: Format variables considering components are along X/Y
direction and not along zonal/meridional direction.
:Return: ``u, v``
"""
# Init and format variables
u = MV2.asarray(us).clone()
v = MV2.asarray(vs).clone()
if alongxy is None:
alongxy = hasattr(u, 'long_name') and 'x' in us.long_name.lower()
if alongxy:
uname, vname = 'ux10m', 'vy10m'
else:
uname, vname = 'u10m', 'v10m'
format_var(u, uname, format_axes=format_axes)
format_var(v, vname, format_axes=format_axes)
# Compute
zero = us.filled(1)==0.
zero &= vs.filled(1)==0.
uvsmod = (us**2+vs**2)**-0.25
uvsmod /= N.sqrt(rhoa*cd)
u.assignValue(uvsmod*us)
v.assignValue(uvsmod*vs)
u.assignValue(MV2.where(zero, 0., u))
v.assignValue(MV2.where(zero, 0., v))
del uvsmod
return u, v
def wind_stress(u, v, rhoa=1.25, cd=0.016, format_axes=False, alongxy=None):
"""Compute the sea surface zonal and meridional wind stress from 10m wind components
Output variables are formatted using :func:`~vacumm.data.cf.format_var`.
:Formula:
.. math::
\\tau_x = \\rho_a C_d (U_{10m}^2+V_{10m}^2)^{\\frac{1}{2}}U_{10m}
\\tau_y = \\rho_a C_d (U_{10m}^2+V_{10m}^2)^{\\frac{1}{2}}V_{10m}
:Params:
- **u/v**: Wind at 10m above sea surface.
- **rhoa**, optional: Air density (in kg.m-3).
- **cd**, optional: Drag coefficient.
- **format_axes**, optional: Also format axes using
:func:`~vacumm.data.cf.format_axis`.
- **alongxy**, optional: Format variables considering components are along X/Y
direction and not along zonal/meridional direction.
:Return: ``us, vs``
"""
# Init and format variables
us = MV2.asarray(u).clone()
vs = MV2.asarray(v).clone()
if alongxy is None:
alongxy = hasattr(u, 'long_name') and 'x' in u.long_name.lower()
if alongxy:
usname, vsname = 'taux', 'tauy'
else:
usname, vsname = 'tauu', 'tauv'
format_var(us, usname, format_axes=format_axes)
format_var(vs, vsname, format_axes=format_axes)
# Compute
uvmod = N.ma.sqrt(u**2+v**2)
us.assignValue(cd*rhoa*uvmod*u)
vs.assignValue(cd*rhoa*uvmod*v)
del uvmod
return us, vs
def coriolis_parameter(lat,
gravity=default_gravity,
fromvar=False,
format_axes=False):
"""Get the coriolis parameters computed at each latitude
:Params:
- **lat**: Latitude or a variable with latitude coordinates.
- **gravity**, optional: Gravity.
- **fromvar**, optional: If True, lat is supposed to be a MV2
array with latitude coordinates.
"""
# Latitude
if fromvar:
if not cdms2.isVariable(lat):
raise VACUMMError('lat must a MV2 array because fromvar is True')
latv = lat * 0
lat = lat.getLatitude()
if lat is None:
raise VACUMMError(
'lat must a MV2 array with a latitude axis because fromvar is True'
)
if cdms2.isVariable(lat): lat = lat.asma() # 2D axes
if lat.shape != latv.shape:
if len(lat.shape) == 2:
latv[:] = N.ma.resize(lat, latv.shape)
else:
yaxis = latv.getOrder().index('y')
new_shape = len(latv.shape) * [1]
new_shape[yaxis] = latv.shape[yaxis]
tile_shape = list(latv.shape)
tile_shape[yaxis] = 1
latv[:] = N.tile(lat[:].reshape(new_shape), tile_shape)
else:
latv = lat if not N.ndim(lat) else lat[:]
# Compute
f0 = 2 * N.ma.sin(N.pi * latv / 180.)
# f0 *= 2*N.pi/(24.*3600.)
f0 *= 2 * N.pi / (86164.) # 86164 = sidereal day....
# Format
if N.isscalar(f0): return f0
f0 = MV2.asarray(f0)
if not fromvar and isaxis(lat) and f0.ndim == 1:
f0.setAxis(0, lat)
return format_var(f0, 'corio', format_axes=format_axes)
def coriolis_parameter(lat, gravity=default_gravity, fromvar=False, format_axes=False):
"""Get the coriolis parameters computed at each latitude
:Params:
- **lat**: Latitude or a variable with latitude coordinates.
- **gravity**, optional: Gravity.
- **fromvar**, optional: If True, lat is supposed to be a MV2
array with latitude coordinates.
"""
# Latitude
if fromvar:
if not cdms2.isVariable(lat):
raise VACUMMError('lat must a MV2 array because fromvar is True')
latv = lat*0
lat = lat.getLatitude()
if lat is None:
raise VACUMMError('lat must a MV2 array with a latitude axis because fromvar is True')
if cdms2.isVariable(lat): lat=lat.asma() # 2D axes
if lat.shape!=latv.shape:
if len(lat.shape)==2:
latv[:] = N.ma.resize(lat, latv.shape)
else:
yaxis = latv.getOrder().index('y')
new_shape = len(latv.shape)*[1]
new_shape[yaxis] = latv.shape[yaxis]
tile_shape = list(latv.shape)
tile_shape[yaxis] = 1
latv[:] = N.tile(lat[:].reshape(new_shape), tile_shape)
else:
latv = lat if N.isscalar(lat) else lat[:]
# Compute
f0 = 2*N.ma.sin(N.pi*latv/180.)
# f0 *= 2*N.pi/(24.*3600.)
f0 *= 2*N.pi/(86164.) # 86164 = sidereal day....
# Format
if N.isscalar(f0): return f0
f0 = MV2.asarray(f0)
if not fromvar and isaxis(lat) and f0.ndim==1:
f0.setAxis(0, lat)
return format_var(f0, 'corio', format_axes=format_axes)
def format_array(self, data, mode=1, firstdims=None, id=None, atts=None,
format_atts=True):
"""Format to a MV2 array similar to input array"""
# Input was not formatted
if not self.ismv2:
return data
# Make sure to have a MV2 array
data = MV2.asarray(data)
# Record or other first dims
rdims = data.shape[:-self.nsdim]
pshape = (N.multiply.reduce(rdims), ) if rdims else ()
pshape += 1, # fake spatial dim
# First dimes
firstdims, firstaxes = self._get_firstdims_(firstdims, pshape=pshape)
# Format
return self._format_array_(data, firstdims, firstaxes, mode,
id=id, atts=atts, format_atts=format_atts)
def _format_var_(vv, name, mask=None, prefix=True, suffix=True, templates=None,
htemplates=None, attributes=None, single=True, **kwargs):
# Some inits
ist = name.startswith('t')
name = name[1:]
if prefix is True:
prefix = 'Temporal ' if ist else 'Spatial '
elif not prefix:
prefix = ''
kwargs['copy'] = 0
# Aways have a list
if isinstance(vv, tuple):
vv = list(vv)
single = False
elif not isinstance(vv, list):
vv = [vv]
single = True
elif single:
single = len(vv)==1
else:
single = False
dual = len(vv)==2
if templates is not None and not isinstance(templates, (list, tuple)):
templates = [templates]*len(vv)
if htemplates is not None and not isinstance(htemplates, (list, tuple)):
htemplates = [htemplates]*len(vv)
if attributes is None:
attributes = ({}, {})
elif not isinstance(attributes, (list, tuple)):
attributes = [attributes]*len(vv)
# Long_name
long_name = prefix
if name=='avail':
long_name += 'availability'
if name=='count':
long_name += 'count'
elif name=='mean':
long_name += 'average'
elif name=='std':
long_name += 'standard deviation'
elif name=='bias':
long_name += 'bias'
elif name=='rms':
long_name += 'absolute RMS difference'
elif name=='crms':
long_name += 'centered RMS difference'
elif name=='corr':
long_name += 'correlation'
elif name=='hist':
long_name += 'histogram'
# Type change
if 'count' in name:
dtype = 'l'
else:
dtype = None
for i, v in enumerate(vv):
# From template or not
hvar = v.ndim==2 # Histogram-like variables: first axis = bins
hist = htemplates and hvar
if templates is not None:
# From axis (sstats)
if isaxis(templates[i]):
# if hvar and not ist:
# v = N.ma.transpose(v) # time first, not bins
var = MV2.asarray(v)
var.setAxis(0, templates[i])
# try:
# var.setAxis(int(hist), templates[i])
# except:
# pass
if hist:
var.setAxis(1, htemplates[i])
# From variable (tstats)
else:
try:
var = (htemplates if hvar else templates)[i].clone()
except:
pass
try:
var[:] = v.reshape(var.shape)
except:
pass
else:
var = MV2.asarray(v)
del v
# Type
if dtype:
var = var.astype(dtype)
# Mask
if mask is not None:
func = N.resize if var.size!=mask.size else N.reshape
var[:] = MV2.masked_where(func(mask, var.shape), var, copy=0)
# Attributes
# - id and long_name
var.id = attributes[i]['id']+'_'+name
var.long_name = long_name
if suffix and isinstance(suffix, basestring):
var.long_name += suffix
# - single stats
if name in ['mean', 'std', 'hist', 'min', 'max']:
if "units" in attributes[i]:
var.units = attributes[i]['units']
if suffix is True and "long_name" in attributes[i]:
var.long_name += ' of '+attributes[i]['long_name']
# - dual stats
elif name in ['bias', 'rms', 'crms']:
for attr in attributes: # loop on both variables attributes
if "units" in attr and not hasattr(var, 'units'):
var.units = attr['units']
if suffix is True and "long_name" in attr and var.long_name==long_name:
var.long_name += ' of '+attr['long_name']
vv[i] = var
if single:
return vv[0]
return tuple(vv)
def _format_var_(vv, name, mask=None, prefix=True, suffix=True, templates=None,
htemplates=None, attributes=None, single=True, **kwargs):
if name=='tstd':
pass
# Some inits
ist = name.startswith('t')
name = name[1:]
if prefix is True:
prefix = 'Temporal ' if ist else 'Spatial '
elif not prefix:
prefix = ''
kwargs['copy'] = 0
# Aways have a list
if isinstance(vv, tuple):
vv = list(vv)
single = False
elif not isinstance(vv, list):
vv = [vv]
single = True
elif single:
single = len(vv)==1
else:
single = False
dual = len(vv)==2
if templates is not None and not isinstance(templates, (list, tuple)):
templates = [templates]*len(vv)
if htemplates is not None and not isinstance(htemplates, (list, tuple)):
htemplates = [htemplates]*len(vv)
if attributes is None:
attributes = ({}, {})
elif not isinstance(attributes, (list, tuple)):
attributes = [attributes]*len(vv)
# Long_name
long_name = prefix
if name=='avail':
long_name += 'availability'
elif name=='mean':
long_name += 'average'
elif name=='std':
long_name += 'standard deviation'
elif name=='bias':
long_name += 'bias'
elif name=='rms':
long_name += 'absolute RMS difference'
elif name=='crms':
long_name += 'centered RMS difference'
elif name=='corr':
long_name += 'correlation'
elif name=='hist':
long_name += 'histogram'
# Type change
if 'count' in name:
dtype = 'l'
else:
dtype = None
for i, v in enumerate(vv):
# From template or not
hvar = v.ndim==2 # Histogram-like variables: first axis = bins
hist = htemplates and hvar
if templates is not None:
# From axis (sstats)
if isaxis(templates[i]):
# if hvar and not ist:
# v = N.ma.transpose(v) # time first, not bins
var = MV2.asarray(v)
var.setAxis(0, templates[i])
# try:
# var.setAxis(int(hist), templates[i])
# except:
# pass
if hist:
var.setAxis(1, htemplates[i])
# From variable (tstats)
else:
try:
var = (htemplates if hvar else templates)[i].clone()
except:
pass
try:
var[:] = v.reshape(var.shape)
except:
pass
else:
var = MV2.asarray(v)
del v
# Type
if dtype:
var = var.astype(dtype)
# Mask
if mask is not None:
func = N.resize if var.size!=mask.size else N.reshape
var[:] = MV2.masked_where(func(mask, var.shape), var, copy=0)
# Attributes
# - id and long_name
var.id = attributes[i]['id']+'_'+name
var.long_name = long_name
if suffix and isinstance(suffix, basestring):
var.long_name += suffix
# - single stats
if name in ['mean', 'std', 'hist', 'min', 'max']:
if "units" in attributes[i]:
var.units = attributes[i]['units']
if suffix is True and "long_name" in attributes[i]:
var.long_name += ' of '+attributes[i]['long_name']
# - dual stats
elif name in ['bias', 'rms', 'crms']:
for attr in attributes: # loop on both variables attributes
if "units" in attr and not hasattr(var, 'units'):
var.units = attr['units']
if suffix is True and "long_name" in attr and var.long_name==long_name:
var.long_name += ' of '+attr['long_name']
vv[i] = var
if single:
return vv[0]
return tuple(vv)
def format_var(var,
name=None,
force=True,
format_axes=True,
order=None,
nodef=True,
mode='warn',
**kwargs):
"""Format a MV2 variable according to its generic name
:Params:
- **var**: A :mod:`numpy` or :mod:`MV2` variabe.
- **name**: Generic name of variable. It should be one of
those listed by :attr:`CF_VAR_SPECS`. If None, it is guessed
with :func:`match_known_var`.
- **force**, optional: Overwrite attributes in all cases.
- **format_axes**, optional: Also format axes.
- **nodef**, optional: Remove location specification when it refers to the
default location (:attr:`DEFAULT_LOCATION`).
- **mode**: "silent", "warn" or "raise".
- Other parameters are passed as attributes, except those:
- present in specifications to allow overriding defaults,
- starting with 'x', 'y', or 't' which are passed
to :func:`format_axis`.
:Examples:
>>> var = format_var(myarray, 'sst', valid_min=-2, valid_max=100)
"""
# Filter keywords for axis formating
axismeths = {'t': 'getTime', 'y': 'getLatitude', 'x': 'getLongitude'}
kwaxes = {}
for k in axismeths.keys():
kwaxes[k] = kwfilter(kwargs, k + '_')
# Always a MV2 array
if not cdms2.isVariable(var):
var = MV2.asarray(var)
# Check specs
if name is None: # guess it
name = match_known_var(var)
if not name:
if mode == 'warn':
warn("Can't guess cf name")
return var
elif mode == 'silent':
return var
else:
raise KeyError("Variable does not match any CF var")
elif name not in CF_VAR_SPECS and name not in CF_AXIS_SPECS:
if var.id in CF_VAR_SPECS or var.id in CF_AXIS_SPECS:
name = var.id
elif mode == 'warn':
warn("Generic var name not found '%s'." % name)
return var
elif mode == 'silent':
return var
else:
raise KeyError(
"Generic var name not found '%s'. Please choose one of: %s" %
(name, ', '.join(CF_VAR_SPECS.keys() + CF_AXIS_SPECS.keys())))
isaxis = name in CF_AXIS_SPECS
if isaxis:
specs = CF_AXIS_SPECS[name].copy()
if 'axis' in specs:
del specs['axis']
else:
specs = CF_VAR_SPECS[name].copy()
# - merge kwargs and specs
for key, val in kwargs.items():
if val is None or key not in specs: continue
# Check type
if not isinstance(val, list) and isinstance(specs[key], list):
val = [val]
# Set
specs[key] = val
del kwargs[key]
# - remove default location
if nodef:
refloc = specs.get('physloc', None) or DEFAULT_LOCATION
for att in 'id', 'long_name', 'standard_name':
if get_loc(specs[att], att) == refloc:
specs[att] = [no_loc_single(specs[att][0], att)]
name = specs['id'][0]
# - id
if ((force is True or force in [2, 'id', 'all'])
or var.id.startswith('variable_')
or (isaxis and var.id.startswith('axis_'))): # FIXME: use regexp
var.id = name
# - attributes
forceatts = (force is True or force in ['atts', 'all']
or (isinstance(force, int) and force > 0))
for att, val in cf2atts(specs, **kwargs).items():
if forceatts or not getattr(var, att, ''):
setattr(var, att, val)
# - physical location
loc = get_loc(var, mode='ext')
if not loc and 'physloc' in specs:
loc = specs['physloc']
if loc:
if 'physloc' in specs and loc == specs['physloc']:
var._vacumm_cf_physloc = loc.lower()
set_loc(var, loc)
# - store cf name
var._vacumm_cf_name = name
# Axes
if format_axes:
# Order
order = var.getOrder() if not isinstance(order, basestring) else order
if order is not None:
if not re.match('^[xyzt-]+$', order):
raise VACUMMError("Wrong cdms order type: " + order)
if len(order) != var.ndim:
raise VACUMMError(
"Cdms order should be of length %s instead of %s" %
(var.ndim, len(order)))
# First check
if 'axes' in specs:
axspecs = specs['axes']
formatted = []
for key, meth in axismeths.items():
axis = getattr(var, meth)()
if order is not None: order.replace(key, '-')
if axis is not None:
format_axis(axis, axspecs[key], **kwaxes[key])
formatted.append(key)
# Check remaining simple axes (DOES NOT WORK FOR 2D AXES)
if order is not None and order != '-' * len(order):
for key in axismeths.keys():
if key in order and key not in formatted:
axis = var.getAxis(order.index(key))
format_axis(axis, axspecs[key], **kwaxes[key])
return var
def generic(var,coefs,units='hours',get_tide=False,only_tide=False,filter_name='Generic',
check_regular=False,strict=True, interp_method='cubic', axis=None):
"""Apply a filter to signal (cdms2 variable) using symetric coefficients. First axis is supposed to be time and regular.
- **var**: Data to filter with first axis supposed to be time.
- **coefs**: Second half of coefficients.
- *strict*: Mask values where at least one value is missing in the filtering interval.
- *filter_name*: Name to give to the filter [default: 'Generic']
%s
"""
# Return?
if only_tide:
get_tide = True
# Coefficients
units = unit_type(units)
sunits = unit_type(units, string_type=True)
coefs = coefs.astype('f')
if coefs[-1] != 0.:
coefs = N.concatenate((coefs,[0.,]))
ncoefs = (len(coefs)-1)*2+1
# Input variable
var = MV2.asarray(var)
check_axes(var)
# Input time axis
if axis is None:
try:
axis = var.getOrder().index('t')
except:
axis = 0
time = var.getAxis(axis)
if check_regular:
assert isregular(time), 'Your time axis seems not regular'
dt = N.diff(time[:]).mean()
if not time.attributes.has_key('units'):
time.units = 'hours since 1970-01-01'
print 'The time axis of your variable has no units. Assuming : '+time.units
if not time.isTime():
time.designateTime(calendar=cdtime.DefaultCalendar)
# Check time range
time_range = time[-1]-time[0]
coefs_units = sunits+' '+' '.join(time.units.split()[1:])
coefs_range0 = cdtime.r2r(cdtime.reltime(time[0],coefs_units),time.units)
coefs_range1 = coefs_range0.add(ncoefs-1,units)
coefs_range = coefs_range1.value - coefs_range0.value
coefs_dt = coefs_range0.add(1,units).value - coefs_range0.value
if time_range < coefs_range:
raise Exception,'Your sample is shorter than the time range of your coefficients'
nt = len(var)
# Remap coefficients to data time axis
tc = var.dtype.char
# print 'dt', dt
# print 'coefs_dt', coefs_dt
if abs((coefs_dt-dt)/dt) > 0.01:
oldx = cdms2.createAxis(N.arange(len(coefs))*coefs_dt)
old_coefs = MV2.array(coefs, axes=[oldx])
newx = cdms2.createAxis(N.arange(0.,max(oldx),dt))
coefs = interp1d(old_coefs, newx, interp_method).filled()
coefs = N.concatenate((coefs[:0:-1],coefs))
coefs /= N.sum(coefs)
ncoefs = len(coefs)
# Filtering using convolution
fvar = convolve1d(var.filled(0.), coefs, axis=axis, mode='constant')
if var.mask is MV2.nomask:
mask = N.zeros(nt)
else:
mask = var.mask.astype('f')
fmask = convolve1d(mask, coefs, axis=axis, mode='constant', cval=1.)!=0.
fvar = MV2.asarray(fvar)
fvar = MV2.masked_where(fmask, fvar, copy=0)
# Output variables
if not only_tide:
fvar.id = var.id+'_cotes'
fvar.name = fvar.id
if var.attributes.has_key('long_name'):
fvar.long_name = 'Tide removed signal from '+var.long_name.lower()
fvar.long_name_fr = 'Signal sans la maree'
if var.attributes.has_key('units'):
fvar.units = var.units
fvar.setAxis(0,time)
fvar.filter_coefficients = coefs
fvar.filter_name = filter_name
res = [fvar,]
if get_tide:
tvar = var-fvar
tvar.id = 'tidal_'+var.id
tvar.name = tvar.id
if var.attributes.has_key('long_name'):
tvar.long_name = 'Tidal signal from '+var.long_name.lower()
if var.attributes.has_key('units'):
tvar.units = var.units
tvar.setAxis(0,time)
tvar.filter_coefficients = coefs
tvar.filter_name = filter_name
if only_tide:
return tvar
res.append(tvar)
if isinstance(res, list) and len(res) == 1:
return res[0]
else:
return res
def format_var(var, name, force=True, format_axes=True, order=None, nodef=True, **kwargs):
"""Format a MV2 variable according to its generic name
:Params:
- **var**: A :mod:`numpy` or :mod:`MV2` variabe.
- **name**: Generic name of variable. It should be one of
those listed by :attr:`GENERIC_VAR_NAMES`.
- **force**, optional: Overwrite attributes in all cases.
- **format_axes**, optional: Also format axes.
- **nodef**, optional: Remove location specification when it refers to the
default location (:attr:`DEFAULT_LOCATION`).
- Other parameters are passed as attributes, except those:
- present in specifications to allow overriding defaults,
- starting with 'x', 'y', or 't' which are passed
to :func:`format_axis`.
:Examples:
>>> var = format_var(myarray, 'sst', valid_min=-2, valid_max=100)
"""
# Filter keywords for axis formating
axismeths = {'t':'getTime', 'y':'getLatitude', 'x':'getLongitude'}
kwaxes = {}
for k in axismeths.keys():
kwaxes[k] = kwfilter(kwargs, k+'_')
# Always a MV2 array
var = MV2.asarray(var)
# Check specs
if name not in GENERIC_VAR_NAMES:
if var.id in GENERIC_VAR_NAMES:
name = var.id
else:
raise KeyError("Generic var name not found '%s'. Please choose one of: %s"%(
name, ', '.join(GENERIC_VAR_NAMES)))
specs = VAR_SPECS[name].copy()
# - merge kwargs and specs
for key, val in kwargs.items():
if val is None or key not in specs: continue
# Check type
if not isinstance(val, list) and isinstance(specs[key], list):
val = [val]
# Set
specs[key] = val
del kwargs[key]
# - remove default location
if nodef:
refloc = specs.get('physloc', DEFAULT_LOCATION)
for stype in 'name', 'long_name', 'standard_name':
sname = stype+'s'
if sname not in specs: continue
if get_loc(specs[sname], stype)==refloc:
specs[sname] = [no_loc_single(specs[sname][0], stype)]
name = specs['names'][0]
# - id
if force or var.id.startswith('variable_'):
var.id = name
# - attributes
for att, val in cf2atts(specs, **kwargs).items():
if force or not getattr(var, att, ''):
setattr(var, att, val)
# - physical location
loc = get_loc(var, mode='ext')
if not loc and 'physloc' in specs:
loc = specs['physloc']
if loc:
if 'physloc' in specs and loc in specs['physloc']:
var._vacumm_cf_physloc = loc.lower()
set_loc(var, loc)
# - store cf name
var._vacumm_cf_name = name
# Axes
if format_axes:
# Order
order = var.getOrder() if not isinstance(order, basestring) else order
if order is not None:
if not re.match('^[xyzt-]+$', order):
raise VACUMMError("Wrong cdms order type: "+order)
if len(order)!=var.ndim:
raise VACUMMError("Cdms order should be of length %s instead of %s"%(var.ndim, len(order)))
# First check
axspecs = specs['axes']
formatted = []
for key, meth in axismeths.items():
axis = getattr(var, meth)()
if order is not None: order.replace(key, '-')
if axis is not None:
format_axis(axis, axspecs[key], **kwaxes[key])
formatted.append(key)
# Check remaining simple axes (DOES NOT WORK FOR 2D AXES)
if order is not None and order!='-'*len(order):
for key in axismeths.keys():
if key in order and key not in formatted:
axis = var.getAxis(order.index(key))
format_axis(axis, axspecs[key], **kwaxes[key])
return var
def generic1d(data, weights, axis=0, mask='same', copy=True, cyclic=False):
"""Generic 1D filter applied to :mod:`MV2` variables using convolution.
:Params:
- **data**: Atleast 1D :mod:`MV2` variable.
- **weights**: integer, 1D weights.
They are expected to be symmetric and of odd sizes.
- **axis**, optional: axis on which to operate
- **mask**, optional: mode of masking.
The mask is also filtered, and its value helps
defining the new mask, depending on this parameter:
- If a float is provided, data are masked if the mask value
is greater than this parameter.
- ``"minimal"``: Equivalent to ``1.``. Data is not masked
if a valid value was used to compute data.
- ``"maximal"``: Equivalent to ``0.``. Data is masked
if a invalid value was used to compute data.
- ``"same"``: Mask is the same as input mask.
- **copy**, optional: Copy variable before filtering it.
:Return:
- The filtered :mod:`MV2` variable.
:Example:
>>> generic1d(data, 3.) # running mean using a 3-points block
>>> generic1d(data, [1.,2.,1], axis=2) # shapiro filter on third axis
:See also: :func:`scipy.signal.convolve2d`
"""
# Setup
# - data
data = MV2.asarray(data)
assert data.ndim, 'Input data array must be at least 1D'
if axis < 0: axis += data.ndim
if axis != data.ndim - 1:
init_order = data.getOrder(ids=1)
data = data.reorder('...%i' % axis)
datan = data.filled(0.).copy()
nx = datan.shape[-1]
datan.shape = -1, nx
# - weights
if isinstance(weights, int):
weights = N.ones(weights)
else:
weights = N.asarray(weights)
assert weights.ndim, 'Input weights array must be at least 1D'
assert weights.shape[-1] % 2 == 1, 'Shape of weights must be of odd size'
ww = (~N.ma.getmaskarray(data)).astype('i') # = good 2D
nw = weights.shape[-1]
nw2 = weights.shape[-1] / 2
weights.shape = 1, -1
ww.shape = datan.shape
if data.mask is not MV2.nomask:
one2d = N.ones(datan.shape, 'i')
# Cyclic case
if cyclic:
mode = 'valid'
fdatan = N.concatenate((datan[:, -nw2:], datan, datan[:, :nw2]),
axis=1)
fww = N.concatenate((ww[:, -nw2:], ww, ww[:, :nw2]), axis=1)
one1d = N.ones(datan.shape[-1] + nw2 * 2, 'i')
else:
mode = 'same'
fdatan = datan
fww = ww
one1d = N.ones(datan.shape[-1], 'i')
# Filter
kwf = dict(mode=mode)
datan = convolve2d(fdatan, weights, **kwf)
one2d = N.resize(N.convolve(one1d, weights[0], **kwf), datan.shape)
del one1d
if data.mask is MV2.nomask:
ww = one2d
else:
ww = convolve2d(fww, weights, **kwf)
del fdatan, fww
bad = ww == 0
ww[bad] = 1
datan[:] = N.where(bad, datan, datan / ww.astype('d'))
ww[bad] = 0
# Set
if copy:
datao = data.clone()
else:
datao = data
datan.shape = data.shape
bad.shape = data.shape
if data.mask is not MV2.nomask:
ww.shape = one2d.shape = data.shape
if mask is 'same':
bad = data.mask
else:
if mask == 'minimal':
mask = 1.
elif mask == 'maximal':
mask = 0.
else:
mask = float(mask)
bad |= (ww / one2d) < (1 - mask) # threshold
datao[:] = N.ma.masked_where(bad, datan, copy=False)
del ww, one2d, bad
else:
datao[:] = datan
if axis != data.ndim - 1:
init_order = cdms2.order2index(datao.getAxisList(), init_order)
return datao.reorder(init_order)
return datao
def generic2d(data, weights, mask='same', copy=True):
"""Generic 2D filter applied to 2D (or more) :mod:`MV2` variables using convolution.
:Params:
- **data**: Atleast 2D :mod:`MV2` variable.
- **weights**: integer, 2D weights.
They are expected to be symmetric and of odd sizes.
If an integer is provided, a ``(weights,weights)``
array of ones is used.
- **mask**, optional: mode of masking.
The mask is also filtered, and its value helps
defining the new mask, depending on this parameter:
- If a float is provided, data are masked if the mask value
is greater than this parameter.
- ``"minimal"``: Equivalent to ``1.``. Data is not masked
if a valid value was used to compute data.
- ``"maximal"``: Equivalent to ``0.``. Data is masked
if a invalid value was used to compute data.
- ``"same"``: Mask is the same as input mask.
- **copy**, optional: Copy variable before filtering it.
:Return:
- The filtered :mod:`MV2` variable.
:Example:
>>> generic2d(data, 3.) # running mean using a 3x3 block
>>> generic2d(data, N.ones(3,3)) # the same
>>> generic2d(data, N.ones(3,3), weights, mode='minimal') # crop coasts
:See also: :func:`scipy.signal.convolve2d`
"""
# Setup
data = MV2.asarray(data)
assert data.ndim > 1, 'Input data array must be at least 2D'
if isinstance(weights, int):
weights = N.ones((weights, weights))
else:
weights = N.asarray(weights)
assert weights.ndim, 'Input weights array must be at least 2D'
for i in -2, -1:
assert weights.shape[i] % 2 == 1, \
'Shape of weights must be of odd size in the two directions'
ww = -N.ma.getmaskarray(data).astype('f') + 1
datan = data.filled(0.).copy()
if datan.ndim > 2:
ny, nx = datan.shape[-2:]
datan.shape = datan.size / (nx * ny), ny, nx
elif datan.ndim == 2:
datan.shape = (1, ) + datan.shape[-2:]
ww.shape = datan.shape
one2d = N.ones(datan.shape[-2:])
# Filter
kwf = dict(mode='same', boundary='fill', fillvalue=0.)
for i in xrange(datan.shape[0]): # TODO: multiproc filter2d
datan[i] = scipy.signal.convolve2d(datan[i], weights, **kwf)
if data.mask is MV2.nomask:
ww[i] = scipy.signal.convolve2d(one2d, weights, **kwf)
else:
ww[i] = scipy.signal.convolve2d(ww[i], weights, **kwf)
if data.mask is not MV2.nomask:
one3d = scipy.signal.convolve2d(one2d, weights, **kwf)
one3d = N.resize(one3d, datan.shape)
bad = ww == 0
ww[bad] = 1.
datan[:] = N.where(bad, datan, datan / ww)
ww[bad] = 0
# del bad
# Set
if copy:
datao = data.clone()
else:
datao = data
datan.shape = data.shape
if data.mask is not MV2.nomask:
ww.shape = bad.shape = one3d.shape = data.shape
# print 'mask crit', ww/one3d
if mask is 'same':
bad = data.mask
else:
if mask == 'minimal':
mask = 1.
elif mask == 'maximal':
mask = 0.
else:
mask = float(mask)
bad |= (ww / one3d) < (1 - mask)
datao[:] = N.ma.masked_where(bad, datan, copy=False)
del ww, one3d
else:
datao[:] = datan
del one2d
return datao
def generic2d(data, weights, mask='same', copy=True):
"""Generic 2D filter applied to 2D (or more) :mod:`MV2` variables using convolution.
:Params:
- **data**: Atleast 2D :mod:`MV2` variable.
- **weights**: integer, 2D weights.
They are expected to be symmetric and of odd sizes.
If an integer is provided, a ``(weights,weights)``
array of ones is used.
- **mask**, optional: mode of masking.
The mask is also filtered, and its value helps
defining the new mask, depending on this parameter:
- If a float is provided, data are masked if the mask value
is greater than this parameter.
- ``"minimal"``: Equivalent to ``1.``. Data is not masked
if a valid value was used to compute data.
- ``"maximal"``: Equivalent to ``0.``. Data is masked
if a invalid value was used to compute data.
- ``"same"``: Mask is the same as input mask.
- **copy**, optional: Copy variable before filtering it.
:Return:
- The filtered :mod:`MV2` variable.
:Example:
>>> generic2d(data, 3.) # running mean using a 3x3 block
>>> generic2d(data, N.ones(3,3)) # the same
>>> generic2d(data, N.ones(3,3), weights, mode='minimal') # crop coasts
:See also: :func:`scipy.signal.convolve2d`
"""
# Setup
data = MV2.asarray(data)
assert data.ndim>1, 'Input data array must be at least 2D'
if isinstance(weights, int):
weights = N.ones((weights, weights))
else:
weights = N.asarray(weights)
assert weights.ndim, 'Input weights array must be at least 2D'
for i in -2,-1:
assert weights.shape[i] % 2 == 1, \
'Shape of weights must be of odd size in the two directions'
ww = -N.ma.getmaskarray(data).astype('f')+1
datan = data.filled(0.).copy()
if datan.ndim > 2:
ny, nx = datan.shape[-2:]
datan.shape = datan.size/(nx*ny), ny, nx
elif datan.ndim == 2:
datan.shape = (1,) + datan.shape[-2:]
ww.shape = datan.shape
if data.mask is not MV2.nomask:
one3d = N.ones(datan.shape)
one2d = N.ones(datan.shape[-2:])
# Filter
kwf = dict(mode='same', boundary='fill', fillvalue=0.)
for i in xrange(datan.shape[0]):
datan[i] = scipy.signal.convolve2d(datan[i], weights, **kwf)
if data.mask is MV2.nomask:
ww[i] = scipy.signal.convolve2d(one2d, weights, **kwf)
else:
ww[i] = scipy.signal.convolve2d(ww[i], weights, **kwf)
one3d[i] = scipy.signal.convolve2d(one2d, weights, **kwf)
bad = ww==0
ww[bad]=1.
datan[:] = N.where(bad, datan, datan/ww)
ww[bad]==0
# del bad
# Set
if copy:
datao = data.clone()
else:
datao = data
datan.shape = data.shape
if data.mask is not MV2.nomask:
ww.shape = bad.shape = one3d.shape = data.shape
# print 'mask crit', ww/one3d
if mask is 'same':
bad = data.mask
else:
if mask == 'minimal':
mask = 1.
elif mask == 'maximal':
mask = 0.
else:
mask = float(mask)
bad |= (ww/one3d)<(1-mask)
datao[:] = N.ma.masked_where(bad, datan, copy=False)
del ww, one3d
else:
datao[:] = datan
del one2d
return datao
def generic1d(data, weights, axis=0, mask='same', copy=True, cyclic=False):
"""Generic 1D filter applied to :mod:`MV2` variables using convolution.
:Params:
- **data**: Atleast 1D :mod:`MV2` variable.
- **weights**: integer, 1D weights.
They are expected to be symmetric and of odd sizes.
- **axis**, optional: axis on which to operate
- **mask**, optional: mode of masking.
The mask is also filtered, and its value helps
defining the new mask, depending on this parameter:
- If a float is provided, data are masked if the mask value
is greater than this parameter.
- ``"minimal"``: Equivalent to ``1.``. Data is not masked
if a valid value was used to compute data.
- ``"maximal"``: Equivalent to ``0.``. Data is masked
if a invalid value was used to compute data.
- ``"same"``: Mask is the same as input mask.
- **copy**, optional: Copy variable before filtering it.
:Return:
- The filtered :mod:`MV2` variable.
:Example:
>>> generic1d(data, 3.) # running mean using a 3-points block
>>> generic1d(data, [1.,2.,1], axis=2) # shapiro filter on third axis
:See also: :func:`scipy.signal.convolve2d`
"""
# Setup
# - data
data = MV2.asarray(data)
assert data.ndim, 'Input data array must be at least 1D'
if axis<0: axis += data.ndim
if axis!=data.ndim-1:
init_order = data.getOrder(ids=1)
data = data.reorder('...%i'%axis)
datan = data.filled(0.).copy()
nx = datan.shape[-1]
datan.shape = -1, nx
# - weights
if isinstance(weights, int):
weights = N.ones(weights)
else:
weights = N.asarray(weights)
assert weights.ndim, 'Input weights array must be at least 1D'
assert weights.shape[-1] % 2 == 1, 'Shape of weights must be of odd size'
ww = (~N.ma.getmaskarray(data)).astype('i') # = good 2D
nw = weights.shape[-1]
nw2 = weights.shape[-1] / 2
weights.shape = 1, -1
ww.shape = datan.shape
if data.mask is not MV2.nomask:
one2d = N.ones(datan.shape, 'i')
# Cyclic case
if cyclic:
mode = 'valid'
fdatan = N.concatenate((datan[:, -nw2:], datan, datan[:, :nw2]), axis=1)
fww = N.concatenate((ww[:, -nw2:], ww, ww[:, :nw2]), axis=1)
one1d = N.ones(datan.shape[-1]+nw2*2, 'i')
else:
mode = 'same'
fdatan = datan
fww = ww
one1d = N.ones(datan.shape[-1], 'i')
# Filter
kwf = dict(mode=mode)
datan = convolve2d(fdatan, weights, **kwf)
one2d = N.resize(N.convolve(one1d, weights[0], **kwf), datan.shape)
del one1d
if data.mask is MV2.nomask:
ww = one2d
else:
ww = convolve2d(fww, weights, **kwf)
del fdatan, fww
bad = ww==0
ww[bad] = 1
datan[:] = N.where(bad, datan, datan/ww.astype('d'))
ww[bad] = 0
# Set
if copy:
datao = data.clone()
else:
datao = data
datan.shape = data.shape
bad.shape = data.shape
if data.mask is not MV2.nomask:
ww.shape = one2d.shape = data.shape
if mask is 'same':
bad = data.mask
else:
if mask == 'minimal':
mask = 1.
elif mask == 'maximal':
mask = 0.
else:
mask = float(mask)
bad |= (ww/one2d)<(1-mask) # threshold
datao[:] = N.ma.masked_where(bad, datan, copy=False)
del ww, one2d, bad
else:
datao[:] = datan
if axis!=data.ndim-1:
init_order = cdms2.order2index(datao.getAxisList(), init_order)
return datao.reorder(init_order)
return datao
def generic(var,
coefs,
units='hours',
get_tide=False,
only_tide=False,
filter_name='Generic',
check_regular=False,
strict=True,
interp_method='cubic',
axis=None):
"""Apply a filter to signal (cdms2 variable) using symetric coefficients. First axis is supposed to be time and regular.
- **var**: Data to filter with first axis supposed to be time.
- **coefs**: Second half of coefficients.
- *strict*: Mask values where at least one value is missing in the filtering interval.
- *filter_name*: Name to give to the filter [default: 'Generic']
%s
"""
# Return?
if only_tide:
get_tide = True
# Coefficients
units = unit_type(units)
sunits = unit_type(units, string_type=True)
coefs = coefs.astype('f')
if coefs[-1] != 0.:
coefs = N.concatenate((coefs, [
0.,
]))
ncoefs = (len(coefs) - 1) * 2 + 1
# Input variable
var = MV2.asarray(var)
check_axes(var)
# Input time axis
if axis is None:
try:
axis = var.getOrder().index('t')
except:
axis = 0
time = var.getAxis(axis)
if check_regular:
assert isregular(time), 'Your time axis seems not regular'
dt = N.diff(time[:]).mean()
if not time.attributes.has_key('units'):
time.units = 'hours since 1970-01-01'
print 'The time axis of your variable has no units. Assuming : ' + time.units
if not time.isTime():
time.designateTime(calendar=cdtime.DefaultCalendar)
# Check time range
time_range = time[-1] - time[0]
coefs_units = sunits + ' ' + ' '.join(time.units.split()[1:])
coefs_range0 = cdtime.r2r(cdtime.reltime(time[0], coefs_units), time.units)
coefs_range1 = coefs_range0.add(ncoefs - 1, units)
coefs_range = coefs_range1.value - coefs_range0.value
coefs_dt = coefs_range0.add(1, units).value - coefs_range0.value
if time_range < coefs_range:
raise Exception, 'Your sample is shorter than the time range of your coefficients'
nt = len(var)
# Remap coefficients to data time axis
tc = var.dtype.char
# print 'dt', dt
# print 'coefs_dt', coefs_dt
if abs((coefs_dt - dt) / dt) > 0.01:
oldx = cdms2.createAxis(N.arange(len(coefs)) * coefs_dt)
old_coefs = MV2.array(coefs, axes=[oldx])
newx = cdms2.createAxis(N.arange(0., max(oldx), dt))
coefs = interp1d(old_coefs, newx, interp_method).filled()
coefs = N.concatenate((coefs[:0:-1], coefs))
coefs /= N.sum(coefs)
ncoefs = len(coefs)
# Filtering using convolution
fvar = convolve1d(var.filled(0.), coefs, axis=axis, mode='constant')
if var.mask is MV2.nomask:
mask = N.zeros(nt)
else:
mask = var.mask.astype('f')
fmask = convolve1d(mask, coefs, axis=axis, mode='constant', cval=1.) != 0.
fvar = MV2.asarray(fvar)
fvar = MV2.masked_where(fmask, fvar, copy=0)
# Output variables
if not only_tide:
fvar.id = var.id + '_cotes'
fvar.name = fvar.id
if var.attributes.has_key('long_name'):
fvar.long_name = 'Tide removed signal from ' + var.long_name.lower(
)
fvar.long_name_fr = 'Signal sans la maree'
if var.attributes.has_key('units'):
fvar.units = var.units
fvar.setAxis(0, time)
fvar.filter_coefficients = coefs
fvar.filter_name = filter_name
res = [
fvar,
]
if get_tide:
tvar = var - fvar
tvar.id = 'tidal_' + var.id
tvar.name = tvar.id
if var.attributes.has_key('long_name'):
tvar.long_name = 'Tidal signal from ' + var.long_name.lower()
if var.attributes.has_key('units'):
tvar.units = var.units
tvar.setAxis(0, time)
tvar.filter_coefficients = coefs
tvar.filter_name = filter_name
if only_tide:
return tvar
res.append(tvar)
if isinstance(res, list) and len(res) == 1:
return res[0]
else:
return res
