def __init__(self, dataset, weights=None, norms=None,
keep_invalids=False, minvalid=None, clean_weights=True,
logger=None, loglevel=None, zerofill=False, **kwargs):
# Logger
Logger.__init__(self, logger=logger, loglevel=loglevel,
**dict_filter(kwargs, 'log_'))
# Input shape
if isinstance(dataset, (list, tuple)):
dataset = list(dataset)
self.map = len(dataset)
else:
dataset = [dataset]
self.map = 0
self.ndataset = self.nd = len(dataset)
self.dataset = dataset
# Other inits
self.data = []
self.nt = None
weights = self.remap(weights, reshape=True)
norms = self.remap(norms, reshape=True)
if self.ndataset==1 and norms[0] is None: norms = [False]
self._invalids = []
self.masked = False
# Loop on datasets
for idata,data in enumerate(dataset):
# Create the Data instance and pack array
dd = Data(data, norm=norms[idata], weights=weights[idata],
keep_invalids=keep_invalids, minvalid=minvalid, clean_weights=clean_weights,
zerofill=zerofill)
self.data.append(dd)
self._invalids.append(dd.invalids)
self.masked |= dd.masked
# Check nt
if self.nt is None:
self.nt = dd.nt
elif self.nt != dd.nt:
self.error('Time dimension of variable %i must have length %i (not %i)'%(idata, self.nt, dd.nt))
# Merge
self.stacked_data = npy.asfortranarray(npy.vstack([d.packed_data for d in self.data]))
self.splits = npy.cumsum([d.packed_data.shape[0] for d in self.data[:-1]])
self.stacked_weights = npy.hstack([d.packed_weights for d in self.data])
self.ns = self.stacked_data.shape[0]
self.ntv = (self.stacked_data!=default_missing_value).any(axis=0).sum()
def __init__(self):
Config.__init__(self)
Logger.__init__(self)
self.base = Firebase()
self.env = self.conf.get("enviroment")
def __init__(self, data, weights=None, norm=None, keep_invalids=False,
minvalid=None, clean_weights=True,
logger=None, loglevel=None, zerofill=False, **kwargs):
# Logger
Logger.__init__(self, logger=logger, loglevel=loglevel, **dict_filter(kwargs, 'log_'))
# Guess data type and copy
if cdms2_isVariable(data):
self.array_type = 'MV2'
self.array_mod = MV2
data = data.clone()
elif npy.ma.isMA(data):
self.array_type = 'numpy.ma'
self.array_mod = numpy.ma
data = data.copy()
else:
self.array_type = 'numpy'
data = data.copy()
self.array_mod = numpy
self.data = data
self.dtype = data.dtype
data = data.astype('d')
# Shape
self.shape = data.shape
self.ndim = data.ndim
self.nt = self.shape[0]
self.nstot = data.size/self.nt
self.nsdim = data.ndim-1
# Check time axis
if cdms2_isVariable(data) and data.getTime() is not None:
order = data.getOrder()
if not order.startswith('t'):
warn('Time axis is not the first axis of input variable (order="%s")'%order)
# Weights ?
if weights is None or weights is False:
if False and weights is not False and data.ndim == 3 and \
cdms2_isVariable(data) and \
'x' in data.getOrder() and 'y' in data.getOrder():
import cdutil# FIXME: WARNING FALSE
weights = cdutil.area_weights(data[0]).data.astype('d') # Geographic weights
elif self.nstot==1:
weights = npy.ones(1)
else:
weights = npy.ones(self.shape[1:])
elif npy.ma.isMA(weights):
weights = weight.astype('d').filled(0.)
else:
weights = npy.asarray(weights, dtype='d')
if data.ndim>1 and self.shape[1:] != weights.shape:
self.error('Weights must be of shape %s (instead of %s)'
%(self.shape[1:], weights.shape))
# Store some info
# - time
if not cdms2_isVariable(data):
self.taxis = data.shape[0]
else:
self.taxis = data.getAxis(0)
# - others axes and attributes
if cdms2_isVariable(data): # cdms -> ids
self.saxes = data.getAxisList()[1:]
self.id = data.id
self.atts = {}
for att in data.listattributes():
self.atts[att] = data.attributes[att]
self.grid = data.getGrid()
data = data.asma()
else: # numpy -> length
self.saxes = data.shape[1:]
self.id = None
self.atts = None
self.grid = None
# - missing value
if npy.ma.isMA(data):
self.missing_value = data.get_fill_value()
else:
self.missing_value = 1.e20
# - special cases
for att in 'long_name', 'units':
if hasattr(data, att):
setattr(self, att, data.attributes[att])
# Masking nans
nans = npy.isnan(data)
if nans.any():
self.warning("Masking %i NaNs"%nans.sum())
if self.array_type == 'numpy':
self.array_type = 'numpy.ma'
self.array_mod = numpy.ma
data = npy.ma.array(data, mask=nans, copy=False)
else:
data[nans] = npy.ma.masked
self.data = data
# Mask (1 means good)
# - real good values
bmask = npy.ma.getmaskarray(data)
good = 1-bmask.astype('l')
# - first from data (integrate) => 1D
count = npy.atleast_1d(good.sum(axis=0))
del good
# - now remove channels where weight is zero
if clean_weights:
count[npy.atleast_1d(weights==0.)] = 0
# - check number of valid data along time
minvalid = kwargs.pop('nvalid', minvalid)
if minvalid is not None and minvalid < 0:
minvalid = -int(round(npy.clip(minvalid, -100., 0)*self.nt/100))
minvalid = npy.clip(int(minvalid), 1, self.nt) if minvalid is not None else 1
count[count<minvalid] = 0 # <minvalid -> 0
count = npy.clip(count, 0, 1)
# - save as 0/1
self.ns = long(count.sum())
self.compress = count.size != self.ns
self.good = count>0 # points in space where there are enough data in time
self.minvalid = self.nvalid = minvalid
# Scale unpacked data
if not self.good.any():
self.warning('No valid data')
self.norm = 1.
self.mean = 0
else:
# - mean
self.mean = data.mean(axis=0)
# - normalisation factor
if norm is True or norm is None:
norm = self.data.std() # Standard norm
elif norm is not False:
if norm <0: # Relative norm, else strict norm
norm = abs(norm)*self.data.std()
else:
norm = 1.
self.norm = norm
# - apply
self.scale(data)
# Fill data
# - fill with missing value or mean (0.) where possible
if minvalid != self.nt:
# invalids = bmask & self.good # invalids = masked data that will be analyzed
# data[invalids] = 0. if zerofill else default_missing_value
# data[invalids] = default_missing_value
data[:, ~self.good] = default_missing_value
if keep_invalids:
self.invalids = bmask & self.good # invalids = masked data that will be analyzed
else:
self.invalids = None
#del invalids
else:
self.invalids = None
# - finally fill with missing values at zero
if npy.ma.isMA(data):
data_num = data.filled(default_missing_value)
else:
data_num = data
# Pack
# - data
self.packed_data = self.core_pack(data_num, force2d=True)
self.masked = npy.isclose(self.packed_data, default_missing_value).any()
# - weights
self.packed_weights = self.core_pack(weights)
def __init__(self, queue=None):
TCPServer.__init__(self)
Logger.__init__(self, 'TCPServer')
self._queue = queue
def __init__(self, queue=None, params=None):
Process.__init__(self, daemon=True)
Logger.__init__(self, 'TCPServerProcess')
self._queue = queue
self._params = params
self._server = None
def __init__(self):
TCPClient.__init__(self)
Logger.__init__(self, 'TCPClient')
def __init__(self, params):
Process.__init__(self, daemon=True)
Logger.__init__(self, 'TCPClientProcess')
self._params = params
self._client = TornadoTCPClient()
