def __init__(self, dataset, weights=None, norms=None, keep_invalids=False, minvalid=None, clean_weights=True, logger=None, loglevel=None, zerofill=False, **kwargs): # Logger Logger.__init__(self, logger=logger, loglevel=loglevel, **dict_filter(kwargs, 'log_')) # Input shape if isinstance(dataset, (list, tuple)): dataset = list(dataset) self.map = len(dataset) else: dataset = [dataset] self.map = 0 self.ndataset = self.nd = len(dataset) self.dataset = dataset # Other inits self.data = [] self.nt = None weights = self.remap(weights, reshape=True) norms = self.remap(norms, reshape=True) if self.ndataset==1 and norms[0] is None: norms = [False] self._invalids = [] self.masked = False # Loop on datasets for idata,data in enumerate(dataset): # Create the Data instance and pack array dd = Data(data, norm=norms[idata], weights=weights[idata], keep_invalids=keep_invalids, minvalid=minvalid, clean_weights=clean_weights, zerofill=zerofill) self.data.append(dd) self._invalids.append(dd.invalids) self.masked |= dd.masked # Check nt if self.nt is None: self.nt = dd.nt elif self.nt != dd.nt: self.error('Time dimension of variable %i must have length %i (not %i)'%(idata, self.nt, dd.nt)) # Merge self.stacked_data = npy.asfortranarray(npy.vstack([d.packed_data for d in self.data])) self.splits = npy.cumsum([d.packed_data.shape[0] for d in self.data[:-1]]) self.stacked_weights = npy.hstack([d.packed_weights for d in self.data]) self.ns = self.stacked_data.shape[0] self.ntv = (self.stacked_data!=default_missing_value).any(axis=0).sum()
def __init__(self): Config.__init__(self) Logger.__init__(self) self.base = Firebase() self.env = self.conf.get("enviroment")
def __init__(self, data, weights=None, norm=None, keep_invalids=False, minvalid=None, clean_weights=True, logger=None, loglevel=None, zerofill=False, **kwargs): # Logger Logger.__init__(self, logger=logger, loglevel=loglevel, **dict_filter(kwargs, 'log_')) # Guess data type and copy if cdms2_isVariable(data): self.array_type = 'MV2' self.array_mod = MV2 data = data.clone() elif npy.ma.isMA(data): self.array_type = 'numpy.ma' self.array_mod = numpy.ma data = data.copy() else: self.array_type = 'numpy' data = data.copy() self.array_mod = numpy self.data = data self.dtype = data.dtype data = data.astype('d') # Shape self.shape = data.shape self.ndim = data.ndim self.nt = self.shape[0] self.nstot = data.size/self.nt self.nsdim = data.ndim-1 # Check time axis if cdms2_isVariable(data) and data.getTime() is not None: order = data.getOrder() if not order.startswith('t'): warn('Time axis is not the first axis of input variable (order="%s")'%order) # Weights ? if weights is None or weights is False: if False and weights is not False and data.ndim == 3 and \ cdms2_isVariable(data) and \ 'x' in data.getOrder() and 'y' in data.getOrder(): import cdutil# FIXME: WARNING FALSE weights = cdutil.area_weights(data[0]).data.astype('d') # Geographic weights elif self.nstot==1: weights = npy.ones(1) else: weights = npy.ones(self.shape[1:]) elif npy.ma.isMA(weights): weights = weight.astype('d').filled(0.) else: weights = npy.asarray(weights, dtype='d') if data.ndim>1 and self.shape[1:] != weights.shape: self.error('Weights must be of shape %s (instead of %s)' %(self.shape[1:], weights.shape)) # Store some info # - time if not cdms2_isVariable(data): self.taxis = data.shape[0] else: self.taxis = data.getAxis(0) # - others axes and attributes if cdms2_isVariable(data): # cdms -> ids self.saxes = data.getAxisList()[1:] self.id = data.id self.atts = {} for att in data.listattributes(): self.atts[att] = data.attributes[att] self.grid = data.getGrid() data = data.asma() else: # numpy -> length self.saxes = data.shape[1:] self.id = None self.atts = None self.grid = None # - missing value if npy.ma.isMA(data): self.missing_value = data.get_fill_value() else: self.missing_value = 1.e20 # - special cases for att in 'long_name', 'units': if hasattr(data, att): setattr(self, att, data.attributes[att]) # Masking nans nans = npy.isnan(data) if nans.any(): self.warning("Masking %i NaNs"%nans.sum()) if self.array_type == 'numpy': self.array_type = 'numpy.ma' self.array_mod = numpy.ma data = npy.ma.array(data, mask=nans, copy=False) else: data[nans] = npy.ma.masked self.data = data # Mask (1 means good) # - real good values bmask = npy.ma.getmaskarray(data) good = 1-bmask.astype('l') # - first from data (integrate) => 1D count = npy.atleast_1d(good.sum(axis=0)) del good # - now remove channels where weight is zero if clean_weights: count[npy.atleast_1d(weights==0.)] = 0 # - check number of valid data along time minvalid = kwargs.pop('nvalid', minvalid) if minvalid is not None and minvalid < 0: minvalid = -int(round(npy.clip(minvalid, -100., 0)*self.nt/100)) minvalid = npy.clip(int(minvalid), 1, self.nt) if minvalid is not None else 1 count[count<minvalid] = 0 # <minvalid -> 0 count = npy.clip(count, 0, 1) # - save as 0/1 self.ns = long(count.sum()) self.compress = count.size != self.ns self.good = count>0 # points in space where there are enough data in time self.minvalid = self.nvalid = minvalid # Scale unpacked data if not self.good.any(): self.warning('No valid data') self.norm = 1. self.mean = 0 else: # - mean self.mean = data.mean(axis=0) # - normalisation factor if norm is True or norm is None: norm = self.data.std() # Standard norm elif norm is not False: if norm <0: # Relative norm, else strict norm norm = abs(norm)*self.data.std() else: norm = 1. self.norm = norm # - apply self.scale(data) # Fill data # - fill with missing value or mean (0.) where possible if minvalid != self.nt: # invalids = bmask & self.good # invalids = masked data that will be analyzed # data[invalids] = 0. if zerofill else default_missing_value # data[invalids] = default_missing_value data[:, ~self.good] = default_missing_value if keep_invalids: self.invalids = bmask & self.good # invalids = masked data that will be analyzed else: self.invalids = None #del invalids else: self.invalids = None # - finally fill with missing values at zero if npy.ma.isMA(data): data_num = data.filled(default_missing_value) else: data_num = data # Pack # - data self.packed_data = self.core_pack(data_num, force2d=True) self.masked = npy.isclose(self.packed_data, default_missing_value).any() # - weights self.packed_weights = self.core_pack(weights)
def __init__(self, queue=None): TCPServer.__init__(self) Logger.__init__(self, 'TCPServer') self._queue = queue
def __init__(self, queue=None, params=None): Process.__init__(self, daemon=True) Logger.__init__(self, 'TCPServerProcess') self._queue = queue self._params = params self._server = None
def __init__(self): TCPClient.__init__(self) Logger.__init__(self, 'TCPClient')
def __init__(self, params): Process.__init__(self, daemon=True) Logger.__init__(self, 'TCPClientProcess') self._params = params self._client = TornadoTCPClient()