def test_fix_slice_with_list(self): shp = (50,) # Verify list is tupled sl = utils.fix_slice([[7,9]], shp) self.assertEqual(sl, ([7,9],)) shp = (50,20) # Verify list of lists are tupled sl = utils.fix_slice([[5,8],[1,3]], shp) self.assertEqual(sl, ([5,8],[1,3])) shp = (50,) #### Passes #### # List of indices sl = utils.fix_slice([[5, 14, 36]], shp) self.assertEqual(sl, ([5, 14, 36],)) # List with some negative indices sl = utils.fix_slice([[5, -36, 23]], shp) self.assertEqual(sl, ([5, 14, 23],)) # List of all negative indices sl = utils.fix_slice([[-44, -32, -5]], shp) self.assertEqual(sl, ([6, 18, 45],)) #### Failures #### # List of indices in non-increasing order self.assertRaises(IndexError, utils.fix_slice, [[20, 14, 36]], shp) # List of negative indices in non-increasing order (after adjustment, list is [45, 30, 5] self.assertRaises(IndexError, utils.fix_slice, [[-5, -20, -45]], shp)
def test_fix_slice_with_int(self): shp = (50,) # Verify int is tupled sl = utils.fix_slice(7, shp) self.assertEqual(sl, (7,)) shp = (50,20) # Verify list of ints are tupled sl = utils.fix_slice([5,8], shp) self.assertEqual(sl, (5,8,)) shp = (50,) #### Passes #### # Index sl = utils.fix_slice(0, shp) self.assertEqual(sl, (0,)) # Negative index sl = utils.fix_slice(-4, shp) self.assertEqual(sl, (46,)) #### Failures #### # Index > size self.assertRaises(IndexError, utils.fix_slice, 52, shp) # Negative index > size (after adjustment, index is -2) self.assertRaises(IndexError, utils.fix_slice, -52, shp)
def test_fix_slice_with_list(self): shp = (50,) # Verify list is tupled sl = utils.fix_slice([[7,9]], shp) self.assertEqual(sl, ([7,9],)) shp = (50,20) # Verify list of lists are tupled sl = utils.fix_slice([[5,8],[1,3]], shp) self.assertEqual(sl, ([5,8],[1,3])) shp = (50,) #### Passes #### # List of indices sl = utils.fix_slice([[5, 14, 36]], shp) self.assertEqual(sl, ([5, 14, 36],)) # List with some negative indices sl = utils.fix_slice([[5, -36, 23]], shp) self.assertEqual(sl, ([5, 14, 23],)) # List of all negative indices sl = utils.fix_slice([[-44, -32, -5]], shp) self.assertEqual(sl, ([6, 18, 45],)) #### Failures #### # List of indices in non-increasing order self.assertRaises(IndexError, utils.fix_slice, [[20, 14, 36]], shp) # List of negative indices in non-increasing order (after adjustment, list is [45, 30, 5] self.assertRaises(IndexError, utils.fix_slice, [[-5, -20, -45]], shp)
def test_fix_slice_with_int(self): shp = (50,) # Verify int is tupled sl = utils.fix_slice(7, shp) self.assertEqual(sl, (7,)) shp = (50,20) # Verify list of ints are tupled sl = utils.fix_slice([5,8], shp) self.assertEqual(sl, (5,8,)) shp = (50,) #### Passes #### # Index sl = utils.fix_slice(0, shp) self.assertEqual(sl, (0,)) # Negative index sl = utils.fix_slice(-4, shp) self.assertEqual(sl, (46,)) #### Failures #### # Index > size self.assertRaises(IndexError, utils.fix_slice, 52, shp) # Negative index > size (after adjustment, index is -2) self.assertRaises(IndexError, utils.fix_slice, -52, shp)
def __getitem__(self, slice_): slice_ = fix_slice(slice_, self.shape) total_indices = prod(self.shape) x = np.arange(total_indices).reshape(self.shape)[slice_] # CBM TODO: This is INDEX based evaluation!!! return ne.evaluate(self.content).astype(self.value_encoding)
def __indexify_slice(self, slice_, total_shape): ## ONLY WORKS FOR 1D ARRAYS!!! fsl = utils.fix_slice(slice_, total_shape) ss = utils.slice_shape(slice_, total_shape) ret = np.empty(ss, dtype=int) rf = ret.flatten() ci = 0 for s, shape in zip(fsl, total_shape): if isinstance(s,slice): ind = range(*s.indices(shape)) ll = len(ind) rf[ci:ll] = ind ci += ll elif isinstance(s, (list,tuple)): ll = len(s) rf[ci:ll] = s ci += ll elif isinstance(s, int): rf[ci] = s ci += 1 else: raise TypeError('Unsupported slice method') # TODO: Better error message return rf.reshape(ss)
def __setitem__(self, slice_, value): if is_well_formed_where(value): slice_ = fix_slice(slice_, self.shape) if len(slice_) == 1 and isinstance(slice_[0], int) and slice_ < len(self._storage[0]): self._storage[0][slice_[0]] = value else: self._storage[0].append(value)
def __indexify_slice(self, slice_, total_shape): ## ONLY WORKS FOR 1D ARRAYS!!! fsl = utils.fix_slice(slice_, total_shape) ss = utils.slice_shape(slice_, total_shape) ret = np.empty(ss, dtype=int) rf = ret.flatten() ci = 0 for s, shape in zip(fsl, total_shape): if isinstance(s, slice): ind = range(*s.indices(shape)) ll = len(ind) rf[ci:ll] = ind ci += ll elif isinstance(s, (list, tuple)): ll = len(s) rf[ci:ll] = s ci += ll elif isinstance(s, int): rf[ci] = s ci += 1 else: raise TypeError( 'Unsupported slice method') # TODO: Better error message return rf.reshape(ss)
def __getitem__(self, slice_): if self._memoized_values is not None: return self._memoized_values else: if self._pval_callback is None: raise ParameterFunctionException( '\'_pval_callback\' is None; cannot evaluate!!') slice_ = utils.fix_slice(slice_, self.shape) try: r = self.content.evaluate(self._pval_callback, slice_, self.parameter_type.fill_value) ve = self.parameter_type.value_encoding if hasattr(self.parameter_type, 'inner_encoding'): ve = self.parameter_type.inner_encoding if ve is not None: r = np.asanyarray(r, dtype=ve) except Exception as ex: import sys raise ParameterFunctionException( ex.message, type(ex)), None, sys.exc_traceback return _cleanse_value(r, slice_)
def __getitem__(self, slice_): slice_ = utils.fix_slice(slice_, self.shape) ret_shape = utils.slice_shape(slice_, self.shape) ret = np.empty(ret_shape, dtype=np.dtype(object)) # Always object type because it's 2 values / element!! ret.fill(self.content) return _cleanse_value(ret, slice_)
def __getitem__(self, slice_): slice_ = utils.fix_slice(slice_, self.shape) ret_shape = utils.slice_shape(slice_, self.shape) ret = np.empty(ret_shape, dtype=np.dtype(self.value_encoding)) ret.fill(self.content) return _cleanse_value(ret, slice_)
def __getitem__(self, slice_): slice_ = utils.fix_slice(slice_, self.shape) ret_shape = utils.slice_shape(slice_, self.shape) ret = np.empty(ret_shape, dtype=np.dtype(self.value_encoding)) ret.fill(self.content) return _cleanse_value(ret, slice_)
def __setitem__(self, slice_, value): if is_well_formed_where(value): slice_ = utils.fix_slice(slice_, self.shape) if len(slice_) == 1 and isinstance( slice_[0], int) and slice_ < len(self._storage[0]): self._storage[0][slice_[0]] = value else: self._storage[0].append(value)
def test_fix_slice(self): shp = (50,) # Initial 'is_valid_constraint' catches bad stuff self.assertRaises(SystemError, utils.fix_slice, 'bob', shp) # Tuples are list-ified sl = utils.fix_slice((20,), shp) # Down-ranking (slice with more dims than shp) shp = (50,) sl = utils.fix_slice([5,23], shp) self.assertEqual(sl, (5,)) # Up-ranking (slice with fewer dims than shp) shp = (50,10,) sl = utils.fix_slice(5, shp) self.assertEqual(sl, (5, slice(None, None, None)))
def __getitem__(self, slice_): slice_ = utils.fix_slice(slice_, self.shape) total_indices = utils.prod(self.shape) x = np.arange(total_indices).reshape( self.shape)[slice_] # CBM TODO: This is INDEX based evaluation!!! return _cleanse_value( ne.evaluate(self.content).astype(self.value_encoding), slice_)
def test_fix_slice(self): shp = (50,) # Initial 'is_valid_constraint' catches bad stuff self.assertRaises(SystemError, utils.fix_slice, 'bob', shp) # Tuples are list-ified sl = utils.fix_slice((20,), shp) # Down-ranking (slice with more dims than shp) shp = (50,) sl = utils.fix_slice([5,23], shp) self.assertEqual(sl, (5,)) # Up-ranking (slice with fewer dims than shp) shp = (50,10,) sl = utils.fix_slice(5, shp) self.assertEqual(sl, (5, slice(None, None, None)))
def __getitem__(self, slice_): slice_ = utils.fix_slice(slice_, self.shape) ret_shape = utils.slice_shape(slice_, self.shape) ret = np.empty(ret_shape, dtype=np.dtype( object)) # Always object type because it's 2 values / element!! ret.fill(self.content) return _cleanse_value(ret, slice_)
def __setitem__(self, slice_, value): slice_ = utils.fix_slice(slice_, self.shape) try: spans = self._storage[0] except ValueError, ve: if ve.message != 'No Bricks!': raise spans = self.fill_value
def __setitem__(self, slice_, value): slice_ = utils.fix_slice(slice_, self.shape) try: spans = self._storage[0] except ValueError, ve: if ve.message != 'No Bricks!': raise spans = self.fill_value
def __getitem__(self, slice_): slice_ = utils.fix_slice(slice_, self.shape) ret = np.atleast_1d(self._storage[slice_]) cats = self.parameter_type.categories if np.iterable(ret): ret = np.array([cats[x] for x in ret], dtype=object) else: ret = cats[ret] return _cleanse_value(ret, slice_)
def __getitem__(self, slice_): slice_ = utils.fix_slice(slice_, self.shape) ret = np.atleast_1d(self._storage[slice_]) cats = self.parameter_type.categories if np.iterable(ret): ret = np.array([cats[x] for x in ret], dtype=object) else: ret = cats[ret] return _cleanse_value(ret, slice_)
def __getitem__(self, slice_): slice_ = fix_slice(slice_, self.shape) ret = self._storage[slice_] cats=self.parameter_type.categories if np.iterable(ret): ret = np.array([cats[x] for x in ret], dtype=object) else: ret = cats[ret.item()] return ret
def __getitem__(self, slice_): slice_ = fix_slice(slice_, self.shape) c = np.ones(self.shape)[slice_] # Make an array of ones of the appropriate shape and slice it as desired ret = ne.evaluate(self.content).astype(self.value_encoding) if ret.size==1: if ret.ndim==0: ret=ret[()] else: ret=ret[0] return ret
def __getitem__(self, slice_): oslice = slice_ = utils.fix_slice(slice_, self.shape) if isinstance(slice_[0], int): # Must always pass a slice so we don't lose the dimension slice_ = (slice(slice_[0], slice_[0] + 1),) + slice_[1:] vals = np.atleast_1d(self._storage[slice_]) vals = self._apply_inner_encoding(vals, self.parameter_type) ret = _cleanse_value(vals, oslice) return ret
def __getitem__(self, slice_): slice_ = utils.fix_slice(slice_, self.shape) # Nothing asked for! if len(slice_) is 0: return np.empty(0, dtype=self.value_encoding) try: spans = self._storage[0] except ValueError, ve: if ve.message != 'No Bricks!': raise return np.empty(0, dtype=self.value_encoding)
def __getitem__(self, slice_): slice_ = utils.fix_slice(slice_, self.shape) # Nothing asked for! if len(slice_) is 0: return np.empty(0, dtype=self.value_encoding) try: spans = self._storage[0] except ValueError, ve: if ve.message != 'No Bricks!': raise return np.empty(0, dtype=self.value_encoding)
def __getitem__(self, slice_): oslice = slice_ = utils.fix_slice(slice_, self.shape) if isinstance( slice_[0], int ): # Must always pass a slice so we don't lose the dimension slice_ = (slice(slice_[0], slice_[0] + 1), ) + slice_[1:] vals = np.atleast_1d(self._storage[slice_]) vals = self._apply_inner_encoding(vals, self.parameter_type) ret = _cleanse_value(vals, oslice) return ret
def get_data_size(self, parameter_name=None, slice_=None, in_bytes=False): """ Returns the size of the <b>data values</b> for the parameter(s) indicated by <i>parameter_name</i>. ParameterContext and Coverage metadata is <b>NOT</b> included in the returned size. If <i>parameter_name</i> is None, all parameters in the coverage are included If more than one parameter is indicated by <i>parameter_name</i>, the sum of the indicated parameters is returned If <i>slice_</i> is not None, it is applied to each parameter (after being run through utils.fix_slice) before calculation of size Sizes are calculated as: size = itemsize * total_extent_size where: itemsize == the per-item size based on the data type of the parameter total_extent_size == the total number of elements after slicing is applied (if applicable) Sizes are in MB unless <i>in_bytes</i> == True @param parameter_name A string parameter name; may be an iterable of such members @param slice_ If not None, applied to each parameter before calculation of size @param in_bytes If True, returns the size in bytes; otherwise, returns the size in MB (default) """ size = 0 if parameter_name is None: for pn in self._range_dictionary.keys(): size += self.get_data_size(pn, in_bytes=in_bytes) for pn in self.__parameter_name_arg_to_params(parameter_name): p = self._range_dictionary.get_context(pn) te=p.dom.total_extents dt = np.dtype(p.param_type.value_encoding) if slice_ is not None: slice_ = utils.fix_slice(slice_, te) a=np.empty(te, dtype=dt)[slice_] size += a.nbytes else: size += dt.itemsize * utils.prod(te) if not in_bytes: size *= 9.53674e-7 return size
def get_values_from_bricks(self, slice_): slice_ = utils.fix_slice(slice_, self.total_domain) bricks = bricking_utils.get_bricks_from_slice( slice_, self.rtree, self.total_domain ) # this is a list of tuples [(b_id, (bounds...),), ...] ret_shp = utils.slice_shape(slice_, self.total_domain) ret_arr = np.empty(ret_shp, dtype=self.dtype) for b in bricks: bid, bbnds = b brick_slice, brick_mm = bricking_utils.get_brick_slice_nd( slice_, bbnds) if None in brick_slice: continue ret_slice = bricking_utils.get_value_slice_nd( slice_, ret_shp, bbnds, brick_slice, brick_mm) if not self.use_hdf: ret_vals = self.bricks[bid][brick_slice] else: fi = self.bricks[bid] with h5py.File(fi) as f: ds = f.require_dataset(str(bid), shape=self.brick_sizes, dtype=self.dtype, chunks=None, fillvalue=-1) ret_vals = ds[brick_slice] ret_arr[ret_slice] = ret_vals ret_arr = ret_arr.squeeze() if ret_arr.size == 1: if ret_arr.ndim == 0: ret_arr = ret_arr[()] else: ret_arr = ret_arr[0] return ret_arr
def __setitem__(self, slice_, value): slice_ = utils.fix_slice(slice_, self.shape) self.parameter_type.is_valid_value(value) value = np.atleast_1d(value) # Replace any None with fill_value np.place(value, value == np.array([None]), self.fill_value) if value.dtype.kind == np.dtype(self.parameter_type.value_encoding).kind: # Set as ordinals self._storage[slice_] = value else: # Set as categories rcats={v:k for k,v in self.parameter_type.categories.iteritems()} try: vals=[rcats[v] if v != self.fill_value else v for v in value] self._storage[slice_] = vals except KeyError, ke: raise ValueError('Invalid category specified: \'{0}\''.format(ke.message))
def __getitem__(self, slice_): if self._memoized_values is not None: return self._memoized_values else: if self._pval_callback is None: raise ParameterFunctionException('\'_pval_callback\' is None; cannot evaluate!!') slice_ = utils.fix_slice(slice_, self.shape) try: r = self.content.evaluate(self._pval_callback, slice_, self.parameter_type.fill_value) ve = self.parameter_type.value_encoding if hasattr(self.parameter_type, 'inner_encoding'): ve = self.parameter_type.inner_encoding if ve is not None: r = np.asanyarray(r, dtype=ve) except Exception as ex: import sys raise ParameterFunctionException(ex.message, type(ex)), None, sys.exc_traceback return _cleanse_value(r, slice_)
def get_values_from_bricks(self, slice_): slice_ = utils.fix_slice(slice_, self.total_domain) bricks = bricking_utils.get_bricks_from_slice( slice_, self.rtree, self.total_domain ) # this is a list of tuples [(b_id, (bounds...),), ...] ret_shp = utils.slice_shape(slice_, self.total_domain) ret_arr = np.empty(ret_shp, dtype=self.dtype) for b in bricks: bid, bbnds = b brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds) if None in brick_slice: continue ret_slice = bricking_utils.get_value_slice_nd(slice_, ret_shp, bbnds, brick_slice, brick_mm) if not self.use_hdf: ret_vals = self.bricks[bid][brick_slice] else: fi = self.bricks[bid] with HDFLockingFile(fi) as f: ds = f.require_dataset( str(bid), shape=self.brick_sizes, dtype=self.dtype, chunks=None, fillvalue=-1 ) ret_vals = ds[brick_slice] ret_arr[ret_slice] = ret_vals ret_arr = ret_arr.squeeze() if ret_arr.size == 1: if ret_arr.ndim == 0: ret_arr = ret_arr[()] else: ret_arr = ret_arr[0] return ret_arr
def __setitem__(self, slice_, value): slice_ = utils.fix_slice(slice_, self.shape) self.parameter_type.is_valid_value(value) value = np.atleast_1d(value) # Replace any None with fill_value np.place(value, value == np.array([None]), self.fill_value) if value.dtype.kind == np.dtype( self.parameter_type.value_encoding).kind: # Set as ordinals self._storage[slice_] = value else: # Set as categories rcats = { v: k for k, v in self.parameter_type.categories.iteritems() } try: vals = [rcats[v] if v != self.fill_value else v for v in value] self._storage[slice_] = vals except KeyError, ke: raise ValueError('Invalid category specified: \'{0}\''.format( ke.message))
def __setitem__(self, slice_, value): slice_ = utils.fix_slice(slice_, self.shape) if isinstance(value, AbstractParameterValue): value = np.atleast_1d(value[:]) value = np.atleast_1d(value) if len(value.shape) > 1: v = np.empty(value.shape[0], dtype=object) for i in xrange(value.shape[0]): iv = value[i,:] if isinstance(iv, np.ndarray): v[i] = iv.tolist() else: v[i] = iv value = v if isinstance(slice_[0], int): slice_ = (slice(slice_[0], slice_[0] + 1),) + slice_[1:] self._storage[slice_] = value[:] self._update_min_max(value)
def __setitem__(self, slice_, value): slice_ = utils.fix_slice(slice_, self.shape) if isinstance(value, AbstractParameterValue): value = np.atleast_1d(value[:]) value = np.atleast_1d(value) if len(value.shape) > 1: v = np.empty(value.shape[0], dtype=object) for i in xrange(value.shape[0]): iv = value[i, :] if isinstance(iv, np.ndarray): v[i] = iv.tolist() else: v[i] = iv value = v if isinstance(slice_[0], int): slice_ = (slice(slice_[0], slice_[0] + 1), ) + slice_[1:] self._storage[slice_] = value[:] self._update_min_max(value)
def __setitem__(self, slice_, value): """ Called to implement assignment of self[slice_, value]. Not implemented by the abstract class @param slice A set of valid constraints - int, [int,], (int,), or slice @param value The value to assign to the storage at location slice_ @raise ValueError when brick contains no values for specified slice """ if self.mode == 'r': raise IOError('PersistenceLayer not open for writing: mode == \'{0}\''.format(self.mode)) from coverage_model import bricking_utils, utils extents = tuple([s for s in self.total_domain.total_extents if s != 0]) # bricks is a list of tuples [(b_ord, b_guid), ...] slice_ = utils.fix_slice(deepcopy(slice_), extents) log.trace('slice_=%s', slice_) bricks = bricking_utils.get_bricks_from_slice(slice_, self.brick_tree, extents) log.trace('Found bricks: %s', bricks) values = np.asanyarray(value) v_shp = values.shape log.trace('value_shape: %s', v_shp) s_shp = utils.slice_shape(slice_, extents) log.trace('slice_shape: %s', s_shp) is_broadcast = False if v_shp == (): log.trace('Broadcast!!') is_broadcast = True value_slice = () elif v_shp != s_shp: if v_shp == tuple([i for i in s_shp if i != 1]): # Missing dimensions are singleton, just reshape to fit values = values.reshape(s_shp) v_shp = values.shape else: raise IndexError( 'Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}'.format( s_shp, v_shp)) else: value_slice = None log.trace('value_shape: %s', v_shp) for b in bricks: # b is (brick_ordinal, brick_guid) _, bid = b # brick_list[brick_guid] contains: [brick_extents, origin, tuple(bD), brick_active_size] _, bori, _, bact = self.brick_list[bid] bbnds = [] bexts = [] for i, bnd in enumerate(bori): bbnds.append((bori[i], bori[i] + bact[i] - 1)) bexts.append(bori[i] + bact[i]) bbnds = tuple(bbnds) bexts = tuple(bexts) log.trace('bid=%s, bbnds=%s, bexts=%s', bid, bbnds, bexts) log.trace('Determining slice for brick: %s', b) brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds) log.trace('brick_slice=%s\tbrick_mm=%s', brick_slice, brick_mm) if None in brick_slice: # Brick does not contain any of the requested indices log.debug('Brick does not contain any of the requested indices: Move to next brick') continue try: brick_slice = utils.fix_slice(brick_slice, bexts) except IndexError: log.debug('Malformed brick_slice: move to next brick') continue if not is_broadcast: value_slice = bricking_utils.get_value_slice_nd(slice_, v_shp, bbnds, brick_slice, brick_mm) try: value_slice = utils.fix_slice(value_slice, v_shp) except IndexError: log.debug('Malformed value_slice: move to next brick') continue log.trace('\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s', b, utils.slice_shape(brick_slice, bexts), brick_slice, brick_mm, utils.slice_shape(value_slice, v_shp), value_slice) v = values[value_slice] self._set_values_to_brick(bid, brick_slice, v) import datetime if self.parameter_manager.parameter_name in self.master_manager.param_groups and v.dtype.type is not np.string_: valid_types = [np.bool_, np.int_, np.intc, np.intp, np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64, np.float_, np.float16, np.float32, np.float64, np.complex_, np.complex64, np.complex128] invalid_fill_values = [None, np.NaN, self.fill_value] try: min_val = None max_val = None # First try to do it fast try: if issubclass(v.dtype.type, numbers.Number) or v.dtype.type in valid_types: tried_it = True min_val = v.min() max_val = v.max() except: min_val = None max_val = None # if fast didn't return valid values, do it slow, but right if min_val in invalid_fill_values or max_val in invalid_fill_values: ts = datetime.datetime.now() mx = [x for x in v if x not in invalid_fill_values and (type(x) in valid_types or issubclass(type(x), numbers.Number))] if len(mx) > 0: min_val = min(mx) max_val = max(mx) time_loss = datetime.datetime.now() - ts log.debug("Repaired numpy statistics inconsistency for parameter/type %s/%s. Time loss of %s seconds ", self.parameter_manager.parameter_name, str(v.dtype.type), str(time_loss)) if min_val is not None and max_val is not None: log.trace("%s min/max %s/%s type %s", self.parameter_manager.parameter_name, min_val, max_val, type(min_val)) self.master_manager.track_data_written_to_brick(bid, brick_slice, self.parameter_manager.parameter_name, min_val, max_val) except Exception as e: log.warn("Could not store Span extents for %s. Unexpected error %s", str( (bid, brick_slice, self.parameter_manager.parameter_name)), e.message ) raise
def __getitem__(self, slice_): """ Called to implement evaluation of self[slice_]. Not implemented by the abstract class @param slice_ A set of valid constraints - int, [int,], (int,), or slice @return The value contained by the storage at location slice @raise ValueError when brick contains no values for specified slice """ from coverage_model import bricking_utils, utils extents = tuple([s for s in self.total_domain.total_extents if s != 0]) if extents == (): # Empty domain(s) - no data, return empty array return np.empty(0, dtype=self.dtype) # bricks is a list of tuples [(b_ord, b_guid), ...] slice_ = utils.fix_slice(deepcopy(slice_), extents) log.trace('slice_=%s', slice_) bricks = bricking_utils.get_bricks_from_slice(slice_, self.brick_tree, extents) log.trace('Found bricks: %s', bricks) ret_shp = utils.slice_shape(slice_, extents) log.trace('Return array shape: %s', ret_shp) ret_arr = np.empty(ret_shp, dtype=self.dtype) ret_arr.fill(self.fill_value) for b in bricks: # b is (brick_ordinal, brick_guid) _, bid = b # brick_list[brick_guid] contains: [brick_extents, origin, tuple(bD), brick_active_size] _, bori, _, bact = self.brick_list[bid] bbnds = [] for i, bnd in enumerate(bori): bbnds.append((bori[i], bori[i] + bact[i] - 1)) bbnds = tuple(bbnds) brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds) log.trace('brick_slice=%s\tbrick_mm=%s', brick_slice, brick_mm) if None in brick_slice: log.debug('Brick does not contain any of the requested indices: Move to next brick') continue ret_slice = bricking_utils.get_value_slice_nd(slice_, ret_shp, bbnds, brick_slice, brick_mm) brick_file_path = '{0}/{1}.hdf5'.format(self.brick_path, bid) if not os.path.exists(brick_file_path): log.trace('Found virtual brick file: %s', brick_file_path) else: log.trace('Found real brick file: %s', brick_file_path) with HDFLockingFile(brick_file_path) as brick_file: ret_vals = brick_file[bid][brick_slice] # Check if object type if self.dtype in ('|O8', '|O4'): if hasattr(ret_vals, '__iter__'): ret_vals = [self._object_unpack_hook(x) for x in ret_vals] else: ret_vals = self._object_unpack_hook(ret_vals) if self.parameter_manager.parameter_name == 'lat': log.trace("values from brick %s %s", str(ret_vals), type(ret_vals)) ret_arr[ret_slice] = ret_vals # ret_arr = np.atleast_1d(ret_arr.squeeze()) # ret_arr = np.atleast_1d(ret_arr) # # # If the array is size 1 AND a slice object was NOT part of the query # if ret_arr.size == 1 and not np.atleast_1d([isinstance(s, slice) for s in slice_]).all(): # ret_arr = ret_arr[0] return ret_arr
def __setitem__(self, slice_, value): """ Called to implement assignment of self[slice_, value]. Not implemented by the abstract class @param slice A set of valid constraints - int, [int,], (int,), or slice @param value The value to assign to the storage at location slice_ @raise ValueError when brick contains no values for specified slice """ if self.mode == 'r': raise IOError('PersistenceLayer not open for writing: mode == \'{0}\''.format(self.mode)) from coverage_model import bricking_utils, utils extents = tuple([s for s in self.total_domain.total_extents if s != 0]) # bricks is a list of tuples [(b_ord, b_guid), ...] slice_ = utils.fix_slice(deepcopy(slice_), extents) log.trace('slice_=%s', slice_) bricks = bricking_utils.get_bricks_from_slice(slice_, self.brick_tree, extents) log.trace('Found bricks: %s', bricks) values = np.asanyarray(value) v_shp = values.shape log.trace('value_shape: %s', v_shp) s_shp = utils.slice_shape(slice_, extents) log.trace('slice_shape: %s', s_shp) is_broadcast = False if v_shp == (): log.trace('Broadcast!!') is_broadcast = True value_slice = () elif v_shp != s_shp: if v_shp == tuple([i for i in s_shp if i != 1]): # Missing dimensions are singleton, just reshape to fit values = values.reshape(s_shp) v_shp = values.shape else: raise IndexError( 'Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}'.format( s_shp, v_shp)) else: value_slice = None log.trace('value_shape: %s', v_shp) for b in bricks: # b is (brick_ordinal, brick_guid) _, bid = b # brick_list[brick_guid] contains: [brick_extents, origin, tuple(bD), brick_active_size] _, bori, _, bact = self.brick_list[bid] bbnds = [] bexts = [] for i, bnd in enumerate(bori): bbnds.append((bori[i], bori[i] + bact[i] - 1)) bexts.append(bori[i] + bact[i]) bbnds = tuple(bbnds) bexts = tuple(bexts) log.trace('bid=%s, bbnds=%s, bexts=%s', bid, bbnds, bexts) log.trace('Determining slice for brick: %s', b) brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds) log.trace('brick_slice=%s\tbrick_mm=%s', brick_slice, brick_mm) if None in brick_slice: # Brick does not contain any of the requested indices log.debug('Brick does not contain any of the requested indices: Move to next brick') continue try: brick_slice = utils.fix_slice(brick_slice, bexts) except IndexError: log.debug('Malformed brick_slice: move to next brick') continue if not is_broadcast: value_slice = bricking_utils.get_value_slice_nd(slice_, v_shp, bbnds, brick_slice, brick_mm) try: value_slice = utils.fix_slice(value_slice, v_shp) except IndexError: log.debug('Malformed value_slice: move to next brick') continue log.trace('\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s', b, utils.slice_shape(brick_slice, bexts), brick_slice, brick_mm, utils.slice_shape(value_slice, v_shp), value_slice) v = values[value_slice] self._set_values_to_brick(bid, brick_slice, v)
def put_values_to_bricks(self, slice_, values): slice_ = utils.fix_slice(slice_, self.total_domain) bricks = bricking_utils.get_bricks_from_slice(slice_, self.rtree, self.total_domain) # this is a list of tuples [(b_id, (bounds...),), ...] values = np.asanyarray(values) v_shp = values.shape log.debug('value_shape: %s', v_shp) s_shp = utils.slice_shape(slice_, self.total_domain) log.debug('slice_shape: %s', s_shp) is_broadcast = False if v_shp == (): log.debug('Broadcast!!') is_broadcast = True value_slice = () elif v_shp != s_shp: if v_shp == tuple([i for i in s_shp if i != 1]): # Missing dimensions are singleton, just reshape to fit values = values.reshape(s_shp) v_shp = values.shape else: raise IndexError( 'Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}'.format( s_shp, v_shp)) else: value_slice = None log.debug('value_shape: %s', v_shp) for b in bricks: # b is (brick_id, (brick_bounds per dim...),) bid, bbnds = b log.debug('Determining slice for brick: %s', b) bexts = tuple([x + 1 for x in zip(*bbnds)[1]]) # Shift from index to size log.debug('bid=%s, bbnds=%s, bexts=%s', bid, bbnds, bexts) brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds) if None in brick_slice: # Brick does not contain any of the requested indices log.debug('Brick does not contain any of the requested indices: Move to next brick') continue try: brick_slice = utils.fix_slice(brick_slice, bexts) except IndexError: log.debug('Malformed brick_slice: move to next brick') continue if not is_broadcast: value_slice = bricking_utils.get_value_slice_nd(slice_, v_shp, bbnds, brick_slice, brick_mm) try: value_slice = utils.fix_slice(value_slice, v_shp) except IndexError: log.debug('Malformed value_slice: move to next brick') continue log.debug('\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s', b, utils.slice_shape(brick_slice, bexts), brick_slice, brick_mm, utils.slice_shape(value_slice, v_shp), value_slice) v = values[value_slice] log.debug('\nvalues %s=\n%s', v.shape, v) if not self.use_hdf: self.bricks[bid][brick_slice] = v else: fi = self.bricks[bid] with HDFLockingFile(fi, 'a') as f: ds = f.require_dataset(str(bid), shape=self.brick_sizes, dtype=self.dtype, chunks=None, fillvalue=-1) ds[brick_slice] = v
def __setitem__(self, slice_, value): slice_ = utils.fix_slice(slice_, self.shape) if self.parameter_type.is_valid_value(value): self._storage[slice_] = np.asanyarray(value, dtype='bool') self._update_min_max(np.asanyarray(value, dtype='int8'))
def __getitem__(self, slice_): slice_ = fix_slice(slice_, self.shape) ret = self._storage[slice_] return ret
def __setitem__(self, slice_, value): slice_ = utils.fix_slice(slice_, self.shape) self._storage[slice_] = value self._update_min_max(value)
def __getitem__(self, slice_): slice_ = utils.fix_slice(slice_, self.shape) return _cleanse_value(self._storage[slice_], slice_)
def put_values_to_bricks(self, slice_, values): slice_ = utils.fix_slice(slice_, self.total_domain) bricks = bricking_utils.get_bricks_from_slice( slice_, self.rtree, self.total_domain ) # this is a list of tuples [(b_id, (bounds...),), ...] values = np.asanyarray(values) v_shp = values.shape log.debug('value_shape: %s', v_shp) s_shp = utils.slice_shape(slice_, self.total_domain) log.debug('slice_shape: %s', s_shp) is_broadcast = False if v_shp == (): log.debug('Broadcast!!') is_broadcast = True value_slice = () elif v_shp != s_shp: if v_shp == tuple([ i for i in s_shp if i != 1 ]): # Missing dimensions are singleton, just reshape to fit values = values.reshape(s_shp) v_shp = values.shape else: raise IndexError( 'Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}' .format(s_shp, v_shp)) else: value_slice = None log.debug('value_shape: %s', v_shp) for b in bricks: # b is (brick_id, (brick_bounds per dim...),) bid, bbnds = b log.debug('Determining slice for brick: %s', b) bexts = tuple([x + 1 for x in zip(*bbnds)[1] ]) # Shift from index to size log.debug('bid=%s, bbnds=%s, bexts=%s', bid, bbnds, bexts) brick_slice, brick_mm = bricking_utils.get_brick_slice_nd( slice_, bbnds) if None in brick_slice: # Brick does not contain any of the requested indices log.debug( 'Brick does not contain any of the requested indices: Move to next brick' ) continue try: brick_slice = utils.fix_slice(brick_slice, bexts) except IndexError: log.debug('Malformed brick_slice: move to next brick') continue if not is_broadcast: value_slice = bricking_utils.get_value_slice_nd( slice_, v_shp, bbnds, brick_slice, brick_mm) try: value_slice = utils.fix_slice(value_slice, v_shp) except IndexError: log.debug('Malformed value_slice: move to next brick') continue log.debug( '\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s', b, utils.slice_shape(brick_slice, bexts), brick_slice, brick_mm, utils.slice_shape(value_slice, v_shp), value_slice) v = values[value_slice] log.debug('\nvalues %s=\n%s', v.shape, v) if not self.use_hdf: self.bricks[bid][brick_slice] = v else: fi = self.bricks[bid] with h5py.File(fi) as f: ds = f.require_dataset(str(bid), shape=self.brick_sizes, dtype=self.dtype, chunks=None, fillvalue=-1) ds[brick_slice] = v
def __setitem__(self, slice_, value): slice_ = utils.fix_slice(slice_, self.shape) if self.parameter_type.is_valid_value(value): self._storage[slice_] = np.asanyarray(value, dtype='bool') self._update_min_max(np.asanyarray(value, dtype='int8'))
def test_fix_slice_with_slice(self): shp = (50,) # Verify slice is tupled sl = utils.fix_slice(slice(None, None, None), shp) self.assertEqual(sl, (slice(None, None, None),)) shp = (50,20) # Verify list of slices are tupled sl = utils.fix_slice([slice(None, None, None), slice(None, None, None)], shp) self.assertEqual(sl, (slice(None, None, None), slice(None, None, None),)) shp = (50,) #### Passes #### # Slice with all None sl = utils.fix_slice(slice(None, None, None), shp) self.assertEqual(sl, (slice(None, None, None),)) # Slice with start=None & stop=positive sl = utils.fix_slice(slice(None, 30, None), shp) self.assertEqual(sl, (slice(None, 30, None),)) # Slice with start=positive & stop=None sl = utils.fix_slice(slice(10, None, None), shp) self.assertEqual(sl, (slice(10, None, None),)) # Slice with start=negative & stop=None sl = utils.fix_slice(slice(-10, None, None), shp) self.assertEqual(sl, (slice(40, None, None),)) # Slice with start=negative & stop=negative sl = utils.fix_slice(slice(-10, -5, None), shp) self.assertEqual(sl, (slice(40, 45, None),)) # Slice with start=None & stop=negative sl = utils.fix_slice(slice(None, -18, None), shp) self.assertEqual(sl, (slice(None, 32, None),)) #### Failures #### # Slice with start > stop self.assertRaises(IndexError, utils.fix_slice, slice(30, 2, None), shp) # Slice with start == stop self.assertRaises(IndexError, utils.fix_slice, slice(18, 18, None), shp) # # Slice with start=None & stop > size # self.assertRaises(IndexError, utils.fix_slice, slice(None, 52, None), shp) # Slice with start > size & stop=None self.assertRaises(IndexError, utils.fix_slice, slice(53, None, None), shp) # Slice with start < 0 & stop=None (after adjustment, -52 => -2) self.assertRaises(IndexError, utils.fix_slice, slice(-52, None, None), shp) # Slice with start=None & stop=0 self.assertRaises(IndexError, utils.fix_slice, slice(None, 0, None), shp) # Slice with start=None & stop < 0 (after adjustment, -52 => -2) self.assertRaises(IndexError, utils.fix_slice, slice(None, -52, None), shp) # Slice with start=negative & stop=negative with start > stop self.assertRaises(IndexError, utils.fix_slice, slice(-2, -10, None), shp) #### Stepping #### # Slice with step != None sl = utils.fix_slice(slice(4, 43, 6), shp) self.assertEqual(sl, (slice(4, 43, 6),)) # Slice with negative step - reverses start/stop sl = utils.fix_slice(slice(10, 2, -2), shp) self.assertEqual(sl, (slice(2, 10, 2),)) # Slice with all negatives (start, stop, step) sl = utils.fix_slice(slice(-3, -22, -5), shp) self.assertEqual(sl, (slice(28, 47, 5),))
def __setitem__(self, slice_, value): slice_ = fix_slice(slice_, self.shape) self._storage[slice_] = value
def __getitem__(self, slice_): """ Called to implement evaluation of self[slice_]. Not implemented by the abstract class @param slice_ A set of valid constraints - int, [int,], (int,), or slice @return The value contained by the storage at location slice @raise ValueError when brick contains no values for specified slice """ from coverage_model import bricking_utils, utils extents = tuple([s for s in self.total_domain.total_extents if s != 0]) if extents == (): # Empty domain(s) - no data, return empty array return np.empty(0, dtype=self.dtype) # bricks is a list of tuples [(b_ord, b_guid), ...] slice_ = utils.fix_slice(deepcopy(slice_), extents) log.trace('slice_=%s', slice_) bricks = bricking_utils.get_bricks_from_slice(slice_, self.brick_tree, extents) log.trace('Found bricks: %s', bricks) ret_shp = utils.slice_shape(slice_, extents) log.trace('Return array shape: %s', ret_shp) ret_arr = np.empty(ret_shp, dtype=self.dtype) ret_arr.fill(self.fill_value) for b in bricks: # b is (brick_ordinal, brick_guid) _, bid = b # brick_list[brick_guid] contains: [brick_extents, origin, tuple(bD), brick_active_size] _, bori, _, bact = self.brick_list[bid] bbnds = [] for i, bnd in enumerate(bori): bbnds.append((bori[i], bori[i] + bact[i] - 1)) bbnds = tuple(bbnds) brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds) log.trace('brick_slice=%s\tbrick_mm=%s', brick_slice, brick_mm) if None in brick_slice: log.debug('Brick does not contain any of the requested indices: Move to next brick') continue ret_slice = bricking_utils.get_value_slice_nd(slice_, ret_shp, bbnds, brick_slice, brick_mm) brick_file_path = '{0}/{1}.hdf5'.format(self.brick_path, bid) if not os.path.exists(brick_file_path): log.trace('Found virtual brick file: %s', brick_file_path) else: log.trace('Found real brick file: %s', brick_file_path) with h5py.File(brick_file_path) as brick_file: ret_vals = brick_file[bid][brick_slice] # Check if object type if self.dtype == '|O8': if hasattr(ret_vals, '__iter__'): ret_vals = [self._object_unpack_hook(x) for x in ret_vals] else: ret_vals = self._object_unpack_hook(ret_vals) ret_arr[ret_slice] = ret_vals # ret_arr = np.atleast_1d(ret_arr.squeeze()) # ret_arr = np.atleast_1d(ret_arr) # # # If the array is size 1 AND a slice object was NOT part of the query # if ret_arr.size == 1 and not np.atleast_1d([isinstance(s, slice) for s in slice_]).all(): # ret_arr = ret_arr[0] return ret_arr
def __setitem__(self, slice_, value): """ Called to implement assignment of self[slice_, value]. Not implemented by the abstract class @param slice A set of valid constraints - int, [int,], (int,), or slice @param value The value to assign to the storage at location slice_ @raise ValueError when brick contains no values for specified slice """ if self.mode == 'r': raise IOError('PersistenceLayer not open for writing: mode == \'{0}\''.format(self.mode)) from coverage_model import bricking_utils, utils extents = tuple([s for s in self.total_domain.total_extents if s != 0]) # bricks is a list of tuples [(b_ord, b_guid), ...] slice_ = utils.fix_slice(deepcopy(slice_), extents) log.trace('slice_=%s', slice_) bricks = bricking_utils.get_bricks_from_slice(slice_, self.brick_tree, extents) log.trace('Found bricks: %s', bricks) values = np.asanyarray(value) v_shp = values.shape log.trace('value_shape: %s', v_shp) s_shp = utils.slice_shape(slice_, extents) log.trace('slice_shape: %s', s_shp) is_broadcast = False if v_shp == (): log.trace('Broadcast!!') is_broadcast = True value_slice = () elif v_shp != s_shp: if v_shp == tuple([i for i in s_shp if i != 1]): # Missing dimensions are singleton, just reshape to fit values = values.reshape(s_shp) v_shp = values.shape else: raise IndexError( 'Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}'.format( s_shp, v_shp)) else: value_slice = None log.trace('value_shape: %s', v_shp) for b in bricks: # b is (brick_ordinal, brick_guid) _, bid = b # brick_list[brick_guid] contains: [brick_extents, origin, tuple(bD), brick_active_size] _, bori, _, bact = self.brick_list[bid] bbnds = [] bexts = [] for i, bnd in enumerate(bori): bbnds.append((bori[i], bori[i] + bact[i] - 1)) bexts.append(bori[i] + bact[i]) bbnds = tuple(bbnds) bexts = tuple(bexts) log.trace('bid=%s, bbnds=%s, bexts=%s', bid, bbnds, bexts) log.trace('Determining slice for brick: %s', b) brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds) log.trace('brick_slice=%s\tbrick_mm=%s', brick_slice, brick_mm) if None in brick_slice: # Brick does not contain any of the requested indices log.debug('Brick does not contain any of the requested indices: Move to next brick') continue try: brick_slice = utils.fix_slice(brick_slice, bexts) except IndexError: log.debug('Malformed brick_slice: move to next brick') continue if not is_broadcast: value_slice = bricking_utils.get_value_slice_nd(slice_, v_shp, bbnds, brick_slice, brick_mm) try: value_slice = utils.fix_slice(value_slice, v_shp) except IndexError: log.debug('Malformed value_slice: move to next brick') continue log.trace('\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s', b, utils.slice_shape(brick_slice, bexts), brick_slice, brick_mm, utils.slice_shape(value_slice, v_shp), value_slice) v = values[value_slice] self._set_values_to_brick(bid, brick_slice, v)
def __setitem__(self, slice_, value): slice_ = utils.fix_slice(slice_, self.shape) self._storage[slice_] = value self._update_min_max(value)
def test_fix_slice_with_slice(self): shp = (50,) # Verify slice is tupled sl = utils.fix_slice(slice(None, None, None), shp) self.assertEqual(sl, (slice(None, None, None),)) shp = (50,20) # Verify list of slices are tupled sl = utils.fix_slice([slice(None, None, None), slice(None, None, None)], shp) self.assertEqual(sl, (slice(None, None, None), slice(None, None, None),)) shp = (50,) #### Passes #### # Slice with all None sl = utils.fix_slice(slice(None, None, None), shp) self.assertEqual(sl, (slice(None, None, None),)) # Slice with start=None & stop=positive sl = utils.fix_slice(slice(None, 30, None), shp) self.assertEqual(sl, (slice(None, 30, None),)) # Slice with start=positive & stop=None sl = utils.fix_slice(slice(10, None, None), shp) self.assertEqual(sl, (slice(10, None, None),)) # Slice with start=negative & stop=None sl = utils.fix_slice(slice(-10, None, None), shp) self.assertEqual(sl, (slice(40, None, None),)) # Slice with start=negative & stop=negative sl = utils.fix_slice(slice(-10, -5, None), shp) self.assertEqual(sl, (slice(40, 45, None),)) # Slice with start=None & stop=negative sl = utils.fix_slice(slice(None, -18, None), shp) self.assertEqual(sl, (slice(None, 32, None),)) #### Failures #### # Slice with start > stop self.assertRaises(IndexError, utils.fix_slice, slice(30, 2, None), shp) # Slice with start == stop self.assertRaises(IndexError, utils.fix_slice, slice(18, 18, None), shp) # # Slice with start=None & stop > size # self.assertRaises(IndexError, utils.fix_slice, slice(None, 52, None), shp) # Slice with start > size & stop=None self.assertRaises(IndexError, utils.fix_slice, slice(53, None, None), shp) # Slice with start < 0 & stop=None (after adjustment, -52 => -2) self.assertRaises(IndexError, utils.fix_slice, slice(-52, None, None), shp) # Slice with start=None & stop=0 self.assertRaises(IndexError, utils.fix_slice, slice(None, 0, None), shp) # Slice with start=None & stop < 0 (after adjustment, -52 => -2) self.assertRaises(IndexError, utils.fix_slice, slice(None, -52, None), shp) # Slice with start=negative & stop=negative with start > stop self.assertRaises(IndexError, utils.fix_slice, slice(-2, -10, None), shp) #### Stepping #### # Slice with step != None sl = utils.fix_slice(slice(4, 43, 6), shp) self.assertEqual(sl, (slice(4, 43, 6),)) # Slice with negative step - reverses start/stop sl = utils.fix_slice(slice(10, 2, -2), shp) self.assertEqual(sl, (slice(2, 10, 2),)) # Slice with all negatives (start, stop, step) sl = utils.fix_slice(slice(-3, -22, -5), shp) self.assertEqual(sl, (slice(28, 47, 5),))
def __getitem__(self, slice_): slice_ = utils.fix_slice(slice_, self.shape) return _cleanse_value(self._storage[slice_], slice_)