def get_values_from_bricks(self, slice_): slice_ = utils.fix_slice(slice_, self.total_domain) bricks = bricking_utils.get_bricks_from_slice( slice_, self.rtree, self.total_domain ) # this is a list of tuples [(b_id, (bounds...),), ...] ret_shp = utils.slice_shape(slice_, self.total_domain) ret_arr = np.empty(ret_shp, dtype=self.dtype) for b in bricks: bid, bbnds = b brick_slice, brick_mm = bricking_utils.get_brick_slice_nd( slice_, bbnds) if None in brick_slice: continue ret_slice = bricking_utils.get_value_slice_nd( slice_, ret_shp, bbnds, brick_slice, brick_mm) if not self.use_hdf: ret_vals = self.bricks[bid][brick_slice] else: fi = self.bricks[bid] with h5py.File(fi) as f: ds = f.require_dataset(str(bid), shape=self.brick_sizes, dtype=self.dtype, chunks=None, fillvalue=-1) ret_vals = ds[brick_slice] ret_arr[ret_slice] = ret_vals ret_arr = ret_arr.squeeze() if ret_arr.size == 1: if ret_arr.ndim == 0: ret_arr = ret_arr[()] else: ret_arr = ret_arr[0] return ret_arr
def get_values_from_bricks(self, slice_): slice_ = utils.fix_slice(slice_, self.total_domain) bricks = bricking_utils.get_bricks_from_slice( slice_, self.rtree, self.total_domain ) # this is a list of tuples [(b_id, (bounds...),), ...] ret_shp = utils.slice_shape(slice_, self.total_domain) ret_arr = np.empty(ret_shp, dtype=self.dtype) for b in bricks: bid, bbnds = b brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds) if None in brick_slice: continue ret_slice = bricking_utils.get_value_slice_nd(slice_, ret_shp, bbnds, brick_slice, brick_mm) if not self.use_hdf: ret_vals = self.bricks[bid][brick_slice] else: fi = self.bricks[bid] with HDFLockingFile(fi) as f: ds = f.require_dataset( str(bid), shape=self.brick_sizes, dtype=self.dtype, chunks=None, fillvalue=-1 ) ret_vals = ds[brick_slice] ret_arr[ret_slice] = ret_vals ret_arr = ret_arr.squeeze() if ret_arr.size == 1: if ret_arr.ndim == 0: ret_arr = ret_arr[()] else: ret_arr = ret_arr[0] return ret_arr
def __setitem__(self, slice_, value): """ Called to implement assignment of self[slice_, value]. Not implemented by the abstract class @param slice A set of valid constraints - int, [int,], (int,), or slice @param value The value to assign to the storage at location slice_ @raise ValueError when brick contains no values for specified slice """ if self.mode == 'r': raise IOError('PersistenceLayer not open for writing: mode == \'{0}\''.format(self.mode)) from coverage_model import bricking_utils, utils extents = tuple([s for s in self.total_domain.total_extents if s != 0]) # bricks is a list of tuples [(b_ord, b_guid), ...] slice_ = utils.fix_slice(deepcopy(slice_), extents) log.trace('slice_=%s', slice_) bricks = bricking_utils.get_bricks_from_slice(slice_, self.brick_tree, extents) log.trace('Found bricks: %s', bricks) values = np.asanyarray(value) v_shp = values.shape log.trace('value_shape: %s', v_shp) s_shp = utils.slice_shape(slice_, extents) log.trace('slice_shape: %s', s_shp) is_broadcast = False if v_shp == (): log.trace('Broadcast!!') is_broadcast = True value_slice = () elif v_shp != s_shp: if v_shp == tuple([i for i in s_shp if i != 1]): # Missing dimensions are singleton, just reshape to fit values = values.reshape(s_shp) v_shp = values.shape else: raise IndexError( 'Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}'.format( s_shp, v_shp)) else: value_slice = None log.trace('value_shape: %s', v_shp) for b in bricks: # b is (brick_ordinal, brick_guid) _, bid = b # brick_list[brick_guid] contains: [brick_extents, origin, tuple(bD), brick_active_size] _, bori, _, bact = self.brick_list[bid] bbnds = [] bexts = [] for i, bnd in enumerate(bori): bbnds.append((bori[i], bori[i] + bact[i] - 1)) bexts.append(bori[i] + bact[i]) bbnds = tuple(bbnds) bexts = tuple(bexts) log.trace('bid=%s, bbnds=%s, bexts=%s', bid, bbnds, bexts) log.trace('Determining slice for brick: %s', b) brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds) log.trace('brick_slice=%s\tbrick_mm=%s', brick_slice, brick_mm) if None in brick_slice: # Brick does not contain any of the requested indices log.debug('Brick does not contain any of the requested indices: Move to next brick') continue try: brick_slice = utils.fix_slice(brick_slice, bexts) except IndexError: log.debug('Malformed brick_slice: move to next brick') continue if not is_broadcast: value_slice = bricking_utils.get_value_slice_nd(slice_, v_shp, bbnds, brick_slice, brick_mm) try: value_slice = utils.fix_slice(value_slice, v_shp) except IndexError: log.debug('Malformed value_slice: move to next brick') continue log.trace('\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s', b, utils.slice_shape(brick_slice, bexts), brick_slice, brick_mm, utils.slice_shape(value_slice, v_shp), value_slice) v = values[value_slice] self._set_values_to_brick(bid, brick_slice, v) import datetime if self.parameter_manager.parameter_name in self.master_manager.param_groups and v.dtype.type is not np.string_: valid_types = [np.bool_, np.int_, np.intc, np.intp, np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64, np.float_, np.float16, np.float32, np.float64, np.complex_, np.complex64, np.complex128] invalid_fill_values = [None, np.NaN, self.fill_value] try: min_val = None max_val = None # First try to do it fast try: if issubclass(v.dtype.type, numbers.Number) or v.dtype.type in valid_types: tried_it = True min_val = v.min() max_val = v.max() except: min_val = None max_val = None # if fast didn't return valid values, do it slow, but right if min_val in invalid_fill_values or max_val in invalid_fill_values: ts = datetime.datetime.now() mx = [x for x in v if x not in invalid_fill_values and (type(x) in valid_types or issubclass(type(x), numbers.Number))] if len(mx) > 0: min_val = min(mx) max_val = max(mx) time_loss = datetime.datetime.now() - ts log.debug("Repaired numpy statistics inconsistency for parameter/type %s/%s. Time loss of %s seconds ", self.parameter_manager.parameter_name, str(v.dtype.type), str(time_loss)) if min_val is not None and max_val is not None: log.trace("%s min/max %s/%s type %s", self.parameter_manager.parameter_name, min_val, max_val, type(min_val)) self.master_manager.track_data_written_to_brick(bid, brick_slice, self.parameter_manager.parameter_name, min_val, max_val) except Exception as e: log.warn("Could not store Span extents for %s. Unexpected error %s", str( (bid, brick_slice, self.parameter_manager.parameter_name)), e.message ) raise
def __getitem__(self, slice_): """ Called to implement evaluation of self[slice_]. Not implemented by the abstract class @param slice_ A set of valid constraints - int, [int,], (int,), or slice @return The value contained by the storage at location slice @raise ValueError when brick contains no values for specified slice """ from coverage_model import bricking_utils, utils extents = tuple([s for s in self.total_domain.total_extents if s != 0]) if extents == (): # Empty domain(s) - no data, return empty array return np.empty(0, dtype=self.dtype) # bricks is a list of tuples [(b_ord, b_guid), ...] slice_ = utils.fix_slice(deepcopy(slice_), extents) log.trace('slice_=%s', slice_) bricks = bricking_utils.get_bricks_from_slice(slice_, self.brick_tree, extents) log.trace('Found bricks: %s', bricks) ret_shp = utils.slice_shape(slice_, extents) log.trace('Return array shape: %s', ret_shp) ret_arr = np.empty(ret_shp, dtype=self.dtype) ret_arr.fill(self.fill_value) for b in bricks: # b is (brick_ordinal, brick_guid) _, bid = b # brick_list[brick_guid] contains: [brick_extents, origin, tuple(bD), brick_active_size] _, bori, _, bact = self.brick_list[bid] bbnds = [] for i, bnd in enumerate(bori): bbnds.append((bori[i], bori[i] + bact[i] - 1)) bbnds = tuple(bbnds) brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds) log.trace('brick_slice=%s\tbrick_mm=%s', brick_slice, brick_mm) if None in brick_slice: log.debug('Brick does not contain any of the requested indices: Move to next brick') continue ret_slice = bricking_utils.get_value_slice_nd(slice_, ret_shp, bbnds, brick_slice, brick_mm) brick_file_path = '{0}/{1}.hdf5'.format(self.brick_path, bid) if not os.path.exists(brick_file_path): log.trace('Found virtual brick file: %s', brick_file_path) else: log.trace('Found real brick file: %s', brick_file_path) with HDFLockingFile(brick_file_path) as brick_file: ret_vals = brick_file[bid][brick_slice] # Check if object type if self.dtype in ('|O8', '|O4'): if hasattr(ret_vals, '__iter__'): ret_vals = [self._object_unpack_hook(x) for x in ret_vals] else: ret_vals = self._object_unpack_hook(ret_vals) if self.parameter_manager.parameter_name == 'lat': log.trace("values from brick %s %s", str(ret_vals), type(ret_vals)) ret_arr[ret_slice] = ret_vals # ret_arr = np.atleast_1d(ret_arr.squeeze()) # ret_arr = np.atleast_1d(ret_arr) # # # If the array is size 1 AND a slice object was NOT part of the query # if ret_arr.size == 1 and not np.atleast_1d([isinstance(s, slice) for s in slice_]).all(): # ret_arr = ret_arr[0] return ret_arr
def __setitem__(self, slice_, value): """ Called to implement assignment of self[slice_, value]. Not implemented by the abstract class @param slice A set of valid constraints - int, [int,], (int,), or slice @param value The value to assign to the storage at location slice_ @raise ValueError when brick contains no values for specified slice """ if self.mode == 'r': raise IOError('PersistenceLayer not open for writing: mode == \'{0}\''.format(self.mode)) from coverage_model import bricking_utils, utils extents = tuple([s for s in self.total_domain.total_extents if s != 0]) # bricks is a list of tuples [(b_ord, b_guid), ...] slice_ = utils.fix_slice(deepcopy(slice_), extents) log.trace('slice_=%s', slice_) bricks = bricking_utils.get_bricks_from_slice(slice_, self.brick_tree, extents) log.trace('Found bricks: %s', bricks) values = np.asanyarray(value) v_shp = values.shape log.trace('value_shape: %s', v_shp) s_shp = utils.slice_shape(slice_, extents) log.trace('slice_shape: %s', s_shp) is_broadcast = False if v_shp == (): log.trace('Broadcast!!') is_broadcast = True value_slice = () elif v_shp != s_shp: if v_shp == tuple([i for i in s_shp if i != 1]): # Missing dimensions are singleton, just reshape to fit values = values.reshape(s_shp) v_shp = values.shape else: raise IndexError( 'Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}'.format( s_shp, v_shp)) else: value_slice = None log.trace('value_shape: %s', v_shp) for b in bricks: # b is (brick_ordinal, brick_guid) _, bid = b # brick_list[brick_guid] contains: [brick_extents, origin, tuple(bD), brick_active_size] _, bori, _, bact = self.brick_list[bid] bbnds = [] bexts = [] for i, bnd in enumerate(bori): bbnds.append((bori[i], bori[i] + bact[i] - 1)) bexts.append(bori[i] + bact[i]) bbnds = tuple(bbnds) bexts = tuple(bexts) log.trace('bid=%s, bbnds=%s, bexts=%s', bid, bbnds, bexts) log.trace('Determining slice for brick: %s', b) brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds) log.trace('brick_slice=%s\tbrick_mm=%s', brick_slice, brick_mm) if None in brick_slice: # Brick does not contain any of the requested indices log.debug('Brick does not contain any of the requested indices: Move to next brick') continue try: brick_slice = utils.fix_slice(brick_slice, bexts) except IndexError: log.debug('Malformed brick_slice: move to next brick') continue if not is_broadcast: value_slice = bricking_utils.get_value_slice_nd(slice_, v_shp, bbnds, brick_slice, brick_mm) try: value_slice = utils.fix_slice(value_slice, v_shp) except IndexError: log.debug('Malformed value_slice: move to next brick') continue log.trace('\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s', b, utils.slice_shape(brick_slice, bexts), brick_slice, brick_mm, utils.slice_shape(value_slice, v_shp), value_slice) v = values[value_slice] self._set_values_to_brick(bid, brick_slice, v)
def __getitem__(self, slice_): """ Called to implement evaluation of self[slice_]. Not implemented by the abstract class @param slice_ A set of valid constraints - int, [int,], (int,), or slice @return The value contained by the storage at location slice @raise ValueError when brick contains no values for specified slice """ from coverage_model import bricking_utils, utils extents = tuple([s for s in self.total_domain.total_extents if s != 0]) if extents == (): # Empty domain(s) - no data, return empty array return np.empty(0, dtype=self.dtype) # bricks is a list of tuples [(b_ord, b_guid), ...] slice_ = utils.fix_slice(deepcopy(slice_), extents) log.trace('slice_=%s', slice_) bricks = bricking_utils.get_bricks_from_slice(slice_, self.brick_tree, extents) log.trace('Found bricks: %s', bricks) ret_shp = utils.slice_shape(slice_, extents) log.trace('Return array shape: %s', ret_shp) ret_arr = np.empty(ret_shp, dtype=self.dtype) ret_arr.fill(self.fill_value) for b in bricks: # b is (brick_ordinal, brick_guid) _, bid = b # brick_list[brick_guid] contains: [brick_extents, origin, tuple(bD), brick_active_size] _, bori, _, bact = self.brick_list[bid] bbnds = [] for i, bnd in enumerate(bori): bbnds.append((bori[i], bori[i] + bact[i] - 1)) bbnds = tuple(bbnds) brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds) log.trace('brick_slice=%s\tbrick_mm=%s', brick_slice, brick_mm) if None in brick_slice: log.debug('Brick does not contain any of the requested indices: Move to next brick') continue ret_slice = bricking_utils.get_value_slice_nd(slice_, ret_shp, bbnds, brick_slice, brick_mm) brick_file_path = '{0}/{1}.hdf5'.format(self.brick_path, bid) if not os.path.exists(brick_file_path): log.trace('Found virtual brick file: %s', brick_file_path) else: log.trace('Found real brick file: %s', brick_file_path) with h5py.File(brick_file_path) as brick_file: ret_vals = brick_file[bid][brick_slice] # Check if object type if self.dtype == '|O8': if hasattr(ret_vals, '__iter__'): ret_vals = [self._object_unpack_hook(x) for x in ret_vals] else: ret_vals = self._object_unpack_hook(ret_vals) ret_arr[ret_slice] = ret_vals # ret_arr = np.atleast_1d(ret_arr.squeeze()) # ret_arr = np.atleast_1d(ret_arr) # # # If the array is size 1 AND a slice object was NOT part of the query # if ret_arr.size == 1 and not np.atleast_1d([isinstance(s, slice) for s in slice_]).all(): # ret_arr = ret_arr[0] return ret_arr
def put_values_to_bricks(self, slice_, values): slice_ = utils.fix_slice(slice_, self.total_domain) bricks = bricking_utils.get_bricks_from_slice( slice_, self.rtree, self.total_domain ) # this is a list of tuples [(b_id, (bounds...),), ...] values = np.asanyarray(values) v_shp = values.shape log.debug('value_shape: %s', v_shp) s_shp = utils.slice_shape(slice_, self.total_domain) log.debug('slice_shape: %s', s_shp) is_broadcast = False if v_shp == (): log.debug('Broadcast!!') is_broadcast = True value_slice = () elif v_shp != s_shp: if v_shp == tuple([ i for i in s_shp if i != 1 ]): # Missing dimensions are singleton, just reshape to fit values = values.reshape(s_shp) v_shp = values.shape else: raise IndexError( 'Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}' .format(s_shp, v_shp)) else: value_slice = None log.debug('value_shape: %s', v_shp) for b in bricks: # b is (brick_id, (brick_bounds per dim...),) bid, bbnds = b log.debug('Determining slice for brick: %s', b) bexts = tuple([x + 1 for x in zip(*bbnds)[1] ]) # Shift from index to size log.debug('bid=%s, bbnds=%s, bexts=%s', bid, bbnds, bexts) brick_slice, brick_mm = bricking_utils.get_brick_slice_nd( slice_, bbnds) if None in brick_slice: # Brick does not contain any of the requested indices log.debug( 'Brick does not contain any of the requested indices: Move to next brick' ) continue try: brick_slice = utils.fix_slice(brick_slice, bexts) except IndexError: log.debug('Malformed brick_slice: move to next brick') continue if not is_broadcast: value_slice = bricking_utils.get_value_slice_nd( slice_, v_shp, bbnds, brick_slice, brick_mm) try: value_slice = utils.fix_slice(value_slice, v_shp) except IndexError: log.debug('Malformed value_slice: move to next brick') continue log.debug( '\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s', b, utils.slice_shape(brick_slice, bexts), brick_slice, brick_mm, utils.slice_shape(value_slice, v_shp), value_slice) v = values[value_slice] log.debug('\nvalues %s=\n%s', v.shape, v) if not self.use_hdf: self.bricks[bid][brick_slice] = v else: fi = self.bricks[bid] with h5py.File(fi) as f: ds = f.require_dataset(str(bid), shape=self.brick_sizes, dtype=self.dtype, chunks=None, fillvalue=-1) ds[brick_slice] = v
def put_values_to_bricks(self, slice_, values): slice_ = utils.fix_slice(slice_, self.total_domain) bricks = bricking_utils.get_bricks_from_slice(slice_, self.rtree, self.total_domain) # this is a list of tuples [(b_id, (bounds...),), ...] values = np.asanyarray(values) v_shp = values.shape log.debug('value_shape: %s', v_shp) s_shp = utils.slice_shape(slice_, self.total_domain) log.debug('slice_shape: %s', s_shp) is_broadcast = False if v_shp == (): log.debug('Broadcast!!') is_broadcast = True value_slice = () elif v_shp != s_shp: if v_shp == tuple([i for i in s_shp if i != 1]): # Missing dimensions are singleton, just reshape to fit values = values.reshape(s_shp) v_shp = values.shape else: raise IndexError( 'Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}'.format( s_shp, v_shp)) else: value_slice = None log.debug('value_shape: %s', v_shp) for b in bricks: # b is (brick_id, (brick_bounds per dim...),) bid, bbnds = b log.debug('Determining slice for brick: %s', b) bexts = tuple([x + 1 for x in zip(*bbnds)[1]]) # Shift from index to size log.debug('bid=%s, bbnds=%s, bexts=%s', bid, bbnds, bexts) brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds) if None in brick_slice: # Brick does not contain any of the requested indices log.debug('Brick does not contain any of the requested indices: Move to next brick') continue try: brick_slice = utils.fix_slice(brick_slice, bexts) except IndexError: log.debug('Malformed brick_slice: move to next brick') continue if not is_broadcast: value_slice = bricking_utils.get_value_slice_nd(slice_, v_shp, bbnds, brick_slice, brick_mm) try: value_slice = utils.fix_slice(value_slice, v_shp) except IndexError: log.debug('Malformed value_slice: move to next brick') continue log.debug('\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s', b, utils.slice_shape(brick_slice, bexts), brick_slice, brick_mm, utils.slice_shape(value_slice, v_shp), value_slice) v = values[value_slice] log.debug('\nvalues %s=\n%s', v.shape, v) if not self.use_hdf: self.bricks[bid][brick_slice] = v else: fi = self.bricks[bid] with HDFLockingFile(fi, 'a') as f: ds = f.require_dataset(str(bid), shape=self.brick_sizes, dtype=self.dtype, chunks=None, fillvalue=-1) ds[brick_slice] = v