def test_fix_slice_with_list(self):
shp = (50,)
# Verify list is tupled
sl = utils.fix_slice([[7,9]], shp)
self.assertEqual(sl, ([7,9],))
shp = (50,20)
# Verify list of lists are tupled
sl = utils.fix_slice([[5,8],[1,3]], shp)
self.assertEqual(sl, ([5,8],[1,3]))
shp = (50,)
#### Passes ####
# List of indices
sl = utils.fix_slice([[5, 14, 36]], shp)
self.assertEqual(sl, ([5, 14, 36],))
# List with some negative indices
sl = utils.fix_slice([[5, -36, 23]], shp)
self.assertEqual(sl, ([5, 14, 23],))
# List of all negative indices
sl = utils.fix_slice([[-44, -32, -5]], shp)
self.assertEqual(sl, ([6, 18, 45],))
#### Failures ####
# List of indices in non-increasing order
self.assertRaises(IndexError, utils.fix_slice, [[20, 14, 36]], shp)
# List of negative indices in non-increasing order (after adjustment, list is [45, 30, 5]
self.assertRaises(IndexError, utils.fix_slice, [[-5, -20, -45]], shp)
def test_fix_slice_with_int(self):
shp = (50,)
# Verify int is tupled
sl = utils.fix_slice(7, shp)
self.assertEqual(sl, (7,))
shp = (50,20)
# Verify list of ints are tupled
sl = utils.fix_slice([5,8], shp)
self.assertEqual(sl, (5,8,))
shp = (50,)
#### Passes ####
# Index
sl = utils.fix_slice(0, shp)
self.assertEqual(sl, (0,))
# Negative index
sl = utils.fix_slice(-4, shp)
self.assertEqual(sl, (46,))
#### Failures ####
# Index > size
self.assertRaises(IndexError, utils.fix_slice, 52, shp)
# Negative index > size (after adjustment, index is -2)
self.assertRaises(IndexError, utils.fix_slice, -52, shp)
def test_fix_slice_with_list(self):
shp = (50,)
# Verify list is tupled
sl = utils.fix_slice([[7,9]], shp)
self.assertEqual(sl, ([7,9],))
shp = (50,20)
# Verify list of lists are tupled
sl = utils.fix_slice([[5,8],[1,3]], shp)
self.assertEqual(sl, ([5,8],[1,3]))
shp = (50,)
#### Passes ####
# List of indices
sl = utils.fix_slice([[5, 14, 36]], shp)
self.assertEqual(sl, ([5, 14, 36],))
# List with some negative indices
sl = utils.fix_slice([[5, -36, 23]], shp)
self.assertEqual(sl, ([5, 14, 23],))
# List of all negative indices
sl = utils.fix_slice([[-44, -32, -5]], shp)
self.assertEqual(sl, ([6, 18, 45],))
#### Failures ####
# List of indices in non-increasing order
self.assertRaises(IndexError, utils.fix_slice, [[20, 14, 36]], shp)
# List of negative indices in non-increasing order (after adjustment, list is [45, 30, 5]
self.assertRaises(IndexError, utils.fix_slice, [[-5, -20, -45]], shp)
def test_fix_slice_with_int(self):
shp = (50,)
# Verify int is tupled
sl = utils.fix_slice(7, shp)
self.assertEqual(sl, (7,))
shp = (50,20)
# Verify list of ints are tupled
sl = utils.fix_slice([5,8], shp)
self.assertEqual(sl, (5,8,))
shp = (50,)
#### Passes ####
# Index
sl = utils.fix_slice(0, shp)
self.assertEqual(sl, (0,))
# Negative index
sl = utils.fix_slice(-4, shp)
self.assertEqual(sl, (46,))
#### Failures ####
# Index > size
self.assertRaises(IndexError, utils.fix_slice, 52, shp)
# Negative index > size (after adjustment, index is -2)
self.assertRaises(IndexError, utils.fix_slice, -52, shp)
def __getitem__(self, slice_):
slice_ = fix_slice(slice_, self.shape)
total_indices = prod(self.shape)
x = np.arange(total_indices).reshape(self.shape)[slice_] # CBM TODO: This is INDEX based evaluation!!!
return ne.evaluate(self.content).astype(self.value_encoding)
def __indexify_slice(self, slice_, total_shape):
## ONLY WORKS FOR 1D ARRAYS!!!
fsl = utils.fix_slice(slice_, total_shape)
ss = utils.slice_shape(slice_, total_shape)
ret = np.empty(ss, dtype=int)
rf = ret.flatten()
ci = 0
for s, shape in zip(fsl, total_shape):
if isinstance(s,slice):
ind = range(*s.indices(shape))
ll = len(ind)
rf[ci:ll] = ind
ci += ll
elif isinstance(s, (list,tuple)):
ll = len(s)
rf[ci:ll] = s
ci += ll
elif isinstance(s, int):
rf[ci] = s
ci += 1
else:
raise TypeError('Unsupported slice method') # TODO: Better error message
return rf.reshape(ss)
def __setitem__(self, slice_, value):
if is_well_formed_where(value):
slice_ = fix_slice(slice_, self.shape)
if len(slice_) == 1 and isinstance(slice_[0], int) and slice_ < len(self._storage[0]):
self._storage[0][slice_[0]] = value
else:
self._storage[0].append(value)
def __indexify_slice(self, slice_, total_shape):
## ONLY WORKS FOR 1D ARRAYS!!!
fsl = utils.fix_slice(slice_, total_shape)
ss = utils.slice_shape(slice_, total_shape)
ret = np.empty(ss, dtype=int)
rf = ret.flatten()
ci = 0
for s, shape in zip(fsl, total_shape):
if isinstance(s, slice):
ind = range(*s.indices(shape))
ll = len(ind)
rf[ci:ll] = ind
ci += ll
elif isinstance(s, (list, tuple)):
ll = len(s)
rf[ci:ll] = s
ci += ll
elif isinstance(s, int):
rf[ci] = s
ci += 1
else:
raise TypeError(
'Unsupported slice method') # TODO: Better error message
return rf.reshape(ss)
def __getitem__(self, slice_):
if self._memoized_values is not None:
return self._memoized_values
else:
if self._pval_callback is None:
raise ParameterFunctionException(
'\'_pval_callback\' is None; cannot evaluate!!')
slice_ = utils.fix_slice(slice_, self.shape)
try:
r = self.content.evaluate(self._pval_callback, slice_,
self.parameter_type.fill_value)
ve = self.parameter_type.value_encoding
if hasattr(self.parameter_type, 'inner_encoding'):
ve = self.parameter_type.inner_encoding
if ve is not None:
r = np.asanyarray(r, dtype=ve)
except Exception as ex:
import sys
raise ParameterFunctionException(
ex.message, type(ex)), None, sys.exc_traceback
return _cleanse_value(r, slice_)
def __getitem__(self, slice_):
slice_ = utils.fix_slice(slice_, self.shape)
ret_shape = utils.slice_shape(slice_, self.shape)
ret = np.empty(ret_shape, dtype=np.dtype(object)) # Always object type because it's 2 values / element!!
ret.fill(self.content)
return _cleanse_value(ret, slice_)
def __getitem__(self, slice_):
slice_ = utils.fix_slice(slice_, self.shape)
ret_shape = utils.slice_shape(slice_, self.shape)
ret = np.empty(ret_shape, dtype=np.dtype(self.value_encoding))
ret.fill(self.content)
return _cleanse_value(ret, slice_)
def __getitem__(self, slice_):
slice_ = utils.fix_slice(slice_, self.shape)
ret_shape = utils.slice_shape(slice_, self.shape)
ret = np.empty(ret_shape, dtype=np.dtype(self.value_encoding))
ret.fill(self.content)
return _cleanse_value(ret, slice_)
def __setitem__(self, slice_, value):
if is_well_formed_where(value):
slice_ = utils.fix_slice(slice_, self.shape)
if len(slice_) == 1 and isinstance(
slice_[0], int) and slice_ < len(self._storage[0]):
self._storage[0][slice_[0]] = value
else:
self._storage[0].append(value)
def test_fix_slice(self):
shp = (50,)
# Initial 'is_valid_constraint' catches bad stuff
self.assertRaises(SystemError, utils.fix_slice, 'bob', shp)
# Tuples are list-ified
sl = utils.fix_slice((20,), shp)
# Down-ranking (slice with more dims than shp)
shp = (50,)
sl = utils.fix_slice([5,23], shp)
self.assertEqual(sl, (5,))
# Up-ranking (slice with fewer dims than shp)
shp = (50,10,)
sl = utils.fix_slice(5, shp)
self.assertEqual(sl, (5, slice(None, None, None)))
def __getitem__(self, slice_):
slice_ = utils.fix_slice(slice_, self.shape)
total_indices = utils.prod(self.shape)
x = np.arange(total_indices).reshape(
self.shape)[slice_] # CBM TODO: This is INDEX based evaluation!!!
return _cleanse_value(
ne.evaluate(self.content).astype(self.value_encoding), slice_)
def test_fix_slice(self):
shp = (50,)
# Initial 'is_valid_constraint' catches bad stuff
self.assertRaises(SystemError, utils.fix_slice, 'bob', shp)
# Tuples are list-ified
sl = utils.fix_slice((20,), shp)
# Down-ranking (slice with more dims than shp)
shp = (50,)
sl = utils.fix_slice([5,23], shp)
self.assertEqual(sl, (5,))
# Up-ranking (slice with fewer dims than shp)
shp = (50,10,)
sl = utils.fix_slice(5, shp)
self.assertEqual(sl, (5, slice(None, None, None)))
def __getitem__(self, slice_):
slice_ = utils.fix_slice(slice_, self.shape)
ret_shape = utils.slice_shape(slice_, self.shape)
ret = np.empty(ret_shape, dtype=np.dtype(
object)) # Always object type because it's 2 values / element!!
ret.fill(self.content)
return _cleanse_value(ret, slice_)
def __setitem__(self, slice_, value):
slice_ = utils.fix_slice(slice_, self.shape)
try:
spans = self._storage[0]
except ValueError, ve:
if ve.message != 'No Bricks!':
raise
spans = self.fill_value
def __setitem__(self, slice_, value):
slice_ = utils.fix_slice(slice_, self.shape)
try:
spans = self._storage[0]
except ValueError, ve:
if ve.message != 'No Bricks!':
raise
spans = self.fill_value
def __getitem__(self, slice_):
slice_ = utils.fix_slice(slice_, self.shape)
ret = np.atleast_1d(self._storage[slice_])
cats = self.parameter_type.categories
if np.iterable(ret):
ret = np.array([cats[x] for x in ret], dtype=object)
else:
ret = cats[ret]
return _cleanse_value(ret, slice_)
def __getitem__(self, slice_):
slice_ = utils.fix_slice(slice_, self.shape)
ret = np.atleast_1d(self._storage[slice_])
cats = self.parameter_type.categories
if np.iterable(ret):
ret = np.array([cats[x] for x in ret], dtype=object)
else:
ret = cats[ret]
return _cleanse_value(ret, slice_)
def __getitem__(self, slice_):
slice_ = fix_slice(slice_, self.shape)
ret = self._storage[slice_]
cats=self.parameter_type.categories
if np.iterable(ret):
ret = np.array([cats[x] for x in ret], dtype=object)
else:
ret = cats[ret.item()]
return ret
def __getitem__(self, slice_):
slice_ = fix_slice(slice_, self.shape)
c = np.ones(self.shape)[slice_] # Make an array of ones of the appropriate shape and slice it as desired
ret = ne.evaluate(self.content).astype(self.value_encoding)
if ret.size==1:
if ret.ndim==0:
ret=ret[()]
else:
ret=ret[0]
return ret
def __getitem__(self, slice_):
oslice = slice_ = utils.fix_slice(slice_, self.shape)
if isinstance(slice_[0], int): # Must always pass a slice so we don't lose the dimension
slice_ = (slice(slice_[0], slice_[0] + 1),) + slice_[1:]
vals = np.atleast_1d(self._storage[slice_])
vals = self._apply_inner_encoding(vals, self.parameter_type)
ret = _cleanse_value(vals, oslice)
return ret
def __getitem__(self, slice_):
slice_ = utils.fix_slice(slice_, self.shape)
# Nothing asked for!
if len(slice_) is 0:
return np.empty(0, dtype=self.value_encoding)
try:
spans = self._storage[0]
except ValueError, ve:
if ve.message != 'No Bricks!':
raise
return np.empty(0, dtype=self.value_encoding)
def __getitem__(self, slice_):
slice_ = utils.fix_slice(slice_, self.shape)
# Nothing asked for!
if len(slice_) is 0:
return np.empty(0, dtype=self.value_encoding)
try:
spans = self._storage[0]
except ValueError, ve:
if ve.message != 'No Bricks!':
raise
return np.empty(0, dtype=self.value_encoding)
def __getitem__(self, slice_):
oslice = slice_ = utils.fix_slice(slice_, self.shape)
if isinstance(
slice_[0], int
): # Must always pass a slice so we don't lose the dimension
slice_ = (slice(slice_[0], slice_[0] + 1), ) + slice_[1:]
vals = np.atleast_1d(self._storage[slice_])
vals = self._apply_inner_encoding(vals, self.parameter_type)
ret = _cleanse_value(vals, oslice)
return ret
def get_data_size(self, parameter_name=None, slice_=None, in_bytes=False):
"""
Returns the size of the <b>data values</b> for the parameter(s) indicated by <i>parameter_name</i>.
ParameterContext and Coverage metadata is <b>NOT</b> included in the returned size.
If <i>parameter_name</i> is None, all parameters in the coverage are included
If more than one parameter is indicated by <i>parameter_name</i>, the sum of the indicated parameters is returned
If <i>slice_</i> is not None, it is applied to each parameter (after being run through utils.fix_slice) before
calculation of size
Sizes are calculated as:
size = itemsize * total_extent_size
where:
itemsize == the per-item size based on the data type of the parameter
total_extent_size == the total number of elements after slicing is applied (if applicable)
Sizes are in MB unless <i>in_bytes</i> == True
@param parameter_name A string parameter name; may be an iterable of such members
@param slice_ If not None, applied to each parameter before calculation of size
@param in_bytes If True, returns the size in bytes; otherwise, returns the size in MB (default)
"""
size = 0
if parameter_name is None:
for pn in self._range_dictionary.keys():
size += self.get_data_size(pn, in_bytes=in_bytes)
for pn in self.__parameter_name_arg_to_params(parameter_name):
p = self._range_dictionary.get_context(pn)
te=p.dom.total_extents
dt = np.dtype(p.param_type.value_encoding)
if slice_ is not None:
slice_ = utils.fix_slice(slice_, te)
a=np.empty(te, dtype=dt)[slice_]
size += a.nbytes
else:
size += dt.itemsize * utils.prod(te)
if not in_bytes:
size *= 9.53674e-7
return size
def get_values_from_bricks(self, slice_):
slice_ = utils.fix_slice(slice_, self.total_domain)
bricks = bricking_utils.get_bricks_from_slice(
slice_, self.rtree, self.total_domain
) # this is a list of tuples [(b_id, (bounds...),), ...]
ret_shp = utils.slice_shape(slice_, self.total_domain)
ret_arr = np.empty(ret_shp, dtype=self.dtype)
for b in bricks:
bid, bbnds = b
brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(
slice_, bbnds)
if None in brick_slice:
continue
ret_slice = bricking_utils.get_value_slice_nd(
slice_, ret_shp, bbnds, brick_slice, brick_mm)
if not self.use_hdf:
ret_vals = self.bricks[bid][brick_slice]
else:
fi = self.bricks[bid]
with h5py.File(fi) as f:
ds = f.require_dataset(str(bid),
shape=self.brick_sizes,
dtype=self.dtype,
chunks=None,
fillvalue=-1)
ret_vals = ds[brick_slice]
ret_arr[ret_slice] = ret_vals
ret_arr = ret_arr.squeeze()
if ret_arr.size == 1:
if ret_arr.ndim == 0:
ret_arr = ret_arr[()]
else:
ret_arr = ret_arr[0]
return ret_arr
def __setitem__(self, slice_, value):
slice_ = utils.fix_slice(slice_, self.shape)
self.parameter_type.is_valid_value(value)
value = np.atleast_1d(value)
# Replace any None with fill_value
np.place(value, value == np.array([None]), self.fill_value)
if value.dtype.kind == np.dtype(self.parameter_type.value_encoding).kind:
# Set as ordinals
self._storage[slice_] = value
else:
# Set as categories
rcats={v:k for k,v in self.parameter_type.categories.iteritems()}
try:
vals=[rcats[v] if v != self.fill_value else v for v in value]
self._storage[slice_] = vals
except KeyError, ke:
raise ValueError('Invalid category specified: \'{0}\''.format(ke.message))
def __getitem__(self, slice_):
if self._memoized_values is not None:
return self._memoized_values
else:
if self._pval_callback is None:
raise ParameterFunctionException('\'_pval_callback\' is None; cannot evaluate!!')
slice_ = utils.fix_slice(slice_, self.shape)
try:
r = self.content.evaluate(self._pval_callback, slice_, self.parameter_type.fill_value)
ve = self.parameter_type.value_encoding
if hasattr(self.parameter_type, 'inner_encoding'):
ve = self.parameter_type.inner_encoding
if ve is not None:
r = np.asanyarray(r, dtype=ve)
except Exception as ex:
import sys
raise ParameterFunctionException(ex.message, type(ex)), None, sys.exc_traceback
return _cleanse_value(r, slice_)
def get_values_from_bricks(self, slice_):
slice_ = utils.fix_slice(slice_, self.total_domain)
bricks = bricking_utils.get_bricks_from_slice(
slice_, self.rtree, self.total_domain
) # this is a list of tuples [(b_id, (bounds...),), ...]
ret_shp = utils.slice_shape(slice_, self.total_domain)
ret_arr = np.empty(ret_shp, dtype=self.dtype)
for b in bricks:
bid, bbnds = b
brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds)
if None in brick_slice:
continue
ret_slice = bricking_utils.get_value_slice_nd(slice_, ret_shp, bbnds, brick_slice, brick_mm)
if not self.use_hdf:
ret_vals = self.bricks[bid][brick_slice]
else:
fi = self.bricks[bid]
with HDFLockingFile(fi) as f:
ds = f.require_dataset(
str(bid), shape=self.brick_sizes, dtype=self.dtype, chunks=None, fillvalue=-1
)
ret_vals = ds[brick_slice]
ret_arr[ret_slice] = ret_vals
ret_arr = ret_arr.squeeze()
if ret_arr.size == 1:
if ret_arr.ndim == 0:
ret_arr = ret_arr[()]
else:
ret_arr = ret_arr[0]
return ret_arr
def __setitem__(self, slice_, value):
slice_ = utils.fix_slice(slice_, self.shape)
self.parameter_type.is_valid_value(value)
value = np.atleast_1d(value)
# Replace any None with fill_value
np.place(value, value == np.array([None]), self.fill_value)
if value.dtype.kind == np.dtype(
self.parameter_type.value_encoding).kind:
# Set as ordinals
self._storage[slice_] = value
else:
# Set as categories
rcats = {
v: k
for k, v in self.parameter_type.categories.iteritems()
}
try:
vals = [rcats[v] if v != self.fill_value else v for v in value]
self._storage[slice_] = vals
except KeyError, ke:
raise ValueError('Invalid category specified: \'{0}\''.format(
ke.message))
def __setitem__(self, slice_, value):
slice_ = utils.fix_slice(slice_, self.shape)
if isinstance(value, AbstractParameterValue):
value = np.atleast_1d(value[:])
value = np.atleast_1d(value)
if len(value.shape) > 1:
v = np.empty(value.shape[0], dtype=object)
for i in xrange(value.shape[0]):
iv = value[i,:]
if isinstance(iv, np.ndarray):
v[i] = iv.tolist()
else:
v[i] = iv
value = v
if isinstance(slice_[0], int):
slice_ = (slice(slice_[0], slice_[0] + 1),) + slice_[1:]
self._storage[slice_] = value[:]
self._update_min_max(value)
def __setitem__(self, slice_, value):
slice_ = utils.fix_slice(slice_, self.shape)
if isinstance(value, AbstractParameterValue):
value = np.atleast_1d(value[:])
value = np.atleast_1d(value)
if len(value.shape) > 1:
v = np.empty(value.shape[0], dtype=object)
for i in xrange(value.shape[0]):
iv = value[i, :]
if isinstance(iv, np.ndarray):
v[i] = iv.tolist()
else:
v[i] = iv
value = v
if isinstance(slice_[0], int):
slice_ = (slice(slice_[0], slice_[0] + 1), ) + slice_[1:]
self._storage[slice_] = value[:]
self._update_min_max(value)
def __setitem__(self, slice_, value):
"""
Called to implement assignment of self[slice_, value].
Not implemented by the abstract class
@param slice A set of valid constraints - int, [int,], (int,), or slice
@param value The value to assign to the storage at location slice_
@raise ValueError when brick contains no values for specified slice
"""
if self.mode == 'r':
raise IOError('PersistenceLayer not open for writing: mode == \'{0}\''.format(self.mode))
from coverage_model import bricking_utils, utils
extents = tuple([s for s in self.total_domain.total_extents if s != 0])
# bricks is a list of tuples [(b_ord, b_guid), ...]
slice_ = utils.fix_slice(deepcopy(slice_), extents)
log.trace('slice_=%s', slice_)
bricks = bricking_utils.get_bricks_from_slice(slice_, self.brick_tree, extents)
log.trace('Found bricks: %s', bricks)
values = np.asanyarray(value)
v_shp = values.shape
log.trace('value_shape: %s', v_shp)
s_shp = utils.slice_shape(slice_, extents)
log.trace('slice_shape: %s', s_shp)
is_broadcast = False
if v_shp == ():
log.trace('Broadcast!!')
is_broadcast = True
value_slice = ()
elif v_shp != s_shp:
if v_shp == tuple([i for i in s_shp if i != 1]): # Missing dimensions are singleton, just reshape to fit
values = values.reshape(s_shp)
v_shp = values.shape
else:
raise IndexError(
'Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}'.format(
s_shp, v_shp))
else:
value_slice = None
log.trace('value_shape: %s', v_shp)
for b in bricks:
# b is (brick_ordinal, brick_guid)
_, bid = b
# brick_list[brick_guid] contains: [brick_extents, origin, tuple(bD), brick_active_size]
_, bori, _, bact = self.brick_list[bid]
bbnds = []
bexts = []
for i, bnd in enumerate(bori):
bbnds.append((bori[i], bori[i] + bact[i] - 1))
bexts.append(bori[i] + bact[i])
bbnds = tuple(bbnds)
bexts = tuple(bexts)
log.trace('bid=%s, bbnds=%s, bexts=%s', bid, bbnds, bexts)
log.trace('Determining slice for brick: %s', b)
brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds)
log.trace('brick_slice=%s\tbrick_mm=%s', brick_slice, brick_mm)
if None in brick_slice: # Brick does not contain any of the requested indices
log.debug('Brick does not contain any of the requested indices: Move to next brick')
continue
try:
brick_slice = utils.fix_slice(brick_slice, bexts)
except IndexError:
log.debug('Malformed brick_slice: move to next brick')
continue
if not is_broadcast:
value_slice = bricking_utils.get_value_slice_nd(slice_, v_shp, bbnds, brick_slice, brick_mm)
try:
value_slice = utils.fix_slice(value_slice, v_shp)
except IndexError:
log.debug('Malformed value_slice: move to next brick')
continue
log.trace('\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s', b,
utils.slice_shape(brick_slice, bexts), brick_slice, brick_mm,
utils.slice_shape(value_slice, v_shp), value_slice)
v = values[value_slice]
self._set_values_to_brick(bid, brick_slice, v)
import datetime
if self.parameter_manager.parameter_name in self.master_manager.param_groups and v.dtype.type is not np.string_:
valid_types = [np.bool_, np.int_, np.intc, np.intp, np.int8, np.int16, np.int32, np.int64, np.uint8,
np.uint16, np.uint32, np.uint64, np.float_, np.float16, np.float32, np.float64,
np.complex_, np.complex64, np.complex128]
invalid_fill_values = [None, np.NaN, self.fill_value]
try:
min_val = None
max_val = None
# First try to do it fast
try:
if issubclass(v.dtype.type, numbers.Number) or v.dtype.type in valid_types:
tried_it = True
min_val = v.min()
max_val = v.max()
except:
min_val = None
max_val = None
# if fast didn't return valid values, do it slow, but right
if min_val in invalid_fill_values or max_val in invalid_fill_values:
ts = datetime.datetime.now()
mx = [x for x in v if x not in invalid_fill_values and (type(x) in valid_types or issubclass(type(x), numbers.Number))]
if len(mx) > 0:
min_val = min(mx)
max_val = max(mx)
time_loss = datetime.datetime.now() - ts
log.debug("Repaired numpy statistics inconsistency for parameter/type %s/%s. Time loss of %s seconds ", self.parameter_manager.parameter_name, str(v.dtype.type), str(time_loss))
if min_val is not None and max_val is not None:
log.trace("%s min/max %s/%s type %s", self.parameter_manager.parameter_name, min_val, max_val, type(min_val))
self.master_manager.track_data_written_to_brick(bid, brick_slice, self.parameter_manager.parameter_name, min_val, max_val)
except Exception as e:
log.warn("Could not store Span extents for %s. Unexpected error %s",
str( (bid, brick_slice, self.parameter_manager.parameter_name)), e.message )
raise
def __getitem__(self, slice_):
"""
Called to implement evaluation of self[slice_].
Not implemented by the abstract class
@param slice_ A set of valid constraints - int, [int,], (int,), or slice
@return The value contained by the storage at location slice
@raise ValueError when brick contains no values for specified slice
"""
from coverage_model import bricking_utils, utils
extents = tuple([s for s in self.total_domain.total_extents if s != 0])
if extents == (): # Empty domain(s) - no data, return empty array
return np.empty(0, dtype=self.dtype)
# bricks is a list of tuples [(b_ord, b_guid), ...]
slice_ = utils.fix_slice(deepcopy(slice_), extents)
log.trace('slice_=%s', slice_)
bricks = bricking_utils.get_bricks_from_slice(slice_, self.brick_tree, extents)
log.trace('Found bricks: %s', bricks)
ret_shp = utils.slice_shape(slice_, extents)
log.trace('Return array shape: %s', ret_shp)
ret_arr = np.empty(ret_shp, dtype=self.dtype)
ret_arr.fill(self.fill_value)
for b in bricks:
# b is (brick_ordinal, brick_guid)
_, bid = b
# brick_list[brick_guid] contains: [brick_extents, origin, tuple(bD), brick_active_size]
_, bori, _, bact = self.brick_list[bid]
bbnds = []
for i, bnd in enumerate(bori):
bbnds.append((bori[i], bori[i] + bact[i] - 1))
bbnds = tuple(bbnds)
brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds)
log.trace('brick_slice=%s\tbrick_mm=%s', brick_slice, brick_mm)
if None in brick_slice:
log.debug('Brick does not contain any of the requested indices: Move to next brick')
continue
ret_slice = bricking_utils.get_value_slice_nd(slice_, ret_shp, bbnds, brick_slice, brick_mm)
brick_file_path = '{0}/{1}.hdf5'.format(self.brick_path, bid)
if not os.path.exists(brick_file_path):
log.trace('Found virtual brick file: %s', brick_file_path)
else:
log.trace('Found real brick file: %s', brick_file_path)
with HDFLockingFile(brick_file_path) as brick_file:
ret_vals = brick_file[bid][brick_slice]
# Check if object type
if self.dtype in ('|O8', '|O4'):
if hasattr(ret_vals, '__iter__'):
ret_vals = [self._object_unpack_hook(x) for x in ret_vals]
else:
ret_vals = self._object_unpack_hook(ret_vals)
if self.parameter_manager.parameter_name == 'lat':
log.trace("values from brick %s %s", str(ret_vals), type(ret_vals))
ret_arr[ret_slice] = ret_vals
# ret_arr = np.atleast_1d(ret_arr.squeeze())
# ret_arr = np.atleast_1d(ret_arr)
#
# # If the array is size 1 AND a slice object was NOT part of the query
# if ret_arr.size == 1 and not np.atleast_1d([isinstance(s, slice) for s in slice_]).all():
# ret_arr = ret_arr[0]
return ret_arr
def __setitem__(self, slice_, value):
"""
Called to implement assignment of self[slice_, value].
Not implemented by the abstract class
@param slice A set of valid constraints - int, [int,], (int,), or slice
@param value The value to assign to the storage at location slice_
@raise ValueError when brick contains no values for specified slice
"""
if self.mode == 'r':
raise IOError('PersistenceLayer not open for writing: mode == \'{0}\''.format(self.mode))
from coverage_model import bricking_utils, utils
extents = tuple([s for s in self.total_domain.total_extents if s != 0])
# bricks is a list of tuples [(b_ord, b_guid), ...]
slice_ = utils.fix_slice(deepcopy(slice_), extents)
log.trace('slice_=%s', slice_)
bricks = bricking_utils.get_bricks_from_slice(slice_, self.brick_tree, extents)
log.trace('Found bricks: %s', bricks)
values = np.asanyarray(value)
v_shp = values.shape
log.trace('value_shape: %s', v_shp)
s_shp = utils.slice_shape(slice_, extents)
log.trace('slice_shape: %s', s_shp)
is_broadcast = False
if v_shp == ():
log.trace('Broadcast!!')
is_broadcast = True
value_slice = ()
elif v_shp != s_shp:
if v_shp == tuple([i for i in s_shp if i != 1]): # Missing dimensions are singleton, just reshape to fit
values = values.reshape(s_shp)
v_shp = values.shape
else:
raise IndexError(
'Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}'.format(
s_shp, v_shp))
else:
value_slice = None
log.trace('value_shape: %s', v_shp)
for b in bricks:
# b is (brick_ordinal, brick_guid)
_, bid = b
# brick_list[brick_guid] contains: [brick_extents, origin, tuple(bD), brick_active_size]
_, bori, _, bact = self.brick_list[bid]
bbnds = []
bexts = []
for i, bnd in enumerate(bori):
bbnds.append((bori[i], bori[i] + bact[i] - 1))
bexts.append(bori[i] + bact[i])
bbnds = tuple(bbnds)
bexts = tuple(bexts)
log.trace('bid=%s, bbnds=%s, bexts=%s', bid, bbnds, bexts)
log.trace('Determining slice for brick: %s', b)
brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds)
log.trace('brick_slice=%s\tbrick_mm=%s', brick_slice, brick_mm)
if None in brick_slice: # Brick does not contain any of the requested indices
log.debug('Brick does not contain any of the requested indices: Move to next brick')
continue
try:
brick_slice = utils.fix_slice(brick_slice, bexts)
except IndexError:
log.debug('Malformed brick_slice: move to next brick')
continue
if not is_broadcast:
value_slice = bricking_utils.get_value_slice_nd(slice_, v_shp, bbnds, brick_slice, brick_mm)
try:
value_slice = utils.fix_slice(value_slice, v_shp)
except IndexError:
log.debug('Malformed value_slice: move to next brick')
continue
log.trace('\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s', b,
utils.slice_shape(brick_slice, bexts), brick_slice, brick_mm,
utils.slice_shape(value_slice, v_shp), value_slice)
v = values[value_slice]
self._set_values_to_brick(bid, brick_slice, v)
def put_values_to_bricks(self, slice_, values):
slice_ = utils.fix_slice(slice_, self.total_domain)
bricks = bricking_utils.get_bricks_from_slice(slice_, self.rtree,
self.total_domain) # this is a list of tuples [(b_id, (bounds...),), ...]
values = np.asanyarray(values)
v_shp = values.shape
log.debug('value_shape: %s', v_shp)
s_shp = utils.slice_shape(slice_, self.total_domain)
log.debug('slice_shape: %s', s_shp)
is_broadcast = False
if v_shp == ():
log.debug('Broadcast!!')
is_broadcast = True
value_slice = ()
elif v_shp != s_shp:
if v_shp == tuple([i for i in s_shp if i != 1]): # Missing dimensions are singleton, just reshape to fit
values = values.reshape(s_shp)
v_shp = values.shape
else:
raise IndexError(
'Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}'.format(
s_shp, v_shp))
else:
value_slice = None
log.debug('value_shape: %s', v_shp)
for b in bricks:
# b is (brick_id, (brick_bounds per dim...),)
bid, bbnds = b
log.debug('Determining slice for brick: %s', b)
bexts = tuple([x + 1 for x in zip(*bbnds)[1]]) # Shift from index to size
log.debug('bid=%s, bbnds=%s, bexts=%s', bid, bbnds, bexts)
brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds)
if None in brick_slice: # Brick does not contain any of the requested indices
log.debug('Brick does not contain any of the requested indices: Move to next brick')
continue
try:
brick_slice = utils.fix_slice(brick_slice, bexts)
except IndexError:
log.debug('Malformed brick_slice: move to next brick')
continue
if not is_broadcast:
value_slice = bricking_utils.get_value_slice_nd(slice_, v_shp, bbnds, brick_slice, brick_mm)
try:
value_slice = utils.fix_slice(value_slice, v_shp)
except IndexError:
log.debug('Malformed value_slice: move to next brick')
continue
log.debug('\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s', b,
utils.slice_shape(brick_slice, bexts), brick_slice, brick_mm,
utils.slice_shape(value_slice, v_shp), value_slice)
v = values[value_slice]
log.debug('\nvalues %s=\n%s', v.shape, v)
if not self.use_hdf:
self.bricks[bid][brick_slice] = v
else:
fi = self.bricks[bid]
with HDFLockingFile(fi, 'a') as f:
ds = f.require_dataset(str(bid), shape=self.brick_sizes, dtype=self.dtype, chunks=None,
fillvalue=-1)
ds[brick_slice] = v
def __setitem__(self, slice_, value):
slice_ = utils.fix_slice(slice_, self.shape)
if self.parameter_type.is_valid_value(value):
self._storage[slice_] = np.asanyarray(value, dtype='bool')
self._update_min_max(np.asanyarray(value, dtype='int8'))
def __getitem__(self, slice_):
slice_ = fix_slice(slice_, self.shape)
ret = self._storage[slice_]
return ret
def __setitem__(self, slice_, value):
slice_ = utils.fix_slice(slice_, self.shape)
self._storage[slice_] = value
self._update_min_max(value)
def __getitem__(self, slice_):
slice_ = utils.fix_slice(slice_, self.shape)
return _cleanse_value(self._storage[slice_], slice_)
def put_values_to_bricks(self, slice_, values):
slice_ = utils.fix_slice(slice_, self.total_domain)
bricks = bricking_utils.get_bricks_from_slice(
slice_, self.rtree, self.total_domain
) # this is a list of tuples [(b_id, (bounds...),), ...]
values = np.asanyarray(values)
v_shp = values.shape
log.debug('value_shape: %s', v_shp)
s_shp = utils.slice_shape(slice_, self.total_domain)
log.debug('slice_shape: %s', s_shp)
is_broadcast = False
if v_shp == ():
log.debug('Broadcast!!')
is_broadcast = True
value_slice = ()
elif v_shp != s_shp:
if v_shp == tuple([
i for i in s_shp if i != 1
]): # Missing dimensions are singleton, just reshape to fit
values = values.reshape(s_shp)
v_shp = values.shape
else:
raise IndexError(
'Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}'
.format(s_shp, v_shp))
else:
value_slice = None
log.debug('value_shape: %s', v_shp)
for b in bricks:
# b is (brick_id, (brick_bounds per dim...),)
bid, bbnds = b
log.debug('Determining slice for brick: %s', b)
bexts = tuple([x + 1 for x in zip(*bbnds)[1]
]) # Shift from index to size
log.debug('bid=%s, bbnds=%s, bexts=%s', bid, bbnds, bexts)
brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(
slice_, bbnds)
if None in brick_slice: # Brick does not contain any of the requested indices
log.debug(
'Brick does not contain any of the requested indices: Move to next brick'
)
continue
try:
brick_slice = utils.fix_slice(brick_slice, bexts)
except IndexError:
log.debug('Malformed brick_slice: move to next brick')
continue
if not is_broadcast:
value_slice = bricking_utils.get_value_slice_nd(
slice_, v_shp, bbnds, brick_slice, brick_mm)
try:
value_slice = utils.fix_slice(value_slice, v_shp)
except IndexError:
log.debug('Malformed value_slice: move to next brick')
continue
log.debug(
'\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s',
b, utils.slice_shape(brick_slice,
bexts), brick_slice, brick_mm,
utils.slice_shape(value_slice, v_shp), value_slice)
v = values[value_slice]
log.debug('\nvalues %s=\n%s', v.shape, v)
if not self.use_hdf:
self.bricks[bid][brick_slice] = v
else:
fi = self.bricks[bid]
with h5py.File(fi) as f:
ds = f.require_dataset(str(bid),
shape=self.brick_sizes,
dtype=self.dtype,
chunks=None,
fillvalue=-1)
ds[brick_slice] = v
def __setitem__(self, slice_, value):
slice_ = utils.fix_slice(slice_, self.shape)
if self.parameter_type.is_valid_value(value):
self._storage[slice_] = np.asanyarray(value, dtype='bool')
self._update_min_max(np.asanyarray(value, dtype='int8'))
def test_fix_slice_with_slice(self):
shp = (50,)
# Verify slice is tupled
sl = utils.fix_slice(slice(None, None, None), shp)
self.assertEqual(sl, (slice(None, None, None),))
shp = (50,20)
# Verify list of slices are tupled
sl = utils.fix_slice([slice(None, None, None), slice(None, None, None)], shp)
self.assertEqual(sl, (slice(None, None, None), slice(None, None, None),))
shp = (50,)
#### Passes ####
# Slice with all None
sl = utils.fix_slice(slice(None, None, None), shp)
self.assertEqual(sl, (slice(None, None, None),))
# Slice with start=None & stop=positive
sl = utils.fix_slice(slice(None, 30, None), shp)
self.assertEqual(sl, (slice(None, 30, None),))
# Slice with start=positive & stop=None
sl = utils.fix_slice(slice(10, None, None), shp)
self.assertEqual(sl, (slice(10, None, None),))
# Slice with start=negative & stop=None
sl = utils.fix_slice(slice(-10, None, None), shp)
self.assertEqual(sl, (slice(40, None, None),))
# Slice with start=negative & stop=negative
sl = utils.fix_slice(slice(-10, -5, None), shp)
self.assertEqual(sl, (slice(40, 45, None),))
# Slice with start=None & stop=negative
sl = utils.fix_slice(slice(None, -18, None), shp)
self.assertEqual(sl, (slice(None, 32, None),))
#### Failures ####
# Slice with start > stop
self.assertRaises(IndexError, utils.fix_slice, slice(30, 2, None), shp)
# Slice with start == stop
self.assertRaises(IndexError, utils.fix_slice, slice(18, 18, None), shp)
# # Slice with start=None & stop > size
# self.assertRaises(IndexError, utils.fix_slice, slice(None, 52, None), shp)
# Slice with start > size & stop=None
self.assertRaises(IndexError, utils.fix_slice, slice(53, None, None), shp)
# Slice with start < 0 & stop=None (after adjustment, -52 => -2)
self.assertRaises(IndexError, utils.fix_slice, slice(-52, None, None), shp)
# Slice with start=None & stop=0
self.assertRaises(IndexError, utils.fix_slice, slice(None, 0, None), shp)
# Slice with start=None & stop < 0 (after adjustment, -52 => -2)
self.assertRaises(IndexError, utils.fix_slice, slice(None, -52, None), shp)
# Slice with start=negative & stop=negative with start > stop
self.assertRaises(IndexError, utils.fix_slice, slice(-2, -10, None), shp)
#### Stepping ####
# Slice with step != None
sl = utils.fix_slice(slice(4, 43, 6), shp)
self.assertEqual(sl, (slice(4, 43, 6),))
# Slice with negative step - reverses start/stop
sl = utils.fix_slice(slice(10, 2, -2), shp)
self.assertEqual(sl, (slice(2, 10, 2),))
# Slice with all negatives (start, stop, step)
sl = utils.fix_slice(slice(-3, -22, -5), shp)
self.assertEqual(sl, (slice(28, 47, 5),))
def __setitem__(self, slice_, value):
slice_ = fix_slice(slice_, self.shape)
self._storage[slice_] = value
def __getitem__(self, slice_):
"""
Called to implement evaluation of self[slice_].
Not implemented by the abstract class
@param slice_ A set of valid constraints - int, [int,], (int,), or slice
@return The value contained by the storage at location slice
@raise ValueError when brick contains no values for specified slice
"""
from coverage_model import bricking_utils, utils
extents = tuple([s for s in self.total_domain.total_extents if s != 0])
if extents == (): # Empty domain(s) - no data, return empty array
return np.empty(0, dtype=self.dtype)
# bricks is a list of tuples [(b_ord, b_guid), ...]
slice_ = utils.fix_slice(deepcopy(slice_), extents)
log.trace('slice_=%s', slice_)
bricks = bricking_utils.get_bricks_from_slice(slice_, self.brick_tree, extents)
log.trace('Found bricks: %s', bricks)
ret_shp = utils.slice_shape(slice_, extents)
log.trace('Return array shape: %s', ret_shp)
ret_arr = np.empty(ret_shp, dtype=self.dtype)
ret_arr.fill(self.fill_value)
for b in bricks:
# b is (brick_ordinal, brick_guid)
_, bid = b
# brick_list[brick_guid] contains: [brick_extents, origin, tuple(bD), brick_active_size]
_, bori, _, bact = self.brick_list[bid]
bbnds = []
for i, bnd in enumerate(bori):
bbnds.append((bori[i], bori[i] + bact[i] - 1))
bbnds = tuple(bbnds)
brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds)
log.trace('brick_slice=%s\tbrick_mm=%s', brick_slice, brick_mm)
if None in brick_slice:
log.debug('Brick does not contain any of the requested indices: Move to next brick')
continue
ret_slice = bricking_utils.get_value_slice_nd(slice_, ret_shp, bbnds, brick_slice, brick_mm)
brick_file_path = '{0}/{1}.hdf5'.format(self.brick_path, bid)
if not os.path.exists(brick_file_path):
log.trace('Found virtual brick file: %s', brick_file_path)
else:
log.trace('Found real brick file: %s', brick_file_path)
with h5py.File(brick_file_path) as brick_file:
ret_vals = brick_file[bid][brick_slice]
# Check if object type
if self.dtype == '|O8':
if hasattr(ret_vals, '__iter__'):
ret_vals = [self._object_unpack_hook(x) for x in ret_vals]
else:
ret_vals = self._object_unpack_hook(ret_vals)
ret_arr[ret_slice] = ret_vals
# ret_arr = np.atleast_1d(ret_arr.squeeze())
# ret_arr = np.atleast_1d(ret_arr)
#
# # If the array is size 1 AND a slice object was NOT part of the query
# if ret_arr.size == 1 and not np.atleast_1d([isinstance(s, slice) for s in slice_]).all():
# ret_arr = ret_arr[0]
return ret_arr
def __setitem__(self, slice_, value):
"""
Called to implement assignment of self[slice_, value].
Not implemented by the abstract class
@param slice A set of valid constraints - int, [int,], (int,), or slice
@param value The value to assign to the storage at location slice_
@raise ValueError when brick contains no values for specified slice
"""
if self.mode == 'r':
raise IOError('PersistenceLayer not open for writing: mode == \'{0}\''.format(self.mode))
from coverage_model import bricking_utils, utils
extents = tuple([s for s in self.total_domain.total_extents if s != 0])
# bricks is a list of tuples [(b_ord, b_guid), ...]
slice_ = utils.fix_slice(deepcopy(slice_), extents)
log.trace('slice_=%s', slice_)
bricks = bricking_utils.get_bricks_from_slice(slice_, self.brick_tree, extents)
log.trace('Found bricks: %s', bricks)
values = np.asanyarray(value)
v_shp = values.shape
log.trace('value_shape: %s', v_shp)
s_shp = utils.slice_shape(slice_, extents)
log.trace('slice_shape: %s', s_shp)
is_broadcast = False
if v_shp == ():
log.trace('Broadcast!!')
is_broadcast = True
value_slice = ()
elif v_shp != s_shp:
if v_shp == tuple([i for i in s_shp if i != 1]): # Missing dimensions are singleton, just reshape to fit
values = values.reshape(s_shp)
v_shp = values.shape
else:
raise IndexError(
'Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}'.format(
s_shp, v_shp))
else:
value_slice = None
log.trace('value_shape: %s', v_shp)
for b in bricks:
# b is (brick_ordinal, brick_guid)
_, bid = b
# brick_list[brick_guid] contains: [brick_extents, origin, tuple(bD), brick_active_size]
_, bori, _, bact = self.brick_list[bid]
bbnds = []
bexts = []
for i, bnd in enumerate(bori):
bbnds.append((bori[i], bori[i] + bact[i] - 1))
bexts.append(bori[i] + bact[i])
bbnds = tuple(bbnds)
bexts = tuple(bexts)
log.trace('bid=%s, bbnds=%s, bexts=%s', bid, bbnds, bexts)
log.trace('Determining slice for brick: %s', b)
brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds)
log.trace('brick_slice=%s\tbrick_mm=%s', brick_slice, brick_mm)
if None in brick_slice: # Brick does not contain any of the requested indices
log.debug('Brick does not contain any of the requested indices: Move to next brick')
continue
try:
brick_slice = utils.fix_slice(brick_slice, bexts)
except IndexError:
log.debug('Malformed brick_slice: move to next brick')
continue
if not is_broadcast:
value_slice = bricking_utils.get_value_slice_nd(slice_, v_shp, bbnds, brick_slice, brick_mm)
try:
value_slice = utils.fix_slice(value_slice, v_shp)
except IndexError:
log.debug('Malformed value_slice: move to next brick')
continue
log.trace('\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s', b,
utils.slice_shape(brick_slice, bexts), brick_slice, brick_mm,
utils.slice_shape(value_slice, v_shp), value_slice)
v = values[value_slice]
self._set_values_to_brick(bid, brick_slice, v)
def __setitem__(self, slice_, value):
slice_ = utils.fix_slice(slice_, self.shape)
self._storage[slice_] = value
self._update_min_max(value)
def test_fix_slice_with_slice(self):
shp = (50,)
# Verify slice is tupled
sl = utils.fix_slice(slice(None, None, None), shp)
self.assertEqual(sl, (slice(None, None, None),))
shp = (50,20)
# Verify list of slices are tupled
sl = utils.fix_slice([slice(None, None, None), slice(None, None, None)], shp)
self.assertEqual(sl, (slice(None, None, None), slice(None, None, None),))
shp = (50,)
#### Passes ####
# Slice with all None
sl = utils.fix_slice(slice(None, None, None), shp)
self.assertEqual(sl, (slice(None, None, None),))
# Slice with start=None & stop=positive
sl = utils.fix_slice(slice(None, 30, None), shp)
self.assertEqual(sl, (slice(None, 30, None),))
# Slice with start=positive & stop=None
sl = utils.fix_slice(slice(10, None, None), shp)
self.assertEqual(sl, (slice(10, None, None),))
# Slice with start=negative & stop=None
sl = utils.fix_slice(slice(-10, None, None), shp)
self.assertEqual(sl, (slice(40, None, None),))
# Slice with start=negative & stop=negative
sl = utils.fix_slice(slice(-10, -5, None), shp)
self.assertEqual(sl, (slice(40, 45, None),))
# Slice with start=None & stop=negative
sl = utils.fix_slice(slice(None, -18, None), shp)
self.assertEqual(sl, (slice(None, 32, None),))
#### Failures ####
# Slice with start > stop
self.assertRaises(IndexError, utils.fix_slice, slice(30, 2, None), shp)
# Slice with start == stop
self.assertRaises(IndexError, utils.fix_slice, slice(18, 18, None), shp)
# # Slice with start=None & stop > size
# self.assertRaises(IndexError, utils.fix_slice, slice(None, 52, None), shp)
# Slice with start > size & stop=None
self.assertRaises(IndexError, utils.fix_slice, slice(53, None, None), shp)
# Slice with start < 0 & stop=None (after adjustment, -52 => -2)
self.assertRaises(IndexError, utils.fix_slice, slice(-52, None, None), shp)
# Slice with start=None & stop=0
self.assertRaises(IndexError, utils.fix_slice, slice(None, 0, None), shp)
# Slice with start=None & stop < 0 (after adjustment, -52 => -2)
self.assertRaises(IndexError, utils.fix_slice, slice(None, -52, None), shp)
# Slice with start=negative & stop=negative with start > stop
self.assertRaises(IndexError, utils.fix_slice, slice(-2, -10, None), shp)
#### Stepping ####
# Slice with step != None
sl = utils.fix_slice(slice(4, 43, 6), shp)
self.assertEqual(sl, (slice(4, 43, 6),))
# Slice with negative step - reverses start/stop
sl = utils.fix_slice(slice(10, 2, -2), shp)
self.assertEqual(sl, (slice(2, 10, 2),))
# Slice with all negatives (start, stop, step)
sl = utils.fix_slice(slice(-3, -22, -5), shp)
self.assertEqual(sl, (slice(28, 47, 5),))
def __getitem__(self, slice_):
slice_ = utils.fix_slice(slice_, self.shape)
return _cleanse_value(self._storage[slice_], slice_)
