def insert(self, count, extents, obj=None):
# The extents from the old rtree impl are [xmin,ymin,xmax,ymax]
minval = extents[0]
maxval = extents[2]
from coverage_model.basic_types import Span
span = Span(minval, maxval, value=obj)
if span not in self._spans:
self._spans.append(span)
def deserialize(cls, src_str):
if isinstance(src_str, basestring) and src_str.startswith(cls.__name__):
tmp = src_str.strip(cls.__name__)
rtp = RTreeProxy()
for span_tpl in tmp.split('::span::'):
if span_tpl == '':
continue
a, b, c, d, e = span_tpl.split('_')
span_tpl = (int(b), int(c), int(d), e)
from coverage_model.basic_types import Span
span = Span.from_iterable(span_tpl)
rtp._spans.append(span)
return rtp
raise TypeError('Improper formatting for RTreeProxy deserialization: %s' % src_str)
def express_slice(slice_, total_shape):
from coverage_model.basic_types import Span
fsl = fix_slice(slice_, total_shape)
ret = []
for i, x in enumerate(fsl):
if isinstance(x, slice):
start = x.start if not x.start is None else 0
stop = x.stop if not x.stop is None else total_shape[i]
step = x.step if not x.step is None else 1
ret.append(slice(start, stop, step))
elif isinstance(x, Span):
low = x.lower_bound if not x.lower_bound is None else 0
up = x.upper_bound if not x.upper_bound is None else total_shape[i]
ret.append(Span(low, up, x.offset, x.value))
else:
ret.append(x)
return tuple(ret)
def test_ndspan(self):
min_int = np.iinfo('int64').min
max_int = np.iinfo('int64').max
s = NdSpan([Span(None, 10), Span(3, 30), Span(20, None)])
self.assertIn((8, 5, 23), s)
self.assertIn((min_int, 29, max_int), s)
self.assertNotIn((12, 5, 28), s)
self.assertNotIn([-29, 30, 21], s)
self.assertNotIn([2, 23, 19], s)
self.assertEqual(s.shape, (0, 27, 0))
self.assertEqual(len(s), 0)
s = NdSpan([Span(0, 20), Span(4, 8), Span(9, 18)])
self.assertIn((7, 4, 15), s)
self.assertIn((0, 7, 10), s)
self.assertNotIn((20, 3, 18), s)
self.assertEqual(s.shape, (20, 4, 9))
self.assertEqual(len(s), 20 * 4 * 9)
def test_span(self):
min_int = np.iinfo('int64').min
max_int = np.iinfo('int64').max
# Unlimited bounds
s = Span(value=10)
self.assertIn(min_int, s)
self.assertIn(0, s)
self.assertIn(max_int, s)
self.assertEqual(len(s), 0)
self.assertEqual(s.value, 10)
self.assertEqual(s.indices(7), (0, 7, 1))
self.assertEqual(s.offset_indices(7), (0, 7, 1))
# Unlimited lower bound
s = Span(None, 5, offset=3, value='bob')
self.assertIn(min_int, s)
self.assertIn(-18, s)
self.assertIn(0, s)
self.assertNotIn(5, s)
self.assertNotIn(6, s)
self.assertNotIn(19, s)
self.assertEqual(len(s), 0)
self.assertEqual(s.value, 'bob')
self.assertEqual(s.indices(10), (0, 5, 1))
self.assertEqual(s.offset_indices(10), (0, 8, 1))
# Unlimited upper bound
s = Span(5, None, offset=-2, value=9.38)
self.assertIn(max_int, s)
self.assertIn(18, s)
self.assertIn(6, s)
self.assertIn(5, s)
self.assertNotIn(4, s)
self.assertNotIn(-12, s)
self.assertEqual(len(s), 0)
self.assertEqual(s.value, 9.38)
self.assertEqual(s.indices(12), (5, 12, 1))
self.assertEqual(s.offset_indices(12), (3, 12, 1))
# Fully bounded
s = Span(10, 68, offset=6, value=4)
self.assertIn(34, s)
self.assertNotIn(8, s)
self.assertNotIn(90, s)
self.assertEqual(len(s), 58)
self.assertEqual(s.value, 4)
self.assertEqual(s.indices(30), (10, 30, 1))
self.assertEqual(s.offset_indices(30), (16, 30, 1))
def inspect_bricks(self, cov_pth, dataset_id, param_name):
brick_domains_new = None
new_brick_list = None
brick_list_spans = None
tD = None
bD = None
min_data_bound = None
max_data_bound = None
spans = []
pdir = os.path.join(cov_pth, dataset_id, param_name)
# TODO: Check for brick files, if none then skip this entirely
if os.path.exists(pdir) and len(os.listdir(pdir)) > 0:
for brick in [
os.path.join(pdir, x) for x in os.listdir(pdir)
if (not param_name in x) and ('.hdf5' in x)
]:
brick_guid = os.path.basename(brick).replace('.hdf5', '')
with h5py.File(brick, 'r') as f:
ds = f[brick_guid]
fv = ds.fillvalue
low = ds[0]
up = ds.value.max()
low = low if low != fv else None
if low < up:
spans.append(
Span(lower_bound=low,
upper_bound=up,
value=brick_guid))
if len(spans) > 0:
spans.sort()
min_data_bound = min([s.lower_bound for s in spans])
max_data_bound = max([s.upper_bound for s in spans])
bricks_sorted = [[
s.value, s.lower_bound, s.upper_bound,
int(s.upper_bound - s.lower_bound + 1)
] for s in spans]
# brick_list
# {'0BC3FF45-60FB-440A-980D-80C9CEB6F799': (((0, 99999),),
# (0,),
# (100000,),
# (100000,)),
# '5AF8B7A8-49DE-47FB-8671-B9BE8164CC8F': (((100000, 199999),),
# (100000,),
# (100000,),
# (29600,))}
# TODO: Surely we can get a hold of the bricking_scheme from some global location???
brick_size = 100000
chunk_size = 100000
start = 0
stop = brick_size - 1
brick_list_spans = []
new_brick_list = {}
for brick in bricks_sorted:
new_brick = [
brick[0],
(((start, stop), ), (start, ), (brick_size, ),
(brick[3], ))
]
brick_list_spans.append(
Span(lower_bound=start,
upper_bound=stop,
value=new_brick))
new_brick_list[brick[0]] = (((start, stop), ), (start, ),
(brick_size, ), (brick[3], ))
start = start + brick_size
stop = start + brick_size - 1
brick_list_spans.sort()
# brick_domains
# [(129600,), (100000,), (100000,), {'brick_size': 100000, 'chunk_size': 100000}]
maxes = [sum(b[3]) for b in new_brick_list.values()]
tD = (sum(maxes), )
bD = (brick_size, )
cD = (chunk_size, )
bricking_scheme = {}
bricking_scheme['brick_size'] = brick_size
bricking_scheme['chunk_size'] = chunk_size
brick_domains_new = [tD, bD, cD, bricking_scheme]
return brick_domains_new, new_brick_list, brick_list_spans, tD, bD, min_data_bound, max_data_bound
if isinstance(
value,
SparseConstantValue): # RDT --> Coverage style assignment
value = value[0, :]
if not isinstance(value, AbstractParameterValue):
# If the value is an array/iterable, we only take the first one of the outermost dimension
value = np.atleast_1d(value)[0, ...]
bv = np.atleast_1d(value)[0]
bnds = (bv, bv)
else:
bnds = value.bounds
if not hasattr(spans, '__iter__') and spans == self.fill_value:
spans = [Span(value=value)]
slice_ = (self.shape[0] - 1, )
# Get the last span
lspn = spans[-1]
if isinstance(lspn.value, AbstractParameterValue):
pndim = len(lspn.value.shape)
else:
pndim = np.atleast_1d(lspn.value).ndim
if isinstance(value, AbstractParameterValue):
nndim = len(value.shape)
else:
nndim = np.atleast_1d(value).ndim
if pndim != nndim:
raise ValueError(
def test_span(self):
min_int = np.iinfo('int64').min
max_int = np.iinfo('int64').max
# Unlimited bounds
s = Span(value=10)
self.assertIn(min_int, s)
self.assertIn(0, s)
self.assertIn(max_int, s)
self.assertEqual(len(s), 0)
self.assertEqual(s.value, 10)
self.assertEqual(s.indices(7), (0, 7, 1))
self.assertEqual(s.offset_indices(7), (0, 7, 1))
# Unlimited lower bound
s = Span(None, 5, offset=3, value='bob')
self.assertIn(min_int, s)
self.assertIn(-18, s)
self.assertIn(0, s)
self.assertNotIn(5, s)
self.assertNotIn(6, s)
self.assertNotIn(19, s)
self.assertEqual(len(s), 0)
self.assertEqual(s.value, 'bob')
self.assertEqual(s.indices(10), (0, 5, 1))
self.assertEqual(s.offset_indices(10), (0, 8, 1))
# Unlimited upper bound
s = Span(5, None, offset=-2, value=9.38)
self.assertIn(max_int, s)
self.assertIn(18, s)
self.assertIn(6, s)
self.assertIn(5, s)
self.assertNotIn(4, s)
self.assertNotIn(-12, s)
self.assertEqual(len(s), 0)
self.assertEqual(s.value, 9.38)
self.assertEqual(s.indices(12), (5, 12, 1))
self.assertEqual(s.offset_indices(12), (3, 12, 1))
# Fully bounded
s = Span(10, 68, offset=6, value=4)
self.assertIn(34, s)
self.assertNotIn(8, s)
self.assertNotIn(90, s)
self.assertEqual(len(s), 58)
self.assertEqual(s.value, 4)
self.assertEqual(s.indices(30), (10, 30, 1))
self.assertEqual(s.offset_indices(30), (16, 30, 1))
