def test_get_bricks_from_slice_1d(self):
total_domain = (15, )
brick_extents = (((0, 4), ), ((5, 9), ), ((10, 14), ))
rtree_extents = ((0, 0, 4, 0), (5, 0, 9, 0), (10, 0, 14, 0))
brick_0 = (0, ((0, 4), ))
brick_1 = (1, ((5, 9), ))
brick_2 = (2, ((10, 14), ))
from coverage_model.test.bricking_assessment_utility import BrickingAssessor
rtree = RTreeProxy()
for x in BrickingAssessor.rtree_populator(rtree_extents,
brick_extents):
rtree.insert(*x)
# Try a variety of slices
self._get_bricks_assert(slice(None), rtree, total_domain, 3,
[brick_0, brick_1, brick_2])
self._get_bricks_assert(slice(None, None, 3), rtree, total_domain, 3,
[brick_0, brick_1, brick_2])
self._get_bricks_assert(slice(0, 3), rtree, total_domain, 1, [brick_0])
self._get_bricks_assert(slice(5, 9), rtree, total_domain, 1, [brick_1])
self._get_bricks_assert(slice(6, None), rtree, total_domain, 2,
[brick_1, brick_2])
self._get_bricks_assert(
slice(None, None, 10), rtree, total_domain, 3,
[brick_0, brick_1, brick_2
]) # three bricks, tho the middle one isn't needed
self._get_bricks_assert(([1, 3], ), rtree, total_domain, 1, [brick_0])
self._get_bricks_assert(([2, 4, 7], ), rtree, total_domain, 2,
[brick_0, brick_1])
self._get_bricks_assert(
([3, 12], ), rtree, total_domain, 3,
[brick_0, brick_1, brick_2
]) # three bricks, tho the middle one isn't needed
self._get_bricks_assert(1, rtree, total_domain, 1, [brick_0])
self._get_bricks_assert(6, rtree, total_domain, 1, [brick_1])
self._get_bricks_assert(13, rtree, total_domain, 1, [brick_2])
def __init__(self,
total_domain=(10, 10),
brick_size=5,
use_hdf=False,
root_dir='test_data/multi_dim_trials',
guid=None,
dtype='int16'):
self.total_domain = total_domain
self.brick_sizes = tuple(brick_size for x in total_domain)
self.use_hdf = use_hdf
self.dtype = np.dtype(dtype).name
if self.use_hdf:
self.guid = guid or create_guid()
name = '%s_%s' % (self.guid, self.dtype)
self.root_dir = root_dir
if not os.path.exists(self.root_dir):
os.makedirs(self.root_dir)
if os.path.exists(os.path.join(self.root_dir, name)):
shutil.rmtree(os.path.join(self.root_dir, name))
self.master_manager = MasterManager(
self.root_dir, name, name='md_test_{0}'.format(name))
self.master_manager.flush()
pc = ParameterContext('test_param',
param_type=QuantityType(self.dtype),
fill_value=-1)
self.param_manager = ParameterManager(
os.path.join(self.root_dir, name, pc.name), pc.name)
self.param_manager.parameter_context = pc
self.master_manager.create_group(pc.name)
self.param_manager.flush()
self.bricks = {}
self.brick_origins = bricking_utils.calc_brick_origins(
self.total_domain, self.brick_sizes)
self.brick_extents, self.rtree_extents = bricking_utils.calc_brick_and_rtree_extents(
self.brick_origins, self.brick_sizes)
self.build_bricks()
self.rtree = RTreeProxy()
for x in BrickingAssessor.rtree_populator(self.rtree_extents,
self.brick_extents):
self.rtree.insert(*x)
def _load(self):
try:
results = DBFactory.get_db().get(self.guid)
for key in results:
val = results[key]
if isinstance(val, basestring) and val.startswith('DICTABLE'):
i = val.index('|', 9)
smod, sclass = val[9:i].split(':')
value = unpack(val[i+1:])
module = __import__(smod, fromlist=[sclass])
classobj = getattr(module, sclass)
value = classobj._fromdict(value)
elif key in ('root_dir', 'file_path'):
# No op - set in constructor
continue
elif key == 'brick_tree':
setattr(self, key, RTreeProxy.deserialize(val))
continue
elif key == 'span_collection':
unpacked = unpack(val)
value = SpanCollectionByFile.from_str(unpacked)
log.trace("Reconstructed SpanCollection for %s: %s", self.guid, str(value))
else:
value = unpack(val)
if isinstance(value, tuple):
value = list(value)
setattr(self, key, value)
except Exception as e:
log.error("Caught exception reconstructing metadata for guid %s : %s", self.guid, e.message)
raise
def __init__(self, filedir, guid, **kwargs):
MetadataManager.__init__(self, **kwargs)
self._ignore.update(['guid', 'file_path', 'root_dir'])
self.guid = guid
self.param_groups = set()
self.root_dir = os.path.join(filedir,guid)
fname = ''.join([guid, '_master.hdf5'])
self.file_path = os.path.join(self.root_dir, fname)
self.brick_tree = RTreeProxy()
if self.is_persisted_in_db(guid):
self._load()
elif self.isPersisted(filedir, guid):
mm = MasterManager(filedir, guid, **kwargs)
for key, value in mm.__dict__.iteritems():
if not key == "_dirty":
self.__setattr__(key, value)
self.flush(deep=False)
# This is odd - You can load an object and override stored values on construction.
# This might lead to unexpected behavior for users.
for k, v in kwargs.iteritems():
if hasattr(self, k) and v is None:
continue
setattr(self, k, v)
if (hasattr(self, 'parameter_bounds') and self.parameter_bounds is None) or not hasattr(self, 'parameter_bounds'):
self.parameter_bounds = {}
def __init__(
self,
total_domain=(10, 10),
brick_size=5,
use_hdf=False,
root_dir="test_data/multi_dim_trials",
guid=None,
dtype="int16",
):
self.total_domain = total_domain
self.brick_sizes = tuple(brick_size for x in total_domain)
self.use_hdf = use_hdf
self.dtype = np.dtype(dtype).name
if self.use_hdf:
self.guid = guid or create_guid()
name = "%s_%s" % (self.guid, self.dtype)
self.root_dir = root_dir
if not os.path.exists(self.root_dir):
os.makedirs(self.root_dir)
if os.path.exists(os.path.join(self.root_dir, name)):
shutil.rmtree(os.path.join(self.root_dir, name))
# self.master_manager = MasterManager(self.root_dir, name, name='md_test_{0}'.format(name))
self.master_manager = MetadataManagerFactory.buildMetadataManager(
self.root_dir, name, name="md_test_{0}".format(name)
)
self.master_manager.flush()
pc = ParameterContext("test_param", param_type=QuantityType(self.dtype), fill_value=-1)
self.param_manager = ParameterManager(os.path.join(self.root_dir, name, pc.name), pc.name)
self.param_manager.parameter_context = pc
self.master_manager.create_group(pc.name)
self.param_manager.flush()
self.bricks = {}
self.brick_origins = bricking_utils.calc_brick_origins(self.total_domain, self.brick_sizes)
self.brick_extents, self.rtree_extents = bricking_utils.calc_brick_and_rtree_extents(
self.brick_origins, self.brick_sizes
)
self.build_bricks()
self.rtree = RTreeProxy()
for x in BrickingAssessor.rtree_populator(self.rtree_extents, self.brick_extents):
self.rtree.insert(*x)
def __init__(self, filedir, guid, **kwargs):
MetadataManager.__init__(self, **kwargs)
self.guid = guid
self._ignore.update(['guid', 'file_path', 'root_dir', 'type'])
self.param_groups = set()
self.root_dir = os.path.join(filedir,guid)
self.file_path = os.path.join(filedir, guid)
self.brick_tree = RTreeProxy()
self.type = ''
self._load()
# This is odd - You can load an object and override stored values on construction.
# This might lead to unexpected behavior for users.
for k, v in kwargs.iteritems():
if hasattr(self, k) and v is None:
continue
setattr(self, k, v)
if (hasattr(self, 'parameter_bounds') and self.parameter_bounds is None) or not hasattr(self, 'parameter_bounds'):
self.parameter_bounds = {}
def test_get_bricks_from_slice_3d(self):
total_domain = (10, 15, 5)
brick_extents = (((0, 4), (0, 4), (0, 4)), ((0, 4), (5, 9), (0, 4)),
((0, 4), (10, 14), (0, 4)), ((5, 9), (0, 4), (0, 4)),
((5, 9), (5, 9), (0, 4)), ((5, 9), (10, 14), (0, 4)))
rtree_extents = ((0, 0, 0, 4, 4, 4), (0, 5, 0, 4, 9, 4),
(0, 10, 0, 4, 14, 4), (5, 0, 0, 9, 4, 4),
(5, 5, 0, 9, 9, 4), (5, 10, 0, 9, 14, 4))
brick_0 = (0, ((0, 4), (0, 4), (0, 4)))
brick_1 = (1, ((0, 4), (5, 9), (0, 4)))
brick_2 = (2, ((0, 4), (10, 14), (0, 4)))
brick_3 = (3, ((5, 9), (0, 4), (0, 4)))
brick_4 = (4, ((5, 9), (5, 9), (0, 4)))
brick_5 = (5, ((5, 9), (10, 14), (0, 4)))
from coverage_model.test.bricking_assessment_utility import BrickingAssessor
rtree = RTreeProxy()
for x in BrickingAssessor.rtree_populator(rtree_extents,
brick_extents):
rtree.insert(*x)
# Get all bricks
self._get_bricks_assert(
(slice(None), ) * 3, rtree, total_domain, 6,
[brick_0, brick_1, brick_2, brick_3, brick_4, brick_5])
self._get_bricks_assert((0, 0, 0), rtree, total_domain, 1, [brick_0])
self._get_bricks_assert((8, 5, 2), rtree, total_domain, 1, [brick_4])
self._get_bricks_assert((4, 12, 1), rtree, total_domain, 1, [brick_2])
self._get_bricks_assert((9, 13, [0, 2]), rtree, total_domain, 1,
[brick_5])
self._get_bricks_assert((8, [3, 5, 12], 0), rtree, total_domain, 3,
[brick_3, brick_4, brick_5])
self._get_bricks_assert(([5, 9], 10, 0), rtree, total_domain, 1,
[brick_5])
self._get_bricks_assert(([5, 9], [4, 12, 13], 0), rtree, total_domain,
3, [brick_3, brick_4, brick_5])
self._get_bricks_assert(([2, 4], [2, 11], [1, 3, 4]), rtree,
total_domain, 3, [brick_0, brick_1, brick_2])
self._get_bricks_assert(([2, 3, 9], 12, [1, 3, 4]), rtree,
total_domain, 2, [brick_2, brick_5])
self._get_bricks_assert((slice(None), 12, [1, 3, 4]), rtree,
total_domain, 2, [brick_2, brick_5])
self._get_bricks_assert((slice(1, 7), 3, [1, 3, 4]), rtree,
total_domain, 2, [brick_0, brick_3])
self._get_bricks_assert((slice(3, 4), 7, [1, 3, 4]), rtree,
total_domain, 1, [brick_1])
self._get_bricks_assert(
(slice(2, 8, 7), [1, 6, 12], 4), rtree, total_domain, 6,
[brick_0, brick_1, brick_2, brick_3, brick_4, brick_5])
self._get_bricks_assert((slice(2, 4, 7), slice(None), 2), rtree,
total_domain, 3, [brick_0, brick_1, brick_2])
self._get_bricks_assert(
(slice(None, 4), slice(9, None, 2), slice(None)), rtree,
total_domain, 2, [brick_1, brick_2])
self._get_bricks_assert(
(slice(None, 6, 4), slice(12, None, 2), slice(3, None)), rtree,
total_domain, 2, [brick_2, brick_5])
self._get_bricks_assert(
(slice(None, 8), slice(6, 13, 4), slice(None, None, 3)), rtree,
total_domain, 4, [brick_1, brick_2, brick_4, brick_5])
def test_get_bricks_from_slice_2d(self):
total_domain = (15, 10)
brick_extents = (((0, 4), (0, 4)), ((0, 4), (5, 9)), ((5, 9), (0, 4)),
((5, 9), (5, 9)), ((10, 14), (0, 4)), ((10, 14), (5,
9)))
rtree_extents = ((0, 0, 4, 4), (0, 5, 4, 9), (5, 0, 9, 4),
(5, 5, 9, 9), (10, 0, 14, 4), (10, 5, 14, 9))
brick_0 = (0, ((0, 4), (0, 4)))
brick_1 = (1, ((0, 4), (5, 9)))
brick_2 = (2, ((5, 9), (0, 4)))
brick_3 = (3, ((5, 9), (5, 9)))
brick_4 = (4, ((10, 14), (0, 4)))
brick_5 = (5, ((10, 14), (5, 9)))
from coverage_model.test.bricking_assessment_utility import BrickingAssessor
rtree = RTreeProxy()
for x in BrickingAssessor.rtree_populator(rtree_extents,
brick_extents):
rtree.insert(*x)
# Get all bricks
self._get_bricks_assert(
(slice(None), ) * 2, rtree, total_domain, 6,
[brick_0, brick_1, brick_2, brick_3, brick_4, brick_5])
self._get_bricks_assert(
(slice(None), slice(None, 8)), rtree, total_domain, 6,
[brick_0, brick_1, brick_2, brick_3, brick_4, brick_5])
self._get_bricks_assert((slice(None), slice(None, 4)), rtree,
total_domain, 3, [brick_0, brick_2, brick_4])
self._get_bricks_assert((slice(7, 12), slice(5, 8)), rtree,
total_domain, 2, [brick_3, brick_5])
self._get_bricks_assert(
(slice(2, 14, 3), slice(2, 7)), rtree, total_domain, 6,
[brick_0, brick_1, brick_2, brick_3, brick_4, brick_5])
self._get_bricks_assert(
(slice(2, 14, 10), slice(2, 7)), rtree, total_domain, 6,
[brick_0, brick_1, brick_2, brick_3, brick_4, brick_5])
self._get_bricks_assert((0, slice(2, 8, 3)), rtree, total_domain, 2,
[brick_0, brick_1])
self._get_bricks_assert((6, slice(2, 7)), rtree, total_domain, 2,
[brick_2, brick_3])
self._get_bricks_assert((slice(None, 12), 7), rtree, total_domain, 3,
[brick_1, brick_3, brick_5])
self._get_bricks_assert((12, slice(2, None, 4)), rtree, total_domain,
2, [brick_4, brick_5])
self._get_bricks_assert(([1, 2], 9), rtree, total_domain, 1, [brick_1])
self._get_bricks_assert(([0, 14], 3), rtree, total_domain, 3,
[brick_0, brick_2, brick_4])
self._get_bricks_assert((3, [1, 8]), rtree, total_domain, 2,
[brick_0, brick_1])
self._get_bricks_assert(([2, 5], [1, 8]), rtree, total_domain, 4,
[brick_0, brick_1, brick_2, brick_3])
self._get_bricks_assert(([6, 9], [1, 8]), rtree, total_domain, 2,
[brick_2, brick_3])
self._get_bricks_assert(([2, 8, 13], [7, 8]), rtree, total_domain, 3,
[brick_1, brick_3, brick_5])
class DbBackedMetadataManager(MetadataManager):
@staticmethod
def dirExists(directory):
return True
@staticmethod
def isPersisted(directory, guid):
return DBFactory.get_db().is_persisted(guid)
@staticmethod
def getCoverageType(directory, guid):
cov_type = DBFactory.get_db().get_coverage_type(guid)
if '' == cov_type:
return ''
else:
cov_type = unpack(cov_type)
return cov_type
def __init__(self, filedir, guid, **kwargs):
MetadataManager.__init__(self, **kwargs)
self.guid = guid
self._ignore.update(['guid', 'file_path', 'root_dir', 'type'])
self.param_groups = set()
self.root_dir = os.path.join(filedir,guid)
self.file_path = os.path.join(filedir, guid)
self.brick_tree = RTreeProxy()
self.type = ''
self._load()
# This is odd - You can load an object and override stored values on construction.
# This might lead to unexpected behavior for users.
for k, v in kwargs.iteritems():
if hasattr(self, k) and v is None:
continue
setattr(self, k, v)
if (hasattr(self, 'parameter_bounds') and self.parameter_bounds is None) or not hasattr(self, 'parameter_bounds'):
self.parameter_bounds = {}
def __setattr__(self, key, value):
super(DbBackedMetadataManager, self).__setattr__(key, value)
if not key in self._ignore and not key.startswith('_'):
self._hmap[key] = hash_any(value)
self._dirty.add(key)
super(DbBackedMetadataManager, self).__setattr__('_is_dirty',True)
def flush(self):
if self.is_dirty(True):
try:
# package for storage
insert_dict = {}
for k in list(self._dirty):
v = getattr(self, k)
log.trace('FLUSH: key=%s v=%s', k, v)
if isinstance(v, Dictable):
prefix='DICTABLE|{0}:{1}|'.format(v.__module__, v.__class__.__name__)
value = prefix + pack(v.dump())
elif k == 'brick_tree':
if hasattr(self, 'brick_tree') and isinstance(self.brick_tree, RTreeProxy):
val = self.brick_tree.serialize()
if val != '':
insert_dict['brick_tree'] = val
continue
else:
value = pack(v)
insert_dict[k] = value
# Update the hash_value in _hmap
self._hmap[k] = hash_any(v)
dirty_spans = self.span_collection.get_dirty_spans()
if len(dirty_spans) > 0:
val = str(self.span_collection)
log.trace("Span tuple: %s", val)
value = pack(val)
insert_dict['span_collection'] = value
DBFactory.get_db().insert(self.guid, insert_dict, dirty_spans)
for span in dirty_spans:
span.is_dirty = False
self._dirty.clear()
except IOError, ex:
if "unable to create file (File accessability: Unable to open file)" in ex.message:
log.info('Issue writing to hdf file during master_manager.flush - this is not likely a huge problem: %s', ex.message)
else:
raise
super(DbBackedMetadataManager, self).__setattr__('_is_dirty',False)
class BrickingAssessor(object):
def __init__(self,
total_domain=(10, 10),
brick_size=5,
use_hdf=False,
root_dir='test_data/multi_dim_trials',
guid=None,
dtype='int16'):
self.total_domain = total_domain
self.brick_sizes = tuple(brick_size for x in total_domain)
self.use_hdf = use_hdf
self.dtype = np.dtype(dtype).name
if self.use_hdf:
self.guid = guid or create_guid()
name = '%s_%s' % (self.guid, self.dtype)
self.root_dir = root_dir
if not os.path.exists(self.root_dir):
os.makedirs(self.root_dir)
if os.path.exists(os.path.join(self.root_dir, name)):
shutil.rmtree(os.path.join(self.root_dir, name))
self.master_manager = MasterManager(
self.root_dir, name, name='md_test_{0}'.format(name))
self.master_manager.flush()
pc = ParameterContext('test_param',
param_type=QuantityType(self.dtype),
fill_value=-1)
self.param_manager = ParameterManager(
os.path.join(self.root_dir, name, pc.name), pc.name)
self.param_manager.parameter_context = pc
self.master_manager.create_group(pc.name)
self.param_manager.flush()
self.bricks = {}
self.brick_origins = bricking_utils.calc_brick_origins(
self.total_domain, self.brick_sizes)
self.brick_extents, self.rtree_extents = bricking_utils.calc_brick_and_rtree_extents(
self.brick_origins, self.brick_sizes)
self.build_bricks()
self.rtree = RTreeProxy()
for x in BrickingAssessor.rtree_populator(self.rtree_extents,
self.brick_extents):
self.rtree.insert(*x)
@classmethod
def rtree_populator(cls, rtree_extents, brick_extents):
for i, e in enumerate(rtree_extents):
yield i, e, brick_extents[i]
def _get_numpy_array(self, shape):
if not isinstance(shape, tuple):
shape = tuple(shape)
return np.arange(utils.prod(shape), dtype=self.dtype).reshape(shape)
def build_bricks(self):
for x in xrange(len(self.brick_origins)):
if not self.use_hdf:
self.bricks[x] = np.empty(self.brick_sizes, dtype=self.dtype)
self.bricks[x].fill(-1)
else:
id = str(x)
fn = '{0}.hdf5'.format(id)
pth = os.path.join(self.param_manager.root_dir, fn)
relpth = os.path.join(
self.param_manager.root_dir.replace(
self.master_manager.root_dir, '.'), fn)
lnpth = '/{0}/{1}'.format(self.param_manager.parameter_name,
id)
self.master_manager.add_external_link(lnpth, relpth, id)
self.bricks[x] = pth
def reset_bricks(self):
for i, arr in enumerate(self.bricks.itervalues()):
if not self.use_hdf:
arr.fill(-1)
else:
with h5py.File(arr) as f:
ds = f.require_dataset(str(i),
shape=self.brick_sizes,
dtype=self.dtype,
chunks=None,
fillvalue=-1)
ds[:] = -1
def put_values_to_bricks(self, slice_, values):
slice_ = utils.fix_slice(slice_, self.total_domain)
bricks = bricking_utils.get_bricks_from_slice(
slice_, self.rtree, self.total_domain
) # this is a list of tuples [(b_id, (bounds...),), ...]
values = np.asanyarray(values)
v_shp = values.shape
log.debug('value_shape: %s', v_shp)
s_shp = utils.slice_shape(slice_, self.total_domain)
log.debug('slice_shape: %s', s_shp)
is_broadcast = False
if v_shp == ():
log.debug('Broadcast!!')
is_broadcast = True
value_slice = ()
elif v_shp != s_shp:
if v_shp == tuple([
i for i in s_shp if i != 1
]): # Missing dimensions are singleton, just reshape to fit
values = values.reshape(s_shp)
v_shp = values.shape
else:
raise IndexError(
'Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}'
.format(s_shp, v_shp))
else:
value_slice = None
log.debug('value_shape: %s', v_shp)
for b in bricks:
# b is (brick_id, (brick_bounds per dim...),)
bid, bbnds = b
log.debug('Determining slice for brick: %s', b)
bexts = tuple([x + 1 for x in zip(*bbnds)[1]
]) # Shift from index to size
log.debug('bid=%s, bbnds=%s, bexts=%s', bid, bbnds, bexts)
brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(
slice_, bbnds)
if None in brick_slice: # Brick does not contain any of the requested indices
log.debug(
'Brick does not contain any of the requested indices: Move to next brick'
)
continue
try:
brick_slice = utils.fix_slice(brick_slice, bexts)
except IndexError:
log.debug('Malformed brick_slice: move to next brick')
continue
if not is_broadcast:
value_slice = bricking_utils.get_value_slice_nd(
slice_, v_shp, bbnds, brick_slice, brick_mm)
try:
value_slice = utils.fix_slice(value_slice, v_shp)
except IndexError:
log.debug('Malformed value_slice: move to next brick')
continue
log.debug(
'\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s',
b, utils.slice_shape(brick_slice,
bexts), brick_slice, brick_mm,
utils.slice_shape(value_slice, v_shp), value_slice)
v = values[value_slice]
log.debug('\nvalues %s=\n%s', v.shape, v)
if not self.use_hdf:
self.bricks[bid][brick_slice] = v
else:
fi = self.bricks[bid]
with h5py.File(fi) as f:
ds = f.require_dataset(str(bid),
shape=self.brick_sizes,
dtype=self.dtype,
chunks=None,
fillvalue=-1)
ds[brick_slice] = v
def get_values_from_bricks(self, slice_):
slice_ = utils.fix_slice(slice_, self.total_domain)
bricks = bricking_utils.get_bricks_from_slice(
slice_, self.rtree, self.total_domain
) # this is a list of tuples [(b_id, (bounds...),), ...]
ret_shp = utils.slice_shape(slice_, self.total_domain)
ret_arr = np.empty(ret_shp, dtype=self.dtype)
for b in bricks:
bid, bbnds = b
brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(
slice_, bbnds)
if None in brick_slice:
continue
ret_slice = bricking_utils.get_value_slice_nd(
slice_, ret_shp, bbnds, brick_slice, brick_mm)
if not self.use_hdf:
ret_vals = self.bricks[bid][brick_slice]
else:
fi = self.bricks[bid]
with h5py.File(fi) as f:
ds = f.require_dataset(str(bid),
shape=self.brick_sizes,
dtype=self.dtype,
chunks=None,
fillvalue=-1)
ret_vals = ds[brick_slice]
ret_arr[ret_slice] = ret_vals
ret_arr = ret_arr.squeeze()
if ret_arr.size == 1:
if ret_arr.ndim == 0:
ret_arr = ret_arr[()]
else:
ret_arr = ret_arr[0]
return ret_arr
class BrickingAssessor(object):
def __init__(self, total_domain=(10, 10), brick_size=5, use_hdf=False, root_dir='test_data/multi_dim_trials',
guid=None, dtype='int16'):
self.total_domain = total_domain
self.brick_sizes = tuple(brick_size for x in total_domain)
self.use_hdf = use_hdf
self.dtype = np.dtype(dtype).name
if self.use_hdf:
self.guid = guid or create_guid()
name = '%s_%s' % (self.guid, self.dtype)
self.root_dir = root_dir
if not os.path.exists(self.root_dir):
os.makedirs(self.root_dir)
if os.path.exists(os.path.join(self.root_dir, name)):
shutil.rmtree(os.path.join(self.root_dir, name))
self.master_manager = MasterManager(self.root_dir, name, name='md_test_{0}'.format(name))
self.master_manager.flush()
pc = ParameterContext('test_param', param_type=QuantityType(self.dtype), fill_value=-1)
self.param_manager = ParameterManager(os.path.join(self.root_dir, name, pc.name), pc.name)
self.param_manager.parameter_context = pc
self.master_manager.create_group(pc.name)
self.param_manager.flush()
self.bricks = {}
self.brick_origins = bricking_utils.calc_brick_origins(self.total_domain, self.brick_sizes)
self.brick_extents, self.rtree_extents = bricking_utils.calc_brick_and_rtree_extents(self.brick_origins,
self.brick_sizes)
self.build_bricks()
self.rtree = RTreeProxy()
for x in BrickingAssessor.rtree_populator(self.rtree_extents, self.brick_extents):
self.rtree.insert(*x)
@classmethod
def rtree_populator(cls, rtree_extents, brick_extents):
for i, e in enumerate(rtree_extents):
yield i, e, brick_extents[i]
def _get_numpy_array(self, shape):
if not isinstance(shape, tuple):
shape = tuple(shape)
return np.arange(utils.prod(shape), dtype=self.dtype).reshape(shape)
def build_bricks(self):
for x in xrange(len(self.brick_origins)):
if not self.use_hdf:
self.bricks[x] = np.empty(self.brick_sizes, dtype=self.dtype)
self.bricks[x].fill(-1)
else:
id = str(x)
fn = '{0}.hdf5'.format(id)
pth = os.path.join(self.param_manager.root_dir, fn)
relpth = os.path.join(self.param_manager.root_dir.replace(self.master_manager.root_dir, '.'), fn)
lnpth = '/{0}/{1}'.format(self.param_manager.parameter_name, id)
self.master_manager.add_external_link(lnpth, relpth, id)
self.bricks[x] = pth
def reset_bricks(self):
for i, arr in enumerate(self.bricks.itervalues()):
if not self.use_hdf:
arr.fill(-1)
else:
with HDFLockingFile(arr, mode='a') as f:
ds = f.require_dataset(str(i), shape=self.brick_sizes, dtype=self.dtype, chunks=None, fillvalue=-1)
ds[:] = -1
def put_values_to_bricks(self, slice_, values):
slice_ = utils.fix_slice(slice_, self.total_domain)
bricks = bricking_utils.get_bricks_from_slice(slice_, self.rtree,
self.total_domain) # this is a list of tuples [(b_id, (bounds...),), ...]
values = np.asanyarray(values)
v_shp = values.shape
log.debug('value_shape: %s', v_shp)
s_shp = utils.slice_shape(slice_, self.total_domain)
log.debug('slice_shape: %s', s_shp)
is_broadcast = False
if v_shp == ():
log.debug('Broadcast!!')
is_broadcast = True
value_slice = ()
elif v_shp != s_shp:
if v_shp == tuple([i for i in s_shp if i != 1]): # Missing dimensions are singleton, just reshape to fit
values = values.reshape(s_shp)
v_shp = values.shape
else:
raise IndexError(
'Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}'.format(
s_shp, v_shp))
else:
value_slice = None
log.debug('value_shape: %s', v_shp)
for b in bricks:
# b is (brick_id, (brick_bounds per dim...),)
bid, bbnds = b
log.debug('Determining slice for brick: %s', b)
bexts = tuple([x + 1 for x in zip(*bbnds)[1]]) # Shift from index to size
log.debug('bid=%s, bbnds=%s, bexts=%s', bid, bbnds, bexts)
brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds)
if None in brick_slice: # Brick does not contain any of the requested indices
log.debug('Brick does not contain any of the requested indices: Move to next brick')
continue
try:
brick_slice = utils.fix_slice(brick_slice, bexts)
except IndexError:
log.debug('Malformed brick_slice: move to next brick')
continue
if not is_broadcast:
value_slice = bricking_utils.get_value_slice_nd(slice_, v_shp, bbnds, brick_slice, brick_mm)
try:
value_slice = utils.fix_slice(value_slice, v_shp)
except IndexError:
log.debug('Malformed value_slice: move to next brick')
continue
log.debug('\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s', b,
utils.slice_shape(brick_slice, bexts), brick_slice, brick_mm,
utils.slice_shape(value_slice, v_shp), value_slice)
v = values[value_slice]
log.debug('\nvalues %s=\n%s', v.shape, v)
if not self.use_hdf:
self.bricks[bid][brick_slice] = v
else:
fi = self.bricks[bid]
with HDFLockingFile(fi, 'a') as f:
ds = f.require_dataset(str(bid), shape=self.brick_sizes, dtype=self.dtype, chunks=None,
fillvalue=-1)
ds[brick_slice] = v
def get_values_from_bricks(self, slice_):
slice_ = utils.fix_slice(slice_, self.total_domain)
bricks = bricking_utils.get_bricks_from_slice(slice_, self.rtree,
self.total_domain) # this is a list of tuples [(b_id, (bounds...),), ...]
ret_shp = utils.slice_shape(slice_, self.total_domain)
ret_arr = np.empty(ret_shp, dtype=self.dtype)
for b in bricks:
bid, bbnds = b
brick_slice, brick_mm = bricking_utils.get_brick_slice_nd(slice_, bbnds)
if None in brick_slice:
continue
ret_slice = bricking_utils.get_value_slice_nd(slice_, ret_shp, bbnds, brick_slice, brick_mm)
if not self.use_hdf:
ret_vals = self.bricks[bid][brick_slice]
else:
fi = self.bricks[bid]
with HDFLockingFile(fi) as f:
ds = f.require_dataset(str(bid), shape=self.brick_sizes, dtype=self.dtype, chunks=None,
fillvalue=-1)
ret_vals = ds[brick_slice]
ret_arr[ret_slice] = ret_vals
ret_arr = ret_arr.squeeze()
if ret_arr.size == 1:
if ret_arr.ndim == 0:
ret_arr = ret_arr[()]
else:
ret_arr = ret_arr[0]
return ret_arr
