def __init__(self, root, guid, name=None, tdom=None, sdom=None, mode=None, bricking_scheme=None, inline_data_writes=True, auto_flush_values=True, value_caching=True, coverage_type=None, **kwargs):
"""
Constructor for PersistenceLayer
@param root The <root> component of the filesystem path for the coverage (/<root>/<guid>)
@param guid The <guid> component of the filesystem path for the coverage (/<root>/<guid>)
@param name CoverageModel's name persisted to the metadata attribute in the master HDF5 file
@param tdom Concrete instance of AbstractDomain for the temporal domain component
@param sdom Concrete instance of AbstractDomain for the spatial domain component
@param bricking_scheme A dictionary containing the brick and chunk sizes
@param auto_flush_values True = Values flushed to HDF5 files automatically, False = Manual
@param value_caching if True (default), value requests should be cached for rapid duplicate retrieval
@param kwargs
@return None
"""
log.debug('Persistence GUID: %s', guid)
root = '.' if root is ('' or None) else root
self.master_manager = MasterManager(root, guid, name=name, tdom=tdom, sdom=sdom, global_bricking_scheme=bricking_scheme, parameter_bounds=None, coverage_type=coverage_type)
self.mode = mode
if not hasattr(self.master_manager, 'auto_flush_values'):
self.master_manager.auto_flush_values = auto_flush_values
if not hasattr(self.master_manager, 'inline_data_writes'):
self.master_manager.inline_data_writes = inline_data_writes
if not hasattr(self.master_manager, 'value_caching'):
self.master_manager.value_caching = value_caching
if not hasattr(self.master_manager, 'coverage_type'):
self.master_manager.coverage_type = coverage_type
# TODO: This is not done correctly
if tdom != None:
self._init_master(tdom.shape.extents, bricking_scheme)
self.value_list = {}
self.parameter_metadata = {} # {parameter_name: [brick_list, parameter_domains, rtree]}
for pname in self.param_groups:
log.debug('parameter group: %s', pname)
self.parameter_metadata[pname] = ParameterManager(os.path.join(self.root_dir, self.guid, pname), pname)
if self.mode != 'r':
if self.master_manager.is_dirty():
self.master_manager.flush()
if self.mode == 'r' or self.inline_data_writes:
self.brick_dispatcher = None
else:
self.brick_dispatcher = BrickWriterDispatcher(self.write_failure_callback)
self.brick_dispatcher.run()
self._closed = False
log.debug('Persistence Layer Successfully Initialized')
def __init__(self,
total_domain=(10, 10),
brick_size=5,
use_hdf=False,
root_dir='test_data/multi_dim_trials',
guid=None,
dtype='int16'):
self.total_domain = total_domain
self.brick_sizes = tuple(brick_size for x in total_domain)
self.use_hdf = use_hdf
self.dtype = np.dtype(dtype).name
if self.use_hdf:
self.guid = guid or create_guid()
name = '%s_%s' % (self.guid, self.dtype)
self.root_dir = root_dir
if not os.path.exists(self.root_dir):
os.makedirs(self.root_dir)
if os.path.exists(os.path.join(self.root_dir, name)):
shutil.rmtree(os.path.join(self.root_dir, name))
self.master_manager = MasterManager(
self.root_dir, name, name='md_test_{0}'.format(name))
self.master_manager.flush()
pc = ParameterContext('test_param',
param_type=QuantityType(self.dtype),
fill_value=-1)
self.param_manager = ParameterManager(
os.path.join(self.root_dir, name, pc.name), pc.name)
self.param_manager.parameter_context = pc
self.master_manager.create_group(pc.name)
self.param_manager.flush()
self.bricks = {}
self.brick_origins = bricking_utils.calc_brick_origins(
self.total_domain, self.brick_sizes)
self.brick_extents, self.rtree_extents = bricking_utils.calc_brick_and_rtree_extents(
self.brick_origins, self.brick_sizes)
self.build_bricks()
self.rtree = RTreeProxy()
for x in BrickingAssessor.rtree_populator(self.rtree_extents,
self.brick_extents):
self.rtree.insert(*x)
def init_parameter(self, parameter_context, bricking_scheme):
"""
Initializes a parameter using a ParameterContext object and a bricking
scheme for that parameter
@param parameter_context ParameterContext object describing the parameter to initialize
@param bricking_scheme A dictionary containing the brick and chunk sizes
@return A PersistedStorage object
"""
if self.mode == 'r':
raise IOError('PersistenceLayer not open for writing: mode == \'{0}\''.format(self.mode))
parameter_name = parameter_context.name
self.global_bricking_scheme = bricking_scheme
pm = ParameterManager(os.path.join(self.root_dir, self.guid, parameter_name), parameter_name, read_only=False)
self.parameter_metadata[parameter_name] = pm
pm.parameter_context = parameter_context
log.debug('Initialize %s', parameter_name)
self.master_manager.create_group(parameter_name)
if parameter_context.param_type._value_class == 'SparseConstantValue':
v = SparsePersistedStorage(pm, self.master_manager, self.brick_dispatcher, dtype=parameter_context.param_type.storage_encoding, fill_value=parameter_context.param_type.fill_value, mode=self.mode, inline_data_writes=self.inline_data_writes, auto_flush=self.auto_flush_values)
else:
v = PersistedStorage(pm, self.master_manager, self.brick_dispatcher, dtype=parameter_context.param_type.storage_encoding, fill_value=parameter_context.param_type.fill_value, mode=self.mode, inline_data_writes=self.inline_data_writes, auto_flush=self.auto_flush_values)
self.value_list[parameter_name] = v
# CBM TODO: Consider making this optional and bulk-flushing from the coverage after all parameters have been initialized
# No need to check if they're dirty, we know they are!
pm.flush()
# Put the pm into read_only mode
pm.read_only = True
# If there are already bricks, ensure there are appropriate links for this new parameter
for brick_guid in self.master_manager.brick_list:
brick_file_name = '{0}.hdf5'.format(brick_guid)
self._add_brick_link(parameter_name, brick_guid, brick_file_name)
self.master_manager.flush()
return v
