def __init__(self, root, guid, name=None, tdom=None, sdom=None, mode=None, bricking_scheme=None, inline_data_writes=True, auto_flush_values=True, value_caching=True, coverage_type=None, **kwargs): """ Constructor for PersistenceLayer @param root The <root> component of the filesystem path for the coverage (/<root>/<guid>) @param guid The <guid> component of the filesystem path for the coverage (/<root>/<guid>) @param name CoverageModel's name persisted to the metadata attribute in the master HDF5 file @param tdom Concrete instance of AbstractDomain for the temporal domain component @param sdom Concrete instance of AbstractDomain for the spatial domain component @param bricking_scheme A dictionary containing the brick and chunk sizes @param auto_flush_values True = Values flushed to HDF5 files automatically, False = Manual @param value_caching if True (default), value requests should be cached for rapid duplicate retrieval @param kwargs @return None """ log.debug('Persistence GUID: %s', guid) root = '.' if root is ('' or None) else root self.master_manager = MasterManager(root, guid, name=name, tdom=tdom, sdom=sdom, global_bricking_scheme=bricking_scheme, parameter_bounds=None, coverage_type=coverage_type) self.mode = mode if not hasattr(self.master_manager, 'auto_flush_values'): self.master_manager.auto_flush_values = auto_flush_values if not hasattr(self.master_manager, 'inline_data_writes'): self.master_manager.inline_data_writes = inline_data_writes if not hasattr(self.master_manager, 'value_caching'): self.master_manager.value_caching = value_caching if not hasattr(self.master_manager, 'coverage_type'): self.master_manager.coverage_type = coverage_type # TODO: This is not done correctly if tdom != None: self._init_master(tdom.shape.extents, bricking_scheme) self.value_list = {} self.parameter_metadata = {} # {parameter_name: [brick_list, parameter_domains, rtree]} for pname in self.param_groups: log.debug('parameter group: %s', pname) self.parameter_metadata[pname] = ParameterManager(os.path.join(self.root_dir, self.guid, pname), pname) if self.mode != 'r': if self.master_manager.is_dirty(): self.master_manager.flush() if self.mode == 'r' or self.inline_data_writes: self.brick_dispatcher = None else: self.brick_dispatcher = BrickWriterDispatcher(self.write_failure_callback) self.brick_dispatcher.run() self._closed = False log.debug('Persistence Layer Successfully Initialized')
def __init__(self, root, guid, name=None, param_dict=None, mode=None, coverage_type=None, **kwargs): root = '.' if root is ('' or None) else root self.master_manager = MasterManager(root_dir=root, guid=guid, name=name, param_dict=param_dict, parameter_bounds=None, tree_rank=2, coverage_type=coverage_type, **kwargs) if not hasattr(self.master_manager, 'coverage_type'): self.master_manager.coverage_type = coverage_type self.mode = mode if self.mode != 'r': self.master_manager.flush() self._closed = False
def __init__(self, total_domain=(10, 10), brick_size=5, use_hdf=False, root_dir='test_data/multi_dim_trials', guid=None, dtype='int16'): self.total_domain = total_domain self.brick_sizes = tuple(brick_size for x in total_domain) self.use_hdf = use_hdf self.dtype = np.dtype(dtype).name if self.use_hdf: self.guid = guid or create_guid() name = '%s_%s' % (self.guid, self.dtype) self.root_dir = root_dir if not os.path.exists(self.root_dir): os.makedirs(self.root_dir) if os.path.exists(os.path.join(self.root_dir, name)): shutil.rmtree(os.path.join(self.root_dir, name)) self.master_manager = MasterManager( self.root_dir, name, name='md_test_{0}'.format(name)) self.master_manager.flush() pc = ParameterContext('test_param', param_type=QuantityType(self.dtype), fill_value=-1) self.param_manager = ParameterManager( os.path.join(self.root_dir, name, pc.name), pc.name) self.param_manager.parameter_context = pc self.master_manager.create_group(pc.name) self.param_manager.flush() self.bricks = {} self.brick_origins = bricking_utils.calc_brick_origins( self.total_domain, self.brick_sizes) self.brick_extents, self.rtree_extents = bricking_utils.calc_brick_and_rtree_extents( self.brick_origins, self.brick_sizes) self.build_bricks() self.rtree = RTreeProxy() for x in BrickingAssessor.rtree_populator(self.rtree_extents, self.brick_extents): self.rtree.insert(*x)