def __init__(self, root, guid, name=None, mode=None, inline_data_writes=True, auto_flush_values=True,
bricking_scheme=None, brick_dispatcher=None, value_caching=True, coverage_type=None,
alignment_parameter=AxisTypeEnum.TIME.lower(), storage_name=None, **kwargs):
"""
Constructor for PersistenceLayer
@param root The <root> component of the filesystem path for the coverage (/<root>/<guid>)
@param guid The <guid> component of the filesystem path for the coverage (/<root>/<guid>)
@param name CoverageModel's name persisted to the metadata attribute in the master HDF5 file
@param auto_flush_values True = Values flushed to HDF5 files automatically, False = Manual
@param value_caching if True (default), value requests should be cached for rapid duplicate retrieval
@param kwargs
@return None
"""
log.debug('Persistence GUID: %s', guid)
root = '.' if root is ('' or None) else root
self.brick_dispatcher=brick_dispatcher
self.bricking_scheme=bricking_scheme
self.master_manager = MetadataManagerFactory.buildMetadataManager(root, guid, name=name, parameter_bounds=None, coverage_type=coverage_type, **kwargs)
self.mode = mode
if not hasattr(self.master_manager, 'auto_flush_values'):
self.master_manager.auto_flush_values = auto_flush_values
if not hasattr(self.master_manager, 'inline_data_writes'):
self.master_manager.inline_data_writes = inline_data_writes
if not hasattr(self.master_manager, 'value_caching'):
self.master_manager.value_caching = value_caching
if not hasattr(self.master_manager, 'coverage_type'):
self.master_manager.coverage_type = coverage_type
if not hasattr(self.master_manager, 'parameter_metadata'):
self.master_manager.parameter_metadata = {}
self.value_list = {}
self.spans = {}
self.span_list = []
self.storage_name = storage_name
for pname in self.param_groups:
log.debug('parameter group: %s', pname)
if pname not in self.master_manager.parameter_metadata:
self.master_manager.parameter_metadata[pname] = MetadataManagerFactory.buildParameterManager(os.path.join(self.root_dir, self.guid, pname), pname)
if self.mode != 'r':
if self.master_manager.is_dirty():
self.master_manager.flush()
self._closed = False
if isinstance(alignment_parameter, basestring):
self.alignment_parameter = alignment_parameter
else:
raise TypeError("alignment_parameter arg must implement %s. Found %s", (basestring.__name__, type(alignment_parameter).__name__))
log.debug('Persistence Layer Successfully Initialized')
def __init__(self, root, guid, name=None, tdom=None, sdom=None, mode=None, bricking_scheme=None, inline_data_writes=True, auto_flush_values=True, value_caching=True, coverage_type=None, **kwargs):
"""
Constructor for PersistenceLayer
@param root The <root> component of the filesystem path for the coverage (/<root>/<guid>)
@param guid The <guid> component of the filesystem path for the coverage (/<root>/<guid>)
@param name CoverageModel's name persisted to the metadata attribute in the master HDF5 file
@param tdom Concrete instance of AbstractDomain for the temporal domain component
@param sdom Concrete instance of AbstractDomain for the spatial domain component
@param bricking_scheme A dictionary containing the brick and chunk sizes
@param auto_flush_values True = Values flushed to HDF5 files automatically, False = Manual
@param value_caching if True (default), value requests should be cached for rapid duplicate retrieval
@param kwargs
@return None
"""
log.debug('Persistence GUID: %s', guid)
root = '.' if root is ('' or None) else root
# self.master_manager = MasterManager(root, guid, name=name, tdom=tdom, sdom=sdom, global_bricking_scheme=bricking_scheme, parameter_bounds=None, coverage_type=coverage_type, **kwargs)
self.master_manager = MetadataManagerFactory.buildMetadataManager(root, guid, name=name, tdom=tdom, sdom=sdom, global_bricking_scheme=bricking_scheme, parameter_bounds=None, coverage_type=coverage_type, **kwargs)
self.mode = mode
if not hasattr(self.master_manager, 'auto_flush_values'):
self.master_manager.auto_flush_values = auto_flush_values
if not hasattr(self.master_manager, 'inline_data_writes'):
self.master_manager.inline_data_writes = inline_data_writes
if not hasattr(self.master_manager, 'value_caching'):
self.master_manager.value_caching = value_caching
if not hasattr(self.master_manager, 'coverage_type'):
self.master_manager.coverage_type = coverage_type
# TODO: This is not done correctly
if tdom != None:
self._init_master(tdom.shape.extents, bricking_scheme)
self.value_list = {}
self.parameter_metadata = {} # {parameter_name: [brick_list, parameter_domains, rtree]}
for pname in self.param_groups:
log.debug('parameter group: %s', pname)
self.parameter_metadata[pname] = ParameterManager(os.path.join(self.root_dir, self.guid, pname), pname)
if self.mode != 'r':
if self.master_manager.is_dirty():
self.master_manager.flush()
if self.mode == 'r' or self.inline_data_writes:
self.brick_dispatcher = None
else:
self.brick_dispatcher = BrickWriterDispatcher(self.write_failure_callback)
self.brick_dispatcher.run()
self._closed = False
log.debug('Persistence Layer Successfully Initialized')
def __init__(
self,
total_domain=(10, 10),
brick_size=5,
use_hdf=False,
root_dir="test_data/multi_dim_trials",
guid=None,
dtype="int16",
):
self.total_domain = total_domain
self.brick_sizes = tuple(brick_size for x in total_domain)
self.use_hdf = use_hdf
self.dtype = np.dtype(dtype).name
if self.use_hdf:
self.guid = guid or create_guid()
name = "%s_%s" % (self.guid, self.dtype)
self.root_dir = root_dir
if not os.path.exists(self.root_dir):
os.makedirs(self.root_dir)
if os.path.exists(os.path.join(self.root_dir, name)):
shutil.rmtree(os.path.join(self.root_dir, name))
# self.master_manager = MasterManager(self.root_dir, name, name='md_test_{0}'.format(name))
self.master_manager = MetadataManagerFactory.buildMetadataManager(
self.root_dir, name, name="md_test_{0}".format(name)
)
self.master_manager.flush()
pc = ParameterContext("test_param", param_type=QuantityType(self.dtype), fill_value=-1)
self.param_manager = ParameterManager(os.path.join(self.root_dir, name, pc.name), pc.name)
self.param_manager.parameter_context = pc
self.master_manager.create_group(pc.name)
self.param_manager.flush()
self.bricks = {}
self.brick_origins = bricking_utils.calc_brick_origins(self.total_domain, self.brick_sizes)
self.brick_extents, self.rtree_extents = bricking_utils.calc_brick_and_rtree_extents(
self.brick_origins, self.brick_sizes
)
self.build_bricks()
self.rtree = RTreeProxy()
for x in BrickingAssessor.rtree_populator(self.rtree_extents, self.brick_extents):
self.rtree.insert(*x)
def __init__(self, root, guid, name=None, param_dict=None, mode=None, coverage_type=None, **kwargs):
root = '.' if root is ('' or None) else root
# self.master_manager = MasterManager(root_dir=root, guid=guid, name=name, param_dict=param_dict,
self.master_manager = MetadataManagerFactory.buildMetadataManager(root, guid, name=name, param_dict=param_dict,
parameter_bounds=None, tree_rank=2, coverage_type=coverage_type, **kwargs)
if not hasattr(self.master_manager, 'coverage_type'):
self.master_manager.coverage_type = coverage_type
self.mode = mode
if self.mode != 'r':
self.master_manager.flush()
self._closed = False
def init_parameter(self, parameter_context, bricking_scheme):
"""
Initializes a parameter using a ParameterContext object and a bricking
scheme for that parameter
@param parameter_context ParameterContext object describing the parameter to initialize
@param bricking_scheme A dictionary containing the brick and chunk sizes
@return A PersistedStorage object
"""
if self.mode == 'r':
raise IOError('PersistenceLayer not open for writing: mode == \'{0}\''.format(self.mode))
parameter_name = parameter_context.name
pm = MetadataManagerFactory.buildParameterManager(self.guid, parameter_name, read_only=False)
self.master_manager.parameter_metadata[parameter_name] = pm
pm.parameter_context = parameter_context
log.debug('Initialize %s', parameter_name)
self.master_manager.create_group(parameter_name)
v = PostgresPersistedStorage(pm, metadata_manager=self.master_manager,
parameter_context=parameter_context,
dtype=parameter_context.param_type.storage_encoding,
fill_value=parameter_context.param_type.fill_value,
mode=self.mode)
self.value_list[parameter_name] = v
# CBM TODO: Consider making this optional and bulk-flushing from the coverage after all parameters have been initialized
# No need to check if they're dirty, we know they are!
pm.flush()
# Put the pm into read_only mode
pm.read_only = True
self.master_manager.flush()
return v
def get_coverage_type(file_dir, guid):
return MetadataManagerFactory.getCoverageType(file_dir,guid)
def dir_exists(file_dir):
return MetadataManagerFactory.dirExists(file_dir)
def is_persisted(file_dir, guid):
return MetadataManagerFactory.isPersisted(file_dir,guid)
