Ejemplo n.º 1
0
    def __init__(self, root, guid, name=None, tdom=None, sdom=None, mode=None, bricking_scheme=None, inline_data_writes=True, auto_flush_values=True, value_caching=True, coverage_type=None, **kwargs):
        """
        Constructor for PersistenceLayer

        @param root The <root> component of the filesystem path for the coverage (/<root>/<guid>)
        @param guid The <guid> component of the filesystem path for the coverage (/<root>/<guid>)
        @param name CoverageModel's name persisted to the metadata attribute in the master HDF5 file
        @param tdom Concrete instance of AbstractDomain for the temporal domain component
        @param sdom Concrete instance of AbstractDomain for the spatial domain component
        @param bricking_scheme  A dictionary containing the brick and chunk sizes
        @param auto_flush_values    True = Values flushed to HDF5 files automatically, False = Manual
        @param value_caching  if True (default), value requests should be cached for rapid duplicate retrieval
        @param kwargs
        @return None
        """

        log.debug('Persistence GUID: %s', guid)
        root = '.' if root is ('' or None) else root

#        self.master_manager = MasterManager(root, guid, name=name, tdom=tdom, sdom=sdom, global_bricking_scheme=bricking_scheme, parameter_bounds=None, coverage_type=coverage_type, **kwargs)
        self.master_manager = MetadataManagerFactory.buildMetadataManager(root, guid, name=name, tdom=tdom, sdom=sdom, global_bricking_scheme=bricking_scheme, parameter_bounds=None, coverage_type=coverage_type, **kwargs)

        self.mode = mode
        if not hasattr(self.master_manager, 'auto_flush_values'):
            self.master_manager.auto_flush_values = auto_flush_values
        if not hasattr(self.master_manager, 'inline_data_writes'):
            self.master_manager.inline_data_writes = inline_data_writes
        if not hasattr(self.master_manager, 'value_caching'):
            self.master_manager.value_caching = value_caching
        if not hasattr(self.master_manager, 'coverage_type'):
            self.master_manager.coverage_type = coverage_type

        # TODO: This is not done correctly
        if tdom != None:
            self._init_master(tdom.shape.extents, bricking_scheme)

        self.value_list = {}

        self.parameter_metadata = {} # {parameter_name: [brick_list, parameter_domains, rtree]}

        for pname in self.param_groups:
            log.debug('parameter group: %s', pname)
            self.parameter_metadata[pname] = ParameterManager(os.path.join(self.root_dir, self.guid, pname), pname)

        if self.mode != 'r':
            if self.master_manager.is_dirty():
                self.master_manager.flush()

        if self.mode == 'r' or self.inline_data_writes:
            self.brick_dispatcher = None
        else:
            self.brick_dispatcher = BrickWriterDispatcher(self.write_failure_callback)
            self.brick_dispatcher.run()

        self._closed = False

        log.debug('Persistence Layer Successfully Initialized')
    def __init__(self, root, guid, name=None, mode=None, inline_data_writes=True, auto_flush_values=True,
                 bricking_scheme=None, brick_dispatcher=None, value_caching=True, coverage_type=None,
                 alignment_parameter=AxisTypeEnum.TIME.lower(), storage_name=None, **kwargs):
        """
        Constructor for PersistenceLayer

        @param root The <root> component of the filesystem path for the coverage (/<root>/<guid>)
        @param guid The <guid> component of the filesystem path for the coverage (/<root>/<guid>)
        @param name CoverageModel's name persisted to the metadata attribute in the master HDF5 file
        @param auto_flush_values    True = Values flushed to HDF5 files automatically, False = Manual
        @param value_caching  if True (default), value requests should be cached for rapid duplicate retrieval
        @param kwargs
        @return None
        """

        log.debug('Persistence GUID: %s', guid)
        root = '.' if root is ('' or None) else root

        self.brick_dispatcher=brick_dispatcher
        self.bricking_scheme=bricking_scheme
        self.master_manager = MetadataManagerFactory.buildMetadataManager(root, guid, name=name, parameter_bounds=None, coverage_type=coverage_type, **kwargs)

        self.mode = mode
        if not hasattr(self.master_manager, 'auto_flush_values'):
            self.master_manager.auto_flush_values = auto_flush_values
        if not hasattr(self.master_manager, 'inline_data_writes'):
            self.master_manager.inline_data_writes = inline_data_writes
        if not hasattr(self.master_manager, 'value_caching'):
            self.master_manager.value_caching = value_caching
        if not hasattr(self.master_manager, 'coverage_type'):
            self.master_manager.coverage_type = coverage_type
        if not hasattr(self.master_manager, 'parameter_metadata'):
            self.master_manager.parameter_metadata = {}

        self.value_list = {}

        self.spans = {}
        self.span_list = []
        self.storage_name = storage_name

        for pname in self.param_groups:
            log.debug('parameter group: %s', pname)
            if pname not in self.master_manager.parameter_metadata:
                self.master_manager.parameter_metadata[pname] = MetadataManagerFactory.buildParameterManager(os.path.join(self.root_dir, self.guid, pname), pname)

        if self.mode != 'r':
            if self.master_manager.is_dirty():
                self.master_manager.flush()

        self._closed = False
        if isinstance(alignment_parameter, basestring):
            self.alignment_parameter = alignment_parameter
        else:
            raise TypeError("alignment_parameter arg must implement %s.  Found %s", (basestring.__name__, type(alignment_parameter).__name__))

        log.debug('Persistence Layer Successfully Initialized')
    def __init__(
        self,
        total_domain=(10, 10),
        brick_size=5,
        use_hdf=False,
        root_dir="test_data/multi_dim_trials",
        guid=None,
        dtype="int16",
    ):
        self.total_domain = total_domain
        self.brick_sizes = tuple(brick_size for x in total_domain)
        self.use_hdf = use_hdf
        self.dtype = np.dtype(dtype).name
        if self.use_hdf:
            self.guid = guid or create_guid()
            name = "%s_%s" % (self.guid, self.dtype)
            self.root_dir = root_dir
            if not os.path.exists(self.root_dir):
                os.makedirs(self.root_dir)

            if os.path.exists(os.path.join(self.root_dir, name)):
                shutil.rmtree(os.path.join(self.root_dir, name))

            #            self.master_manager = MasterManager(self.root_dir, name, name='md_test_{0}'.format(name))
            self.master_manager = MetadataManagerFactory.buildMetadataManager(
                self.root_dir, name, name="md_test_{0}".format(name)
            )

            self.master_manager.flush()

            pc = ParameterContext("test_param", param_type=QuantityType(self.dtype), fill_value=-1)
            self.param_manager = ParameterManager(os.path.join(self.root_dir, name, pc.name), pc.name)
            self.param_manager.parameter_context = pc
            self.master_manager.create_group(pc.name)

            self.param_manager.flush()

        self.bricks = {}

        self.brick_origins = bricking_utils.calc_brick_origins(self.total_domain, self.brick_sizes)
        self.brick_extents, self.rtree_extents = bricking_utils.calc_brick_and_rtree_extents(
            self.brick_origins, self.brick_sizes
        )
        self.build_bricks()

        self.rtree = RTreeProxy()
        for x in BrickingAssessor.rtree_populator(self.rtree_extents, self.brick_extents):
            self.rtree.insert(*x)
Ejemplo n.º 4
0
    def __init__(self, root, guid, name=None, param_dict=None, mode=None, coverage_type=None, **kwargs):
        root = '.' if root is ('' or None) else root

#        self.master_manager = MasterManager(root_dir=root, guid=guid, name=name, param_dict=param_dict,
        self.master_manager = MetadataManagerFactory.buildMetadataManager(root, guid, name=name, param_dict=param_dict,
                                            parameter_bounds=None, tree_rank=2, coverage_type=coverage_type, **kwargs)

        if not hasattr(self.master_manager, 'coverage_type'):
            self.master_manager.coverage_type = coverage_type

        self.mode = mode

        if self.mode != 'r':
            self.master_manager.flush()

        self._closed = False