def test_array_init(self): # normal initialization store = dict() init_array(store, shape=100, chunks=10) a = Array(store) assert_is_instance(a, Array) eq((100,), a.shape) eq((10,), a.chunks) eq('', a.path) assert_is_none(a.name) assert_is(store, a.store) # initialize at path store = dict() init_array(store, shape=100, chunks=10, path='foo/bar') a = Array(store, path='foo/bar') assert_is_instance(a, Array) eq((100,), a.shape) eq((10,), a.chunks) eq('foo/bar', a.path) eq('/foo/bar', a.name) assert_is(store, a.store) # store not initialized store = dict() with assert_raises(ValueError): Array(store) # group is in the way store = dict() init_group(store, path='baz') with assert_raises(ValueError): Array(store, path='baz')
def create_meta_store(slide: OpenSlide, tilesize: int) -> Dict[str, bytes]: """Creates a dict containing the zarr metadata for the multiscale openslide image.""" store = dict() root_attrs = { "multiscales": [{ "name": Path(slide._filename).name, "datasets": [{ "path": str(i) } for i in range(slide.level_count)], "version": "0.1", }] } init_group(store) init_attrs(store, root_attrs) for i, (x, y) in enumerate(slide.level_dimensions): init_array( store, path=str(i), shape=(y, x, 4), chunks=(tilesize, tilesize, 4), dtype="|u1", compressor=None, ) return store
def test_init_group_overwrite_path(self): # setup path = 'foo/bar' store = self.create_store() meta = dict(shape=(2000,), chunks=(200,), dtype=np.dtype('u1'), compressor=None, fill_value=0, order='F', filters=None) store[array_meta_key] = encode_array_metadata(meta) store[path + '/' + array_meta_key] = encode_array_metadata(meta) # don't overwrite with assert_raises(ValueError): init_group(store, path=path) # do overwrite try: init_group(store, overwrite=True, path=path) except NotImplementedError: pass else: assert array_meta_key not in store assert group_meta_key in store assert (path + '/' + array_meta_key) not in store assert (path + '/' + group_meta_key) in store # should have been overwritten meta = decode_group_metadata(store[path + '/' + group_meta_key]) eq(ZARR_FORMAT, meta['zarr_format'])
def group(store=None, overwrite=False, chunk_store=None, synchronizer=None, path=None): """Create a group. Parameters ---------- store : MutableMapping or string, optional Store or path to directory in file system. overwrite : bool, optional If True, delete any pre-existing data in `store` at `path` before creating the group. chunk_store : MutableMapping, optional Separate storage for chunks. If not provided, `store` will be used for storage of both chunks and metadata. synchronizer : object, optional Array synchronizer. path : string, optional Group path within store. Returns ------- g : zarr.hierarchy.Group Examples -------- Create a group in memory:: >>> import zarr >>> g = zarr.group() >>> g <zarr.hierarchy.Group '/'> Create a group with a different store:: >>> store = zarr.DirectoryStore('data/example.zarr') >>> g = zarr.group(store=store, overwrite=True) >>> g <zarr.hierarchy.Group '/'> """ # handle polymorphic store arg store = _normalize_store_arg(store) path = normalize_storage_path(path) # require group if overwrite or not contains_group(store): init_group(store, overwrite=overwrite, chunk_store=chunk_store, path=path) return Group(store, read_only=False, chunk_store=chunk_store, synchronizer=synchronizer, path=path)
def test_init_group_overwrite_path(self): # setup path = 'foo/bar' store = self.create_store() meta = dict(shape=(2000, ), chunks=(200, ), dtype=np.dtype('u1'), compressor=None, fill_value=0, order='F', filters=None) store[array_meta_key] = encode_array_metadata(meta) store[path + '/' + array_meta_key] = encode_array_metadata(meta) # don't overwrite with pytest.raises(ValueError): init_group(store, path=path) # do overwrite try: init_group(store, overwrite=True, path=path) except NotImplementedError: pass else: assert array_meta_key not in store assert group_meta_key in store assert (path + '/' + array_meta_key) not in store assert (path + '/' + group_meta_key) in store # should have been overwritten meta = decode_group_metadata(store[path + '/' + group_meta_key]) assert ZARR_FORMAT == meta['zarr_format']
def test_array_init(self): # normal initialization store = dict() init_array(store, shape=100, chunks=10) a = Array(store) assert_is_instance(a, Array) eq((100,), a.shape) eq((10,), a.chunks) eq('', a.path) assert_is_none(a.name) assert_is(store, a.store) # initialize at path store = dict() init_array(store, shape=100, chunks=10, path='foo/bar') a = Array(store, path='foo/bar') assert_is_instance(a, Array) eq((100,), a.shape) eq((10,), a.chunks) eq('foo/bar', a.path) eq('/foo/bar', a.name) assert_is(store, a.store) # store not initialized store = dict() with assert_raises(KeyError): Array(store) # group is in the way store = dict() init_group(store, path='baz') with assert_raises(KeyError): Array(store, path='baz')
def test_init_group(self): store = self.create_store() init_group(store) # check metadata assert group_meta_key in store meta = decode_group_metadata(store[group_meta_key]) eq(ZARR_FORMAT, meta['zarr_format'])
def test_init_group(self): store = self.create_store() init_group(store) # check metadata assert group_meta_key in store meta = decode_group_metadata(store[group_meta_key]) assert ZARR_FORMAT == meta['zarr_format']
def create_group(self, store=None, path=None, read_only=False, chunk_store=None, synchronizer=None): # can be overridden in sub-classes if store is None: store, chunk_store = self.create_store() init_group(store, path=path, chunk_store=chunk_store) g = Group(store, path=path, read_only=read_only, chunk_store=chunk_store, synchronizer=synchronizer) return g
def create_group(self, store=None, path=None, read_only=False, chunk_store=None, synchronizer=None): if store is None: store, chunk_store = self.create_store() init_group(store, path=path, chunk_store=chunk_store) synchronizer = ThreadSynchronizer() g = Group(store, path=path, read_only=read_only, chunk_store=chunk_store, synchronizer=synchronizer) return g
def _create_group_nosync(self, name, overwrite=False): path = self._item_path(name) # create terminal group init_group(self._store, path=path, chunk_store=self._chunk_store, overwrite=overwrite) return Group(self._store, path=path, read_only=self._read_only, chunk_store=self._chunk_store, synchronizer=self._synchronizer)
def group(store=None, overwrite=False, chunk_store=None, cache_attrs=True, synchronizer=None, path=None): """Create a group. Parameters ---------- store : MutableMapping or string, optional Store or path to directory in file system. overwrite : bool, optional If True, delete any pre-existing data in `store` at `path` before creating the group. chunk_store : MutableMapping, optional Separate storage for chunks. If not provided, `store` will be used for storage of both chunks and metadata. cache_attrs : bool, optional If True (default), user attributes will be cached for attribute read operations. If False, user attributes are reloaded from the store prior to all attribute read operations. synchronizer : object, optional Array synchronizer. path : string, optional Group path within store. Returns ------- g : zarr.hierarchy.Group Examples -------- Create a group in memory:: >>> import zarr >>> g = zarr.group() >>> g <zarr.hierarchy.Group '/'> Create a group with a different store:: >>> store = zarr.DirectoryStore('data/example.zarr') >>> g = zarr.group(store=store, overwrite=True) >>> g <zarr.hierarchy.Group '/'> """ # handle polymorphic store arg store = _normalize_store_arg(store) path = normalize_storage_path(path) # require group if overwrite or not contains_group(store): init_group(store, overwrite=overwrite, chunk_store=chunk_store, path=path) return Group(store, read_only=False, chunk_store=chunk_store, cache_attrs=cache_attrs, synchronizer=synchronizer, path=path)
def _create_group_nosync(self, name, overwrite=False): path = self._item_path(name) # create terminal group init_group(self._store, path=path, chunk_store=self._chunk_store, overwrite=overwrite) return Group(self._store, path=path, read_only=self._read_only, chunk_store=self._chunk_store, cache_attrs=self.attrs.cache, synchronizer=self._synchronizer)
def _require_group_nosync(self, name, overwrite=False): path = self._item_path(name) # create terminal group if necessary if not contains_group(self._store, path): init_group(store=self._store, path=path, chunk_store=self._chunk_store, overwrite=overwrite) return Group(self._store, path=path, read_only=self._read_only, chunk_store=self._chunk_store, synchronizer=self._synchronizer)
def group(store=None, overwrite=False, chunk_store=None, synchronizer=None, path=None): """Create a group. Parameters ---------- store : MutableMapping or string Store or path to directory in file system. overwrite : bool, optional If True, delete any pre-existing data in `store` at `path` before creating the group. chunk_store : MutableMapping, optional Separate storage for chunks. If not provided, `store` will be used for storage of both chunks and metadata. synchronizer : object, optional Array synchronizer. path : string, optional Group path. Returns ------- g : zarr.hierarchy.Group Examples -------- Create a group in memory:: >>> import zarr >>> g = zarr.group() >>> g Group(/, 0) store: DictStore Create a group with a different store:: >>> store = zarr.DirectoryStore('example') >>> g = zarr.group(store=store, overwrite=True) >>> g Group(/, 0) store: DirectoryStore """ # handle polymorphic store arg store = _handle_store_arg(store) path = normalize_storage_path(path) # require group if overwrite or not contains_group(store): init_group(store, overwrite=overwrite, chunk_store=chunk_store, path=path) return Group(store, read_only=False, chunk_store=chunk_store, synchronizer=synchronizer, path=path)
def create_group(self, store=None, path=None, read_only=False, chunk_store=None, synchronizer=None): if store is None: store, chunk_store = self.create_store() init_group(store, path=path, chunk_store=chunk_store) sync_path = tempfile.mkdtemp() atexit.register(atexit_rmtree, sync_path) synchronizer = ProcessSynchronizer(sync_path) g = Group(store, path=path, read_only=read_only, synchronizer=synchronizer, chunk_store=chunk_store) return g
def test_init_group(self): store = self.create_store() init_group(store) # check metadata assert group_meta_key in store meta = decode_group_metadata(store[group_meta_key]) eq(ZARR_FORMAT, meta['zarr_format']) # check attributes assert attrs_key in store eq(dict(), json.loads(text_type(store[attrs_key], 'ascii')))
def test_group_init_from_dict(chunk_dict): if chunk_dict: store, chunk_store = dict(), dict() else: store, chunk_store = dict(), None init_group(store, path=None, chunk_store=chunk_store) g = Group(store, path=None, read_only=False, chunk_store=chunk_store) assert store is not g.store assert isinstance(g.store, KVStore) if chunk_store is None: assert g.store is g.chunk_store else: assert chunk_store is not g.chunk_store
def test_misc(klass): from zarr3 import V2from3Adapter from zarr.storage import init_group _store = klass() _store.initialize() store = V2from3Adapter(_store) init_group(store) if isinstance(_store, MemoryStoreV3): assert store._v3store._backend == { "meta/root.group": b'{\n "zarr_format": "https://purl.org/zarr/spec/protocol/core/3.0"\n}' } assert store[".zgroup"] == b'{\n "zarr_format": 2\n}'
def test_init_group_overwrite(self): # setup store = self.create_store() store[array_meta_key] = encode_array_metadata( dict(shape=(2000, ), chunks=(200, ), dtype=np.dtype('u1'), compressor=None, fill_value=0, order='F', filters=None)) # don't overwrite array (default) with assert_raises(ValueError): init_group(store) # do overwrite try: init_group(store, overwrite=True) except NotImplementedError: pass else: assert array_meta_key not in store assert group_meta_key in store meta = decode_group_metadata(store[group_meta_key]) eq(ZARR_FORMAT, meta['zarr_format']) # don't overwrite group with assert_raises(ValueError): init_group(store)
def test_init_group_overwrite_chunk_store(self): # setup store = self.create_store() chunk_store = self.create_store() store[array_meta_key] = encode_array_metadata( dict(shape=(2000, ), chunks=(200, ), dtype=np.dtype('u1'), compressor=None, fill_value=0, filters=None, order='F')) chunk_store['foo'] = b'bar' chunk_store['baz'] = b'quux' # don't overwrite array (default) with pytest.raises(ValueError): init_group(store, chunk_store=chunk_store) # do overwrite try: init_group(store, overwrite=True, chunk_store=chunk_store) except NotImplementedError: pass else: assert array_meta_key not in store assert group_meta_key in store meta = decode_group_metadata(store[group_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert 'foo' not in chunk_store assert 'baz' not in chunk_store # don't overwrite group with pytest.raises(ValueError): init_group(store)
def test_init_group_overwrite(self): # setup store = self.create_store() store[array_meta_key] = encode_array_metadata( dict(shape=(2000,), chunks=(200,), dtype=np.dtype('u1'), compressor=None, fill_value=0, order='F', filters=None) ) # don't overwrite array (default) with assert_raises(ValueError): init_group(store) # do overwrite try: init_group(store, overwrite=True) except NotImplementedError: pass else: assert array_meta_key not in store assert group_meta_key in store meta = decode_group_metadata(store[group_meta_key]) eq(ZARR_FORMAT, meta['zarr_format']) # don't overwrite group with assert_raises(ValueError): init_group(store)
def open_group(store, mode='a', cache_attrs=True, synchronizer=None, path=None): """Open a group using file-mode-like semantics. Parameters ---------- store : MutableMapping or string Store or path to directory in file system or name of zip file. mode : {'r', 'r+', 'a', 'w', 'w-'}, optional Persistence mode: 'r' means read only (must exist); 'r+' means read/write (must exist); 'a' means read/write (create if doesn't exist); 'w' means create (overwrite if exists); 'w-' means create (fail if exists). cache_attrs : bool, optional If True (default), user attributes will be cached for attribute read operations. If False, user attributes are reloaded from the store prior to all attribute read operations. synchronizer : object, optional Array synchronizer. path : string, optional Group path within store. Returns ------- g : zarr.hierarchy.Group Examples -------- >>> import zarr >>> root = zarr.open_group('data/example.zarr', mode='w') >>> foo = root.create_group('foo') >>> bar = root.create_group('bar') >>> root <zarr.hierarchy.Group '/'> >>> root2 = zarr.open_group('data/example.zarr', mode='a') >>> root2 <zarr.hierarchy.Group '/'> >>> root == root2 True """ # handle polymorphic store arg store = _normalize_store_arg(store) path = normalize_storage_path(path) # ensure store is initialized if mode in ['r', 'r+']: if contains_array(store, path=path): err_contains_array(path) elif not contains_group(store, path=path): err_group_not_found(path) elif mode == 'w': init_group(store, overwrite=True, path=path) elif mode == 'a': if contains_array(store, path=path): err_contains_array(path) if not contains_group(store, path=path): init_group(store, path=path) elif mode in ['w-', 'x']: if contains_array(store, path=path): err_contains_array(path) elif contains_group(store, path=path): err_contains_group(path) else: init_group(store, path=path) # determine read only status read_only = mode == 'r' return Group(store, read_only=read_only, cache_attrs=cache_attrs, synchronizer=synchronizer, path=path)
def open_group(store=None, mode='a', synchronizer=None, path=None): """Open a group using mode-like semantics. Parameters ---------- store : MutableMapping or string Store or path to directory in file system. mode : {'r', 'r+', 'a', 'w', 'w-'} Persistence mode: 'r' means read only (must exist); 'r+' means read/write (must exist); 'a' means read/write (create if doesn't exist); 'w' means create (overwrite if exists); 'w-' means create (fail if exists). synchronizer : object, optional Array synchronizer. path : string, optional Group path. Returns ------- g : zarr.hierarchy.Group Examples -------- >>> import zarr >>> root = zarr.open_group('example', mode='w') >>> foo = root.create_group('foo') >>> bar = root.create_group('bar') >>> root Group(/, 2) groups: 2; bar, foo store: DirectoryStore >>> root2 = zarr.open_group('example', mode='a') >>> root2 Group(/, 2) groups: 2; bar, foo store: DirectoryStore >>> root == root2 True """ # handle polymorphic store arg store = _handle_store_arg(store) path = normalize_storage_path(path) # ensure store is initialized if mode in ['r', 'r+']: if contains_array(store, path=path): err_contains_array(path) elif not contains_group(store, path=path): err_group_not_found(path) elif mode == 'w': init_group(store, overwrite=True, path=path) elif mode == 'a': if contains_array(store, path=path): err_contains_array(path) if not contains_group(store, path=path): init_group(store, path=path) elif mode in ['w-', 'x']: if contains_array(store, path=path): err_contains_array(path) elif contains_group(store, path=path): err_contains_group(path) else: init_group(store, path=path) # determine read only status read_only = mode == 'r' return Group(store, read_only=read_only, synchronizer=synchronizer, path=path)
def group(store=None, overwrite=False, chunk_store=None, cache_attrs=True, synchronizer=None, path=None, *, zarr_version=None): """Create a group. Parameters ---------- store : MutableMapping or string, optional Store or path to directory in file system. overwrite : bool, optional If True, delete any pre-existing data in `store` at `path` before creating the group. chunk_store : MutableMapping, optional Separate storage for chunks. If not provided, `store` will be used for storage of both chunks and metadata. cache_attrs : bool, optional If True (default), user attributes will be cached for attribute read operations. If False, user attributes are reloaded from the store prior to all attribute read operations. synchronizer : object, optional Array synchronizer. path : string, optional Group path within store. Returns ------- g : zarr.hierarchy.Group Examples -------- Create a group in memory:: >>> import zarr >>> g = zarr.group() >>> g <zarr.hierarchy.Group '/'> Create a group with a different store:: >>> store = zarr.DirectoryStore('data/example.zarr') >>> g = zarr.group(store=store, overwrite=True) >>> g <zarr.hierarchy.Group '/'> """ # handle polymorphic store arg store = _normalize_store_arg(store, zarr_version=zarr_version) if zarr_version is None: zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION) if zarr_version == 3 and path is None: raise ValueError(f"path must be provided for a v{zarr_version} group") path = normalize_storage_path(path) if zarr_version == 2: requires_init = overwrite or not contains_group(store) elif zarr_version == 3: requires_init = overwrite or not contains_group(store, path) if requires_init: init_group(store, overwrite=overwrite, chunk_store=chunk_store, path=path) return Group(store, read_only=False, chunk_store=chunk_store, cache_attrs=cache_attrs, synchronizer=synchronizer, path=path, zarr_version=zarr_version)
def open_group(store=None, mode='a', cache_attrs=True, synchronizer=None, path=None, chunk_store=None, storage_options=None): """Open a group using file-mode-like semantics. Parameters ---------- store : MutableMapping or string, optional Store or path to directory in file system or name of zip file. mode : {'r', 'r+', 'a', 'w', 'w-'}, optional Persistence mode: 'r' means read only (must exist); 'r+' means read/write (must exist); 'a' means read/write (create if doesn't exist); 'w' means create (overwrite if exists); 'w-' means create (fail if exists). cache_attrs : bool, optional If True (default), user attributes will be cached for attribute read operations. If False, user attributes are reloaded from the store prior to all attribute read operations. synchronizer : object, optional Array synchronizer. path : string, optional Group path within store. chunk_store : MutableMapping or string, optional Store or path to directory in file system or name of zip file. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. Returns ------- g : zarr.hierarchy.Group Examples -------- >>> import zarr >>> root = zarr.open_group('data/example.zarr', mode='w') >>> foo = root.create_group('foo') >>> bar = root.create_group('bar') >>> root <zarr.hierarchy.Group '/'> >>> root2 = zarr.open_group('data/example.zarr', mode='a') >>> root2 <zarr.hierarchy.Group '/'> >>> root == root2 True """ # handle polymorphic store arg clobber = mode != "r" store = _normalize_store_arg( store, clobber=clobber, storage_options=storage_options, mode=mode ) if chunk_store is not None: chunk_store = _normalize_store_arg(chunk_store, clobber=clobber, storage_options=storage_options) path = normalize_storage_path(path) # ensure store is initialized if mode in ['r', 'r+']: if contains_array(store, path=path): raise ContainsArrayError(path) elif not contains_group(store, path=path): raise GroupNotFoundError(path) elif mode == 'w': init_group(store, overwrite=True, path=path, chunk_store=chunk_store) elif mode == 'a': if contains_array(store, path=path): raise ContainsArrayError(path) if not contains_group(store, path=path): init_group(store, path=path, chunk_store=chunk_store) elif mode in ['w-', 'x']: if contains_array(store, path=path): raise ContainsArrayError(path) elif contains_group(store, path=path): raise ContainsGroupError(path) else: init_group(store, path=path, chunk_store=chunk_store) # determine read only status read_only = mode == 'r' return Group(store, read_only=read_only, cache_attrs=cache_attrs, synchronizer=synchronizer, path=path, chunk_store=chunk_store)