def test_group_init_errors_2(self): store, chunk_store = self.create_store() init_array(store, shape=1000, chunks=100, chunk_store=chunk_store) # array blocks group with pytest.raises(ValueError): Group(store, chunk_store=chunk_store) store.close()
def test_array_init(self): # normal initialization store = dict() init_array(store, shape=100, chunks=10) a = Array(store) assert_is_instance(a, Array) eq((100,), a.shape) eq((10,), a.chunks) eq('', a.path) assert_is_none(a.name) assert_is(store, a.store) # initialize at path store = dict() init_array(store, shape=100, chunks=10, path='foo/bar') a = Array(store, path='foo/bar') assert_is_instance(a, Array) eq((100,), a.shape) eq((10,), a.chunks) eq('foo/bar', a.path) eq('/foo/bar', a.name) assert_is(store, a.store) # store not initialized store = dict() with assert_raises(KeyError): Array(store) # group is in the way store = dict() init_group(store, path='baz') with assert_raises(KeyError): Array(store, path='baz')
def test_init_array_overwrite_chunk_store(self): # setup store = self.create_store() chunk_store = self.create_store() store[array_meta_key] = encode_array_metadata( dict(shape=(2000,), chunks=(200,), dtype=np.dtype('u1'), compressor=None, fill_value=0, filters=None, order='F') ) chunk_store['0'] = b'aaa' chunk_store['1'] = b'bbb' # don't overwrite (default) with assert_raises(ValueError): init_array(store, shape=1000, chunks=100, chunk_store=chunk_store) # do overwrite try: init_array(store, shape=1000, chunks=100, dtype='i4', overwrite=True, chunk_store=chunk_store) except NotImplementedError: pass else: assert array_meta_key in store meta = decode_array_metadata(store[array_meta_key]) eq(ZARR_FORMAT, meta['zarr_format']) eq((1000,), meta['shape']) eq((100,), meta['chunks']) eq(np.dtype('i4'), meta['dtype']) assert '0' not in chunk_store assert '1' not in chunk_store
def test_init_array_overwrite(self): # setup store = self.create_store() store[array_meta_key] = encode_array_metadata( dict(shape=(2000, ), chunks=(200, ), dtype=np.dtype('u1'), compressor=Zlib(1).get_config(), fill_value=0, order='F', filters=None)) # don't overwrite (default) with pytest.raises(ValueError): init_array(store, shape=1000, chunks=100) # do overwrite try: init_array(store, shape=1000, chunks=100, dtype='i4', overwrite=True) except NotImplementedError: pass else: assert array_meta_key in store meta = decode_array_metadata(store[array_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000, ) == meta['shape'] assert (100, ) == meta['chunks'] assert np.dtype('i4') == meta['dtype']
def test_init_array_overwrite_path(self): # setup path = 'foo/bar' store = self.create_store() meta = dict(shape=(2000,), chunks=(200,), dtype=np.dtype('u1'), compressor=Zlib(1).get_config(), fill_value=0, order='F', filters=None) store[array_meta_key] = encode_array_metadata(meta) store[path + '/' + array_meta_key] = encode_array_metadata(meta) # don't overwrite with assert_raises(ValueError): init_array(store, shape=1000, chunks=100, path=path) # do overwrite try: init_array(store, shape=1000, chunks=100, dtype='i4', path=path, overwrite=True) except NotImplementedError: pass else: assert group_meta_key in store assert array_meta_key not in store assert (path + '/' + array_meta_key) in store # should have been overwritten meta = decode_array_metadata(store[path + '/' + array_meta_key]) eq(ZARR_FORMAT, meta['zarr_format']) eq((1000,), meta['shape']) eq((100,), meta['chunks']) eq(np.dtype('i4'), meta['dtype'])
def create_array(read_only=False, **kwargs): path = mkdtemp() atexit.register(shutil.rmtree, path) store = DirectoryStore(path) kwargs.setdefault('compressor', Zlib(1)) init_array(store, **kwargs) return Array(store, read_only=read_only)
def test_init_array_overwrite(self): # setup store = self.create_store() store[array_meta_key] = encode_array_metadata( dict(shape=(2000,), chunks=(200,), dtype=np.dtype('u1'), compressor=Zlib(1).get_config(), fill_value=0, order='F', filters=None) ) # don't overwrite (default) with assert_raises(ValueError): init_array(store, shape=1000, chunks=100) # do overwrite try: init_array(store, shape=1000, chunks=100, dtype='i4', overwrite=True) except NotImplementedError: pass else: assert array_meta_key in store meta = decode_array_metadata(store[array_meta_key]) eq(ZARR_FORMAT, meta['zarr_format']) eq((1000,), meta['shape']) eq((100,), meta['chunks']) eq(np.dtype('i4'), meta['dtype'])
def create_meta_store(slide: OpenSlide, tilesize: int) -> Dict[str, bytes]: """Creates a dict containing the zarr metadata for the multiscale openslide image.""" store = dict() root_attrs = { "multiscales": [{ "name": Path(slide._filename).name, "datasets": [{ "path": str(i) } for i in range(slide.level_count)], "version": "0.1", }] } init_group(store) init_attrs(store, root_attrs) for i, (x, y) in enumerate(slide.level_dimensions): init_array( store, path=str(i), shape=(y, x, 4), chunks=(tilesize, tilesize, 4), dtype="|u1", compressor=None, ) return store
def create_array(read_only=False, **kwargs): path = mktemp(suffix='.dbm') atexit.register(os.remove, path) store = DBMStore(path, flag='n', open=bsddb3.btopen) kwargs.setdefault('compressor', Zlib(1)) init_array(store, **kwargs) return Array(store, read_only=read_only)
def test_init_array_overwrite_group(self): # setup path = 'foo/bar' store = self.create_store() store[path + '/' + group_meta_key] = encode_group_metadata() # don't overwrite with pytest.raises(ValueError): init_array(store, shape=1000, chunks=100, path=path) # do overwrite try: init_array(store, shape=1000, chunks=100, dtype='i4', path=path, overwrite=True) except NotImplementedError: pass else: assert (path + '/' + group_meta_key) not in store assert (path + '/' + array_meta_key) in store meta = decode_array_metadata(store[path + '/' + array_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000, ) == meta['shape'] assert (100, ) == meta['chunks'] assert np.dtype('i4') == meta['dtype']
def test_array_init(self): # normal initialization store = dict() init_array(store, shape=100, chunks=10) a = Array(store) assert_is_instance(a, Array) eq((100,), a.shape) eq((10,), a.chunks) eq('', a.path) assert_is_none(a.name) assert_is(store, a.store) # initialize at path store = dict() init_array(store, shape=100, chunks=10, path='foo/bar') a = Array(store, path='foo/bar') assert_is_instance(a, Array) eq((100,), a.shape) eq((10,), a.chunks) eq('foo/bar', a.path) eq('/foo/bar', a.name) assert_is(store, a.store) # store not initialized store = dict() with assert_raises(ValueError): Array(store) # group is in the way store = dict() init_group(store, path='baz') with assert_raises(ValueError): Array(store, path='baz')
def create_array(read_only=False, **kwargs): path = mkdtemp() atexit.register(shutil.rmtree, path) store = NestedDirectoryStore(path) kwargs.setdefault('compressor', Zlib(1)) init_array(store, **kwargs) return Array(store, read_only=read_only)
def create_array(self, read_only=False, **kwargs): store = dict() cache_metadata = kwargs.pop('cache_metadata', True) cache_attrs = kwargs.pop('cache_attrs', True) init_array(store, **kwargs) return Array(store, synchronizer=ThreadSynchronizer(), read_only=read_only, cache_metadata=cache_metadata, cache_attrs=cache_attrs)
def create_array(read_only=False, **kwargs): path = mktemp(suffix='.lmdb') atexit_rmtree(path) try: store = LMDBStore(path, buffers=False) except ImportError: # pragma: no cover raise SkipTest('lmdb not installed') kwargs.setdefault('compressor', Zlib(1)) init_array(store, **kwargs) return Array(store, read_only=read_only)
def create_array(self, read_only=False, **kwargs): path = tempfile.mkdtemp() atexit.register(atexit_rmtree, path) store = DirectoryStore(path) init_array(store, **kwargs) sync_path = tempfile.mkdtemp() atexit.register(atexit_rmtree, sync_path) synchronizer = ProcessSynchronizer(sync_path) return Array(store, synchronizer=synchronizer, read_only=read_only, cache_metadata=False)
def create_array(self, read_only=False, **kwargs): store = KVStore(dict()) cache_metadata = kwargs.pop('cache_metadata', True) cache_attrs = kwargs.pop('cache_attrs', True) write_empty_chunks = kwargs.pop('write_empty_chunks', True) init_array(store, **kwargs) return Array(store, synchronizer=ThreadSynchronizer(), read_only=read_only, cache_metadata=cache_metadata, cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks)
def create_array(read_only=False, **kwargs): store = dict() dtype = kwargs.get('dtype', None) filters = [ Delta(dtype=dtype), FixedScaleOffset(dtype=dtype, scale=1, offset=0), ] kwargs.setdefault('filters', filters) compressor = Zlib(1) kwargs.setdefault('compressor', compressor) init_array(store, **kwargs) return Array(store, read_only=read_only)
def test_init_array(self): store = self.create_store() init_array(store, shape=1000, chunks=100) # check metadata assert array_meta_key in store meta = decode_array_metadata(store[array_meta_key]) eq(ZARR_FORMAT, meta['zarr_format']) eq((1000,), meta['shape']) eq((100,), meta['chunks']) eq(np.dtype(None), meta['dtype']) eq(default_compressor.get_config(), meta['compressor']) assert_is_none(meta['fill_value'])
def test_init_array(self): store = self.create_store() path = 'arr1' init_array(store, path=path, shape=1000, chunks=100) # check metadata mkey = meta_root + path + '.array.json' assert mkey in store meta = store._metadata_class.decode_array_metadata(store[mkey]) assert (1000, ) == meta['shape'] assert (100, ) == meta['chunk_grid']['chunk_shape'] assert np.dtype(None) == meta['data_type'] assert meta['chunk_grid']['separator'] == "/"
def test_init_array(self): store = self.create_store() init_array(store, shape=1000, chunks=100) # check metadata assert array_meta_key in store meta = decode_array_metadata(store[array_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000, ) == meta['shape'] assert (100, ) == meta['chunks'] assert np.dtype(None) == meta['dtype'] assert default_compressor.get_config() == meta['compressor'] assert meta['fill_value'] is None
def test_list_prefix(self): store = self.create_store() path = 'arr1' init_array(store, path=path, shape=1000, chunks=100) expected = [meta_root + 'arr1.array.json', 'zarr.json'] assert sorted(store.list_prefix('')) == expected expected = [meta_root + 'arr1.array.json'] assert sorted(store.list_prefix(meta_root.rstrip('/'))) == expected # cannot start prefix with '/' with pytest.raises(ValueError): store.list_prefix(prefix='/' + meta_root.rstrip('/'))
def test_astype_no_filters(self): shape = (100,) dtype = np.dtype(np.int8) astype = np.dtype(np.float32) store = dict() init_array(store, shape=shape, chunks=10, dtype=dtype) data = np.arange(np.prod(shape), dtype=dtype).reshape(shape) z1 = Array(store) z1[...] = data z2 = z1.astype(astype) expected = data.astype(astype) assert_array_equal(expected, z2) eq(z2.read_only, True)
def test_init_array(self): store = self.create_store() init_array(store, shape=1000, chunks=100) # check metadata assert array_meta_key in store meta = decode_array_metadata(store[array_meta_key]) eq(ZARR_FORMAT, meta['zarr_format']) eq((1000, ), meta['shape']) eq((100, ), meta['chunks']) eq(np.dtype(None), meta['dtype']) eq(default_compressor.get_config(), meta['compressor']) assert_is_none(meta['fill_value']) # check attributes assert attrs_key in store eq(dict(), json.loads(text_type(store[attrs_key], 'ascii')))
def create_array(self, read_only=False, **kwargs): path = tempfile.mkdtemp() atexit.register(atexit_rmtree, path) store = DirectoryStore(path) cache_metadata = kwargs.pop('cache_metadata', False) cache_attrs = kwargs.pop('cache_attrs', False) write_empty_chunks = kwargs.pop('write_empty_chunks', True) init_array(store, **kwargs) sync_path = tempfile.mkdtemp() atexit.register(atexit_rmtree, sync_path) synchronizer = ProcessSynchronizer(sync_path) return Array(store, synchronizer=synchronizer, read_only=read_only, cache_metadata=cache_metadata, cache_attrs=cache_attrs, write_empty_chunks=write_empty_chunks)
def test_init_array_path(self): path = 'foo/bar' store = self.create_store() init_array(store, shape=1000, chunks=100, path=path) # check metadata key = path + '/' + array_meta_key assert key in store meta = decode_array_metadata(store[key]) eq(ZARR_FORMAT, meta['zarr_format']) eq((1000,), meta['shape']) eq((100,), meta['chunks']) eq(np.dtype(None), meta['dtype']) eq(default_compressor.get_config(), meta['compressor']) assert_is_none(meta['fill_value']) # check attributes key = path + '/' + attrs_key assert key in store eq(dict(), json.loads(text_type(store[key], 'ascii')))
def test_init_array(self, dimension_separator_fixture_v3): pass_dim_sep, want_dim_sep = dimension_separator_fixture_v3 store = self.create_store() path = 'arr1' init_array(store, path=path, shape=1000, chunks=100, dimension_separator=pass_dim_sep) # check metadata mkey = meta_root + path + '.array.json' assert mkey in store meta = store._metadata_class.decode_array_metadata(store[mkey]) assert (1000,) == meta['shape'] assert (100,) == meta['chunk_grid']['chunk_shape'] assert np.dtype(None) == meta['data_type'] assert default_compressor == meta['compressor'] assert meta['fill_value'] is None # Missing MUST be assumed to be "/" assert meta['chunk_grid']['separator'] is want_dim_sep store.close()
def test_init_array_overwrite_path(self): # setup path = 'foo/bar' store = self.create_store() meta = dict(shape=(2000, ), chunks=(200, ), dtype=np.dtype('u1'), compressor=Zlib(1).get_config(), fill_value=0, order='F', filters=None) store[array_meta_key] = encode_array_metadata(meta) store[path + '/' + array_meta_key] = encode_array_metadata(meta) # don't overwrite with pytest.raises(ValueError): init_array(store, shape=1000, chunks=100, path=path) # do overwrite try: init_array(store, shape=1000, chunks=100, dtype='i4', path=path, overwrite=True) except NotImplementedError: pass else: assert group_meta_key in store assert array_meta_key not in store assert (path + '/' + array_meta_key) in store # should have been overwritten meta = decode_array_metadata(store[path + '/' + array_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000, ) == meta['shape'] assert (100, ) == meta['chunks'] assert np.dtype('i4') == meta['dtype']
def test_init_array_overwrite_chunk_store(self): # setup store = self.create_store() chunk_store = self.create_store() store[array_meta_key] = encode_array_metadata( dict(shape=(2000, ), chunks=(200, ), dtype=np.dtype('u1'), compressor=None, fill_value=0, filters=None, order='F')) chunk_store['0'] = b'aaa' chunk_store['1'] = b'bbb' # don't overwrite (default) with assert_raises(ValueError): init_array(store, shape=1000, chunks=100, chunk_store=chunk_store) # do overwrite try: init_array(store, shape=1000, chunks=100, dtype='i4', overwrite=True, chunk_store=chunk_store) except NotImplementedError: pass else: assert array_meta_key in store meta = decode_array_metadata(store[array_meta_key]) eq(ZARR_FORMAT, meta['zarr_format']) eq((1000, ), meta['shape']) eq((100, ), meta['chunks']) eq(np.dtype('i4'), meta['dtype']) assert '0' not in chunk_store assert '1' not in chunk_store
def test_group(): # test the group() convenience function # basic usage g = group() assert_is_instance(g, Group) eq('', g.path) eq('/', g.name) # usage with custom store store = dict() g = group(store=store) assert_is_instance(g, Group) assert_is(store, g.store) # overwrite behaviour store = dict() init_array(store, shape=100, chunks=10) with assert_raises(KeyError): group(store) g = group(store, overwrite=True) assert_is_instance(g, Group) assert_is(store, g.store)
def test_group(): # test the group() convenience function # basic usage g = group() assert isinstance(g, Group) assert '' == g.path assert '/' == g.name # usage with custom store store = dict() g = group(store=store) assert isinstance(g, Group) assert store is g.store # overwrite behaviour store = dict() init_array(store, shape=100, chunks=10) with pytest.raises(ValueError): group(store) g = group(store, overwrite=True) assert isinstance(g, Group) assert store is g.store
def test_group(): # test the group() convenience function # basic usage g = group() assert_is_instance(g, Group) eq('', g.path) eq('/', g.name) # usage with custom store store = dict() g = group(store=store) assert_is_instance(g, Group) assert_is(store, g.store) # overwrite behaviour store = dict() init_array(store, shape=100, chunks=10) with assert_raises(ValueError): group(store) g = group(store, overwrite=True) assert_is_instance(g, Group) assert_is(store, g.store)
def test_init_array_overwrite_group(self): # setup path = 'foo/bar' store = self.create_store() store[path + '/' + group_meta_key] = encode_group_metadata() # don't overwrite with assert_raises(ValueError): init_array(store, shape=1000, chunks=100, path=path) # do overwrite try: init_array(store, shape=1000, chunks=100, dtype='i4', path=path, overwrite=True) except NotImplementedError: pass else: assert (path + '/' + group_meta_key) not in store assert (path + '/' + array_meta_key) in store meta = decode_array_metadata(store[path + '/' + array_meta_key]) eq(ZARR_FORMAT, meta['zarr_format']) eq((1000,), meta['shape']) eq((100,), meta['chunks']) eq(np.dtype('i4'), meta['dtype'])
def open_array(store, mode='a', shape=None, chunks=True, dtype=None, compressor='default', fill_value=0, order='C', synchronizer=None, filters=None, cache_metadata=True, cache_attrs=True, path=None, object_codec=None, **kwargs): """Open an array using file-mode-like semantics. Parameters ---------- store : MutableMapping or string Store or path to directory in file system or name of zip file. mode : {'r', 'r+', 'a', 'w', 'w-'}, optional Persistence mode: 'r' means read only (must exist); 'r+' means read/write (must exist); 'a' means read/write (create if doesn't exist); 'w' means create (overwrite if exists); 'w-' means create (fail if exists). shape : int or tuple of ints, optional Array shape. chunks : int or tuple of ints, optional Chunk shape. If True, will be guessed from `shape` and `dtype`. If False, will be set to `shape`, i.e., single chunk for the whole array. dtype : string or dtype, optional NumPy dtype. compressor : Codec, optional Primary compressor. fill_value : object, optional Default value to use for uninitialized portions of the array. order : {'C', 'F'}, optional Memory layout to be used within each chunk. synchronizer : object, optional Array synchronizer. filters : sequence, optional Sequence of filters to use to encode chunk data prior to compression. cache_metadata : bool, optional If True, array configuration metadata will be cached for the lifetime of the object. If False, array metadata will be reloaded prior to all data access and modification operations (may incur overhead depending on storage and data access pattern). cache_attrs : bool, optional If True (default), user attributes will be cached for attribute read operations. If False, user attributes are reloaded from the store prior to all attribute read operations. path : string, optional Array path within store. object_codec : Codec, optional A codec to encode object arrays, only needed if dtype=object. Returns ------- z : zarr.core.Array Examples -------- >>> import numpy as np >>> import zarr >>> z1 = zarr.open_array('data/example.zarr', mode='w', shape=(10000, 10000), ... chunks=(1000, 1000), fill_value=0) >>> z1[:] = np.arange(100000000).reshape(10000, 10000) >>> z1 <zarr.core.Array (10000, 10000) float64> >>> z2 = zarr.open_array('data/example.zarr', mode='r') >>> z2 <zarr.core.Array (10000, 10000) float64 read-only> >>> np.all(z1[:] == z2[:]) True Notes ----- There is no need to close an array. Data are automatically flushed to the file system. """ # use same mode semantics as h5py # r : read only, must exist # r+ : read/write, must exist # w : create, delete if exists # w- or x : create, fail if exists # a : read/write if exists, create otherwise (default) # handle polymorphic store arg store = normalize_store_arg(store, clobber=(mode == 'w')) path = normalize_storage_path(path) # API compatibility with h5py compressor, fill_value = _kwargs_compat(compressor, fill_value, kwargs) # ensure fill_value of correct type if fill_value is not None: fill_value = np.array(fill_value, dtype=dtype)[()] # ensure store is initialized if mode in ['r', 'r+']: if contains_group(store, path=path): err_contains_group(path) elif not contains_array(store, path=path): err_array_not_found(path) elif mode == 'w': init_array(store, shape=shape, chunks=chunks, dtype=dtype, compressor=compressor, fill_value=fill_value, order=order, filters=filters, overwrite=True, path=path, object_codec=object_codec) elif mode == 'a': if contains_group(store, path=path): err_contains_group(path) elif not contains_array(store, path=path): init_array(store, shape=shape, chunks=chunks, dtype=dtype, compressor=compressor, fill_value=fill_value, order=order, filters=filters, path=path, object_codec=object_codec) elif mode in ['w-', 'x']: if contains_group(store, path=path): err_contains_group(path) elif contains_array(store, path=path): err_contains_array(path) else: init_array(store, shape=shape, chunks=chunks, dtype=dtype, compressor=compressor, fill_value=fill_value, order=order, filters=filters, path=path, object_codec=object_codec) # determine read only status read_only = mode == 'r' # instantiate array z = Array(store, read_only=read_only, synchronizer=synchronizer, cache_metadata=cache_metadata, cache_attrs=cache_attrs, path=path) return z
def create_array(read_only=False, **kwargs): store = dict() init_array(store, path='foo/bar', **kwargs) return Array(store, path='foo/bar', read_only=read_only)
def create_array(read_only=False, **kwargs): store = dict() # separate chunk store chunk_store = dict() init_array(store, chunk_store=chunk_store, **kwargs) return Array(store, read_only=read_only, chunk_store=chunk_store)
def test_init_array_compat(self): store = self.create_store() init_array(store, shape=1000, chunks=100, compressor='none') meta = decode_array_metadata(store[array_meta_key]) assert meta['compressor'] is None
def open_array(store=None, mode='a', shape=None, chunks=None, dtype=None, compressor='default', fill_value=0, order='C', synchronizer=None, filters=None, cache_metadata=True, path=None, **kwargs): """Open array using mode-like semantics. Parameters ---------- store : MutableMapping or string Store or path to directory in file system. mode : {'r', 'r+', 'a', 'w', 'w-'} Persistence mode: 'r' means read only (must exist); 'r+' means read/write (must exist); 'a' means read/write (create if doesn't exist); 'w' means create (overwrite if exists); 'w-' means create (fail if exists). shape : int or tuple of ints Array shape. chunks : int or tuple of ints, optional Chunk shape. If not provided, will be guessed from `shape` and `dtype`. dtype : string or dtype, optional NumPy dtype. compressor : Codec, optional Primary compressor. fill_value : object Default value to use for uninitialized portions of the array. order : {'C', 'F'}, optional Memory layout to be used within each chunk. synchronizer : object, optional Array synchronizer. filters : sequence, optional Sequence of filters to use to encode chunk data prior to compression. cache_metadata : bool, optional If True, array configuration metadata will be cached for the lifetime of the object. If False, array metadata will be reloaded prior to all data access and modification operations (may incur overhead depending on storage and data access pattern). path : string, optional Array path. Returns ------- z : zarr.core.Array Examples -------- >>> import numpy as np >>> import zarr >>> z1 = zarr.open_array('example.zarr', mode='w', shape=(10000, 10000), ... chunks=(1000, 1000), fill_value=0) >>> z1[:] = np.arange(100000000).reshape(10000, 10000) >>> z1 Array((10000, 10000), float64, chunks=(1000, 1000), order=C) nbytes: 762.9M; nbytes_stored: 23.0M; ratio: 33.2; initialized: 100/100 compressor: Blosc(cname='lz4', clevel=5, shuffle=1) store: DirectoryStore >>> z2 = zarr.open_array('example.zarr', mode='r') >>> z2 Array((10000, 10000), float64, chunks=(1000, 1000), order=C) nbytes: 762.9M; nbytes_stored: 23.0M; ratio: 33.2; initialized: 100/100 compressor: Blosc(cname='lz4', clevel=5, shuffle=1) store: DirectoryStore >>> np.all(z1[:] == z2[:]) True Notes ----- There is no need to close an array. Data are automatically flushed to the file system. """ # flake8: noqa # use same mode semantics as h5py # r : read only, must exist # r+ : read/write, must exist # w : create, delete if exists # w- or x : create, fail if exists # a : read/write if exists, create otherwise (default) # handle polymorphic store arg store = _handle_store_arg(store) path = normalize_storage_path(path) # compatibility compressor, fill_value = _handle_kwargs(compressor, fill_value, kwargs) # ensure store is initialized if mode in ['r', 'r+']: if contains_group(store, path=path): err_contains_group(path) elif not contains_array(store, path=path): err_array_not_found(path) elif mode == 'w': init_array(store, shape=shape, chunks=chunks, dtype=dtype, compressor=compressor, fill_value=fill_value, order=order, filters=filters, overwrite=True, path=path) elif mode == 'a': if contains_group(store, path=path): err_contains_group(path) elif not contains_array(store, path=path): init_array(store, shape=shape, chunks=chunks, dtype=dtype, compressor=compressor, fill_value=fill_value, order=order, filters=filters, path=path) elif mode in ['w-', 'x']: if contains_group(store, path=path): err_contains_group(path) elif contains_array(store, path=path): err_contains_array(path) else: init_array(store, shape=shape, chunks=chunks, dtype=dtype, compressor=compressor, fill_value=fill_value, order=order, filters=filters, path=path) # determine read only status read_only = mode == 'r' # instantiate array z = Array(store, read_only=read_only, synchronizer=synchronizer, cache_metadata=cache_metadata, path=path) return z
def create(shape, chunks=True, dtype=None, compressor='default', fill_value=0, order='C', store=None, synchronizer=None, overwrite=False, path=None, chunk_store=None, filters=None, cache_metadata=True, cache_attrs=True, read_only=False, object_codec=None, **kwargs): """Create an array. Parameters ---------- shape : int or tuple of ints Array shape. chunks : int or tuple of ints, optional Chunk shape. If True, will be guessed from `shape` and `dtype`. If False, will be set to `shape`, i.e., single chunk for the whole array. dtype : string or dtype, optional NumPy dtype. compressor : Codec, optional Primary compressor. fill_value : object Default value to use for uninitialized portions of the array. order : {'C', 'F'}, optional Memory layout to be used within each chunk. store : MutableMapping or string Store or path to directory in file system or name of zip file. synchronizer : object, optional Array synchronizer. overwrite : bool, optional If True, delete all pre-existing data in `store` at `path` before creating the array. path : string, optional Path under which array is stored. chunk_store : MutableMapping, optional Separate storage for chunks. If not provided, `store` will be used for storage of both chunks and metadata. filters : sequence of Codecs, optional Sequence of filters to use to encode chunk data prior to compression. cache_metadata : bool, optional If True, array configuration metadata will be cached for the lifetime of the object. If False, array metadata will be reloaded prior to all data access and modification operations (may incur overhead depending on storage and data access pattern). cache_attrs : bool, optional If True (default), user attributes will be cached for attribute read operations. If False, user attributes are reloaded from the store prior to all attribute read operations. read_only : bool, optional True if array should be protected against modification. object_codec : Codec, optional A codec to encode object arrays, only needed if dtype=object. Returns ------- z : zarr.core.Array Examples -------- Create an array with default settings:: >>> import zarr >>> z = zarr.create((10000, 10000), chunks=(1000, 1000)) >>> z <zarr.core.Array (10000, 10000) float64> Create an array with different some different configuration options:: >>> from numcodecs import Blosc >>> compressor = Blosc(cname='zstd', clevel=1, shuffle=Blosc.BITSHUFFLE) >>> z = zarr.create((10000, 10000), chunks=(1000, 1000), dtype='i1', order='F', ... compressor=compressor) >>> z <zarr.core.Array (10000, 10000) int8> To create an array with object dtype requires a filter that can handle Python object encoding, e.g., `MsgPack` or `Pickle` from `numcodecs`:: >>> from numcodecs import MsgPack >>> z = zarr.create((10000, 10000), chunks=(1000, 1000), dtype=object, ... object_codec=MsgPack()) >>> z <zarr.core.Array (10000, 10000) object> Example with some filters, and also storing chunks separately from metadata:: >>> from numcodecs import Quantize, Adler32 >>> store, chunk_store = dict(), dict() >>> z = zarr.create((10000, 10000), chunks=(1000, 1000), dtype='f8', ... filters=[Quantize(digits=2, dtype='f8'), Adler32()], ... store=store, chunk_store=chunk_store) >>> z <zarr.core.Array (10000, 10000) float64> """ # handle polymorphic store arg store = normalize_store_arg(store) # API compatibility with h5py compressor, fill_value = _kwargs_compat(compressor, fill_value, kwargs) # initialize array metadata init_array(store, shape=shape, chunks=chunks, dtype=dtype, compressor=compressor, fill_value=fill_value, order=order, overwrite=overwrite, path=path, chunk_store=chunk_store, filters=filters, object_codec=object_codec) # instantiate array z = Array(store, path=path, chunk_store=chunk_store, synchronizer=synchronizer, cache_metadata=cache_metadata, cache_attrs=cache_attrs, read_only=read_only) return z
def create_array(read_only=False, **kwargs): store = CustomMapping() kwargs.setdefault('compressor', Zlib(1)) init_array(store, **kwargs) return Array(store, read_only=read_only)
def create_array(self, read_only=False, **kwargs): store = dict() compressor = LZMA(preset=1) kwargs.setdefault('compressor', compressor) init_array(store, **kwargs) return Array(store, read_only=read_only)
def create_array(self, read_only=False, **kwargs): store = dict() compressor = Blosc(cname='zstd', clevel=1, shuffle=1) kwargs.setdefault('compressor', compressor) init_array(store, **kwargs) return Array(store, read_only=read_only)
def create_array(self, read_only=False, **kwargs): store = dict() kwargs.setdefault('compressor', None) init_array(store, **kwargs) return Array(store, read_only=read_only)
def create_array(read_only=False, **kwargs): store = dict() kwargs.setdefault('compressor', Zlib(level=1)) init_array(store, **kwargs) return Array(store, read_only=read_only, cache_metadata=False)
def test_init_array_compat(self): store = self.create_store() init_array(store, shape=1000, chunks=100, compressor='none') meta = decode_array_metadata(store[array_meta_key]) assert_is_none(meta['compressor'])
def create_array(self, read_only=False, **kwargs): store = dict() init_array(store, **kwargs) return Array(store, synchronizer=ThreadSynchronizer(), read_only=read_only)
def create(shape, chunks=None, dtype=None, compressor='default', fill_value=0, order='C', store=None, synchronizer=None, overwrite=False, path=None, chunk_store=None, filters=None, cache_metadata=True, **kwargs): """Create an array. Parameters ---------- shape : int or tuple of ints Array shape. chunks : int or tuple of ints, optional Chunk shape. If not provided, will be guessed from `shape` and `dtype`. dtype : string or dtype, optional NumPy dtype. compressor : Codec, optional Primary compressor. fill_value : object Default value to use for uninitialized portions of the array. order : {'C', 'F'}, optional Memory layout to be used within each chunk. store : MutableMapping or string Store or path to directory in file system. synchronizer : object, optional Array synchronizer. overwrite : bool, optional If True, delete all pre-existing data in `store` at `path` before creating the array. path : string, optional Path under which array is stored. chunk_store : MutableMapping, optional Separate storage for chunks. If not provided, `store` will be used for storage of both chunks and metadata. filters : sequence of Codecs, optional Sequence of filters to use to encode chunk data prior to compression. cache_metadata : bool, optional If True, array configuration metadata will be cached for the lifetime of the object. If False, array metadata will be reloaded prior to all data access and modification operations (may incur overhead depending on storage and data access pattern). Returns ------- z : zarr.core.Array Examples -------- Create an array with default settings:: >>> import zarr >>> z = zarr.create((10000, 10000), chunks=(1000, 1000)) >>> z Array((10000, 10000), float64, chunks=(1000, 1000), order=C) nbytes: 762.9M; nbytes_stored: 323; ratio: 2476780.2; initialized: 0/100 compressor: Blosc(cname='lz4', clevel=5, shuffle=1) store: dict """ # flake8: noqa # handle polymorphic store arg store = _handle_store_arg(store) # compatibility compressor, fill_value = _handle_kwargs(compressor, fill_value, kwargs) # initialize array metadata init_array(store, shape=shape, chunks=chunks, dtype=dtype, compressor=compressor, fill_value=fill_value, order=order, overwrite=overwrite, path=path, chunk_store=chunk_store, filters=filters) # instantiate array z = Array(store, path=path, chunk_store=chunk_store, synchronizer=synchronizer, cache_metadata=cache_metadata) return z
def test_group_init_errors_2(self): store, chunk_store = self.create_store() init_array(store, shape=1000, chunks=100, chunk_store=chunk_store) # array blocks group with assert_raises(KeyError): Group(store, chunk_store=chunk_store)