def test_create():
# defaults
z = create(100)
assert isinstance(z, Array)
assert (100,) == z.shape
assert (100,) == z.chunks # auto-chunks
assert np.dtype(None) == z.dtype
assert 'blosc' == z.compressor.codec_id
assert 0 == z.fill_value
# all specified
z = create(100, chunks=10, dtype='i4', compressor=Zlib(1),
fill_value=42, order='F')
assert isinstance(z, Array)
assert (100,) == z.shape
assert (10,) == z.chunks
assert np.dtype('i4') == z.dtype
assert 'zlib' == z.compressor.codec_id
assert 1 == z.compressor.level
assert 42 == z.fill_value
assert 'F' == z.order
# with synchronizer
synchronizer = ThreadSynchronizer()
z = create(100, chunks=10, synchronizer=synchronizer)
assert isinstance(z, Array)
assert (100,) == z.shape
assert (10,) == z.chunks
assert synchronizer is z.synchronizer
# don't allow string as compressor arg
with pytest.raises(ValueError):
create(100, chunks=10, compressor='zlib')
# h5py compatibility
z = create(100, compression='zlib', compression_opts=9)
assert 'zlib' == z.compressor.codec_id
assert 9 == z.compressor.level
z = create(100, compression='default')
assert 'blosc' == z.compressor.codec_id
# errors
with pytest.raises(ValueError):
# bad compression argument
create(100, compression=1)
with pytest.raises(ValueError):
# bad fill value
create(100, dtype='i4', fill_value='foo')
# auto chunks
z = create(1000000000, chunks=True)
assert z.chunks[0] < z.shape[0]
z = create(1000000000, chunks=None) # backwards-compatibility
assert z.chunks[0] < z.shape[0]
# no chunks
z = create(1000000000, chunks=False)
assert z.chunks == z.shape
def init_attributes(self, store, read_only=False, cache=True):
key = 'attrs'
synchronizer = ThreadSynchronizer()
return Attributes(store,
synchronizer=synchronizer,
key=key,
read_only=read_only,
cache=cache)
def init_attributes(self, store, read_only=False):
key = 'attrs'
store[key] = json.dumps(dict()).encode('ascii')
synchronizer = ThreadSynchronizer()
return Attributes(store,
synchronizer=synchronizer,
key=key,
read_only=read_only)
def create_array(self, read_only=False, **kwargs):
store = dict()
cache_metadata = kwargs.pop('cache_metadata', True)
cache_attrs = kwargs.pop('cache_attrs', True)
init_array(store, **kwargs)
return Array(store,
synchronizer=ThreadSynchronizer(),
read_only=read_only,
cache_metadata=cache_metadata,
cache_attrs=cache_attrs)
def init_attributes(self,
store,
read_only=False,
cache=True,
zarr_version=zarr_version):
key = '.zattrs' if zarr_version == 2 else meta_root + 'attrs'
synchronizer = ThreadSynchronizer()
return Attributes(store,
synchronizer=synchronizer,
key=key,
read_only=read_only,
cache=cache)
def create_array(self, read_only=False, **kwargs):
store = KVStore(dict())
cache_metadata = kwargs.pop('cache_metadata', True)
cache_attrs = kwargs.pop('cache_attrs', True)
write_empty_chunks = kwargs.pop('write_empty_chunks', True)
init_array(store, **kwargs)
return Array(store,
synchronizer=ThreadSynchronizer(),
read_only=read_only,
cache_metadata=cache_metadata,
cache_attrs=cache_attrs,
write_empty_chunks=write_empty_chunks)
def test_create():
# defaults
z = create(100)
assert_is_instance(z, Array)
eq((100, ), z.shape)
eq((100, ), z.chunks) # auto-chunks
eq(np.dtype(None), z.dtype)
eq('blosc', z.compressor.codec_id)
eq(0, z.fill_value)
# all specified
z = create(100,
chunks=10,
dtype='i4',
compressor=Zlib(1),
fill_value=42,
order='F')
assert_is_instance(z, Array)
eq((100, ), z.shape)
eq((10, ), z.chunks)
eq(np.dtype('i4'), z.dtype)
eq('zlib', z.compressor.codec_id)
eq(1, z.compressor.level)
eq(42, z.fill_value)
eq('F', z.order)
# with synchronizer
synchronizer = ThreadSynchronizer()
z = create(100, chunks=10, synchronizer=synchronizer)
assert_is_instance(z, Array)
eq((100, ), z.shape)
eq((10, ), z.chunks)
assert synchronizer is z.synchronizer
# don't allow string as compressor arg
with assert_raises(ValueError):
create(100, chunks=10, compressor='zlib')
# compatibility
z = create(100, compression='zlib', compression_opts=9)
eq('zlib', z.compressor.codec_id)
eq(9, z.compressor.level)
z = create(100, compression='default')
eq('blosc', z.compressor.codec_id)
# errors
with assert_raises(ValueError):
create(100, compression=1)
def create_group(self,
store=None,
path=None,
read_only=False,
chunk_store=None,
synchronizer=None):
if store is None:
store, chunk_store = self.create_store()
init_group(store, path=path, chunk_store=chunk_store)
synchronizer = ThreadSynchronizer()
g = Group(store,
path=path,
read_only=read_only,
chunk_store=chunk_store,
synchronizer=synchronizer)
return g
def test_open_array():
store = 'data/array.zarr'
# mode == 'w'
z = open_array(store, mode='w', shape=100, chunks=10)
z[:] = 42
assert isinstance(z, Array)
assert isinstance(z.store, DirectoryStore)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
# mode in 'r', 'r+'
open_group('data/group.zarr', mode='w')
for mode in 'r', 'r+':
with pytest.raises(ValueError):
open_array('doesnotexist', mode=mode)
with pytest.raises(ValueError):
open_array('data/group.zarr', mode=mode)
z = open_array(store, mode='r')
assert isinstance(z, Array)
assert isinstance(z.store, DirectoryStore)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
with pytest.raises(PermissionError):
z[:] = 43
z = open_array(store, mode='r+')
assert isinstance(z, Array)
assert isinstance(z.store, DirectoryStore)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
z[:] = 43
assert_array_equal(np.full(100, fill_value=43), z[:])
# mode == 'a'
shutil.rmtree(store)
z = open_array(store, mode='a', shape=100, chunks=10)
z[:] = 42
assert isinstance(z, Array)
assert isinstance(z.store, DirectoryStore)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
with pytest.raises(ValueError):
open_array('data/group.zarr', mode='a')
# mode in 'w-', 'x'
for mode in 'w-', 'x':
shutil.rmtree(store)
z = open_array(store, mode=mode, shape=100, chunks=10)
z[:] = 42
assert isinstance(z, Array)
assert isinstance(z.store, DirectoryStore)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
with pytest.raises(ValueError):
open_array(store, mode=mode)
with pytest.raises(ValueError):
open_array('data/group.zarr', mode=mode)
# with synchronizer
z = open_array(store, synchronizer=ThreadSynchronizer())
assert isinstance(z, Array)
# with path
z = open_array(store, shape=100, path='foo/bar', mode='w')
assert isinstance(z, Array)
assert 'foo/bar' == z.path
# with chunk store
meta_store = 'data/meta.zarr'
chunk_store = 'data/chunks.zarr'
z = open_array(store=meta_store,
chunk_store=chunk_store,
shape=11,
mode='w')
z[:] = 42
assert os.path.abspath(meta_store) == z.store.path
assert os.path.abspath(chunk_store) == z.chunk_store.path
# for N5 store
store = 'data/array.n5'
z = open_array(store, mode='w', shape=100, chunks=10)
z[:] = 42
assert isinstance(z, Array)
assert isinstance(z.store, N5Store)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
def test_open_array():
store = 'data/array.zarr'
# mode == 'w'
z = open_array(store, mode='w', shape=100, chunks=10)
z[:] = 42
assert isinstance(z, Array)
assert isinstance(z.store, DirectoryStore)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
# mode in 'r', 'r+'
open_group('data/group.zarr', mode='w')
for mode in 'r', 'r+':
with pytest.raises(ValueError):
open_array('doesnotexist', mode=mode)
with pytest.raises(ValueError):
open_array('data/group.zarr', mode=mode)
z = open_array(store, mode='r')
assert isinstance(z, Array)
assert isinstance(z.store, DirectoryStore)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
with pytest.raises(PermissionError):
z[:] = 43
z = open_array(store, mode='r+')
assert isinstance(z, Array)
assert isinstance(z.store, DirectoryStore)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
z[:] = 43
assert_array_equal(np.full(100, fill_value=43), z[:])
# mode == 'a'
shutil.rmtree(store)
z = open_array(store, mode='a', shape=100, chunks=10)
z[:] = 42
assert isinstance(z, Array)
assert isinstance(z.store, DirectoryStore)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
with pytest.raises(ValueError):
open_array('data/group.zarr', mode='a')
# mode in 'w-', 'x'
for mode in 'w-', 'x':
shutil.rmtree(store)
z = open_array(store, mode=mode, shape=100, chunks=10)
z[:] = 42
assert isinstance(z, Array)
assert isinstance(z.store, DirectoryStore)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
with pytest.raises(ValueError):
open_array(store, mode=mode)
with pytest.raises(ValueError):
open_array('data/group.zarr', mode=mode)
# with synchronizer
z = open_array(store, synchronizer=ThreadSynchronizer())
assert isinstance(z, Array)
# with path
z = open_array(store, shape=100, path='foo/bar', mode='w')
assert isinstance(z, Array)
assert 'foo/bar' == z.path
# with chunk store
meta_store = 'data/meta.zarr'
chunk_store = 'data/chunks.zarr'
z = open_array(store=meta_store,
chunk_store=chunk_store,
shape=11,
mode='w')
z[:] = 42
assert os.path.abspath(meta_store) == z.store.path
assert os.path.abspath(chunk_store) == z.chunk_store.path
# for N5 store
store = 'data/array.n5'
z = open_array(store, mode='w', shape=100, chunks=10)
z[:] = 42
assert isinstance(z, Array)
assert isinstance(z.store, N5Store)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
store = 'data/group.n5'
z = open_group(store, mode='w')
i = z.create_group('inner')
a = i.zeros("array", shape=100, chunks=10)
a[:] = 42
# Edit inner/attributes.json to not include "n5"
with open('data/group.n5/inner/attributes.json', 'w') as o:
o.write("{}")
# Re-open
a = open_group(store)["inner"]["array"]
assert isinstance(a, Array)
assert isinstance(z.store, N5Store)
assert (100, ) == a.shape
assert (10, ) == a.chunks
assert_array_equal(np.full(100, fill_value=42), a[:])
def test_open_array():
store = 'example'
# mode == 'w'
z = open_array(store, mode='w', shape=100, chunks=10)
z[:] = 42
assert_is_instance(z, Array)
assert_is_instance(z.store, DirectoryStore)
eq((100, ), z.shape)
eq((10, ), z.chunks)
assert_array_equal(np.full(100, fill_value=42), z[:])
# mode in 'r', 'r+'
open_group('example_group', mode='w')
for mode in 'r', 'r+':
with assert_raises(KeyError):
open_array('doesnotexist', mode=mode)
with assert_raises(KeyError):
open_array('example_group', mode=mode)
z = open_array(store, mode='r')
assert_is_instance(z, Array)
assert_is_instance(z.store, DirectoryStore)
eq((100, ), z.shape)
eq((10, ), z.chunks)
assert_array_equal(np.full(100, fill_value=42), z[:])
with assert_raises(PermissionError):
z[:] = 43
z = open_array(store, mode='r+')
assert_is_instance(z, Array)
assert_is_instance(z.store, DirectoryStore)
eq((100, ), z.shape)
eq((10, ), z.chunks)
assert_array_equal(np.full(100, fill_value=42), z[:])
z[:] = 43
assert_array_equal(np.full(100, fill_value=43), z[:])
# mode == 'a'
shutil.rmtree(store)
z = open_array(store, mode='a', shape=100, chunks=10)
z[:] = 42
assert_is_instance(z, Array)
assert_is_instance(z.store, DirectoryStore)
eq((100, ), z.shape)
eq((10, ), z.chunks)
assert_array_equal(np.full(100, fill_value=42), z[:])
with assert_raises(KeyError):
open_array('example_group', mode='a')
# mode in 'w-', 'x'
for mode in 'w-', 'x':
shutil.rmtree(store)
z = open_array(store, mode=mode, shape=100, chunks=10)
z[:] = 42
assert_is_instance(z, Array)
assert_is_instance(z.store, DirectoryStore)
eq((100, ), z.shape)
eq((10, ), z.chunks)
assert_array_equal(np.full(100, fill_value=42), z[:])
with assert_raises(KeyError):
open_array(store, mode=mode)
with assert_raises(KeyError):
open_array('example_group', mode=mode)
# with synchronizer
z = open_array(store, synchronizer=ThreadSynchronizer())
assert_is_instance(z, Array)
# with path
z = open_array(store, shape=100, path='foo/bar', mode='w')
assert_is_instance(z, Array)
eq('foo/bar', z.path)
def test_create(zarr_version):
kwargs = _init_creation_kwargs(zarr_version)
expected_zarr_version = DEFAULT_ZARR_VERSION if zarr_version is None else zarr_version
# defaults
z = create(100, **kwargs)
if zarr_version == 3:
with pytest.raises(ValueError):
# cannot create without specifying a path
z = create(100, zarr_version=3)
assert isinstance(z, Array)
assert (100, ) == z.shape
assert (100, ) == z.chunks # auto-chunks
assert np.dtype(None) == z.dtype
assert 'blosc' == z.compressor.codec_id
assert 0 == z.fill_value
assert z._store._store_version == expected_zarr_version
# all specified
z = create(100,
chunks=10,
dtype='i4',
compressor=Zlib(1),
fill_value=42,
order='F',
**kwargs)
assert isinstance(z, Array)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert np.dtype('i4') == z.dtype
assert 'zlib' == z.compressor.codec_id
assert 1 == z.compressor.level
assert 42 == z.fill_value
assert 'F' == z.order
assert z._store._store_version == expected_zarr_version
# with synchronizer
synchronizer = ThreadSynchronizer()
z = create(100, chunks=10, synchronizer=synchronizer, **kwargs)
assert isinstance(z, Array)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert synchronizer is z.synchronizer
assert z._store._store_version == expected_zarr_version
# don't allow string as compressor arg
with pytest.raises(ValueError):
create(100, chunks=10, compressor='zlib', **kwargs)
# h5py compatibility
z = create(100, compression='zlib', compression_opts=9, **kwargs)
assert 'zlib' == z.compressor.codec_id
assert 9 == z.compressor.level
z = create(100, compression='default', **kwargs)
assert 'blosc' == z.compressor.codec_id
# errors
with pytest.raises(ValueError):
# bad compression argument
create(100, compression=1, **kwargs)
with pytest.raises(ValueError):
# bad fill value
create(100, dtype='i4', fill_value='foo', **kwargs)
# auto chunks
z = create(1000000000, chunks=True, **kwargs)
assert z.chunks[0] < z.shape[0]
z = create(1000000000, chunks=None, **kwargs) # backwards-compatibility
assert z.chunks[0] < z.shape[0]
# no chunks
z = create(1000000000, chunks=False, **kwargs)
assert z.chunks == z.shape
def test_open_array(zarr_version, dimension_separator):
store = 'data/array.zarr'
kwargs = _init_creation_kwargs(zarr_version)
# mode == 'w'
z = open_array(store,
mode='w',
shape=100,
chunks=10,
dimension_separator=dimension_separator,
**kwargs)
z[:] = 42
assert isinstance(z, Array)
if z._store._store_version == 2:
assert isinstance(z.store, DirectoryStore)
else:
assert isinstance(z.store, DirectoryStoreV3)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
if dimension_separator is None:
assert z._dimension_separator == '/' if zarr_version == 3 else '.'
else:
assert z._dimension_separator == dimension_separator
# mode in 'r', 'r+'
group_kwargs = kwargs.copy()
if zarr_version == 3:
group_kwargs['path'] = 'group'
open_group('data/group.zarr', mode='w', **group_kwargs)
for mode in 'r', 'r+':
with pytest.raises(ValueError):
open_array('doesnotexist', mode=mode)
with pytest.raises(ValueError):
open_array('data/group.zarr', mode=mode)
z = open_array(store, mode='r', **kwargs)
assert isinstance(z, Array)
if z._store._store_version == 2:
assert isinstance(z.store, DirectoryStore)
else:
assert isinstance(z.store, DirectoryStoreV3)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
with pytest.raises(PermissionError):
z[:] = 43
z = open_array(store, mode='r+', **kwargs)
assert isinstance(z, Array)
if z._store._store_version == 2:
assert isinstance(z.store, DirectoryStore)
else:
assert isinstance(z.store, DirectoryStoreV3)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
z[:] = 43
assert_array_equal(np.full(100, fill_value=43), z[:])
# mode == 'a'
shutil.rmtree(store)
z = open_array(store, mode='a', shape=100, chunks=10, **kwargs)
z[:] = 42
assert isinstance(z, Array)
if z._store._store_version == 2:
assert isinstance(z.store, DirectoryStore)
else:
assert isinstance(z.store, DirectoryStoreV3)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
expected_error = TypeError if zarr_version == 3 else ValueError
# v3 path does not conflict, but will raise TypeError without shape kwarg
with pytest.raises(expected_error):
# array would end up at data/group.zarr/meta/root/array.array.json
open_array('data/group.zarr', mode='a', **kwargs)
# mode in 'w-', 'x'
for mode in 'w-', 'x':
shutil.rmtree(store)
z = open_array(store, mode=mode, shape=100, chunks=10, **kwargs)
z[:] = 42
assert isinstance(z, Array)
if z._store._store_version == 2:
assert isinstance(z.store, DirectoryStore)
else:
assert isinstance(z.store, DirectoryStoreV3)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
with pytest.raises(ValueError):
open_array(store, mode=mode, **kwargs)
expected_error = TypeError if zarr_version == 3 else ValueError
# v3 path does not conflict, but will raise TypeError without shape kwarg
with pytest.raises(expected_error):
open_array('data/group.zarr', mode=mode, **kwargs)
# with synchronizer
z = open_array(store, synchronizer=ThreadSynchronizer(), **kwargs)
assert isinstance(z, Array)
# with path
kwargs_no_path = kwargs.copy()
kwargs_no_path.pop('path', None)
z = open_array(store,
shape=100,
path='foo/bar',
mode='w',
**kwargs_no_path)
assert isinstance(z, Array)
assert 'foo/bar' == z.path
# with chunk store
meta_store = 'data/meta.zarr'
chunk_store = 'data/chunks.zarr'
z = open_array(store=meta_store,
chunk_store=chunk_store,
shape=11,
mode='w',
**kwargs)
z[:] = 42
assert os.path.abspath(meta_store) == z.store.path
assert os.path.abspath(chunk_store) == z.chunk_store.path
def create_array(self, read_only=False, **kwargs):
store = dict()
init_array(store, **kwargs)
return Array(store,
synchronizer=ThreadSynchronizer(),
read_only=read_only)