def test_group_init_errors_2(self):
store, chunk_store = self.create_store()
init_array(store, shape=1000, chunks=100, chunk_store=chunk_store)
# array blocks group
with pytest.raises(ValueError):
Group(store, chunk_store=chunk_store)
store.close()
def test_array_init(self):
# normal initialization
store = dict()
init_array(store, shape=100, chunks=10)
a = Array(store)
assert_is_instance(a, Array)
eq((100,), a.shape)
eq((10,), a.chunks)
eq('', a.path)
assert_is_none(a.name)
assert_is(store, a.store)
# initialize at path
store = dict()
init_array(store, shape=100, chunks=10, path='foo/bar')
a = Array(store, path='foo/bar')
assert_is_instance(a, Array)
eq((100,), a.shape)
eq((10,), a.chunks)
eq('foo/bar', a.path)
eq('/foo/bar', a.name)
assert_is(store, a.store)
# store not initialized
store = dict()
with assert_raises(KeyError):
Array(store)
# group is in the way
store = dict()
init_group(store, path='baz')
with assert_raises(KeyError):
Array(store, path='baz')
def test_init_array_overwrite_chunk_store(self):
# setup
store = self.create_store()
chunk_store = self.create_store()
store[array_meta_key] = encode_array_metadata(
dict(shape=(2000,),
chunks=(200,),
dtype=np.dtype('u1'),
compressor=None,
fill_value=0,
filters=None,
order='F')
)
chunk_store['0'] = b'aaa'
chunk_store['1'] = b'bbb'
# don't overwrite (default)
with assert_raises(ValueError):
init_array(store, shape=1000, chunks=100, chunk_store=chunk_store)
# do overwrite
try:
init_array(store, shape=1000, chunks=100, dtype='i4',
overwrite=True, chunk_store=chunk_store)
except NotImplementedError:
pass
else:
assert array_meta_key in store
meta = decode_array_metadata(store[array_meta_key])
eq(ZARR_FORMAT, meta['zarr_format'])
eq((1000,), meta['shape'])
eq((100,), meta['chunks'])
eq(np.dtype('i4'), meta['dtype'])
assert '0' not in chunk_store
assert '1' not in chunk_store
def test_init_array_overwrite(self):
# setup
store = self.create_store()
store[array_meta_key] = encode_array_metadata(
dict(shape=(2000, ),
chunks=(200, ),
dtype=np.dtype('u1'),
compressor=Zlib(1).get_config(),
fill_value=0,
order='F',
filters=None))
# don't overwrite (default)
with pytest.raises(ValueError):
init_array(store, shape=1000, chunks=100)
# do overwrite
try:
init_array(store,
shape=1000,
chunks=100,
dtype='i4',
overwrite=True)
except NotImplementedError:
pass
else:
assert array_meta_key in store
meta = decode_array_metadata(store[array_meta_key])
assert ZARR_FORMAT == meta['zarr_format']
assert (1000, ) == meta['shape']
assert (100, ) == meta['chunks']
assert np.dtype('i4') == meta['dtype']
def test_init_array_overwrite_path(self):
# setup
path = 'foo/bar'
store = self.create_store()
meta = dict(shape=(2000,),
chunks=(200,),
dtype=np.dtype('u1'),
compressor=Zlib(1).get_config(),
fill_value=0,
order='F',
filters=None)
store[array_meta_key] = encode_array_metadata(meta)
store[path + '/' + array_meta_key] = encode_array_metadata(meta)
# don't overwrite
with assert_raises(ValueError):
init_array(store, shape=1000, chunks=100, path=path)
# do overwrite
try:
init_array(store, shape=1000, chunks=100, dtype='i4', path=path,
overwrite=True)
except NotImplementedError:
pass
else:
assert group_meta_key in store
assert array_meta_key not in store
assert (path + '/' + array_meta_key) in store
# should have been overwritten
meta = decode_array_metadata(store[path + '/' + array_meta_key])
eq(ZARR_FORMAT, meta['zarr_format'])
eq((1000,), meta['shape'])
eq((100,), meta['chunks'])
eq(np.dtype('i4'), meta['dtype'])
def create_array(read_only=False, **kwargs):
path = mkdtemp()
atexit.register(shutil.rmtree, path)
store = DirectoryStore(path)
kwargs.setdefault('compressor', Zlib(1))
init_array(store, **kwargs)
return Array(store, read_only=read_only)
def test_init_array_overwrite(self):
# setup
store = self.create_store()
store[array_meta_key] = encode_array_metadata(
dict(shape=(2000,),
chunks=(200,),
dtype=np.dtype('u1'),
compressor=Zlib(1).get_config(),
fill_value=0,
order='F',
filters=None)
)
# don't overwrite (default)
with assert_raises(ValueError):
init_array(store, shape=1000, chunks=100)
# do overwrite
try:
init_array(store, shape=1000, chunks=100, dtype='i4',
overwrite=True)
except NotImplementedError:
pass
else:
assert array_meta_key in store
meta = decode_array_metadata(store[array_meta_key])
eq(ZARR_FORMAT, meta['zarr_format'])
eq((1000,), meta['shape'])
eq((100,), meta['chunks'])
eq(np.dtype('i4'), meta['dtype'])
def create_meta_store(slide: OpenSlide, tilesize: int) -> Dict[str, bytes]:
"""Creates a dict containing the zarr metadata for the multiscale openslide image."""
store = dict()
root_attrs = {
"multiscales": [{
"name":
Path(slide._filename).name,
"datasets": [{
"path": str(i)
} for i in range(slide.level_count)],
"version":
"0.1",
}]
}
init_group(store)
init_attrs(store, root_attrs)
for i, (x, y) in enumerate(slide.level_dimensions):
init_array(
store,
path=str(i),
shape=(y, x, 4),
chunks=(tilesize, tilesize, 4),
dtype="|u1",
compressor=None,
)
return store
def create_array(read_only=False, **kwargs):
path = mktemp(suffix='.dbm')
atexit.register(os.remove, path)
store = DBMStore(path, flag='n', open=bsddb3.btopen)
kwargs.setdefault('compressor', Zlib(1))
init_array(store, **kwargs)
return Array(store, read_only=read_only)
def test_init_array_overwrite_group(self):
# setup
path = 'foo/bar'
store = self.create_store()
store[path + '/' + group_meta_key] = encode_group_metadata()
# don't overwrite
with pytest.raises(ValueError):
init_array(store, shape=1000, chunks=100, path=path)
# do overwrite
try:
init_array(store,
shape=1000,
chunks=100,
dtype='i4',
path=path,
overwrite=True)
except NotImplementedError:
pass
else:
assert (path + '/' + group_meta_key) not in store
assert (path + '/' + array_meta_key) in store
meta = decode_array_metadata(store[path + '/' + array_meta_key])
assert ZARR_FORMAT == meta['zarr_format']
assert (1000, ) == meta['shape']
assert (100, ) == meta['chunks']
assert np.dtype('i4') == meta['dtype']
def test_array_init(self):
# normal initialization
store = dict()
init_array(store, shape=100, chunks=10)
a = Array(store)
assert_is_instance(a, Array)
eq((100,), a.shape)
eq((10,), a.chunks)
eq('', a.path)
assert_is_none(a.name)
assert_is(store, a.store)
# initialize at path
store = dict()
init_array(store, shape=100, chunks=10, path='foo/bar')
a = Array(store, path='foo/bar')
assert_is_instance(a, Array)
eq((100,), a.shape)
eq((10,), a.chunks)
eq('foo/bar', a.path)
eq('/foo/bar', a.name)
assert_is(store, a.store)
# store not initialized
store = dict()
with assert_raises(ValueError):
Array(store)
# group is in the way
store = dict()
init_group(store, path='baz')
with assert_raises(ValueError):
Array(store, path='baz')
def create_array(read_only=False, **kwargs):
path = mkdtemp()
atexit.register(shutil.rmtree, path)
store = NestedDirectoryStore(path)
kwargs.setdefault('compressor', Zlib(1))
init_array(store, **kwargs)
return Array(store, read_only=read_only)
def create_array(self, read_only=False, **kwargs):
store = dict()
cache_metadata = kwargs.pop('cache_metadata', True)
cache_attrs = kwargs.pop('cache_attrs', True)
init_array(store, **kwargs)
return Array(store, synchronizer=ThreadSynchronizer(),
read_only=read_only, cache_metadata=cache_metadata,
cache_attrs=cache_attrs)
def create_array(self, read_only=False, **kwargs):
store = dict()
cache_metadata = kwargs.pop('cache_metadata', True)
cache_attrs = kwargs.pop('cache_attrs', True)
init_array(store, **kwargs)
return Array(store,
synchronizer=ThreadSynchronizer(),
read_only=read_only,
cache_metadata=cache_metadata,
cache_attrs=cache_attrs)
def create_array(read_only=False, **kwargs):
path = mktemp(suffix='.lmdb')
atexit_rmtree(path)
try:
store = LMDBStore(path, buffers=False)
except ImportError: # pragma: no cover
raise SkipTest('lmdb not installed')
kwargs.setdefault('compressor', Zlib(1))
init_array(store, **kwargs)
return Array(store, read_only=read_only)
def create_array(self, read_only=False, **kwargs):
path = tempfile.mkdtemp()
atexit.register(atexit_rmtree, path)
store = DirectoryStore(path)
init_array(store, **kwargs)
sync_path = tempfile.mkdtemp()
atexit.register(atexit_rmtree, sync_path)
synchronizer = ProcessSynchronizer(sync_path)
return Array(store, synchronizer=synchronizer,
read_only=read_only, cache_metadata=False)
def create_array(self, read_only=False, **kwargs):
store = KVStore(dict())
cache_metadata = kwargs.pop('cache_metadata', True)
cache_attrs = kwargs.pop('cache_attrs', True)
write_empty_chunks = kwargs.pop('write_empty_chunks', True)
init_array(store, **kwargs)
return Array(store,
synchronizer=ThreadSynchronizer(),
read_only=read_only,
cache_metadata=cache_metadata,
cache_attrs=cache_attrs,
write_empty_chunks=write_empty_chunks)
def create_array(self, read_only=False, **kwargs):
path = tempfile.mkdtemp()
atexit.register(atexit_rmtree, path)
store = DirectoryStore(path)
init_array(store, **kwargs)
sync_path = tempfile.mkdtemp()
atexit.register(atexit_rmtree, sync_path)
synchronizer = ProcessSynchronizer(sync_path)
return Array(store,
synchronizer=synchronizer,
read_only=read_only,
cache_metadata=False)
def create_array(read_only=False, **kwargs):
store = dict()
dtype = kwargs.get('dtype', None)
filters = [
Delta(dtype=dtype),
FixedScaleOffset(dtype=dtype, scale=1, offset=0),
]
kwargs.setdefault('filters', filters)
compressor = Zlib(1)
kwargs.setdefault('compressor', compressor)
init_array(store, **kwargs)
return Array(store, read_only=read_only)
def create_array(read_only=False, **kwargs):
store = dict()
dtype = kwargs.get('dtype', None)
filters = [
Delta(dtype=dtype),
FixedScaleOffset(dtype=dtype, scale=1, offset=0),
]
kwargs.setdefault('filters', filters)
compressor = Zlib(1)
kwargs.setdefault('compressor', compressor)
init_array(store, **kwargs)
return Array(store, read_only=read_only)
def test_init_array(self):
store = self.create_store()
init_array(store, shape=1000, chunks=100)
# check metadata
assert array_meta_key in store
meta = decode_array_metadata(store[array_meta_key])
eq(ZARR_FORMAT, meta['zarr_format'])
eq((1000,), meta['shape'])
eq((100,), meta['chunks'])
eq(np.dtype(None), meta['dtype'])
eq(default_compressor.get_config(), meta['compressor'])
assert_is_none(meta['fill_value'])
def test_init_array(self):
store = self.create_store()
path = 'arr1'
init_array(store, path=path, shape=1000, chunks=100)
# check metadata
mkey = meta_root + path + '.array.json'
assert mkey in store
meta = store._metadata_class.decode_array_metadata(store[mkey])
assert (1000, ) == meta['shape']
assert (100, ) == meta['chunk_grid']['chunk_shape']
assert np.dtype(None) == meta['data_type']
assert meta['chunk_grid']['separator'] == "/"
def test_init_array(self):
store = self.create_store()
init_array(store, shape=1000, chunks=100)
# check metadata
assert array_meta_key in store
meta = decode_array_metadata(store[array_meta_key])
assert ZARR_FORMAT == meta['zarr_format']
assert (1000, ) == meta['shape']
assert (100, ) == meta['chunks']
assert np.dtype(None) == meta['dtype']
assert default_compressor.get_config() == meta['compressor']
assert meta['fill_value'] is None
def test_list_prefix(self):
store = self.create_store()
path = 'arr1'
init_array(store, path=path, shape=1000, chunks=100)
expected = [meta_root + 'arr1.array.json', 'zarr.json']
assert sorted(store.list_prefix('')) == expected
expected = [meta_root + 'arr1.array.json']
assert sorted(store.list_prefix(meta_root.rstrip('/'))) == expected
# cannot start prefix with '/'
with pytest.raises(ValueError):
store.list_prefix(prefix='/' + meta_root.rstrip('/'))
def test_astype_no_filters(self):
shape = (100,)
dtype = np.dtype(np.int8)
astype = np.dtype(np.float32)
store = dict()
init_array(store, shape=shape, chunks=10, dtype=dtype)
data = np.arange(np.prod(shape), dtype=dtype).reshape(shape)
z1 = Array(store)
z1[...] = data
z2 = z1.astype(astype)
expected = data.astype(astype)
assert_array_equal(expected, z2)
eq(z2.read_only, True)
def test_init_array(self):
store = self.create_store()
init_array(store, shape=1000, chunks=100)
# check metadata
assert array_meta_key in store
meta = decode_array_metadata(store[array_meta_key])
eq(ZARR_FORMAT, meta['zarr_format'])
eq((1000, ), meta['shape'])
eq((100, ), meta['chunks'])
eq(np.dtype(None), meta['dtype'])
eq(default_compressor.get_config(), meta['compressor'])
assert_is_none(meta['fill_value'])
# check attributes
assert attrs_key in store
eq(dict(), json.loads(text_type(store[attrs_key], 'ascii')))
def create_array(self, read_only=False, **kwargs):
path = tempfile.mkdtemp()
atexit.register(atexit_rmtree, path)
store = DirectoryStore(path)
cache_metadata = kwargs.pop('cache_metadata', False)
cache_attrs = kwargs.pop('cache_attrs', False)
write_empty_chunks = kwargs.pop('write_empty_chunks', True)
init_array(store, **kwargs)
sync_path = tempfile.mkdtemp()
atexit.register(atexit_rmtree, sync_path)
synchronizer = ProcessSynchronizer(sync_path)
return Array(store,
synchronizer=synchronizer,
read_only=read_only,
cache_metadata=cache_metadata,
cache_attrs=cache_attrs,
write_empty_chunks=write_empty_chunks)
def test_init_array_path(self):
path = 'foo/bar'
store = self.create_store()
init_array(store, shape=1000, chunks=100, path=path)
# check metadata
key = path + '/' + array_meta_key
assert key in store
meta = decode_array_metadata(store[key])
eq(ZARR_FORMAT, meta['zarr_format'])
eq((1000,), meta['shape'])
eq((100,), meta['chunks'])
eq(np.dtype(None), meta['dtype'])
eq(default_compressor.get_config(), meta['compressor'])
assert_is_none(meta['fill_value'])
# check attributes
key = path + '/' + attrs_key
assert key in store
eq(dict(), json.loads(text_type(store[key], 'ascii')))
def test_init_array(self, dimension_separator_fixture_v3):
pass_dim_sep, want_dim_sep = dimension_separator_fixture_v3
store = self.create_store()
path = 'arr1'
init_array(store, path=path, shape=1000, chunks=100,
dimension_separator=pass_dim_sep)
# check metadata
mkey = meta_root + path + '.array.json'
assert mkey in store
meta = store._metadata_class.decode_array_metadata(store[mkey])
assert (1000,) == meta['shape']
assert (100,) == meta['chunk_grid']['chunk_shape']
assert np.dtype(None) == meta['data_type']
assert default_compressor == meta['compressor']
assert meta['fill_value'] is None
# Missing MUST be assumed to be "/"
assert meta['chunk_grid']['separator'] is want_dim_sep
store.close()
def test_init_array_overwrite_path(self):
# setup
path = 'foo/bar'
store = self.create_store()
meta = dict(shape=(2000, ),
chunks=(200, ),
dtype=np.dtype('u1'),
compressor=Zlib(1).get_config(),
fill_value=0,
order='F',
filters=None)
store[array_meta_key] = encode_array_metadata(meta)
store[path + '/' + array_meta_key] = encode_array_metadata(meta)
# don't overwrite
with pytest.raises(ValueError):
init_array(store, shape=1000, chunks=100, path=path)
# do overwrite
try:
init_array(store,
shape=1000,
chunks=100,
dtype='i4',
path=path,
overwrite=True)
except NotImplementedError:
pass
else:
assert group_meta_key in store
assert array_meta_key not in store
assert (path + '/' + array_meta_key) in store
# should have been overwritten
meta = decode_array_metadata(store[path + '/' + array_meta_key])
assert ZARR_FORMAT == meta['zarr_format']
assert (1000, ) == meta['shape']
assert (100, ) == meta['chunks']
assert np.dtype('i4') == meta['dtype']
def test_init_array_overwrite_chunk_store(self):
# setup
store = self.create_store()
chunk_store = self.create_store()
store[array_meta_key] = encode_array_metadata(
dict(shape=(2000, ),
chunks=(200, ),
dtype=np.dtype('u1'),
compressor=None,
fill_value=0,
filters=None,
order='F'))
chunk_store['0'] = b'aaa'
chunk_store['1'] = b'bbb'
# don't overwrite (default)
with assert_raises(ValueError):
init_array(store, shape=1000, chunks=100, chunk_store=chunk_store)
# do overwrite
try:
init_array(store,
shape=1000,
chunks=100,
dtype='i4',
overwrite=True,
chunk_store=chunk_store)
except NotImplementedError:
pass
else:
assert array_meta_key in store
meta = decode_array_metadata(store[array_meta_key])
eq(ZARR_FORMAT, meta['zarr_format'])
eq((1000, ), meta['shape'])
eq((100, ), meta['chunks'])
eq(np.dtype('i4'), meta['dtype'])
assert '0' not in chunk_store
assert '1' not in chunk_store
def test_group():
# test the group() convenience function
# basic usage
g = group()
assert_is_instance(g, Group)
eq('', g.path)
eq('/', g.name)
# usage with custom store
store = dict()
g = group(store=store)
assert_is_instance(g, Group)
assert_is(store, g.store)
# overwrite behaviour
store = dict()
init_array(store, shape=100, chunks=10)
with assert_raises(KeyError):
group(store)
g = group(store, overwrite=True)
assert_is_instance(g, Group)
assert_is(store, g.store)
def test_group():
# test the group() convenience function
# basic usage
g = group()
assert isinstance(g, Group)
assert '' == g.path
assert '/' == g.name
# usage with custom store
store = dict()
g = group(store=store)
assert isinstance(g, Group)
assert store is g.store
# overwrite behaviour
store = dict()
init_array(store, shape=100, chunks=10)
with pytest.raises(ValueError):
group(store)
g = group(store, overwrite=True)
assert isinstance(g, Group)
assert store is g.store
def test_group():
# test the group() convenience function
# basic usage
g = group()
assert_is_instance(g, Group)
eq('', g.path)
eq('/', g.name)
# usage with custom store
store = dict()
g = group(store=store)
assert_is_instance(g, Group)
assert_is(store, g.store)
# overwrite behaviour
store = dict()
init_array(store, shape=100, chunks=10)
with assert_raises(ValueError):
group(store)
g = group(store, overwrite=True)
assert_is_instance(g, Group)
assert_is(store, g.store)
def test_group():
# test the group() convenience function
# basic usage
g = group()
assert isinstance(g, Group)
assert '' == g.path
assert '/' == g.name
# usage with custom store
store = dict()
g = group(store=store)
assert isinstance(g, Group)
assert store is g.store
# overwrite behaviour
store = dict()
init_array(store, shape=100, chunks=10)
with pytest.raises(ValueError):
group(store)
g = group(store, overwrite=True)
assert isinstance(g, Group)
assert store is g.store
def test_init_array_overwrite_group(self):
# setup
path = 'foo/bar'
store = self.create_store()
store[path + '/' + group_meta_key] = encode_group_metadata()
# don't overwrite
with assert_raises(ValueError):
init_array(store, shape=1000, chunks=100, path=path)
# do overwrite
try:
init_array(store, shape=1000, chunks=100, dtype='i4', path=path,
overwrite=True)
except NotImplementedError:
pass
else:
assert (path + '/' + group_meta_key) not in store
assert (path + '/' + array_meta_key) in store
meta = decode_array_metadata(store[path + '/' + array_meta_key])
eq(ZARR_FORMAT, meta['zarr_format'])
eq((1000,), meta['shape'])
eq((100,), meta['chunks'])
eq(np.dtype('i4'), meta['dtype'])
def open_array(store, mode='a', shape=None, chunks=True, dtype=None, compressor='default',
fill_value=0, order='C', synchronizer=None, filters=None,
cache_metadata=True, cache_attrs=True, path=None, object_codec=None,
**kwargs):
"""Open an array using file-mode-like semantics.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
mode : {'r', 'r+', 'a', 'w', 'w-'}, optional
Persistence mode: 'r' means read only (must exist); 'r+' means
read/write (must exist); 'a' means read/write (create if doesn't
exist); 'w' means create (overwrite if exists); 'w-' means create
(fail if exists).
shape : int or tuple of ints, optional
Array shape.
chunks : int or tuple of ints, optional
Chunk shape. If True, will be guessed from `shape` and `dtype`. If
False, will be set to `shape`, i.e., single chunk for the whole array.
dtype : string or dtype, optional
NumPy dtype.
compressor : Codec, optional
Primary compressor.
fill_value : object, optional
Default value to use for uninitialized portions of the array.
order : {'C', 'F'}, optional
Memory layout to be used within each chunk.
synchronizer : object, optional
Array synchronizer.
filters : sequence, optional
Sequence of filters to use to encode chunk data prior to compression.
cache_metadata : bool, optional
If True, array configuration metadata will be cached for the
lifetime of the object. If False, array metadata will be reloaded
prior to all data access and modification operations (may incur
overhead depending on storage and data access pattern).
cache_attrs : bool, optional
If True (default), user attributes will be cached for attribute read
operations. If False, user attributes are reloaded from the store prior
to all attribute read operations.
path : string, optional
Array path within store.
object_codec : Codec, optional
A codec to encode object arrays, only needed if dtype=object.
Returns
-------
z : zarr.core.Array
Examples
--------
>>> import numpy as np
>>> import zarr
>>> z1 = zarr.open_array('data/example.zarr', mode='w', shape=(10000, 10000),
... chunks=(1000, 1000), fill_value=0)
>>> z1[:] = np.arange(100000000).reshape(10000, 10000)
>>> z1
<zarr.core.Array (10000, 10000) float64>
>>> z2 = zarr.open_array('data/example.zarr', mode='r')
>>> z2
<zarr.core.Array (10000, 10000) float64 read-only>
>>> np.all(z1[:] == z2[:])
True
Notes
-----
There is no need to close an array. Data are automatically flushed to the
file system.
"""
# use same mode semantics as h5py
# r : read only, must exist
# r+ : read/write, must exist
# w : create, delete if exists
# w- or x : create, fail if exists
# a : read/write if exists, create otherwise (default)
# handle polymorphic store arg
store = normalize_store_arg(store, clobber=(mode == 'w'))
path = normalize_storage_path(path)
# API compatibility with h5py
compressor, fill_value = _kwargs_compat(compressor, fill_value, kwargs)
# ensure fill_value of correct type
if fill_value is not None:
fill_value = np.array(fill_value, dtype=dtype)[()]
# ensure store is initialized
if mode in ['r', 'r+']:
if contains_group(store, path=path):
err_contains_group(path)
elif not contains_array(store, path=path):
err_array_not_found(path)
elif mode == 'w':
init_array(store, shape=shape, chunks=chunks, dtype=dtype,
compressor=compressor, fill_value=fill_value,
order=order, filters=filters, overwrite=True, path=path,
object_codec=object_codec)
elif mode == 'a':
if contains_group(store, path=path):
err_contains_group(path)
elif not contains_array(store, path=path):
init_array(store, shape=shape, chunks=chunks, dtype=dtype,
compressor=compressor, fill_value=fill_value,
order=order, filters=filters, path=path,
object_codec=object_codec)
elif mode in ['w-', 'x']:
if contains_group(store, path=path):
err_contains_group(path)
elif contains_array(store, path=path):
err_contains_array(path)
else:
init_array(store, shape=shape, chunks=chunks, dtype=dtype,
compressor=compressor, fill_value=fill_value,
order=order, filters=filters, path=path,
object_codec=object_codec)
# determine read only status
read_only = mode == 'r'
# instantiate array
z = Array(store, read_only=read_only, synchronizer=synchronizer,
cache_metadata=cache_metadata, cache_attrs=cache_attrs, path=path)
return z
def create_array(read_only=False, **kwargs):
store = dict()
init_array(store, path='foo/bar', **kwargs)
return Array(store, path='foo/bar', read_only=read_only)
def create_array(read_only=False, **kwargs):
store = dict()
# separate chunk store
chunk_store = dict()
init_array(store, chunk_store=chunk_store, **kwargs)
return Array(store, read_only=read_only, chunk_store=chunk_store)
def test_init_array_compat(self):
store = self.create_store()
init_array(store, shape=1000, chunks=100, compressor='none')
meta = decode_array_metadata(store[array_meta_key])
assert meta['compressor'] is None
def open_array(store=None, mode='a', shape=None, chunks=None, dtype=None,
compressor='default', fill_value=0, order='C',
synchronizer=None, filters=None, cache_metadata=True,
path=None, **kwargs):
"""Open array using mode-like semantics.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system.
mode : {'r', 'r+', 'a', 'w', 'w-'}
Persistence mode: 'r' means read only (must exist); 'r+' means
read/write (must exist); 'a' means read/write (create if doesn't
exist); 'w' means create (overwrite if exists); 'w-' means create
(fail if exists).
shape : int or tuple of ints
Array shape.
chunks : int or tuple of ints, optional
Chunk shape. If not provided, will be guessed from `shape` and `dtype`.
dtype : string or dtype, optional
NumPy dtype.
compressor : Codec, optional
Primary compressor.
fill_value : object
Default value to use for uninitialized portions of the array.
order : {'C', 'F'}, optional
Memory layout to be used within each chunk.
synchronizer : object, optional
Array synchronizer.
filters : sequence, optional
Sequence of filters to use to encode chunk data prior to compression.
cache_metadata : bool, optional
If True, array configuration metadata will be cached for the
lifetime of the object. If False, array metadata will be reloaded
prior to all data access and modification operations (may incur
overhead depending on storage and data access pattern).
path : string, optional
Array path.
Returns
-------
z : zarr.core.Array
Examples
--------
>>> import numpy as np
>>> import zarr
>>> z1 = zarr.open_array('example.zarr', mode='w', shape=(10000, 10000),
... chunks=(1000, 1000), fill_value=0)
>>> z1[:] = np.arange(100000000).reshape(10000, 10000)
>>> z1
Array((10000, 10000), float64, chunks=(1000, 1000), order=C)
nbytes: 762.9M; nbytes_stored: 23.0M; ratio: 33.2; initialized: 100/100
compressor: Blosc(cname='lz4', clevel=5, shuffle=1)
store: DirectoryStore
>>> z2 = zarr.open_array('example.zarr', mode='r')
>>> z2
Array((10000, 10000), float64, chunks=(1000, 1000), order=C)
nbytes: 762.9M; nbytes_stored: 23.0M; ratio: 33.2; initialized: 100/100
compressor: Blosc(cname='lz4', clevel=5, shuffle=1)
store: DirectoryStore
>>> np.all(z1[:] == z2[:])
True
Notes
-----
There is no need to close an array. Data are automatically flushed to the
file system.
""" # flake8: noqa
# use same mode semantics as h5py
# r : read only, must exist
# r+ : read/write, must exist
# w : create, delete if exists
# w- or x : create, fail if exists
# a : read/write if exists, create otherwise (default)
# handle polymorphic store arg
store = _handle_store_arg(store)
path = normalize_storage_path(path)
# compatibility
compressor, fill_value = _handle_kwargs(compressor, fill_value, kwargs)
# ensure store is initialized
if mode in ['r', 'r+']:
if contains_group(store, path=path):
err_contains_group(path)
elif not contains_array(store, path=path):
err_array_not_found(path)
elif mode == 'w':
init_array(store, shape=shape, chunks=chunks, dtype=dtype,
compressor=compressor, fill_value=fill_value,
order=order, filters=filters, overwrite=True, path=path)
elif mode == 'a':
if contains_group(store, path=path):
err_contains_group(path)
elif not contains_array(store, path=path):
init_array(store, shape=shape, chunks=chunks, dtype=dtype,
compressor=compressor, fill_value=fill_value,
order=order, filters=filters, path=path)
elif mode in ['w-', 'x']:
if contains_group(store, path=path):
err_contains_group(path)
elif contains_array(store, path=path):
err_contains_array(path)
else:
init_array(store, shape=shape, chunks=chunks, dtype=dtype,
compressor=compressor, fill_value=fill_value,
order=order, filters=filters, path=path)
# determine read only status
read_only = mode == 'r'
# instantiate array
z = Array(store, read_only=read_only, synchronizer=synchronizer,
cache_metadata=cache_metadata, path=path)
return z
def create(shape, chunks=True, dtype=None, compressor='default',
fill_value=0, order='C', store=None, synchronizer=None,
overwrite=False, path=None, chunk_store=None, filters=None,
cache_metadata=True, cache_attrs=True, read_only=False,
object_codec=None, **kwargs):
"""Create an array.
Parameters
----------
shape : int or tuple of ints
Array shape.
chunks : int or tuple of ints, optional
Chunk shape. If True, will be guessed from `shape` and `dtype`. If
False, will be set to `shape`, i.e., single chunk for the whole array.
dtype : string or dtype, optional
NumPy dtype.
compressor : Codec, optional
Primary compressor.
fill_value : object
Default value to use for uninitialized portions of the array.
order : {'C', 'F'}, optional
Memory layout to be used within each chunk.
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
synchronizer : object, optional
Array synchronizer.
overwrite : bool, optional
If True, delete all pre-existing data in `store` at `path` before
creating the array.
path : string, optional
Path under which array is stored.
chunk_store : MutableMapping, optional
Separate storage for chunks. If not provided, `store` will be used
for storage of both chunks and metadata.
filters : sequence of Codecs, optional
Sequence of filters to use to encode chunk data prior to compression.
cache_metadata : bool, optional
If True, array configuration metadata will be cached for the
lifetime of the object. If False, array metadata will be reloaded
prior to all data access and modification operations (may incur
overhead depending on storage and data access pattern).
cache_attrs : bool, optional
If True (default), user attributes will be cached for attribute read
operations. If False, user attributes are reloaded from the store prior
to all attribute read operations.
read_only : bool, optional
True if array should be protected against modification.
object_codec : Codec, optional
A codec to encode object arrays, only needed if dtype=object.
Returns
-------
z : zarr.core.Array
Examples
--------
Create an array with default settings::
>>> import zarr
>>> z = zarr.create((10000, 10000), chunks=(1000, 1000))
>>> z
<zarr.core.Array (10000, 10000) float64>
Create an array with different some different configuration options::
>>> from numcodecs import Blosc
>>> compressor = Blosc(cname='zstd', clevel=1, shuffle=Blosc.BITSHUFFLE)
>>> z = zarr.create((10000, 10000), chunks=(1000, 1000), dtype='i1', order='F',
... compressor=compressor)
>>> z
<zarr.core.Array (10000, 10000) int8>
To create an array with object dtype requires a filter that can handle Python object
encoding, e.g., `MsgPack` or `Pickle` from `numcodecs`::
>>> from numcodecs import MsgPack
>>> z = zarr.create((10000, 10000), chunks=(1000, 1000), dtype=object,
... object_codec=MsgPack())
>>> z
<zarr.core.Array (10000, 10000) object>
Example with some filters, and also storing chunks separately from metadata::
>>> from numcodecs import Quantize, Adler32
>>> store, chunk_store = dict(), dict()
>>> z = zarr.create((10000, 10000), chunks=(1000, 1000), dtype='f8',
... filters=[Quantize(digits=2, dtype='f8'), Adler32()],
... store=store, chunk_store=chunk_store)
>>> z
<zarr.core.Array (10000, 10000) float64>
"""
# handle polymorphic store arg
store = normalize_store_arg(store)
# API compatibility with h5py
compressor, fill_value = _kwargs_compat(compressor, fill_value, kwargs)
# initialize array metadata
init_array(store, shape=shape, chunks=chunks, dtype=dtype, compressor=compressor,
fill_value=fill_value, order=order, overwrite=overwrite, path=path,
chunk_store=chunk_store, filters=filters, object_codec=object_codec)
# instantiate array
z = Array(store, path=path, chunk_store=chunk_store, synchronizer=synchronizer,
cache_metadata=cache_metadata, cache_attrs=cache_attrs, read_only=read_only)
return z
def create_array(read_only=False, **kwargs):
store = CustomMapping()
kwargs.setdefault('compressor', Zlib(1))
init_array(store, **kwargs)
return Array(store, read_only=read_only)
def create_array(self, read_only=False, **kwargs):
store = dict()
compressor = LZMA(preset=1)
kwargs.setdefault('compressor', compressor)
init_array(store, **kwargs)
return Array(store, read_only=read_only)
def create_array(self, read_only=False, **kwargs):
store = dict()
compressor = Blosc(cname='zstd', clevel=1, shuffle=1)
kwargs.setdefault('compressor', compressor)
init_array(store, **kwargs)
return Array(store, read_only=read_only)
def create_array(self, read_only=False, **kwargs):
store = dict()
kwargs.setdefault('compressor', None)
init_array(store, **kwargs)
return Array(store, read_only=read_only)
def create_array(read_only=False, **kwargs):
store = dict()
kwargs.setdefault('compressor', Zlib(level=1))
init_array(store, **kwargs)
return Array(store, read_only=read_only, cache_metadata=False)
def create_array(read_only=False, **kwargs):
store = dict()
kwargs.setdefault('compressor', Zlib(level=1))
init_array(store, **kwargs)
return Array(store, read_only=read_only, cache_metadata=False)
def test_init_array_compat(self):
store = self.create_store()
init_array(store, shape=1000, chunks=100, compressor='none')
meta = decode_array_metadata(store[array_meta_key])
assert_is_none(meta['compressor'])
def create_array(self, read_only=False, **kwargs):
store = dict()
compressor = LZMA(preset=1)
kwargs.setdefault('compressor', compressor)
init_array(store, **kwargs)
return Array(store, read_only=read_only)
def create_array(self, read_only=False, **kwargs):
store = dict()
compressor = Blosc(cname='zstd', clevel=1, shuffle=1)
kwargs.setdefault('compressor', compressor)
init_array(store, **kwargs)
return Array(store, read_only=read_only)
def create_array(self, read_only=False, **kwargs):
store = dict()
init_array(store, **kwargs)
return Array(store, synchronizer=ThreadSynchronizer(),
read_only=read_only)
def create_array(self, read_only=False, **kwargs):
store = dict()
kwargs.setdefault('compressor', None)
init_array(store, **kwargs)
return Array(store, read_only=read_only)
def create(shape, chunks=None, dtype=None, compressor='default',
fill_value=0, order='C', store=None, synchronizer=None,
overwrite=False, path=None, chunk_store=None, filters=None,
cache_metadata=True, **kwargs):
"""Create an array.
Parameters
----------
shape : int or tuple of ints
Array shape.
chunks : int or tuple of ints, optional
Chunk shape. If not provided, will be guessed from `shape` and `dtype`.
dtype : string or dtype, optional
NumPy dtype.
compressor : Codec, optional
Primary compressor.
fill_value : object
Default value to use for uninitialized portions of the array.
order : {'C', 'F'}, optional
Memory layout to be used within each chunk.
store : MutableMapping or string
Store or path to directory in file system.
synchronizer : object, optional
Array synchronizer.
overwrite : bool, optional
If True, delete all pre-existing data in `store` at `path` before
creating the array.
path : string, optional
Path under which array is stored.
chunk_store : MutableMapping, optional
Separate storage for chunks. If not provided, `store` will be used
for storage of both chunks and metadata.
filters : sequence of Codecs, optional
Sequence of filters to use to encode chunk data prior to compression.
cache_metadata : bool, optional
If True, array configuration metadata will be cached for the
lifetime of the object. If False, array metadata will be reloaded
prior to all data access and modification operations (may incur
overhead depending on storage and data access pattern).
Returns
-------
z : zarr.core.Array
Examples
--------
Create an array with default settings::
>>> import zarr
>>> z = zarr.create((10000, 10000), chunks=(1000, 1000))
>>> z
Array((10000, 10000), float64, chunks=(1000, 1000), order=C)
nbytes: 762.9M; nbytes_stored: 323; ratio: 2476780.2; initialized: 0/100
compressor: Blosc(cname='lz4', clevel=5, shuffle=1)
store: dict
""" # flake8: noqa
# handle polymorphic store arg
store = _handle_store_arg(store)
# compatibility
compressor, fill_value = _handle_kwargs(compressor, fill_value, kwargs)
# initialize array metadata
init_array(store, shape=shape, chunks=chunks, dtype=dtype,
compressor=compressor, fill_value=fill_value, order=order,
overwrite=overwrite, path=path, chunk_store=chunk_store,
filters=filters)
# instantiate array
z = Array(store, path=path, chunk_store=chunk_store,
synchronizer=synchronizer, cache_metadata=cache_metadata)
return z
def create_array(read_only=False, **kwargs):
store = dict()
# separate chunk store
chunk_store = dict()
init_array(store, chunk_store=chunk_store, **kwargs)
return Array(store, read_only=read_only, chunk_store=chunk_store)
def test_group_init_errors_2(self):
store, chunk_store = self.create_store()
init_array(store, shape=1000, chunks=100, chunk_store=chunk_store)
# array blocks group
with assert_raises(KeyError):
Group(store, chunk_store=chunk_store)
def create_array(read_only=False, **kwargs):
store = dict()
init_array(store, path='foo/bar', **kwargs)
return Array(store, path='foo/bar', read_only=read_only)