def test_compression_args():
warnings.resetwarnings()
warnings.simplefilter('always')
z = create(100, compression='zlib', compression_opts=9)
assert_is_instance(z, Array)
eq('zlib', z.compressor.codec_id)
eq(9, z.compressor.level)
# 'compressor' overrides 'compression'
z = create(100, compressor=Zlib(9), compression='bz2', compression_opts=1)
assert_is_instance(z, Array)
eq('zlib', z.compressor.codec_id)
eq(9, z.compressor.level)
# 'compressor' ignores 'compression_opts'
z = create(100, compressor=Zlib(9), compression_opts=1)
assert_is_instance(z, Array)
eq('zlib', z.compressor.codec_id)
eq(9, z.compressor.level)
warnings.resetwarnings()
warnings.simplefilter('error')
with assert_raises(UserWarning):
# 'compressor' overrides 'compression'
create(100, compressor=Zlib(9), compression='bz2', compression_opts=1)
with assert_raises(UserWarning):
# 'compressor' ignores 'compression_opts'
create(100, compressor=Zlib(9), compression_opts=1)
warnings.resetwarnings()
warnings.simplefilter('always')
def test_compression_args():
z = create(100, compression='zlib', compression_opts=9)
assert isinstance(z, Array)
assert 'zlib' == z.compressor.codec_id
assert 9 == z.compressor.level
# 'compressor' overrides 'compression'
z = create(100, compressor=Zlib(9), compression='bz2', compression_opts=1)
assert isinstance(z, Array)
assert 'zlib' == z.compressor.codec_id
assert 9 == z.compressor.level
# 'compressor' ignores 'compression_opts'
z = create(100, compressor=Zlib(9), compression_opts=1)
assert isinstance(z, Array)
assert 'zlib' == z.compressor.codec_id
assert 9 == z.compressor.level
# cannot get warning tests to work on PY2
if not PY2: # pragma: py2 no cover
with pytest.warns(UserWarning):
# 'compressor' overrides 'compression'
create(100,
compressor=Zlib(9),
compression='bz2',
compression_opts=1)
with pytest.warns(UserWarning):
# 'compressor' ignores 'compression_opts'
create(100, compressor=Zlib(9), compression_opts=1)
def test_compression_args():
z = create(100, compression='zlib', compression_opts=9)
assert isinstance(z, Array)
assert 'zlib' == z.compressor.codec_id
assert 9 == z.compressor.level
# 'compressor' overrides 'compression'
z = create(100, compressor=Zlib(9), compression='bz2', compression_opts=1)
assert isinstance(z, Array)
assert 'zlib' == z.compressor.codec_id
assert 9 == z.compressor.level
# 'compressor' ignores 'compression_opts'
z = create(100, compressor=Zlib(9), compression_opts=1)
assert isinstance(z, Array)
assert 'zlib' == z.compressor.codec_id
assert 9 == z.compressor.level
with pytest.warns(UserWarning):
# 'compressor' overrides 'compression'
create(100, compressor=Zlib(9), compression='bz2', compression_opts=1)
with pytest.warns(UserWarning):
# 'compressor' ignores 'compression_opts'
create(100, compressor=Zlib(9), compression_opts=1)
def test_compression_args():
with warnings.catch_warnings():
warnings.simplefilter("default")
z = create(100, compression="zlib", compression_opts=9)
assert isinstance(z, Array)
assert "zlib" == z.compressor.codec_id
assert 9 == z.compressor.level
# 'compressor' overrides 'compression'
with pytest.warns(UserWarning):
z = create(100,
compressor=Zlib(9),
compression="bz2",
compression_opts=1)
assert isinstance(z, Array)
assert "zlib" == z.compressor.codec_id
assert 9 == z.compressor.level
# 'compressor' ignores 'compression_opts'
with pytest.warns(UserWarning):
z = create(100, compressor=Zlib(9), compression_opts=1)
assert isinstance(z, Array)
assert "zlib" == z.compressor.codec_id
assert 9 == z.compressor.level
with pytest.warns(UserWarning):
# 'compressor' overrides 'compression'
create(100,
compressor=Zlib(9),
compression="bz2",
compression_opts=1)
with pytest.warns(UserWarning):
# 'compressor' ignores 'compression_opts'
create(100, compressor=Zlib(9), compression_opts=1)
def test_open_like():
# zarr array
path = tempfile.mktemp()
atexit.register(shutil.rmtree, path)
z = full(100,
chunks=10,
dtype='f4',
compressor=Zlib(5),
fill_value=42,
order='F')
z2 = open_like(z, path)
eq(z.shape, z2.shape)
eq(z.chunks, z2.chunks)
eq(z.dtype, z2.dtype)
eq(z.compressor.get_config(), z2.compressor.get_config())
eq(z.fill_value, z2.fill_value)
eq(z.order, z2.order)
# numpy array
path = tempfile.mktemp()
atexit.register(shutil.rmtree, path)
a = np.empty(100, dtype='f4')
z3 = open_like(a, path, chunks=10)
eq(a.shape, z3.shape)
eq((10, ), z3.chunks)
eq(a.dtype, z3.dtype)
eq(0, z3.fill_value)
def test_encode_decode_array_1():
meta = dict(shape=(100, ),
chunks=(10, ),
dtype=np.dtype('f8'),
compressor=Zlib(1).get_config(),
fill_value=None,
filters=None,
order='C')
meta_json = '''{
"chunks": [10],
"compressor": {"id": "zlib", "level": 1},
"dtype": "<f8",
"fill_value": null,
"filters": null,
"order": "C",
"shape": [100],
"zarr_format": %s
}''' % ZARR_FORMAT
# test encoding
meta_enc = encode_array_metadata(meta)
assert_json_equal(meta_json, meta_enc)
# test decoding
meta_dec = decode_array_metadata(meta_enc)
assert ZARR_FORMAT == meta_dec['zarr_format']
assert meta['shape'] == meta_dec['shape']
assert meta['chunks'] == meta_dec['chunks']
assert meta['dtype'] == meta_dec['dtype']
assert meta['compressor'] == meta_dec['compressor']
assert meta['order'] == meta_dec['order']
assert meta_dec['fill_value'] is None
assert meta_dec['filters'] is None
def test_create():
# defaults
z = create(100)
assert isinstance(z, Array)
assert (100,) == z.shape
assert (100,) == z.chunks # auto-chunks
assert np.dtype(None) == z.dtype
assert 'blosc' == z.compressor.codec_id
assert 0 == z.fill_value
# all specified
z = create(100, chunks=10, dtype='i4', compressor=Zlib(1),
fill_value=42, order='F')
assert isinstance(z, Array)
assert (100,) == z.shape
assert (10,) == z.chunks
assert np.dtype('i4') == z.dtype
assert 'zlib' == z.compressor.codec_id
assert 1 == z.compressor.level
assert 42 == z.fill_value
assert 'F' == z.order
# with synchronizer
synchronizer = ThreadSynchronizer()
z = create(100, chunks=10, synchronizer=synchronizer)
assert isinstance(z, Array)
assert (100,) == z.shape
assert (10,) == z.chunks
assert synchronizer is z.synchronizer
# don't allow string as compressor arg
with pytest.raises(ValueError):
create(100, chunks=10, compressor='zlib')
# h5py compatibility
z = create(100, compression='zlib', compression_opts=9)
assert 'zlib' == z.compressor.codec_id
assert 9 == z.compressor.level
z = create(100, compression='default')
assert 'blosc' == z.compressor.codec_id
# errors
with pytest.raises(ValueError):
# bad compression argument
create(100, compression=1)
with pytest.raises(ValueError):
# bad fill value
create(100, dtype='i4', fill_value='foo')
# auto chunks
z = create(1000000000, chunks=True)
assert z.chunks[0] < z.shape[0]
z = create(1000000000, chunks=None) # backwards-compatibility
assert z.chunks[0] < z.shape[0]
# no chunks
z = create(1000000000, chunks=False)
assert z.chunks == z.shape
def test_init_array_overwrite(self):
# setup
store = self.create_store()
store[array_meta_key] = encode_array_metadata(
dict(shape=(2000, ),
chunks=(200, ),
dtype=np.dtype('u1'),
compressor=Zlib(1).get_config(),
fill_value=0,
order='F',
filters=None))
# don't overwrite (default)
with pytest.raises(ValueError):
init_array(store, shape=1000, chunks=100)
# do overwrite
try:
init_array(store,
shape=1000,
chunks=100,
dtype='i4',
overwrite=True)
except NotImplementedError:
pass
else:
assert array_meta_key in store
meta = decode_array_metadata(store[array_meta_key])
assert ZARR_FORMAT == meta['zarr_format']
assert (1000, ) == meta['shape']
assert (100, ) == meta['chunks']
assert np.dtype('i4') == meta['dtype']
def test_open_like(zarr_version):
kwargs = _init_creation_kwargs(zarr_version)
expected_zarr_version = DEFAULT_ZARR_VERSION if zarr_version is None else zarr_version
# zarr array
path = tempfile.mktemp()
atexit.register(shutil.rmtree, path)
z = full(100,
chunks=10,
dtype='f4',
compressor=Zlib(5),
fill_value=42,
order='F',
**kwargs)
z2 = open_like(z, path)
assert z.shape == z2.shape
assert z.chunks == z2.chunks
assert z.dtype == z2.dtype
assert z.compressor.get_config() == z2.compressor.get_config()
assert z.fill_value == z2.fill_value
assert z.order == z2.order
assert (z._store._store_version == z2._store._store_version ==
expected_zarr_version)
# numpy array
path = tempfile.mktemp()
atexit.register(shutil.rmtree, path)
a = np.empty(100, dtype='f4')
z3 = open_like(a, path, chunks=10, zarr_version=zarr_version)
assert a.shape == z3.shape
assert (10, ) == z3.chunks
assert a.dtype == z3.dtype
assert 0 == z3.fill_value
assert z3._store._store_version == expected_zarr_version
def test_full_like(zarr_version):
kwargs = _init_creation_kwargs(zarr_version)
expected_zarr_version = DEFAULT_ZARR_VERSION if zarr_version is None else zarr_version
z = full(100,
chunks=10,
dtype='f4',
compressor=Zlib(5),
fill_value=42,
order='F',
**kwargs)
z2 = full_like(z, path=kwargs.get('path', None))
assert z.shape == z2.shape
assert z.chunks == z2.chunks
assert z.dtype == z2.dtype
assert z.compressor.get_config() == z2.compressor.get_config()
assert z.fill_value == z2.fill_value
assert z.order == z2.order
assert (z._store._store_version == z2._store._store_version ==
expected_zarr_version)
# numpy array
a = np.empty(100, dtype='f4')
z3 = full_like(a, chunks=10, fill_value=42, **kwargs)
assert a.shape == z3.shape
assert (10, ) == z3.chunks
assert a.dtype == z3.dtype
assert 42 == z3.fill_value
assert z3._store._store_version == expected_zarr_version
with pytest.raises(TypeError):
# fill_value missing
full_like(a, chunks=10, **kwargs)
def test_ones_like(zarr_version):
kwargs = _init_creation_kwargs(zarr_version)
expected_zarr_version = DEFAULT_ZARR_VERSION if zarr_version is None else zarr_version
# zarr array
z = ones(100,
chunks=10,
dtype='f4',
compressor=Zlib(5),
order='F',
**kwargs)
z2 = ones_like(z, path=kwargs.get('path', None))
assert z.shape == z2.shape
assert z.chunks == z2.chunks
assert z.dtype == z2.dtype
assert z.compressor.get_config() == z2.compressor.get_config()
assert z.fill_value == z2.fill_value
assert z.order == z2.order
assert (z._store._store_version == z2._store._store_version ==
expected_zarr_version)
# numpy array
a = np.empty(100, dtype='f4')
z3 = ones_like(a, chunks=10, **kwargs)
assert a.shape == z3.shape
assert (10, ) == z3.chunks
assert a.dtype == z3.dtype
assert 1 == z3.fill_value
assert z3._store._store_version == expected_zarr_version
def create_array(read_only=False, **kwargs):
path = mkdtemp()
atexit.register(shutil.rmtree, path)
store = DirectoryStore(path)
kwargs.setdefault('compressor', Zlib(1))
init_array(store, **kwargs)
return Array(store, read_only=read_only)
def test_open_like():
# zarr array
path = tempfile.mktemp()
atexit.register(shutil.rmtree, path)
z = full(100,
chunks=10,
dtype='f4',
compressor=Zlib(5),
fill_value=42,
order='F')
z2 = open_like(z, path)
assert z.shape == z2.shape
assert z.chunks == z2.chunks
assert z.dtype == z2.dtype
assert z.compressor.get_config() == z2.compressor.get_config()
assert z.fill_value == z2.fill_value
assert z.order == z2.order
# numpy array
path = tempfile.mktemp()
atexit.register(shutil.rmtree, path)
a = np.empty(100, dtype='f4')
z3 = open_like(a, path, chunks=10)
assert a.shape == z3.shape
assert (10, ) == z3.chunks
assert a.dtype == z3.dtype
assert 0 == z3.fill_value
def test_compression_args():
z = create(100, compression='zlib', compression_opts=9)
assert_is_instance(z, Array)
eq('zlib', z.compressor.codec_id)
eq(9, z.compressor.level)
# 'compressor' overrides 'compression'
z = create(100, compressor=Zlib(9), compression='bz2', compression_opts=1)
assert_is_instance(z, Array)
eq('zlib', z.compressor.codec_id)
eq(9, z.compressor.level)
# 'compressor' ignores 'compression_opts'
z = create(100, compressor=Zlib(9), compression_opts=1)
assert_is_instance(z, Array)
eq('zlib', z.compressor.codec_id)
eq(9, z.compressor.level)
def create_array(read_only=False, **kwargs):
store = dict()
dtype = kwargs.get('dtype', None)
filters = [
Delta(dtype=dtype),
FixedScaleOffset(dtype=dtype, scale=1, offset=0),
]
kwargs.setdefault('filters', filters)
compressor = Zlib(1)
kwargs.setdefault('compressor', compressor)
init_array(store, **kwargs)
return Array(store, read_only=read_only)
def test_create():
# defaults
z = create(100)
assert_is_instance(z, Array)
eq((100, ), z.shape)
eq((100, ), z.chunks) # auto-chunks
eq(np.dtype(None), z.dtype)
eq('blosc', z.compressor.codec_id)
eq(0, z.fill_value)
# all specified
z = create(100,
chunks=10,
dtype='i4',
compressor=Zlib(1),
fill_value=42,
order='F')
assert_is_instance(z, Array)
eq((100, ), z.shape)
eq((10, ), z.chunks)
eq(np.dtype('i4'), z.dtype)
eq('zlib', z.compressor.codec_id)
eq(1, z.compressor.level)
eq(42, z.fill_value)
eq('F', z.order)
# with synchronizer
synchronizer = ThreadSynchronizer()
z = create(100, chunks=10, synchronizer=synchronizer)
assert_is_instance(z, Array)
eq((100, ), z.shape)
eq((10, ), z.chunks)
assert synchronizer is z.synchronizer
# don't allow string as compressor arg
with assert_raises(ValueError):
create(100, chunks=10, compressor='zlib')
# compatibility
z = create(100, compression='zlib', compression_opts=9)
eq('zlib', z.compressor.codec_id)
eq(9, z.compressor.level)
z = create(100, compression='default')
eq('blosc', z.compressor.codec_id)
# errors
with assert_raises(ValueError):
create(100, compression=1)
def test_empty_like(zarr_version):
kwargs = _init_creation_kwargs(zarr_version)
expected_zarr_version = DEFAULT_ZARR_VERSION if zarr_version is None else zarr_version
# zarr array
z = empty(100,
chunks=10,
dtype='f4',
compressor=Zlib(5),
order='F',
**kwargs)
# zarr_version will be inferred from z, but have to specify a path in v3
z2 = empty_like(z, path=kwargs.get('path', None))
assert z.shape == z2.shape
assert z.chunks == z2.chunks
assert z.dtype == z2.dtype
assert z.compressor.get_config() == z2.compressor.get_config()
assert z.fill_value == z2.fill_value
assert z.order == z2.order
assert (z._store._store_version == z2._store._store_version ==
expected_zarr_version)
# numpy array
a = np.empty(100, dtype='f4')
z3 = empty_like(a, **kwargs)
assert a.shape == z3.shape
assert (100, ) == z3.chunks
assert a.dtype == z3.dtype
assert z3.fill_value is None
assert z3._store._store_version == expected_zarr_version
# something slightly silly
a = [0] * 100
z3 = empty_like(a, shape=200, **kwargs)
assert (200, ) == z3.shape
# other array-likes
b = np.arange(1000).reshape(100, 10)
c = MockBcolzArray(b, 10)
z = empty_like(c, **kwargs)
assert b.shape == z.shape
assert (10, 10) == z.chunks
c = MockH5pyDataset(b, chunks=(10, 2))
z = empty_like(c, **kwargs)
assert b.shape == z.shape
assert (10, 2) == z.chunks
c = MockH5pyDataset(b, chunks=None)
z = empty_like(c, **kwargs)
assert b.shape == z.shape
assert isinstance(z.chunks, tuple)
def test_repr(self):
if not PY2:
z = self.create_array(shape=100,
chunks=10,
dtype='f4',
compressor=Zlib(1))
# flake8: noqa
expect = """Array((100,), float32, chunks=(10,), order=C)
nbytes: 400; nbytes_stored: 245; ratio: 1.6; initialized: 0/10
compressor: Zlib(level=1)
store: DirectoryStore; synchronizer: ProcessSynchronizer
"""
actual = repr(z)
for l1, l2 in zip(expect.split('\n'), actual.split('\n')):
eq(l1, l2)
def test_encode_decode_array_dtype_shape_v3():
meta = dict(shape=(100, ),
chunk_grid=dict(type='regular',
chunk_shape=(10, ),
separator=('/')),
data_type=np.dtype('(10, 10)<f8'),
compressor=Zlib(1),
fill_value=None,
chunk_memory_layout='C')
meta_json = '''{
"attributes": {},
"chunk_grid": {
"chunk_shape": [10],
"separator": "/",
"type": "regular"
},
"chunk_memory_layout": "C",
"compressor": {
"codec": "https://purl.org/zarr/spec/codec/zlib/1.0",
"configuration": {
"level": 1
}
},
"data_type": "<f8",
"extensions": [],
"fill_value": null,
"shape": [100, 10, 10 ]
}'''
# test encoding
meta_enc = Metadata3.encode_array_metadata(meta)
assert_json_equal(meta_json, meta_enc)
# test decoding
meta_dec = Metadata3.decode_array_metadata(meta_enc)
# to maintain consistency with numpy unstructured arrays, unpack dimensions into shape
assert meta['shape'] + meta['data_type'].shape == meta_dec['shape']
assert meta['chunk_grid'] == meta_dec['chunk_grid']
# to maintain consistency with numpy unstructured arrays, unpack dtypes
assert meta['data_type'].base == meta_dec['data_type']
assert meta['compressor'] == meta_dec['compressor']
assert meta['chunk_memory_layout'] == meta_dec['chunk_memory_layout']
assert meta_dec['fill_value'] is None
assert 'filters' not in meta_dec
def test_ones_like():
# zarr array
z = ones(100, chunks=10, dtype='f4', compressor=Zlib(5), order='F')
z2 = ones_like(z)
eq(z.shape, z2.shape)
eq(z.chunks, z2.chunks)
eq(z.dtype, z2.dtype)
eq(z.compressor.get_config(), z2.compressor.get_config())
eq(z.fill_value, z2.fill_value)
eq(z.order, z2.order)
# numpy array
a = np.empty(100, dtype='f4')
z3 = ones_like(a, chunks=10)
eq(a.shape, z3.shape)
eq((10, ), z3.chunks)
eq(a.dtype, z3.dtype)
eq(1, z3.fill_value)
def test_ones_like():
# zarr array
z = ones(100, chunks=10, dtype='f4', compressor=Zlib(5), order='F')
z2 = ones_like(z)
assert z.shape == z2.shape
assert z.chunks == z2.chunks
assert z.dtype == z2.dtype
assert z.compressor.get_config() == z2.compressor.get_config()
assert z.fill_value == z2.fill_value
assert z.order == z2.order
# numpy array
a = np.empty(100, dtype='f4')
z3 = ones_like(a, chunks=10)
assert a.shape == z3.shape
assert (10, ) == z3.chunks
assert a.dtype == z3.dtype
assert 1 == z3.fill_value
def test_encode_decode_fill_values_bytes():
dtype = np.dtype('S10')
fills = b'foo', bytes(10)
for v in fills:
# setup and encode metadata
meta = dict(
shape=(100,),
chunks=(10,),
dtype=dtype,
compressor=Zlib(1).get_config(),
fill_value=v,
filters=None,
order='C'
)
meta_enc = encode_array_metadata(meta)
# define expected metadata encoded as JSON
s = base64.standard_b64encode(v)
if not PY2:
s = s.decode()
meta_json = '''{
"chunks": [10],
"compressor": {"id": "zlib", "level": 1},
"dtype": "|S10",
"fill_value": "%s",
"filters": null,
"order": "C",
"shape": [100],
"zarr_format": %s
}''' % (s, ZARR_FORMAT)
# test encoding
assert_json_equal(meta_json, meta_enc)
# test decoding
meta_dec = decode_array_metadata(meta_enc)
actual = meta_dec['fill_value']
expect = np.array(v, dtype=dtype)[()]
assert expect == actual
def test_full_like():
z = full(100, chunks=10, dtype='f4', compressor=Zlib(5),
fill_value=42, order='F')
z2 = full_like(z)
assert z.shape == z2.shape
assert z.chunks == z2.chunks
assert z.dtype == z2.dtype
assert z.compressor.get_config() == z2.compressor.get_config()
assert z.fill_value == z2.fill_value
assert z.order == z2.order
# numpy array
a = np.empty(100, dtype='f4')
z3 = full_like(a, chunks=10, fill_value=42)
assert a.shape == z3.shape
assert (10,) == z3.chunks
assert a.dtype == z3.dtype
assert 42 == z3.fill_value
with pytest.raises(TypeError):
# fill_value missing
full_like(a, chunks=10)
def test_empty_like():
# zarr array
z = empty(100, chunks=10, dtype='f4', compressor=Zlib(5),
order='F')
z2 = empty_like(z)
assert z.shape == z2.shape
assert z.chunks == z2.chunks
assert z.dtype == z2.dtype
assert z.compressor.get_config() == z2.compressor.get_config()
assert z.fill_value == z2.fill_value
assert z.order == z2.order
# numpy array
a = np.empty(100, dtype='f4')
z3 = empty_like(a)
assert a.shape == z3.shape
assert (100,) == z3.chunks
assert a.dtype == z3.dtype
assert z3.fill_value is None
# something slightly silly
a = [0] * 100
z3 = empty_like(a, shape=200)
assert (200,) == z3.shape
# other array-likes
b = np.arange(1000).reshape(100, 10)
c = MockBcolzArray(b, 10)
z = empty_like(c)
assert b.shape == z.shape
assert (10, 10) == z.chunks
c = MockH5pyDataset(b, chunks=(10, 2))
z = empty_like(c)
assert b.shape == z.shape
assert (10, 2) == z.chunks
c = MockH5pyDataset(b, chunks=None)
z = empty_like(c)
assert b.shape == z.shape
assert isinstance(z.chunks, tuple)
def test_encode_decode_fill_values_nan():
fills = (
(np.nan, "NaN", np.isnan),
(np.NINF, "-Infinity", np.isneginf),
(np.PINF, "Infinity", np.isposinf),
)
for v, s, f in fills:
meta = dict(
shape=(100,),
chunks=(10,),
dtype=np.dtype('f8'),
compressor=Zlib(1).get_config(),
fill_value=v,
filters=None,
order='C'
)
meta_json = '''{
"chunks": [10],
"compressor": {"id": "zlib", "level": 1},
"dtype": "<f8",
"fill_value": "%s",
"filters": null,
"order": "C",
"shape": [100],
"zarr_format": %s
}''' % (s, ZARR_FORMAT)
# test encoding
meta_enc = encode_array_metadata(meta)
assert_json_equal(meta_json, meta_enc)
# test decoding
meta_dec = decode_array_metadata(meta_enc)
actual = meta_dec['fill_value']
assert f(actual)
def test_encode_decode_array_structured():
meta = dict(
shape=(100,),
chunks=(10,),
dtype=np.dtype('i8, (10, 10)f8, (5, 10, 15)u1'),
compressor=Zlib(1).get_config(),
fill_value=None,
filters=None,
order='C'
)
meta_json = '''{
"chunks": [10],
"compressor": {"id": "zlib", "level": 1},
"dtype": [["f0", "<i8"], ["f1", "<f8", [10, 10]], ["f2", "|u1", [5, 10, 15]]],
"fill_value": null,
"filters": null,
"order": "C",
"shape": [100],
"zarr_format": %s
}''' % ZARR_FORMAT
# test encoding
meta_enc = encode_array_metadata(meta)
assert_json_equal(meta_json, meta_enc)
# test decoding
meta_dec = decode_array_metadata(meta_enc)
assert ZARR_FORMAT == meta_dec['zarr_format']
# to maintain consistency with numpy unstructured arrays, unpack dimensions into shape
assert meta['shape'] + meta['dtype'].shape == meta_dec['shape']
assert meta['chunks'] == meta_dec['chunks']
# to maintain consistency with numpy unstructured arrays, unpack dimensions into shape
assert meta['dtype'].base == meta_dec['dtype']
assert meta['compressor'] == meta_dec['compressor']
assert meta['order'] == meta_dec['order']
assert meta_dec['fill_value'] is None
assert meta_dec['filters'] is None
def test_empty_like():
# zarr array
z = empty(100, chunks=10, dtype='f4', compressor=Zlib(5), order='F')
z2 = empty_like(z)
eq(z.shape, z2.shape)
eq(z.chunks, z2.chunks)
eq(z.dtype, z2.dtype)
eq(z.compressor.get_config(), z2.compressor.get_config())
eq(z.fill_value, z2.fill_value)
eq(z.order, z2.order)
# numpy array
a = np.empty(100, dtype='f4')
z3 = empty_like(a)
eq(a.shape, z3.shape)
eq((100, ), z3.chunks)
eq(a.dtype, z3.dtype)
assert_is_none(z3.fill_value)
# something slightly silly
a = [0] * 100
z3 = empty_like(a, shape=200)
eq((200, ), z3.shape)
# other array-likes
b = np.arange(1000).reshape(100, 10)
c = MockBcolzArray(b, 10)
z = empty_like(c)
eq(b.shape, z.shape)
eq((10, 10), z.chunks)
c = MockH5pyDataset(b, chunks=(10, 2))
z = empty_like(c)
eq(b.shape, z.shape)
eq((10, 2), z.chunks)
c = MockH5pyDataset(b, chunks=None)
z = empty_like(c)
eq(b.shape, z.shape)
assert_is_instance(z.chunks, tuple)
def test_init_array_overwrite_path(self):
# setup
path = 'foo/bar'
store = self.create_store()
meta = dict(shape=(2000, ),
chunks=(200, ),
dtype=np.dtype('u1'),
compressor=Zlib(1).get_config(),
fill_value=0,
order='F',
filters=None)
store[array_meta_key] = encode_array_metadata(meta)
store[path + '/' + array_meta_key] = encode_array_metadata(meta)
# don't overwrite
with pytest.raises(ValueError):
init_array(store, shape=1000, chunks=100, path=path)
# do overwrite
try:
init_array(store,
shape=1000,
chunks=100,
dtype='i4',
path=path,
overwrite=True)
except NotImplementedError:
pass
else:
assert group_meta_key in store
assert array_meta_key not in store
assert (path + '/' + array_meta_key) in store
# should have been overwritten
meta = decode_array_metadata(store[path + '/' + array_meta_key])
assert ZARR_FORMAT == meta['zarr_format']
assert (1000, ) == meta['shape']
assert (100, ) == meta['chunks']
assert np.dtype('i4') == meta['dtype']
def test_full_like():
z = full(100,
chunks=10,
dtype='f4',
compressor=Zlib(5),
fill_value=42,
order='F')
z2 = full_like(z)
eq(z.shape, z2.shape)
eq(z.chunks, z2.chunks)
eq(z.dtype, z2.dtype)
eq(z.compressor.get_config(), z2.compressor.get_config())
eq(z.fill_value, z2.fill_value)
eq(z.order, z2.order)
# numpy array
a = np.empty(100, dtype='f4')
z3 = full_like(a, chunks=10, fill_value=42)
eq(a.shape, z3.shape)
eq((10, ), z3.chunks)
eq(a.dtype, z3.dtype)
eq(42, z3.fill_value)
with assert_raises(TypeError):
# fill_value missing
full_like(a, chunks=10)
def test_format_compatibility():
# This test is intended to catch any unintended changes that break the ability to
# read data stored with a previous minor version (which should be format-compatible).
# fixture data
fixture = group(store=DirectoryStore('fixture'))
# set seed to get consistent random data
np.random.seed(42)
arrays_chunks = [
(np.arange(1111, dtype='i1'), 100),
(np.arange(1111, dtype='i2'), 100),
(np.arange(1111, dtype='i4'), 100),
(np.arange(1111, dtype='i8'), 1000),
(np.random.randint(0, 200, size=2222, dtype='u1'), 100),
(np.random.randint(0, 2000, size=2222, dtype='u2'), 100),
(np.random.randint(0, 2000, size=2222, dtype='u4'), 100),
(np.random.randint(0, 2000, size=2222, dtype='u8'), 100),
(np.linspace(0, 1, 3333, dtype='f2'), 100),
(np.linspace(0, 1, 3333, dtype='f4'), 100),
(np.linspace(0, 1, 3333, dtype='f8'), 100),
(np.random.normal(loc=0, scale=1, size=4444).astype('f2'), 100),
(np.random.normal(loc=0, scale=1, size=4444).astype('f4'), 100),
(np.random.normal(loc=0, scale=1, size=4444).astype('f8'), 100),
(np.random.choice([b'A', b'C', b'G', b'T'], size=5555,
replace=True).astype('S'), 100),
(np.random.choice(['foo', 'bar', 'baz', 'quux'],
size=5555,
replace=True).astype('U'), 100),
(np.random.choice([0, 1 / 3, 1 / 7, 1 / 9, np.nan],
size=5555,
replace=True).astype('f8'), 100),
(np.random.randint(0, 2, size=5555, dtype=bool), 100),
(np.arange(20000, dtype='i4').reshape(2000, 10, order='C'), (100, 3)),
(np.arange(20000, dtype='i4').reshape(200, 100, order='F'), (100, 30)),
(np.arange(20000, dtype='i4').reshape(200, 10, 10,
order='C'), (100, 3, 3)),
(np.arange(20000, dtype='i4').reshape(20, 100, 10,
order='F'), (10, 30, 3)),
(np.arange(20000, dtype='i4').reshape(20, 10, 10, 10,
order='C'), (10, 3, 3, 3)),
(np.arange(20000, dtype='i4').reshape(20, 10, 10, 10,
order='F'), (10, 3, 3, 3)),
]
compressors = [
None,
Zlib(level=1),
BZ2(level=1),
Blosc(cname='zstd', clevel=1, shuffle=0),
Blosc(cname='zstd', clevel=1, shuffle=1),
Blosc(cname='zstd', clevel=1, shuffle=2),
Blosc(cname='lz4', clevel=1, shuffle=0),
]
for i, (arr, chunks) in enumerate(arrays_chunks):
if arr.flags.f_contiguous:
order = 'F'
else:
order = 'C'
for j, compressor in enumerate(compressors):
path = '{}/{}'.format(i, j)
if path not in fixture: # pragma: no cover
# store the data - should be one-time operation
fixture.array(path,
data=arr,
chunks=chunks,
order=order,
compressor=compressor)
# setup array
z = fixture[path]
# check contents
if arr.dtype.kind == 'f':
assert_array_almost_equal(arr, z[:])
else:
assert_array_equal(arr, z[:])
# check dtype
assert arr.dtype == z.dtype
# check compressor
if compressor is None:
assert z.compressor is None
else:
assert compressor.codec_id == z.compressor.codec_id
assert compressor.get_config() == z.compressor.get_config()
