def test_decode_array_missing_fields():
# missing fields
meta_json = '''{
"zarr_format": %s
}''' % ZARR_FORMAT
with assert_raises(MetadataError):
decode_array_metadata(meta_json)
def test_decode_array_missing_fields():
# missing fields
meta_json = '''{
"zarr_format": %s
}''' % ZARR_FORMAT
with pytest.raises(MetadataError):
decode_array_metadata(meta_json)
def test_decode_array_unsupported_format():
# unsupported format
meta_json = '''{
"zarr_format": %s,
"shape": [100],
"chunks": [10],
"dtype": "<f8",
"compressor": {"id": "zlib", "level": 1},
"fill_value": null,
"order": "C"
}''' % (ZARR_FORMAT - 1)
with pytest.raises(MetadataError):
decode_array_metadata(meta_json)
def test_decode_array_unsupported_format():
# unsupported format
meta_json = '''{
"zarr_format": %s,
"shape": [100],
"chunks": [10],
"dtype": "<f8",
"compressor": {"id": "zlib", "level": 1},
"fill_value": null,
"order": "C"
}''' % (ZARR_FORMAT - 1)
with assert_raises(MetadataError):
decode_array_metadata(meta_json)
def test_init_array_overwrite_chunk_store(self):
# setup
store = self.create_store()
chunk_store = self.create_store()
store[array_meta_key] = encode_array_metadata(
dict(shape=(2000,),
chunks=(200,),
dtype=np.dtype('u1'),
compressor=None,
fill_value=0,
filters=None,
order='F')
)
chunk_store['0'] = b'aaa'
chunk_store['1'] = b'bbb'
# don't overwrite (default)
with assert_raises(ValueError):
init_array(store, shape=1000, chunks=100, chunk_store=chunk_store)
# do overwrite
try:
init_array(store, shape=1000, chunks=100, dtype='i4',
overwrite=True, chunk_store=chunk_store)
except NotImplementedError:
pass
else:
assert array_meta_key in store
meta = decode_array_metadata(store[array_meta_key])
eq(ZARR_FORMAT, meta['zarr_format'])
eq((1000,), meta['shape'])
eq((100,), meta['chunks'])
eq(np.dtype('i4'), meta['dtype'])
assert '0' not in chunk_store
assert '1' not in chunk_store
def _load_metadata_nosync(self):
try:
mkey = self._key_prefix + array_meta_key
meta_bytes = self._store[mkey]
except KeyError:
err_array_not_found(self._path)
else:
# decode and store metadata
meta = decode_array_metadata(meta_bytes)
self._meta = meta
self._shape = meta['shape']
self._chunks = meta['chunks']
self._dtype = meta['dtype']
self._fill_value = meta['fill_value']
self._order = meta['order']
# setup compressor
config = meta['compressor']
if config is None:
self._compressor = None
else:
self._compressor = get_codec(config)
# setup filters
filters = meta['filters']
if filters:
filters = [get_codec(config) for config in filters]
self._filters = filters
def test_init_array_overwrite_path(self):
# setup
path = 'foo/bar'
store = self.create_store()
meta = dict(shape=(2000,),
chunks=(200,),
dtype=np.dtype('u1'),
compressor=Zlib(1).get_config(),
fill_value=0,
order='F',
filters=None)
store[array_meta_key] = encode_array_metadata(meta)
store[path + '/' + array_meta_key] = encode_array_metadata(meta)
# don't overwrite
with assert_raises(ValueError):
init_array(store, shape=1000, chunks=100, path=path)
# do overwrite
try:
init_array(store, shape=1000, chunks=100, dtype='i4', path=path,
overwrite=True)
except NotImplementedError:
pass
else:
assert group_meta_key in store
assert array_meta_key not in store
assert (path + '/' + array_meta_key) in store
# should have been overwritten
meta = decode_array_metadata(store[path + '/' + array_meta_key])
eq(ZARR_FORMAT, meta['zarr_format'])
eq((1000,), meta['shape'])
eq((100,), meta['chunks'])
eq(np.dtype('i4'), meta['dtype'])
def test_init_array_overwrite(self):
# setup
store = self.create_store()
store[array_meta_key] = encode_array_metadata(
dict(shape=(2000, ),
chunks=(200, ),
dtype=np.dtype('u1'),
compressor=Zlib(1).get_config(),
fill_value=0,
order='F',
filters=None))
# don't overwrite (default)
with pytest.raises(ValueError):
init_array(store, shape=1000, chunks=100)
# do overwrite
try:
init_array(store,
shape=1000,
chunks=100,
dtype='i4',
overwrite=True)
except NotImplementedError:
pass
else:
assert array_meta_key in store
meta = decode_array_metadata(store[array_meta_key])
assert ZARR_FORMAT == meta['zarr_format']
assert (1000, ) == meta['shape']
assert (100, ) == meta['chunks']
assert np.dtype('i4') == meta['dtype']
def test_init_array_overwrite(self):
# setup
store = self.create_store()
store[array_meta_key] = encode_array_metadata(
dict(shape=(2000,),
chunks=(200,),
dtype=np.dtype('u1'),
compressor=Zlib(1).get_config(),
fill_value=0,
order='F',
filters=None)
)
# don't overwrite (default)
with assert_raises(ValueError):
init_array(store, shape=1000, chunks=100)
# do overwrite
try:
init_array(store, shape=1000, chunks=100, dtype='i4',
overwrite=True)
except NotImplementedError:
pass
else:
assert array_meta_key in store
meta = decode_array_metadata(store[array_meta_key])
eq(ZARR_FORMAT, meta['zarr_format'])
eq((1000,), meta['shape'])
eq((100,), meta['chunks'])
eq(np.dtype('i4'), meta['dtype'])
def _load_metadata_nosync(self):
try:
mkey = self._key_prefix + array_meta_key
meta_bytes = self._store[mkey]
except KeyError:
err_array_not_found(self._path)
else:
# decode and store metadata
meta = decode_array_metadata(meta_bytes)
self._meta = meta
self._shape = meta['shape']
self._chunks = meta['chunks']
self._dtype = meta['dtype']
self._fill_value = meta['fill_value']
self._order = meta['order']
# setup compressor
config = meta['compressor']
if config is None:
self._compressor = None
else:
self._compressor = get_codec(config)
# setup filters
filters = meta['filters']
if filters:
filters = [get_codec(config) for config in filters]
self._filters = filters
def test_init_array_overwrite_group(self):
# setup
path = 'foo/bar'
store = self.create_store()
store[path + '/' + group_meta_key] = encode_group_metadata()
# don't overwrite
with pytest.raises(ValueError):
init_array(store, shape=1000, chunks=100, path=path)
# do overwrite
try:
init_array(store,
shape=1000,
chunks=100,
dtype='i4',
path=path,
overwrite=True)
except NotImplementedError:
pass
else:
assert (path + '/' + group_meta_key) not in store
assert (path + '/' + array_meta_key) in store
meta = decode_array_metadata(store[path + '/' + array_meta_key])
assert ZARR_FORMAT == meta['zarr_format']
assert (1000, ) == meta['shape']
assert (100, ) == meta['chunks']
assert np.dtype('i4') == meta['dtype']
def test_encode_decode_array_1():
meta = dict(shape=(100, ),
chunks=(10, ),
dtype=np.dtype('f8'),
compressor=Zlib(1).get_config(),
fill_value=None,
filters=None,
order='C')
meta_json = '''{
"chunks": [10],
"compressor": {"id": "zlib", "level": 1},
"dtype": "<f8",
"fill_value": null,
"filters": null,
"order": "C",
"shape": [100],
"zarr_format": %s
}''' % ZARR_FORMAT
# test encoding
meta_enc = encode_array_metadata(meta)
assert_json_equal(meta_json, meta_enc)
# test decoding
meta_dec = decode_array_metadata(meta_enc)
assert ZARR_FORMAT == meta_dec['zarr_format']
assert meta['shape'] == meta_dec['shape']
assert meta['chunks'] == meta_dec['chunks']
assert meta['dtype'] == meta_dec['dtype']
assert meta['compressor'] == meta_dec['compressor']
assert meta['order'] == meta_dec['order']
assert meta_dec['fill_value'] is None
assert meta_dec['filters'] is None
def test_encode_decode_array_datetime_timedelta():
# some variations
for k in ['m8[s]', 'M8[s]']:
compressor = Blosc(cname='lz4', clevel=3, shuffle=2)
dtype = np.dtype(k)
fill_value = dtype.type("NaT")
meta = dict(
shape=(100, 100),
chunks=(10, 10),
dtype=dtype,
compressor=compressor.get_config(),
fill_value=fill_value,
order=dtype.char,
filters=[]
)
meta_json = '''{
"chunks": [10, 10],
"compressor": {
"id": "blosc",
"clevel": 3,
"cname": "lz4",
"shuffle": 2,
"blocksize": 0
},
"dtype": "%s",
"fill_value": -9223372036854775808,
"filters": [],
"order": "%s",
"shape": [100, 100],
"zarr_format": %s
}''' % (dtype.str, dtype.char, ZARR_FORMAT)
# test encoding
meta_enc = encode_array_metadata(meta)
assert_json_equal(meta_json, meta_enc)
# test decoding
meta_dec = decode_array_metadata(meta_enc)
assert ZARR_FORMAT == meta_dec['zarr_format']
assert meta['shape'] == meta_dec['shape']
assert meta['chunks'] == meta_dec['chunks']
assert meta['dtype'] == meta_dec['dtype']
assert meta['compressor'] == meta_dec['compressor']
assert meta['order'] == meta_dec['order']
# Based off of this SO answer: https://stackoverflow.com/a/49972198
assert np.all(
fill_value.view((np.uint8, fill_value.itemsize)) ==
meta_dec['fill_value'].view((np.uint8, meta_dec['fill_value'].itemsize))
)
assert [] == meta_dec['filters']
def test_init_array(self):
store = self.create_store()
init_array(store, shape=1000, chunks=100)
# check metadata
assert array_meta_key in store
meta = decode_array_metadata(store[array_meta_key])
eq(ZARR_FORMAT, meta['zarr_format'])
eq((1000,), meta['shape'])
eq((100,), meta['chunks'])
eq(np.dtype(None), meta['dtype'])
eq(default_compressor.get_config(), meta['compressor'])
assert_is_none(meta['fill_value'])
def test_init_array(self):
store = self.create_store()
init_array(store, shape=1000, chunks=100)
# check metadata
assert array_meta_key in store
meta = decode_array_metadata(store[array_meta_key])
assert ZARR_FORMAT == meta['zarr_format']
assert (1000, ) == meta['shape']
assert (100, ) == meta['chunks']
assert np.dtype(None) == meta['dtype']
assert default_compressor.get_config() == meta['compressor']
assert meta['fill_value'] is None
def test_encode_decode_array_2():
# some variations
df = Delta(astype='u2', dtype='V14')
compressor = Blosc(cname='lz4', clevel=3, shuffle=2)
dtype = np.dtype([('a', 'i4'), ('b', 'S10')])
fill_value = np.zeros((), dtype=dtype)[()]
meta = dict(
shape=(100, 100),
chunks=(10, 10),
dtype=dtype,
compressor=compressor.get_config(),
fill_value=fill_value,
order='F',
filters=[df.get_config()]
)
meta_json = '''{
"chunks": [10, 10],
"compressor": {
"id": "blosc",
"clevel": 3,
"cname": "lz4",
"shuffle": 2,
"blocksize": 0
},
"dtype": [["a", "<i4"], ["b", "|S10"]],
"fill_value": "AAAAAAAAAAAAAAAAAAA=",
"filters": [
{"id": "delta", "astype": "<u2", "dtype": "|V14"}
],
"order": "F",
"shape": [100, 100],
"zarr_format": %s
}''' % ZARR_FORMAT
# test encoding
meta_enc = encode_array_metadata(meta)
assert_json_equal(meta_json, meta_enc)
# test decoding
meta_dec = decode_array_metadata(meta_enc)
assert ZARR_FORMAT == meta_dec['zarr_format']
assert meta['shape'] == meta_dec['shape']
assert meta['chunks'] == meta_dec['chunks']
assert meta['dtype'] == meta_dec['dtype']
assert meta['compressor'] == meta_dec['compressor']
assert meta['order'] == meta_dec['order']
assert fill_value == meta_dec['fill_value']
assert [df.get_config()] == meta_dec['filters']
def test_encode_decode_array_2():
# some variations
df = Delta(astype='u2', dtype='V14')
compressor = Blosc(cname='lz4', clevel=3, shuffle=2)
dtype = np.dtype([('a', 'i4'), ('b', 'S10')])
fill_value = np.zeros((), dtype=dtype)[()]
meta = dict(
shape=(100, 100),
chunks=(10, 10),
dtype=dtype,
compressor=compressor.get_config(),
fill_value=fill_value,
order='F',
filters=[df.get_config()]
)
meta_json = '''{
"chunks": [10, 10],
"compressor": {
"id": "blosc",
"clevel": 3,
"cname": "lz4",
"shuffle": 2,
"blocksize": 0
},
"dtype": [["a", "<i4"], ["b", "|S10"]],
"fill_value": "AAAAAAAAAAAAAAAAAAA=",
"filters": [
{"id": "delta", "astype": "<u2", "dtype": "|V14"}
],
"order": "F",
"shape": [100, 100],
"zarr_format": %s
}''' % ZARR_FORMAT
# test encoding
meta_enc = encode_array_metadata(meta)
assert_json_equal(meta_json, meta_enc)
# test decoding
meta_dec = decode_array_metadata(meta_enc)
assert ZARR_FORMAT == meta_dec['zarr_format']
assert meta['shape'] == meta_dec['shape']
assert meta['chunks'] == meta_dec['chunks']
assert meta['dtype'] == meta_dec['dtype']
assert meta['compressor'] == meta_dec['compressor']
assert meta['order'] == meta_dec['order']
assert fill_value == meta_dec['fill_value']
assert [df.get_config()] == meta_dec['filters']
def test_init_array(self):
store = self.create_store()
init_array(store, shape=1000, chunks=100)
# check metadata
assert array_meta_key in store
meta = decode_array_metadata(store[array_meta_key])
eq(ZARR_FORMAT, meta['zarr_format'])
eq((1000, ), meta['shape'])
eq((100, ), meta['chunks'])
eq(np.dtype(None), meta['dtype'])
eq(default_compressor.get_config(), meta['compressor'])
assert_is_none(meta['fill_value'])
# check attributes
assert attrs_key in store
eq(dict(), json.loads(text_type(store[attrs_key], 'ascii')))
def test_encode_decode_fill_values_bytes():
dtype = np.dtype('S10')
fills = b'foo', bytes(10)
for v in fills:
# setup and encode metadata
meta = dict(
shape=(100,),
chunks=(10,),
dtype=dtype,
compressor=Zlib(1).get_config(),
fill_value=v,
filters=None,
order='C'
)
meta_enc = encode_array_metadata(meta)
# define expected metadata encoded as JSON
s = base64.standard_b64encode(v)
if not PY2:
s = s.decode()
meta_json = '''{
"chunks": [10],
"compressor": {"id": "zlib", "level": 1},
"dtype": "|S10",
"fill_value": "%s",
"filters": null,
"order": "C",
"shape": [100],
"zarr_format": %s
}''' % (s, ZARR_FORMAT)
# test encoding
assert_json_equal(meta_json, meta_enc)
# test decoding
meta_dec = decode_array_metadata(meta_enc)
actual = meta_dec['fill_value']
expect = np.array(v, dtype=dtype)[()]
assert expect == actual
def test_encode_decode_fill_values_bytes():
dtype = np.dtype('S10')
fills = b'foo', bytes(10)
for v in fills:
# setup and encode metadata
meta = dict(
shape=(100,),
chunks=(10,),
dtype=dtype,
compressor=Zlib(1).get_config(),
fill_value=v,
filters=None,
order='C'
)
meta_enc = encode_array_metadata(meta)
# define expected metadata encoded as JSON
s = base64.standard_b64encode(v)
if not PY2:
s = s.decode()
meta_json = '''{
"chunks": [10],
"compressor": {"id": "zlib", "level": 1},
"dtype": "|S10",
"fill_value": "%s",
"filters": null,
"order": "C",
"shape": [100],
"zarr_format": %s
}''' % (s, ZARR_FORMAT)
# test encoding
assert_json_equal(meta_json, meta_enc)
# test decoding
meta_dec = decode_array_metadata(meta_enc)
actual = meta_dec['fill_value']
expect = np.array(v, dtype=dtype)[()]
assert expect == actual
def test_init_array_path(self):
path = 'foo/bar'
store = self.create_store()
init_array(store, shape=1000, chunks=100, path=path)
# check metadata
key = path + '/' + array_meta_key
assert key in store
meta = decode_array_metadata(store[key])
eq(ZARR_FORMAT, meta['zarr_format'])
eq((1000,), meta['shape'])
eq((100,), meta['chunks'])
eq(np.dtype(None), meta['dtype'])
eq(default_compressor.get_config(), meta['compressor'])
assert_is_none(meta['fill_value'])
# check attributes
key = path + '/' + attrs_key
assert key in store
eq(dict(), json.loads(text_type(store[key], 'ascii')))
def test_encode_decode_fill_values_nan():
fills = (
(np.nan, "NaN", np.isnan),
(np.NINF, "-Infinity", np.isneginf),
(np.PINF, "Infinity", np.isposinf),
)
for v, s, f in fills:
meta = dict(
shape=(100,),
chunks=(10,),
dtype=np.dtype('f8'),
compressor=Zlib(1).get_config(),
fill_value=v,
filters=None,
order='C'
)
meta_json = '''{
"chunks": [10],
"compressor": {"id": "zlib", "level": 1},
"dtype": "<f8",
"fill_value": "%s",
"filters": null,
"order": "C",
"shape": [100],
"zarr_format": %s
}''' % (s, ZARR_FORMAT)
# test encoding
meta_enc = encode_array_metadata(meta)
assert_json_equal(meta_json, meta_enc)
# test decoding
meta_dec = decode_array_metadata(meta_enc)
actual = meta_dec['fill_value']
assert f(actual)
def test_encode_decode_array_structured():
meta = dict(
shape=(100,),
chunks=(10,),
dtype=np.dtype('i8, (10, 10)f8, (5, 10, 15)u1'),
compressor=Zlib(1).get_config(),
fill_value=None,
filters=None,
order='C'
)
meta_json = '''{
"chunks": [10],
"compressor": {"id": "zlib", "level": 1},
"dtype": [["f0", "<i8"], ["f1", "<f8", [10, 10]], ["f2", "|u1", [5, 10, 15]]],
"fill_value": null,
"filters": null,
"order": "C",
"shape": [100],
"zarr_format": %s
}''' % ZARR_FORMAT
# test encoding
meta_enc = encode_array_metadata(meta)
assert_json_equal(meta_json, meta_enc)
# test decoding
meta_dec = decode_array_metadata(meta_enc)
assert ZARR_FORMAT == meta_dec['zarr_format']
# to maintain consistency with numpy unstructured arrays, unpack dimensions into shape
assert meta['shape'] + meta['dtype'].shape == meta_dec['shape']
assert meta['chunks'] == meta_dec['chunks']
# to maintain consistency with numpy unstructured arrays, unpack dimensions into shape
assert meta['dtype'].base == meta_dec['dtype']
assert meta['compressor'] == meta_dec['compressor']
assert meta['order'] == meta_dec['order']
assert meta_dec['fill_value'] is None
assert meta_dec['filters'] is None
def test_encode_decode_array_fill_values():
fills = (
(np.nan, "NaN", np.isnan),
(np.NINF, "-Infinity", np.isneginf),
(np.PINF, "Infinity", np.isposinf),
)
for v, s, f in fills:
meta = dict(
shape=(100,),
chunks=(10,),
dtype=np.dtype('f8'),
compressor=Zlib(1).get_config(),
fill_value=v,
filters=None,
order='C'
)
meta_json = '''{
"chunks": [10],
"compressor": {"id": "zlib", "level": 1},
"dtype": "<f8",
"fill_value": "%s",
"filters": null,
"order": "C",
"shape": [100],
"zarr_format": %s
}''' % (s, ZARR_FORMAT)
# test encoding
meta_enc = encode_array_metadata(meta)
assert_json_eq(meta_json, meta_enc)
# test decoding
meta_dec = decode_array_metadata(meta_enc)
actual = meta_dec['fill_value']
assert f(actual)
def test_init_array_overwrite_path(self):
# setup
path = 'foo/bar'
store = self.create_store()
meta = dict(shape=(2000, ),
chunks=(200, ),
dtype=np.dtype('u1'),
compressor=Zlib(1).get_config(),
fill_value=0,
order='F',
filters=None)
store[array_meta_key] = encode_array_metadata(meta)
store[path + '/' + array_meta_key] = encode_array_metadata(meta)
# don't overwrite
with pytest.raises(ValueError):
init_array(store, shape=1000, chunks=100, path=path)
# do overwrite
try:
init_array(store,
shape=1000,
chunks=100,
dtype='i4',
path=path,
overwrite=True)
except NotImplementedError:
pass
else:
assert group_meta_key in store
assert array_meta_key not in store
assert (path + '/' + array_meta_key) in store
# should have been overwritten
meta = decode_array_metadata(store[path + '/' + array_meta_key])
assert ZARR_FORMAT == meta['zarr_format']
assert (1000, ) == meta['shape']
assert (100, ) == meta['chunks']
assert np.dtype('i4') == meta['dtype']
def test_init_array_overwrite_chunk_store(self):
# setup
store = self.create_store()
chunk_store = self.create_store()
store[array_meta_key] = encode_array_metadata(
dict(shape=(2000, ),
chunks=(200, ),
dtype=np.dtype('u1'),
compressor=None,
fill_value=0,
filters=None,
order='F'))
chunk_store['0'] = b'aaa'
chunk_store['1'] = b'bbb'
# don't overwrite (default)
with assert_raises(ValueError):
init_array(store, shape=1000, chunks=100, chunk_store=chunk_store)
# do overwrite
try:
init_array(store,
shape=1000,
chunks=100,
dtype='i4',
overwrite=True,
chunk_store=chunk_store)
except NotImplementedError:
pass
else:
assert array_meta_key in store
meta = decode_array_metadata(store[array_meta_key])
eq(ZARR_FORMAT, meta['zarr_format'])
eq((1000, ), meta['shape'])
eq((100, ), meta['chunks'])
eq(np.dtype('i4'), meta['dtype'])
assert '0' not in chunk_store
assert '1' not in chunk_store
def test_encode_decode_array_1():
meta = dict(
shape=(100,),
chunks=(10,),
dtype=np.dtype('f8'),
compressor=Zlib(1).get_config(),
fill_value=None,
filters=None,
order='C'
)
meta_json = '''{
"chunks": [10],
"compressor": {"id": "zlib", "level": 1},
"dtype": "<f8",
"fill_value": null,
"filters": null,
"order": "C",
"shape": [100],
"zarr_format": %s
}''' % ZARR_FORMAT
# test encoding
meta_enc = encode_array_metadata(meta)
assert_json_eq(meta_json, meta_enc)
# test decoding
meta_dec = decode_array_metadata(meta_enc)
eq(ZARR_FORMAT, meta_dec['zarr_format'])
eq(meta['shape'], meta_dec['shape'])
eq(meta['chunks'], meta_dec['chunks'])
eq(meta['dtype'], meta_dec['dtype'])
eq(meta['compressor'], meta_dec['compressor'])
eq(meta['order'], meta_dec['order'])
assert_is_none(meta_dec['fill_value'])
assert_is_none(meta_dec['filters'])
def test_init_array_overwrite_group(self):
# setup
path = 'foo/bar'
store = self.create_store()
store[path + '/' + group_meta_key] = encode_group_metadata()
# don't overwrite
with assert_raises(ValueError):
init_array(store, shape=1000, chunks=100, path=path)
# do overwrite
try:
init_array(store, shape=1000, chunks=100, dtype='i4', path=path,
overwrite=True)
except NotImplementedError:
pass
else:
assert (path + '/' + group_meta_key) not in store
assert (path + '/' + array_meta_key) in store
meta = decode_array_metadata(store[path + '/' + array_meta_key])
eq(ZARR_FORMAT, meta['zarr_format'])
eq((1000,), meta['shape'])
eq((100,), meta['chunks'])
eq(np.dtype('i4'), meta['dtype'])
def test_migrate_1to2():
from zarr import meta_v1
# N.B., version 1 did not support hierarchies, so we only have to be
# concerned about migrating a single array at the root of the store
# setup
store = dict()
meta = dict(shape=(100, ),
chunks=(10, ),
dtype=np.dtype('f4'),
compression='zlib',
compression_opts=1,
fill_value=None,
order='C')
meta_json = meta_v1.encode_metadata(meta)
store['meta'] = meta_json
store['attrs'] = json.dumps(dict()).encode('ascii')
# run migration
migrate_1to2(store)
# check results
assert 'meta' not in store
assert array_meta_key in store
assert 'attrs' not in store
assert attrs_key in store
meta_migrated = decode_array_metadata(store[array_meta_key])
assert 2 == meta_migrated['zarr_format']
# preserved fields
for f in 'shape', 'chunks', 'dtype', 'fill_value', 'order':
assert meta[f] == meta_migrated[f]
# migrate should have added empty filters field
assert meta_migrated['filters'] is None
# check compression and compression_opts migrated to compressor
assert 'compression' not in meta_migrated
assert 'compression_opts' not in meta_migrated
assert meta_migrated['compressor'] == Zlib(1).get_config()
# check dict compression_opts
store = dict()
meta['compression'] = 'blosc'
meta['compression_opts'] = dict(cname='lz4', clevel=5, shuffle=1)
meta_json = meta_v1.encode_metadata(meta)
store['meta'] = meta_json
store['attrs'] = json.dumps(dict()).encode('ascii')
migrate_1to2(store)
meta_migrated = decode_array_metadata(store[array_meta_key])
assert 'compression' not in meta_migrated
assert 'compression_opts' not in meta_migrated
assert (meta_migrated['compressor'] == Blosc(cname='lz4',
clevel=5,
shuffle=1).get_config())
# check 'none' compression is migrated to None (null in JSON)
store = dict()
meta['compression'] = 'none'
meta_json = meta_v1.encode_metadata(meta)
store['meta'] = meta_json
store['attrs'] = json.dumps(dict()).encode('ascii')
migrate_1to2(store)
meta_migrated = decode_array_metadata(store[array_meta_key])
assert 'compression' not in meta_migrated
assert 'compression_opts' not in meta_migrated
assert meta_migrated['compressor'] is None
def test_migrate_1to2():
from zarr import meta_v1
# N.B., version 1 did not support hierarchies, so we only have to be
# concerned about migrating a single array at the root of the store
# setup
store = dict()
meta = dict(
shape=(100,),
chunks=(10,),
dtype=np.dtype('f4'),
compression='zlib',
compression_opts=1,
fill_value=None,
order='C'
)
meta_json = meta_v1.encode_metadata(meta)
store['meta'] = meta_json
store['attrs'] = json.dumps(dict()).encode('ascii')
# run migration
migrate_1to2(store)
# check results
assert 'meta' not in store
assert array_meta_key in store
assert 'attrs' not in store
assert attrs_key in store
meta_migrated = decode_array_metadata(store[array_meta_key])
eq(2, meta_migrated['zarr_format'])
# preserved fields
for f in 'shape', 'chunks', 'dtype', 'fill_value', 'order':
eq(meta[f], meta_migrated[f])
# migrate should have added empty filters field
assert_is_none(meta_migrated['filters'])
# check compression and compression_opts migrated to compressor
assert 'compression' not in meta_migrated
assert 'compression_opts' not in meta_migrated
eq(meta_migrated['compressor'], Zlib(1).get_config())
# check dict compression_opts
store = dict()
meta['compression'] = 'blosc'
meta['compression_opts'] = dict(cname='lz4', clevel=5, shuffle=1)
meta_json = meta_v1.encode_metadata(meta)
store['meta'] = meta_json
store['attrs'] = json.dumps(dict()).encode('ascii')
migrate_1to2(store)
meta_migrated = decode_array_metadata(store[array_meta_key])
assert 'compression' not in meta_migrated
assert 'compression_opts' not in meta_migrated
eq(meta_migrated['compressor'],
Blosc(cname='lz4', clevel=5, shuffle=1).get_config())
# check 'none' compression is migrated to None (null in JSON)
store = dict()
meta['compression'] = 'none'
meta_json = meta_v1.encode_metadata(meta)
store['meta'] = meta_json
store['attrs'] = json.dumps(dict()).encode('ascii')
migrate_1to2(store)
meta_migrated = decode_array_metadata(store[array_meta_key])
assert 'compression' not in meta_migrated
assert 'compression_opts' not in meta_migrated
assert_is_none(meta_migrated['compressor'])
def test_init_array_compat(self):
store = self.create_store()
init_array(store, shape=1000, chunks=100, compressor='none')
meta = decode_array_metadata(store[array_meta_key])
assert meta['compressor'] is None
def test_init_array_compat(self):
store = self.create_store()
init_array(store, shape=1000, chunks=100, compressor='none')
meta = decode_array_metadata(store[array_meta_key])
assert_is_none(meta['compressor'])
