def test_decode_array_missing_fields(): # missing fields meta_json = '''{ "zarr_format": %s }''' % ZARR_FORMAT with assert_raises(MetadataError): decode_array_metadata(meta_json)
def test_decode_array_missing_fields(): # missing fields meta_json = '''{ "zarr_format": %s }''' % ZARR_FORMAT with pytest.raises(MetadataError): decode_array_metadata(meta_json)
def test_decode_array_unsupported_format(): # unsupported format meta_json = '''{ "zarr_format": %s, "shape": [100], "chunks": [10], "dtype": "<f8", "compressor": {"id": "zlib", "level": 1}, "fill_value": null, "order": "C" }''' % (ZARR_FORMAT - 1) with pytest.raises(MetadataError): decode_array_metadata(meta_json)
def test_decode_array_unsupported_format(): # unsupported format meta_json = '''{ "zarr_format": %s, "shape": [100], "chunks": [10], "dtype": "<f8", "compressor": {"id": "zlib", "level": 1}, "fill_value": null, "order": "C" }''' % (ZARR_FORMAT - 1) with assert_raises(MetadataError): decode_array_metadata(meta_json)
def test_init_array_overwrite_chunk_store(self): # setup store = self.create_store() chunk_store = self.create_store() store[array_meta_key] = encode_array_metadata( dict(shape=(2000,), chunks=(200,), dtype=np.dtype('u1'), compressor=None, fill_value=0, filters=None, order='F') ) chunk_store['0'] = b'aaa' chunk_store['1'] = b'bbb' # don't overwrite (default) with assert_raises(ValueError): init_array(store, shape=1000, chunks=100, chunk_store=chunk_store) # do overwrite try: init_array(store, shape=1000, chunks=100, dtype='i4', overwrite=True, chunk_store=chunk_store) except NotImplementedError: pass else: assert array_meta_key in store meta = decode_array_metadata(store[array_meta_key]) eq(ZARR_FORMAT, meta['zarr_format']) eq((1000,), meta['shape']) eq((100,), meta['chunks']) eq(np.dtype('i4'), meta['dtype']) assert '0' not in chunk_store assert '1' not in chunk_store
def _load_metadata_nosync(self): try: mkey = self._key_prefix + array_meta_key meta_bytes = self._store[mkey] except KeyError: err_array_not_found(self._path) else: # decode and store metadata meta = decode_array_metadata(meta_bytes) self._meta = meta self._shape = meta['shape'] self._chunks = meta['chunks'] self._dtype = meta['dtype'] self._fill_value = meta['fill_value'] self._order = meta['order'] # setup compressor config = meta['compressor'] if config is None: self._compressor = None else: self._compressor = get_codec(config) # setup filters filters = meta['filters'] if filters: filters = [get_codec(config) for config in filters] self._filters = filters
def test_init_array_overwrite_path(self): # setup path = 'foo/bar' store = self.create_store() meta = dict(shape=(2000,), chunks=(200,), dtype=np.dtype('u1'), compressor=Zlib(1).get_config(), fill_value=0, order='F', filters=None) store[array_meta_key] = encode_array_metadata(meta) store[path + '/' + array_meta_key] = encode_array_metadata(meta) # don't overwrite with assert_raises(ValueError): init_array(store, shape=1000, chunks=100, path=path) # do overwrite try: init_array(store, shape=1000, chunks=100, dtype='i4', path=path, overwrite=True) except NotImplementedError: pass else: assert group_meta_key in store assert array_meta_key not in store assert (path + '/' + array_meta_key) in store # should have been overwritten meta = decode_array_metadata(store[path + '/' + array_meta_key]) eq(ZARR_FORMAT, meta['zarr_format']) eq((1000,), meta['shape']) eq((100,), meta['chunks']) eq(np.dtype('i4'), meta['dtype'])
def test_init_array_overwrite(self): # setup store = self.create_store() store[array_meta_key] = encode_array_metadata( dict(shape=(2000, ), chunks=(200, ), dtype=np.dtype('u1'), compressor=Zlib(1).get_config(), fill_value=0, order='F', filters=None)) # don't overwrite (default) with pytest.raises(ValueError): init_array(store, shape=1000, chunks=100) # do overwrite try: init_array(store, shape=1000, chunks=100, dtype='i4', overwrite=True) except NotImplementedError: pass else: assert array_meta_key in store meta = decode_array_metadata(store[array_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000, ) == meta['shape'] assert (100, ) == meta['chunks'] assert np.dtype('i4') == meta['dtype']
def test_init_array_overwrite(self): # setup store = self.create_store() store[array_meta_key] = encode_array_metadata( dict(shape=(2000,), chunks=(200,), dtype=np.dtype('u1'), compressor=Zlib(1).get_config(), fill_value=0, order='F', filters=None) ) # don't overwrite (default) with assert_raises(ValueError): init_array(store, shape=1000, chunks=100) # do overwrite try: init_array(store, shape=1000, chunks=100, dtype='i4', overwrite=True) except NotImplementedError: pass else: assert array_meta_key in store meta = decode_array_metadata(store[array_meta_key]) eq(ZARR_FORMAT, meta['zarr_format']) eq((1000,), meta['shape']) eq((100,), meta['chunks']) eq(np.dtype('i4'), meta['dtype'])
def test_init_array_overwrite_group(self): # setup path = 'foo/bar' store = self.create_store() store[path + '/' + group_meta_key] = encode_group_metadata() # don't overwrite with pytest.raises(ValueError): init_array(store, shape=1000, chunks=100, path=path) # do overwrite try: init_array(store, shape=1000, chunks=100, dtype='i4', path=path, overwrite=True) except NotImplementedError: pass else: assert (path + '/' + group_meta_key) not in store assert (path + '/' + array_meta_key) in store meta = decode_array_metadata(store[path + '/' + array_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000, ) == meta['shape'] assert (100, ) == meta['chunks'] assert np.dtype('i4') == meta['dtype']
def test_encode_decode_array_1(): meta = dict(shape=(100, ), chunks=(10, ), dtype=np.dtype('f8'), compressor=Zlib(1).get_config(), fill_value=None, filters=None, order='C') meta_json = '''{ "chunks": [10], "compressor": {"id": "zlib", "level": 1}, "dtype": "<f8", "fill_value": null, "filters": null, "order": "C", "shape": [100], "zarr_format": %s }''' % ZARR_FORMAT # test encoding meta_enc = encode_array_metadata(meta) assert_json_equal(meta_json, meta_enc) # test decoding meta_dec = decode_array_metadata(meta_enc) assert ZARR_FORMAT == meta_dec['zarr_format'] assert meta['shape'] == meta_dec['shape'] assert meta['chunks'] == meta_dec['chunks'] assert meta['dtype'] == meta_dec['dtype'] assert meta['compressor'] == meta_dec['compressor'] assert meta['order'] == meta_dec['order'] assert meta_dec['fill_value'] is None assert meta_dec['filters'] is None
def test_encode_decode_array_datetime_timedelta(): # some variations for k in ['m8[s]', 'M8[s]']: compressor = Blosc(cname='lz4', clevel=3, shuffle=2) dtype = np.dtype(k) fill_value = dtype.type("NaT") meta = dict( shape=(100, 100), chunks=(10, 10), dtype=dtype, compressor=compressor.get_config(), fill_value=fill_value, order=dtype.char, filters=[] ) meta_json = '''{ "chunks": [10, 10], "compressor": { "id": "blosc", "clevel": 3, "cname": "lz4", "shuffle": 2, "blocksize": 0 }, "dtype": "%s", "fill_value": -9223372036854775808, "filters": [], "order": "%s", "shape": [100, 100], "zarr_format": %s }''' % (dtype.str, dtype.char, ZARR_FORMAT) # test encoding meta_enc = encode_array_metadata(meta) assert_json_equal(meta_json, meta_enc) # test decoding meta_dec = decode_array_metadata(meta_enc) assert ZARR_FORMAT == meta_dec['zarr_format'] assert meta['shape'] == meta_dec['shape'] assert meta['chunks'] == meta_dec['chunks'] assert meta['dtype'] == meta_dec['dtype'] assert meta['compressor'] == meta_dec['compressor'] assert meta['order'] == meta_dec['order'] # Based off of this SO answer: https://stackoverflow.com/a/49972198 assert np.all( fill_value.view((np.uint8, fill_value.itemsize)) == meta_dec['fill_value'].view((np.uint8, meta_dec['fill_value'].itemsize)) ) assert [] == meta_dec['filters']
def test_init_array(self): store = self.create_store() init_array(store, shape=1000, chunks=100) # check metadata assert array_meta_key in store meta = decode_array_metadata(store[array_meta_key]) eq(ZARR_FORMAT, meta['zarr_format']) eq((1000,), meta['shape']) eq((100,), meta['chunks']) eq(np.dtype(None), meta['dtype']) eq(default_compressor.get_config(), meta['compressor']) assert_is_none(meta['fill_value'])
def test_init_array(self): store = self.create_store() init_array(store, shape=1000, chunks=100) # check metadata assert array_meta_key in store meta = decode_array_metadata(store[array_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000, ) == meta['shape'] assert (100, ) == meta['chunks'] assert np.dtype(None) == meta['dtype'] assert default_compressor.get_config() == meta['compressor'] assert meta['fill_value'] is None
def test_encode_decode_array_2(): # some variations df = Delta(astype='u2', dtype='V14') compressor = Blosc(cname='lz4', clevel=3, shuffle=2) dtype = np.dtype([('a', 'i4'), ('b', 'S10')]) fill_value = np.zeros((), dtype=dtype)[()] meta = dict( shape=(100, 100), chunks=(10, 10), dtype=dtype, compressor=compressor.get_config(), fill_value=fill_value, order='F', filters=[df.get_config()] ) meta_json = '''{ "chunks": [10, 10], "compressor": { "id": "blosc", "clevel": 3, "cname": "lz4", "shuffle": 2, "blocksize": 0 }, "dtype": [["a", "<i4"], ["b", "|S10"]], "fill_value": "AAAAAAAAAAAAAAAAAAA=", "filters": [ {"id": "delta", "astype": "<u2", "dtype": "|V14"} ], "order": "F", "shape": [100, 100], "zarr_format": %s }''' % ZARR_FORMAT # test encoding meta_enc = encode_array_metadata(meta) assert_json_equal(meta_json, meta_enc) # test decoding meta_dec = decode_array_metadata(meta_enc) assert ZARR_FORMAT == meta_dec['zarr_format'] assert meta['shape'] == meta_dec['shape'] assert meta['chunks'] == meta_dec['chunks'] assert meta['dtype'] == meta_dec['dtype'] assert meta['compressor'] == meta_dec['compressor'] assert meta['order'] == meta_dec['order'] assert fill_value == meta_dec['fill_value'] assert [df.get_config()] == meta_dec['filters']
def test_init_array(self): store = self.create_store() init_array(store, shape=1000, chunks=100) # check metadata assert array_meta_key in store meta = decode_array_metadata(store[array_meta_key]) eq(ZARR_FORMAT, meta['zarr_format']) eq((1000, ), meta['shape']) eq((100, ), meta['chunks']) eq(np.dtype(None), meta['dtype']) eq(default_compressor.get_config(), meta['compressor']) assert_is_none(meta['fill_value']) # check attributes assert attrs_key in store eq(dict(), json.loads(text_type(store[attrs_key], 'ascii')))
def test_encode_decode_fill_values_bytes(): dtype = np.dtype('S10') fills = b'foo', bytes(10) for v in fills: # setup and encode metadata meta = dict( shape=(100,), chunks=(10,), dtype=dtype, compressor=Zlib(1).get_config(), fill_value=v, filters=None, order='C' ) meta_enc = encode_array_metadata(meta) # define expected metadata encoded as JSON s = base64.standard_b64encode(v) if not PY2: s = s.decode() meta_json = '''{ "chunks": [10], "compressor": {"id": "zlib", "level": 1}, "dtype": "|S10", "fill_value": "%s", "filters": null, "order": "C", "shape": [100], "zarr_format": %s }''' % (s, ZARR_FORMAT) # test encoding assert_json_equal(meta_json, meta_enc) # test decoding meta_dec = decode_array_metadata(meta_enc) actual = meta_dec['fill_value'] expect = np.array(v, dtype=dtype)[()] assert expect == actual
def test_init_array_path(self): path = 'foo/bar' store = self.create_store() init_array(store, shape=1000, chunks=100, path=path) # check metadata key = path + '/' + array_meta_key assert key in store meta = decode_array_metadata(store[key]) eq(ZARR_FORMAT, meta['zarr_format']) eq((1000,), meta['shape']) eq((100,), meta['chunks']) eq(np.dtype(None), meta['dtype']) eq(default_compressor.get_config(), meta['compressor']) assert_is_none(meta['fill_value']) # check attributes key = path + '/' + attrs_key assert key in store eq(dict(), json.loads(text_type(store[key], 'ascii')))
def test_encode_decode_fill_values_nan(): fills = ( (np.nan, "NaN", np.isnan), (np.NINF, "-Infinity", np.isneginf), (np.PINF, "Infinity", np.isposinf), ) for v, s, f in fills: meta = dict( shape=(100,), chunks=(10,), dtype=np.dtype('f8'), compressor=Zlib(1).get_config(), fill_value=v, filters=None, order='C' ) meta_json = '''{ "chunks": [10], "compressor": {"id": "zlib", "level": 1}, "dtype": "<f8", "fill_value": "%s", "filters": null, "order": "C", "shape": [100], "zarr_format": %s }''' % (s, ZARR_FORMAT) # test encoding meta_enc = encode_array_metadata(meta) assert_json_equal(meta_json, meta_enc) # test decoding meta_dec = decode_array_metadata(meta_enc) actual = meta_dec['fill_value'] assert f(actual)
def test_encode_decode_array_structured(): meta = dict( shape=(100,), chunks=(10,), dtype=np.dtype('i8, (10, 10)f8, (5, 10, 15)u1'), compressor=Zlib(1).get_config(), fill_value=None, filters=None, order='C' ) meta_json = '''{ "chunks": [10], "compressor": {"id": "zlib", "level": 1}, "dtype": [["f0", "<i8"], ["f1", "<f8", [10, 10]], ["f2", "|u1", [5, 10, 15]]], "fill_value": null, "filters": null, "order": "C", "shape": [100], "zarr_format": %s }''' % ZARR_FORMAT # test encoding meta_enc = encode_array_metadata(meta) assert_json_equal(meta_json, meta_enc) # test decoding meta_dec = decode_array_metadata(meta_enc) assert ZARR_FORMAT == meta_dec['zarr_format'] # to maintain consistency with numpy unstructured arrays, unpack dimensions into shape assert meta['shape'] + meta['dtype'].shape == meta_dec['shape'] assert meta['chunks'] == meta_dec['chunks'] # to maintain consistency with numpy unstructured arrays, unpack dimensions into shape assert meta['dtype'].base == meta_dec['dtype'] assert meta['compressor'] == meta_dec['compressor'] assert meta['order'] == meta_dec['order'] assert meta_dec['fill_value'] is None assert meta_dec['filters'] is None
def test_encode_decode_array_fill_values(): fills = ( (np.nan, "NaN", np.isnan), (np.NINF, "-Infinity", np.isneginf), (np.PINF, "Infinity", np.isposinf), ) for v, s, f in fills: meta = dict( shape=(100,), chunks=(10,), dtype=np.dtype('f8'), compressor=Zlib(1).get_config(), fill_value=v, filters=None, order='C' ) meta_json = '''{ "chunks": [10], "compressor": {"id": "zlib", "level": 1}, "dtype": "<f8", "fill_value": "%s", "filters": null, "order": "C", "shape": [100], "zarr_format": %s }''' % (s, ZARR_FORMAT) # test encoding meta_enc = encode_array_metadata(meta) assert_json_eq(meta_json, meta_enc) # test decoding meta_dec = decode_array_metadata(meta_enc) actual = meta_dec['fill_value'] assert f(actual)
def test_init_array_overwrite_path(self): # setup path = 'foo/bar' store = self.create_store() meta = dict(shape=(2000, ), chunks=(200, ), dtype=np.dtype('u1'), compressor=Zlib(1).get_config(), fill_value=0, order='F', filters=None) store[array_meta_key] = encode_array_metadata(meta) store[path + '/' + array_meta_key] = encode_array_metadata(meta) # don't overwrite with pytest.raises(ValueError): init_array(store, shape=1000, chunks=100, path=path) # do overwrite try: init_array(store, shape=1000, chunks=100, dtype='i4', path=path, overwrite=True) except NotImplementedError: pass else: assert group_meta_key in store assert array_meta_key not in store assert (path + '/' + array_meta_key) in store # should have been overwritten meta = decode_array_metadata(store[path + '/' + array_meta_key]) assert ZARR_FORMAT == meta['zarr_format'] assert (1000, ) == meta['shape'] assert (100, ) == meta['chunks'] assert np.dtype('i4') == meta['dtype']
def test_init_array_overwrite_chunk_store(self): # setup store = self.create_store() chunk_store = self.create_store() store[array_meta_key] = encode_array_metadata( dict(shape=(2000, ), chunks=(200, ), dtype=np.dtype('u1'), compressor=None, fill_value=0, filters=None, order='F')) chunk_store['0'] = b'aaa' chunk_store['1'] = b'bbb' # don't overwrite (default) with assert_raises(ValueError): init_array(store, shape=1000, chunks=100, chunk_store=chunk_store) # do overwrite try: init_array(store, shape=1000, chunks=100, dtype='i4', overwrite=True, chunk_store=chunk_store) except NotImplementedError: pass else: assert array_meta_key in store meta = decode_array_metadata(store[array_meta_key]) eq(ZARR_FORMAT, meta['zarr_format']) eq((1000, ), meta['shape']) eq((100, ), meta['chunks']) eq(np.dtype('i4'), meta['dtype']) assert '0' not in chunk_store assert '1' not in chunk_store
def test_encode_decode_array_1(): meta = dict( shape=(100,), chunks=(10,), dtype=np.dtype('f8'), compressor=Zlib(1).get_config(), fill_value=None, filters=None, order='C' ) meta_json = '''{ "chunks": [10], "compressor": {"id": "zlib", "level": 1}, "dtype": "<f8", "fill_value": null, "filters": null, "order": "C", "shape": [100], "zarr_format": %s }''' % ZARR_FORMAT # test encoding meta_enc = encode_array_metadata(meta) assert_json_eq(meta_json, meta_enc) # test decoding meta_dec = decode_array_metadata(meta_enc) eq(ZARR_FORMAT, meta_dec['zarr_format']) eq(meta['shape'], meta_dec['shape']) eq(meta['chunks'], meta_dec['chunks']) eq(meta['dtype'], meta_dec['dtype']) eq(meta['compressor'], meta_dec['compressor']) eq(meta['order'], meta_dec['order']) assert_is_none(meta_dec['fill_value']) assert_is_none(meta_dec['filters'])
def test_init_array_overwrite_group(self): # setup path = 'foo/bar' store = self.create_store() store[path + '/' + group_meta_key] = encode_group_metadata() # don't overwrite with assert_raises(ValueError): init_array(store, shape=1000, chunks=100, path=path) # do overwrite try: init_array(store, shape=1000, chunks=100, dtype='i4', path=path, overwrite=True) except NotImplementedError: pass else: assert (path + '/' + group_meta_key) not in store assert (path + '/' + array_meta_key) in store meta = decode_array_metadata(store[path + '/' + array_meta_key]) eq(ZARR_FORMAT, meta['zarr_format']) eq((1000,), meta['shape']) eq((100,), meta['chunks']) eq(np.dtype('i4'), meta['dtype'])
def test_migrate_1to2(): from zarr import meta_v1 # N.B., version 1 did not support hierarchies, so we only have to be # concerned about migrating a single array at the root of the store # setup store = dict() meta = dict(shape=(100, ), chunks=(10, ), dtype=np.dtype('f4'), compression='zlib', compression_opts=1, fill_value=None, order='C') meta_json = meta_v1.encode_metadata(meta) store['meta'] = meta_json store['attrs'] = json.dumps(dict()).encode('ascii') # run migration migrate_1to2(store) # check results assert 'meta' not in store assert array_meta_key in store assert 'attrs' not in store assert attrs_key in store meta_migrated = decode_array_metadata(store[array_meta_key]) assert 2 == meta_migrated['zarr_format'] # preserved fields for f in 'shape', 'chunks', 'dtype', 'fill_value', 'order': assert meta[f] == meta_migrated[f] # migrate should have added empty filters field assert meta_migrated['filters'] is None # check compression and compression_opts migrated to compressor assert 'compression' not in meta_migrated assert 'compression_opts' not in meta_migrated assert meta_migrated['compressor'] == Zlib(1).get_config() # check dict compression_opts store = dict() meta['compression'] = 'blosc' meta['compression_opts'] = dict(cname='lz4', clevel=5, shuffle=1) meta_json = meta_v1.encode_metadata(meta) store['meta'] = meta_json store['attrs'] = json.dumps(dict()).encode('ascii') migrate_1to2(store) meta_migrated = decode_array_metadata(store[array_meta_key]) assert 'compression' not in meta_migrated assert 'compression_opts' not in meta_migrated assert (meta_migrated['compressor'] == Blosc(cname='lz4', clevel=5, shuffle=1).get_config()) # check 'none' compression is migrated to None (null in JSON) store = dict() meta['compression'] = 'none' meta_json = meta_v1.encode_metadata(meta) store['meta'] = meta_json store['attrs'] = json.dumps(dict()).encode('ascii') migrate_1to2(store) meta_migrated = decode_array_metadata(store[array_meta_key]) assert 'compression' not in meta_migrated assert 'compression_opts' not in meta_migrated assert meta_migrated['compressor'] is None
def test_migrate_1to2(): from zarr import meta_v1 # N.B., version 1 did not support hierarchies, so we only have to be # concerned about migrating a single array at the root of the store # setup store = dict() meta = dict( shape=(100,), chunks=(10,), dtype=np.dtype('f4'), compression='zlib', compression_opts=1, fill_value=None, order='C' ) meta_json = meta_v1.encode_metadata(meta) store['meta'] = meta_json store['attrs'] = json.dumps(dict()).encode('ascii') # run migration migrate_1to2(store) # check results assert 'meta' not in store assert array_meta_key in store assert 'attrs' not in store assert attrs_key in store meta_migrated = decode_array_metadata(store[array_meta_key]) eq(2, meta_migrated['zarr_format']) # preserved fields for f in 'shape', 'chunks', 'dtype', 'fill_value', 'order': eq(meta[f], meta_migrated[f]) # migrate should have added empty filters field assert_is_none(meta_migrated['filters']) # check compression and compression_opts migrated to compressor assert 'compression' not in meta_migrated assert 'compression_opts' not in meta_migrated eq(meta_migrated['compressor'], Zlib(1).get_config()) # check dict compression_opts store = dict() meta['compression'] = 'blosc' meta['compression_opts'] = dict(cname='lz4', clevel=5, shuffle=1) meta_json = meta_v1.encode_metadata(meta) store['meta'] = meta_json store['attrs'] = json.dumps(dict()).encode('ascii') migrate_1to2(store) meta_migrated = decode_array_metadata(store[array_meta_key]) assert 'compression' not in meta_migrated assert 'compression_opts' not in meta_migrated eq(meta_migrated['compressor'], Blosc(cname='lz4', clevel=5, shuffle=1).get_config()) # check 'none' compression is migrated to None (null in JSON) store = dict() meta['compression'] = 'none' meta_json = meta_v1.encode_metadata(meta) store['meta'] = meta_json store['attrs'] = json.dumps(dict()).encode('ascii') migrate_1to2(store) meta_migrated = decode_array_metadata(store[array_meta_key]) assert 'compression' not in meta_migrated assert 'compression_opts' not in meta_migrated assert_is_none(meta_migrated['compressor'])
def test_init_array_compat(self): store = self.create_store() init_array(store, shape=1000, chunks=100, compressor='none') meta = decode_array_metadata(store[array_meta_key]) assert meta['compressor'] is None
def test_init_array_compat(self): store = self.create_store() init_array(store, shape=1000, chunks=100, compressor='none') meta = decode_array_metadata(store[array_meta_key]) assert_is_none(meta['compressor'])