def encode_array_metadata(cls, meta: MappingType[str, Any]) -> bytes:
dtype = meta["dtype"]
sdshape = ()
if dtype.subdtype is not None:
dtype, sdshape = dtype.subdtype
dimension_separator = meta.get("dimension_separator")
if dtype.hasobject:
import numcodecs
object_codec = numcodecs.get_codec(meta['filters'][0])
else:
object_codec = None
meta = dict(
zarr_format=cls.ZARR_FORMAT,
shape=meta["shape"] + sdshape,
chunks=meta["chunks"],
dtype=cls.encode_dtype(dtype),
compressor=meta["compressor"],
fill_value=cls.encode_fill_value(meta["fill_value"], dtype, object_codec),
order=meta["order"],
filters=meta["filters"],
)
if dimension_separator:
meta['dimension_separator'] = dimension_separator
if dimension_separator:
meta["dimension_separator"] = dimension_separator
return json_dumps(meta)
def chunks_info(zarray, chunks_loc):
"""Store chunks location information for a Zarr array.
Parameters
----------
zarray : zarr.core.Array
Zarr array that will use the chunk data.
chunks_loc : dict
File storage information for the chunks belonging to the Zarr array.
"""
if 'source' not in chunks_loc:
raise ValueError('Chunk source information missing')
if any([k not in chunks_loc['source'] for k in ('uri', 'array_name')]):
raise ValueError(
f'{chunks_loc["source"]}: Chunk source information incomplete')
key = _path_to_prefix(zarray.path) + chunks_meta_key
chunks_meta = dict()
for k, v in chunks_loc.items():
if k != 'source':
k = zarray._chunk_key(k)
if any([a not in v for a in ('offset', 'size')]):
raise ValueError(
f'{k}: Incomplete chunk location information')
chunks_meta[k] = v
# Store Zarr array chunk location metadata...
zarray.store[key] = json_dumps(chunks_meta)
def consolidate_metadata(self, metadata_key='.zmetadata'):
'''
Wrapper over zarr.consolidate_metadata to pass chunk store when opening the zarr store
'''
# same as zarr.consolidate_metadata(self.store, metadata_key) call,
# only with key.endswith('.zchunkstore') in is_zarr_key, and passing chunk store
def is_zarr_key(key):
return (key.endswith('.zchunkstore') or key.endswith('.zarray')
or key.endswith('.zgroup') or key.endswith('.zattrs'))
out = {
'zarr_consolidated_format': 1,
'metadata': {
key: json_loads(self.store[key])
for key in self.store if is_zarr_key(key)
}
}
self.store[metadata_key] = json_dumps(out)
meta_store = ConsolidatedMetadataStore(self.store,
metadata_key=metadata_key)
store_mode_cons = 'r' if self.store_mode == 'r' else 'r+'
self.zgroup = zarr.open(store=meta_store,
mode=store_mode_cons,
chunk_store=self.zgroup.chunk_store,
path=self.store_path)
return self.zgroup
def _init_meta_store(dzi_meta: DZIMetadata, csize: int) -> dict:
"""Generates zarr metadata key-value mapping for all levels of DZI pyramid"""
d = dict()
# DZI generates all levels of the pyramid
# Level 0 is 1x1 image, so we need to calculate the max level (highest resolution)
# and trim the pyramid to just the tiled levels.
max_size = max(dzi_meta.width, dzi_meta.height)
max_level = np.ceil(np.log2(max_size)).astype(int)
nlevels = max_level - np.ceil(np.log2(dzi_meta.tilesize)).astype(int)
levels = list(reversed(range(max_level + 1)))[:nlevels]
# Create root group
group_meta = dict(zarr_format=ZARR_FORMAT)
d[ZARR_GROUP_META_KEY] = json_dumps(group_meta)
# Create root attrs (multiscale meta)
datasets = [dict(path=str(i)) for i in levels]
root_attrs = dict(multiscales=[dict(datasets=datasets, version="0.1")])
d[ZARR_META_KEY] = json_dumps(root_attrs)
# Create zarr array meta for each level of DZI pyramid
for level in range(nlevels):
xsize, ysize = (dzi_meta.width // 2 ** level, dzi_meta.height // 2 ** level)
arr_meta_key = f"{max_level - level}/{ZARR_ARRAY_META_KEY}"
arr_meta = dict(
shape=(ysize, xsize, csize),
chunks=(dzi_meta.tilesize, dzi_meta.tilesize, csize),
compressor=None, # chunk is decoded with store, so no zarr compression
dtype="|u1", # RGB/A images only
fill_value=0,
filters=None,
order="C",
zarr_format=ZARR_FORMAT,
)
d[arr_meta_key] = json_dumps(arr_meta)
return d
def _put_nosync(self, d):
if self._version == 2:
self.store[self.key] = json_dumps(d)
if self.cache:
self._cached_asdict = d
else:
if self.key in self.store:
# Cannot write the attributes directly to JSON, but have to
# store it within the pre-existing attributes key of the v3
# metadata.
# Note: this changes the store.counter result in test_caching_on!
meta = self.store._metadata_class.parse_metadata(self.store[self.key])
if 'attributes' in meta and 'filters' in meta['attributes']:
# need to preserve any existing "filters" attribute
d['attributes']['filters'] = meta['attributes']['filters']
meta['attributes'] = d['attributes']
else:
meta = d
self.store[self.key] = json_dumps(meta)
if self.cache:
self._cached_asdict = d['attributes']
def encode_array_metadata(meta: MappingType[str, Any]) -> bytes:
dtype = meta['dtype']
sdshape = ()
if dtype.subdtype is not None:
dtype, sdshape = dtype.subdtype
meta = dict(
zarr_format=ZARR_FORMAT,
shape=meta['shape'] + sdshape,
chunks=meta['chunks'],
dtype=encode_dtype(dtype),
compressor=meta['compressor'],
fill_value=encode_fill_value(meta['fill_value'], dtype),
order=meta['order'],
filters=meta['filters'],
)
return json_dumps(meta)
def generate_fixtures():
# Empty fixture
create_simple_array(os.path.join(FIXTURES_FOLDER, "empty.zarr"), dtype="<i4")
# little endian
for codec in (None, "gzip", "zlib"):
path = os.path.join(
FIXTURES_FOLDER, "simple{}_LE.zarr".format(f"_{codec}" if codec else "")
)
create_simple_array(
store=path,
dtype="<i4",
compression=codec,
write_chunks=True,
)
# big endian
for codec in (None, "gzip", "zlib"):
path = os.path.join(
FIXTURES_FOLDER, "simple{}_BE.zarr".format(f"_{codec}" if codec else "")
)
create_simple_array(
store=path,
dtype=">i4",
compression=codec,
write_chunks=True,
)
# nested
# TODO: Use latest zarr-python once https://github.com/zarr-developers/zarr-python/pull/716 is merged
store = zarr.storage.FSStore(
os.path.join(FIXTURES_FOLDER, "simple_nested.zarr"),
key_separator="/",
auto_mkdir=True,
)
create_simple_array(
store=store, dtype=">i4", compression="blosc", write_chunks=True
)
# Manually add dimension separator to array meta
meta = json_loads(store[".zarray"])
meta["dimension_separator"] = "/"
store[".zarray"] = json_dumps(meta)
def _put_nosync(self, d):
self.store[self.key] = json_dumps(d)
if self.cache:
self._cached_asdict = d
def init_attrs(store: MutableMapping,
attrs: Mapping[str, Any],
path: str = None):
path = normalize_storage_path(path)
path = _path_to_prefix(path)
store[path + attrs_key] = json_dumps(attrs)
def encode_group_metadata(cls, meta=None) -> bytes:
meta = dict(zarr_format=cls.ZARR_FORMAT)
return json_dumps(meta)
def encode_group_metadata(meta=None):
meta = dict(zarr_format=ZARR_FORMAT, )
return json_dumps(meta)
def test_json_dumps_numpy_dtype():
assert json_dumps(np.int64(0)) == json_dumps(0)
assert json_dumps(np.float32(0)) == json_dumps(float(0))
# Check that we raise the error of the superclass for unsupported object
with pytest.raises(TypeError):
json_dumps(Array)