def consolidate_metadata(self, metadata_key='.zmetadata'):
'''
Wrapper over zarr.consolidate_metadata to pass chunk store when opening the zarr store
'''
# same as zarr.consolidate_metadata(self.store, metadata_key) call,
# only with key.endswith('.zchunkstore') in is_zarr_key, and passing chunk store
def is_zarr_key(key):
return (key.endswith('.zchunkstore') or key.endswith('.zarray')
or key.endswith('.zgroup') or key.endswith('.zattrs'))
out = {
'zarr_consolidated_format': 1,
'metadata': {
key: json_loads(self.store[key])
for key in self.store if is_zarr_key(key)
}
}
self.store[metadata_key] = json_dumps(out)
meta_store = ConsolidatedMetadataStore(self.store,
metadata_key=metadata_key)
store_mode_cons = 'r' if self.store_mode == 'r' else 'r+'
self.zgroup = zarr.open(store=meta_store,
mode=store_mode_cons,
chunk_store=self.zgroup.chunk_store,
path=self.store_path)
return self.zgroup
def parse_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]:
# Here we allow that a store may return an already-parsed metadata object,
# or a string of JSON that we will parse here. We allow for an already-parsed
# object to accommodate a consolidated metadata store, where all the metadata for
# all groups and arrays will already have been parsed from JSON.
if isinstance(s, Mapping):
# assume metadata has already been parsed into a mapping object
meta = s
else:
# assume metadata needs to be parsed as JSON
meta = json_loads(s)
return meta
def __init__(self,
store: StoreLike,
metadata_key=meta_root + "consolidated/.zmetadata"):
self.store = StoreV3._ensure_store(store)
# retrieve consolidated metadata
meta = json_loads(self.store[metadata_key])
# check format of consolidated metadata
consolidated_format = meta.get('zarr_consolidated_format', None)
if consolidated_format != 1:
raise MetadataError(
'unsupported zarr consolidated metadata format: %s' %
consolidated_format)
# decode metadata
self.meta_store: Store = KVStoreV3(meta["metadata"])
def generate_fixtures():
# Empty fixture
create_simple_array(os.path.join(FIXTURES_FOLDER, "empty.zarr"), dtype="<i4")
# little endian
for codec in (None, "gzip", "zlib"):
path = os.path.join(
FIXTURES_FOLDER, "simple{}_LE.zarr".format(f"_{codec}" if codec else "")
)
create_simple_array(
store=path,
dtype="<i4",
compression=codec,
write_chunks=True,
)
# big endian
for codec in (None, "gzip", "zlib"):
path = os.path.join(
FIXTURES_FOLDER, "simple{}_BE.zarr".format(f"_{codec}" if codec else "")
)
create_simple_array(
store=path,
dtype=">i4",
compression=codec,
write_chunks=True,
)
# nested
# TODO: Use latest zarr-python once https://github.com/zarr-developers/zarr-python/pull/716 is merged
store = zarr.storage.FSStore(
os.path.join(FIXTURES_FOLDER, "simple_nested.zarr"),
key_separator="/",
auto_mkdir=True,
)
create_simple_array(
store=store, dtype=">i4", compression="blosc", write_chunks=True
)
# Manually add dimension separator to array meta
meta = json_loads(store[".zarray"])
meta["dimension_separator"] = "/"
store[".zarray"] = json_dumps(meta)
def _ensure_dict(self, obj):
if isinstance(obj, bytes):
return json_loads(obj)
else:
return obj