def chunks_info(zarray, chunks_loc):
"""Store chunks location information for a Zarr array.
Parameters
----------
zarray : zarr.core.Array
Zarr array that will use the chunk data.
chunks_loc : dict
File storage information for the chunks belonging to the Zarr array.
"""
if 'source' not in chunks_loc:
raise ValueError('Chunk source information missing')
if any([k not in chunks_loc['source'] for k in ('uri', 'array_name')]):
raise ValueError(
f'{chunks_loc["source"]}: Chunk source information incomplete')
key = _path_to_prefix(zarray.path) + chunks_meta_key
chunks_meta = dict()
for k, v in chunks_loc.items():
if k != 'source':
k = zarray._chunk_key(k)
if any([a not in v for a in ('offset', 'size')]):
raise ValueError(
f'{k}: Incomplete chunk location information')
chunks_meta[k] = v
# Store Zarr array chunk location metadata...
zarray.store[key] = json_dumps(chunks_meta)
def consolidate_metadata(fs, root):
if root.rstrip("/").endswith(".zarr"):
logger.info(f"Consolidating metadata of {root}")
meta = _get_metadata_fs(fs, root)
with fs.open(os.path.join(root, ".zmetadata"), "wb") as f:
f.write(json_dumps(meta))
elif fs.isdir(root):
logger.info(f"Recursing {root}")
dirs = fs.ls(root)
for dir_ in dirs:
consolidate_metadata(fs, dir_)
def consolidate_metadata(store, metadata_key='.zmetadata'):
"""
Consolidate all metadata for groups and arrays within the given store
into a single resource and put it under the given key.
This produces a single object in the backend store, containing all the
metadata read from all the zarr-related keys that can be found. After
metadata have been consolidated, use :func:`open_consolidated` to open
the root group in optimised, read-only mode, using the consolidated
metadata to reduce the number of read operations on the backend store.
Note, that if the metadata in the store is changed after this
consolidation, then the metadata read by :func:`open_consolidated`
would be incorrect unless this function is called again.
.. note:: This is an experimental feature.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
metadata_key : str
Key to put the consolidated metadata under.
Returns
-------
g : :class:`zarr.hierarchy.Group`
Group instance, opened with the new consolidated metadata.
See Also
--------
open_consolidated
"""
import json
store = normalize_store_arg(store)
def is_zarr_key(key):
return (key.endswith('.zarray') or key.endswith('.zgroup')
or key.endswith('.zattrs'))
out = {
'zarr_consolidated_format': 1,
'metadata': {
key: json.loads(ensure_str(store[key]))
for key in store if is_zarr_key(key)
}
}
store[metadata_key] = json_dumps(out).encode()
return open_consolidated(store, metadata_key=metadata_key)
def consolidate_metadata(store: BaseStore,
metadata_key=".zmetadata",
*,
path=''):
"""
Consolidate all metadata for groups and arrays within the given store
into a single resource and put it under the given key.
This produces a single object in the backend store, containing all the
metadata read from all the zarr-related keys that can be found. After
metadata have been consolidated, use :func:`open_consolidated` to open
the root group in optimised, read-only mode, using the consolidated
metadata to reduce the number of read operations on the backend store.
Note, that if the metadata in the store is changed after this
consolidation, then the metadata read by :func:`open_consolidated`
would be incorrect unless this function is called again.
.. note:: This is an experimental feature.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
metadata_key : str
Key to put the consolidated metadata under.
path : str or None
Path corresponding to the group that is being consolidated. Not required
for zarr v2 stores.
Returns
-------
g : :class:`zarr.hierarchy.Group`
Group instance, opened with the new consolidated metadata.
See Also
--------
open_consolidated
"""
store = normalize_store_arg(store, mode="w")
version = store._store_version
if version == 2:
def is_zarr_key(key):
return (key.endswith('.zarray') or key.endswith('.zgroup')
or key.endswith('.zattrs'))
else:
sfx = _get_metadata_suffix(store) # type: ignore
def is_zarr_key(key):
return (key.endswith('.array' + sfx)
or key.endswith('.group' + sfx) or key == 'zarr.json')
# cannot create a group without a path in v3
# so create /meta/root/consolidated group to store the metadata
if 'consolidated' not in store:
_create_group(store, path='consolidated')
if not metadata_key.startswith('meta/root/'):
metadata_key = 'meta/root/consolidated/' + metadata_key
# path = 'consolidated'
out = {
'zarr_consolidated_format': 1,
'metadata':
{key: json_loads(store[key])
for key in store if is_zarr_key(key)}
}
store[metadata_key] = json_dumps(out)
return open_consolidated(store, metadata_key=metadata_key, path=path)
