def consolidate_metadata(store: BaseStore,
metadata_key=".zmetadata",
*,
path=''):
"""
Consolidate all metadata for groups and arrays within the given store
into a single resource and put it under the given key.
This produces a single object in the backend store, containing all the
metadata read from all the zarr-related keys that can be found. After
metadata have been consolidated, use :func:`open_consolidated` to open
the root group in optimised, read-only mode, using the consolidated
metadata to reduce the number of read operations on the backend store.
Note, that if the metadata in the store is changed after this
consolidation, then the metadata read by :func:`open_consolidated`
would be incorrect unless this function is called again.
.. note:: This is an experimental feature.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
metadata_key : str
Key to put the consolidated metadata under.
path : str or None
Path corresponding to the group that is being consolidated. Not required
for zarr v2 stores.
Returns
-------
g : :class:`zarr.hierarchy.Group`
Group instance, opened with the new consolidated metadata.
See Also
--------
open_consolidated
"""
store = normalize_store_arg(store, mode="w")
version = store._store_version
if version == 2:
def is_zarr_key(key):
return (key.endswith('.zarray') or key.endswith('.zgroup')
or key.endswith('.zattrs'))
else:
sfx = _get_metadata_suffix(store) # type: ignore
def is_zarr_key(key):
return (key.endswith('.array' + sfx)
or key.endswith('.group' + sfx) or key == 'zarr.json')
# cannot create a group without a path in v3
# so create /meta/root/consolidated group to store the metadata
if 'consolidated' not in store:
_create_group(store, path='consolidated')
if not metadata_key.startswith('meta/root/'):
metadata_key = 'meta/root/consolidated/' + metadata_key
# path = 'consolidated'
out = {
'zarr_consolidated_format': 1,
'metadata':
{key: json_loads(store[key])
for key in store if is_zarr_key(key)}
}
store[metadata_key] = json_dumps(out)
return open_consolidated(store, metadata_key=metadata_key, path=path)
def save_group(store, *args, **kwargs):
"""Convenience function to save several NumPy arrays to the local file system, following a
similar API to the NumPy savez()/savez_compressed() functions.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
args : ndarray
NumPy arrays with data to save.
kwargs
NumPy arrays with data to save.
Examples
--------
Save several arrays to a directory on the file system (uses a
:class:`DirectoryStore`):
>>> import zarr
>>> import numpy as np
>>> a1 = np.arange(10000)
>>> a2 = np.arange(10000, 0, -1)
>>> zarr.save_group('data/example.zarr', a1, a2)
>>> loader = zarr.load('data/example.zarr')
>>> loader
<LazyLoader: arr_0, arr_1>
>>> loader['arr_0']
array([ 0, 1, 2, ..., 9997, 9998, 9999])
>>> loader['arr_1']
array([10000, 9999, 9998, ..., 3, 2, 1])
Save several arrays using named keyword arguments::
>>> zarr.save_group('data/example.zarr', foo=a1, bar=a2)
>>> loader = zarr.load('data/example.zarr')
>>> loader
<LazyLoader: bar, foo>
>>> loader['foo']
array([ 0, 1, 2, ..., 9997, 9998, 9999])
>>> loader['bar']
array([10000, 9999, 9998, ..., 3, 2, 1])
Store several arrays in a single zip file (uses a :class:`ZipStore`)::
>>> zarr.save_group('data/example.zip', foo=a1, bar=a2)
>>> loader = zarr.load('data/example.zip')
>>> loader
<LazyLoader: bar, foo>
>>> loader['foo']
array([ 0, 1, 2, ..., 9997, 9998, 9999])
>>> loader['bar']
array([10000, 9999, 9998, ..., 3, 2, 1])
Notes
-----
Default compression options will be used.
"""
if len(args) == 0 and len(kwargs) == 0:
raise ValueError('at least one array must be provided')
# handle polymorphic store arg
may_need_closing = isinstance(store, str)
store = normalize_store_arg(store, clobber=True)
try:
grp = _create_group(store, overwrite=True)
for i, arr in enumerate(args):
k = 'arr_{}'.format(i)
grp.create_dataset(k, data=arr, overwrite=True)
for k, arr in kwargs.items():
grp.create_dataset(k, data=arr, overwrite=True)
finally:
if may_need_closing and hasattr(store, 'close'):
# needed to ensure zip file records are written
store.close()
def save_group(store, *args, **kwargs):
"""Convenience function to save several NumPy arrays to the local file system, following a
similar API to the NumPy savez()/savez_compressed() functions.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
args : ndarray
NumPy arrays with data to save.
kwargs
NumPy arrays with data to save.
Examples
--------
Save several arrays to a directory on the file system (uses a
:class:`DirectoryStore`):
>>> import zarr
>>> import numpy as np
>>> a1 = np.arange(10000)
>>> a2 = np.arange(10000, 0, -1)
>>> zarr.save_group('data/example.zarr', a1, a2)
>>> loader = zarr.load('data/example.zarr')
>>> loader
<LazyLoader: arr_0, arr_1>
>>> loader['arr_0']
array([ 0, 1, 2, ..., 9997, 9998, 9999])
>>> loader['arr_1']
array([10000, 9999, 9998, ..., 3, 2, 1])
Save several arrays using named keyword arguments::
>>> zarr.save_group('data/example.zarr', foo=a1, bar=a2)
>>> loader = zarr.load('data/example.zarr')
>>> loader
<LazyLoader: bar, foo>
>>> loader['foo']
array([ 0, 1, 2, ..., 9997, 9998, 9999])
>>> loader['bar']
array([10000, 9999, 9998, ..., 3, 2, 1])
Store several arrays in a single zip file (uses a :class:`ZipStore`)::
>>> zarr.save_group('data/example.zip', foo=a1, bar=a2)
>>> loader = zarr.load('data/example.zip')
>>> loader
<LazyLoader: bar, foo>
>>> loader['foo']
array([ 0, 1, 2, ..., 9997, 9998, 9999])
>>> loader['bar']
array([10000, 9999, 9998, ..., 3, 2, 1])
Notes
-----
Default compression options will be used.
"""
if len(args) == 0 and len(kwargs) == 0:
raise ValueError('at least one array must be provided')
# handle polymorphic store arg
may_need_closing = isinstance(store, str)
store = normalize_store_arg(store, clobber=True)
try:
grp = _create_group(store, overwrite=True)
for i, arr in enumerate(args):
k = 'arr_{}'.format(i)
grp.create_dataset(k, data=arr, overwrite=True)
for k, arr in kwargs.items():
grp.create_dataset(k, data=arr, overwrite=True)
finally:
if may_need_closing and hasattr(store, 'close'):
# needed to ensure zip file records are written
store.close()
def save_group(store: StoreLike,
*args,
zarr_version=None,
path=None,
**kwargs):
"""Convenience function to save several NumPy arrays to the local file system, following a
similar API to the NumPy savez()/savez_compressed() functions.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
args : ndarray
NumPy arrays with data to save.
zarr_version : {2, 3, None}, optional
The zarr protocol version to use when saving. The default value of None
will attempt to infer the version from `store` if possible, otherwise
it will fall back to 2.
path : str or None, optional
Path within the store where the group will be saved.
kwargs
NumPy arrays with data to save.
Examples
--------
Save several arrays to a directory on the file system (uses a
:class:`DirectoryStore`):
>>> import zarr
>>> import numpy as np
>>> a1 = np.arange(10000)
>>> a2 = np.arange(10000, 0, -1)
>>> zarr.save_group('data/example.zarr', a1, a2)
>>> loader = zarr.load('data/example.zarr')
>>> loader
<LazyLoader: arr_0, arr_1>
>>> loader['arr_0']
array([ 0, 1, 2, ..., 9997, 9998, 9999])
>>> loader['arr_1']
array([10000, 9999, 9998, ..., 3, 2, 1])
Save several arrays using named keyword arguments::
>>> zarr.save_group('data/example.zarr', foo=a1, bar=a2)
>>> loader = zarr.load('data/example.zarr')
>>> loader
<LazyLoader: bar, foo>
>>> loader['foo']
array([ 0, 1, 2, ..., 9997, 9998, 9999])
>>> loader['bar']
array([10000, 9999, 9998, ..., 3, 2, 1])
Store several arrays in a single zip file (uses a :class:`ZipStore`)::
>>> zarr.save_group('data/example.zip', foo=a1, bar=a2)
>>> loader = zarr.load('data/example.zip')
>>> loader
<LazyLoader: bar, foo>
>>> loader['foo']
array([ 0, 1, 2, ..., 9997, 9998, 9999])
>>> loader['bar']
array([10000, 9999, 9998, ..., 3, 2, 1])
Notes
-----
Default compression options will be used.
"""
if len(args) == 0 and len(kwargs) == 0:
raise ValueError('at least one array must be provided')
# handle polymorphic store arg
may_need_closing = _might_close(store)
_store: BaseStore = normalize_store_arg(store,
mode="w",
zarr_version=zarr_version)
path = _check_and_update_path(_store, path)
try:
grp = _create_group(_store,
path=path,
overwrite=True,
zarr_version=zarr_version)
for i, arr in enumerate(args):
k = 'arr_{}'.format(i)
grp.create_dataset(k,
data=arr,
overwrite=True,
zarr_version=zarr_version)
for k, arr in kwargs.items():
grp.create_dataset(k,
data=arr,
overwrite=True,
zarr_version=zarr_version)
finally:
if may_need_closing:
# needed to ensure zip file records are written
_store.close()