def load(store: StoreLike):
"""Load data from an array or group into memory.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
Returns
-------
out
If the store contains an array, out will be a numpy array. If the store contains
a group, out will be a dict-like object where keys are array names and values
are numpy arrays.
See Also
--------
save, savez
Notes
-----
If loading data from a group of arrays, data will not be immediately loaded into
memory. Rather, arrays will be loaded into memory as they are requested.
"""
# handle polymorphic store arg
_store = normalize_store_arg(store)
if contains_array(_store, path=None):
return Array(store=_store, path=None)[...]
elif contains_group(_store, path=None):
grp = Group(store=_store, path=None)
return LazyLoader(grp)
def load(store):
"""Load data from an array or group into memory.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
Returns
-------
out
If the store contains an array, out will be a numpy array. If the store contains
a group, out will be a dict-like object where keys are array names and values
are numpy arrays.
See Also
--------
save, savez
Notes
-----
If loading data from a group of arrays, data will not be immediately loaded into
memory. Rather, arrays will be loaded into memory as they are requested.
"""
# handle polymorphic store arg
store = normalize_store_arg(store)
if contains_array(store, path=None):
return Array(store=store, path=None)[...]
elif contains_group(store, path=None):
grp = Group(store=store, path=None)
return LazyLoader(grp)
def open_consolidated(store, metadata_key='.zmetadata', mode='r+', **kwargs):
"""Open group using metadata previously consolidated into a single key.
This is an optimised method for opening a Zarr group, where instead of
traversing the group/array hierarchy by accessing the metadata keys at
each level, a single key contains all of the metadata for everything.
For remote data sources where the overhead of accessing a key is large
compared to the time to read data.
The group accessed must have already had its metadata consolidated into a
single key using the function :func:`consolidate_metadata`.
This optimised method only works in modes which do not change the
metadata, although the data may still be written/updated.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
metadata_key : str
Key to read the consolidated metadata from. The default (.zmetadata)
corresponds to the default used by :func:`consolidate_metadata`.
mode : {'r', 'r+'}, optional
Persistence mode: 'r' means read only (must exist); 'r+' means
read/write (must exist) although only writes to data are allowed,
changes to metadata including creation of new arrays or group
are not allowed.
**kwargs
Additional parameters are passed through to :func:`zarr.creation.open_array` or
:func:`zarr.hierarchy.open_group`.
Returns
-------
g : :class:`zarr.hierarchy.Group`
Group instance, opened with the consolidated metadata.
See Also
--------
consolidate_metadata
"""
from .storage import ConsolidatedMetadataStore
# normalize parameters
store = normalize_store_arg(store,
storage_options=kwargs.get(
"storage_options", None))
if mode not in {'r', 'r+'}:
raise ValueError(
"invalid mode, expected either 'r' or 'r+'; found {!r}".format(
mode))
# setup metadata store
meta_store = ConsolidatedMetadataStore(store, metadata_key=metadata_key)
# pass through
chunk_store = kwargs.pop('chunk_store', None) or store
return open(store=meta_store, chunk_store=chunk_store, mode=mode, **kwargs)
def _normalize_store_arg(store,
*,
clobber=False,
storage_options=None,
mode=None):
if store is None:
return MemoryStore()
return normalize_store_arg(store,
clobber=clobber,
storage_options=storage_options,
mode=mode)
def consolidate_metadata(store, metadata_key='.zmetadata'):
"""
Consolidate all metadata for groups and arrays within the given store
into a single resource and put it under the given key.
This produces a single object in the backend store, containing all the
metadata read from all the zarr-related keys that can be found. After
metadata have been consolidated, use :func:`open_consolidated` to open
the root group in optimised, read-only mode, using the consolidated
metadata to reduce the number of read operations on the backend store.
Note, that if the metadata in the store is changed after this
consolidation, then the metadata read by :func:`open_consolidated`
would be incorrect unless this function is called again.
.. note:: This is an experimental feature.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
metadata_key : str
Key to put the consolidated metadata under.
Returns
-------
g : :class:`zarr.hierarchy.Group`
Group instance, opened with the new consolidated metadata.
See Also
--------
open_consolidated
"""
import json
store = normalize_store_arg(store)
def is_zarr_key(key):
return (key.endswith('.zarray') or key.endswith('.zgroup')
or key.endswith('.zattrs'))
out = {
'zarr_consolidated_format': 1,
'metadata': {
key: json.loads(ensure_str(store[key]))
for key in store if is_zarr_key(key)
}
}
store[metadata_key] = json_dumps(out).encode()
return open_consolidated(store, metadata_key=metadata_key)
def save_array(store: StoreLike, arr, **kwargs):
"""Convenience function to save a NumPy array to the local file system, following a
similar API to the NumPy save() function.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
arr : ndarray
NumPy array with data to save.
kwargs
Passed through to :func:`create`, e.g., compressor.
Examples
--------
Save an array to a directory on the file system (uses a :class:`DirectoryStore`)::
>>> import zarr
>>> import numpy as np
>>> arr = np.arange(10000)
>>> zarr.save_array('data/example.zarr', arr)
>>> zarr.load('data/example.zarr')
array([ 0, 1, 2, ..., 9997, 9998, 9999])
Save an array to a single file (uses a :class:`ZipStore`)::
>>> zarr.save_array('data/example.zip', arr)
>>> zarr.load('data/example.zip')
array([ 0, 1, 2, ..., 9997, 9998, 9999])
"""
may_need_closing = _might_close(store)
_store: BaseStore = normalize_store_arg(store, clobber=True)
try:
_create_array(arr, store=_store, overwrite=True, **kwargs)
finally:
if may_need_closing:
# needed to ensure zip file records are written
_store.close()
def save_array(store, arr, **kwargs):
"""Convenience function to save a NumPy array to the local file system, following a
similar API to the NumPy save() function.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
arr : ndarray
NumPy array with data to save.
kwargs
Passed through to :func:`create`, e.g., compressor.
Examples
--------
Save an array to a directory on the file system (uses a :class:`DirectoryStore`)::
>>> import zarr
>>> import numpy as np
>>> arr = np.arange(10000)
>>> zarr.save_array('data/example.zarr', arr)
>>> zarr.load('data/example.zarr')
array([ 0, 1, 2, ..., 9997, 9998, 9999])
Save an array to a single file (uses a :class:`ZipStore`)::
>>> zarr.save_array('data/example.zip', arr)
>>> zarr.load('data/example.zip')
array([ 0, 1, 2, ..., 9997, 9998, 9999])
"""
may_need_closing = isinstance(store, str)
store = normalize_store_arg(store, clobber=True)
try:
_create_array(arr, store=store, overwrite=True, **kwargs)
finally:
if may_need_closing and hasattr(store, 'close'):
# needed to ensure zip file records are written
store.close()
def _normalize_store_arg(store, clobber=False):
return normalize_store_arg(store, clobber=clobber, default=DictStore)
def open(store: StoreLike = None, mode: str = "a", **kwargs):
"""Convenience function to open a group or array using file-mode-like semantics.
Parameters
----------
store : Store or string, optional
Store or path to directory in file system or name of zip file.
mode : {'r', 'r+', 'a', 'w', 'w-'}, optional
Persistence mode: 'r' means read only (must exist); 'r+' means
read/write (must exist); 'a' means read/write (create if doesn't
exist); 'w' means create (overwrite if exists); 'w-' means create
(fail if exists).
**kwargs
Additional parameters are passed through to :func:`zarr.creation.open_array` or
:func:`zarr.hierarchy.open_group`.
Returns
-------
z : :class:`zarr.core.Array` or :class:`zarr.hierarchy.Group`
Array or group, depending on what exists in the given store.
See Also
--------
zarr.creation.open_array, zarr.hierarchy.open_group
Examples
--------
Storing data in a directory 'data/example.zarr' on the local file system::
>>> import zarr
>>> store = 'data/example.zarr'
>>> zw = zarr.open(store, mode='w', shape=100, dtype='i4') # open new array
>>> zw
<zarr.core.Array (100,) int32>
>>> za = zarr.open(store, mode='a') # open existing array for reading and writing
>>> za
<zarr.core.Array (100,) int32>
>>> zr = zarr.open(store, mode='r') # open existing array read-only
>>> zr
<zarr.core.Array (100,) int32 read-only>
>>> gw = zarr.open(store, mode='w') # open new group, overwriting previous data
>>> gw
<zarr.hierarchy.Group '/'>
>>> ga = zarr.open(store, mode='a') # open existing group for reading and writing
>>> ga
<zarr.hierarchy.Group '/'>
>>> gr = zarr.open(store, mode='r') # open existing group read-only
>>> gr
<zarr.hierarchy.Group '/' read-only>
"""
path = kwargs.get('path', None)
# handle polymorphic store arg
clobber = mode == 'w'
# we pass storage options explicitly, since normalize_store_arg might construct
# a store if the input is a fsspec-compatible URL
_store: BaseStore = normalize_store_arg(store,
clobber=clobber,
storage_options=kwargs.pop(
"storage_options", {}))
path = normalize_storage_path(path)
if mode in {'w', 'w-', 'x'}:
if 'shape' in kwargs:
return open_array(_store, mode=mode, **kwargs)
else:
return open_group(_store, mode=mode, **kwargs)
elif mode == "a":
if "shape" in kwargs or contains_array(_store, path):
return open_array(_store, mode=mode, **kwargs)
else:
return open_group(_store, mode=mode, **kwargs)
else:
if contains_array(_store, path):
return open_array(_store, mode=mode, **kwargs)
elif contains_group(_store, path):
return open_group(_store, mode=mode, **kwargs)
else:
raise PathNotFoundError(path)
def save_group(store: StoreLike, *args, **kwargs):
"""Convenience function to save several NumPy arrays to the local file system, following a
similar API to the NumPy savez()/savez_compressed() functions.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
args : ndarray
NumPy arrays with data to save.
kwargs
NumPy arrays with data to save.
Examples
--------
Save several arrays to a directory on the file system (uses a
:class:`DirectoryStore`):
>>> import zarr
>>> import numpy as np
>>> a1 = np.arange(10000)
>>> a2 = np.arange(10000, 0, -1)
>>> zarr.save_group('data/example.zarr', a1, a2)
>>> loader = zarr.load('data/example.zarr')
>>> loader
<LazyLoader: arr_0, arr_1>
>>> loader['arr_0']
array([ 0, 1, 2, ..., 9997, 9998, 9999])
>>> loader['arr_1']
array([10000, 9999, 9998, ..., 3, 2, 1])
Save several arrays using named keyword arguments::
>>> zarr.save_group('data/example.zarr', foo=a1, bar=a2)
>>> loader = zarr.load('data/example.zarr')
>>> loader
<LazyLoader: bar, foo>
>>> loader['foo']
array([ 0, 1, 2, ..., 9997, 9998, 9999])
>>> loader['bar']
array([10000, 9999, 9998, ..., 3, 2, 1])
Store several arrays in a single zip file (uses a :class:`ZipStore`)::
>>> zarr.save_group('data/example.zip', foo=a1, bar=a2)
>>> loader = zarr.load('data/example.zip')
>>> loader
<LazyLoader: bar, foo>
>>> loader['foo']
array([ 0, 1, 2, ..., 9997, 9998, 9999])
>>> loader['bar']
array([10000, 9999, 9998, ..., 3, 2, 1])
Notes
-----
Default compression options will be used.
"""
if len(args) == 0 and len(kwargs) == 0:
raise ValueError('at least one array must be provided')
# handle polymorphic store arg
may_need_closing = _might_close(store)
_store: BaseStore = normalize_store_arg(store, clobber=True)
try:
grp = _create_group(_store, overwrite=True)
for i, arr in enumerate(args):
k = 'arr_{}'.format(i)
grp.create_dataset(k, data=arr, overwrite=True)
for k, arr in kwargs.items():
grp.create_dataset(k, data=arr, overwrite=True)
finally:
if may_need_closing:
# needed to ensure zip file records are written
_store.close()
def open(store, mode='a', **kwargs):
"""Convenience function to open a group or array using file-mode-like semantics.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
mode : {'r', 'r+', 'a', 'w', 'w-'}, optional
Persistence mode: 'r' means read only (must exist); 'r+' means
read/write (must exist); 'a' means read/write (create if doesn't
exist); 'w' means create (overwrite if exists); 'w-' means create
(fail if exists).
**kwargs
Additional parameters are passed through to :func:`zarr.open_array` or
:func:`zarr.open_group`.
See Also
--------
zarr.open_array, zarr.open_group
Examples
--------
Storing data in a directory 'data/example.zarr' on the local file system::
>>> import zarr
>>> store = 'data/example.zarr'
>>> zw = zarr.open(store, mode='w', shape=100, dtype='i4') # open new array
>>> zw
<zarr.core.Array (100,) int32>
>>> za = zarr.open(store, mode='a') # open existing array for reading and writing
>>> za
<zarr.core.Array (100,) int32>
>>> zr = zarr.open(store, mode='r') # open existing array read-only
>>> zr
<zarr.core.Array (100,) int32 read-only>
>>> gw = zarr.open(store, mode='w') # open new group, overwriting previous data
>>> gw
<zarr.hierarchy.Group '/'>
>>> ga = zarr.open(store, mode='a') # open existing group for reading and writing
>>> ga
<zarr.hierarchy.Group '/'>
>>> gr = zarr.open(store, mode='r') # open existing group read-only
>>> gr
<zarr.hierarchy.Group '/' read-only>
"""
path = kwargs.get('path', None)
# handle polymorphic store arg
store = normalize_store_arg(store, clobber=(mode == 'w'))
path = normalize_storage_path(path)
if mode in {'w', 'w-', 'x'}:
if 'shape' in kwargs:
return open_array(store, mode=mode, **kwargs)
else:
return open_group(store, mode=mode, **kwargs)
elif mode == 'a':
if contains_array(store, path):
return open_array(store, mode=mode, **kwargs)
elif contains_group(store, path):
return open_group(store, mode=mode, **kwargs)
elif 'shape' in kwargs:
return open_array(store, mode=mode, **kwargs)
else:
return open_group(store, mode=mode, **kwargs)
else:
if contains_array(store, path):
return open_array(store, mode=mode, **kwargs)
elif contains_group(store, path):
return open_group(store, mode=mode, **kwargs)
else:
err_path_not_found(path)
def _normalize_store_arg(store, clobber=False, storage_options=None):
return normalize_store_arg(store,
clobber=clobber,
default=MemoryStore,
storage_options=storage_options)
def open(store, mode='a', **kwargs):
"""Convenience function to open a group or array using file-mode-like semantics.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
mode : {'r', 'r+', 'a', 'w', 'w-'}, optional
Persistence mode: 'r' means read only (must exist); 'r+' means
read/write (must exist); 'a' means read/write (create if doesn't
exist); 'w' means create (overwrite if exists); 'w-' means create
(fail if exists).
**kwargs
Additional parameters are passed through to :func:`zarr.open_array` or
:func:`zarr.open_group`.
See Also
--------
zarr.open_array, zarr.open_group
Examples
--------
Storing data in a directory 'data/example.zarr' on the local file system::
>>> import zarr
>>> store = 'data/example.zarr'
>>> zw = zarr.open(store, mode='w', shape=100, dtype='i4') # open new array
>>> zw
<zarr.core.Array (100,) int32>
>>> za = zarr.open(store, mode='a') # open existing array for reading and writing
>>> za
<zarr.core.Array (100,) int32>
>>> zr = zarr.open(store, mode='r') # open existing array read-only
>>> zr
<zarr.core.Array (100,) int32 read-only>
>>> gw = zarr.open(store, mode='w') # open new group, overwriting previous data
>>> gw
<zarr.hierarchy.Group '/'>
>>> ga = zarr.open(store, mode='a') # open existing group for reading and writing
>>> ga
<zarr.hierarchy.Group '/'>
>>> gr = zarr.open(store, mode='r') # open existing group read-only
>>> gr
<zarr.hierarchy.Group '/' read-only>
"""
path = kwargs.get('path', None)
# handle polymorphic store arg
store = normalize_store_arg(store, clobber=(mode == 'w'))
path = normalize_storage_path(path)
if mode in {'w', 'w-', 'x'}:
if 'shape' in kwargs:
return open_array(store, mode=mode, **kwargs)
else:
return open_group(store, mode=mode, **kwargs)
elif mode == 'a':
if contains_array(store, path):
return open_array(store, mode=mode, **kwargs)
elif contains_group(store, path):
return open_group(store, mode=mode, **kwargs)
elif 'shape' in kwargs:
return open_array(store, mode=mode, **kwargs)
else:
return open_group(store, mode=mode, **kwargs)
else:
if contains_array(store, path):
return open_array(store, mode=mode, **kwargs)
elif contains_group(store, path):
return open_group(store, mode=mode, **kwargs)
else:
err_path_not_found(path)
def save_group(store, *args, **kwargs):
"""Convenience function to save several NumPy arrays to the local file system, following a
similar API to the NumPy savez()/savez_compressed() functions.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
args : ndarray
NumPy arrays with data to save.
kwargs
NumPy arrays with data to save.
Examples
--------
Save several arrays to a directory on the file system (uses a
:class:`DirectoryStore`):
>>> import zarr
>>> import numpy as np
>>> a1 = np.arange(10000)
>>> a2 = np.arange(10000, 0, -1)
>>> zarr.save_group('data/example.zarr', a1, a2)
>>> loader = zarr.load('data/example.zarr')
>>> loader
<LazyLoader: arr_0, arr_1>
>>> loader['arr_0']
array([ 0, 1, 2, ..., 9997, 9998, 9999])
>>> loader['arr_1']
array([10000, 9999, 9998, ..., 3, 2, 1])
Save several arrays using named keyword arguments::
>>> zarr.save_group('data/example.zarr', foo=a1, bar=a2)
>>> loader = zarr.load('data/example.zarr')
>>> loader
<LazyLoader: bar, foo>
>>> loader['foo']
array([ 0, 1, 2, ..., 9997, 9998, 9999])
>>> loader['bar']
array([10000, 9999, 9998, ..., 3, 2, 1])
Store several arrays in a single zip file (uses a :class:`ZipStore`)::
>>> zarr.save_group('data/example.zip', foo=a1, bar=a2)
>>> loader = zarr.load('data/example.zip')
>>> loader
<LazyLoader: bar, foo>
>>> loader['foo']
array([ 0, 1, 2, ..., 9997, 9998, 9999])
>>> loader['bar']
array([10000, 9999, 9998, ..., 3, 2, 1])
Notes
-----
Default compression options will be used.
"""
if len(args) == 0 and len(kwargs) == 0:
raise ValueError('at least one array must be provided')
# handle polymorphic store arg
may_need_closing = isinstance(store, str)
store = normalize_store_arg(store, clobber=True)
try:
grp = _create_group(store, overwrite=True)
for i, arr in enumerate(args):
k = 'arr_{}'.format(i)
grp.create_dataset(k, data=arr, overwrite=True)
for k, arr in kwargs.items():
grp.create_dataset(k, data=arr, overwrite=True)
finally:
if may_need_closing and hasattr(store, 'close'):
# needed to ensure zip file records are written
store.close()
def _normalize_store_arg(store, clobber=False):
return normalize_store_arg(store, clobber=clobber, default=DictStore)