def test_open_array_n5(zarr_version):
store = 'data/array.zarr'
kwargs = _init_creation_kwargs(zarr_version)
# for N5 store
store = 'data/array.n5'
z = open_array(store, mode='w', shape=100, chunks=10, **kwargs)
z[:] = 42
assert isinstance(z, Array)
assert isinstance(z.store, N5Store)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
store = 'data/group.n5'
group_kwargs = kwargs.copy()
# if zarr_version == 3:
# group_kwargs['path'] = 'group'
z = open_group(store, mode='w', **group_kwargs)
i = z.create_group('inner')
a = i.zeros("array", shape=100, chunks=10)
a[:] = 42
# Edit inner/attributes.json to not include "n5"
with open('data/group.n5/inner/attributes.json', 'w') as o:
o.write("{}")
# Re-open
a = open_group(store, **group_kwargs)["inner"]["array"]
assert isinstance(a, Array)
assert isinstance(z.store, N5Store)
assert (100, ) == a.shape
assert (10, ) == a.chunks
assert_array_equal(np.full(100, fill_value=42), a[:])
def test_round_trip_nd(self):
data = np.arange(1000).reshape(10, 10, 10)
name = 'raw'
store, _ = self.create_store()
f = open_group(store, mode='w')
f.create_dataset(name, data=data, chunks=(5, 5, 5), compressor=None)
h = open_group(store, mode='r')
np.testing.assert_array_equal(h[name][:], data)
def test_inconsistent_dimension_separator(self):
data = np.arange(1000).reshape(10, 10, 10)
name = 'raw'
store, _ = self.create_store()
f = open_group(store, mode='w')
# cannot specify dimension_separator that conflicts with the store
with pytest.raises(ValueError):
f.create_dataset(name, data=data, chunks=(5, 5, 5),
compressor=None, dimension_separator='.')
def image_to_zarr(image: omero.gateway.ImageWrapper,
args: argparse.Namespace) -> None:
target_dir = args.output
cache_dir = target_dir if args.cache_numpy else None
name = os.path.join(target_dir, "%s.zarr" % image.id)
print(f"Exporting to {name} ({VERSION})")
store = open_store(name)
root = open_group(store)
n_levels, axes = add_image(image, root, cache_dir=cache_dir)
add_multiscales_metadata(root, axes, n_levels)
add_omero_metadata(root, image)
add_toplevel_metadata(root)
print("Finished.")
def test_open_group():
# test the open_group() convenience function
store = 'example'
# mode == 'w'
g = open_group(store, mode='w')
assert_is_instance(g, Group)
assert_is_instance(g.store, DirectoryStore)
eq(0, len(g))
g.create_groups('foo', 'bar')
eq(2, len(g))
# mode in 'r', 'r+'
open_array('example_array', shape=100, chunks=10, mode='w')
for mode in 'r', 'r+':
with assert_raises(KeyError):
open_group('doesnotexist', mode=mode)
with assert_raises(KeyError):
open_group('example_array', mode=mode)
g = open_group(store, mode='r')
assert_is_instance(g, Group)
eq(2, len(g))
with assert_raises(PermissionError):
g.create_group('baz')
g = open_group(store, mode='r+')
assert_is_instance(g, Group)
eq(2, len(g))
g.create_groups('baz', 'quux')
eq(4, len(g))
# mode == 'a'
shutil.rmtree(store)
g = open_group(store, mode='a')
assert_is_instance(g, Group)
assert_is_instance(g.store, DirectoryStore)
eq(0, len(g))
g.create_groups('foo', 'bar')
eq(2, len(g))
with assert_raises(KeyError):
open_group('example_array', mode='a')
# mode in 'w-', 'x'
for mode in 'w-', 'x':
shutil.rmtree(store)
g = open_group(store, mode=mode)
assert_is_instance(g, Group)
assert_is_instance(g.store, DirectoryStore)
eq(0, len(g))
g.create_groups('foo', 'bar')
eq(2, len(g))
with assert_raises(KeyError):
open_group(store, mode=mode)
with assert_raises(KeyError):
open_group('example_array', mode=mode)
# open with path
g = open_group(store, path='foo/bar')
assert_is_instance(g, Group)
eq('foo/bar', g.path)
def plate_to_zarr(plate: omero.gateway._PlateWrapper,
args: argparse.Namespace) -> None:
"""
Exports a plate to a zarr file using the hierarchy discussed here ('Option 3'):
https://github.com/ome/omero-ms-zarr/issues/73#issuecomment-706770955
"""
gs = plate.getGridSize()
n_rows = gs["rows"]
n_cols = gs["columns"]
n_fields = plate.getNumberOfFields()
total = n_rows * n_cols * (n_fields[1] - n_fields[0] + 1)
target_dir = args.output
cache_dir = target_dir if args.cache_numpy else None
name = os.path.join(target_dir, "%s.zarr" % plate.id)
print(f"Exporting to {name}")
root = open_group(name, mode="w")
count = 0
max_fields = 0
t0 = time.time()
row_names = set()
col_names = set()
well_paths = set()
col_names = plate.getColumnLabels()
row_names = plate.getRowLabels()
plate_acqs = list(plate.listPlateAcquisitions())
ac_names = [pa.name for pa in plate_acqs] if plate_acqs else ["0"]
plate_metadata = {
"name": plate.name,
"rows": [{
"name": str(name)
} for name in row_names],
"columns": [{
"name": str(name)
} for name in col_names],
"acquisitions": [{
"path": x
} for x in ac_names],
}
root.attrs["plate"] = plate_metadata
for well in plate.listChildren():
row = plate.getRowLabels()[well.row]
col = plate.getColumnLabels()[well.column]
field_paths = []
for field in range(n_fields[0], n_fields[1] + 1):
ws = well.getWellSample(field)
field_name = "%d" % field
count += 1
if ws and ws.getImage():
img = ws.getImage()
ac = ws.getPlateAcquisition()
ac_name = ac.getName() if ac else "0"
well_paths.add(f"{ac_name}/{row}/{col}/")
field_paths.append(f"{field_name}/")
ac_group = root.require_group(ac_name)
row_group = ac_group.require_group(row)
col_group = row_group.require_group(col)
field_group = col_group.require_group(field_name)
n_levels = add_image(img, field_group, cache_dir=cache_dir)
add_group_metadata(field_group, img, n_levels)
# Update Well metadata after each image
col_group.attrs["well"] = {
"images": [{
"path": x
} for x in field_paths]
}
max_fields = max(max_fields, field + 1)
print_status(int(t0), int(time.time()), count, total)
# Update plate_metadata after each Well
plate_metadata["wells"] = [{"path": x} for x in well_paths]
plate_metadata["field_count"] = max_fields
root.attrs["plate"] = plate_metadata
add_toplevel_metadata(root)
print("Finished.")
def test_open_array():
store = 'data/array.zarr'
# mode == 'w'
z = open_array(store, mode='w', shape=100, chunks=10)
z[:] = 42
assert_is_instance(z, Array)
assert_is_instance(z.store, DirectoryStore)
eq((100,), z.shape)
eq((10,), z.chunks)
assert_array_equal(np.full(100, fill_value=42), z[:])
# mode in 'r', 'r+'
open_group('data/group.zarr', mode='w')
for mode in 'r', 'r+':
with assert_raises(ValueError):
open_array('doesnotexist', mode=mode)
with assert_raises(ValueError):
open_array('data/group.zarr', mode=mode)
z = open_array(store, mode='r')
assert_is_instance(z, Array)
assert_is_instance(z.store, DirectoryStore)
eq((100,), z.shape)
eq((10,), z.chunks)
assert_array_equal(np.full(100, fill_value=42), z[:])
with assert_raises(PermissionError):
z[:] = 43
z = open_array(store, mode='r+')
assert_is_instance(z, Array)
assert_is_instance(z.store, DirectoryStore)
eq((100,), z.shape)
eq((10,), z.chunks)
assert_array_equal(np.full(100, fill_value=42), z[:])
z[:] = 43
assert_array_equal(np.full(100, fill_value=43), z[:])
# mode == 'a'
shutil.rmtree(store)
z = open_array(store, mode='a', shape=100, chunks=10)
z[:] = 42
assert_is_instance(z, Array)
assert_is_instance(z.store, DirectoryStore)
eq((100,), z.shape)
eq((10,), z.chunks)
assert_array_equal(np.full(100, fill_value=42), z[:])
with assert_raises(ValueError):
open_array('data/group.zarr', mode='a')
# mode in 'w-', 'x'
for mode in 'w-', 'x':
shutil.rmtree(store)
z = open_array(store, mode=mode, shape=100, chunks=10)
z[:] = 42
assert_is_instance(z, Array)
assert_is_instance(z.store, DirectoryStore)
eq((100,), z.shape)
eq((10,), z.chunks)
assert_array_equal(np.full(100, fill_value=42), z[:])
with assert_raises(ValueError):
open_array(store, mode=mode)
with assert_raises(ValueError):
open_array('data/group.zarr', mode=mode)
# with synchronizer
z = open_array(store, synchronizer=ThreadSynchronizer())
assert_is_instance(z, Array)
# with path
z = open_array(store, shape=100, path='foo/bar', mode='w')
assert_is_instance(z, Array)
eq('foo/bar', z.path)
def test_open_array():
store = 'data/array.zarr'
# mode == 'w'
z = open_array(store, mode='w', shape=100, chunks=10)
z[:] = 42
assert isinstance(z, Array)
assert isinstance(z.store, DirectoryStore)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
# mode in 'r', 'r+'
open_group('data/group.zarr', mode='w')
for mode in 'r', 'r+':
with pytest.raises(ValueError):
open_array('doesnotexist', mode=mode)
with pytest.raises(ValueError):
open_array('data/group.zarr', mode=mode)
z = open_array(store, mode='r')
assert isinstance(z, Array)
assert isinstance(z.store, DirectoryStore)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
with pytest.raises(PermissionError):
z[:] = 43
z = open_array(store, mode='r+')
assert isinstance(z, Array)
assert isinstance(z.store, DirectoryStore)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
z[:] = 43
assert_array_equal(np.full(100, fill_value=43), z[:])
# mode == 'a'
shutil.rmtree(store)
z = open_array(store, mode='a', shape=100, chunks=10)
z[:] = 42
assert isinstance(z, Array)
assert isinstance(z.store, DirectoryStore)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
with pytest.raises(ValueError):
open_array('data/group.zarr', mode='a')
# mode in 'w-', 'x'
for mode in 'w-', 'x':
shutil.rmtree(store)
z = open_array(store, mode=mode, shape=100, chunks=10)
z[:] = 42
assert isinstance(z, Array)
assert isinstance(z.store, DirectoryStore)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
with pytest.raises(ValueError):
open_array(store, mode=mode)
with pytest.raises(ValueError):
open_array('data/group.zarr', mode=mode)
# with synchronizer
z = open_array(store, synchronizer=ThreadSynchronizer())
assert isinstance(z, Array)
# with path
z = open_array(store, shape=100, path='foo/bar', mode='w')
assert isinstance(z, Array)
assert 'foo/bar' == z.path
# with chunk store
meta_store = 'data/meta.zarr'
chunk_store = 'data/chunks.zarr'
z = open_array(store=meta_store,
chunk_store=chunk_store,
shape=11,
mode='w')
z[:] = 42
assert os.path.abspath(meta_store) == z.store.path
assert os.path.abspath(chunk_store) == z.chunk_store.path
# for N5 store
store = 'data/array.n5'
z = open_array(store, mode='w', shape=100, chunks=10)
z[:] = 42
assert isinstance(z, Array)
assert isinstance(z.store, N5Store)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
store = 'data/group.n5'
z = open_group(store, mode='w')
i = z.create_group('inner')
a = i.zeros("array", shape=100, chunks=10)
a[:] = 42
# Edit inner/attributes.json to not include "n5"
with open('data/group.n5/inner/attributes.json', 'w') as o:
o.write("{}")
# Re-open
a = open_group(store)["inner"]["array"]
assert isinstance(a, Array)
assert isinstance(z.store, N5Store)
assert (100, ) == a.shape
assert (10, ) == a.chunks
assert_array_equal(np.full(100, fill_value=42), a[:])
def open(store: StoreLike = None, mode: str = "a", **kwargs):
"""Convenience function to open a group or array using file-mode-like semantics.
Parameters
----------
store : Store or string, optional
Store or path to directory in file system or name of zip file.
mode : {'r', 'r+', 'a', 'w', 'w-'}, optional
Persistence mode: 'r' means read only (must exist); 'r+' means
read/write (must exist); 'a' means read/write (create if doesn't
exist); 'w' means create (overwrite if exists); 'w-' means create
(fail if exists).
**kwargs
Additional parameters are passed through to :func:`zarr.creation.open_array` or
:func:`zarr.hierarchy.open_group`.
Returns
-------
z : :class:`zarr.core.Array` or :class:`zarr.hierarchy.Group`
Array or group, depending on what exists in the given store.
See Also
--------
zarr.creation.open_array, zarr.hierarchy.open_group
Examples
--------
Storing data in a directory 'data/example.zarr' on the local file system::
>>> import zarr
>>> store = 'data/example.zarr'
>>> zw = zarr.open(store, mode='w', shape=100, dtype='i4') # open new array
>>> zw
<zarr.core.Array (100,) int32>
>>> za = zarr.open(store, mode='a') # open existing array for reading and writing
>>> za
<zarr.core.Array (100,) int32>
>>> zr = zarr.open(store, mode='r') # open existing array read-only
>>> zr
<zarr.core.Array (100,) int32 read-only>
>>> gw = zarr.open(store, mode='w') # open new group, overwriting previous data
>>> gw
<zarr.hierarchy.Group '/'>
>>> ga = zarr.open(store, mode='a') # open existing group for reading and writing
>>> ga
<zarr.hierarchy.Group '/'>
>>> gr = zarr.open(store, mode='r') # open existing group read-only
>>> gr
<zarr.hierarchy.Group '/' read-only>
"""
path = kwargs.get('path', None)
# handle polymorphic store arg
clobber = mode == 'w'
# we pass storage options explicitly, since normalize_store_arg might construct
# a store if the input is a fsspec-compatible URL
_store: BaseStore = normalize_store_arg(store,
clobber=clobber,
storage_options=kwargs.pop(
"storage_options", {}))
path = normalize_storage_path(path)
if mode in {'w', 'w-', 'x'}:
if 'shape' in kwargs:
return open_array(_store, mode=mode, **kwargs)
else:
return open_group(_store, mode=mode, **kwargs)
elif mode == "a":
if "shape" in kwargs or contains_array(_store, path):
return open_array(_store, mode=mode, **kwargs)
else:
return open_group(_store, mode=mode, **kwargs)
else:
if contains_array(_store, path):
return open_array(_store, mode=mode, **kwargs)
elif contains_group(_store, path):
return open_group(_store, mode=mode, **kwargs)
else:
raise PathNotFoundError(path)
def open(store, mode='a', **kwargs):
"""Convenience function to open a group or array using file-mode-like semantics.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
mode : {'r', 'r+', 'a', 'w', 'w-'}, optional
Persistence mode: 'r' means read only (must exist); 'r+' means
read/write (must exist); 'a' means read/write (create if doesn't
exist); 'w' means create (overwrite if exists); 'w-' means create
(fail if exists).
**kwargs
Additional parameters are passed through to :func:`zarr.open_array` or
:func:`zarr.open_group`.
See Also
--------
zarr.open_array, zarr.open_group
Examples
--------
Storing data in a directory 'data/example.zarr' on the local file system::
>>> import zarr
>>> store = 'data/example.zarr'
>>> zw = zarr.open(store, mode='w', shape=100, dtype='i4') # open new array
>>> zw
<zarr.core.Array (100,) int32>
>>> za = zarr.open(store, mode='a') # open existing array for reading and writing
>>> za
<zarr.core.Array (100,) int32>
>>> zr = zarr.open(store, mode='r') # open existing array read-only
>>> zr
<zarr.core.Array (100,) int32 read-only>
>>> gw = zarr.open(store, mode='w') # open new group, overwriting previous data
>>> gw
<zarr.hierarchy.Group '/'>
>>> ga = zarr.open(store, mode='a') # open existing group for reading and writing
>>> ga
<zarr.hierarchy.Group '/'>
>>> gr = zarr.open(store, mode='r') # open existing group read-only
>>> gr
<zarr.hierarchy.Group '/' read-only>
"""
path = kwargs.get('path', None)
# handle polymorphic store arg
store = normalize_store_arg(store, clobber=(mode == 'w'))
path = normalize_storage_path(path)
if mode in {'w', 'w-', 'x'}:
if 'shape' in kwargs:
return open_array(store, mode=mode, **kwargs)
else:
return open_group(store, mode=mode, **kwargs)
elif mode == 'a':
if contains_array(store, path):
return open_array(store, mode=mode, **kwargs)
elif contains_group(store, path):
return open_group(store, mode=mode, **kwargs)
elif 'shape' in kwargs:
return open_array(store, mode=mode, **kwargs)
else:
return open_group(store, mode=mode, **kwargs)
else:
if contains_array(store, path):
return open_array(store, mode=mode, **kwargs)
elif contains_group(store, path):
return open_group(store, mode=mode, **kwargs)
else:
err_path_not_found(path)
def test_open_array():
store = 'example'
# mode == 'w'
z = open_array(store, mode='w', shape=100, chunks=10)
z[:] = 42
assert_is_instance(z, Array)
assert_is_instance(z.store, DirectoryStore)
eq((100, ), z.shape)
eq((10, ), z.chunks)
assert_array_equal(np.full(100, fill_value=42), z[:])
# mode in 'r', 'r+'
open_group('example_group', mode='w')
for mode in 'r', 'r+':
with assert_raises(KeyError):
open_array('doesnotexist', mode=mode)
with assert_raises(KeyError):
open_array('example_group', mode=mode)
z = open_array(store, mode='r')
assert_is_instance(z, Array)
assert_is_instance(z.store, DirectoryStore)
eq((100, ), z.shape)
eq((10, ), z.chunks)
assert_array_equal(np.full(100, fill_value=42), z[:])
with assert_raises(PermissionError):
z[:] = 43
z = open_array(store, mode='r+')
assert_is_instance(z, Array)
assert_is_instance(z.store, DirectoryStore)
eq((100, ), z.shape)
eq((10, ), z.chunks)
assert_array_equal(np.full(100, fill_value=42), z[:])
z[:] = 43
assert_array_equal(np.full(100, fill_value=43), z[:])
# mode == 'a'
shutil.rmtree(store)
z = open_array(store, mode='a', shape=100, chunks=10)
z[:] = 42
assert_is_instance(z, Array)
assert_is_instance(z.store, DirectoryStore)
eq((100, ), z.shape)
eq((10, ), z.chunks)
assert_array_equal(np.full(100, fill_value=42), z[:])
with assert_raises(KeyError):
open_array('example_group', mode='a')
# mode in 'w-', 'x'
for mode in 'w-', 'x':
shutil.rmtree(store)
z = open_array(store, mode=mode, shape=100, chunks=10)
z[:] = 42
assert_is_instance(z, Array)
assert_is_instance(z.store, DirectoryStore)
eq((100, ), z.shape)
eq((10, ), z.chunks)
assert_array_equal(np.full(100, fill_value=42), z[:])
with assert_raises(KeyError):
open_array(store, mode=mode)
with assert_raises(KeyError):
open_array('example_group', mode=mode)
# with synchronizer
z = open_array(store, synchronizer=ThreadSynchronizer())
assert_is_instance(z, Array)
# with path
z = open_array(store, shape=100, path='foo/bar', mode='w')
assert_is_instance(z, Array)
eq('foo/bar', z.path)
def plate_to_zarr(plate: omero.gateway._PlateWrapper,
args: argparse.Namespace) -> None:
"""
Exports a plate to a zarr file using the hierarchy discussed here ('Option 3'):
https://github.com/ome/omero-ms-zarr/issues/73#issuecomment-706770955
"""
gs = plate.getGridSize()
n_rows = gs["rows"]
n_cols = gs["columns"]
n_fields = plate.getNumberOfFields()
total = n_rows * n_cols * (n_fields[1] - n_fields[0] + 1)
target_dir = args.output
cache_dir = target_dir if args.cache_numpy else None
name = os.path.join(target_dir, "%s.zarr" % plate.id)
store = open_store(name)
print(f"Exporting to {name} ({VERSION})")
root = open_group(store)
count = 0
max_fields = 0
t0 = time.time()
well_paths = set()
col_names = plate.getColumnLabels()
row_names = plate.getRowLabels()
plate_metadata = {
"name": plate.name,
"rows": [{
"name": str(name)
} for name in row_names],
"columns": [{
"name": str(name)
} for name in col_names],
"version": VERSION,
}
# Add acquisitions key if at least one plate acquisition exists
acquisitions = list(plate.listPlateAcquisitions())
if acquisitions:
plate_metadata["acquisitions"] = [
marshal_acquisition(x) for x in acquisitions
]
root.attrs["plate"] = plate_metadata
for well in plate.listChildren():
row = plate.getRowLabels()[well.row]
col = plate.getColumnLabels()[well.column]
fields = []
for field in range(n_fields[0], n_fields[1] + 1):
ws = well.getWellSample(field)
if ws and ws.getImage():
ac = ws.getPlateAcquisition()
field_name = "%d" % field
count += 1
img = ws.getImage()
well_paths.add(f"{row}/{col}")
field_info = {"path": f"{field_name}"}
if ac:
field_info["acquisition"] = ac.id
fields.append(field_info)
row_group = root.require_group(row)
col_group = row_group.require_group(col)
field_group = col_group.require_group(field_name)
n_levels, axes = add_image(img,
field_group,
cache_dir=cache_dir)
add_multiscales_metadata(field_group, axes, n_levels)
add_omero_metadata(field_group, img)
# Update Well metadata after each image
col_group.attrs["well"] = {
"images": fields,
"version": VERSION
}
max_fields = max(max_fields, field + 1)
print_status(int(t0), int(time.time()), count, total)
# Update plate_metadata after each Well
plate_metadata["wells"] = [{"path": x} for x in well_paths]
plate_metadata["field_count"] = max_fields
root.attrs["plate"] = plate_metadata
add_toplevel_metadata(root)
print("Finished.")
def open(store, mode='a', **kwargs):
"""Convenience function to open a group or array using file-mode-like semantics.
Parameters
----------
store : MutableMapping or string
Store or path to directory in file system or name of zip file.
mode : {'r', 'r+', 'a', 'w', 'w-'}, optional
Persistence mode: 'r' means read only (must exist); 'r+' means
read/write (must exist); 'a' means read/write (create if doesn't
exist); 'w' means create (overwrite if exists); 'w-' means create
(fail if exists).
**kwargs
Additional parameters are passed through to :func:`zarr.open_array` or
:func:`zarr.open_group`.
See Also
--------
zarr.open_array, zarr.open_group
Examples
--------
Storing data in a directory 'data/example.zarr' on the local file system::
>>> import zarr
>>> store = 'data/example.zarr'
>>> zw = zarr.open(store, mode='w', shape=100, dtype='i4') # open new array
>>> zw
<zarr.core.Array (100,) int32>
>>> za = zarr.open(store, mode='a') # open existing array for reading and writing
>>> za
<zarr.core.Array (100,) int32>
>>> zr = zarr.open(store, mode='r') # open existing array read-only
>>> zr
<zarr.core.Array (100,) int32 read-only>
>>> gw = zarr.open(store, mode='w') # open new group, overwriting previous data
>>> gw
<zarr.hierarchy.Group '/'>
>>> ga = zarr.open(store, mode='a') # open existing group for reading and writing
>>> ga
<zarr.hierarchy.Group '/'>
>>> gr = zarr.open(store, mode='r') # open existing group read-only
>>> gr
<zarr.hierarchy.Group '/' read-only>
"""
path = kwargs.get('path', None)
# handle polymorphic store arg
store = normalize_store_arg(store, clobber=(mode == 'w'))
path = normalize_storage_path(path)
if mode in {'w', 'w-', 'x'}:
if 'shape' in kwargs:
return open_array(store, mode=mode, **kwargs)
else:
return open_group(store, mode=mode, **kwargs)
elif mode == 'a':
if contains_array(store, path):
return open_array(store, mode=mode, **kwargs)
elif contains_group(store, path):
return open_group(store, mode=mode, **kwargs)
elif 'shape' in kwargs:
return open_array(store, mode=mode, **kwargs)
else:
return open_group(store, mode=mode, **kwargs)
else:
if contains_array(store, path):
return open_array(store, mode=mode, **kwargs)
elif contains_group(store, path):
return open_group(store, mode=mode, **kwargs)
else:
err_path_not_found(path)
def test_open_group():
# test the open_group() convenience function
store = 'data/group.zarr'
# mode == 'w'
g = open_group(store, mode='w')
assert isinstance(g, Group)
assert isinstance(g.store, DirectoryStore)
assert 0 == len(g)
g.create_groups('foo', 'bar')
assert 2 == len(g)
# mode in 'r', 'r+'
open_array('data/array.zarr', shape=100, chunks=10, mode='w')
for mode in 'r', 'r+':
with pytest.raises(ValueError):
open_group('doesnotexist', mode=mode)
with pytest.raises(ValueError):
open_group('data/array.zarr', mode=mode)
g = open_group(store, mode='r')
assert isinstance(g, Group)
assert 2 == len(g)
with pytest.raises(PermissionError):
g.create_group('baz')
g = open_group(store, mode='r+')
assert isinstance(g, Group)
assert 2 == len(g)
g.create_groups('baz', 'quux')
assert 4 == len(g)
# mode == 'a'
shutil.rmtree(store)
g = open_group(store, mode='a')
assert isinstance(g, Group)
assert isinstance(g.store, DirectoryStore)
assert 0 == len(g)
g.create_groups('foo', 'bar')
assert 2 == len(g)
with pytest.raises(ValueError):
open_group('data/array.zarr', mode='a')
# mode in 'w-', 'x'
for mode in 'w-', 'x':
shutil.rmtree(store)
g = open_group(store, mode=mode)
assert isinstance(g, Group)
assert isinstance(g.store, DirectoryStore)
assert 0 == len(g)
g.create_groups('foo', 'bar')
assert 2 == len(g)
with pytest.raises(ValueError):
open_group(store, mode=mode)
with pytest.raises(ValueError):
open_group('data/array.zarr', mode=mode)
# open with path
g = open_group(store, path='foo/bar')
assert isinstance(g, Group)
assert 'foo/bar' == g.path
def save(self, masks: List[omero.model.Shape], name: str) -> None:
"""
Save the masks/labels. In case of plate, make sure to set_image first.
:param masks: The masks
:param name: The name
:return: None
"""
# Figure out whether we can flatten some dimensions
unique_dims: Dict[str, Set[int]] = {
"T": {unwrap(mask.theT)
for shapes in masks for mask in shapes},
"C": {unwrap(mask.theC)
for shapes in masks for mask in shapes},
"Z": {unwrap(mask.theZ)
for shapes in masks for mask in shapes},
}
ignored_dimensions: Set[str] = set()
print(f"Unique dimensions: {unique_dims}")
for d in "TCZ":
if unique_dims[d] == {None}:
ignored_dimensions.add(d)
if self.plate:
filename = f"{self.plate.id}.zarr"
else:
filename = f"{self.image.id}.zarr"
# Verify that we are linking this mask to a real ome-zarr
source_image = self.source_image
source_image_link = self.source_image
if source_image is None:
# Assume that we're using the output directory
source_image = filename
source_image_link = "../.." # Drop "labels/0"
if self.plate:
assert self.plate_path, "Need image path within the plate"
source_image = f"{source_image}/{self.plate_path}"
current_path = f"{self.plate_path}/{self.path}"
else:
current_path = self.path
print(f"source_image {source_image}")
src = parse_url(source_image)
assert src, "Source image does not exist"
input_pyramid = Node(src, [])
assert input_pyramid.load(Multiscales), "No multiscales metadata found"
input_pyramid_levels = len(input_pyramid.data)
store = open_store(filename)
root = open_group(store)
if current_path in root.group_keys():
out_labels = getattr(root, current_path)
else:
out_labels = root.require_group(current_path)
_mask_shape: List[int] = list(self.image_shape)
for d in ignored_dimensions:
_mask_shape[DIMENSION_ORDER[d]] = 1
mask_shape: Tuple[int, ...] = tuple(_mask_shape)
del _mask_shape
print(f"Ignoring dimensions {ignored_dimensions}")
if self.style not in ("labeled", "split"):
assert False, "6d has been removed"
# Create and store binary data
labels, fill_colors, properties = self.masks_to_labels(
masks,
mask_shape,
ignored_dimensions,
check_overlaps=True,
)
# For v0.3 ngff we want to reduce the number of dimensions to
# match the dims of the Image.
dims_to_squeeze = []
axes = []
for dim, size in enumerate(self.image_shape):
if size == 1:
dims_to_squeeze.append(dim)
else:
axes.append("tczyx"[dim])
labels = np.squeeze(labels, axis=tuple(dims_to_squeeze))
scaler = Scaler(max_layer=input_pyramid_levels)
label_pyramid = scaler.nearest(labels)
pyramid_grp = out_labels.require_group(name)
write_multiscale(label_pyramid, pyramid_grp,
axes=axes) # TODO: dtype, chunks, overwite
# Specify and store metadata
image_label_colors: List[JSONDict] = []
label_properties: List[JSONDict] = []
image_label = {
"version": "0.3",
"colors": image_label_colors,
"properties": label_properties,
"source": {
"image": source_image_link
},
}
if properties:
for label_value, props_dict in sorted(properties.items()):
new_dict: Dict = {"label-value": label_value, **props_dict}
label_properties.append(new_dict)
if fill_colors:
for label_value, rgba_int in sorted(fill_colors.items()):
image_label_colors.append({
"label-value": label_value,
"rgba": int_to_rgba_255(rgba_int)
})
# TODO: move to write method
pyramid_grp.attrs["image-label"] = image_label
# Register with labels metadata
print(f"Created {filename}/{current_path}/{name}")
attrs = out_labels.attrs.asdict()
# TODO: could temporarily support "masks" here as well
if "labels" in attrs:
if name not in attrs["labels"]:
attrs["labels"].append(name)
else:
attrs["labels"] = [name]
out_labels.attrs.update(attrs)
def test_open_array(zarr_version, dimension_separator):
store = 'data/array.zarr'
kwargs = _init_creation_kwargs(zarr_version)
# mode == 'w'
z = open_array(store,
mode='w',
shape=100,
chunks=10,
dimension_separator=dimension_separator,
**kwargs)
z[:] = 42
assert isinstance(z, Array)
if z._store._store_version == 2:
assert isinstance(z.store, DirectoryStore)
else:
assert isinstance(z.store, DirectoryStoreV3)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
if dimension_separator is None:
assert z._dimension_separator == '/' if zarr_version == 3 else '.'
else:
assert z._dimension_separator == dimension_separator
# mode in 'r', 'r+'
group_kwargs = kwargs.copy()
if zarr_version == 3:
group_kwargs['path'] = 'group'
open_group('data/group.zarr', mode='w', **group_kwargs)
for mode in 'r', 'r+':
with pytest.raises(ValueError):
open_array('doesnotexist', mode=mode)
with pytest.raises(ValueError):
open_array('data/group.zarr', mode=mode)
z = open_array(store, mode='r', **kwargs)
assert isinstance(z, Array)
if z._store._store_version == 2:
assert isinstance(z.store, DirectoryStore)
else:
assert isinstance(z.store, DirectoryStoreV3)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
with pytest.raises(PermissionError):
z[:] = 43
z = open_array(store, mode='r+', **kwargs)
assert isinstance(z, Array)
if z._store._store_version == 2:
assert isinstance(z.store, DirectoryStore)
else:
assert isinstance(z.store, DirectoryStoreV3)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
z[:] = 43
assert_array_equal(np.full(100, fill_value=43), z[:])
# mode == 'a'
shutil.rmtree(store)
z = open_array(store, mode='a', shape=100, chunks=10, **kwargs)
z[:] = 42
assert isinstance(z, Array)
if z._store._store_version == 2:
assert isinstance(z.store, DirectoryStore)
else:
assert isinstance(z.store, DirectoryStoreV3)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
expected_error = TypeError if zarr_version == 3 else ValueError
# v3 path does not conflict, but will raise TypeError without shape kwarg
with pytest.raises(expected_error):
# array would end up at data/group.zarr/meta/root/array.array.json
open_array('data/group.zarr', mode='a', **kwargs)
# mode in 'w-', 'x'
for mode in 'w-', 'x':
shutil.rmtree(store)
z = open_array(store, mode=mode, shape=100, chunks=10, **kwargs)
z[:] = 42
assert isinstance(z, Array)
if z._store._store_version == 2:
assert isinstance(z.store, DirectoryStore)
else:
assert isinstance(z.store, DirectoryStoreV3)
assert (100, ) == z.shape
assert (10, ) == z.chunks
assert_array_equal(np.full(100, fill_value=42), z[:])
with pytest.raises(ValueError):
open_array(store, mode=mode, **kwargs)
expected_error = TypeError if zarr_version == 3 else ValueError
# v3 path does not conflict, but will raise TypeError without shape kwarg
with pytest.raises(expected_error):
open_array('data/group.zarr', mode=mode, **kwargs)
# with synchronizer
z = open_array(store, synchronizer=ThreadSynchronizer(), **kwargs)
assert isinstance(z, Array)
# with path
kwargs_no_path = kwargs.copy()
kwargs_no_path.pop('path', None)
z = open_array(store,
shape=100,
path='foo/bar',
mode='w',
**kwargs_no_path)
assert isinstance(z, Array)
assert 'foo/bar' == z.path
# with chunk store
meta_store = 'data/meta.zarr'
chunk_store = 'data/chunks.zarr'
z = open_array(store=meta_store,
chunk_store=chunk_store,
shape=11,
mode='w',
**kwargs)
z[:] = 42
assert os.path.abspath(meta_store) == z.store.path
assert os.path.abspath(chunk_store) == z.chunk_store.path