def test_open_array_n5(zarr_version): store = 'data/array.zarr' kwargs = _init_creation_kwargs(zarr_version) # for N5 store store = 'data/array.n5' z = open_array(store, mode='w', shape=100, chunks=10, **kwargs) z[:] = 42 assert isinstance(z, Array) assert isinstance(z.store, N5Store) assert (100, ) == z.shape assert (10, ) == z.chunks assert_array_equal(np.full(100, fill_value=42), z[:]) store = 'data/group.n5' group_kwargs = kwargs.copy() # if zarr_version == 3: # group_kwargs['path'] = 'group' z = open_group(store, mode='w', **group_kwargs) i = z.create_group('inner') a = i.zeros("array", shape=100, chunks=10) a[:] = 42 # Edit inner/attributes.json to not include "n5" with open('data/group.n5/inner/attributes.json', 'w') as o: o.write("{}") # Re-open a = open_group(store, **group_kwargs)["inner"]["array"] assert isinstance(a, Array) assert isinstance(z.store, N5Store) assert (100, ) == a.shape assert (10, ) == a.chunks assert_array_equal(np.full(100, fill_value=42), a[:])
def test_round_trip_nd(self): data = np.arange(1000).reshape(10, 10, 10) name = 'raw' store, _ = self.create_store() f = open_group(store, mode='w') f.create_dataset(name, data=data, chunks=(5, 5, 5), compressor=None) h = open_group(store, mode='r') np.testing.assert_array_equal(h[name][:], data)
def test_inconsistent_dimension_separator(self): data = np.arange(1000).reshape(10, 10, 10) name = 'raw' store, _ = self.create_store() f = open_group(store, mode='w') # cannot specify dimension_separator that conflicts with the store with pytest.raises(ValueError): f.create_dataset(name, data=data, chunks=(5, 5, 5), compressor=None, dimension_separator='.')
def image_to_zarr(image: omero.gateway.ImageWrapper, args: argparse.Namespace) -> None: target_dir = args.output cache_dir = target_dir if args.cache_numpy else None name = os.path.join(target_dir, "%s.zarr" % image.id) print(f"Exporting to {name} ({VERSION})") store = open_store(name) root = open_group(store) n_levels, axes = add_image(image, root, cache_dir=cache_dir) add_multiscales_metadata(root, axes, n_levels) add_omero_metadata(root, image) add_toplevel_metadata(root) print("Finished.")
def test_open_group(): # test the open_group() convenience function store = 'example' # mode == 'w' g = open_group(store, mode='w') assert_is_instance(g, Group) assert_is_instance(g.store, DirectoryStore) eq(0, len(g)) g.create_groups('foo', 'bar') eq(2, len(g)) # mode in 'r', 'r+' open_array('example_array', shape=100, chunks=10, mode='w') for mode in 'r', 'r+': with assert_raises(KeyError): open_group('doesnotexist', mode=mode) with assert_raises(KeyError): open_group('example_array', mode=mode) g = open_group(store, mode='r') assert_is_instance(g, Group) eq(2, len(g)) with assert_raises(PermissionError): g.create_group('baz') g = open_group(store, mode='r+') assert_is_instance(g, Group) eq(2, len(g)) g.create_groups('baz', 'quux') eq(4, len(g)) # mode == 'a' shutil.rmtree(store) g = open_group(store, mode='a') assert_is_instance(g, Group) assert_is_instance(g.store, DirectoryStore) eq(0, len(g)) g.create_groups('foo', 'bar') eq(2, len(g)) with assert_raises(KeyError): open_group('example_array', mode='a') # mode in 'w-', 'x' for mode in 'w-', 'x': shutil.rmtree(store) g = open_group(store, mode=mode) assert_is_instance(g, Group) assert_is_instance(g.store, DirectoryStore) eq(0, len(g)) g.create_groups('foo', 'bar') eq(2, len(g)) with assert_raises(KeyError): open_group(store, mode=mode) with assert_raises(KeyError): open_group('example_array', mode=mode) # open with path g = open_group(store, path='foo/bar') assert_is_instance(g, Group) eq('foo/bar', g.path)
def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace) -> None: """ Exports a plate to a zarr file using the hierarchy discussed here ('Option 3'): https://github.com/ome/omero-ms-zarr/issues/73#issuecomment-706770955 """ gs = plate.getGridSize() n_rows = gs["rows"] n_cols = gs["columns"] n_fields = plate.getNumberOfFields() total = n_rows * n_cols * (n_fields[1] - n_fields[0] + 1) target_dir = args.output cache_dir = target_dir if args.cache_numpy else None name = os.path.join(target_dir, "%s.zarr" % plate.id) print(f"Exporting to {name}") root = open_group(name, mode="w") count = 0 max_fields = 0 t0 = time.time() row_names = set() col_names = set() well_paths = set() col_names = plate.getColumnLabels() row_names = plate.getRowLabels() plate_acqs = list(plate.listPlateAcquisitions()) ac_names = [pa.name for pa in plate_acqs] if plate_acqs else ["0"] plate_metadata = { "name": plate.name, "rows": [{ "name": str(name) } for name in row_names], "columns": [{ "name": str(name) } for name in col_names], "acquisitions": [{ "path": x } for x in ac_names], } root.attrs["plate"] = plate_metadata for well in plate.listChildren(): row = plate.getRowLabels()[well.row] col = plate.getColumnLabels()[well.column] field_paths = [] for field in range(n_fields[0], n_fields[1] + 1): ws = well.getWellSample(field) field_name = "%d" % field count += 1 if ws and ws.getImage(): img = ws.getImage() ac = ws.getPlateAcquisition() ac_name = ac.getName() if ac else "0" well_paths.add(f"{ac_name}/{row}/{col}/") field_paths.append(f"{field_name}/") ac_group = root.require_group(ac_name) row_group = ac_group.require_group(row) col_group = row_group.require_group(col) field_group = col_group.require_group(field_name) n_levels = add_image(img, field_group, cache_dir=cache_dir) add_group_metadata(field_group, img, n_levels) # Update Well metadata after each image col_group.attrs["well"] = { "images": [{ "path": x } for x in field_paths] } max_fields = max(max_fields, field + 1) print_status(int(t0), int(time.time()), count, total) # Update plate_metadata after each Well plate_metadata["wells"] = [{"path": x} for x in well_paths] plate_metadata["field_count"] = max_fields root.attrs["plate"] = plate_metadata add_toplevel_metadata(root) print("Finished.")
def test_open_array(): store = 'data/array.zarr' # mode == 'w' z = open_array(store, mode='w', shape=100, chunks=10) z[:] = 42 assert_is_instance(z, Array) assert_is_instance(z.store, DirectoryStore) eq((100,), z.shape) eq((10,), z.chunks) assert_array_equal(np.full(100, fill_value=42), z[:]) # mode in 'r', 'r+' open_group('data/group.zarr', mode='w') for mode in 'r', 'r+': with assert_raises(ValueError): open_array('doesnotexist', mode=mode) with assert_raises(ValueError): open_array('data/group.zarr', mode=mode) z = open_array(store, mode='r') assert_is_instance(z, Array) assert_is_instance(z.store, DirectoryStore) eq((100,), z.shape) eq((10,), z.chunks) assert_array_equal(np.full(100, fill_value=42), z[:]) with assert_raises(PermissionError): z[:] = 43 z = open_array(store, mode='r+') assert_is_instance(z, Array) assert_is_instance(z.store, DirectoryStore) eq((100,), z.shape) eq((10,), z.chunks) assert_array_equal(np.full(100, fill_value=42), z[:]) z[:] = 43 assert_array_equal(np.full(100, fill_value=43), z[:]) # mode == 'a' shutil.rmtree(store) z = open_array(store, mode='a', shape=100, chunks=10) z[:] = 42 assert_is_instance(z, Array) assert_is_instance(z.store, DirectoryStore) eq((100,), z.shape) eq((10,), z.chunks) assert_array_equal(np.full(100, fill_value=42), z[:]) with assert_raises(ValueError): open_array('data/group.zarr', mode='a') # mode in 'w-', 'x' for mode in 'w-', 'x': shutil.rmtree(store) z = open_array(store, mode=mode, shape=100, chunks=10) z[:] = 42 assert_is_instance(z, Array) assert_is_instance(z.store, DirectoryStore) eq((100,), z.shape) eq((10,), z.chunks) assert_array_equal(np.full(100, fill_value=42), z[:]) with assert_raises(ValueError): open_array(store, mode=mode) with assert_raises(ValueError): open_array('data/group.zarr', mode=mode) # with synchronizer z = open_array(store, synchronizer=ThreadSynchronizer()) assert_is_instance(z, Array) # with path z = open_array(store, shape=100, path='foo/bar', mode='w') assert_is_instance(z, Array) eq('foo/bar', z.path)
def test_open_array(): store = 'data/array.zarr' # mode == 'w' z = open_array(store, mode='w', shape=100, chunks=10) z[:] = 42 assert isinstance(z, Array) assert isinstance(z.store, DirectoryStore) assert (100, ) == z.shape assert (10, ) == z.chunks assert_array_equal(np.full(100, fill_value=42), z[:]) # mode in 'r', 'r+' open_group('data/group.zarr', mode='w') for mode in 'r', 'r+': with pytest.raises(ValueError): open_array('doesnotexist', mode=mode) with pytest.raises(ValueError): open_array('data/group.zarr', mode=mode) z = open_array(store, mode='r') assert isinstance(z, Array) assert isinstance(z.store, DirectoryStore) assert (100, ) == z.shape assert (10, ) == z.chunks assert_array_equal(np.full(100, fill_value=42), z[:]) with pytest.raises(PermissionError): z[:] = 43 z = open_array(store, mode='r+') assert isinstance(z, Array) assert isinstance(z.store, DirectoryStore) assert (100, ) == z.shape assert (10, ) == z.chunks assert_array_equal(np.full(100, fill_value=42), z[:]) z[:] = 43 assert_array_equal(np.full(100, fill_value=43), z[:]) # mode == 'a' shutil.rmtree(store) z = open_array(store, mode='a', shape=100, chunks=10) z[:] = 42 assert isinstance(z, Array) assert isinstance(z.store, DirectoryStore) assert (100, ) == z.shape assert (10, ) == z.chunks assert_array_equal(np.full(100, fill_value=42), z[:]) with pytest.raises(ValueError): open_array('data/group.zarr', mode='a') # mode in 'w-', 'x' for mode in 'w-', 'x': shutil.rmtree(store) z = open_array(store, mode=mode, shape=100, chunks=10) z[:] = 42 assert isinstance(z, Array) assert isinstance(z.store, DirectoryStore) assert (100, ) == z.shape assert (10, ) == z.chunks assert_array_equal(np.full(100, fill_value=42), z[:]) with pytest.raises(ValueError): open_array(store, mode=mode) with pytest.raises(ValueError): open_array('data/group.zarr', mode=mode) # with synchronizer z = open_array(store, synchronizer=ThreadSynchronizer()) assert isinstance(z, Array) # with path z = open_array(store, shape=100, path='foo/bar', mode='w') assert isinstance(z, Array) assert 'foo/bar' == z.path # with chunk store meta_store = 'data/meta.zarr' chunk_store = 'data/chunks.zarr' z = open_array(store=meta_store, chunk_store=chunk_store, shape=11, mode='w') z[:] = 42 assert os.path.abspath(meta_store) == z.store.path assert os.path.abspath(chunk_store) == z.chunk_store.path # for N5 store store = 'data/array.n5' z = open_array(store, mode='w', shape=100, chunks=10) z[:] = 42 assert isinstance(z, Array) assert isinstance(z.store, N5Store) assert (100, ) == z.shape assert (10, ) == z.chunks assert_array_equal(np.full(100, fill_value=42), z[:]) store = 'data/group.n5' z = open_group(store, mode='w') i = z.create_group('inner') a = i.zeros("array", shape=100, chunks=10) a[:] = 42 # Edit inner/attributes.json to not include "n5" with open('data/group.n5/inner/attributes.json', 'w') as o: o.write("{}") # Re-open a = open_group(store)["inner"]["array"] assert isinstance(a, Array) assert isinstance(z.store, N5Store) assert (100, ) == a.shape assert (10, ) == a.chunks assert_array_equal(np.full(100, fill_value=42), a[:])
def open(store: StoreLike = None, mode: str = "a", **kwargs): """Convenience function to open a group or array using file-mode-like semantics. Parameters ---------- store : Store or string, optional Store or path to directory in file system or name of zip file. mode : {'r', 'r+', 'a', 'w', 'w-'}, optional Persistence mode: 'r' means read only (must exist); 'r+' means read/write (must exist); 'a' means read/write (create if doesn't exist); 'w' means create (overwrite if exists); 'w-' means create (fail if exists). **kwargs Additional parameters are passed through to :func:`zarr.creation.open_array` or :func:`zarr.hierarchy.open_group`. Returns ------- z : :class:`zarr.core.Array` or :class:`zarr.hierarchy.Group` Array or group, depending on what exists in the given store. See Also -------- zarr.creation.open_array, zarr.hierarchy.open_group Examples -------- Storing data in a directory 'data/example.zarr' on the local file system:: >>> import zarr >>> store = 'data/example.zarr' >>> zw = zarr.open(store, mode='w', shape=100, dtype='i4') # open new array >>> zw <zarr.core.Array (100,) int32> >>> za = zarr.open(store, mode='a') # open existing array for reading and writing >>> za <zarr.core.Array (100,) int32> >>> zr = zarr.open(store, mode='r') # open existing array read-only >>> zr <zarr.core.Array (100,) int32 read-only> >>> gw = zarr.open(store, mode='w') # open new group, overwriting previous data >>> gw <zarr.hierarchy.Group '/'> >>> ga = zarr.open(store, mode='a') # open existing group for reading and writing >>> ga <zarr.hierarchy.Group '/'> >>> gr = zarr.open(store, mode='r') # open existing group read-only >>> gr <zarr.hierarchy.Group '/' read-only> """ path = kwargs.get('path', None) # handle polymorphic store arg clobber = mode == 'w' # we pass storage options explicitly, since normalize_store_arg might construct # a store if the input is a fsspec-compatible URL _store: BaseStore = normalize_store_arg(store, clobber=clobber, storage_options=kwargs.pop( "storage_options", {})) path = normalize_storage_path(path) if mode in {'w', 'w-', 'x'}: if 'shape' in kwargs: return open_array(_store, mode=mode, **kwargs) else: return open_group(_store, mode=mode, **kwargs) elif mode == "a": if "shape" in kwargs or contains_array(_store, path): return open_array(_store, mode=mode, **kwargs) else: return open_group(_store, mode=mode, **kwargs) else: if contains_array(_store, path): return open_array(_store, mode=mode, **kwargs) elif contains_group(_store, path): return open_group(_store, mode=mode, **kwargs) else: raise PathNotFoundError(path)
def open(store, mode='a', **kwargs): """Convenience function to open a group or array using file-mode-like semantics. Parameters ---------- store : MutableMapping or string Store or path to directory in file system or name of zip file. mode : {'r', 'r+', 'a', 'w', 'w-'}, optional Persistence mode: 'r' means read only (must exist); 'r+' means read/write (must exist); 'a' means read/write (create if doesn't exist); 'w' means create (overwrite if exists); 'w-' means create (fail if exists). **kwargs Additional parameters are passed through to :func:`zarr.open_array` or :func:`zarr.open_group`. See Also -------- zarr.open_array, zarr.open_group Examples -------- Storing data in a directory 'data/example.zarr' on the local file system:: >>> import zarr >>> store = 'data/example.zarr' >>> zw = zarr.open(store, mode='w', shape=100, dtype='i4') # open new array >>> zw <zarr.core.Array (100,) int32> >>> za = zarr.open(store, mode='a') # open existing array for reading and writing >>> za <zarr.core.Array (100,) int32> >>> zr = zarr.open(store, mode='r') # open existing array read-only >>> zr <zarr.core.Array (100,) int32 read-only> >>> gw = zarr.open(store, mode='w') # open new group, overwriting previous data >>> gw <zarr.hierarchy.Group '/'> >>> ga = zarr.open(store, mode='a') # open existing group for reading and writing >>> ga <zarr.hierarchy.Group '/'> >>> gr = zarr.open(store, mode='r') # open existing group read-only >>> gr <zarr.hierarchy.Group '/' read-only> """ path = kwargs.get('path', None) # handle polymorphic store arg store = normalize_store_arg(store, clobber=(mode == 'w')) path = normalize_storage_path(path) if mode in {'w', 'w-', 'x'}: if 'shape' in kwargs: return open_array(store, mode=mode, **kwargs) else: return open_group(store, mode=mode, **kwargs) elif mode == 'a': if contains_array(store, path): return open_array(store, mode=mode, **kwargs) elif contains_group(store, path): return open_group(store, mode=mode, **kwargs) elif 'shape' in kwargs: return open_array(store, mode=mode, **kwargs) else: return open_group(store, mode=mode, **kwargs) else: if contains_array(store, path): return open_array(store, mode=mode, **kwargs) elif contains_group(store, path): return open_group(store, mode=mode, **kwargs) else: err_path_not_found(path)
def test_open_array(): store = 'example' # mode == 'w' z = open_array(store, mode='w', shape=100, chunks=10) z[:] = 42 assert_is_instance(z, Array) assert_is_instance(z.store, DirectoryStore) eq((100, ), z.shape) eq((10, ), z.chunks) assert_array_equal(np.full(100, fill_value=42), z[:]) # mode in 'r', 'r+' open_group('example_group', mode='w') for mode in 'r', 'r+': with assert_raises(KeyError): open_array('doesnotexist', mode=mode) with assert_raises(KeyError): open_array('example_group', mode=mode) z = open_array(store, mode='r') assert_is_instance(z, Array) assert_is_instance(z.store, DirectoryStore) eq((100, ), z.shape) eq((10, ), z.chunks) assert_array_equal(np.full(100, fill_value=42), z[:]) with assert_raises(PermissionError): z[:] = 43 z = open_array(store, mode='r+') assert_is_instance(z, Array) assert_is_instance(z.store, DirectoryStore) eq((100, ), z.shape) eq((10, ), z.chunks) assert_array_equal(np.full(100, fill_value=42), z[:]) z[:] = 43 assert_array_equal(np.full(100, fill_value=43), z[:]) # mode == 'a' shutil.rmtree(store) z = open_array(store, mode='a', shape=100, chunks=10) z[:] = 42 assert_is_instance(z, Array) assert_is_instance(z.store, DirectoryStore) eq((100, ), z.shape) eq((10, ), z.chunks) assert_array_equal(np.full(100, fill_value=42), z[:]) with assert_raises(KeyError): open_array('example_group', mode='a') # mode in 'w-', 'x' for mode in 'w-', 'x': shutil.rmtree(store) z = open_array(store, mode=mode, shape=100, chunks=10) z[:] = 42 assert_is_instance(z, Array) assert_is_instance(z.store, DirectoryStore) eq((100, ), z.shape) eq((10, ), z.chunks) assert_array_equal(np.full(100, fill_value=42), z[:]) with assert_raises(KeyError): open_array(store, mode=mode) with assert_raises(KeyError): open_array('example_group', mode=mode) # with synchronizer z = open_array(store, synchronizer=ThreadSynchronizer()) assert_is_instance(z, Array) # with path z = open_array(store, shape=100, path='foo/bar', mode='w') assert_is_instance(z, Array) eq('foo/bar', z.path)
def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace) -> None: """ Exports a plate to a zarr file using the hierarchy discussed here ('Option 3'): https://github.com/ome/omero-ms-zarr/issues/73#issuecomment-706770955 """ gs = plate.getGridSize() n_rows = gs["rows"] n_cols = gs["columns"] n_fields = plate.getNumberOfFields() total = n_rows * n_cols * (n_fields[1] - n_fields[0] + 1) target_dir = args.output cache_dir = target_dir if args.cache_numpy else None name = os.path.join(target_dir, "%s.zarr" % plate.id) store = open_store(name) print(f"Exporting to {name} ({VERSION})") root = open_group(store) count = 0 max_fields = 0 t0 = time.time() well_paths = set() col_names = plate.getColumnLabels() row_names = plate.getRowLabels() plate_metadata = { "name": plate.name, "rows": [{ "name": str(name) } for name in row_names], "columns": [{ "name": str(name) } for name in col_names], "version": VERSION, } # Add acquisitions key if at least one plate acquisition exists acquisitions = list(plate.listPlateAcquisitions()) if acquisitions: plate_metadata["acquisitions"] = [ marshal_acquisition(x) for x in acquisitions ] root.attrs["plate"] = plate_metadata for well in plate.listChildren(): row = plate.getRowLabels()[well.row] col = plate.getColumnLabels()[well.column] fields = [] for field in range(n_fields[0], n_fields[1] + 1): ws = well.getWellSample(field) if ws and ws.getImage(): ac = ws.getPlateAcquisition() field_name = "%d" % field count += 1 img = ws.getImage() well_paths.add(f"{row}/{col}") field_info = {"path": f"{field_name}"} if ac: field_info["acquisition"] = ac.id fields.append(field_info) row_group = root.require_group(row) col_group = row_group.require_group(col) field_group = col_group.require_group(field_name) n_levels, axes = add_image(img, field_group, cache_dir=cache_dir) add_multiscales_metadata(field_group, axes, n_levels) add_omero_metadata(field_group, img) # Update Well metadata after each image col_group.attrs["well"] = { "images": fields, "version": VERSION } max_fields = max(max_fields, field + 1) print_status(int(t0), int(time.time()), count, total) # Update plate_metadata after each Well plate_metadata["wells"] = [{"path": x} for x in well_paths] plate_metadata["field_count"] = max_fields root.attrs["plate"] = plate_metadata add_toplevel_metadata(root) print("Finished.")
def open(store, mode='a', **kwargs): """Convenience function to open a group or array using file-mode-like semantics. Parameters ---------- store : MutableMapping or string Store or path to directory in file system or name of zip file. mode : {'r', 'r+', 'a', 'w', 'w-'}, optional Persistence mode: 'r' means read only (must exist); 'r+' means read/write (must exist); 'a' means read/write (create if doesn't exist); 'w' means create (overwrite if exists); 'w-' means create (fail if exists). **kwargs Additional parameters are passed through to :func:`zarr.open_array` or :func:`zarr.open_group`. See Also -------- zarr.open_array, zarr.open_group Examples -------- Storing data in a directory 'data/example.zarr' on the local file system:: >>> import zarr >>> store = 'data/example.zarr' >>> zw = zarr.open(store, mode='w', shape=100, dtype='i4') # open new array >>> zw <zarr.core.Array (100,) int32> >>> za = zarr.open(store, mode='a') # open existing array for reading and writing >>> za <zarr.core.Array (100,) int32> >>> zr = zarr.open(store, mode='r') # open existing array read-only >>> zr <zarr.core.Array (100,) int32 read-only> >>> gw = zarr.open(store, mode='w') # open new group, overwriting previous data >>> gw <zarr.hierarchy.Group '/'> >>> ga = zarr.open(store, mode='a') # open existing group for reading and writing >>> ga <zarr.hierarchy.Group '/'> >>> gr = zarr.open(store, mode='r') # open existing group read-only >>> gr <zarr.hierarchy.Group '/' read-only> """ path = kwargs.get('path', None) # handle polymorphic store arg store = normalize_store_arg(store, clobber=(mode == 'w')) path = normalize_storage_path(path) if mode in {'w', 'w-', 'x'}: if 'shape' in kwargs: return open_array(store, mode=mode, **kwargs) else: return open_group(store, mode=mode, **kwargs) elif mode == 'a': if contains_array(store, path): return open_array(store, mode=mode, **kwargs) elif contains_group(store, path): return open_group(store, mode=mode, **kwargs) elif 'shape' in kwargs: return open_array(store, mode=mode, **kwargs) else: return open_group(store, mode=mode, **kwargs) else: if contains_array(store, path): return open_array(store, mode=mode, **kwargs) elif contains_group(store, path): return open_group(store, mode=mode, **kwargs) else: err_path_not_found(path)
def test_open_group(): # test the open_group() convenience function store = 'data/group.zarr' # mode == 'w' g = open_group(store, mode='w') assert isinstance(g, Group) assert isinstance(g.store, DirectoryStore) assert 0 == len(g) g.create_groups('foo', 'bar') assert 2 == len(g) # mode in 'r', 'r+' open_array('data/array.zarr', shape=100, chunks=10, mode='w') for mode in 'r', 'r+': with pytest.raises(ValueError): open_group('doesnotexist', mode=mode) with pytest.raises(ValueError): open_group('data/array.zarr', mode=mode) g = open_group(store, mode='r') assert isinstance(g, Group) assert 2 == len(g) with pytest.raises(PermissionError): g.create_group('baz') g = open_group(store, mode='r+') assert isinstance(g, Group) assert 2 == len(g) g.create_groups('baz', 'quux') assert 4 == len(g) # mode == 'a' shutil.rmtree(store) g = open_group(store, mode='a') assert isinstance(g, Group) assert isinstance(g.store, DirectoryStore) assert 0 == len(g) g.create_groups('foo', 'bar') assert 2 == len(g) with pytest.raises(ValueError): open_group('data/array.zarr', mode='a') # mode in 'w-', 'x' for mode in 'w-', 'x': shutil.rmtree(store) g = open_group(store, mode=mode) assert isinstance(g, Group) assert isinstance(g.store, DirectoryStore) assert 0 == len(g) g.create_groups('foo', 'bar') assert 2 == len(g) with pytest.raises(ValueError): open_group(store, mode=mode) with pytest.raises(ValueError): open_group('data/array.zarr', mode=mode) # open with path g = open_group(store, path='foo/bar') assert isinstance(g, Group) assert 'foo/bar' == g.path
def save(self, masks: List[omero.model.Shape], name: str) -> None: """ Save the masks/labels. In case of plate, make sure to set_image first. :param masks: The masks :param name: The name :return: None """ # Figure out whether we can flatten some dimensions unique_dims: Dict[str, Set[int]] = { "T": {unwrap(mask.theT) for shapes in masks for mask in shapes}, "C": {unwrap(mask.theC) for shapes in masks for mask in shapes}, "Z": {unwrap(mask.theZ) for shapes in masks for mask in shapes}, } ignored_dimensions: Set[str] = set() print(f"Unique dimensions: {unique_dims}") for d in "TCZ": if unique_dims[d] == {None}: ignored_dimensions.add(d) if self.plate: filename = f"{self.plate.id}.zarr" else: filename = f"{self.image.id}.zarr" # Verify that we are linking this mask to a real ome-zarr source_image = self.source_image source_image_link = self.source_image if source_image is None: # Assume that we're using the output directory source_image = filename source_image_link = "../.." # Drop "labels/0" if self.plate: assert self.plate_path, "Need image path within the plate" source_image = f"{source_image}/{self.plate_path}" current_path = f"{self.plate_path}/{self.path}" else: current_path = self.path print(f"source_image {source_image}") src = parse_url(source_image) assert src, "Source image does not exist" input_pyramid = Node(src, []) assert input_pyramid.load(Multiscales), "No multiscales metadata found" input_pyramid_levels = len(input_pyramid.data) store = open_store(filename) root = open_group(store) if current_path in root.group_keys(): out_labels = getattr(root, current_path) else: out_labels = root.require_group(current_path) _mask_shape: List[int] = list(self.image_shape) for d in ignored_dimensions: _mask_shape[DIMENSION_ORDER[d]] = 1 mask_shape: Tuple[int, ...] = tuple(_mask_shape) del _mask_shape print(f"Ignoring dimensions {ignored_dimensions}") if self.style not in ("labeled", "split"): assert False, "6d has been removed" # Create and store binary data labels, fill_colors, properties = self.masks_to_labels( masks, mask_shape, ignored_dimensions, check_overlaps=True, ) # For v0.3 ngff we want to reduce the number of dimensions to # match the dims of the Image. dims_to_squeeze = [] axes = [] for dim, size in enumerate(self.image_shape): if size == 1: dims_to_squeeze.append(dim) else: axes.append("tczyx"[dim]) labels = np.squeeze(labels, axis=tuple(dims_to_squeeze)) scaler = Scaler(max_layer=input_pyramid_levels) label_pyramid = scaler.nearest(labels) pyramid_grp = out_labels.require_group(name) write_multiscale(label_pyramid, pyramid_grp, axes=axes) # TODO: dtype, chunks, overwite # Specify and store metadata image_label_colors: List[JSONDict] = [] label_properties: List[JSONDict] = [] image_label = { "version": "0.3", "colors": image_label_colors, "properties": label_properties, "source": { "image": source_image_link }, } if properties: for label_value, props_dict in sorted(properties.items()): new_dict: Dict = {"label-value": label_value, **props_dict} label_properties.append(new_dict) if fill_colors: for label_value, rgba_int in sorted(fill_colors.items()): image_label_colors.append({ "label-value": label_value, "rgba": int_to_rgba_255(rgba_int) }) # TODO: move to write method pyramid_grp.attrs["image-label"] = image_label # Register with labels metadata print(f"Created {filename}/{current_path}/{name}") attrs = out_labels.attrs.asdict() # TODO: could temporarily support "masks" here as well if "labels" in attrs: if name not in attrs["labels"]: attrs["labels"].append(name) else: attrs["labels"] = [name] out_labels.attrs.update(attrs)
def test_open_array(zarr_version, dimension_separator): store = 'data/array.zarr' kwargs = _init_creation_kwargs(zarr_version) # mode == 'w' z = open_array(store, mode='w', shape=100, chunks=10, dimension_separator=dimension_separator, **kwargs) z[:] = 42 assert isinstance(z, Array) if z._store._store_version == 2: assert isinstance(z.store, DirectoryStore) else: assert isinstance(z.store, DirectoryStoreV3) assert (100, ) == z.shape assert (10, ) == z.chunks assert_array_equal(np.full(100, fill_value=42), z[:]) if dimension_separator is None: assert z._dimension_separator == '/' if zarr_version == 3 else '.' else: assert z._dimension_separator == dimension_separator # mode in 'r', 'r+' group_kwargs = kwargs.copy() if zarr_version == 3: group_kwargs['path'] = 'group' open_group('data/group.zarr', mode='w', **group_kwargs) for mode in 'r', 'r+': with pytest.raises(ValueError): open_array('doesnotexist', mode=mode) with pytest.raises(ValueError): open_array('data/group.zarr', mode=mode) z = open_array(store, mode='r', **kwargs) assert isinstance(z, Array) if z._store._store_version == 2: assert isinstance(z.store, DirectoryStore) else: assert isinstance(z.store, DirectoryStoreV3) assert (100, ) == z.shape assert (10, ) == z.chunks assert_array_equal(np.full(100, fill_value=42), z[:]) with pytest.raises(PermissionError): z[:] = 43 z = open_array(store, mode='r+', **kwargs) assert isinstance(z, Array) if z._store._store_version == 2: assert isinstance(z.store, DirectoryStore) else: assert isinstance(z.store, DirectoryStoreV3) assert (100, ) == z.shape assert (10, ) == z.chunks assert_array_equal(np.full(100, fill_value=42), z[:]) z[:] = 43 assert_array_equal(np.full(100, fill_value=43), z[:]) # mode == 'a' shutil.rmtree(store) z = open_array(store, mode='a', shape=100, chunks=10, **kwargs) z[:] = 42 assert isinstance(z, Array) if z._store._store_version == 2: assert isinstance(z.store, DirectoryStore) else: assert isinstance(z.store, DirectoryStoreV3) assert (100, ) == z.shape assert (10, ) == z.chunks assert_array_equal(np.full(100, fill_value=42), z[:]) expected_error = TypeError if zarr_version == 3 else ValueError # v3 path does not conflict, but will raise TypeError without shape kwarg with pytest.raises(expected_error): # array would end up at data/group.zarr/meta/root/array.array.json open_array('data/group.zarr', mode='a', **kwargs) # mode in 'w-', 'x' for mode in 'w-', 'x': shutil.rmtree(store) z = open_array(store, mode=mode, shape=100, chunks=10, **kwargs) z[:] = 42 assert isinstance(z, Array) if z._store._store_version == 2: assert isinstance(z.store, DirectoryStore) else: assert isinstance(z.store, DirectoryStoreV3) assert (100, ) == z.shape assert (10, ) == z.chunks assert_array_equal(np.full(100, fill_value=42), z[:]) with pytest.raises(ValueError): open_array(store, mode=mode, **kwargs) expected_error = TypeError if zarr_version == 3 else ValueError # v3 path does not conflict, but will raise TypeError without shape kwarg with pytest.raises(expected_error): open_array('data/group.zarr', mode=mode, **kwargs) # with synchronizer z = open_array(store, synchronizer=ThreadSynchronizer(), **kwargs) assert isinstance(z, Array) # with path kwargs_no_path = kwargs.copy() kwargs_no_path.pop('path', None) z = open_array(store, shape=100, path='foo/bar', mode='w', **kwargs_no_path) assert isinstance(z, Array) assert 'foo/bar' == z.path # with chunk store meta_store = 'data/meta.zarr' chunk_store = 'data/chunks.zarr' z = open_array(store=meta_store, chunk_store=chunk_store, shape=11, mode='w', **kwargs) z[:] = 42 assert os.path.abspath(meta_store) == z.store.path assert os.path.abspath(chunk_store) == z.chunk_store.path