def ds_dtype(request, tmp_path): dtype = request.param n5_path = str(tmp_path / "test.n5") ds_name = "ds" + dtype pyn5.create_dataset(n5_path, ds_name, DS_SIZE, BLOCKSIZE, dtype) yield pyn5.open(n5_path, ds_name, dtype, False), np.dtype(dtype.lower())
def setUp(self): self.root = "test.n5" self.dataset = "test" self.dtype = "UINT8" self.dataset_size = [10, 10, 10] self.block_size = [2, 2, 2] pyn5.create_dataset(self.root, self.dataset, self.dataset_size, self.block_size, self.dtype) self.n5 = pyn5.open(self.root, self.dataset, self.dtype, False)
def setUp(self): self.root = "test.n5" self.dataset = "test_{}".format(self.dtype) self.dataset_size = [10, 10, 10] self.block_size = [2, 2, 2] if Path(self.root).is_dir(): shutil.rmtree(str(Path(self.root).absolute())) pyn5.create_dataset(self.root, self.dataset, self.dataset_size, self.block_size, self.dtype) self.n5 = pyn5.open(self.root, self.dataset, self.dtype, False)
def write_to_n5(self, folder, dataset): """ Note: because diluvian works with z,y,x and pyn5 assumes x,y,z: when writing coordinates must be flipped """ pyn5.create_dataset( folder, dataset, self.bounds[1], self.leaf_shape, str(self.dtype).upper() ) dataset = pyn5.open(folder, dataset) for leaf in self.iter_leaves(): pyn5.write(dataset, leaf.bounds, leaf.data, self.dtype)
def test_compression(tmp_path, compression_dict): root = tmp_path / "test.n5" data = np.arange(100, dtype=np.uint8).reshape((10, 10)) pyn5.create_dataset(str(root), "ds", data.shape, (5, 5), data.dtype.name.upper(), json.dumps(compression_dict)) with open(root / "ds" / "attributes.json") as f: attrs = json.load(f) assert attrs["compression"] == compression_dict ds = pyn5.DatasetUINT8(str(root), "ds", True) ds.write_ndarray((0, 0), data, 0)
def test_vs_z5_hash(tmp_path, z5_file): """Check different block hashes to z5""" root = tmp_path / "test.n5" z5_path = Path(z5_file.path) shape = (10, 20) data = np.arange(np.product(shape)).reshape(shape) chunks = (6, 7) pyn5.create_dataset(str(root), "ds", shape, chunks, data.dtype.name.upper(), json.dumps({"type": "raw"})) ds = pyn5.DatasetINT64(str(root), "ds", False) ds.write_ndarray((0, 0), data, 0) z5_file.create_dataset("ds", data=data, chunks=(6, 7), compression="raw") assert blocks_hash(root) != blocks_hash(z5_path)
def test_data_ordering(tmp_path): root = tmp_path / "test.n5" shape = (10, 20) chunks = (10, 10) pyn5.create_dataset(str(root), "ds", shape, chunks, "UINT8") ds = pyn5.DatasetUINT8(str(root), "ds", False) arr = np.array(ds.read_ndarray((0, 0), shape)) arr += 1 ds.write_ndarray((0, 0), arr, 0) ds_path = root / "ds" assert blocks_in(ds_path) == {"0", "0/0", "0/1"} with open(ds_path / "attributes.json") as f: attrs = json.load(f) assert list(shape) == attrs["dimensions"]
def test_vs_z5(tmp_path, z5_file): """Check different dimensions, same dtype/compression as z5""" root = tmp_path / "test.n5" z5_path = Path(z5_file.path) shape = (10, 20) data = np.arange(np.product(shape)).reshape(shape) chunks = (6, 7) pyn5.create_dataset(str(root), "ds", shape, chunks, data.dtype.name.upper()) ds = pyn5.DatasetINT64(str(root), "ds", False) ds.write_ndarray((0, 0), data, 0) z5_file.create_dataset("ds", data=data, chunks=(6, 7), compression="gzip", level=-1) assert np.allclose(ds.read_ndarray((0, 0), shape), z5_file["ds"][:]) assert blocks_in(root / "ds") != blocks_in(z5_path / "ds") attrs = attrs_in(root / "ds") z5_attrs = attrs_in(z5_path / "ds") for key in ("blockSize", "dimensions"): assert attrs[key] != z5_attrs[key] for key in ("dataType", "compression"): assert attrs[key] == z5_attrs[key] data2 = pyn5.DatasetINT64(str(z5_path), "ds", False).read_ndarray((0, 0), shape) data3 = z5py.N5File(root)["ds"][:] assert not all([np.array_equal(data, data2), np.array_equal(data, data3)])
def save_data_for_CATMAID(self, folder_path: Path): """ Save the segmentation confidence score """ pyn5.create_dataset( folder_path + "/segmentations.n5", "confidence", [int(x) for x in self.end_voxel], [int(x) for x in self.leaf_shape_voxels], "UINT8", ) dataset = pyn5.open(folder_path + "/segmentations.n5", "confidence") for leaf in self.distances.iter_leaves(): pyn5.write( dataset, leaf.bounds, ( 255 * self._view_weighted_mask( tuple(map(slice, leaf.bounds[0], leaf.bounds[1])) ) ).astype(np.uint8), np.uint8, )
for file_path in (zarr_root / "volumes").iterdir(): if not file_path.name.startswith("affs") and not file_path.name.startswith( "."): dataset_name = "volumes/{}".format(file_path.name) zarr_dataset = zarr_data[dataset_name] dtype = "{}".format(zarr_dataset.dtype).upper() if (n5_root / "volumes" / file_path.name).exists(): shutil.rmtree(n5_root / "volumes" / file_path.name) all_data = zarr_dataset[:, :, :].transpose([2, 1, 0]) # if dtype == "UINT64": # vals = list(set(all_data.flatten())) # mapping = {v: i for i, v in enumerate(vals)} # fv = np.vectorize(lambda x: mapping[x]) # all_data = fv(all_data) # dtype = "UINT64" pyn5.create_dataset( str(n5_root.absolute()), dataset_name, zarr_dataset.shape[::-1], [64, 64, 64], dtype, ) n5_dataset = pyn5.open(str(n5_root.absolute()), dataset_name) pyn5.write( n5_dataset, (np.array([0, 0, 0]), np.array(zarr_dataset.shape[::-1])), all_data, dtype, )
def test_volume_transforms_n5_volume(): # Create test n5 dataset test_dataset_path = Path("test.n5") if test_dataset_path.is_dir(): shutil.rmtree(str(test_dataset_path.absolute())) pyn5.create_dataset("test.n5", "test", [10, 10, 10], [2, 2, 2], "UINT8") test_dataset = pyn5.open("test.n5", "test") test_data = np.zeros([10, 10, 10]).astype(int) x = np.linspace(0, 9, 10).reshape([10, 1, 1]).astype(int) test_data = test_data + x + x.transpose([1, 2, 0]) + x.transpose([2, 0, 1]) block_starts = [(i % 5, i // 5 % 5, i // 25 % 5) for i in range(5**3)] for block_start in block_starts: current_bound = list( map(slice, [2 * x for x in block_start], [2 * x + 2 for x in block_start])) flattened = test_data[current_bound].reshape(-1) try: test_dataset.write_block(block_start, flattened) except Exception as e: raise AssertionError( "Writing to n5 failed! Could not create test dataset.\nError: {}" .format(e)) v = volumes.N5Volume("test.n5", {"image": { "path": "test", "dtype": "UINT8" }}, bounds=[10, 10, 10], resolution=[1, 1, 1]) pv = v.partition( [2, 1, 1], [1, 0, 0]) # Note axes are flipped after volume initialization dpv = pv.downsample((2, 2, 2)) np.testing.assert_array_equal( dpv.local_coord_to_world(np.array([2, 2, 2])), np.array([9, 4, 4])) np.testing.assert_array_equal( dpv.world_coord_to_local(np.array([9, 4, 4])), np.array([2, 2, 2])) svb = volumes.SubvolumeBounds(np.array((5, 0, 0), dtype=np.int64), np.array((7, 2, 2), dtype=np.int64)) sv = v.get_subvolume(svb) dpsvb = volumes.SubvolumeBounds(np.array((0, 0, 0), dtype=np.int64), np.array((1, 1, 1), dtype=np.int64)) dpsv = dpv.get_subvolume(dpsvb) np.testing.assert_array_equal( dpsv.image, sv.image.reshape((1, 2, 1, 2, 1, 2)).mean(5).mean(3).mean(1)) # sanity check that test.n5 contains varying data svb2 = volumes.SubvolumeBounds(np.array((5, 0, 1), dtype=np.int64), np.array((7, 2, 3), dtype=np.int64)) sv2 = v.get_subvolume(svb2) assert not all(sv.image.flatten() == sv2.image.flatten()) if test_dataset_path.is_dir(): shutil.rmtree(str(test_dataset_path.absolute()))
from tqdm import tqdm import shutil from pathlib import Path INTERP = [True, False] for interp in INTERP: root = "/home/pattonw/Work/Data/n5_datasets/L1-segmented/L1.n5" dataset = "volumes/segmentation_20" out_dataset = "volumes/segmentation_20_skeleton_mapped_{}".format( "interp" if interp else "no_interp") mapping = pickle.load( open("mapping_{}.obj".format("interpolated" if interp else "plain"), "rb")) image_dataset = pyn5.open(root, dataset) image_data = image_dataset.read_ndarray((0, 0, 0), (1125, 1125, 80)) for key, value in tqdm(mapping.items()): if len(value) == 1: image_data = np.where(image_data == key, value.pop(), image_data) try: shutil.rmtree(Path(root, out_dataset)) except Exception as e: print(e) pass pyn5.create_dataset(root, out_dataset, (1125, 1125, 80), (125, 125, 10), "UINT32") out_ds = pyn5.open(root, out_dataset) out_ds.write_ndarray(np.array((0, 0, 0)), image_data, 0)