def test_retrieve_roi_smaller_than_tile():
# fmt: off
data = Array5D(np.asarray([
[[ 1, 2, 3, 4, 5],
[ 6, 7, 8, 9, 10],
[ 11, 12, 13, 14, 15],
[ 16, 17, 18, 19, 20]],
[[ 100, 200, 300, 400, 500],
[ 600, 700, 800, 900, 1000],
[1100, 1200, 1300, 1400, 1500],
[1600, 1700, 1800, 1900, 2000]],
]).astype(np.uint32), axiskeys="cyx")
expected_cyx = np.asarray([
[[ 100, 200, 300, 400],
[ 600, 700, 800, 900],
[1100, 1200, 1300, 1400],
[1600, 1700, 1800, 1900]]
])
# fmt: on
path = PurePosixPath(create_n5(data, chunk_size=Shape5D(c=2, y=4, x=4)))
ds = N5DataSource(path=path / "data", filesystem=OsFs("/"))
smaller_than_tile = ds.retrieve(c=1, y=(0, 4), x=(0, 4))
assert np.all(smaller_than_tile.raw("cyx") == expected_cyx)
def test_distributed_n5_datasink():
tmp_path = create_tmp_dir(prefix="test_distributed_n5_datasink")
filesystem = OsFs(tmp_path.as_posix())
outer_path = PurePosixPath("test_distributed_n5_datasink.n5")
inner_path = PurePosixPath("/data")
full_path = PurePosixPath("test_distributed_n5_datasink.n5/data")
attributes = N5DatasetAttributes(
dimensions=datasource.shape,
blockSize=datasource.tile_shape,
c_axiskeys=data.axiskeys, #FIXME: double check this
dataType=datasource.dtype,
compression=RawCompressor())
sink = N5DatasetSink(outer_path=outer_path,
inner_path=inner_path,
filesystem=filesystem,
attributes=attributes)
sink_writer = sink.create()
assert not isinstance(sink_writer, Exception)
sink_writers = [sink_writer] * 4
for idx, piece in enumerate(DataRoi(datasource).default_split()):
sink = sink_writers[idx % len(sink_writers)]
sink.write(piece.retrieve())
n5ds = N5DataSource(filesystem=filesystem, path=full_path)
assert n5ds.retrieve() == data
def test_distributed_n5_datasink(tmp_path: Path, data: Array5D,
datasource: DataSource):
filesystem = OsFs(tmp_path.as_posix())
outer_path = Path("test_distributed_n5_datasink.n5")
inner_path = PurePosixPath("/data")
full_path = Path("test_distributed_n5_datasink.n5/data")
attributes = N5DatasetAttributes(dimensions=datasource.shape,
blockSize=datasource.tile_shape,
axiskeys=datasource.axiskeys,
dataType=datasource.dtype,
compression=RawCompressor())
sinks = [
N5DatasetSink.create(outer_path=outer_path,
inner_path=inner_path,
filesystem=filesystem,
attributes=attributes),
N5DatasetSink.open(path=full_path, filesystem=filesystem),
N5DatasetSink.open(path=full_path, filesystem=filesystem),
N5DatasetSink.open(path=full_path, filesystem=filesystem),
]
for idx, piece in enumerate(DataRoi(datasource).default_split()):
sink = sinks[idx % len(sinks)]
sink.write(piece.retrieve())
n5ds = N5DataSource(filesystem=filesystem, path=full_path)
assert n5ds.retrieve() == data
def test_n5_datasink(tmp_path: Path, data: Array5D, datasource: DataSource):
sink = N5DatasetSink.create(filesystem=OsFs(tmp_path.as_posix()),
outer_path=Path("test_n5_datasink.n5"),
inner_path=PurePosixPath("/data"),
attributes=N5DatasetAttributes(
dimensions=datasource.shape,
blockSize=Shape5D(x=10, y=10),
axiskeys=datasource.axiskeys,
dataType=datasource.dtype,
compression=RawCompressor(),
location=Point5D.zero(x=7, y=13)))
for tile in DataRoi(datasource).split(sink.tile_shape):
sink.write(tile.retrieve().translated(Point5D.zero(x=7, y=13)))
n5ds = N5DataSource(filesystem=sink.filesystem, path=sink.path)
saved_data = n5ds.retrieve()
assert saved_data.location == Point5D.zero(x=7, y=13)
assert saved_data == data
def test_n5_datasource():
# fmt: off
data = Array5D(np.asarray([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10],
[11, 12, 13, 14, 15], [16, 17, 18, 19,
20]]).astype(np.uint8),
axiskeys="yx")
# fmt: on
path = Path(create_n5(data, chunk_size=Shape5D(x=2, y=2)))
ds = N5DataSource(path=path / "data", filesystem=OsFs("/"))
assert ds.shape == data.shape
# fmt: off
expected_raw_piece = Array5D(np.asarray([[1, 2, 3], [6, 7,
8]]).astype(np.uint8),
axiskeys="yx")
# fmt: on
assert ds.retrieve(x=(0, 3), y=(0, 2)) == expected_raw_piece
ds2 = pickle.loads(pickle.dumps(ds))
assert ds2.retrieve(x=(0, 3), y=(0, 2)) == expected_raw_piece
def test_n5_datasink():
tmp_path = create_tmp_dir(prefix="test_n5_datasink")
sink = N5DatasetSink(
filesystem=OsFs(tmp_path.as_posix()),
outer_path=PurePosixPath("test_n5_datasink.n5"),
inner_path=PurePosixPath("/data"),
attributes=N5DatasetAttributes(
dimensions=datasource.shape,
blockSize=Shape5D(x=10, y=10),
c_axiskeys=data.axiskeys, #FIXME: double check this
dataType=datasource.dtype,
compression=RawCompressor(),
location=Point5D.zero(x=7, y=13)))
sink_writer = sink.create()
assert not isinstance(sink_writer, Exception)
for tile in DataRoi(datasource).split(sink.tile_shape):
sink_writer.write(tile.retrieve().translated(Point5D.zero(x=7, y=13)))
n5ds = N5DataSource(filesystem=sink.filesystem, path=sink.full_path)
saved_data = n5ds.retrieve()
assert saved_data.location == Point5D.zero(x=7, y=13)
assert saved_data == data