def test_get_borders():
slc = Slice5D.zero(x=slice(100, 200), y=slice(300, 400), c=slice(0, 4))
thickness = Shape5D.zero(x=1, y=1)
expected_borders = {
slc.with_coord(x=slice(100, 101)),
slc.with_coord(y=slice(300, 301)),
slc.with_coord(x=slice(199, 200)),
slc.with_coord(y=slice(399, 400)),
}
assert expected_borders == set(slc.get_borders(thickness))
assert len(list(slc.get_borders(thickness))) == 4
thickness = Shape5D.zero(x=10, y=20)
expected_thick_borders = {
slc.with_coord(x=slice(100, 110)),
slc.with_coord(x=slice(190, 200)),
slc.with_coord(y=slice(300, 320)),
slc.with_coord(y=slice(380, 400)),
}
assert expected_thick_borders == set(slc.get_borders(thickness=thickness))
assert len(list(slc.get_borders(thickness=thickness))) == 4
z2_slc = Slice5D.zero(x=slice(100, 200), y=slice(300, 400), z=slice(8, 10))
thickness = Shape5D.zero(x=10, z=2)
expected_z2_borders = {
z2_slc.with_coord(x=slice(100, 110)),
z2_slc.with_coord(x=slice(190, 200)),
z2_slc.with_coord(z=slice(8, 10)),
}
assert expected_z2_borders == set(z2_slc.get_borders(thickness=thickness))
assert len(list(z2_slc.get_borders(thickness=thickness))) == 4
def test_get_borders():
slc = Slice5D.zero(x=slice(100, 200), y=slice(300, 400), c=slice(0, 4))
thickness = Shape5D.zero(x=1, y=1)
expected_borders = {
slc.with_coord(x=slice(100, 101)),
slc.with_coord(y=slice(300, 301)),
slc.with_coord(x=slice(199, 200)),
slc.with_coord(y=slice(399, 400)),
}
for border_slc in slc.get_borders(thickness):
expected_borders.remove(border_slc)
assert len(expected_borders) == 0
thickness = Shape5D.zero(x=10, y=20)
expected_thick_borders = {
slc.with_coord(x=slice(100, 110)),
slc.with_coord(x=slice(190, 200)),
slc.with_coord(y=slice(300, 320)),
slc.with_coord(y=slice(380, 400)),
}
for border_slc in slc.get_borders(thickness=thickness):
expected_thick_borders.remove(border_slc)
assert len(expected_thick_borders) == 0
z2_slc = Slice5D.zero(x=slice(100, 200), y=slice(300, 400), z=slice(8, 10))
thickness = Shape5D.zero(x=10, z=2)
expected_z2_borders = {
z2_slc.with_coord(x=slice(100, 110)),
z2_slc.with_coord(x=slice(190, 200)),
z2_slc.with_coord(z=slice(8, 10)),
}
for border_slc in z2_slc.get_borders(thickness=thickness):
expected_z2_borders.remove(border_slc)
assert len(expected_z2_borders) == 0
def __init__(self,
path: Path,
*,
location: Point5D = Point5D.zero(),
filesystem: FS):
url = filesystem.geturl(path.as_posix())
match = re.search(r"[^/]+\.n5/.*$", url, re.IGNORECASE)
if not match:
raise UnsupportedUrlException(url)
name = match.group(0)
self.filesystem = filesystem.opendir(path.as_posix())
with self.filesystem.openbin("attributes.json", "r") as f:
attributes_json_bytes = f.read()
attributes = json.loads(attributes_json_bytes.decode("utf8"))
dimensions = attributes["dimensions"][::-1]
blockSize = attributes["blockSize"][::-1]
axiskeys = "".join(attributes["axes"]).lower(
)[::-1] if "axes" in attributes else guess_axiskeys(dimensions)
super().__init__(
url=url,
name=name,
tile_shape=Shape5D.create(raw_shape=blockSize, axiskeys=axiskeys),
shape=Shape5D.create(raw_shape=dimensions, axiskeys=axiskeys),
dtype=np.dtype(attributes["dataType"]).newbyteorder(">"),
location=location,
axiskeys=axiskeys,
)
self.compression_type = attributes["compression"]["type"]
if self.compression_type not in N5Block.DECOMPRESSORS.keys():
raise NotImplementedError(
f"Don't know how to decompress from {self.compression_type}")
def __init__(self,
*,
outer_path: Path,
inner_path: PurePosixPath,
location: Point5D = Point5D.zero(),
filesystem: JsonableFilesystem):
self.outer_path = outer_path
self.inner_path = inner_path
self.filesystem = filesystem
binfile = filesystem.openbin(outer_path.as_posix())
f = h5py.File(binfile, "r")
try:
dataset = f[inner_path.as_posix()]
if not isinstance(dataset, h5py.Dataset):
raise ValueError(f"{inner_path} is not a Dataset")
axiskeys = self.getAxisKeys(dataset)
self._dataset = cast(h5py.Dataset, dataset)
tile_shape = Shape5D.create(raw_shape=self._dataset.chunks
or self._dataset.shape,
axiskeys=axiskeys)
super().__init__(
tile_shape=tile_shape,
interval=Shape5D.create(
raw_shape=self._dataset.shape,
axiskeys=axiskeys).to_interval5d(location),
dtype=self._dataset.dtype,
axiskeys=axiskeys,
)
except Exception as e:
f.close()
raise e
def test_h5_datasource():
data_2d = Array5D(np.arange(100).reshape(10, 10), axiskeys="yx")
h5_path = create_h5(data_2d,
axiskeys_style="vigra",
chunk_shape=Shape5D(x=3, y=3))
ds = H5DataSource(outer_path=h5_path,
inner_path=PurePosixPath("/data"),
filesystem=OsFs("/"))
assert ds.shape == data_2d.shape
assert ds.tile_shape == Shape5D(x=3, y=3)
slc = ds.interval.updated(x=(0, 3), y=(0, 2))
assert (ds.retrieve(slc).raw("yx") == data_2d.cut(slc).raw("yx")
).all() #type: ignore
data_3d = Array5D(np.arange(10 * 10 * 10).reshape(10, 10, 10),
axiskeys="zyx")
h5_path = create_h5(data_3d,
axiskeys_style="vigra",
chunk_shape=Shape5D(x=3, y=3))
ds = H5DataSource(outer_path=h5_path,
inner_path=PurePosixPath("/data"),
filesystem=OsFs("/"))
assert ds.shape == data_3d.shape
assert ds.tile_shape == Shape5D(x=3, y=3)
slc = ds.interval.updated(x=(0, 3), y=(0, 2), z=3)
assert (ds.retrieve(slc).raw("yxz") == data_3d.cut(slc).raw("yxz")
).all() #type: ignore
def __init__(self,
path: Path,
*,
location: Point5D = Point5D.zero(),
filesystem: FS):
self._dataset: Optional[h5py.Dataset] = None
try:
self._dataset, outer_path, inner_path = self.openDataset(
path, filesystem=filesystem)
axiskeys = self.getAxisKeys(self._dataset)
tile_shape = Shape5D.create(raw_shape=self._dataset.chunks
or self._dataset.shape,
axiskeys=axiskeys)
super().__init__(
url=filesystem.desc(outer_path.as_posix()) + "/" +
inner_path.as_posix(),
tile_shape=tile_shape,
shape=Shape5D.create(raw_shape=self._dataset.shape,
axiskeys=axiskeys),
dtype=self._dataset.dtype,
name=self._dataset.file.filename.split("/")[-1] +
self._dataset.name,
location=location,
axiskeys=axiskeys,
)
except Exception as e:
if self._dataset:
self._dataset.file.close()
raise e
def test_allocation():
arr = Array5D.allocate(Slice5D.zero(x=slice(100, 200), y=slice(200, 300)), numpy.uint8)
assert arr.shape == Shape5D(x=100, y=100)
assert arr.location == Point5D.zero(x=100, y=200)
arr = Array5D.allocate(Slice5D.zero(x=slice(-100, 200), y=slice(200, 300)), numpy.uint8)
assert arr.shape == Shape5D(x=300, y=100)
assert arr.location == Point5D.zero(x=-100, y=200)
def test_unique_border_colors():
# fmt: off
arr = Array5D(numpy.asarray([
[7, 7, 0, 0, 0, 0],
[7, 7, 0, 0, 0, 0],
[7, 0, 0, 0, 0, 0],
[0, 0, 0, 3, 0, 0],
[0, 0, 3, 3, 3, 0],
[0, 0, 0, 3, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 5, 5, 0, 0]]), axiskeys="yx")
# fmt: on
border_colors = arr.unique_border_colors()
assert border_colors.shape == Shape5D(x=len([7, 5, 0]))
raw_colors = border_colors.raw("x")
assert 7 in raw_colors
assert 5 in raw_colors
assert 0 in raw_colors
# fmt: off
arr_zyx = Array5D(numpy.asarray([
[[7, 7, 0, 0, 0, 0],
[7, 7, 0, 0, 0, 0],
[7, 0, 0, 0, 0, 0],
[0, 0, 0, 3, 0, 0],
[0, 0, 3, 3, 3, 0],
[0, 0, 0, 3, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 5, 5, 0, 0]],
[[0, 0, 0, 2, 2, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 9, 0],
[0, 0, 0, 0, 9, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0]],
]), axiskeys="zyx")
# fmt: on
# import pydevd; pydevd.settrace()
# get borders as if this was two separate plaes, as opposed to a single 3d block
border_colors = arr_zyx.unique_border_colors(border_thickness=Shape5D.zero(x=1, y=1))
print("===>>>>>", border_colors.raw("x"))
assert border_colors.shape == Shape5D(x=len([7, 5, 0, 2]))
raw_colors = border_colors.raw("x")
assert 7 in raw_colors
assert 5 in raw_colors
assert 0 in raw_colors
assert 2 in border_colors._data
def test_n5_attributes():
attributes = N5DatasetAttributes(
dimensions=Shape5D(x=100, y=200),
blockSize=Shape5D(x=10, y=20),
axiskeys="yx",
dataType=np.dtype("uint16").newbyteorder(">"), #type: ignore
compression=GzipCompressor(level=3))
reserialized_attributes = N5DatasetAttributes.from_json_data(
attributes.to_json_data())
assert reserialized_attributes == attributes
assert attributes.to_json_data()["axes"] == ("x", "y")
def test_writing_to_precomputed_chunks(tmp_path: Path, data: Array5D):
datasource = ArrayDataSource.from_array5d(data,
tile_shape=Shape5D(x=10, y=10))
scale = PrecomputedChunksScale.from_datasource(datasource=datasource,
key=Path("my_test_data"),
encoding=RawEncoder())
info = PrecomputedChunksInfo(
data_type=datasource.dtype,
type_="image",
num_channels=datasource.shape.c,
scales=tuple([scale]),
)
sink_path = Path("mytest.precomputed")
filesystem = OsFs(tmp_path.as_posix())
datasink = PrecomputedChunksSink.create(
filesystem=filesystem,
base_path=sink_path,
info=info,
).scale_sinks[0]
for tile in datasource.roi.get_datasource_tiles():
datasink.write(tile.retrieve())
precomp_datasource = PrecomputedChunksDataSource(
path=sink_path, filesystem=filesystem, resolution=scale.resolution)
reloaded_data = precomp_datasource.retrieve()
assert reloaded_data == data
def create_h5(array: Array5D,
axiskeys_style: str,
chunk_shape: Optional[Shape5D] = None,
axiskeys: str = "xyztc"):
raw_chunk_shape = (chunk_shape or Shape5D() * 2).clamped(
maximum=array.shape).to_tuple(axiskeys)
path = tempfile.mkstemp()[1] + ".h5"
f = h5py.File(path, "w")
ds = f.create_dataset("data",
chunks=raw_chunk_shape,
data=array.raw(axiskeys))
if axiskeys_style == "dims":
for key, dim in zip(axiskeys, ds.dims): # type: ignore
dim.label = key
elif axiskeys_style == "vigra":
type_flags = {"x": 2, "y": 2, "z": 2, "t": 2, "c": 1}
axistags = [{
"key": key,
"typeflags": type_flags[key],
"resolution": 0,
"description": ""
} for key in axiskeys]
ds.attrs["axistags"] = json.dumps({"axes": axistags})
else:
raise Exception(f"Bad axiskeys_style: {axiskeys_style}")
return Path(path)
def create_n5(array: Array5D, axiskeys: str = "xyztc", chunk_size: Optional[Shape5D] = None):
data_slice = DataSourceSlice(ArrayDataSource.from_array5d(array))
chunk_size = chunk_size or Shape5D.hypercube(10)
path = Path(tempfile.mkstemp()[1] + ".n5/data")
sink = N5DataSink(path=path, data_slice=data_slice, axiskeys=axiskeys, tile_shape=chunk_size)
sink.process()
return path.as_posix()
def test_get_tiles_when_slice_is_NOT_multiple_of_tile():
slc = Slice5D.zero(x=slice(90, 210), y=slice(200, 320), z=slice(10, 20))
pieces = list(slc.get_tiles(Shape5D(x=50, y=50, z=10)))
assert Slice5D.zero(x=slice(50, 100), y=slice(200, 250),
z=slice(10, 20)) in pieces
assert Slice5D.zero(x=slice(50, 100), y=slice(250, 300),
z=slice(10, 20)) in pieces
assert Slice5D.zero(x=slice(50, 100), y=slice(300, 350),
z=slice(10, 20)) in pieces
assert Slice5D.zero(x=slice(100, 150), y=slice(200, 250),
z=slice(10, 20)) in pieces
assert Slice5D.zero(x=slice(100, 150), y=slice(250, 300),
z=slice(10, 20)) in pieces
assert Slice5D.zero(x=slice(100, 150), y=slice(300, 350),
z=slice(10, 20)) in pieces
assert Slice5D.zero(x=slice(150, 200), y=slice(200, 250),
z=slice(10, 20)) in pieces
assert Slice5D.zero(x=slice(150, 200), y=slice(250, 300),
z=slice(10, 20)) in pieces
assert Slice5D.zero(x=slice(150, 200), y=slice(300, 350),
z=slice(10, 20)) in pieces
assert Slice5D.zero(x=slice(200, 250), y=slice(200, 250),
z=slice(10, 20)) in pieces
assert Slice5D.zero(x=slice(200, 250), y=slice(250, 300),
z=slice(10, 20)) in pieces
assert Slice5D.zero(x=slice(200, 250), y=slice(300, 350),
z=slice(10, 20)) in pieces
assert len(pieces) == 12
def test_slice_defined_with():
slc = Slice5D(x=slice(10, 20))
assert slc.defined_with(Shape5D(x=100, y=15, z=17)) == Slice5D.zero(x=slice(10, 20), y=slice(0, 15), z=slice(0, 17))
assert slc.defined_with(Slice5D.zero(x=slice(1, 3), y=slice(10, 20))) == Slice5D.zero(
x=slice(10, 20), y=slice(10, 20)
)
def test_get_tiles_when_slice_is_multiple_of_tile():
slc = Slice5D.zero(x=slice(100, 200), y=slice(200, 300))
tiles = list(slc.get_tiles(Shape5D(x=50, y=50)))
assert Slice5D.zero(x=slice(100, 150), y=slice(200, 250)) in tiles
assert Slice5D.zero(x=slice(100, 150), y=slice(250, 300)) in tiles
assert Slice5D.zero(x=slice(150, 200), y=slice(200, 250)) in tiles
assert Slice5D.zero(x=slice(150, 200), y=slice(250, 300)) in tiles
assert len(tiles) == 4
def test_split_when_slice_is_multiple_of_block_shape():
slc = Slice5D.zero(x=slice(100, 200), y=slice(200, 300))
pieces = list(slc.split(Shape5D(x=50, y=50)))
assert Slice5D.zero(x=slice(100, 150), y=slice(200, 250)) in pieces
assert Slice5D.zero(x=slice(100, 150), y=slice(250, 300)) in pieces
assert Slice5D.zero(x=slice(150, 200), y=slice(200, 250)) in pieces
assert Slice5D.zero(x=slice(150, 200), y=slice(250, 300)) in pieces
assert len(pieces) == 4
def test_neighboring_tiles():
# fmt: off
arr = Array5D(np.asarray(
[[10, 11, 12, 20, 21, 22, 30], [13, 14, 15, 23, 24, 25, 33],
[16, 17, 18, 26, 27, 28, 36], [40, 41, 42, 50, 51, 52, 60],
[43, 44, 45, 53, 54, 55, 63], [46, 47, 48, 56, 57, 58, 66],
[70, 71, 72, 80, 81, 82, 90], [73, 74, 75, 83, 84, 85, 93],
[76, 77, 78, 86, 87, 88, 96], [0, 1, 2, 3, 4, 5, 6]],
dtype=np.uint8),
axiskeys="yx")
ds = DataSource.create(create_png(arr))
fifties_slice = DataSourceSlice(ds).clamped(
Slice5D(x=slice(3, 6), y=slice(3, 6)))
expected_fifties_slice = Array5D(np.asarray([[50, 51, 52], [53, 54, 55],
[56, 57, 58]]),
axiskeys="yx")
# fmt: on
top_slice = DataSourceSlice(ds, x=slice(3, 6), y=slice(0, 3))
bottom_slice = DataSourceSlice(ds, x=slice(3, 6), y=slice(6, 9))
right_slice = DataSourceSlice(ds, x=slice(6, 7), y=slice(3, 6))
left_slice = DataSourceSlice(ds, x=slice(0, 3), y=slice(3, 6))
# fmt: off
fifties_neighbor_data = {
top_slice:
Array5D(np.asarray([[20, 21, 22], [23, 24, 25], [26, 27, 28]]),
axiskeys="yx"),
right_slice:
Array5D(np.asarray([[60], [63], [66]]), axiskeys="yx"),
bottom_slice:
Array5D(np.asarray([[80, 81, 82], [83, 84, 85], [86, 87, 88]]),
axiskeys="yx"),
left_slice:
Array5D(np.asarray([[40, 41, 42], [43, 44, 45], [46, 47, 48]]),
axiskeys="yx"),
}
expected_fifties_neighbors = {}
# fmt: on
assert (fifties_slice.retrieve().raw("yx") == expected_fifties_slice.raw(
"yx")).all()
for neighbor in fifties_slice.get_neighboring_tiles(
tile_shape=Shape5D(x=3, y=3)):
try:
expected_slice = fifties_neighbor_data.pop(neighbor)
assert (expected_slice.raw("yx") == neighbor.retrieve().raw("yx")
).all()
except KeyError:
print(f"\nWas searching for ", neighbor, "\n")
for k in fifties_neighbor_data.keys():
print("--->>> ", k)
assert len(fifties_neighbor_data) == 0
def test_n5_datasink_saves_roi(tmp_path: Path, data: Array5D, datasource: DataSource):
roi = DataSourceSlice(datasource, x=slice(5, 8), y=slice(2, 4))
dataset_path = tmp_path / "test_n5_datasink_saves_roi.n5/data"
sink = N5DataSink(path=dataset_path, data_slice=roi, tile_shape=Shape5D(x=10, y=10))
sink.process(Slice5D.all())
n5ds = DataSource.create(dataset_path)
assert n5ds.retrieve(Slice5D.all()) == roi.retrieve()
def test_h5_datasource():
data_2d = Array5D(np.arange(100).reshape(10, 10), axiskeys="yx")
h5_path = create_h5(data_2d, axiskeys_style="vigra", chunk_shape=Shape5D(x=3, y=3))
ds = DataSource.create(h5_path)
assert ds.shape == data_2d.shape
assert ds.tile_shape == Shape5D(x=3, y=3)
slc = Slice5D(x=slice(0, 3), y=slice(0, 2))
assert (ds.retrieve(slc).raw("yx") == data_2d.cut(slc).raw("yx")).all()
data_3d = Array5D(np.arange(10 * 10 * 10).reshape(10, 10, 10), axiskeys="zyx")
h5_path = create_h5(data_3d, axiskeys_style="vigra", chunk_shape=Shape5D(x=3, y=3))
ds = DataSource.create(h5_path)
assert ds.shape == data_3d.shape
assert ds.tile_shape == Shape5D(x=3, y=3)
slc = Slice5D(x=slice(0, 3), y=slice(0, 2), z=3)
assert (ds.retrieve(slc).raw("yxz") == data_3d.cut(slc).raw("yxz")).all()
def __init__(
self,
url: str,
*,
tile_shape: Optional[Shape5D] = None,
dtype: np.dtype,
name: str = "",
shape: Shape5D,
location: Point5D = Point5D.zero(),
axiskeys: str,
):
self.url = url
self.tile_shape = (tile_shape or Shape5D.hypercube(256)).to_slice_5d().clamped(shape.to_slice_5d()).shape
self.dtype = dtype
self.name = name or self.url.split("/")[-1]
self.shape = shape
self.roi = shape.to_slice_5d(offset=location)
self.location = location
self.axiskeys = axiskeys
def test_data_roi_get_tiles_can_clamp_to_datasource_tiles():
# fmt: off
data = Array5D(np.asarray([
[1, 2, 3, 4, 5],
[6, 7, 8, 9, 10],
[11, 12, 13, 14, 15],
[16, 17, 18, 19, 20],
]).astype(np.uint8),
axiskeys="yx")
# fmt: on
ds = ArrayDataSource.from_array5d(data, tile_shape=Shape5D(x=2, y=2))
data_slice = DataRoi(datasource=ds, x=(1, 4), y=(0, 3))
# fmt: off
dataslice_expected_data = Array5D(np.asarray([[2, 3, 4], [7, 8, 9],
[12, 13,
14]]).astype(np.uint8),
axiskeys="yx",
location=Point5D.zero(x=1))
# fmt: on
assert data_slice.retrieve() == dataslice_expected_data
# fmt: off
dataslice_expected_slices = [
Array5D(np.asarray([[1, 2], [6, 7]]).astype(np.uint8),
axiskeys="yx",
location=Point5D.zero()),
Array5D(np.asarray([
[3, 4],
[8, 9],
]).astype(np.uint8),
axiskeys="yx",
location=Point5D.zero(x=2)),
Array5D(np.asarray([
[11, 12],
[16, 17],
]).astype(np.uint8),
axiskeys="yx",
location=Point5D.zero(y=2)),
Array5D(np.asarray([
[13, 14],
[18, 19],
]).astype(np.uint8),
axiskeys="yx",
location=Point5D.zero(x=2, y=2))
]
# fmt: on
expected_slice_dict = {a.interval: a for a in dataslice_expected_slices}
for piece in data_slice.get_datasource_tiles(clamp_to_datasource=True):
expected_data = expected_slice_dict.pop(piece.interval)
assert expected_data == piece.retrieve()
assert len(expected_slice_dict) == 0
def from_json_value(cls, value: JsonValue) -> "SkimageDataSource":
value_obj = ensureJsonObject(value)
raw_location = value_obj.get("location")
raw_tile_shape = value_obj.get("tile_shape")
return SkimageDataSource(
path=Path(ensureJsonString(value_obj.get("path"))),
location=Point5D.zero()
if raw_location is None else Point5D.from_json_value(raw_location),
filesystem=JsonableFilesystem.from_json_value(
value_obj.get("filesystem")),
tile_shape=None if raw_tile_shape is None else
Shape5D.from_json_value(raw_tile_shape))
def test_slice_clamp():
outer = Slice5D(x=slice(10, 100), y=slice(20, 200))
inner = Slice5D(x=slice(20, 50), y=slice(30, 40), z=0, t=0, c=0)
assert outer.clamped(inner) == inner
assert inner.clamped(outer) == inner
intersecting_outer = Slice5D(x=slice(50, 200), y=slice(30, 900))
assert intersecting_outer.clamped(outer) == Slice5D(x=slice(50, 100), y=slice(30, 200))
intersecting_outer = Slice5D(x=slice(-100, 50), y=slice(10, 100))
assert intersecting_outer.clamped(outer) == Slice5D(x=slice(10, 50), y=slice(20, 100))
outside_outer = Slice5D(x=slice(200, 300), y=slice(400, 500))
assert outside_outer.clamped(outer).defined_with(Shape5D()).shape.volume == 0
def test_get_borders():
# fmt: off
arr = Array5D(
numpy.asarray([
[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15],
[16, 17, 18, 19, 20]],
[[-1, -2, -3, -4, -5], [-6, -7, -8, -9, -10],
[-11, -12, -13, -14, -15], [-16, -17, -18, -19, -20]],
[[10, 20, 30, 40, 50], [11, 21, 31, 41, 51], [12, 22, 32, 42, 52],
[13, 23, 33, 43, 53]],
]), "cyx")
expected_thin_borders = {
"left_border":
Array5D(
numpy.asarray([
[[1], [6], [11], [16]],
[[-1], [-6], [-11], [-16]],
[[10], [11], [12], [13]],
]), "cyx"),
"top_border":
Array5D(
numpy.asarray([[[1, 2, 3, 4, 5]], [[-1, -2, -3, -4, -5]],
[[10, 20, 30, 40, 50]]]), "cyx"),
"right_border":
Array5D(
numpy.asarray([
[[5], [10], [15], [20]],
[[-5], [-10], [-15], [-20]],
[[50], [51], [52], [53]],
]), "cyx"),
"bottom_border":
Array5D(
numpy.asarray([
[[16, 17, 18, 19, 20]],
[[-16, -17, -18, -19, -20]],
[[13, 23, 33, 43, 53]],
]), "cyx")
}
# fmt: on
for border_data in arr.get_borders(thickness=Shape5D.zero(x=1, y=1)):
for expected_border in expected_thin_borders.values():
if (border_data.raw("cyx") == expected_border.raw("cyx")).all():
break
else:
raise Exception(
f"Could not find this border in the expected set:\n{border_data.raw('cyx')}"
)
def test_retrieve_roi_smaller_than_tile():
# fmt: off
data = Array5D(np.asarray([
[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15],
[16, 17, 18, 19, 20]],
[[100, 200, 300, 400, 500], [600, 700, 800, 900, 1000],
[1100, 1200, 1300, 1400, 1500], [1600, 1700, 1800, 1900, 2000]],
]).astype(np.uint32),
axiskeys="cyx")
# fmt: on
path = Path(create_n5(data, chunk_size=Shape5D(c=2, y=4, x=4)))
ds = N5DataSource(path=path / "data", filesystem=OsFs("/"))
print(f"\n\n====>> tile shape: {ds.shape}")
smaller_than_tile = ds.retrieve(c=1, y=(0, 4), x=(0, 4))
print(smaller_than_tile.raw("cyx"))
def from_json_value(cls,
value: JsonValue) -> "PrecomputedChunksDataSource":
value_obj = ensureJsonObject(value)
raw_location = value_obj.get("location")
raw_chunk_size = value_obj.get("chunk_size")
return PrecomputedChunksDataSource(
path=Path(ensureJsonString(value_obj.get("path"))),
resolution=ensureJsonIntTripplet(
value_obj.get("spatial_resolution")
), # FIXME? change to just resolution?
location=None
if raw_location is None else Point5D.from_json_value(raw_location),
chunk_size=None if raw_chunk_size is None else
Shape5D.from_json_value(raw_chunk_size),
filesystem=JsonableFilesystem.from_json_value(
value_obj.get("filesystem")),
)
def test_n5_datasink(tmp_path: Path, data: Array5D, datasource: DataSource):
sink = N5DatasetSink.create(filesystem=OsFs(tmp_path.as_posix()),
outer_path=Path("test_n5_datasink.n5"),
inner_path=PurePosixPath("/data"),
attributes=N5DatasetAttributes(
dimensions=datasource.shape,
blockSize=Shape5D(x=10, y=10),
axiskeys=datasource.axiskeys,
dataType=datasource.dtype,
compression=RawCompressor(),
location=Point5D.zero(x=7, y=13)))
for tile in DataRoi(datasource).split(sink.tile_shape):
sink.write(tile.retrieve().translated(Point5D.zero(x=7, y=13)))
n5ds = N5DataSource(filesystem=sink.filesystem, path=sink.path)
saved_data = n5ds.retrieve()
assert saved_data.location == Point5D.zero(x=7, y=13)
assert saved_data == data
def __init__(
self,
*,
data: np.ndarray,
axiskeys: str,
tile_shape: Optional[Shape5D] = None,
location: Point5D = Point5D.zero(),
):
self._data = Array5D(data, axiskeys=axiskeys, location=location)
if tile_shape is None:
tile_shape = Shape5D.hypercube(256).to_interval5d().clamped(
self._data.shape).shape
super().__init__(
dtype=self._data.dtype,
tile_shape=tile_shape,
interval=self._data.interval,
axiskeys=axiskeys,
)
def test_skimage_datasource_tiles(png_image: Path):
bs = DataRoi(SkimageDataSource(png_image, filesystem=OsFs("/")))
num_checked_tiles = 0
for tile in bs.split(Shape5D(x=2, y=2)):
if tile == Interval5D.zero(x=(0, 2), y=(0, 2)):
expected_raw = raw_0_2x0_2y
elif tile == Interval5D.zero(x=(0, 2), y=(2, 4)):
expected_raw = raw_0_2x2_4y
elif tile == Interval5D.zero(x=(2, 4), y=(0, 2)):
expected_raw = raw_2_4x0_2y
elif tile == Interval5D.zero(x=(2, 4), y=(2, 4)):
expected_raw = raw_2_4x2_4y
elif tile == Interval5D.zero(x=(4, 5), y=(0, 2)):
expected_raw = raw_4_5x0_2y
elif tile == Interval5D.zero(x=(4, 5), y=(2, 4)):
expected_raw = raw_4_5x2_4y
else:
raise Exception(f"Unexpected tile {tile}")
assert (tile.retrieve().raw("yx") == expected_raw).all()
num_checked_tiles += 1
assert num_checked_tiles == 6
def test_n5_datasource():
# fmt: off
data = Array5D(np.asarray([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10],
[11, 12, 13, 14, 15], [16, 17, 18, 19,
20]]).astype(np.uint8),
axiskeys="yx")
# fmt: on
path = Path(create_n5(data, chunk_size=Shape5D(x=2, y=2)))
ds = N5DataSource(path=path / "data", filesystem=OsFs("/"))
assert ds.shape == data.shape
# fmt: off
expected_raw_piece = Array5D(np.asarray([[1, 2, 3], [6, 7,
8]]).astype(np.uint8),
axiskeys="yx")
# fmt: on
assert ds.retrieve(x=(0, 3), y=(0, 2)) == expected_raw_piece
ds2 = pickle.loads(pickle.dumps(ds))
assert ds2.retrieve(x=(0, 3), y=(0, 2)) == expected_raw_piece