def test_n5_datasink(tmp_path: Path, data: Array5D, datasource: DataSource):
dataset_path = tmp_path / "test_n5_datasink.n5/data"
sink = N5DataSink(path=dataset_path, data_slice=DataSourceSlice(datasource), tile_shape=Shape5D(x=10, y=10))
sink.process(Slice5D.all())
n5ds = DataSource.create(dataset_path)
assert n5ds.retrieve(Slice5D.all()) == data
def test_n5_datasink_saves_roi(tmp_path: Path, data: Array5D, datasource: DataSource):
roi = DataSourceSlice(datasource, x=slice(5, 8), y=slice(2, 4))
dataset_path = tmp_path / "test_n5_datasink_saves_roi.n5/data"
sink = N5DataSink(path=dataset_path, data_slice=roi, tile_shape=Shape5D(x=10, y=10))
sink.process(Slice5D.all())
n5ds = DataSource.create(dataset_path)
assert n5ds.retrieve(Slice5D.all()) == roi.retrieve()
def process(self,
roi: Slice5D = Slice5D.all(),
address_mode: AddressMode = AddressMode.BLACK) -> None:
defined_roi = roi.defined_with(self.data_slice)
assert self.data_slice.contains(defined_roi)
for piece in defined_roi.split(self.tile_shape):
source_data = self.data_slice.datasource.retrieve(
piece, address_mode=address_mode)
self._process_tile(source_data)
def parse_known_cmdline_args(self, cmdline_args):
# We use the same parser as the DataSelectionApplet
parser = DataSelectionApplet.get_arg_parser(
self.dataSelectionApplet.role_names)
parser.add_argument(
"--distributed",
help=
"Distributed mode. Used for running ilastik on HPCs via SLURM/srun/mpirun",
action="store_true",
)
default_block_roi = Slice5D.all(x=slice(0, 256),
y=slice(0, 256),
z=slice(0, 256),
t=slice(0, 1))
def parse_distributed_block_roi(value: str) -> Slice5D:
parsed = ast.literal_eval(value)
if isinstance(parsed, dict):
if not set(parsed.keys()).issubset("xyztc"):
raise ValueError(
f"Bad keys for distributed-block-roi: {value}")
if not all(
isinstance(v, (int, None.__class__))
for v in parsed.values()):
raise ValueError(
f"Bad values for distributed-block-roi: {value}")
overrides = {k: slice(0, int(v)) for k, v in parsed.items()}
elif isinstance(parsed, int):
overrides = {k: slice(0, parsed) for k in "xyz"}
else:
raise TypeError(
f"Could not convert value {value} into a Slice5D")
return Slice5D(**{**default_block_roi.to_dict(), **overrides})
parser.add_argument(
"--distributed_block_roi",
"--distributed-block-roi",
help=textwrap.dedent("""
Determines the dimensions of the blocks used to split the data in distributed mode.
Values can be either:"
An integer, which will be interpreted as if the following dict was passed in:
{'x': value, 'y': value, 'z': value, 't': 1, 'c': None}
or a literal python Dict[str, Optional[int]], with keys in 'xyztc'.
Missing keys will default like so:
{'x': 256, 'y': 256, 'z': 256, 't': 1, 'c': None}
Use None anywhere in the dict to mean "the whole dimension".
"""),
type=parse_distributed_block_roi,
default=default_block_roi,
)
parsed_args, unused_args = parser.parse_known_args(cmdline_args)
return parsed_args, unused_args
def test_retrieve_roi_smaller_than_tile():
# fmt: off
data = Array5D(np.asarray([
[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15],
[16, 17, 18, 19, 20]],
[[100, 200, 300, 400, 500], [600, 700, 800, 900, 1000],
[1100, 1200, 1300, 1400, 1500], [1600, 1700, 1800, 1900, 2000]],
]).astype(np.uint32),
axiskeys="cyx")
# fmt: on
path = Path(create_n5(data, chunk_size=Shape5D(c=2, y=4, x=4)))
ds = DataSource.create(path)
print(f"\n\n====>> tile shape: {ds.roi}")
smaller_than_tile = ds.retrieve(
Slice5D.all(c=1, y=slice(0, 4), x=slice(0, 4)))
print(smaller_than_tile.raw("cyx"))
def test_distributed_n5_datasink(tmp_path: Path, data: Array5D, datasource: DataSource):
dataset_path = tmp_path / "test_distributed_n5_datasink.n5/data"
data_slice = DataSourceSlice(datasource)
sinks = [
N5DataSink(path=dataset_path, data_slice=data_slice, mode=N5DataSink.Mode.CREATE),
N5DataSink(path=dataset_path, data_slice=data_slice, mode=N5DataSink.Mode.OPEN),
N5DataSink(path=dataset_path, data_slice=data_slice, mode=N5DataSink.Mode.OPEN),
N5DataSink(path=dataset_path, data_slice=data_slice, mode=N5DataSink.Mode.OPEN),
]
for idx, piece in enumerate(data_slice.split()):
sink = sinks[idx % len(sinks)]
sink.process(piece)
n5ds = DataSource.create(dataset_path)
assert n5ds.retrieve(Slice5D.all()) == data
def test_all_constructor():
slc = Slice5D.all(x=3, z=slice(10, 20))
assert slc.to_slices("xyztc") == (slice(3, 4), slice(None), slice(10, 20),
slice(None), slice(None))
