def test_n5_datasink(tmp_path: Path, data: Array5D, datasource: DataSource): dataset_path = tmp_path / "test_n5_datasink.n5/data" sink = N5DataSink(path=dataset_path, data_slice=DataSourceSlice(datasource), tile_shape=Shape5D(x=10, y=10)) sink.process(Slice5D.all()) n5ds = DataSource.create(dataset_path) assert n5ds.retrieve(Slice5D.all()) == data
def test_n5_datasink_saves_roi(tmp_path: Path, data: Array5D, datasource: DataSource): roi = DataSourceSlice(datasource, x=slice(5, 8), y=slice(2, 4)) dataset_path = tmp_path / "test_n5_datasink_saves_roi.n5/data" sink = N5DataSink(path=dataset_path, data_slice=roi, tile_shape=Shape5D(x=10, y=10)) sink.process(Slice5D.all()) n5ds = DataSource.create(dataset_path) assert n5ds.retrieve(Slice5D.all()) == roi.retrieve()
def process(self, roi: Slice5D = Slice5D.all(), address_mode: AddressMode = AddressMode.BLACK) -> None: defined_roi = roi.defined_with(self.data_slice) assert self.data_slice.contains(defined_roi) for piece in defined_roi.split(self.tile_shape): source_data = self.data_slice.datasource.retrieve( piece, address_mode=address_mode) self._process_tile(source_data)
def parse_known_cmdline_args(self, cmdline_args): # We use the same parser as the DataSelectionApplet parser = DataSelectionApplet.get_arg_parser( self.dataSelectionApplet.role_names) parser.add_argument( "--distributed", help= "Distributed mode. Used for running ilastik on HPCs via SLURM/srun/mpirun", action="store_true", ) default_block_roi = Slice5D.all(x=slice(0, 256), y=slice(0, 256), z=slice(0, 256), t=slice(0, 1)) def parse_distributed_block_roi(value: str) -> Slice5D: parsed = ast.literal_eval(value) if isinstance(parsed, dict): if not set(parsed.keys()).issubset("xyztc"): raise ValueError( f"Bad keys for distributed-block-roi: {value}") if not all( isinstance(v, (int, None.__class__)) for v in parsed.values()): raise ValueError( f"Bad values for distributed-block-roi: {value}") overrides = {k: slice(0, int(v)) for k, v in parsed.items()} elif isinstance(parsed, int): overrides = {k: slice(0, parsed) for k in "xyz"} else: raise TypeError( f"Could not convert value {value} into a Slice5D") return Slice5D(**{**default_block_roi.to_dict(), **overrides}) parser.add_argument( "--distributed_block_roi", "--distributed-block-roi", help=textwrap.dedent(""" Determines the dimensions of the blocks used to split the data in distributed mode. Values can be either:" An integer, which will be interpreted as if the following dict was passed in: {'x': value, 'y': value, 'z': value, 't': 1, 'c': None} or a literal python Dict[str, Optional[int]], with keys in 'xyztc'. Missing keys will default like so: {'x': 256, 'y': 256, 'z': 256, 't': 1, 'c': None} Use None anywhere in the dict to mean "the whole dimension". """), type=parse_distributed_block_roi, default=default_block_roi, ) parsed_args, unused_args = parser.parse_known_args(cmdline_args) return parsed_args, unused_args
def test_retrieve_roi_smaller_than_tile(): # fmt: off data = Array5D(np.asarray([ [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15], [16, 17, 18, 19, 20]], [[100, 200, 300, 400, 500], [600, 700, 800, 900, 1000], [1100, 1200, 1300, 1400, 1500], [1600, 1700, 1800, 1900, 2000]], ]).astype(np.uint32), axiskeys="cyx") # fmt: on path = Path(create_n5(data, chunk_size=Shape5D(c=2, y=4, x=4))) ds = DataSource.create(path) print(f"\n\n====>> tile shape: {ds.roi}") smaller_than_tile = ds.retrieve( Slice5D.all(c=1, y=slice(0, 4), x=slice(0, 4))) print(smaller_than_tile.raw("cyx"))
def test_distributed_n5_datasink(tmp_path: Path, data: Array5D, datasource: DataSource): dataset_path = tmp_path / "test_distributed_n5_datasink.n5/data" data_slice = DataSourceSlice(datasource) sinks = [ N5DataSink(path=dataset_path, data_slice=data_slice, mode=N5DataSink.Mode.CREATE), N5DataSink(path=dataset_path, data_slice=data_slice, mode=N5DataSink.Mode.OPEN), N5DataSink(path=dataset_path, data_slice=data_slice, mode=N5DataSink.Mode.OPEN), N5DataSink(path=dataset_path, data_slice=data_slice, mode=N5DataSink.Mode.OPEN), ] for idx, piece in enumerate(data_slice.split()): sink = sinks[idx % len(sinks)] sink.process(piece) n5ds = DataSource.create(dataset_path) assert n5ds.retrieve(Slice5D.all()) == data
def test_all_constructor(): slc = Slice5D.all(x=3, z=slice(10, 20)) assert slc.to_slices("xyztc") == (slice(3, 4), slice(None), slice(10, 20), slice(None), slice(None))