def test_paint_point(): # fmt: off img = Array5D(numpy.asarray([ [[100, 0, 0, 100], [ 0, 17, 0, 0], [ 0, 0, 17, 0], [ 0, 0, 0, 0]], [[200, 0, 0, 200], [ 0, 40, 0, 0], [ 0, 0, 40, 0], [ 0, 0, 0, 0]] ]), axiskeys="cyx") # fmt: on # fmt: off expected_painted = Array5D( numpy.asarray( [ [[107, 0, 0, 100], [0, 17, 0, 0], [0, 0, 17, 0], [0, 0, 0, 0]], [[200, 0, 0, 200], [0, 40, 123, 0], [0, 0, 40, 0], [0, 0, 0, 0]], ] ), axiskeys="cyx", ) # fmt: on img.paint_point(Point5D.zero(c=0, y=0, x=0), value=107) img.paint_point(Point5D.zero(c=1, y=1, x=2), value=123) assert img == expected_painted
def test_combine(): # fmt: off arr = Array5D(numpy.asarray([ [7, 7, 0, 0, 0, 0], [7, 7, 0, 0, 0, 0], [7, 0, 0, 0, 0, 0], [0, 0, 0, 3, 0, 0], [0, 0, 3, 3, 3, 0], [0, 0, 0, 3, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 5, 5, 0, 0]]), axiskeys="yx") piece1 = Array5D(numpy.asarray([ [7, 7,], [7, 7,], [7, 0,]]), axiskeys="yx", location=Point5D.zero()) piece2 = Array5D(numpy.asarray([ [0, 3, 0, 0], [3, 3, 3, 0], [0, 3, 0, 0]]), axiskeys="yx", location=Point5D.zero(y=3, x=2)) piece3 = Array5D(numpy.asarray([ [5, 5]]), axiskeys="yx", location=Point5D.zero(y=8, x=2)) # fmt: on combined = piece1.combine([piece2, piece3]) assert (combined.raw("yx") == arr.raw("yx")).all()
def test_slice_enclosing(): p1 = Point5D.zero(x=-13, y=40) p2 = Point5D.zero(z=-1, c=6) p3 = Point5D.zero(t=3, x=4) p4 = Point5D.zero(t=100, y=400) expected_slice = Slice5D(x=slice(-13, 4 + 1), y=slice(40, 400 + 1), z=slice(-1, -1 + 1), c=slice(6, 6 + 1)) assert Slice5D.enclosing([p1, p2, p3, p4])
def test_allocation(): arr = Array5D.allocate(Slice5D.zero(x=slice(100, 200), y=slice(200, 300)), numpy.uint8) assert arr.shape == Shape5D(x=100, y=100) assert arr.location == Point5D.zero(x=100, y=200) arr = Array5D.allocate(Slice5D.zero(x=slice(-100, 200), y=slice(200, 300)), numpy.uint8) assert arr.shape == Shape5D(x=300, y=100) assert arr.location == Point5D.zero(x=-100, y=200)
def test_data_roi_get_tiles_can_clamp_to_datasource_tiles(): # fmt: off data = Array5D(np.asarray([ [1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15], [16, 17, 18, 19, 20], ]).astype(np.uint8), axiskeys="yx") # fmt: on ds = ArrayDataSource.from_array5d(data, tile_shape=Shape5D(x=2, y=2)) data_slice = DataRoi(datasource=ds, x=(1, 4), y=(0, 3)) # fmt: off dataslice_expected_data = Array5D(np.asarray([[2, 3, 4], [7, 8, 9], [12, 13, 14]]).astype(np.uint8), axiskeys="yx", location=Point5D.zero(x=1)) # fmt: on assert data_slice.retrieve() == dataslice_expected_data # fmt: off dataslice_expected_slices = [ Array5D(np.asarray([[1, 2], [6, 7]]).astype(np.uint8), axiskeys="yx", location=Point5D.zero()), Array5D(np.asarray([ [3, 4], [8, 9], ]).astype(np.uint8), axiskeys="yx", location=Point5D.zero(x=2)), Array5D(np.asarray([ [11, 12], [16, 17], ]).astype(np.uint8), axiskeys="yx", location=Point5D.zero(y=2)), Array5D(np.asarray([ [13, 14], [18, 19], ]).astype(np.uint8), axiskeys="yx", location=Point5D.zero(x=2, y=2)) ] # fmt: on expected_slice_dict = {a.interval: a for a in dataslice_expected_slices} for piece in data_slice.get_datasource_tiles(clamp_to_datasource=True): expected_data = expected_slice_dict.pop(piece.interval) assert expected_data == piece.retrieve() assert len(expected_slice_dict) == 0
def test_setting_rois(): # fmt: off raw = numpy.asarray([ [[1, 2, 3, 4, 5 ], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15], [16, 17, 18, 19, 20]], [[-1, -2, -3, -4, -5 ], [-6, -7, -8, -9, -10], [-11, -12, -13, -14, -15], [-16, -17, -18, -19, -20]], ]) piece = Array5D(numpy.asarray([ [100, 200], [300, 400] ]), "yx", location=Point5D.zero(x=2, y=1, c=1)) expected_cyx_raw_with_piece = numpy.asarray([ [[1, 2, 3, 4, 5 ], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15], [16, 17, 18, 19, 20]], [[-1, -2, -3, -4, -5 ], [-6, -7, 100, 200, -10], [-11, -12, 300, 400, -15], [-16, -17, -18, -19, -20]], ]) extrapolating_piece = Array5D(numpy.asarray([ [111, 222, 333], [444, 555, 6661] ]), "yx", location=Point5D.zero(x=3, y=2, c=0)) expected_cyx_raw_with_extrapolating_piece = numpy.asarray([ [[1, 2, 3, 4, 5 ], [6, 7, 8, 9, 10], [11, 12, 13, 111, 222], [16, 17, 18, 444, 555]], [[-1, -2, -3, -4, -5 ], [-6, -7, 100, 200, -10], [-11, -12, 300, 400, -15], [-16, -17, -18, -19, -20]], ]) # fmt: on arr = Array5D(raw, "cyx") arr.set(piece) assert (arr.raw("cyx") == expected_cyx_raw_with_piece).all() arr.set(extrapolating_piece, autocrop=True) assert (arr.raw("cyx") == expected_cyx_raw_with_extrapolating_piece).all()
def stack_h5s(stack_axis: str) -> List[H5DataSource]: offset = Point5D.zero() stack: List[H5DataSource] = [] for outer_path in h5_outer_paths: stack.append( H5DataSource(outer_path=outer_path, inner_path=PurePosixPath("/data"), filesystem=OsFs("/"), location=offset)) offset += Point5D.zero(**{stack_axis: stack[-1].shape[stack_axis]}) return stack
def __init__(self, *, outer_path: Path, inner_path: PurePosixPath, location: Point5D = Point5D.zero(), filesystem: JsonableFilesystem): self.outer_path = outer_path self.inner_path = inner_path self.filesystem = filesystem binfile = filesystem.openbin(outer_path.as_posix()) f = h5py.File(binfile, "r") try: dataset = f[inner_path.as_posix()] if not isinstance(dataset, h5py.Dataset): raise ValueError(f"{inner_path} is not a Dataset") axiskeys = self.getAxisKeys(dataset) self._dataset = cast(h5py.Dataset, dataset) tile_shape = Shape5D.create(raw_shape=self._dataset.chunks or self._dataset.shape, axiskeys=axiskeys) super().__init__( tile_shape=tile_shape, interval=Shape5D.create( raw_shape=self._dataset.shape, axiskeys=axiskeys).to_interval5d(location), dtype=self._dataset.dtype, axiskeys=axiskeys, ) except Exception as e: f.close() raise e
def __init__(self, arr: np.ndarray, axiskeys: str, channel_colors: Sequence[Color], location: Point5D = Point5D.zero()): super().__init__(arr, axiskeys, location=location) self.channel_colors = tuple(channel_colors)
def __init__(self, path: Path, *, location: Point5D = Point5D.zero(), filesystem: FS): self._dataset: Optional[h5py.Dataset] = None try: self._dataset, outer_path, inner_path = self.openDataset( path, filesystem=filesystem) axiskeys = self.getAxisKeys(self._dataset) tile_shape = Shape5D.create(raw_shape=self._dataset.chunks or self._dataset.shape, axiskeys=axiskeys) super().__init__( url=filesystem.desc(outer_path.as_posix()) + "/" + inner_path.as_posix(), tile_shape=tile_shape, shape=Shape5D.create(raw_shape=self._dataset.shape, axiskeys=axiskeys), dtype=self._dataset.dtype, name=self._dataset.file.filename.split("/")[-1] + self._dataset.name, location=location, axiskeys=axiskeys, ) except Exception as e: if self._dataset: self._dataset.file.close() raise e
def __init__(self, path: Path, *, location: Point5D = Point5D.zero(), filesystem: FS): url = filesystem.geturl(path.as_posix()) match = re.search(r"[^/]+\.n5/.*$", url, re.IGNORECASE) if not match: raise UnsupportedUrlException(url) name = match.group(0) self.filesystem = filesystem.opendir(path.as_posix()) with self.filesystem.openbin("attributes.json", "r") as f: attributes_json_bytes = f.read() attributes = json.loads(attributes_json_bytes.decode("utf8")) dimensions = attributes["dimensions"][::-1] blockSize = attributes["blockSize"][::-1] axiskeys = "".join(attributes["axes"]).lower( )[::-1] if "axes" in attributes else guess_axiskeys(dimensions) super().__init__( url=url, name=name, tile_shape=Shape5D.create(raw_shape=blockSize, axiskeys=axiskeys), shape=Shape5D.create(raw_shape=dimensions, axiskeys=axiskeys), dtype=np.dtype(attributes["dataType"]).newbyteorder(">"), location=location, axiskeys=axiskeys, ) self.compression_type = attributes["compression"]["type"] if self.compression_type not in N5Block.DECOMPRESSORS.keys(): raise NotImplementedError( f"Don't know how to decompress from {self.compression_type}")
def __init__(self, path: Path, *, location: Point5D = Point5D.zero(), filesystem: FS): try: raw_data = skimage.io.imread(filesystem.openbin(path.as_posix())) except ValueError: raise UnsupportedUrlException(path) axiskeys = "yxc"[: len(raw_data.shape)] super().__init__(url=filesystem.desc(path.as_posix()), data=raw_data, axiskeys=axiskeys, location=location)
def test_cut(): # fmt: off raw = numpy.asarray([ [1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15], [16, 17, 18, 19, 20], ]) expected_piece = numpy.asarray([ [2, 3], [7, 8], [12, 13], [17, 18] ]) expected_global_sub_piece = numpy.asarray([ [3], [8], [13], [18] ]) # fmt: on arr = Array5D(raw, "zy") piece = arr.cut(Slice5D(y=slice(1, 3))) assert (piece.raw("zy") == expected_piece).all() assert piece.location == Point5D.zero(y=1) global_sub_piece = piece.cut(Slice5D(y=2)) assert (global_sub_piece.raw("zy") == expected_global_sub_piece).all() local_sub_piece = piece.local_cut(Slice5D(y=1)) assert (local_sub_piece.raw("zy") == global_sub_piece.raw("zy")).all()
def test_zero_factory_method_defaults_coords_to_zero(): p = Point5D.zero(x=123, c=456) assert p.x == 123 assert p.y == 0 assert p.z == 0 assert p.t == 0 assert p.c == 456
def create(cls, path: Path, *, location: Point5D = Point5D.zero(), filesystem: Optional[FS] = None) -> "DataSource": filesystem = filesystem or OSFS(path.anchor) for klass in cls.REGISTRY if cls == DataSource else [cls]: try: return klass(path, location=location, filesystem=filesystem) except UnsupportedUrlException: pass raise UnsupportedUrlException(path)
def __init__( self, paths: List[Path], *, stack_axis: str, layer_axiskeys: Union[str, Sequence[str]] = "", location: Point5D = Point5D.zero(), filesystems: Sequence[FS] = (), ): layer_axiskeys = layer_axiskeys or [""] * len(paths) assert len(layer_axiskeys) == len(paths) self.stack_axis = stack_axis self.layers: List[DataSource] = [] self.layer_offsets: List[int] = [] layer_offset = Point5D.zero() for layer_path, layer_fs in itertools.zip_longest(paths, filesystems): layer = DataSource.create(layer_path, location=layer_offset, filesystem=layer_fs) self.layers.append(layer) self.layer_offsets.append(layer_offset[stack_axis]) layer_offset += Point5D.zero( **{stack_axis: layer.shape[stack_axis]}) if len( set( layer.shape.with_coord(**{stack_axis: 1}) for layer in self.layers)) > 1: raise ValueError( "Provided files have different dimensions on the non-stacking axis" ) if any(layer.dtype != self.layers[0].dtype for layer in self.layers): raise ValueError("All layers must have the same data type!") stack_size = sum(layer.shape[self.stack_axis] for layer in self.layers) full_shape = self.layers[0].shape.with_coord( **{self.stack_axis: stack_size}) super().__init__( url=":".join(p.as_posix() for p in paths), shape=full_shape, name="Stack from " + ":".join(p.name for p in paths), dtype=self.layers[0].dtype, location=location, axiskeys=stack_axis + Point5D.LABELS.replace(stack_axis, ""), )
def test_n5_datasink(tmp_path: Path, data: Array5D, datasource: DataSource): sink = N5DatasetSink.create(filesystem=OsFs(tmp_path.as_posix()), outer_path=Path("test_n5_datasink.n5"), inner_path=PurePosixPath("/data"), attributes=N5DatasetAttributes( dimensions=datasource.shape, blockSize=Shape5D(x=10, y=10), axiskeys=datasource.axiskeys, dataType=datasource.dtype, compression=RawCompressor(), location=Point5D.zero(x=7, y=13))) for tile in DataRoi(datasource).split(sink.tile_shape): sink.write(tile.retrieve().translated(Point5D.zero(x=7, y=13))) n5ds = N5DataSource(filesystem=sink.filesystem, path=sink.path) saved_data = n5ds.retrieve() assert saved_data.location == Point5D.zero(x=7, y=13) assert saved_data == data
def show(self): data = self.data_tile.retrieve().cut(copy=True) for axis in "xyz": increment = Point5D.zero(**{axis: 1}) for pos in (self.position + increment, self.position - increment): if data.interval.contains(Interval5D.enclosing([pos])): data.paint_point(pos, 0) data.paint_point(self.position, 255) data.show_images()
def from_array5d(cls, arr: Array5D, *, tile_shape: Optional[Shape5D] = None, location: Point5D = Point5D.zero()): return cls(data=arr.raw(Point5D.LABELS), axiskeys=Point5D.LABELS, location=location, tile_shape=tile_shape)
def test_point_arithmetic(): p = Point5D(x=100, y=200, z=300, t=400, c=500) assert p + Point5D.zero(x=100) == Point5D(x=200, y=200, z=300, t=400, c=500) assert p + Point5D.inf(x=100) == Point5D.inf(x=200) assert p + Point5D(x=1, y=2, z=3, t=4, c=5) == Point5D(x=101, y=202, z=303, t=404, c=505) other = Point5D(x=1, y=2, z=3, t=4, c=5) for op in ("__add__", "__sub__", "__mul__", "__floordiv__"): p_as_np = p.to_np(Point5D.LABELS) np_result = getattr(p_as_np, op)(other.to_np(Point5D.LABELS)) assert all(getattr(p, op)(other).to_np(Point5D.LABELS) == np_result)
def from_json_value(cls, value: JsonValue) -> "SkimageDataSource": value_obj = ensureJsonObject(value) raw_location = value_obj.get("location") raw_tile_shape = value_obj.get("tile_shape") return SkimageDataSource( path=Path(ensureJsonString(value_obj.get("path"))), location=Point5D.zero() if raw_location is None else Point5D.from_json_value(raw_location), filesystem=JsonableFilesystem.from_json_value( value_obj.get("filesystem")), tile_shape=None if raw_tile_shape is None else Shape5D.from_json_value(raw_tile_shape))
def from_json_value(cls, value: JsonValue) -> "H5DataSource": value_obj = ensureJsonObject(value) raw_location = value_obj.get("location") return H5DataSource( outer_path=Path(ensureJsonString(value_obj.get("outer_path"))), inner_path=PurePosixPath( ensureJsonString(value_obj.get("inner_path"))), filesystem=JsonableFilesystem.from_json_value( value_obj.get("filesystem")), location=Point5D.zero() if raw_location is None else Point5D.from_json_value(raw_location), )
def compute(self, roi: DataRoi) -> FeatureData: features: List[FeatureData] = [] channel_offset: int = 0 for fx in self.extractors: result = fx.compute(roi).translated(Point5D.zero(c=channel_offset)) features.append(result) channel_offset += result.shape.c out = Array5D.combine(features) return FeatureData( arr=out.raw(out.axiskeys), axiskeys=out.axiskeys, location=out.location )
def get_neighboring_tiles(self, tile_shape: Shape5D = None ) -> Iterator["DataSourceSlice"]: if not self.is_defined(): return self.defined().get_neighboring_tiles(tile_shape=tile_shape) tile_shape = tile_shape or self.tile_shape assert self.is_tile(tile_shape) for axis in Point5D.LABELS: for axis_offset in (tile_shape[axis], -tile_shape[axis]): offset = Point5D.zero(**{axis: axis_offset}) neighbor = self.translated(offset).clamped(self.full()) if neighbor.shape.hypervolume > 0 and neighbor != self: yield neighbor
def __init__(self, arr: np.ndarray, *, axiskeys: str, location: Point5D = Point5D.zero(), color: Color, raw_data: DataSource): super().__init__(arr.astype(bool), axiskeys=axiskeys, location=location) if not raw_data.interval.contains(self.interval): raise AnnotationOutOfBounds(annotation_roi=self.interval, raw_data=raw_data) self.color = color self.raw_data = raw_data
def __init__(self, path: Path, *, location: Point5D = Point5D.zero(), filesystem: JsonableFilesystem, tile_shape: Optional[Shape5D] = None): self.path = path self.filesystem = filesystem raw_data: np.ndarray = skimage.io.imread( filesystem.openbin(path.as_posix())) # type: ignore axiskeys = "yxc"[:len(raw_data.shape)] super().__init__( data=raw_data, axiskeys=axiskeys, location=location, tile_shape=tile_shape, )
def __init__( self, url: str = "", *, data: np.ndarray, axiskeys: str, tile_shape: Optional[Shape5D] = None, location: Point5D = Point5D.zero(), ): self._data = Array5D(data, axiskeys=axiskeys) super().__init__( url=url or "memory://{id(data)}]", shape=self._data.shape, dtype=self._data.dtype, tile_shape=tile_shape, location=location, axiskeys=axiskeys, )
def __init__( self, *, data: np.ndarray, axiskeys: str, tile_shape: Optional[Shape5D] = None, location: Point5D = Point5D.zero(), ): self._data = Array5D(data, axiskeys=axiskeys, location=location) if tile_shape is None: tile_shape = Shape5D.hypercube(256).to_interval5d().clamped( self._data.shape).shape super().__init__( dtype=self._data.dtype, tile_shape=tile_shape, interval=self._data.interval, axiskeys=axiskeys, )
def __init__( self, url: str, *, tile_shape: Optional[Shape5D] = None, dtype: np.dtype, name: str = "", shape: Shape5D, location: Point5D = Point5D.zero(), axiskeys: str, ): self.url = url self.tile_shape = (tile_shape or Shape5D.hypercube(256)).to_slice_5d().clamped(shape.to_slice_5d()).shape self.dtype = dtype self.name = name or self.url.split("/")[-1] self.shape = shape self.roi = shape.to_slice_5d(offset=location) self.location = location self.axiskeys = axiskeys
def __init__(self, path: Path, *, location: Point5D = Point5D.zero(), chunk_size: Optional[Shape5D] = None, filesystem: FS): """A DataSource that handles Neurogancer's precomputed chunks path: a path all the pay down to the scale, i.e., if some scale has "key": "my_scale" then your path should end in "my_scale" chunk_size: a valid chunk_size for the scale selected by 'path' """ self.filesystem = filesystem.opendir(path.parent.as_posix()) self.info = PrecomputedChunksInfo.load(path=Path("info"), filesystem=self.filesystem) self.scale = self.info.get_scale(key=path.name) super().__init__( url="precomputed://" + filesystem.desc(path.as_posix()), tile_shape=self.scale.get_tile_shape_5d( self.info.num_channels, tile_shape_hint=chunk_size), shape=self.scale.get_shape_5d(self.info.num_channels), dtype=self.info.data_type, name=path.name, location=location, axiskeys=self.scale. axiskeys[:: -1], # externally reported axiskeys are always c-ordered ) encoding_type = self.scale.encoding if encoding_type == "raw": noop = lambda data: data self.decompressor = noop self.compressor = noop else: raise NotImplementedError( f"Don't know how to decompress {compression_type}")