def from_json_data(cls, data: JsonValue, location_override: Optional[Point5D] = None ) -> "N5DatasetAttributes": raw_attributes = ensureJsonObject(data) dimensions = ensureJsonIntArray(raw_attributes.get("dimensions")) blockSize = ensureJsonIntArray(raw_attributes.get("blockSize")) axes = raw_attributes.get("axes") if axes is None: axiskeys = guess_axiskeys(dimensions) else: axiskeys = "".join(ensureJsonStringArray(axes)[::-1]).lower() location = raw_attributes.get("location") if location is None: location_5d = Point5D.zero() else: location_5d = Point5D.zero( **dict(zip(axiskeys, ensureJsonIntArray(location)[::-1]))) return N5DatasetAttributes( blockSize=Shape5D.create(raw_shape=blockSize[::-1], axiskeys=axiskeys), dimensions=Shape5D.create(raw_shape=dimensions[::-1], axiskeys=axiskeys), dataType=np.dtype(ensureJsonString(raw_attributes.get( "dataType"))).newbyteorder(">"), # type: ignore axiskeys=axiskeys, compression=N5Compressor.from_json_data( raw_attributes["compression"]), location=location_override or location_5d, )
def stack_h5s(stack_axis: str) -> List[H5DataSource]: offset = Point5D.zero() stack: List[H5DataSource] = [] for outer_path in h5_outer_paths: stack.append(H5DataSource(outer_path=outer_path, inner_path=PurePosixPath("/data"), filesystem=OsFs("/"), location=offset)) offset += Point5D.zero(**{stack_axis: stack[-1].shape[stack_axis]}) return stack
def test_data_roi_get_tiles_can_clamp_to_datasource_tiles(): # fmt: off data = Array5D(np.asarray([ [1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15], [16, 17, 18, 19, 20], ]).astype(np.uint8), axiskeys="yx") # fmt: on ds = ArrayDataSource(data=data, tile_shape=Shape5D(x=2, y=2)) data_slice = DataRoi(datasource=ds, x=(1, 4), y=(0, 3)) # fmt: off dataslice_expected_data = Array5D(np.asarray([ [2, 3, 4], [7, 8, 9], [12, 13, 14] ]).astype(np.uint8), axiskeys="yx", location=Point5D.zero(x=1)) # fmt: on assert data_slice.retrieve() == dataslice_expected_data # fmt: off dataslice_expected_slices = [ Array5D(np.asarray([ [1, 2], [6, 7] ]).astype(np.uint8), axiskeys="yx", location=Point5D.zero()), Array5D(np.asarray([ [3, 4], [8, 9], ]).astype(np.uint8), axiskeys="yx", location=Point5D.zero(x=2)), Array5D(np.asarray([ [11, 12], [16, 17], ]).astype(np.uint8), axiskeys="yx", location=Point5D.zero(y=2)), Array5D(np.asarray([ [13, 14], [18, 19], ]).astype(np.uint8), axiskeys="yx", location=Point5D.zero(x=2, y=2)) ] # fmt: on expected_slice_dict = {a.interval: a for a in dataslice_expected_slices} for piece in data_slice.get_datasource_tiles(clamp_to_datasource=True): expected_data = expected_slice_dict.pop(piece.interval) assert expected_data == piece.retrieve() assert len(expected_slice_dict) == 0
def get_sample_c_cells_pixel_annotations( override_datasource: "FsDataSource | None" = None) -> Sequence[Label]: raw_data_source = override_datasource or get_sample_c_cells_datasource() return [ Label(name="Foreground", color=Color(r=np.uint8(255), g=np.uint8(0), b=np.uint8(0)), annotations=[ Annotation.interpolate_from_points(voxels=[ Point5D.zero(x=140, y=150), Point5D.zero(x=145, y=155) ], raw_data=raw_data_source), Annotation.interpolate_from_points(voxels=[ Point5D.zero(x=238, y=101), Point5D.zero(x=229, y=139) ], raw_data=raw_data_source), ]), Label(name="Background", color=Color(r=np.uint8(255), g=np.uint8(0), b=np.uint8(0)), annotations=[ Annotation.interpolate_from_points(voxels=[ Point5D.zero(x=283, y=87), Point5D.zero(x=288, y=92) ], raw_data=raw_data_source), Annotation.interpolate_from_points(voxels=[ Point5D.zero(x=274, y=168), Point5D.zero(x=256, y=191) ], raw_data=raw_data_source), ]), ]
def __init__( self, *, path: PurePosixPath, location: Point5D = Point5D.zero(), filesystem: JsonableFilesystem, tile_shape: Optional[Shape5D] = None, spatial_resolution: Optional[Tuple[int, int, int]] = None, ): raw_data: "np.ndarray[Any, Any]" = skimage.io.imread(filesystem.openbin(path.as_posix())) # type: ignore c_axiskeys_on_disk = "yxc"[: len(raw_data.shape)] self._data = Array5D(raw_data, axiskeys=c_axiskeys_on_disk, location=location) if tile_shape is None: tile_shape = Shape5D.hypercube(256).to_interval5d().clamped(self._data.shape).shape super().__init__( c_axiskeys_on_disk=c_axiskeys_on_disk, filesystem=filesystem, path=path, dtype=self._data.dtype, interval=self._data.interval, tile_shape=tile_shape, spatial_resolution=spatial_resolution, )
def __init__(self, arr: "np.ndarray[Any, Any]", *, axiskeys: str, location: Point5D = Point5D.zero(), raw_data: DataSource): assert arr.dtype == np.dtype(bool) super().__init__(arr, axiskeys=axiskeys, location=location) if not raw_data.interval.contains(self.interval): raise AnnotationOutOfBounds(annotation_roi=self.interval, raw_data=raw_data) self.raw_data = raw_data
def test_n5_datasink(): tmp_path = create_tmp_dir(prefix="test_n5_datasink") sink = N5DatasetSink( filesystem=OsFs(tmp_path.as_posix()), outer_path=PurePosixPath("test_n5_datasink.n5"), inner_path=PurePosixPath("/data"), attributes=N5DatasetAttributes( dimensions=datasource.shape, blockSize=Shape5D(x=10, y=10), c_axiskeys=data.axiskeys, #FIXME: double check this dataType=datasource.dtype, compression=RawCompressor(), location=Point5D.zero(x=7, y=13))) sink_writer = sink.create() assert not isinstance(sink_writer, Exception) for tile in DataRoi(datasource).split(sink.tile_shape): sink_writer.write(tile.retrieve().translated(Point5D.zero(x=7, y=13))) n5ds = N5DataSource(filesystem=sink.filesystem, path=sink.full_path) saved_data = n5ds.retrieve() assert saved_data.location == Point5D.zero(x=7, y=13) assert saved_data == data
def from_json_value(cls, value: JsonValue) -> "H5DataSource": value_obj = ensureJsonObject(value) return H5DataSource(outer_path=PurePosixPath( ensureJsonString(value_obj.get("outer_path"))), inner_path=PurePosixPath( ensureJsonString(value_obj.get("inner_path"))), filesystem=JsonableFilesystem.from_json_value( value_obj.get("filesystem")), location=ensureOptional(Point5D.from_json_value, value_obj.get("location")) or Point5D.zero(), spatial_resolution=ensureJsonIntTripplet( value_obj.get("spatial_resolution")))
class FeatureExtractorCollection(FeatureExtractor): def __init__(self, extractors: Iterable[FeatureExtractor]): self.extractors = tuple(extractors) assert len(self.extractors) > 0 super().__init__() def is_applicable_to(self, datasource: DataSource) -> bool: return all(fx.is_applicable_to(datasource) for fx in self.extractors) def __call__(self, /, roi: DataRoi) -> FeatureData: assert roi.interval.c[0] == 0 feature_promises: Dict[int, Future[FeatureData]] = {} executor = get_executor(hint="feature_extraction", max_workers=len(self.extractors)) from webilastik.features.ilp_filter import IlpGaussianSmoothing feature_promises = { fx_index: executor.submit(fx, roi) for fx_index, fx in enumerate(self.extractors) if isinstance(fx, IlpGaussianSmoothing) } feature_promises.update({ fx_index: executor.submit(fx, roi) for fx_index, fx in enumerate(self.extractors) if not isinstance(fx, IlpGaussianSmoothing) }) assert len(feature_promises) == len(self.extractors) features = [ feature_promises[i].result() for i in range(len(self.extractors)) ] out = Array5D.allocate( dtype=np.dtype("float32"), interval=roi.shape.updated(c=sum(feat.shape.c for feat in features)), axiskeys="tzyxc", ).translated(roi.start) channel_offset: int = 0 for feature in features: out.set(feature.translated(Point5D.zero(c=channel_offset))) channel_offset += feature.shape.c return FeatureData(arr=out.raw(out.axiskeys), axiskeys=out.axiskeys, location=out.location)
def __init__( self, *, dimensions: Shape5D, blockSize: Shape5D, axiskeys: str, dataType: np.dtype, compression: N5Compressor, location: Point5D = Point5D.zero(), ): """axiskeys follows ndstructs conventions (c-order), despite 'axes' in N5 datasets being F-order""" self.dimensions = dimensions self.blockSize = blockSize self.axiskeys = axiskeys self.dataType = dataType self.compression = compression self.location = location self.interval = self.dimensions.to_interval5d(self.location)
def __init__(self, *, outer_path: PurePosixPath, inner_path: PurePosixPath, location: Point5D = Point5D.zero(), filesystem: JsonableFilesystem, spatial_resolution: Optional[Tuple[int, int, int]] = None): self.outer_path = outer_path self.inner_path = inner_path self.filesystem = filesystem binfile = filesystem.openbin(outer_path.as_posix()) # FIXME: h5py might not like this if the filesystem isn't OSFS f = h5py.File(binfile, "r") #type: ignore try: dataset = f[inner_path.as_posix()] if not isinstance(dataset, h5py.Dataset): raise ValueError(f"{inner_path} is not a Dataset") self.axiskeys = self.getAxisKeys(dataset) self._dataset = dataset tile_shape = Shape5D.create(raw_shape=self._dataset.chunks or self._dataset.shape, axiskeys=self.axiskeys) base_url = Url.parse(filesystem.geturl(outer_path.as_posix())) assert base_url is not None super().__init__( c_axiskeys_on_disk=self.axiskeys, tile_shape=tile_shape, interval=Shape5D.create( raw_shape=self._dataset.shape, axiskeys=self.axiskeys).to_interval5d(location), dtype=self._dataset.dtype, spatial_resolution=spatial_resolution or (1, 1, 1), # FIXME filesystem=filesystem, path=self.outer_path) except Exception as e: f.close() raise e
def __init__(self, arr: "np.ndarray[Any, np.dtype[np.float32]]", axiskeys: str, location: Point5D = Point5D.zero()): super().__init__(arr, axiskeys=axiskeys, location=location) assert self.dtype == np.dtype('float32')
Point5D(x=2191, y=1329, z=0), Point5D(x=2192, y=1329, z=0), Point5D(x=2192, y=1328, z=0), Point5D(x=2193, y=1328, z=0), Point5D(x=2194, y=1328, z=0), Point5D(x=2194, y=1327, z=0), ], raw_data=mouse_datasources[0]), ] elif args.datasource == "c_cells": datasource = SkimageDataSource(filesystem=OsFs( Path(__file__).joinpath("../../public/images/").as_posix()), path=PurePosixPath("c_cells_1.png")) class1_annotations = [ Annotation.interpolate_from_points(voxels=[ Point5D.zero(x=140, y=150), Point5D.zero(x=145, y=155) ], raw_data=datasource), Annotation.interpolate_from_points(voxels=[ Point5D.zero(x=238, y=101), Point5D.zero(x=229, y=139) ], raw_data=datasource), ] class_2_annotations = [ Annotation.interpolate_from_points( voxels=[Point5D.zero(x=283, y=87), Point5D.zero(x=288, y=92)], raw_data=datasource), Annotation.interpolate_from_points(voxels=[
def __setstate__(self, value_obj: JsonObject): self.__init__( path=PurePosixPath(ensureJsonString(value_obj.get("path"))), location=ensureOptional(Point5D.from_json_value, value_obj.get("location")) or Point5D.zero(), filesystem=JsonableFilesystem.from_json_value(value_obj.get("filesystem")), tile_shape=ensureOptional(Shape5D.from_json_value, value_obj.get("tile_shape")), spatial_resolution=ensureOptional(ensureJsonIntTripplet, value_obj.get("spatial_resolution")), )
def from_json_value(cls, value: JsonValue) -> "SkimageDataSource": value_obj = ensureJsonObject(value) return SkimageDataSource( path=PurePosixPath(ensureJsonString(value_obj.get("path"))), location=ensureOptional(Point5D.from_json_value, value_obj.get("location")) or Point5D.zero(), filesystem=JsonableFilesystem.from_json_value(value_obj.get("filesystem")), tile_shape=ensureOptional(Shape5D.from_json_value, value_obj.get("tile_shape")), spatial_resolution=ensureOptional(ensureJsonIntTripplet, value_obj.get("spatial_resolution")), )
def __init__( self, arr: "np.ndarray[Any, Any]", *, axiskeys: str, location: Point5D = Point5D.zero(), labels: Optional[Set[int]] = None ): super().__init__(arr, axiskeys=axiskeys, location=location) self._border_colors: Optional[Set[int]] = None self._labels = labels