def stack_h5s(stack_axis: str) -> List[H5DataSource]: offset = Point5D.zero() stack: List[H5DataSource] = [] for outer_path in h5_outer_paths: stack.append(H5DataSource(outer_path=outer_path, inner_path=PurePosixPath("/data"), filesystem=OsFs("/"), location=offset)) offset += Point5D.zero(**{stack_axis: stack[-1].shape[stack_axis]}) return stack
def interpolate_from_points(cls, voxels: Sequence[Point5D], raw_data: DataSource): start = Point5D.min_coords(voxels) stop = Point5D.max_coords( voxels ) + 1 # +1 because slice.stop is exclusive, but max_point isinclusive scribbling_roi = Interval5D.create_from_start_stop(start=start, stop=stop) if scribbling_roi.shape.c != 1: raise ValueError( f"Annotations must not span multiple channels: {voxels}") scribblings = Array5D.allocate(scribbling_roi, dtype=np.dtype(bool), value=False) anchor = voxels[0] for voxel in voxels: for interp_voxel in anchor.interpolate_until(voxel): scribblings.paint_point(point=interp_voxel, value=True) anchor = voxel return cls(scribblings._data, axiskeys=scribblings.axiskeys, raw_data=raw_data, location=start)
def from_json_data(cls, data: JsonValue, location_override: Optional[Point5D] = None ) -> "N5DatasetAttributes": raw_attributes = ensureJsonObject(data) dimensions = ensureJsonIntArray(raw_attributes.get("dimensions")) blockSize = ensureJsonIntArray(raw_attributes.get("blockSize")) axes = raw_attributes.get("axes") if axes is None: axiskeys = guess_axiskeys(dimensions) else: axiskeys = "".join(ensureJsonStringArray(axes)[::-1]).lower() location = raw_attributes.get("location") if location is None: location_5d = Point5D.zero() else: location_5d = Point5D.zero( **dict(zip(axiskeys, ensureJsonIntArray(location)[::-1]))) return N5DatasetAttributes( blockSize=Shape5D.create(raw_shape=blockSize[::-1], axiskeys=axiskeys), dimensions=Shape5D.create(raw_shape=dimensions[::-1], axiskeys=axiskeys), dataType=np.dtype(ensureJsonString(raw_attributes.get( "dataType"))).newbyteorder(">"), # type: ignore axiskeys=axiskeys, compression=N5Compressor.from_json_data( raw_attributes["compression"]), location=location_override or location_5d, )
def test_data_roi_get_tiles_can_clamp_to_datasource_tiles(): # fmt: off data = Array5D(np.asarray([ [1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15], [16, 17, 18, 19, 20], ]).astype(np.uint8), axiskeys="yx") # fmt: on ds = ArrayDataSource(data=data, tile_shape=Shape5D(x=2, y=2)) data_slice = DataRoi(datasource=ds, x=(1, 4), y=(0, 3)) # fmt: off dataslice_expected_data = Array5D(np.asarray([ [2, 3, 4], [7, 8, 9], [12, 13, 14] ]).astype(np.uint8), axiskeys="yx", location=Point5D.zero(x=1)) # fmt: on assert data_slice.retrieve() == dataslice_expected_data # fmt: off dataslice_expected_slices = [ Array5D(np.asarray([ [1, 2], [6, 7] ]).astype(np.uint8), axiskeys="yx", location=Point5D.zero()), Array5D(np.asarray([ [3, 4], [8, 9], ]).astype(np.uint8), axiskeys="yx", location=Point5D.zero(x=2)), Array5D(np.asarray([ [11, 12], [16, 17], ]).astype(np.uint8), axiskeys="yx", location=Point5D.zero(y=2)), Array5D(np.asarray([ [13, 14], [18, 19], ]).astype(np.uint8), axiskeys="yx", location=Point5D.zero(x=2, y=2)) ] # fmt: on expected_slice_dict = {a.interval: a for a in dataslice_expected_slices} for piece in data_slice.get_datasource_tiles(clamp_to_datasource=True): expected_data = expected_slice_dict.pop(piece.interval) assert expected_data == piece.retrieve() assert len(expected_slice_dict) == 0
def get_sample_c_cells_pixel_annotations( override_datasource: "FsDataSource | None" = None) -> Sequence[Label]: raw_data_source = override_datasource or get_sample_c_cells_datasource() return [ Label(name="Foreground", color=Color(r=np.uint8(255), g=np.uint8(0), b=np.uint8(0)), annotations=[ Annotation.interpolate_from_points(voxels=[ Point5D.zero(x=140, y=150), Point5D.zero(x=145, y=155) ], raw_data=raw_data_source), Annotation.interpolate_from_points(voxels=[ Point5D.zero(x=238, y=101), Point5D.zero(x=229, y=139) ], raw_data=raw_data_source), ]), Label(name="Background", color=Color(r=np.uint8(255), g=np.uint8(0), b=np.uint8(0)), annotations=[ Annotation.interpolate_from_points(voxels=[ Point5D.zero(x=283, y=87), Point5D.zero(x=288, y=92) ], raw_data=raw_data_source), Annotation.interpolate_from_points(voxels=[ Point5D.zero(x=274, y=168), Point5D.zero(x=256, y=191) ], raw_data=raw_data_source), ]), ]
def __init__( self, *, filesystem: JsonableFilesystem, info_dir: PurePosixPath, #path to dir containing with the info file scale: PrecomputedChunksScale, dtype: "np.dtype[Any]", num_channels: int, ): self.info_dir = info_dir self.scale = scale shape = Shape5D(x=scale.size[0], y=scale.size[1], z=scale.size[2], c=num_channels) location = Point5D(x=scale.voxel_offset[0], y=scale.voxel_offset[1], z=scale.voxel_offset[2]) interval = shape.to_interval5d(offset=location) chunk_sizes_5d = [ Shape5D(x=cs[0], y=cs[1], z=cs[2], c=num_channels) for cs in scale.chunk_sizes ] super().__init__( filesystem=filesystem, path=info_dir, tile_shape=chunk_sizes_5d[0], #FIXME interval=interval, dtype=dtype, )
def __init__( self, *, key: PurePosixPath, size: Tuple[int, int, int], resolution: Tuple[int, int, int], voxel_offset: Tuple[int, int, int], chunk_sizes: Tuple[Tuple[int, int, int], ...], encoding: PrecomputedChunksEncoder, num_channels: int, ): super().__init__(key=key, size=size, resolution=resolution, voxel_offset=voxel_offset, chunk_sizes=chunk_sizes, encoding=encoding) self.num_channels = num_channels self.shape = Shape5D(x=self.size[0], y=self.size[1], z=self.size[2], c=num_channels) self.location = Point5D(x=self.voxel_offset[0], y=self.voxel_offset[1], z=self.voxel_offset[2]) self.interval = self.shape.to_interval5d(offset=self.location) self.chunk_sizes_5d = [ Shape5D(x=cs[0], y=cs[1], z=cs[2], c=num_channels) for cs in self.chunk_sizes ]
def __init__( self, *, path: PurePosixPath, location: Point5D = Point5D.zero(), filesystem: JsonableFilesystem, tile_shape: Optional[Shape5D] = None, spatial_resolution: Optional[Tuple[int, int, int]] = None, ): raw_data: "np.ndarray[Any, Any]" = skimage.io.imread(filesystem.openbin(path.as_posix())) # type: ignore c_axiskeys_on_disk = "yxc"[: len(raw_data.shape)] self._data = Array5D(raw_data, axiskeys=c_axiskeys_on_disk, location=location) if tile_shape is None: tile_shape = Shape5D.hypercube(256).to_interval5d().clamped(self._data.shape).shape super().__init__( c_axiskeys_on_disk=c_axiskeys_on_disk, filesystem=filesystem, path=path, dtype=self._data.dtype, interval=self._data.interval, tile_shape=tile_shape, spatial_resolution=spatial_resolution, )
def from_json_value(cls, value: JsonValue) -> "N5DataSource": value_obj = ensureJsonObject(value) raw_location = value_obj.get("location") return N5DataSource( path=PurePosixPath(ensureJsonString(value_obj.get("path"))), filesystem=JsonableFilesystem.from_json_value(value_obj.get("filesystem")), location=raw_location if raw_location is None else Point5D.from_json_value(raw_location), spatial_resolution=ensureOptional(ensureJsonIntTripplet, value_obj.get("spatial_resolution")), )
def from_json_value(cls, data: JsonValue) -> "Annotation": data_dict = ensureJsonObject(data) raw_voxels = ensureJsonArray(data_dict.get("voxels")) voxels: Sequence[Point5D] = [ Point5D.from_json_value(raw_voxel) for raw_voxel in raw_voxels ] raw_data = DataSource.from_json_value(data_dict.get("raw_data")) return cls.from_voxels(voxels=voxels, raw_data=raw_data)
def __init__(self, arr: "np.ndarray[Any, Any]", *, axiskeys: str, location: Point5D = Point5D.zero(), raw_data: DataSource): assert arr.dtype == np.dtype(bool) super().__init__(arr, axiskeys=axiskeys, location=location) if not raw_data.interval.contains(self.interval): raise AnnotationOutOfBounds(annotation_roi=self.interval, raw_data=raw_data) self.raw_data = raw_data
def from_json_value(cls, value: JsonValue) -> "H5DataSource": value_obj = ensureJsonObject(value) return H5DataSource(outer_path=PurePosixPath( ensureJsonString(value_obj.get("outer_path"))), inner_path=PurePosixPath( ensureJsonString(value_obj.get("inner_path"))), filesystem=JsonableFilesystem.from_json_value( value_obj.get("filesystem")), location=ensureOptional(Point5D.from_json_value, value_obj.get("location")) or Point5D.zero(), spatial_resolution=ensureJsonIntTripplet( value_obj.get("spatial_resolution")))
def test_n5_datasink(): tmp_path = create_tmp_dir(prefix="test_n5_datasink") sink = N5DatasetSink( filesystem=OsFs(tmp_path.as_posix()), outer_path=PurePosixPath("test_n5_datasink.n5"), inner_path=PurePosixPath("/data"), attributes=N5DatasetAttributes( dimensions=datasource.shape, blockSize=Shape5D(x=10, y=10), c_axiskeys=data.axiskeys, #FIXME: double check this dataType=datasource.dtype, compression=RawCompressor(), location=Point5D.zero(x=7, y=13))) sink_writer = sink.create() assert not isinstance(sink_writer, Exception) for tile in DataRoi(datasource).split(sink.tile_shape): sink_writer.write(tile.retrieve().translated(Point5D.zero(x=7, y=13))) n5ds = N5DataSource(filesystem=sink.filesystem, path=sink.full_path) saved_data = n5ds.retrieve() assert saved_data.location == Point5D.zero(x=7, y=13) assert saved_data == data
class FeatureExtractorCollection(FeatureExtractor): def __init__(self, extractors: Iterable[FeatureExtractor]): self.extractors = tuple(extractors) assert len(self.extractors) > 0 super().__init__() def is_applicable_to(self, datasource: DataSource) -> bool: return all(fx.is_applicable_to(datasource) for fx in self.extractors) def __call__(self, /, roi: DataRoi) -> FeatureData: assert roi.interval.c[0] == 0 feature_promises: Dict[int, Future[FeatureData]] = {} executor = get_executor(hint="feature_extraction", max_workers=len(self.extractors)) from webilastik.features.ilp_filter import IlpGaussianSmoothing feature_promises = { fx_index: executor.submit(fx, roi) for fx_index, fx in enumerate(self.extractors) if isinstance(fx, IlpGaussianSmoothing) } feature_promises.update({ fx_index: executor.submit(fx, roi) for fx_index, fx in enumerate(self.extractors) if not isinstance(fx, IlpGaussianSmoothing) }) assert len(feature_promises) == len(self.extractors) features = [ feature_promises[i].result() for i in range(len(self.extractors)) ] out = Array5D.allocate( dtype=np.dtype("float32"), interval=roi.shape.updated(c=sum(feat.shape.c for feat in features)), axiskeys="tzyxc", ).translated(roi.start) channel_offset: int = 0 for feature in features: out.set(feature.translated(Point5D.zero(c=channel_offset))) channel_offset += feature.shape.c return FeatureData(arr=out.raw(out.axiskeys), axiskeys=out.axiskeys, location=out.location)
def __init__( self, *, dimensions: Shape5D, blockSize: Shape5D, axiskeys: str, dataType: np.dtype, compression: N5Compressor, location: Point5D = Point5D.zero(), ): """axiskeys follows ndstructs conventions (c-order), despite 'axes' in N5 datasets being F-order""" self.dimensions = dimensions self.blockSize = blockSize self.axiskeys = axiskeys self.dataType = dataType self.compression = compression self.location = location self.interval = self.dimensions.to_interval5d(self.location)
def __init__(self, *, outer_path: PurePosixPath, inner_path: PurePosixPath, location: Point5D = Point5D.zero(), filesystem: JsonableFilesystem, spatial_resolution: Optional[Tuple[int, int, int]] = None): self.outer_path = outer_path self.inner_path = inner_path self.filesystem = filesystem binfile = filesystem.openbin(outer_path.as_posix()) # FIXME: h5py might not like this if the filesystem isn't OSFS f = h5py.File(binfile, "r") #type: ignore try: dataset = f[inner_path.as_posix()] if not isinstance(dataset, h5py.Dataset): raise ValueError(f"{inner_path} is not a Dataset") self.axiskeys = self.getAxisKeys(dataset) self._dataset = dataset tile_shape = Shape5D.create(raw_shape=self._dataset.chunks or self._dataset.shape, axiskeys=self.axiskeys) base_url = Url.parse(filesystem.geturl(outer_path.as_posix())) assert base_url is not None super().__init__( c_axiskeys_on_disk=self.axiskeys, tile_shape=tile_shape, interval=Shape5D.create( raw_shape=self._dataset.shape, axiskeys=self.axiskeys).to_interval5d(location), dtype=self._dataset.dtype, spatial_resolution=spatial_resolution or (1, 1, 1), # FIXME filesystem=filesystem, path=self.outer_path) except Exception as e: f.close() raise e
def test_writing_to_offset_precomputed_chunks(): tmp_path = create_tmp_dir( prefix="test_writing_to_offset_precomputed_chunks") data_at_1000_1000 = data.translated( Point5D(x=1000, y=1000) - data.location) datasource = ArrayDataSource(data=data_at_1000_1000, tile_shape=Shape5D(x=10, y=10)) scale = PrecomputedChunksScale.from_datasource( datasource=datasource, key=PurePosixPath("my_test_data"), encoding=RawEncoder()) sink_path = PurePosixPath("mytest.precomputed") filesystem = OsFs(tmp_path.as_posix()) print(f"\n\n will write to '{filesystem.geturl(sink_path.as_posix())}' ") datasink = PrecomputedChunksScaleSink( filesystem=filesystem, info_dir=sink_path, scale=scale, num_channels=datasource.shape.c, dtype=datasource.dtype, ) creation_result = datasink.create() if isinstance(creation_result, Exception): raise creation_result for tile in datasource.roi.get_datasource_tiles(): datasink.write(tile.retrieve()) precomp_datasource = PrecomputedChunksDataSource( path=sink_path, filesystem=filesystem, resolution=scale.resolution) reloaded_data = precomp_datasource.retrieve( interval=data_at_1000_1000.interval) assert (reloaded_data.raw("xyz") == data.raw("xyz")).all()
def to_points(self) -> Iterable[Point5D]: # FIXME: annotation should probably not be an Array6D for x, y, z in zip(*self.raw("xyz").nonzero()): # type: ignore yield Point5D(x=x, y=y, z=z) + self.location
def halo(self) -> Point5D: # FIXME: Add appropriate halo property to filters args = {"x": 30, "y": 30, "z": 30, "c": 0} if self.axis_2d: args[self.axis_2d] = 0 return Point5D(**args)
feature_extractors = [ #computes in 2D, slicing along the axis_2d. set axis_2d to None to compute in 3D IlpGaussianSmoothing(ilp_scale=0.3, axis_2d="z"), IlpGaussianSmoothing(ilp_scale=0.7, axis_2d="z"), IlpGaussianSmoothing(ilp_scale=1.0, axis_2d="z"), IlpGaussianSmoothing(ilp_scale=1.6, axis_2d="z"), IlpGaussianSmoothing(ilp_scale=3.5, axis_2d="z"), IlpGaussianSmoothing(ilp_scale=5.0, axis_2d="z"), IlpGaussianSmoothing(ilp_scale=10.0, axis_2d="z"), ] label_classes = [ #first label/class [ Annotation.interpolate_from_points( voxels=[Point5D.zero(x=140, y=150), Point5D.zero(x=145, y=155)], raw_data=data_source # All Annotations know what they have annotated ), Annotation.interpolate_from_points( voxels=[Point5D.zero(x=238, y=101), Point5D.zero(x=229, y=139)], raw_data=data_source ), ], #second label/class [ Annotation.interpolate_from_points( voxels=[Point5D.zero(x=283, y=87), Point5D.zero(x=288, y=92)], raw_data=data_source ), Annotation.interpolate_from_points( voxels=[Point5D.zero(x=274, y=168), Point5D.zero(x=256, y=191)],
filesystem=OsFs( Path(__file__).joinpath("../../public/images/").as_posix()), path=PurePosixPath(f"mouse{i}.precomputed"), resolution=(1, 1, 1)) for i in range(1, 3 + 1) ] if args.datasource == "brain": datasource = PrecomputedChunksDataSource( filesystem=OsFs( Path(__file__).joinpath("../../public/images/").as_posix()), path=PurePosixPath(f"mouse1.precomputed"), resolution=(1, 1, 1)) class1_annotations = [ Annotation.from_voxels( voxels=[ Point5D(x=2156, y=1326, z=0), Point5D(x=2157, y=1326, z=0), Point5D(x=2157, y=1327, z=0), Point5D(x=2157, y=1328, z=0), Point5D(x=2157, y=1329, z=0), Point5D(x=2157, y=1330, z=0), Point5D(x=2158, y=1330, z=0), Point5D(x=2159, y=1330, z=0), Point5D(x=2159, y=1331, z=0), Point5D(x=2160, y=1331, z=0), Point5D(x=2161, y=1331, z=0), Point5D(x=2162, y=1331, z=0), Point5D(x=2163, y=1331, z=0), Point5D(x=2163, y=1332, z=0), Point5D(x=2164, y=1332, z=0), Point5D(x=2164, y=1333, z=0),
def test_pixel_classification_ilp_serialization(): sample_trained_ilp_path = Path( "tests/projects/TrainedPixelClassification.ilp") output_ilp_path = Path("/tmp/rewritten.ilp") with h5py.File(sample_trained_ilp_path, "r") as f: sample_workflow_data = IlpPixelClassificationWorkflowGroup.parse( group=f, ilp_fs=OsFs("."), allowed_protocols=[Protocol.FILE], ) assert not isinstance(sample_workflow_data, Exception) with open(output_ilp_path, "wb") as rewritten: _ = rewritten.write(sample_workflow_data.to_h5_file_bytes()) shutil.copy("tests/projects/c_cells_1.png", output_ilp_path.parent.joinpath("c_cells_1.png")) with h5py.File(output_ilp_path, "r") as rewritten: reloaded_data = IlpPixelClassificationWorkflowGroup.parse( group=rewritten, ilp_fs=OsFs("/"), allowed_protocols=[Protocol.FILE], ) assert not isinstance(reloaded_data, Exception) loaded_feature_extractors = reloaded_data.FeatureSelections.feature_extractors assert IlpGaussianSmoothing(ilp_scale=0.3, axis_2d="z") in loaded_feature_extractors assert IlpLaplacianOfGaussian(ilp_scale=0.7, axis_2d="z") in loaded_feature_extractors assert IlpGaussianGradientMagnitude( ilp_scale=1.0, axis_2d="z") in loaded_feature_extractors assert IlpDifferenceOfGaussians( ilp_scale=1.6, axis_2d="z") in loaded_feature_extractors assert IlpStructureTensorEigenvalues( ilp_scale=3.5, axis_2d="z") in loaded_feature_extractors assert IlpHessianOfGaussianEigenvalues( ilp_scale=5.0, axis_2d="z") in loaded_feature_extractors assert len(loaded_feature_extractors) == 6 loaded_labels = reloaded_data.PixelClassification.labels for label in loaded_labels: loaded_points: Set[Point5D] = set() for a in label.annotations: loaded_points.update(a.to_points()) if label.color == Color(r=np.uint8(255), g=np.uint8(225), b=np.uint8(25)): assert loaded_points == set([ Point5D(x=200, y=200), Point5D(x=201, y=201), Point5D(x=202, y=202) ]) elif label.color == Color(r=np.uint8(0), g=np.uint8(130), b=np.uint8(200)): assert loaded_points == set([ Point5D(x=400, y=400), Point5D(x=401, y=401), Point5D(x=402, y=402) ]) else: assert False, f"Unexpected label color: {label.color}" some_executor = ProcessPoolExecutor(max_workers=2) priority_executor = PriorityExecutor(executor=some_executor, max_active_job_steps=2) workflow = PixelClassificationWorkflow.from_ilp( allowed_protocols=[Protocol.FILE], executor=some_executor, priority_executor=priority_executor, ilp_path=output_ilp_path, on_async_change=lambda: None, ) assert not isinstance(workflow, Exception) print(f"These are the deserialized brush strokes:") from pprint import pprint pprint(workflow.brushing_applet.label_classes()) annotation_raw_data = workflow.brushing_applet.labels( )[0].annotations[0].raw_data expected_annotation1 = Annotation.from_voxels( voxels=[ Point5D(x=200, y=200), Point5D(x=201, y=201), Point5D(x=202, y=202) ], raw_data=annotation_raw_data, ) expected_annotation2 = Annotation.from_voxels( voxels=[ Point5D(x=400, y=400), Point5D(x=401, y=401), Point5D(x=402, y=402) ], raw_data=annotation_raw_data, ) assert set( a for annotations in workflow.brushing_applet.label_classes().values() for a in annotations) == set( [expected_annotation1, expected_annotation2]) res = workflow.brushing_applet.remove_annotation( user_prompt=dummy_prompt, label_name="Label 1", annotation=expected_annotation1) print(res) pprint(workflow.brushing_applet.label_classes()) priority_executor.shutdown() some_executor.shutdown()
def __init__( self, arr: "np.ndarray[Any, Any]", *, axiskeys: str, location: Point5D = Point5D.zero(), labels: Optional[Set[int]] = None ): super().__init__(arr, axiskeys=axiskeys, location=location) self._border_colors: Optional[Set[int]] = None self._labels = labels
def from_json_value(cls, value: JsonValue) -> "SkimageDataSource": value_obj = ensureJsonObject(value) return SkimageDataSource( path=PurePosixPath(ensureJsonString(value_obj.get("path"))), location=ensureOptional(Point5D.from_json_value, value_obj.get("location")) or Point5D.zero(), filesystem=JsonableFilesystem.from_json_value(value_obj.get("filesystem")), tile_shape=ensureOptional(Shape5D.from_json_value, value_obj.get("tile_shape")), spatial_resolution=ensureOptional(ensureJsonIntTripplet, value_obj.get("spatial_resolution")), )
def __init__(self, arr: "np.ndarray[Any, np.dtype[np.float32]]", axiskeys: str, location: Point5D = Point5D.zero()): super().__init__(arr, axiskeys=axiskeys, location=location) assert self.dtype == np.dtype('float32')
def __setstate__(self, value_obj: JsonObject): self.__init__( path=PurePosixPath(ensureJsonString(value_obj.get("path"))), location=ensureOptional(Point5D.from_json_value, value_obj.get("location")) or Point5D.zero(), filesystem=JsonableFilesystem.from_json_value(value_obj.get("filesystem")), tile_shape=ensureOptional(Shape5D.from_json_value, value_obj.get("tile_shape")), spatial_resolution=ensureOptional(ensureJsonIntTripplet, value_obj.get("spatial_resolution")), )