def from_json_data(cls,
data: JsonValue,
location_override: Optional[Point5D] = None
) -> "N5DatasetAttributes":
raw_attributes = ensureJsonObject(data)
dimensions = ensureJsonIntArray(raw_attributes.get("dimensions"))
blockSize = ensureJsonIntArray(raw_attributes.get("blockSize"))
axes = raw_attributes.get("axes")
if axes is None:
axiskeys = guess_axiskeys(dimensions)
else:
axiskeys = "".join(ensureJsonStringArray(axes)[::-1]).lower()
location = raw_attributes.get("location")
if location is None:
location_5d = Point5D.zero()
else:
location_5d = Point5D.zero(
**dict(zip(axiskeys,
ensureJsonIntArray(location)[::-1])))
return N5DatasetAttributes(
blockSize=Shape5D.create(raw_shape=blockSize[::-1],
axiskeys=axiskeys),
dimensions=Shape5D.create(raw_shape=dimensions[::-1],
axiskeys=axiskeys),
dataType=np.dtype(ensureJsonString(raw_attributes.get(
"dataType"))).newbyteorder(">"), # type: ignore
axiskeys=axiskeys,
compression=N5Compressor.from_json_data(
raw_attributes["compression"]),
location=location_override or location_5d,
)
def stack_h5s(stack_axis: str) -> List[H5DataSource]:
offset = Point5D.zero()
stack: List[H5DataSource] = []
for outer_path in h5_outer_paths:
stack.append(H5DataSource(outer_path=outer_path, inner_path=PurePosixPath("/data"), filesystem=OsFs("/"), location=offset))
offset += Point5D.zero(**{stack_axis: stack[-1].shape[stack_axis]})
return stack
def test_data_roi_get_tiles_can_clamp_to_datasource_tiles():
# fmt: off
data = Array5D(np.asarray([
[1, 2, 3, 4, 5],
[6, 7, 8, 9, 10],
[11, 12, 13, 14, 15],
[16, 17, 18, 19, 20],
]).astype(np.uint8), axiskeys="yx")
# fmt: on
ds = ArrayDataSource(data=data, tile_shape=Shape5D(x=2, y=2))
data_slice = DataRoi(datasource=ds, x=(1, 4), y=(0, 3))
# fmt: off
dataslice_expected_data = Array5D(np.asarray([
[2, 3, 4],
[7, 8, 9],
[12, 13, 14]
]).astype(np.uint8), axiskeys="yx", location=Point5D.zero(x=1))
# fmt: on
assert data_slice.retrieve() == dataslice_expected_data
# fmt: off
dataslice_expected_slices = [
Array5D(np.asarray([
[1, 2],
[6, 7]
]).astype(np.uint8), axiskeys="yx", location=Point5D.zero()),
Array5D(np.asarray([
[3, 4],
[8, 9],
]).astype(np.uint8), axiskeys="yx", location=Point5D.zero(x=2)),
Array5D(np.asarray([
[11, 12],
[16, 17],
]).astype(np.uint8), axiskeys="yx", location=Point5D.zero(y=2)),
Array5D(np.asarray([
[13, 14],
[18, 19],
]).astype(np.uint8), axiskeys="yx", location=Point5D.zero(x=2, y=2))
]
# fmt: on
expected_slice_dict = {a.interval: a for a in dataslice_expected_slices}
for piece in data_slice.get_datasource_tiles(clamp_to_datasource=True):
expected_data = expected_slice_dict.pop(piece.interval)
assert expected_data == piece.retrieve()
assert len(expected_slice_dict) == 0
def get_sample_c_cells_pixel_annotations(
override_datasource: "FsDataSource | None" = None) -> Sequence[Label]:
raw_data_source = override_datasource or get_sample_c_cells_datasource()
return [
Label(name="Foreground",
color=Color(r=np.uint8(255), g=np.uint8(0), b=np.uint8(0)),
annotations=[
Annotation.interpolate_from_points(voxels=[
Point5D.zero(x=140, y=150),
Point5D.zero(x=145, y=155)
],
raw_data=raw_data_source),
Annotation.interpolate_from_points(voxels=[
Point5D.zero(x=238, y=101),
Point5D.zero(x=229, y=139)
],
raw_data=raw_data_source),
]),
Label(name="Background",
color=Color(r=np.uint8(255), g=np.uint8(0), b=np.uint8(0)),
annotations=[
Annotation.interpolate_from_points(voxels=[
Point5D.zero(x=283, y=87),
Point5D.zero(x=288, y=92)
],
raw_data=raw_data_source),
Annotation.interpolate_from_points(voxels=[
Point5D.zero(x=274, y=168),
Point5D.zero(x=256, y=191)
],
raw_data=raw_data_source),
]),
]
def __init__(
self,
*,
path: PurePosixPath,
location: Point5D = Point5D.zero(),
filesystem: JsonableFilesystem,
tile_shape: Optional[Shape5D] = None,
spatial_resolution: Optional[Tuple[int, int, int]] = None,
):
raw_data: "np.ndarray[Any, Any]" = skimage.io.imread(filesystem.openbin(path.as_posix())) # type: ignore
c_axiskeys_on_disk = "yxc"[: len(raw_data.shape)]
self._data = Array5D(raw_data, axiskeys=c_axiskeys_on_disk, location=location)
if tile_shape is None:
tile_shape = Shape5D.hypercube(256).to_interval5d().clamped(self._data.shape).shape
super().__init__(
c_axiskeys_on_disk=c_axiskeys_on_disk,
filesystem=filesystem,
path=path,
dtype=self._data.dtype,
interval=self._data.interval,
tile_shape=tile_shape,
spatial_resolution=spatial_resolution,
)
def __init__(self,
arr: "np.ndarray[Any, Any]",
*,
axiskeys: str,
location: Point5D = Point5D.zero(),
raw_data: DataSource):
assert arr.dtype == np.dtype(bool)
super().__init__(arr, axiskeys=axiskeys, location=location)
if not raw_data.interval.contains(self.interval):
raise AnnotationOutOfBounds(annotation_roi=self.interval,
raw_data=raw_data)
self.raw_data = raw_data
def test_n5_datasink():
tmp_path = create_tmp_dir(prefix="test_n5_datasink")
sink = N5DatasetSink(
filesystem=OsFs(tmp_path.as_posix()),
outer_path=PurePosixPath("test_n5_datasink.n5"),
inner_path=PurePosixPath("/data"),
attributes=N5DatasetAttributes(
dimensions=datasource.shape,
blockSize=Shape5D(x=10, y=10),
c_axiskeys=data.axiskeys, #FIXME: double check this
dataType=datasource.dtype,
compression=RawCompressor(),
location=Point5D.zero(x=7, y=13)))
sink_writer = sink.create()
assert not isinstance(sink_writer, Exception)
for tile in DataRoi(datasource).split(sink.tile_shape):
sink_writer.write(tile.retrieve().translated(Point5D.zero(x=7, y=13)))
n5ds = N5DataSource(filesystem=sink.filesystem, path=sink.full_path)
saved_data = n5ds.retrieve()
assert saved_data.location == Point5D.zero(x=7, y=13)
assert saved_data == data
def from_json_value(cls, value: JsonValue) -> "H5DataSource":
value_obj = ensureJsonObject(value)
return H5DataSource(outer_path=PurePosixPath(
ensureJsonString(value_obj.get("outer_path"))),
inner_path=PurePosixPath(
ensureJsonString(value_obj.get("inner_path"))),
filesystem=JsonableFilesystem.from_json_value(
value_obj.get("filesystem")),
location=ensureOptional(Point5D.from_json_value,
value_obj.get("location"))
or Point5D.zero(),
spatial_resolution=ensureJsonIntTripplet(
value_obj.get("spatial_resolution")))
class FeatureExtractorCollection(FeatureExtractor):
def __init__(self, extractors: Iterable[FeatureExtractor]):
self.extractors = tuple(extractors)
assert len(self.extractors) > 0
super().__init__()
def is_applicable_to(self, datasource: DataSource) -> bool:
return all(fx.is_applicable_to(datasource) for fx in self.extractors)
def __call__(self, /, roi: DataRoi) -> FeatureData:
assert roi.interval.c[0] == 0
feature_promises: Dict[int, Future[FeatureData]] = {}
executor = get_executor(hint="feature_extraction",
max_workers=len(self.extractors))
from webilastik.features.ilp_filter import IlpGaussianSmoothing
feature_promises = {
fx_index: executor.submit(fx, roi)
for fx_index, fx in enumerate(self.extractors)
if isinstance(fx, IlpGaussianSmoothing)
}
feature_promises.update({
fx_index: executor.submit(fx, roi)
for fx_index, fx in enumerate(self.extractors)
if not isinstance(fx, IlpGaussianSmoothing)
})
assert len(feature_promises) == len(self.extractors)
features = [
feature_promises[i].result() for i in range(len(self.extractors))
]
out = Array5D.allocate(
dtype=np.dtype("float32"),
interval=roi.shape.updated(c=sum(feat.shape.c
for feat in features)),
axiskeys="tzyxc",
).translated(roi.start)
channel_offset: int = 0
for feature in features:
out.set(feature.translated(Point5D.zero(c=channel_offset)))
channel_offset += feature.shape.c
return FeatureData(arr=out.raw(out.axiskeys),
axiskeys=out.axiskeys,
location=out.location)
def __init__(
self,
*,
dimensions: Shape5D,
blockSize: Shape5D,
axiskeys: str,
dataType: np.dtype,
compression: N5Compressor,
location: Point5D = Point5D.zero(),
):
"""axiskeys follows ndstructs conventions (c-order), despite 'axes' in N5 datasets being F-order"""
self.dimensions = dimensions
self.blockSize = blockSize
self.axiskeys = axiskeys
self.dataType = dataType
self.compression = compression
self.location = location
self.interval = self.dimensions.to_interval5d(self.location)
def __init__(self,
*,
outer_path: PurePosixPath,
inner_path: PurePosixPath,
location: Point5D = Point5D.zero(),
filesystem: JsonableFilesystem,
spatial_resolution: Optional[Tuple[int, int, int]] = None):
self.outer_path = outer_path
self.inner_path = inner_path
self.filesystem = filesystem
binfile = filesystem.openbin(outer_path.as_posix())
# FIXME: h5py might not like this if the filesystem isn't OSFS
f = h5py.File(binfile, "r") #type: ignore
try:
dataset = f[inner_path.as_posix()]
if not isinstance(dataset, h5py.Dataset):
raise ValueError(f"{inner_path} is not a Dataset")
self.axiskeys = self.getAxisKeys(dataset)
self._dataset = dataset
tile_shape = Shape5D.create(raw_shape=self._dataset.chunks
or self._dataset.shape,
axiskeys=self.axiskeys)
base_url = Url.parse(filesystem.geturl(outer_path.as_posix()))
assert base_url is not None
super().__init__(
c_axiskeys_on_disk=self.axiskeys,
tile_shape=tile_shape,
interval=Shape5D.create(
raw_shape=self._dataset.shape,
axiskeys=self.axiskeys).to_interval5d(location),
dtype=self._dataset.dtype,
spatial_resolution=spatial_resolution or (1, 1, 1), # FIXME
filesystem=filesystem,
path=self.outer_path)
except Exception as e:
f.close()
raise e
def __init__(self,
arr: "np.ndarray[Any, np.dtype[np.float32]]",
axiskeys: str,
location: Point5D = Point5D.zero()):
super().__init__(arr, axiskeys=axiskeys, location=location)
assert self.dtype == np.dtype('float32')
Point5D(x=2191, y=1329, z=0),
Point5D(x=2192, y=1329, z=0),
Point5D(x=2192, y=1328, z=0),
Point5D(x=2193, y=1328, z=0),
Point5D(x=2194, y=1328, z=0),
Point5D(x=2194, y=1327, z=0),
],
raw_data=mouse_datasources[0]),
]
elif args.datasource == "c_cells":
datasource = SkimageDataSource(filesystem=OsFs(
Path(__file__).joinpath("../../public/images/").as_posix()),
path=PurePosixPath("c_cells_1.png"))
class1_annotations = [
Annotation.interpolate_from_points(voxels=[
Point5D.zero(x=140, y=150),
Point5D.zero(x=145, y=155)
],
raw_data=datasource),
Annotation.interpolate_from_points(voxels=[
Point5D.zero(x=238, y=101),
Point5D.zero(x=229, y=139)
],
raw_data=datasource),
]
class_2_annotations = [
Annotation.interpolate_from_points(
voxels=[Point5D.zero(x=283, y=87),
Point5D.zero(x=288, y=92)],
raw_data=datasource),
Annotation.interpolate_from_points(voxels=[
def __setstate__(self, value_obj: JsonObject):
self.__init__(
path=PurePosixPath(ensureJsonString(value_obj.get("path"))),
location=ensureOptional(Point5D.from_json_value, value_obj.get("location")) or Point5D.zero(),
filesystem=JsonableFilesystem.from_json_value(value_obj.get("filesystem")),
tile_shape=ensureOptional(Shape5D.from_json_value, value_obj.get("tile_shape")),
spatial_resolution=ensureOptional(ensureJsonIntTripplet, value_obj.get("spatial_resolution")),
)
def from_json_value(cls, value: JsonValue) -> "SkimageDataSource":
value_obj = ensureJsonObject(value)
return SkimageDataSource(
path=PurePosixPath(ensureJsonString(value_obj.get("path"))),
location=ensureOptional(Point5D.from_json_value, value_obj.get("location")) or Point5D.zero(),
filesystem=JsonableFilesystem.from_json_value(value_obj.get("filesystem")),
tile_shape=ensureOptional(Shape5D.from_json_value, value_obj.get("tile_shape")),
spatial_resolution=ensureOptional(ensureJsonIntTripplet, value_obj.get("spatial_resolution")),
)
def __init__(
self, arr: "np.ndarray[Any, Any]", *, axiskeys: str, location: Point5D = Point5D.zero(), labels: Optional[Set[int]] = None
):
super().__init__(arr, axiskeys=axiskeys, location=location)
self._border_colors: Optional[Set[int]] = None
self._labels = labels