def tryLoad(cls, filesystem: JsonableFilesystem,
path: PurePosixPath) -> "PrecomputedChunksInfo | Exception":
url = filesystem.geturl(path.as_posix())
if not filesystem.exists(path.as_posix()):
return FileNotFoundError(f"Could not find info file at {url}")
with filesystem.openbin(path.as_posix(), "r") as f:
try:
info_json = f.read().decode("utf8")
return PrecomputedChunksInfo.from_json_value(
json.loads(info_json))
except Exception:
return ValueError(
f"Could not interpret json info file at {url}")
def __setstate__(self, data: JsonValue):
data_obj = ensureJsonObject(data)
self.__init__(
path=Path(ensureJsonString(data_obj.get("path"))),
location=Interval5D.from_json_value(data_obj.get("interval")).start,
filesystem=JsonableFilesystem.from_json_value(data_obj.get("filesystem"))
)
def __init__(
self,
*,
path: PurePosixPath,
location: Point5D = Point5D.zero(),
filesystem: JsonableFilesystem,
tile_shape: Optional[Shape5D] = None,
spatial_resolution: Optional[Tuple[int, int, int]] = None,
):
raw_data: "np.ndarray[Any, Any]" = skimage.io.imread(filesystem.openbin(path.as_posix())) # type: ignore
c_axiskeys_on_disk = "yxc"[: len(raw_data.shape)]
self._data = Array5D(raw_data, axiskeys=c_axiskeys_on_disk, location=location)
if tile_shape is None:
tile_shape = Shape5D.hypercube(256).to_interval5d().clamped(self._data.shape).shape
super().__init__(
c_axiskeys_on_disk=c_axiskeys_on_disk,
filesystem=filesystem,
path=path,
dtype=self._data.dtype,
interval=self._data.interval,
tile_shape=tile_shape,
spatial_resolution=spatial_resolution,
)
def __init__(self,
*,
outer_path: Path,
inner_path: PurePosixPath,
location: Point5D = Point5D.zero(),
filesystem: JsonableFilesystem):
self.outer_path = outer_path
self.inner_path = inner_path
self.filesystem = filesystem
binfile = filesystem.openbin(outer_path.as_posix())
f = h5py.File(binfile, "r")
try:
dataset = f[inner_path.as_posix()]
if not isinstance(dataset, h5py.Dataset):
raise ValueError(f"{inner_path} is not a Dataset")
axiskeys = self.getAxisKeys(dataset)
self._dataset = cast(h5py.Dataset, dataset)
tile_shape = Shape5D.create(raw_shape=self._dataset.chunks
or self._dataset.shape,
axiskeys=axiskeys)
super().__init__(
tile_shape=tile_shape,
interval=Shape5D.create(
raw_shape=self._dataset.shape,
axiskeys=axiskeys).to_interval5d(location),
dtype=self._dataset.dtype,
axiskeys=axiskeys,
)
except Exception as e:
f.close()
raise e
def from_json_value(cls, value: JsonValue) -> "N5DataSource":
value_obj = ensureJsonObject(value)
raw_location = value_obj.get("location")
return N5DataSource(
path=Path(ensureJsonString(value_obj.get("path"))),
filesystem=JsonableFilesystem.from_json_value(value_obj.get("filesystem")),
location=raw_location if raw_location is None else Point5D.from_json_value(raw_location),
)
def __setstate__(self, value_obj: JsonObject):
self.__init__(
path=PurePosixPath(ensureJsonString(value_obj.get("path"))),
location=ensureOptional(Point5D.from_json_value, value_obj.get("location")) or Point5D.zero(),
filesystem=JsonableFilesystem.from_json_value(value_obj.get("filesystem")),
tile_shape=ensureOptional(Shape5D.from_json_value, value_obj.get("tile_shape")),
spatial_resolution=ensureOptional(ensureJsonIntTripplet, value_obj.get("spatial_resolution")),
)
def from_json_value(cls, value: JsonValue) -> "SkimageDataSource":
value_obj = ensureJsonObject(value)
return SkimageDataSource(
path=PurePosixPath(ensureJsonString(value_obj.get("path"))),
location=ensureOptional(Point5D.from_json_value, value_obj.get("location")) or Point5D.zero(),
filesystem=JsonableFilesystem.from_json_value(value_obj.get("filesystem")),
tile_shape=ensureOptional(Shape5D.from_json_value, value_obj.get("tile_shape")),
spatial_resolution=ensureOptional(ensureJsonIntTripplet, value_obj.get("spatial_resolution")),
)
def open(cls, *, path: Path,
filesystem: JsonableFilesystem) -> "N5DatasetSink":
with filesystem.openbin(
path.joinpath("attributes.json").as_posix(), "r") as f:
attributes_json = f.read().decode("utf8")
attributes = N5DatasetAttributes.from_json_data(
json.loads(attributes_json))
return N5DatasetSink(filesystem=filesystem,
path=path,
attributes=attributes)
def from_json_value(cls, value: JsonValue) -> "PrecomputedChunksScaleSink":
value_obj = ensureJsonObject(value)
return PrecomputedChunksScaleSink(
filesystem=JsonableFilesystem.from_json_value(
value_obj.get("filesystem")),
info_dir=PurePosixPath(ensureJsonString(
value_obj.get("info_dir"))),
scale=PrecomputedChunksScale.from_json_value(
value_obj.get("scale")),
dtype=np.dtype(ensureJsonString(value_obj.get("dtype"))),
num_channels=ensureJsonInt(value_obj.get("num_channels")))
def from_json_value(cls, value: JsonValue) -> "PrecomputedChunksScaleSink":
value_obj = ensureJsonObject(value)
return PrecomputedChunksScaleSink(
filesystem=JsonableFilesystem.from_json_value(
value_obj.get("filesystem")),
base_path=Path(ensureJsonString(value_obj.get("base_path"))),
scale=PrecomputedChunksScale5D.from_json_value(
value_obj.get("scale")),
dtype=np.dtype(ensureJsonString(
value_obj.get("dtype"))), #type: ignore
)
def from_json_value(cls, value: JsonValue) -> "H5DataSource":
value_obj = ensureJsonObject(value)
raw_location = value_obj.get("location")
return H5DataSource(
outer_path=Path(ensureJsonString(value_obj.get("outer_path"))),
inner_path=PurePosixPath(
ensureJsonString(value_obj.get("inner_path"))),
filesystem=JsonableFilesystem.from_json_value(
value_obj.get("filesystem")),
location=Point5D.zero()
if raw_location is None else Point5D.from_json_value(raw_location),
)
def from_json_value(cls, value: JsonValue) -> "SkimageDataSource":
value_obj = ensureJsonObject(value)
raw_location = value_obj.get("location")
raw_tile_shape = value_obj.get("tile_shape")
return SkimageDataSource(
path=Path(ensureJsonString(value_obj.get("path"))),
location=Point5D.zero()
if raw_location is None else Point5D.from_json_value(raw_location),
filesystem=JsonableFilesystem.from_json_value(
value_obj.get("filesystem")),
tile_shape=None if raw_tile_shape is None else
Shape5D.from_json_value(raw_tile_shape))
def from_url(cls, url: Url) -> "Sequence[SkimageDataSource] | Exception":
if not cls.supports_url(url):
return Exception(f"Unsupported url: {url}")
fs_url = url.parent.schemeless().hashless()
fs_result = JsonableFilesystem.from_url(url=fs_url)
if isinstance(fs_result, Exception):
return fs_result
path = PurePosixPath(url.path.name)
try:
return [SkimageDataSource(path=path, filesystem=fs_result)]
except Exception as e:
return e
def create(
cls,
*,
outer_path: Path,
inner_path: PurePosixPath,
filesystem: JsonableFilesystem,
attributes: N5DatasetAttributes,
) -> "N5DatasetSink":
full_path = outer_path.joinpath(inner_path.as_posix().lstrip("/"))
filesystem.makedirs(full_path.as_posix(), recreate=True)
with filesystem.openbin(
outer_path.joinpath("attributes.json").as_posix(), "w") as f:
f.write(json.dumps({"n5": "2.0.0"}).encode("utf8"))
with filesystem.openbin(
full_path.joinpath("attributes.json").as_posix(), "w") as f:
f.write(json.dumps(attributes.to_json_data()).encode("utf-8"))
# create all directories in the constructor to avoid races when processing tiles
created_dirs: Set[Path] = set()
for tile in attributes.interval.split(attributes.blockSize):
dir_path = full_path / attributes.get_tile_path(tile).parent
if dir_path and dir_path not in created_dirs:
# print(f"Will create dir at {dir_path}")
filesystem.makedirs(dir_path.as_posix())
created_dirs.add(dir_path)
return N5DatasetSink(
path=full_path,
filesystem=filesystem,
attributes=attributes,
)
def from_json_value(cls, value: JsonValue) -> "H5DataSource":
value_obj = ensureJsonObject(value)
return H5DataSource(outer_path=PurePosixPath(
ensureJsonString(value_obj.get("outer_path"))),
inner_path=PurePosixPath(
ensureJsonString(value_obj.get("inner_path"))),
filesystem=JsonableFilesystem.from_json_value(
value_obj.get("filesystem")),
location=ensureOptional(Point5D.from_json_value,
value_obj.get("location"))
or Point5D.zero(),
spatial_resolution=ensureJsonIntTripplet(
value_obj.get("spatial_resolution")))
def __init__(self,
*,
outer_path: PurePosixPath,
inner_path: PurePosixPath,
location: Point5D = Point5D.zero(),
filesystem: JsonableFilesystem,
spatial_resolution: Optional[Tuple[int, int, int]] = None):
self.outer_path = outer_path
self.inner_path = inner_path
self.filesystem = filesystem
binfile = filesystem.openbin(outer_path.as_posix())
# FIXME: h5py might not like this if the filesystem isn't OSFS
f = h5py.File(binfile, "r") #type: ignore
try:
dataset = f[inner_path.as_posix()]
if not isinstance(dataset, h5py.Dataset):
raise ValueError(f"{inner_path} is not a Dataset")
self.axiskeys = self.getAxisKeys(dataset)
self._dataset = dataset
tile_shape = Shape5D.create(raw_shape=self._dataset.chunks
or self._dataset.shape,
axiskeys=self.axiskeys)
base_url = Url.parse(filesystem.geturl(outer_path.as_posix()))
assert base_url is not None
super().__init__(
c_axiskeys_on_disk=self.axiskeys,
tile_shape=tile_shape,
interval=Shape5D.create(
raw_shape=self._dataset.shape,
axiskeys=self.axiskeys).to_interval5d(location),
dtype=self._dataset.dtype,
spatial_resolution=spatial_resolution or (1, 1, 1), # FIXME
filesystem=filesystem,
path=self.outer_path)
except Exception as e:
f.close()
raise e
def from_json_value(cls,
value: JsonValue) -> "PrecomputedChunksDataSource":
value_obj = ensureJsonObject(value)
raw_location = value_obj.get("location")
raw_chunk_size = value_obj.get("chunk_size")
return PrecomputedChunksDataSource(
path=Path(ensureJsonString(value_obj.get("path"))),
resolution=ensureJsonIntTripplet(
value_obj.get("spatial_resolution")
), # FIXME? change to just resolution?
location=None
if raw_location is None else Point5D.from_json_value(raw_location),
chunk_size=None if raw_chunk_size is None else
Shape5D.from_json_value(raw_chunk_size),
filesystem=JsonableFilesystem.from_json_value(
value_obj.get("filesystem")),
)
def __init__(self,
path: Path,
*,
location: Point5D = Point5D.zero(),
filesystem: JsonableFilesystem,
tile_shape: Optional[Shape5D] = None):
self.path = path
self.filesystem = filesystem
raw_data: np.ndarray = skimage.io.imread(
filesystem.openbin(path.as_posix())) # type: ignore
axiskeys = "yxc"[:len(raw_data.shape)]
super().__init__(
data=raw_data,
axiskeys=axiskeys,
location=location,
tile_shape=tile_shape,
)
def try_to_datasource(
self,
*,
ilp_fs: JsonableFilesystem,
ilp_path: PurePosixPath,
allowed_protocols: Sequence[Protocol] = (Protocol.HTTP, Protocol.HTTPS)
) -> "FsDataSource | Exception":
url = Url.parse(self.filePath)
if url is None: # filePath was probably a path, not an URL
path = ilp_path.parent.joinpath(self.filePath)
url = Url.parse(ilp_fs.geturl(path.as_posix()))
if url is None:
return Exception(f"Could not parse {self.filePath} as URL")
datasources_result = try_get_datasources_from_url(url=url, allowed_protocols=allowed_protocols)
if isinstance(datasources_result, Exception):
return Exception(f"Could not open {url} as a data source: {datasources_result}")
if len(datasources_result) != 1:
return Exception(f"Expected a single datasource from {url}, found {len(datasources_result)}")
return datasources_result[0]
def __init__(
self,
*,
filesystem: JsonableFilesystem,
path: PurePosixPath,
location: Optional[Point5D] = None,
spatial_resolution: Optional[Tuple[int, int, int]] = None,
):
with filesystem.openbin(path.joinpath("attributes.json").as_posix(), "r") as f:
attributes_json = f.read().decode("utf8")
self.attributes = N5DatasetAttributes.from_json_data(json.loads(attributes_json), location_override=location)
super().__init__(
c_axiskeys_on_disk=self.attributes.c_axiskeys,
filesystem=filesystem,
path=path,
tile_shape=self.attributes.blockSize,
interval=self.attributes.interval,
dtype=self.attributes.dataType,
spatial_resolution=spatial_resolution,
)
def create(
cls,
*,
filesystem: JsonableFilesystem,
base_path:
Path, # path to the "directory" that should contain the info file
info: PrecomputedChunksInfo,
) -> "PrecomputedChunksSink":
if filesystem.exists(base_path.as_posix()):
filesystem.removedir(base_path.as_posix())
filesystem.makedirs(base_path.as_posix())
with filesystem.openbin(base_path.joinpath("info").as_posix(),
"w") as info_file:
info_file.write(json.dumps(info.to_json_value()).encode("utf8"))
for scale in info.scales_5d:
scale_path = base_path.joinpath(scale.key.as_posix().lstrip("/"))
filesystem.makedirs(scale_path.as_posix())
return PrecomputedChunksSink(filesystem=filesystem,
base_path=base_path,
info=info)
def save_project(self, fs: JsonableFilesystem, path: PurePosixPath) -> int:
with fs.openbin(path.as_posix(), "w") as f:
return f.write(self.get_ilp_contents())