def create(
cls,
*,
outer_path: Path,
inner_path: PurePosixPath,
filesystem: JsonableFilesystem,
attributes: N5DatasetAttributes,
) -> "N5DatasetSink":
full_path = outer_path.joinpath(inner_path.as_posix().lstrip("/"))
filesystem.makedirs(full_path.as_posix(), recreate=True)
with filesystem.openbin(
outer_path.joinpath("attributes.json").as_posix(), "w") as f:
f.write(json.dumps({"n5": "2.0.0"}).encode("utf8"))
with filesystem.openbin(
full_path.joinpath("attributes.json").as_posix(), "w") as f:
f.write(json.dumps(attributes.to_json_data()).encode("utf-8"))
# create all directories in the constructor to avoid races when processing tiles
created_dirs: Set[Path] = set()
for tile in attributes.interval.split(attributes.blockSize):
dir_path = full_path / attributes.get_tile_path(tile).parent
if dir_path and dir_path not in created_dirs:
# print(f"Will create dir at {dir_path}")
filesystem.makedirs(dir_path.as_posix())
created_dirs.add(dir_path)
return N5DatasetSink(
path=full_path,
filesystem=filesystem,
attributes=attributes,
)
def __init__(self,
*,
outer_path: Path,
inner_path: PurePosixPath,
location: Point5D = Point5D.zero(),
filesystem: JsonableFilesystem):
self.outer_path = outer_path
self.inner_path = inner_path
self.filesystem = filesystem
binfile = filesystem.openbin(outer_path.as_posix())
f = h5py.File(binfile, "r")
try:
dataset = f[inner_path.as_posix()]
if not isinstance(dataset, h5py.Dataset):
raise ValueError(f"{inner_path} is not a Dataset")
axiskeys = self.getAxisKeys(dataset)
self._dataset = cast(h5py.Dataset, dataset)
tile_shape = Shape5D.create(raw_shape=self._dataset.chunks
or self._dataset.shape,
axiskeys=axiskeys)
super().__init__(
tile_shape=tile_shape,
interval=Shape5D.create(
raw_shape=self._dataset.shape,
axiskeys=axiskeys).to_interval5d(location),
dtype=self._dataset.dtype,
axiskeys=axiskeys,
)
except Exception as e:
f.close()
raise e
def __init__(
self,
*,
path: PurePosixPath,
location: Point5D = Point5D.zero(),
filesystem: JsonableFilesystem,
tile_shape: Optional[Shape5D] = None,
spatial_resolution: Optional[Tuple[int, int, int]] = None,
):
raw_data: "np.ndarray[Any, Any]" = skimage.io.imread(filesystem.openbin(path.as_posix())) # type: ignore
c_axiskeys_on_disk = "yxc"[: len(raw_data.shape)]
self._data = Array5D(raw_data, axiskeys=c_axiskeys_on_disk, location=location)
if tile_shape is None:
tile_shape = Shape5D.hypercube(256).to_interval5d().clamped(self._data.shape).shape
super().__init__(
c_axiskeys_on_disk=c_axiskeys_on_disk,
filesystem=filesystem,
path=path,
dtype=self._data.dtype,
interval=self._data.interval,
tile_shape=tile_shape,
spatial_resolution=spatial_resolution,
)
def open(cls, *, path: Path,
filesystem: JsonableFilesystem) -> "N5DatasetSink":
with filesystem.openbin(
path.joinpath("attributes.json").as_posix(), "r") as f:
attributes_json = f.read().decode("utf8")
attributes = N5DatasetAttributes.from_json_data(
json.loads(attributes_json))
return N5DatasetSink(filesystem=filesystem,
path=path,
attributes=attributes)
def tryLoad(cls, filesystem: JsonableFilesystem,
path: PurePosixPath) -> "PrecomputedChunksInfo | Exception":
url = filesystem.geturl(path.as_posix())
if not filesystem.exists(path.as_posix()):
return FileNotFoundError(f"Could not find info file at {url}")
with filesystem.openbin(path.as_posix(), "r") as f:
try:
info_json = f.read().decode("utf8")
return PrecomputedChunksInfo.from_json_value(
json.loads(info_json))
except Exception:
return ValueError(
f"Could not interpret json info file at {url}")
def __init__(self,
path: Path,
*,
location: Point5D = Point5D.zero(),
filesystem: JsonableFilesystem,
tile_shape: Optional[Shape5D] = None):
self.path = path
self.filesystem = filesystem
raw_data: np.ndarray = skimage.io.imread(
filesystem.openbin(path.as_posix())) # type: ignore
axiskeys = "yxc"[:len(raw_data.shape)]
super().__init__(
data=raw_data,
axiskeys=axiskeys,
location=location,
tile_shape=tile_shape,
)
def __init__(
self,
*,
filesystem: JsonableFilesystem,
path: PurePosixPath,
location: Optional[Point5D] = None,
spatial_resolution: Optional[Tuple[int, int, int]] = None,
):
with filesystem.openbin(path.joinpath("attributes.json").as_posix(), "r") as f:
attributes_json = f.read().decode("utf8")
self.attributes = N5DatasetAttributes.from_json_data(json.loads(attributes_json), location_override=location)
super().__init__(
c_axiskeys_on_disk=self.attributes.c_axiskeys,
filesystem=filesystem,
path=path,
tile_shape=self.attributes.blockSize,
interval=self.attributes.interval,
dtype=self.attributes.dataType,
spatial_resolution=spatial_resolution,
)
def create(
cls,
*,
filesystem: JsonableFilesystem,
base_path:
Path, # path to the "directory" that should contain the info file
info: PrecomputedChunksInfo,
) -> "PrecomputedChunksSink":
if filesystem.exists(base_path.as_posix()):
filesystem.removedir(base_path.as_posix())
filesystem.makedirs(base_path.as_posix())
with filesystem.openbin(base_path.joinpath("info").as_posix(),
"w") as info_file:
info_file.write(json.dumps(info.to_json_value()).encode("utf8"))
for scale in info.scales_5d:
scale_path = base_path.joinpath(scale.key.as_posix().lstrip("/"))
filesystem.makedirs(scale_path.as_posix())
return PrecomputedChunksSink(filesystem=filesystem,
base_path=base_path,
info=info)
def __init__(self,
*,
outer_path: PurePosixPath,
inner_path: PurePosixPath,
location: Point5D = Point5D.zero(),
filesystem: JsonableFilesystem,
spatial_resolution: Optional[Tuple[int, int, int]] = None):
self.outer_path = outer_path
self.inner_path = inner_path
self.filesystem = filesystem
binfile = filesystem.openbin(outer_path.as_posix())
# FIXME: h5py might not like this if the filesystem isn't OSFS
f = h5py.File(binfile, "r") #type: ignore
try:
dataset = f[inner_path.as_posix()]
if not isinstance(dataset, h5py.Dataset):
raise ValueError(f"{inner_path} is not a Dataset")
self.axiskeys = self.getAxisKeys(dataset)
self._dataset = dataset
tile_shape = Shape5D.create(raw_shape=self._dataset.chunks
or self._dataset.shape,
axiskeys=self.axiskeys)
base_url = Url.parse(filesystem.geturl(outer_path.as_posix()))
assert base_url is not None
super().__init__(
c_axiskeys_on_disk=self.axiskeys,
tile_shape=tile_shape,
interval=Shape5D.create(
raw_shape=self._dataset.shape,
axiskeys=self.axiskeys).to_interval5d(location),
dtype=self._dataset.dtype,
spatial_resolution=spatial_resolution or (1, 1, 1), # FIXME
filesystem=filesystem,
path=self.outer_path)
except Exception as e:
f.close()
raise e
def save_project(self, fs: JsonableFilesystem, path: PurePosixPath) -> int:
with fs.openbin(path.as_posix(), "w") as f:
return f.write(self.get_ilp_contents())