def get_partitions(self):
ds_shape = Shape(self.shape, sig_dims=self.sig_dims)
ds_slice = Slice(origin=[0] * len(self.shape), shape=ds_shape)
target_size = self.target_size
if target_size is None:
if self._compression is None:
target_size = 512 * 1024 * 1024
else:
target_size = 256 * 1024 * 1024
partition_shape = self.partition_shape(
target_size=target_size,
dtype=self.dtype,
) + tuple(self.shape.sig)
# if the data is chunked in the navigation axes, choose a compatible
# partition size (even important for non-compressed data!)
chunks = self._chunks
if chunks is not None and not _have_contig_chunks(chunks, ds_shape):
partition_shape = _partition_shape_for_chunking(chunks, ds_shape)
for pslice in ds_slice.subslices(partition_shape):
yield H5Partition(
meta=self._meta,
reader=self.get_reader(),
partition_slice=pslice.flatten_nav(self.shape),
slice_nd=pslice,
io_backend=self.get_io_backend(),
chunks=self._chunks,
decoder=None,
)
def _get_subslices_chunked_tiled(self, tiling_scheme, scheme_lookup,
nav_dims, tileshape_nd):
"""
general tiled reading w/ chunking outer loop is a chunk in
signal dimensions, inner loop is over "rows in nav"
"""
slice_nd_sig = self.slice_nd.sig
slice_nd_nav = self.slice_nd.nav
chunks_nav = self._chunks[:nav_dims]
sig_slices = slice_nd_sig.subslices(tiling_scheme.shape.sig)
logger.debug(
"_get_subslices_chunked_tiled: chunking first by sig %r, then nav %r, finally %r",
tiling_scheme.shape.sig, chunks_nav, tileshape_nd)
for sig_slice in sig_slices:
chunk_slices = slice_nd_nav.subslices(shape=chunks_nav)
for chunk_slice_nav in chunk_slices:
chunk_slice = Slice(
origin=chunk_slice_nav.origin + sig_slice.origin,
shape=chunk_slice_nav.shape + tuple(sig_slice.shape),
)
subslices = chunk_slice.subslices(shape=tileshape_nd)
for subslice in subslices:
scheme_key = (subslice.origin[nav_dims:],
subslice.shape[nav_dims:])
idx = scheme_lookup[scheme_key]
yield idx, subslice
def test_subslices_non_even_division_with_origin_2():
top_slice = Slice(
origin=(0, 3, 0, 0),
shape=Shape((3, 3, 3, 3), sig_dims=2),
)
assert list(top_slice.subslices(shape=(2, 2, 2, 2))) == [
Slice(origin=(0, 3, 0, 0), shape=Shape((2, 2, 2, 2), sig_dims=2)),
Slice(origin=(0, 3, 0, 2), shape=Shape((2, 2, 2, 1), sig_dims=2)),
Slice(origin=(0, 3, 2, 0), shape=Shape((2, 2, 1, 2), sig_dims=2)),
Slice(origin=(0, 3, 2, 2), shape=Shape((2, 2, 1, 1), sig_dims=2)),
Slice(origin=(0, 5, 0, 0), shape=Shape((2, 1, 2, 2), sig_dims=2)),
Slice(origin=(0, 5, 0, 2), shape=Shape((2, 1, 2, 1), sig_dims=2)),
Slice(origin=(0, 5, 2, 0), shape=Shape((2, 1, 1, 2), sig_dims=2)),
Slice(origin=(0, 5, 2, 2), shape=Shape((2, 1, 1, 1), sig_dims=2)),
Slice(origin=(2, 3, 0, 0), shape=Shape((1, 2, 2, 2), sig_dims=2)),
Slice(origin=(2, 3, 0, 2), shape=Shape((1, 2, 2, 1), sig_dims=2)),
Slice(origin=(2, 3, 2, 0), shape=Shape((1, 2, 1, 2), sig_dims=2)),
Slice(origin=(2, 3, 2, 2), shape=Shape((1, 2, 1, 1), sig_dims=2)),
Slice(origin=(2, 5, 0, 0), shape=Shape((1, 1, 2, 2), sig_dims=2)),
Slice(origin=(2, 5, 0, 2), shape=Shape((1, 1, 2, 1), sig_dims=2)),
Slice(origin=(2, 5, 2, 0), shape=Shape((1, 1, 1, 2), sig_dims=2)),
Slice(origin=(2, 5, 2, 2), shape=Shape((1, 1, 1, 1), sig_dims=2)),
]
def test_subslices_non_even_division_1():
top_slice = Slice(
origin=(0, 0, 0, 0),
shape=Shape((5, 1, 1, 1), sig_dims=2),
)
assert list(top_slice.subslices(shape=(2, 1, 1, 1))) == [
Slice(origin=(0, 0, 0, 0), shape=Shape((2, 1, 1, 1), sig_dims=2)),
Slice(origin=(2, 0, 0, 0), shape=Shape((2, 1, 1, 1), sig_dims=2)),
Slice(origin=(4, 0, 0, 0), shape=Shape((1, 1, 1, 1), sig_dims=2)),
]
def get_partitions(self):
ds_slice = Slice(origin=tuple([0] * self.raw_shape.dims),
shape=self.raw_shape)
for pslice in ds_slice.subslices(self.partition_shape):
yield MemoryPartition(
tileshape=self.tileshape,
meta=self._meta,
reader=self.get_reader(),
partition_slice=pslice,
)
def test_subslices_simple():
top_slice = Slice(
origin=(0, 0, 0, 0),
shape=Shape((4, 4, 4, 4), sig_dims=2),
)
assert list(top_slice.subslices(shape=(2, 2, 4, 4))) == [
Slice(origin=(0, 0, 0, 0), shape=Shape((2, 2, 4, 4), sig_dims=2)),
Slice(origin=(0, 2, 0, 0), shape=Shape((2, 2, 4, 4), sig_dims=2)),
Slice(origin=(2, 0, 0, 0), shape=Shape((2, 2, 4, 4), sig_dims=2)),
Slice(origin=(2, 2, 0, 0), shape=Shape((2, 2, 4, 4), sig_dims=2)),
]
def get_partitions(self):
ds_shape = Shape(self.shape, sig_dims=self.sig_dims)
ds_slice = Slice(origin=[0] * len(self.shape), shape=ds_shape)
partition_shape = self.partition_shape(
target_size=self.target_size,
dtype=self.dtype,
) + tuple(self.shape.sig)
for pslice in ds_slice.subslices(partition_shape):
yield H5Partition(
meta=self._meta,
reader=self.get_reader(),
partition_slice=pslice.flatten_nav(self.shape),
slice_nd=pslice,
)
def get_partitions(self):
ds_slice = Slice(origin=(0, 0, 0, 0), shape=self.shape)
partition_shape = self.partition_shape(
datashape=self.shape,
framesize=self.shape[2] * self.shape[3],
dtype=self.dtype,
target_size=256*1024*1024,
)
for pslice in ds_slice.subslices(partition_shape):
yield BloPartition(
tileshape=self._tileshape,
meta=self._meta,
reader=self.get_reader(),
partition_slice=pslice,
)
def make_for_shape(
cls,
tileshape: Shape,
dataset_shape: Shape,
intent: Optional[TilingIntent] = None,
debug=None,
) -> "TilingScheme":
"""
Make a TilingScheme from `tileshape` and `dataset_shape`.
Note that both in signal and navigation direction there are border
effects, i.e. if the depth doesn't evenly divide the number of
frames in the partition (simplified, ROI also applies...), or if
the signal dimensions of `tileshape` doesn't evenly divide the signal
dimensions of the `dataset_shape`.
Parameters
----------
tileshape
Uniform shape of all tiles. Should have flat navigation axis
(meaning tileshape.nav.dims == 1) and be contiguous in signal
dimensions.
dataset_shape
Shape of the whole data set. Only the signal part is used.
intent
The intent of this scheme (whole partitions, frames or tiles)
Needs to be set for correct per-partition tiling!
"""
# FIXME: validate navigation part of the tileshape to be contiguous
# (i.e. a shape like (1, 1, ..., 1, X1, ..., XN))
# where X1 is <= the dataset shape at that index, and X2, ..., XN are
# equal to the dataset shape at that index
sig_slice = Slice(
origin=tuple([0] * dataset_shape.sig.dims),
shape=dataset_shape.sig
)
subslices = list(sig_slice.subslices(tileshape.sig))
return cls(
slices=subslices,
tileshape=tileshape,
dataset_shape=dataset_shape,
debug=debug,
intent=intent,
)
def get_partitions(self):
ds_slice = Slice(origin=(0, 0, 0, 0), shape=self.shape)
partition_shape = self.partition_shape(
datashape=self.shape,
framesize=self._detector_size[0] * self._detector_size[1],
dtype=self.dtype,
target_size=256*1024*1024,
min_num_partitions=self._min_num_partitions,
)
for pslice in ds_slice.subslices(partition_shape):
# TODO: where should the tileshape be set? let the user choose for now
yield RawFilePartition(
tileshape=self._tileshape,
meta=self._meta,
reader=self.get_reader(),
partition_slice=pslice,
)
def get_partitions(self):
ds_shape = Shape(self.shape, sig_dims=self.sig_dims)
ds_slice = Slice(origin=(0, 0, 0, 0), shape=ds_shape)
dtype = self.dtype
partition_shape = self.partition_shape(
datashape=self.shape,
framesize=self.shape.sig.size,
dtype=dtype,
target_size=self.target_size,
min_num_partitions=self.min_num_partitions,
)
for pslice in ds_slice.subslices(partition_shape):
# TODO: where should the tileshape be set? let the user choose for now
yield H5Partition(
tileshape=self.tileshape,
meta=self._meta,
reader=self.get_reader(),
partition_slice=pslice.flatten_nav(self.shape),
)
def get_partitions(self):
ds_slice = Slice(origin=(0, 0, 0), shape=self.raw_shape)
num_partitions = 16 # FIXME: remove hardcoded
num_frames = self.raw_shape[0]
partition_shape = Shape(
(num_frames // num_partitions, ) + tuple(self.raw_shape.sig),
sig_dims=self._sig_dims)
for pslice in ds_slice.subslices(partition_shape):
files_slice = pslice.get(nav_only=True)
assert len(files_slice) == 1
pfiles = self._files[files_slice[0]]
yield RawFilesPartition(
tileshape=self._tileshape,
file_shape=self._file_shape,
dataset=self,
dtype=self.dtype,
partition_slice=pslice,
files=pfiles,
)
def make_for_shape(cls,
tileshape: Shape,
dataset_shape: Shape,
debug=None):
"""
Make a TilingScheme from `tileshape` and `dataset_shape`.
Note that both in signal and navigation direction there are border
effects, i.e. if the depth doesn't evenly divide the number of
frames in the partition (simplified, ROI also applies...), or if
the signal dimensions of `tileshape` doesn't evenly divide the signal
dimensions of the `dataset_shape`.
Parameters
----------
tileshape
Uniform shape of all tiles. Should have flat navigation axis
(meaning tileshape.nav.dims == 1) and be contiguous in signal
dimensions.
dataset_shape
Shape of the whole data set. Only the signal part is used.
"""
# FIXME: validate tileshape to be contiguous
# (i.e. a shape like (1, 1, ..., 1, X1, ..., XN))
# where X1 is <= the dataset shape at that index, and X2, ..., XN are
# equal to the dataset shape at that index
# NOTE: not applicable to all file formats! see k2is
sig_slice = Slice(origin=tuple([0] * dataset_shape.sig.dims),
shape=dataset_shape.sig)
subslices = list(sig_slice.subslices(tileshape.sig))
return cls(
slices=subslices,
tileshape=tileshape,
dataset_shape=dataset_shape,
debug=debug,
)
def test_subslices_non_even_division_4():
top_slice = Slice(
origin=(1, 0, 0, 0),
shape=Shape((1, 10, 3838, 3710), sig_dims=2),
)
list(top_slice.subslices(shape=(1, 2, 128, 128)))
