def get_partitions(self): ds_shape = Shape(self.shape, sig_dims=self.sig_dims) ds_slice = Slice(origin=[0] * len(self.shape), shape=ds_shape) target_size = self.target_size if target_size is None: if self._compression is None: target_size = 512 * 1024 * 1024 else: target_size = 256 * 1024 * 1024 partition_shape = self.partition_shape( target_size=target_size, dtype=self.dtype, ) + tuple(self.shape.sig) # if the data is chunked in the navigation axes, choose a compatible # partition size (even important for non-compressed data!) chunks = self._chunks if chunks is not None and not _have_contig_chunks(chunks, ds_shape): partition_shape = _partition_shape_for_chunking(chunks, ds_shape) for pslice in ds_slice.subslices(partition_shape): yield H5Partition( meta=self._meta, reader=self.get_reader(), partition_slice=pslice.flatten_nav(self.shape), slice_nd=pslice, io_backend=self.get_io_backend(), chunks=self._chunks, decoder=None, )
def _get_subslices_chunked_tiled(self, tiling_scheme, scheme_lookup, nav_dims, tileshape_nd): """ general tiled reading w/ chunking outer loop is a chunk in signal dimensions, inner loop is over "rows in nav" """ slice_nd_sig = self.slice_nd.sig slice_nd_nav = self.slice_nd.nav chunks_nav = self._chunks[:nav_dims] sig_slices = slice_nd_sig.subslices(tiling_scheme.shape.sig) logger.debug( "_get_subslices_chunked_tiled: chunking first by sig %r, then nav %r, finally %r", tiling_scheme.shape.sig, chunks_nav, tileshape_nd) for sig_slice in sig_slices: chunk_slices = slice_nd_nav.subslices(shape=chunks_nav) for chunk_slice_nav in chunk_slices: chunk_slice = Slice( origin=chunk_slice_nav.origin + sig_slice.origin, shape=chunk_slice_nav.shape + tuple(sig_slice.shape), ) subslices = chunk_slice.subslices(shape=tileshape_nd) for subslice in subslices: scheme_key = (subslice.origin[nav_dims:], subslice.shape[nav_dims:]) idx = scheme_lookup[scheme_key] yield idx, subslice
def test_subslices_non_even_division_with_origin_2(): top_slice = Slice( origin=(0, 3, 0, 0), shape=Shape((3, 3, 3, 3), sig_dims=2), ) assert list(top_slice.subslices(shape=(2, 2, 2, 2))) == [ Slice(origin=(0, 3, 0, 0), shape=Shape((2, 2, 2, 2), sig_dims=2)), Slice(origin=(0, 3, 0, 2), shape=Shape((2, 2, 2, 1), sig_dims=2)), Slice(origin=(0, 3, 2, 0), shape=Shape((2, 2, 1, 2), sig_dims=2)), Slice(origin=(0, 3, 2, 2), shape=Shape((2, 2, 1, 1), sig_dims=2)), Slice(origin=(0, 5, 0, 0), shape=Shape((2, 1, 2, 2), sig_dims=2)), Slice(origin=(0, 5, 0, 2), shape=Shape((2, 1, 2, 1), sig_dims=2)), Slice(origin=(0, 5, 2, 0), shape=Shape((2, 1, 1, 2), sig_dims=2)), Slice(origin=(0, 5, 2, 2), shape=Shape((2, 1, 1, 1), sig_dims=2)), Slice(origin=(2, 3, 0, 0), shape=Shape((1, 2, 2, 2), sig_dims=2)), Slice(origin=(2, 3, 0, 2), shape=Shape((1, 2, 2, 1), sig_dims=2)), Slice(origin=(2, 3, 2, 0), shape=Shape((1, 2, 1, 2), sig_dims=2)), Slice(origin=(2, 3, 2, 2), shape=Shape((1, 2, 1, 1), sig_dims=2)), Slice(origin=(2, 5, 0, 0), shape=Shape((1, 1, 2, 2), sig_dims=2)), Slice(origin=(2, 5, 0, 2), shape=Shape((1, 1, 2, 1), sig_dims=2)), Slice(origin=(2, 5, 2, 0), shape=Shape((1, 1, 1, 2), sig_dims=2)), Slice(origin=(2, 5, 2, 2), shape=Shape((1, 1, 1, 1), sig_dims=2)), ]
def test_subslices_non_even_division_1(): top_slice = Slice( origin=(0, 0, 0, 0), shape=Shape((5, 1, 1, 1), sig_dims=2), ) assert list(top_slice.subslices(shape=(2, 1, 1, 1))) == [ Slice(origin=(0, 0, 0, 0), shape=Shape((2, 1, 1, 1), sig_dims=2)), Slice(origin=(2, 0, 0, 0), shape=Shape((2, 1, 1, 1), sig_dims=2)), Slice(origin=(4, 0, 0, 0), shape=Shape((1, 1, 1, 1), sig_dims=2)), ]
def get_partitions(self): ds_slice = Slice(origin=tuple([0] * self.raw_shape.dims), shape=self.raw_shape) for pslice in ds_slice.subslices(self.partition_shape): yield MemoryPartition( tileshape=self.tileshape, meta=self._meta, reader=self.get_reader(), partition_slice=pslice, )
def test_subslices_simple(): top_slice = Slice( origin=(0, 0, 0, 0), shape=Shape((4, 4, 4, 4), sig_dims=2), ) assert list(top_slice.subslices(shape=(2, 2, 4, 4))) == [ Slice(origin=(0, 0, 0, 0), shape=Shape((2, 2, 4, 4), sig_dims=2)), Slice(origin=(0, 2, 0, 0), shape=Shape((2, 2, 4, 4), sig_dims=2)), Slice(origin=(2, 0, 0, 0), shape=Shape((2, 2, 4, 4), sig_dims=2)), Slice(origin=(2, 2, 0, 0), shape=Shape((2, 2, 4, 4), sig_dims=2)), ]
def get_partitions(self): ds_shape = Shape(self.shape, sig_dims=self.sig_dims) ds_slice = Slice(origin=[0] * len(self.shape), shape=ds_shape) partition_shape = self.partition_shape( target_size=self.target_size, dtype=self.dtype, ) + tuple(self.shape.sig) for pslice in ds_slice.subslices(partition_shape): yield H5Partition( meta=self._meta, reader=self.get_reader(), partition_slice=pslice.flatten_nav(self.shape), slice_nd=pslice, )
def get_partitions(self): ds_slice = Slice(origin=(0, 0, 0, 0), shape=self.shape) partition_shape = self.partition_shape( datashape=self.shape, framesize=self.shape[2] * self.shape[3], dtype=self.dtype, target_size=256*1024*1024, ) for pslice in ds_slice.subslices(partition_shape): yield BloPartition( tileshape=self._tileshape, meta=self._meta, reader=self.get_reader(), partition_slice=pslice, )
def make_for_shape( cls, tileshape: Shape, dataset_shape: Shape, intent: Optional[TilingIntent] = None, debug=None, ) -> "TilingScheme": """ Make a TilingScheme from `tileshape` and `dataset_shape`. Note that both in signal and navigation direction there are border effects, i.e. if the depth doesn't evenly divide the number of frames in the partition (simplified, ROI also applies...), or if the signal dimensions of `tileshape` doesn't evenly divide the signal dimensions of the `dataset_shape`. Parameters ---------- tileshape Uniform shape of all tiles. Should have flat navigation axis (meaning tileshape.nav.dims == 1) and be contiguous in signal dimensions. dataset_shape Shape of the whole data set. Only the signal part is used. intent The intent of this scheme (whole partitions, frames or tiles) Needs to be set for correct per-partition tiling! """ # FIXME: validate navigation part of the tileshape to be contiguous # (i.e. a shape like (1, 1, ..., 1, X1, ..., XN)) # where X1 is <= the dataset shape at that index, and X2, ..., XN are # equal to the dataset shape at that index sig_slice = Slice( origin=tuple([0] * dataset_shape.sig.dims), shape=dataset_shape.sig ) subslices = list(sig_slice.subslices(tileshape.sig)) return cls( slices=subslices, tileshape=tileshape, dataset_shape=dataset_shape, debug=debug, intent=intent, )
def get_partitions(self): ds_slice = Slice(origin=(0, 0, 0, 0), shape=self.shape) partition_shape = self.partition_shape( datashape=self.shape, framesize=self._detector_size[0] * self._detector_size[1], dtype=self.dtype, target_size=256*1024*1024, min_num_partitions=self._min_num_partitions, ) for pslice in ds_slice.subslices(partition_shape): # TODO: where should the tileshape be set? let the user choose for now yield RawFilePartition( tileshape=self._tileshape, meta=self._meta, reader=self.get_reader(), partition_slice=pslice, )
def get_partitions(self): ds_shape = Shape(self.shape, sig_dims=self.sig_dims) ds_slice = Slice(origin=(0, 0, 0, 0), shape=ds_shape) dtype = self.dtype partition_shape = self.partition_shape( datashape=self.shape, framesize=self.shape.sig.size, dtype=dtype, target_size=self.target_size, min_num_partitions=self.min_num_partitions, ) for pslice in ds_slice.subslices(partition_shape): # TODO: where should the tileshape be set? let the user choose for now yield H5Partition( tileshape=self.tileshape, meta=self._meta, reader=self.get_reader(), partition_slice=pslice.flatten_nav(self.shape), )
def get_partitions(self): ds_slice = Slice(origin=(0, 0, 0), shape=self.raw_shape) num_partitions = 16 # FIXME: remove hardcoded num_frames = self.raw_shape[0] partition_shape = Shape( (num_frames // num_partitions, ) + tuple(self.raw_shape.sig), sig_dims=self._sig_dims) for pslice in ds_slice.subslices(partition_shape): files_slice = pslice.get(nav_only=True) assert len(files_slice) == 1 pfiles = self._files[files_slice[0]] yield RawFilesPartition( tileshape=self._tileshape, file_shape=self._file_shape, dataset=self, dtype=self.dtype, partition_slice=pslice, files=pfiles, )
def make_for_shape(cls, tileshape: Shape, dataset_shape: Shape, debug=None): """ Make a TilingScheme from `tileshape` and `dataset_shape`. Note that both in signal and navigation direction there are border effects, i.e. if the depth doesn't evenly divide the number of frames in the partition (simplified, ROI also applies...), or if the signal dimensions of `tileshape` doesn't evenly divide the signal dimensions of the `dataset_shape`. Parameters ---------- tileshape Uniform shape of all tiles. Should have flat navigation axis (meaning tileshape.nav.dims == 1) and be contiguous in signal dimensions. dataset_shape Shape of the whole data set. Only the signal part is used. """ # FIXME: validate tileshape to be contiguous # (i.e. a shape like (1, 1, ..., 1, X1, ..., XN)) # where X1 is <= the dataset shape at that index, and X2, ..., XN are # equal to the dataset shape at that index # NOTE: not applicable to all file formats! see k2is sig_slice = Slice(origin=tuple([0] * dataset_shape.sig.dims), shape=dataset_shape.sig) subslices = list(sig_slice.subslices(tileshape.sig)) return cls( slices=subslices, tileshape=tileshape, dataset_shape=dataset_shape, debug=debug, )
def test_subslices_non_even_division_4(): top_slice = Slice( origin=(1, 0, 0, 0), shape=Shape((1, 10, 3838, 3710), sig_dims=2), ) list(top_slice.subslices(shape=(1, 2, 128, 128)))