def _do_initialize(self):
if self.ds_path is None:
datasets = _get_datasets(self.path)
datasets_list = sorted(datasets, key=lambda i: i[1], reverse=True)
name, size, shape, dtype, compression, chunks = datasets_list[0]
self.ds_path = name
with self.get_reader().get_h5ds() as h5ds:
self._dtype = h5ds.dtype
shape = h5ds.shape
if len(shape) == self.sig_dims:
# shape = (1,) + shape -> this leads to indexing errors down the line
# so we currently don't support opening 2D HDF5 files
raise DataSetException(
"2D HDF5 files are currently not supported")
self._shape = Shape(shape, sig_dims=self.sig_dims)
self._image_count = self._shape.nav.size
self._meta = DataSetMeta(
shape=self.shape,
raw_dtype=self._dtype,
sync_offset=self._sync_offset,
image_count=self._image_count,
)
self._chunks = h5ds.chunks
self._compression = h5ds.compression
if self._compression is not None:
warnings.warn(
"Loading compressed HDF5, performance can be worse than with other formats",
RuntimeWarning)
self._nav_shape_product = self._shape.nav.size
self._sync_offset_info = self.get_sync_offset_info()
return self
def get_macrotile(self, dest_dtype="float32", roi=None):
'''
Return a single tile for the entire partition.
This is useful to support process_partiton() in UDFs and to construct dask arrays
from datasets.
'''
tiling_scheme = TilingScheme.make_for_shape(
tileshape=self.shape,
dataset_shape=self.meta.shape,
)
try:
return next(
self.get_tiles(
tiling_scheme=tiling_scheme,
dest_dtype=dest_dtype,
roi=roi,
))
except StopIteration:
tile_slice = Slice(
origin=(self.slice.origin[0], 0, 0),
shape=Shape((0, ) + tuple(self.slice.shape.sig), sig_dims=2),
)
return DataTile(
np.zeros(tile_slice.shape, dtype=dest_dtype),
tile_slice=tile_slice,
scheme_idx=0,
)
def test_mask_caching_1():
input_masks = [
lambda: np.ones((128, 128)),
lambda: np.zeros((128, 128)),
]
mask_container = MaskContainer(mask_factories=input_masks, dtype="float32")
shape = Shape((16, 16, 128, 128), sig_dims=2)
slice_ = Slice(origin=(0, 0, 0, 0), shape=shape)
mask_container[slice_]
cache_info = mask_container._get_masks_for_slice.cache_info()
assert cache_info.hits == 0
assert cache_info.misses == 1
mask_container[slice_]
cache_info = mask_container._get_masks_for_slice.cache_info()
assert cache_info.hits == 1
assert cache_info.misses == 1
slice_ = Slice(origin=(0, 1, 0, 0), shape=shape)
mask_container[slice_]
cache_info = mask_container._get_masks_for_slice.cache_info()
assert cache_info.hits == 2
assert cache_info.misses == 1
def get_partitions(self):
fs = self._fileset
num_frames = self.shape.nav.size
f_per_part = num_frames // self._get_num_partitions()
c0 = itertools.count(start=0, step=f_per_part)
c1 = itertools.count(start=f_per_part, step=f_per_part)
for (start, stop) in zip(c0, c1):
if start >= num_frames:
break
stop = min(stop, num_frames)
part_slice = Slice(origin=(
start,
0,
0,
),
shape=Shape(((stop - start), SECTOR_SIZE[0],
SECTOR_SIZE[1] * NUM_SECTORS),
sig_dims=2))
yield K2ISPartition(
meta=self._meta,
partition_slice=part_slice,
sectors=fs.sectors,
start_frame=start,
num_frames=stop - start,
)
def test_roi_3(hdf5, lt_ctx):
ds = H5DataSet(
path=hdf5.filename,
ds_path="data",
target_size=12800 * 2,
)
ds = ds.initialize(lt_ctx.executor)
roi = np.zeros(ds.shape.flatten_nav().nav, dtype=bool)
roi[24] = 1
tileshape = Shape((16, ) + tuple(ds.shape.sig), sig_dims=ds.shape.sig.dims)
tiling_scheme = TilingScheme.make_for_shape(
tileshape=tileshape,
dataset_shape=ds.shape,
)
tiles = []
for p in ds.get_partitions():
for tile in p.get_tiles(tiling_scheme=tiling_scheme,
dest_dtype="float32",
roi=roi):
print("tile:", tile)
tiles.append(tile)
assert len(tiles) == 1
assert tiles[0].tile_slice.shape.nav.size == 1
assert tuple(tiles[0].tile_slice.shape.sig) == (16, 16)
assert tiles[0].tile_slice.origin == (0, 0, 0)
assert np.allclose(tiles[0].data, hdf5['data'][4, 4])
def __init__(self, path, tileshape=None, scan_size=None):
super().__init__()
self._sig_dims = 2
self._path = path
if tileshape is None:
tileshape = (1, 3, 256, 256)
tileshape = Shape(tileshape, sig_dims=self._sig_dims)
self._tileshape = tileshape
if scan_size is not None:
scan_size = tuple(scan_size)
else:
if not path.lower().endswith(".hdr"):
raise ValueError(
"either scan_size needs to be passed, or path needs to point to a .hdr file"
)
self._scan_size = scan_size
self._filename_cache = None
self._files_sorted = None
# ._preread_headers() calls ._files() which passes the cached headers down to MIBFile,
# if they exist. So we need to make sure to initialize self._headers
# before calling _preread_headers!
self._headers = {}
self._meta = None
self._total_filesize = None
self._sequence_start = None
def initialize(self, executor):
self._filesize = executor.run_function(self._get_filesize)
if self._same_offset:
metadata = executor.run_function(_get_metadata,
self._get_files()[0])
self._offsets = {
fn: metadata['offset']
for fn in self._get_files()
}
self._z_sizes = {fn: metadata['zsize'] for fn in self._get_files()}
else:
metadata = dict(
zip(
self._get_files(),
executor.map(_get_metadata, self._get_files()),
))
self._offsets = {
fn: metadata[fn]['offset']
for fn in self._get_files()
}
self._z_sizes = {
fn: metadata[fn]['zsize']
for fn in self._get_files()
}
self._fileset = executor.run_function(self._get_fileset)
first_file = next(self._fileset.files_from(0))
nav_dims = self._get_scan_size()
shape = nav_dims + tuple(first_file.sig_shape)
sig_dims = len(first_file.sig_shape)
self._meta = DataSetMeta(
shape=Shape(shape, sig_dims=sig_dims),
raw_dtype=first_file.native_dtype,
)
return self
def get_tiles(self, crop_to=None, full_frames=False):
stackheight = self.tileshape.nav.size
num_tiles = self.partfile.fields['num_images'] // stackheight
tshape = self.tileshape.flatten_nav()
sig_origin = (0, 0)
if crop_to is not None and tshape.sig != crop_to.shape.sig:
tshape = Shape(tuple(tshape.nav) + tuple(crop_to.shape.sig),
sig_dims=tshape.sig.dims)
sig_origin = crop_to.origin[1:]
data = np.ndarray(tshape, dtype=self.dtype)
for t in range(num_tiles):
tile_slice = Slice(
origin=(t * stackheight + self.slice.origin[0], ) + sig_origin,
shape=tshape)
if crop_to is not None:
intersection = tile_slice.intersection_with(crop_to)
if intersection.is_null():
continue
self.partfile.read_frames(num=stackheight,
offset=t * stackheight,
out=data,
crop_to=crop_to)
assert all(
[item > 0 for item in tile_slice.shift(self.slice).shape])
assert all(
[item >= 0 for item in tile_slice.shift(self.slice).origin])
yield DataTile(data=data, tile_slice=tile_slice)
def test_partition_shape_4():
assert get_partition_shape(Shape((128, 15, 15, 16, 16), sig_dims=2),
target_size_items=15 * 512) == ((
1,
2,
15,
))
def get_partitions(self):
num_frames = self.shape.nav.size
f_per_part = num_frames // self._get_num_partitions()
c0 = itertools.count(start=0, step=f_per_part)
c1 = itertools.count(start=f_per_part, step=f_per_part)
for (start, stop) in zip(c0, c1):
if start >= num_frames:
break
stop = min(stop, num_frames)
part_slice = Slice(origin=(
start,
0,
0,
),
shape=Shape(
((stop - start), ) + tuple(self.shape.sig),
sig_dims=self.shape.sig.dims))
yield SERPartition(
meta=self._meta,
partition_slice=part_slice,
reader=SERReader(path=self._path, emipath=self._emipath),
start_frame=start,
num_frames=stop - start,
)
def test_apply_mask_job(default_k2is, lt_ctx):
mask = np.ones((1860, 2048))
tileshape = Shape(
(16, 930, 16),
sig_dims=2,
)
tiling_scheme = TilingScheme.make_for_shape(
tileshape=tileshape,
dataset_shape=default_k2is.shape,
)
job = ApplyMasksJob(
dataset=default_k2is, mask_factories=[lambda: mask],
tiling_scheme=tiling_scheme,
)
out = job.get_result_buffer()
executor = InlineJobExecutor()
for tiles in executor.run_job(job):
for tile in tiles:
tile.reduce_into_result(out)
results = lt_ctx.run(job)
assert results[0].shape == (34 * 35,)
# there should be _something_ in each result pixel
for px in results[0].reshape((-1,)):
assert not np.isclose(px, 0)
def test_missing_frames(lt_ctx, raw_data_8x8x8x8_path, io_backend):
ds = RawFileDataSet(
path=raw_data_8x8x8x8_path,
nav_shape=(10, 8),
sig_shape=(8, 8),
dtype="float32",
io_backend=io_backend,
)
ds.set_num_cores(4)
ds = ds.initialize(lt_ctx.executor)
tileshape = Shape((4, ) + tuple(ds.shape.sig), sig_dims=ds.shape.sig.dims)
tiling_scheme = TilingScheme.make_for_shape(
tileshape=tileshape,
dataset_shape=ds.shape,
)
for p in ds.get_partitions():
for t in p.get_tiles(tiling_scheme=tiling_scheme):
pass
assert p._start_frame == 60
assert p._num_frames == 20
assert p.slice.origin == (60, 0, 0)
assert p.slice.shape[0] == 20
assert t.tile_slice.origin == (60, 0, 0)
assert t.tile_slice.shape[0] == 4
def get_tiles(self, tiling_scheme, dest_dtype="float32", roi=None):
shape = Shape((1, ) + tuple(self.shape.sig),
sig_dims=self.shape.sig.dims)
self.validate_tiling_scheme(tiling_scheme)
start = self._start_frame
stop = start + self._num_frames
if roi is None:
indices = np.arange(start, stop)
offset = start
else:
indices = _roi_to_indices(roi, start, stop)
offset = np.count_nonzero(roi.reshape((-1, ))[:start])
with fileSER(self._path) as f:
for num, idx in enumerate(indices):
tile_slice = Slice(origin=(num + offset, 0, 0), shape=shape)
data, metadata = f.getDataset(int(idx))
if data.dtype != np.dtype(dest_dtype):
data = data.astype(dest_dtype)
data = data.reshape(shape)
self._preprocess(data, tile_slice)
yield DataTile(
data,
tile_slice=tile_slice,
scheme_idx=0,
)
def get_partitions(self):
ds_shape = Shape(self.shape, sig_dims=self.sig_dims)
ds_slice = Slice(origin=[0] * len(self.shape), shape=ds_shape)
target_size = self.target_size
if target_size is None:
if self._compression is None:
target_size = 512 * 1024 * 1024
else:
target_size = 256 * 1024 * 1024
partition_shape = self.partition_shape(
target_size=target_size,
dtype=self.dtype,
) + tuple(self.shape.sig)
# if the data is chunked in the navigation axes, choose a compatible
# partition size (even important for non-compressed data!)
chunks = self._chunks
if chunks is not None and not _have_contig_chunks(chunks, ds_shape):
partition_shape = _partition_shape_for_chunking(chunks, ds_shape)
for pslice in ds_slice.subslices(partition_shape):
yield H5Partition(
meta=self._meta,
reader=self.get_reader(),
partition_slice=pslice.flatten_nav(self.shape),
slice_nd=pslice,
io_backend=self.get_io_backend(),
chunks=self._chunks,
)
def test_for_datatile_2(masks):
tile = DataTile(
tile_slice=Slice(origin=(0, 0, 0), shape=Shape((2 * 2, 10, 10), sig_dims=2)),
data=np.ones((2 * 2, 10, 10))
)
slice_ = masks.get_masks_for_slice(tile.tile_slice)
assert slice_.shape == (100, 5)
def test_simple(default_cached_ds):
parts = list(default_cached_ds.get_partitions())
p = parts[0]
datadir = default_cached_ds._cache_path
default_raw = default_cached_ds._source_ds
tileshape = Shape((16, ) + tuple(default_raw.shape.sig),
sig_dims=default_raw.shape.sig.dims)
tiling_scheme = TilingScheme.make_for_shape(
tileshape=tileshape,
dataset_shape=default_raw.shape,
)
print(datadir, os.listdir(datadir))
print(p)
t_cache = next(p.get_tiles(tiling_scheme=tiling_scheme))
t_orig = next(
next(default_raw.get_partitions()).get_tiles(
tiling_scheme=tiling_scheme))
print(t_cache)
print(t_orig)
assert np.allclose(t_cache.data, t_orig.data)
for p in default_cached_ds.get_partitions():
for tile in p.get_tiles(tiling_scheme=tiling_scheme):
pass
def __init__(self, tileshape=None, num_partitions=None, data=None, sig_dims=2,
check_cast=True, crop_frames=False, tiledelay=None, datashape=None):
# For HTTP API testing purposes: Allow to create empty dataset with given shape
if data is None:
assert datashape is not None
data = np.zeros(datashape, dtype=np.float32)
if num_partitions is None:
num_partitions = psutil.cpu_count(logical=False)
if tileshape is None:
sig_shape = data.shape[-sig_dims:]
target = 2**20
framesize = np.prod(sig_shape)
framecount = max(1, min(np.prod(data.shape[:-sig_dims]), int(target / framesize)))
tileshape = (framecount, ) + sig_shape
assert len(tileshape) == sig_dims + 1
self.data = data
self.tileshape = Shape(tileshape, sig_dims=sig_dims)
self.num_partitions = num_partitions
self.sig_dims = sig_dims
self._meta = DataSetMeta(
shape=self.shape,
raw_dtype=self.data.dtype,
)
self._check_cast = check_cast
self._crop_frames = crop_frames
self._tiledelay = tiledelay
async def test_fd_limit(async_executor):
import resource
import psutil
# set soft limit, throws errors but allows to raise it
# again afterwards:
proc = psutil.Process()
oldlimit = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE,
(proc.num_fds() + 24, oldlimit[1]))
print("fds", proc.num_fds())
try:
data = _mk_random(size=(1, 16, 16), dtype='<u2')
dataset = MemoryDataSet(data=data,
tileshape=(1, 16, 16),
num_partitions=1)
slice_ = Slice(origin=(0, 0, 0), shape=Shape((1, 16, 16), sig_dims=2))
job = PickFrameJob(dataset=dataset, slice_=slice_)
for i in range(32):
print(i)
print(proc.num_fds())
async for tiles in async_executor.run_job(job, cancel_id="42"):
pass
finally:
resource.setrlimit(resource.RLIMIT_NOFILE, oldlimit)
def _do_initialize(self):
self._headers = self._preread_headers()
self._files_sorted = list(
sorted(self._files(),
key=lambda f: f.fields['sequence_first_image']))
try:
first_file = self._files_sorted[0]
except IndexError:
raise DataSetException("no files found")
if self._scan_size is None:
hdr = read_hdr_file(self._path)
self._scan_size = scan_size_from_hdr(hdr)
shape = Shape(self._scan_size + first_file.fields['image_size'],
sig_dims=self._sig_dims)
dtype = first_file.fields['dtype']
meta = DataSetMeta(shape=shape,
raw_dtype=dtype,
iocaps={"FRAME_CROPS", "MMAP", "FULL_FRAMES"})
if first_file.fields['mib_dtype'] == "r64":
meta.iocaps.remove("MMAP")
self._meta = meta
self._total_filesize = sum(
os.stat(path).st_size for path in self._filenames())
self._sequence_start = first_file.fields['sequence_first_image']
self._files_sorted = list(
sorted(self._files(),
key=lambda f: f.fields['sequence_first_image']))
return self
def _get_tiles_w_copy(
self, tiling_scheme, fileset, read_ranges, read_dtype, native_dtype,
decoder=None, corrections=None,
):
if decoder is None:
decoder = DtypeConversionDecoder()
decode = decoder.get_decode(
native_dtype=np.dtype(native_dtype),
read_dtype=np.dtype(read_dtype),
)
r_n_d = self._r_n_d = self.get_read_and_decode(decode)
native_dtype = decoder.get_native_dtype(native_dtype, read_dtype)
mmaps = List()
for fh in fileset:
mmaps.append(np.frombuffer(fh.raw_mmap(), dtype=np.uint8))
sig_dims = tiling_scheme.shape.sig.dims
ds_shape = np.array(tiling_scheme.dataset_shape)
largest_slice = sorted([
(np.prod(s_.shape), s_)
for _, s_ in tiling_scheme.slices
], key=lambda x: x[0], reverse=True)[0][1]
buf_shape = (tiling_scheme.depth,) + tuple(largest_slice.shape)
need_clear = decoder.do_clear()
with self._buffer_pool.empty(buf_shape, dtype=read_dtype) as out_decoded:
out_decoded = out_decoded.reshape((-1,))
slices = read_ranges[0]
shape_prods = np.prod(slices[..., 1, :], axis=1)
ranges = read_ranges[1]
scheme_indices = read_ranges[2]
for idx in range(slices.shape[0]):
origin = slices[idx, 0]
shape = slices[idx, 1]
tile_slice = Slice(
origin=origin,
shape=Shape(shape, sig_dims=sig_dims)
)
tile_ranges = ranges[idx]
scheme_idx = scheme_indices[idx]
# if idx < slices.shape[0] - 1:
# self._prefetch_for_tile(fileset, ranges[idx + 1])
# pass
out_cut = out_decoded[:shape_prods[idx]].reshape((shape[0], -1))
data = r_n_d(
idx,
mmaps, sig_dims, tile_ranges,
out_cut, native_dtype, do_zero=need_clear,
origin=origin, shape=shape, ds_shape=ds_shape,
)
data = data.reshape(shape)
self.preprocess(data, tile_slice, corrections)
yield DataTile(
data,
tile_slice=tile_slice,
scheme_idx=scheme_idx,
)
def test_missing_frames(lt_ctx, io_backend):
"""
there can be some frames missing at the end
"""
# one full row of additional frames in the data set than the number of files
nav_shape = (3, 5)
ds = lt_ctx.load(
"dm",
files=list(sorted(glob(os.path.join(DM_TESTDATA_PATH, '*.dm4')))),
nav_shape=nav_shape,
io_backend=io_backend,
)
ds.set_num_cores(4)
tileshape = Shape((1, ) + tuple(ds.shape.sig), sig_dims=ds.shape.sig.dims)
tiling_scheme = TilingScheme.make_for_shape(
tileshape=tileshape,
dataset_shape=ds.shape,
)
for p in ds.get_partitions():
for t in p.get_tiles(tiling_scheme=tiling_scheme):
pass
assert p._start_frame == 12
assert p._num_frames == 3
assert p.slice.origin == (12, 0, 0)
assert p.slice.shape[0] == 3
assert t.tile_slice.origin == (9, 0, 0)
assert t.tile_slice.shape[0] == 1
def _do_initialize(self):
self._filenames = list(sorted(glob.glob(self._pattern())))
self._hdr_info = self._read_hdr_info()
self._headers = [
_read_file_header(path)
for path in self._files()
]
header = self._headers[0]
raw_frame_size = header['height'], header['width']
# frms6 frames are folded in a specific way, this is the shape after unfolding:
frame_size = 2 * header['height'], header['width'] // 2
assert header['width'] % 2 == 0
hdr = self._get_hdr_info()
bin_factor = hdr['readoutmode']['bin']
if bin_factor > 1:
frame_size = (frame_size[0] * bin_factor, frame_size[1])
sig_dims = 2 # FIXME: is there a different cameraMode that doesn't output 2D signals?
preferred_dtype = np.dtype("<u2")
if self._enable_offset_correction:
preferred_dtype = np.dtype("float32")
self._meta = DataSetMeta(
raw_dtype=np.dtype("<u2"),
dtype=preferred_dtype,
metadata={'raw_frame_size': raw_frame_size},
shape=Shape(tuple(hdr['stemimagesize']) + frame_size, sig_dims=sig_dims),
)
self._dark_frame = self._get_dark_frame()
self._gain_map = self._get_gain_map()
self._total_filesize = sum(
os.stat(path).st_size
for path in self._files()
)
return self
def _read_full_frames(self, crop_to=None, dest_dtype="float32", roi=None):
with contextlib.ExitStack() as stack:
frame_buf = zeros_aligned((1, 1860, 2048), dtype=dest_dtype)
open_sectors = [
stack.enter_context(sector) for sector in self._sectors
]
frame_offset = 0
if roi is not None:
roi = roi.reshape((-1, ))
frame_offset = np.count_nonzero(roi[:self._start_frame])
frames_read = 0
for frame in range(self._start_frame,
self._start_frame + self._num_frames):
if roi is not None and not roi[frame]:
continue
origin = frame
if roi is not None:
origin = frame_offset + frames_read
tile_slice = Slice(
origin=(origin, 0, 0),
shape=Shape(frame_buf.shape, sig_dims=2),
)
if crop_to is not None:
intersection = tile_slice.intersection_with(crop_to)
if intersection.is_null():
continue
for s in open_sectors:
s.read_full_frame(
frame=frame,
buf=frame_buf[:, :,
s.idx * SECTOR_SIZE[1]:(s.idx + 1) *
SECTOR_SIZE[1]])
yield DataTile(data=frame_buf, tile_slice=tile_slice)
frames_read += 1
def _get_tiles_fullframe(self,
tiling_scheme,
dest_dtype="float32",
roi=None):
# assert len(tiling_scheme) == 1
logger.debug("reading up to frame idx %d for this partition",
self._end_idx)
pool = self._data_source.pool.get_impl(
read_upto_frame=self._end_idx,
# chunk_size=11,
chunk_size=tiling_scheme.depth,
)
to_read = self._end_idx - self._start_idx
with pool:
while to_read > 0:
# logger.debug(
# "LivePartition.get_tiles: to_read=%d, start_idx=%d end_idx=%d",
# to_read, self._start_idx, self._end_idx,
# )
with pool.get_result() as res_wrapped:
frames_in_tile = res_wrapped.stop - res_wrapped.start
tile_shape = Shape(
(frames_in_tile, ) + tuple(tiling_scheme[0].shape),
sig_dims=2)
tile_slice = Slice(
origin=(res_wrapped.start, ) + (0, 0),
shape=tile_shape,
)
yield DataTile(
res_wrapped.buf,
tile_slice=tile_slice,
scheme_idx=0,
)
to_read -= frames_in_tile
logger.debug("LivePartition.get_tiles: end of method")
def _get_tiles_with_roi(self, roi, dest_dtype):
flat_roi = roi.reshape((-1,))
roi = roi.reshape(self.meta.shape.nav)
result_shape = Shape((1,) + tuple(self.meta.shape.sig), sig_dims=self.meta.shape.sig.dims)
sig_origin = tuple([0] * self.meta.shape.sig.dims)
frames_read = 0
start_at_frame = self.slice.origin[0]
frame_offset = np.count_nonzero(flat_roi[:start_at_frame])
indices = _roi_to_nd_indices(roi, self.slice_nd)
with self.reader.get_h5ds() as h5ds:
for idx in indices:
tile_slice = Slice(
origin=(frames_read + frame_offset,) + sig_origin,
shape=result_shape,
)
yield DataTile(
h5ds[idx].reshape(result_shape),
tile_slice=tile_slice,
# there is only a single slice in the tiling scheme, so our
# scheme_idx is constant 0
scheme_idx=0,
)
frames_read += 1
def get_tiles(self, tiling_scheme, dest_dtype="float32", roi=None):
yield from self._get_tiles_fullframe(tiling_scheme, dest_dtype, roi)
return
# assert len(tiling_scheme) == 1
print(tiling_scheme)
pool = self._data_source.pool.get_impl(
read_upto_frame=self._end_idx,
chunk_size=tiling_scheme.depth,
)
to_read = int(self._end_idx - self._start_idx)
nav_slices_raw = [(..., ) + slice_.get(sig_only=True)
for idx, slice_ in tiling_scheme.slices]
with pool:
while to_read > 0:
with pool.get_result() as res_wrapped:
frames_in_tile = res_wrapped.stop - res_wrapped.start
for (idx,
slice_), nav_slice_raw in zip(tiling_scheme.slices,
nav_slices_raw):
tile_shape = Shape(
(frames_in_tile, ) + tuple(slice_.shape),
sig_dims=2)
tile_slice = Slice(
origin=(res_wrapped.start, ) +
tuple(slice_.origin),
shape=tile_shape,
)
sliced_res = res_wrapped.buf[nav_slice_raw]
yield DataTile(sliced_res,
tile_slice=tile_slice,
scheme_idx=idx)
to_read -= frames_in_tile
def test_for_datatile_with_frame_origin(masks):
tile = DataTile(tile_slice=Slice(origin=(10, 10, 10, 10),
shape=Shape((2, 2, 1, 5), sig_dims=2)),
data=np.ones((2, 2, 1, 5)))
slice_ = masks.get_masks_for_slice(tile.tile_slice)
print(slice_)
np.allclose(
slice_,
np.array([
1,
0,
1,
0,
10,
1,
0,
1,
0,
11,
1,
0,
1,
0,
12,
1,
0,
1,
0,
13,
1,
0,
1,
0,
14,
]).reshape((5, 5)))
def test_get_signal_only():
s = Slice(origin=(0, 0, 0, 0), shape=Shape((1, 1, 1, 1), sig_dims=2))
assert s.get(sig_only=True) == (
slice(0, 1),
slice(0, 1),
)
def test_mask_caching_1():
input_masks = [
lambda: np.ones((128, 128)),
lambda: np.zeros((128, 128)),
]
mask_container = MaskContainer(mask_factories=input_masks, dtype="float32")
shape = Shape((16 * 16, 128, 128), sig_dims=2)
slice_ = Slice(origin=(0, 0, 0), shape=shape)
mask_container.get(slice_)
key = (mask_container.dtype, False, True, 'numpy')
cache_info = mask_container._get_masks_for_slice[key].cache_info()
assert cache_info.hits == 0
assert cache_info.misses == 1
mask_container.get(slice_)
cache_info = mask_container._get_masks_for_slice[key].cache_info()
assert cache_info.hits == 1
assert cache_info.misses == 1
slice_ = Slice(origin=(1, 0, 0), shape=shape)
mask_container.get(slice_)
cache_info = mask_container._get_masks_for_slice[key].cache_info()
assert cache_info.hits == 2
assert cache_info.misses == 1
def initialize(self, executor):
self._header = h = executor.run_function(self._read_header)
NY = int(h['NY'])
NX = int(h['NX'])
DP_SZ = int(h['DP_SZ'])
self._image_count = NY * NX
if self._nav_shape is None:
self._nav_shape = (NY, NX)
if self._sig_shape is None:
self._sig_shape = (DP_SZ, DP_SZ)
elif int(prod(self._sig_shape)) != (DP_SZ * DP_SZ):
raise DataSetException("sig_shape must be of size: %s" %
(DP_SZ * DP_SZ))
self._nav_shape_product = int(prod(self._nav_shape))
self._sync_offset_info = self.get_sync_offset_info()
self._shape = Shape(self._nav_shape + self._sig_shape,
sig_dims=len(self._sig_shape))
self._meta = DataSetMeta(
shape=self._shape,
raw_dtype=np.dtype("u1"),
sync_offset=self._sync_offset,
image_count=self._image_count,
)
self._filesize = executor.run_function(self._get_filesize)
return self