def _read_full_frames(self, crop_to=None, dest_dtype="float32", roi=None):
with contextlib.ExitStack() as stack:
frame_buf = zeros_aligned((1, 1860, 2048), dtype=dest_dtype)
open_sectors = [
stack.enter_context(sector) for sector in self._sectors
]
frame_offset = 0
if roi is not None:
roi = roi.reshape((-1, ))
frame_offset = np.count_nonzero(roi[:self._start_frame])
frames_read = 0
for frame in range(self._start_frame,
self._start_frame + self._num_frames):
if roi is not None and not roi[frame]:
continue
origin = frame
if roi is not None:
origin = frame_offset + frames_read
tile_slice = Slice(
origin=(origin, 0, 0),
shape=Shape(frame_buf.shape, sig_dims=2),
)
if crop_to is not None:
intersection = tile_slice.intersection_with(crop_to)
if intersection.is_null():
continue
for s in open_sectors:
s.read_full_frame(
frame=frame,
buf=frame_buf[:, :,
s.idx * SECTOR_SIZE[1]:(s.idx + 1) *
SECTOR_SIZE[1]])
yield DataTile(data=frame_buf, tile_slice=tile_slice)
frames_read += 1
def get_tiles(self, crop_to=None, full_frames=False):
stackheight = self.tileshape.nav.size
num_tiles = self.partfile.fields['num_images'] // stackheight
tshape = self.tileshape.flatten_nav()
sig_origin = (0, 0)
if crop_to is not None and tshape.sig != crop_to.shape.sig:
tshape = Shape(tuple(tshape.nav) + tuple(crop_to.shape.sig),
sig_dims=tshape.sig.dims)
sig_origin = crop_to.origin[1:]
data = np.ndarray(tshape, dtype=self.dtype)
for t in range(num_tiles):
tile_slice = Slice(
origin=(t * stackheight + self.slice.origin[0], ) + sig_origin,
shape=tshape)
if crop_to is not None:
intersection = tile_slice.intersection_with(crop_to)
if intersection.is_null():
continue
self.partfile.read_frames(num=stackheight,
offset=t * stackheight,
out=data,
crop_to=crop_to)
assert all(
[item > 0 for item in tile_slice.shift(self.slice).shape])
assert all(
[item >= 0 for item in tile_slice.shift(self.slice).origin])
yield DataTile(data=data, tile_slice=tile_slice)
def test_slice_intersect_0():
s1 = Slice(
origin=(0, 0, 0, 0),
shape=Shape((2, 2, 2, 2), sig_dims=2),
)
s2 = Slice(
origin=(0, 0, 0, 0),
shape=Shape((1, 1, 1, 1), sig_dims=2),
)
assert s1.intersection_with(s2) == s2
def test_slice_intersect_3():
s1 = Slice(
origin=(1, 1, 1, 1),
shape=Shape((2, 2, 2, 2), sig_dims=2)
)
s2 = Slice(
origin=(0, 0, 0, 0),
shape=Shape((4, 4, 4, 4), sig_dims=2)
)
res = s1.intersection_with(s2)
assert res == s1
def _get_tiles_normal(self,
crop_to,
full_frames,
dest_dtype,
target_size=None):
start_at_frame = self._start_frame
num_frames = self._num_frames
sig_shape = self.meta.shape.sig
sig_origin = tuple([0] * len(sig_shape))
if crop_to is not None:
sig_origin = tuple(crop_to.origin[-sig_shape.dims:])
sig_shape = crop_to.shape.sig
if full_frames:
sig_shape = self.meta.shape.sig
stackheight = self._get_stackheight(sig_shape=sig_shape,
dest_dtype=dest_dtype,
target_size=target_size)
tile_buf_full = zeros_aligned((stackheight, ) + tuple(sig_shape),
dtype=dest_dtype)
tileshape = (stackheight, ) + tuple(sig_shape)
with self._fileset as fileset:
for outer_frame in range(start_at_frame,
start_at_frame + num_frames, stackheight):
if start_at_frame + num_frames - outer_frame < stackheight:
end_frame = start_at_frame + num_frames
current_stackheight = end_frame - outer_frame
current_tileshape = (
current_stackheight, ) + tuple(sig_shape)
tile_buf = zeros_aligned(current_tileshape,
dtype=dest_dtype)
else:
current_stackheight = stackheight
current_tileshape = tileshape
tile_buf = tile_buf_full
tile_slice = Slice(origin=(outer_frame, ) + sig_origin,
shape=Shape(current_tileshape,
sig_dims=sig_shape.dims))
if crop_to is not None:
intersection = tile_slice.intersection_with(crop_to)
if intersection.is_null():
continue
fileset.read_images_multifile(
start=outer_frame,
stop=outer_frame + current_stackheight,
out=tile_buf,
crop_to=crop_to,
)
yield DataTile(data=tile_buf, tile_slice=tile_slice)
def test_slice_intersect_2():
s1 = Slice(
origin=(1, 1, 1, 1),
shape=Shape((2, 2, 2, 2), sig_dims=2),
)
s2 = Slice(
origin=(0, 0, 0, 0),
shape=Shape((1, 1, 1, 1), sig_dims=2),
)
res = s1.intersection_with(s2)
assert res == Slice(
origin=(1, 1, 1, 1),
shape=Shape((0, 0, 0, 0), sig_dims=2),
)
assert res.is_null()
def get_tiles(self, crop_to=None):
stackheight = self.tileshape.nav.size
tile_buffer = np.zeros(self.tileshape.flatten_nav(),
dtype=self.dataset.dtype)
files_by_tile = list(grouper(self.files, stackheight))
sig_dims = self.tileshape.sig.dims
dtype = self.dataset.dtype
partition_origin = self.slice.origin
tile_slice_shape = Shape(tile_buffer.shape, sig_dims=sig_dims)
files_done = 0
for files in files_by_tile:
if len(files) != stackheight:
# allocate different size buffer for "rest-tile":
tile_buffer = np.zeros(
(len(files), ) + tuple(self.dataset.raw_shape.sig),
dtype=dtype)
tile_slice_shape = Shape(tile_buffer.shape, sig_dims=sig_dims)
origin = tuple(
sum(x) for x in zip(partition_origin, (files_done, 0, 0)))
tile_slice = Slice(
origin=origin,
shape=tile_slice_shape,
)
files_done += len(files)
if crop_to is not None:
intersection = tile_slice.intersection_with(crop_to)
if intersection.is_null():
continue
if stackheight == 1:
with open(files[0], "rb") as f:
f.readinto(tile_buffer)
yield DataTile(data=tile_buffer, tile_slice=tile_slice)
continue
else:
for idx, fn in enumerate(files):
with open(fn, "rb") as f:
f.readinto(tile_buffer[idx, ...])
yield DataTile(data=tile_buffer, tile_slice=tile_slice)
def get_tiles(self, crop_to=None, full_frames=False):
# NOTE: full_frames is ignored, as we currently read whole frames only
start_at_frame = self._start_frame
num_frames = self._num_frames
stackheight = self._get_stackheight()
dtype = self.meta.dtype
sig_shape = self.meta.shape.sig
sig_origin = tuple([0] * len(sig_shape))
if crop_to is not None:
sig_origin = tuple(crop_to.origin[-sig_shape.dims:])
sig_shape = crop_to.shape.sig
tile_buf_full = np.zeros((stackheight, ) + tuple(sig_shape),
dtype=dtype)
tileshape = (stackheight, ) + tuple(sig_shape)
for outer_frame in range(start_at_frame, start_at_frame + num_frames,
stackheight):
if start_at_frame + num_frames - outer_frame < stackheight:
end_frame = start_at_frame + num_frames
current_stackheight = end_frame - outer_frame
current_tileshape = (current_stackheight, ) + tuple(sig_shape)
tile_buf = np.zeros(current_tileshape, dtype=dtype)
else:
current_stackheight = stackheight
current_tileshape = tileshape
tile_buf = tile_buf_full
tile_slice = Slice(origin=(outer_frame, ) + sig_origin,
shape=Shape(current_tileshape,
sig_dims=sig_shape.dims))
if crop_to is not None:
intersection = tile_slice.intersection_with(crop_to)
if intersection.is_null():
continue
self._fileset.read_images(
start=outer_frame,
stop=outer_frame + current_stackheight,
out=tile_buf,
crop_to=crop_to,
)
yield DataTile(data=tile_buf, tile_slice=tile_slice)
def _get_tiles_mmap(self, crop_to, full_frames, dest_dtype):
start_at_frame = self._start_frame
num_frames = self._num_frames
sig_shape = self.meta.shape.sig
sig_origin = tuple([0] * len(sig_shape))
if dest_dtype != self.meta.raw_dtype:
raise ValueError(
"using mmap with dtype conversion is not efficient")
if crop_to is not None:
sig_origin = tuple(crop_to.origin[-sig_shape.dims:])
sig_shape = crop_to.shape.sig
if full_frames:
sig_shape = self.meta.shape.sig
fileset = self._fileset.get_for_range(
start=start_at_frame,
stop=start_at_frame + num_frames,
)
with self._fileset as fileset:
for f in fileset:
# global start/stop indices:
start = max(f.start_idx, self._start_frame)
stop = min(f.end_idx, self._start_frame + num_frames)
tile_slice = Slice(origin=(start, ) + sig_origin,
shape=Shape(
(stop - start, ) + tuple(sig_shape),
sig_dims=sig_shape.dims))
arr = f.mmap()
# limit to this partition (translate to file-local coords)
arr = arr[start - f.start_idx:stop - f.start_idx]
if crop_to is not None:
intersection = tile_slice.intersection_with(crop_to)
if intersection.is_null():
continue
# crop to, signal part:
arr = arr[(..., ) + tile_slice.get(sig_only=True)]
yield DataTile(data=arr, tile_slice=tile_slice)
def get_tiles(self, crop_to=None, full_frames=False):
# NOTE: full_frames ignored here because we currently only read full frames
if crop_to is not None:
if crop_to.shape.sig != self.meta.shape.sig:
raise DataSetException(
"DirectRawFileDataSet only supports whole-frame crops for now"
)
stackheight = self.stackheight
start_frame = self.start_frame
num_frames = self.num_frames
shape_sig = tuple(self.shape.sig)
sig_dims = self.shape.sig.dims
sig_size = self.shape.sig.size
stop = start_frame + num_frames
with self.reader.open_file() as reader:
buf = reader.get_buffer(stackheight)
c0 = itertools.count(start=start_frame, step=stackheight)
for tile_start in c0:
if tile_start >= stop:
break
# tile_height is the real height, which may be smaller than stackheight
# at the end
tile_height = min(stackheight, stop - tile_start)
tileslice = Slice(origin=(
tile_start,
0,
0,
),
shape=Shape((tile_height, ) + shape_sig,
sig_dims=sig_dims))
if crop_to is not None:
intersection = tileslice.intersection_with(crop_to)
if intersection.is_null():
continue
reader.seek_frame(tile_start)
data = reader.readinto(buf)
# at partition boundary we may read more than requested, cut it off:
data = data[:tile_height * sig_size]
yield DataTile(data=data.reshape((tile_height, ) + shape_sig),
tile_slice=tileslice)
def _read_full_frames(self, crop_to=None):
with contextlib.ExitStack() as stack:
frame_buf = np.zeros((1, 1860, 2048), dtype="float32")
open_sectors = [
stack.enter_context(sector) for sector in self._sectors
]
for frame in range(self._start_frame,
self._start_frame + self._num_frames):
tile_slice = Slice(
origin=(frame, 0, 0),
shape=Shape(frame_buf.shape, sig_dims=2),
)
if crop_to is not None:
intersection = tile_slice.intersection_with(crop_to)
if intersection.is_null():
continue
for s in open_sectors:
s.read_full_frame(
frame=frame,
buf=frame_buf[:, :,
s.idx * SECTOR_SIZE[1]:(s.idx + 1) *
SECTOR_SIZE[1]])
yield DataTile(data=frame_buf, tile_slice=tile_slice)
def read_stacked(self,
start_at_frame,
num_frames,
stackheight=16,
dtype="float32",
crop_to=None):
"""
Reads `stackheight` blocks into a single buffer.
The blocks are read from consecutive frames, always
from the same coordinates inside the sector of the frame.
yields DataTiles of the shape (stackheight, 930, 16)
(different tiles at the borders may be yielded if the stackheight doesn't evenly divide
the total number of frames to read)
"""
tileshape = (stackheight, ) + BLOCK_SHAPE
raw_data = mmap.mmap(
fileno=self.f.fileno(),
length=0, # whole file
access=mmap.ACCESS_READ,
)
tile_buf_full = zeros_aligned(tileshape, dtype=dtype)
assert DATA_SIZE % 3 == 0
log.debug(
"starting read_stacked with start_at_frame=%d, num_frames=%d, stackheight=%d",
start_at_frame, num_frames, stackheight)
for outer_frame in range(start_at_frame, start_at_frame + num_frames,
stackheight):
# log.debug("outer_frame=%d", outer_frame)
# end of the selected frame range, calculate rest of stack:
if start_at_frame + num_frames - outer_frame < stackheight:
end_frame = start_at_frame + num_frames
current_stackheight = end_frame - outer_frame
current_tileshape = (current_stackheight, ) + BLOCK_SHAPE
tile_buf = zeros_aligned(current_tileshape, dtype=dtype)
else:
current_stackheight = stackheight
current_tileshape = tileshape
tile_buf = tile_buf_full
for blockidx in range(BLOCKS_PER_SECTOR_PER_FRAME):
start_x = (self.idx + 1) * 256 - (16 * (blockidx % 16 + 1))
start_y = 930 * (blockidx // 16)
tile_slice = Slice(
origin=(
outer_frame,
start_y,
start_x,
),
shape=Shape(current_tileshape, sig_dims=self.sig_dims),
)
if crop_to is not None:
intersection = tile_slice.intersection_with(crop_to)
if intersection.is_null():
continue
offset = (
self.first_block_offset +
outer_frame * BLOCK_SIZE * BLOCKS_PER_SECTOR_PER_FRAME +
blockidx * BLOCK_SIZE)
for frame in range(current_stackheight):
block_offset = (
offset +
frame * BLOCK_SIZE * BLOCKS_PER_SECTOR_PER_FRAME)
input_start = block_offset + HEADER_SIZE
input_end = block_offset + HEADER_SIZE + DATA_SIZE
out = tile_buf[frame].reshape((-1, ))
decode_uint12_le(
inp=raw_data[input_start:input_end],
out=out,
)
yield DataTile(data=tile_buf, tile_slice=tile_slice)
raw_data.close()
