def _read_full_frames(self, crop_to=None, dest_dtype="float32", roi=None): with contextlib.ExitStack() as stack: frame_buf = zeros_aligned((1, 1860, 2048), dtype=dest_dtype) open_sectors = [ stack.enter_context(sector) for sector in self._sectors ] frame_offset = 0 if roi is not None: roi = roi.reshape((-1, )) frame_offset = np.count_nonzero(roi[:self._start_frame]) frames_read = 0 for frame in range(self._start_frame, self._start_frame + self._num_frames): if roi is not None and not roi[frame]: continue origin = frame if roi is not None: origin = frame_offset + frames_read tile_slice = Slice( origin=(origin, 0, 0), shape=Shape(frame_buf.shape, sig_dims=2), ) if crop_to is not None: intersection = tile_slice.intersection_with(crop_to) if intersection.is_null(): continue for s in open_sectors: s.read_full_frame( frame=frame, buf=frame_buf[:, :, s.idx * SECTOR_SIZE[1]:(s.idx + 1) * SECTOR_SIZE[1]]) yield DataTile(data=frame_buf, tile_slice=tile_slice) frames_read += 1
def get_tiles(self, crop_to=None, full_frames=False): stackheight = self.tileshape.nav.size num_tiles = self.partfile.fields['num_images'] // stackheight tshape = self.tileshape.flatten_nav() sig_origin = (0, 0) if crop_to is not None and tshape.sig != crop_to.shape.sig: tshape = Shape(tuple(tshape.nav) + tuple(crop_to.shape.sig), sig_dims=tshape.sig.dims) sig_origin = crop_to.origin[1:] data = np.ndarray(tshape, dtype=self.dtype) for t in range(num_tiles): tile_slice = Slice( origin=(t * stackheight + self.slice.origin[0], ) + sig_origin, shape=tshape) if crop_to is not None: intersection = tile_slice.intersection_with(crop_to) if intersection.is_null(): continue self.partfile.read_frames(num=stackheight, offset=t * stackheight, out=data, crop_to=crop_to) assert all( [item > 0 for item in tile_slice.shift(self.slice).shape]) assert all( [item >= 0 for item in tile_slice.shift(self.slice).origin]) yield DataTile(data=data, tile_slice=tile_slice)
def test_slice_intersect_0(): s1 = Slice( origin=(0, 0, 0, 0), shape=Shape((2, 2, 2, 2), sig_dims=2), ) s2 = Slice( origin=(0, 0, 0, 0), shape=Shape((1, 1, 1, 1), sig_dims=2), ) assert s1.intersection_with(s2) == s2
def test_slice_intersect_3(): s1 = Slice( origin=(1, 1, 1, 1), shape=Shape((2, 2, 2, 2), sig_dims=2) ) s2 = Slice( origin=(0, 0, 0, 0), shape=Shape((4, 4, 4, 4), sig_dims=2) ) res = s1.intersection_with(s2) assert res == s1
def _get_tiles_normal(self, crop_to, full_frames, dest_dtype, target_size=None): start_at_frame = self._start_frame num_frames = self._num_frames sig_shape = self.meta.shape.sig sig_origin = tuple([0] * len(sig_shape)) if crop_to is not None: sig_origin = tuple(crop_to.origin[-sig_shape.dims:]) sig_shape = crop_to.shape.sig if full_frames: sig_shape = self.meta.shape.sig stackheight = self._get_stackheight(sig_shape=sig_shape, dest_dtype=dest_dtype, target_size=target_size) tile_buf_full = zeros_aligned((stackheight, ) + tuple(sig_shape), dtype=dest_dtype) tileshape = (stackheight, ) + tuple(sig_shape) with self._fileset as fileset: for outer_frame in range(start_at_frame, start_at_frame + num_frames, stackheight): if start_at_frame + num_frames - outer_frame < stackheight: end_frame = start_at_frame + num_frames current_stackheight = end_frame - outer_frame current_tileshape = ( current_stackheight, ) + tuple(sig_shape) tile_buf = zeros_aligned(current_tileshape, dtype=dest_dtype) else: current_stackheight = stackheight current_tileshape = tileshape tile_buf = tile_buf_full tile_slice = Slice(origin=(outer_frame, ) + sig_origin, shape=Shape(current_tileshape, sig_dims=sig_shape.dims)) if crop_to is not None: intersection = tile_slice.intersection_with(crop_to) if intersection.is_null(): continue fileset.read_images_multifile( start=outer_frame, stop=outer_frame + current_stackheight, out=tile_buf, crop_to=crop_to, ) yield DataTile(data=tile_buf, tile_slice=tile_slice)
def test_slice_intersect_2(): s1 = Slice( origin=(1, 1, 1, 1), shape=Shape((2, 2, 2, 2), sig_dims=2), ) s2 = Slice( origin=(0, 0, 0, 0), shape=Shape((1, 1, 1, 1), sig_dims=2), ) res = s1.intersection_with(s2) assert res == Slice( origin=(1, 1, 1, 1), shape=Shape((0, 0, 0, 0), sig_dims=2), ) assert res.is_null()
def get_tiles(self, crop_to=None): stackheight = self.tileshape.nav.size tile_buffer = np.zeros(self.tileshape.flatten_nav(), dtype=self.dataset.dtype) files_by_tile = list(grouper(self.files, stackheight)) sig_dims = self.tileshape.sig.dims dtype = self.dataset.dtype partition_origin = self.slice.origin tile_slice_shape = Shape(tile_buffer.shape, sig_dims=sig_dims) files_done = 0 for files in files_by_tile: if len(files) != stackheight: # allocate different size buffer for "rest-tile": tile_buffer = np.zeros( (len(files), ) + tuple(self.dataset.raw_shape.sig), dtype=dtype) tile_slice_shape = Shape(tile_buffer.shape, sig_dims=sig_dims) origin = tuple( sum(x) for x in zip(partition_origin, (files_done, 0, 0))) tile_slice = Slice( origin=origin, shape=tile_slice_shape, ) files_done += len(files) if crop_to is not None: intersection = tile_slice.intersection_with(crop_to) if intersection.is_null(): continue if stackheight == 1: with open(files[0], "rb") as f: f.readinto(tile_buffer) yield DataTile(data=tile_buffer, tile_slice=tile_slice) continue else: for idx, fn in enumerate(files): with open(fn, "rb") as f: f.readinto(tile_buffer[idx, ...]) yield DataTile(data=tile_buffer, tile_slice=tile_slice)
def get_tiles(self, crop_to=None, full_frames=False): # NOTE: full_frames is ignored, as we currently read whole frames only start_at_frame = self._start_frame num_frames = self._num_frames stackheight = self._get_stackheight() dtype = self.meta.dtype sig_shape = self.meta.shape.sig sig_origin = tuple([0] * len(sig_shape)) if crop_to is not None: sig_origin = tuple(crop_to.origin[-sig_shape.dims:]) sig_shape = crop_to.shape.sig tile_buf_full = np.zeros((stackheight, ) + tuple(sig_shape), dtype=dtype) tileshape = (stackheight, ) + tuple(sig_shape) for outer_frame in range(start_at_frame, start_at_frame + num_frames, stackheight): if start_at_frame + num_frames - outer_frame < stackheight: end_frame = start_at_frame + num_frames current_stackheight = end_frame - outer_frame current_tileshape = (current_stackheight, ) + tuple(sig_shape) tile_buf = np.zeros(current_tileshape, dtype=dtype) else: current_stackheight = stackheight current_tileshape = tileshape tile_buf = tile_buf_full tile_slice = Slice(origin=(outer_frame, ) + sig_origin, shape=Shape(current_tileshape, sig_dims=sig_shape.dims)) if crop_to is not None: intersection = tile_slice.intersection_with(crop_to) if intersection.is_null(): continue self._fileset.read_images( start=outer_frame, stop=outer_frame + current_stackheight, out=tile_buf, crop_to=crop_to, ) yield DataTile(data=tile_buf, tile_slice=tile_slice)
def _get_tiles_mmap(self, crop_to, full_frames, dest_dtype): start_at_frame = self._start_frame num_frames = self._num_frames sig_shape = self.meta.shape.sig sig_origin = tuple([0] * len(sig_shape)) if dest_dtype != self.meta.raw_dtype: raise ValueError( "using mmap with dtype conversion is not efficient") if crop_to is not None: sig_origin = tuple(crop_to.origin[-sig_shape.dims:]) sig_shape = crop_to.shape.sig if full_frames: sig_shape = self.meta.shape.sig fileset = self._fileset.get_for_range( start=start_at_frame, stop=start_at_frame + num_frames, ) with self._fileset as fileset: for f in fileset: # global start/stop indices: start = max(f.start_idx, self._start_frame) stop = min(f.end_idx, self._start_frame + num_frames) tile_slice = Slice(origin=(start, ) + sig_origin, shape=Shape( (stop - start, ) + tuple(sig_shape), sig_dims=sig_shape.dims)) arr = f.mmap() # limit to this partition (translate to file-local coords) arr = arr[start - f.start_idx:stop - f.start_idx] if crop_to is not None: intersection = tile_slice.intersection_with(crop_to) if intersection.is_null(): continue # crop to, signal part: arr = arr[(..., ) + tile_slice.get(sig_only=True)] yield DataTile(data=arr, tile_slice=tile_slice)
def get_tiles(self, crop_to=None, full_frames=False): # NOTE: full_frames ignored here because we currently only read full frames if crop_to is not None: if crop_to.shape.sig != self.meta.shape.sig: raise DataSetException( "DirectRawFileDataSet only supports whole-frame crops for now" ) stackheight = self.stackheight start_frame = self.start_frame num_frames = self.num_frames shape_sig = tuple(self.shape.sig) sig_dims = self.shape.sig.dims sig_size = self.shape.sig.size stop = start_frame + num_frames with self.reader.open_file() as reader: buf = reader.get_buffer(stackheight) c0 = itertools.count(start=start_frame, step=stackheight) for tile_start in c0: if tile_start >= stop: break # tile_height is the real height, which may be smaller than stackheight # at the end tile_height = min(stackheight, stop - tile_start) tileslice = Slice(origin=( tile_start, 0, 0, ), shape=Shape((tile_height, ) + shape_sig, sig_dims=sig_dims)) if crop_to is not None: intersection = tileslice.intersection_with(crop_to) if intersection.is_null(): continue reader.seek_frame(tile_start) data = reader.readinto(buf) # at partition boundary we may read more than requested, cut it off: data = data[:tile_height * sig_size] yield DataTile(data=data.reshape((tile_height, ) + shape_sig), tile_slice=tileslice)
def _read_full_frames(self, crop_to=None): with contextlib.ExitStack() as stack: frame_buf = np.zeros((1, 1860, 2048), dtype="float32") open_sectors = [ stack.enter_context(sector) for sector in self._sectors ] for frame in range(self._start_frame, self._start_frame + self._num_frames): tile_slice = Slice( origin=(frame, 0, 0), shape=Shape(frame_buf.shape, sig_dims=2), ) if crop_to is not None: intersection = tile_slice.intersection_with(crop_to) if intersection.is_null(): continue for s in open_sectors: s.read_full_frame( frame=frame, buf=frame_buf[:, :, s.idx * SECTOR_SIZE[1]:(s.idx + 1) * SECTOR_SIZE[1]]) yield DataTile(data=frame_buf, tile_slice=tile_slice)
def read_stacked(self, start_at_frame, num_frames, stackheight=16, dtype="float32", crop_to=None): """ Reads `stackheight` blocks into a single buffer. The blocks are read from consecutive frames, always from the same coordinates inside the sector of the frame. yields DataTiles of the shape (stackheight, 930, 16) (different tiles at the borders may be yielded if the stackheight doesn't evenly divide the total number of frames to read) """ tileshape = (stackheight, ) + BLOCK_SHAPE raw_data = mmap.mmap( fileno=self.f.fileno(), length=0, # whole file access=mmap.ACCESS_READ, ) tile_buf_full = zeros_aligned(tileshape, dtype=dtype) assert DATA_SIZE % 3 == 0 log.debug( "starting read_stacked with start_at_frame=%d, num_frames=%d, stackheight=%d", start_at_frame, num_frames, stackheight) for outer_frame in range(start_at_frame, start_at_frame + num_frames, stackheight): # log.debug("outer_frame=%d", outer_frame) # end of the selected frame range, calculate rest of stack: if start_at_frame + num_frames - outer_frame < stackheight: end_frame = start_at_frame + num_frames current_stackheight = end_frame - outer_frame current_tileshape = (current_stackheight, ) + BLOCK_SHAPE tile_buf = zeros_aligned(current_tileshape, dtype=dtype) else: current_stackheight = stackheight current_tileshape = tileshape tile_buf = tile_buf_full for blockidx in range(BLOCKS_PER_SECTOR_PER_FRAME): start_x = (self.idx + 1) * 256 - (16 * (blockidx % 16 + 1)) start_y = 930 * (blockidx // 16) tile_slice = Slice( origin=( outer_frame, start_y, start_x, ), shape=Shape(current_tileshape, sig_dims=self.sig_dims), ) if crop_to is not None: intersection = tile_slice.intersection_with(crop_to) if intersection.is_null(): continue offset = ( self.first_block_offset + outer_frame * BLOCK_SIZE * BLOCKS_PER_SECTOR_PER_FRAME + blockidx * BLOCK_SIZE) for frame in range(current_stackheight): block_offset = ( offset + frame * BLOCK_SIZE * BLOCKS_PER_SECTOR_PER_FRAME) input_start = block_offset + HEADER_SIZE input_end = block_offset + HEADER_SIZE + DATA_SIZE out = tile_buf[frame].reshape((-1, )) decode_uint12_le( inp=raw_data[input_start:input_end], out=out, ) yield DataTile(data=tile_buf, tile_slice=tile_slice) raw_data.close()