def flatten_nav(self, containing_shape): sig_dims = self.shape.sig.dims nav_dims = self.shape.dims - sig_dims containing_shape = tuple(containing_shape)[:nav_dims] origin = self.origin[:nav_dims] # validation for the nav_shape: # what are the preconditions that allow flattening? # # - nav part of the shape: must be in the form of: # # (1, 1, ..., N, M, M, ...) # # where N<=M and M is the corresponding part of # the shape of the dataset. # # - the origin must match the shape in the following way: # # (o1, o2, ..., oi, 0, 0, ...) # # where all oj are arbitraty (but in bounds) # state = 0 for cs, s, o in zip(containing_shape, self.shape.nav, origin): if state == 0: if s != 1: state = 1 assert s <= cs, "invalid nav_shape #1" elif state == 1: assert s == cs, "invalid nav_shape #2" assert o == 0, "invalid origin" nav_origin = np.ravel_multi_index(origin, containing_shape) nav_shape = np.product(tuple(self.shape.nav)) return Slice(origin=(nav_origin, ) + self.origin[nav_dims:], shape=Shape((nav_shape, ) + tuple(self.shape.sig), sig_dims=sig_dims))
def test_readranges_quad(bits_per_pixel): # make some read ranges: dataset_shape = Shape((4, 4, 512, 512), sig_dims=2) tiling_scheme = TilingScheme.make_for_shape( dataset_shape=dataset_shape, tileshape=Shape((2, 32, 512), sig_dims=2), ) sync_offset = 0 start_at_frame = 2 stop_before_frame = 6 roi = None # fileset_arr with one "file": fileset_arr = np.zeros((1, 4), dtype=np.int64) frame_header_bytes = 768 frame_footer_bytes = 0 # This is for formats like DM where the offset in each file can be different. # In this case, we only have the per-frame header: file_header_bytes = 0 # (start_idx, end_idx, file_idx, file_header_bytes) fileset_arr[0] = (0, dataset_shape.nav.size, 0, file_header_bytes) kwargs = dict( start_at_frame=start_at_frame, stop_before_frame=stop_before_frame, roi=roi, depth=tiling_scheme.depth, slices_arr=tiling_scheme.slices_array, fileset_arr=fileset_arr, sig_shape=tuple(tiling_scheme.dataset_shape.sig), sync_offset=sync_offset, bpp=bits_per_pixel, frame_header_bytes=frame_header_bytes, frame_footer_bytes=frame_footer_bytes, ) read_ranges = mib_2x2_get_read_ranges(**kwargs) # a 3-tuple is returned from get_read_ranges functions assert len(read_ranges) == 3 rr_slices, rr_ranges, rr_scheme_indices = read_ranges # - we read four frames, [2, 6) # - each frame is divided into 512/32 = 16 tiles in sig dimensions # - each tile has depth=2, so we have 4/2*16 = 32 tiles in total assert rr_slices.shape == ( 32, # number of tiles 2, # origin and shape 3, # indices in flattened dimensions ) # first and last tile slices: assert tuple(rr_slices[0, 0]) == (2, 0, 0) assert tuple(rr_slices[0, 1]) == (2, 32, 512) assert tuple(rr_slices[-1, 0]) == (4, 480, 0) assert tuple(rr_slices[-1, 1]) == (2, 32, 512) assert rr_ranges.shape == ( 32, # 32 tiles 128, # 128 reads per file per tile (ugh...) 4 # (file_idx, start, stop, flip) ) # each rr corresponds to a single row: row_size = 256 * bits_per_pixel // 8 # row size for a single quadrant, in bytes assert set((rr_ranges[:, :, 2] - rr_ranges[:, :, 1]).reshape( (-1, ))) == {row_size} # the offset for the first frame we are reading. # we have 3 headers and two full frames ahead of us: sig_size_bytes = 512 * 512 * bits_per_pixel // 8 frame_2_start = 3 * frame_header_bytes + 2 * sig_size_bytes q1_offset = 3 * row_size q2_offset = 2 * row_size q3_offset = 1 * row_size q4_offset = 0 * row_size stride = 4 * row_size # input layout is: [4 | 3 | 2 | 1] # # output layout is: # _________ # | 1 | 2 | # --------- # | 3 | 4 | # --------- # first tile reads only from Q1/Q2: # 0, 0 is rr for the first row of Q1 assert tuple(rr_ranges[0, 0]) == ( 0, # file index frame_2_start + q1_offset + 0 * stride, frame_2_start + q1_offset + 0 * stride + row_size, 0, # don't flip ) # 0, 1 is rr for the first row of Q2 assert tuple(rr_ranges[0, 1]) == ( 0, # file index frame_2_start + q2_offset + 0 * stride, frame_2_start + q2_offset + 0 * stride + row_size, 0, # don't flip ) # 0, 2 is rr for the second row of Q1 assert tuple(rr_ranges[0, 2]) == ( 0, # file index frame_2_start + q1_offset + 1 * stride, frame_2_start + q1_offset + 1 * stride + row_size, 0, # don't flip ) # 0, 3 is rr for the second row of Q2 assert tuple(rr_ranges[0, 3]) == ( 0, # file index frame_2_start + q2_offset + 1 * stride, frame_2_start + q2_offset + 1 * stride + row_size, 0, # don't flip ) # flip is set in half of the rr's assert set(rr_ranges[:8, :, -1].reshape((-1, ))) == {0} assert set(rr_ranges[8:16, :, -1].reshape((-1, ))) == {1} # and so on... # Let's check out another tile that actually reads from the flipped Q3/Q4 # (in this case, let's have a look at the last row of the *output*, but still # in the first "tile block", so depth starting at frame 2): last_tile_idx = 15 assert tuple(rr_slices[last_tile_idx, 0]) == (2, 480, 0) assert tuple(rr_slices[last_tile_idx, 1]) == (2, 32, 512) # we have in total a tile depth of 2, which translates to 128 # read ranges (depth 2, 32 rows per tile, two input rows per output row) assert rr_ranges[last_tile_idx].shape == (128, 4) # so this means the first input row of Q3 appears at the end of the # first half of the read ranges for the full tile. # 15, 62 is rr for the first *input* row of Q3, in depth=0 of the tile assert tuple(rr_ranges[last_tile_idx, 62]) == ( 0, # file index frame_2_start + q3_offset + 0 * stride, frame_2_start + q3_offset + 0 * stride + row_size, 1, # flip ) # 15, 63 is rr for the first *input* row of Q4, in depth=0 of the tile assert tuple(rr_ranges[last_tile_idx, 63]) == ( 0, # file index frame_2_start + q4_offset + 0 * stride, frame_2_start + q4_offset + 0 * stride + row_size, 1, # flip ) # for each tile, this is an index into the tiling scheme and indirectly # gives us the sig part of the tile slice: assert rr_scheme_indices.shape == (32, ) assert set(rr_scheme_indices) == set(range(16))
def encode_roundtrip_quad(encode, bits_per_pixel, input_data=None, dataset_shape=None, tileshape=None, start_at_frame=2, stop_before_frame=6): if dataset_shape is None: # make some read ranges: dataset_shape = (6, 512, 512) dataset_shape = Shape(dataset_shape, sig_dims=2) if tileshape is None: tileshape = (2, 128, 512) tiling_scheme = TilingScheme.make_for_shape( dataset_shape=dataset_shape, tileshape=Shape(tileshape, sig_dims=2), ) sync_offset = 0 roi = None frame_header_bytes = 768 image_size_bytes = dataset_shape.sig.size * bits_per_pixel // 8 if bits_per_pixel in (1, 8): native_dtype = np.uint8 elif bits_per_pixel == 16: native_dtype = np.uint16 fields: HeaderDict = { 'header_size_bytes': frame_header_bytes, 'dtype': native_dtype, 'mib_dtype': 'R64', 'mib_kind': 'r', # remove padding from `bits_per_pixel` 'bits_per_pixel': { 1: 1, 8: 6, 16: 12 }[bits_per_pixel], 'image_size': (512, 512), 'image_size_bytes': image_size_bytes, 'sequence_first_image': 1, 'filesize': dataset_shape.nav.size * (image_size_bytes + frame_header_bytes), 'num_images': dataset_shape.nav.size, 'num_chips': 4, 'sensor_layout': (2, 2), } file = MIBFile( path="", start_idx=0, end_idx=dataset_shape.nav.size, native_dtype=native_dtype, sig_shape=dataset_shape.sig, frame_header=frame_header_bytes, file_header=0, header=fields, ) fileset = MIBFileSet(files=[file], header=fields, frame_header_bytes=frame_header_bytes) backend = MMapBackendImplInMem() max_value = (1 << bits_per_pixel) - 1 if input_data is None: data_full = np.random.randint(0, max_value + 1, tuple(dataset_shape.flatten_nav())) # make sure min/max values are indeed hit: data_full.reshape((-1, ))[0] = max_value data_full.reshape((-1, ))[-1] = 0 assert np.max(data_full) == max_value assert np.min(data_full) == 0 else: data_full = input_data.reshape(dataset_shape.flatten_nav()) data = data_full[start_at_frame:stop_before_frame] # we need headers in-between, in contrast to the frame-by-frame decoding, the decoder # expects contiguous input data and we can't slice them away beforehand: encoded_data = encode_quad(encode, data_full, bits_per_pixel, with_headers=True) decoded = np.zeros_like(data) # that's the "interface" we made up for the in-mem mmap file above: file.data = encoded_data # wrapping the numba decoder function: decoder = MIBDecoder(header=fields) outer_slice = Slice( origin=(start_at_frame, 0, 0), shape=dataset_shape.flatten_nav(), ) read_ranges = fileset.get_read_ranges( start_at_frame=start_at_frame, stop_before_frame=stop_before_frame, dtype=native_dtype, tiling_scheme=tiling_scheme, sync_offset=sync_offset, roi=roi, ) for tile in backend.get_tiles( tiling_scheme=tiling_scheme, fileset=fileset, read_ranges=read_ranges, roi=roi, native_dtype=np.uint8, read_dtype=np.float32, decoder=decoder, sync_offset=0, corrections=None, ): slice_shifted = tile.tile_slice.shift(outer_slice) decoded[slice_shifted.get()] = tile.reshape(tile.tile_slice.shape) assert_allclose(data, decoded) return data, decoded
import numpy as np import pytest from libertem.common.math import prod from libertem.common.shape import Shape @pytest.mark.parametrize( ('sequence', 'ref', 'typ'), [([], 1, int), (np.array([]), 1, int), ((1, 2, 3), 6, int), ((-11, 2, 3), -66, int), ((1., 2, False), 0, ValueError), (np.array((1., 1 + 2j, 1), dtype=np.complex128), 1, ValueError), ((2**32, 2**32, 2**32), 2**96, int), (np.array((2**62, 2**62, 2**62), dtype=np.int64), 2**(3 * 62), int), (Shape((1, 2, 3), sig_dims=1).nav, 2, int)]) def test_prod(sequence, ref, typ): if typ is int: res = prod(sequence) assert res == ref assert isinstance(res, typ) else: with pytest.raises(typ): res = prod(sequence)