def _do_initialize(self): if self.ds_path is None: datasets = _get_datasets(self.path) datasets_list = sorted(datasets, key=lambda i: i[1], reverse=True) name, size, shape, dtype, compression, chunks = datasets_list[0] self.ds_path = name with self.get_reader().get_h5ds() as h5ds: self._dtype = h5ds.dtype shape = h5ds.shape if len(shape) == self.sig_dims: # shape = (1,) + shape -> this leads to indexing errors down the line # so we currently don't support opening 2D HDF5 files raise DataSetException( "2D HDF5 files are currently not supported") self._shape = Shape(shape, sig_dims=self.sig_dims) self._image_count = self._shape.nav.size self._meta = DataSetMeta( shape=self.shape, raw_dtype=self._dtype, sync_offset=self._sync_offset, image_count=self._image_count, ) self._chunks = h5ds.chunks self._compression = h5ds.compression if self._compression is not None: warnings.warn( "Loading compressed HDF5, performance can be worse than with other formats", RuntimeWarning) self._nav_shape_product = self._shape.nav.size self._sync_offset_info = self.get_sync_offset_info() return self
def get_macrotile(self, dest_dtype="float32", roi=None): ''' Return a single tile for the entire partition. This is useful to support process_partiton() in UDFs and to construct dask arrays from datasets. ''' tiling_scheme = TilingScheme.make_for_shape( tileshape=self.shape, dataset_shape=self.meta.shape, ) try: return next( self.get_tiles( tiling_scheme=tiling_scheme, dest_dtype=dest_dtype, roi=roi, )) except StopIteration: tile_slice = Slice( origin=(self.slice.origin[0], 0, 0), shape=Shape((0, ) + tuple(self.slice.shape.sig), sig_dims=2), ) return DataTile( np.zeros(tile_slice.shape, dtype=dest_dtype), tile_slice=tile_slice, scheme_idx=0, )
def test_mask_caching_1(): input_masks = [ lambda: np.ones((128, 128)), lambda: np.zeros((128, 128)), ] mask_container = MaskContainer(mask_factories=input_masks, dtype="float32") shape = Shape((16, 16, 128, 128), sig_dims=2) slice_ = Slice(origin=(0, 0, 0, 0), shape=shape) mask_container[slice_] cache_info = mask_container._get_masks_for_slice.cache_info() assert cache_info.hits == 0 assert cache_info.misses == 1 mask_container[slice_] cache_info = mask_container._get_masks_for_slice.cache_info() assert cache_info.hits == 1 assert cache_info.misses == 1 slice_ = Slice(origin=(0, 1, 0, 0), shape=shape) mask_container[slice_] cache_info = mask_container._get_masks_for_slice.cache_info() assert cache_info.hits == 2 assert cache_info.misses == 1
def get_partitions(self): fs = self._fileset num_frames = self.shape.nav.size f_per_part = num_frames // self._get_num_partitions() c0 = itertools.count(start=0, step=f_per_part) c1 = itertools.count(start=f_per_part, step=f_per_part) for (start, stop) in zip(c0, c1): if start >= num_frames: break stop = min(stop, num_frames) part_slice = Slice(origin=( start, 0, 0, ), shape=Shape(((stop - start), SECTOR_SIZE[0], SECTOR_SIZE[1] * NUM_SECTORS), sig_dims=2)) yield K2ISPartition( meta=self._meta, partition_slice=part_slice, sectors=fs.sectors, start_frame=start, num_frames=stop - start, )
def test_roi_3(hdf5, lt_ctx): ds = H5DataSet( path=hdf5.filename, ds_path="data", target_size=12800 * 2, ) ds = ds.initialize(lt_ctx.executor) roi = np.zeros(ds.shape.flatten_nav().nav, dtype=bool) roi[24] = 1 tileshape = Shape((16, ) + tuple(ds.shape.sig), sig_dims=ds.shape.sig.dims) tiling_scheme = TilingScheme.make_for_shape( tileshape=tileshape, dataset_shape=ds.shape, ) tiles = [] for p in ds.get_partitions(): for tile in p.get_tiles(tiling_scheme=tiling_scheme, dest_dtype="float32", roi=roi): print("tile:", tile) tiles.append(tile) assert len(tiles) == 1 assert tiles[0].tile_slice.shape.nav.size == 1 assert tuple(tiles[0].tile_slice.shape.sig) == (16, 16) assert tiles[0].tile_slice.origin == (0, 0, 0) assert np.allclose(tiles[0].data, hdf5['data'][4, 4])
def __init__(self, path, tileshape=None, scan_size=None): super().__init__() self._sig_dims = 2 self._path = path if tileshape is None: tileshape = (1, 3, 256, 256) tileshape = Shape(tileshape, sig_dims=self._sig_dims) self._tileshape = tileshape if scan_size is not None: scan_size = tuple(scan_size) else: if not path.lower().endswith(".hdr"): raise ValueError( "either scan_size needs to be passed, or path needs to point to a .hdr file" ) self._scan_size = scan_size self._filename_cache = None self._files_sorted = None # ._preread_headers() calls ._files() which passes the cached headers down to MIBFile, # if they exist. So we need to make sure to initialize self._headers # before calling _preread_headers! self._headers = {} self._meta = None self._total_filesize = None self._sequence_start = None
def initialize(self, executor): self._filesize = executor.run_function(self._get_filesize) if self._same_offset: metadata = executor.run_function(_get_metadata, self._get_files()[0]) self._offsets = { fn: metadata['offset'] for fn in self._get_files() } self._z_sizes = {fn: metadata['zsize'] for fn in self._get_files()} else: metadata = dict( zip( self._get_files(), executor.map(_get_metadata, self._get_files()), )) self._offsets = { fn: metadata[fn]['offset'] for fn in self._get_files() } self._z_sizes = { fn: metadata[fn]['zsize'] for fn in self._get_files() } self._fileset = executor.run_function(self._get_fileset) first_file = next(self._fileset.files_from(0)) nav_dims = self._get_scan_size() shape = nav_dims + tuple(first_file.sig_shape) sig_dims = len(first_file.sig_shape) self._meta = DataSetMeta( shape=Shape(shape, sig_dims=sig_dims), raw_dtype=first_file.native_dtype, ) return self
def get_tiles(self, crop_to=None, full_frames=False): stackheight = self.tileshape.nav.size num_tiles = self.partfile.fields['num_images'] // stackheight tshape = self.tileshape.flatten_nav() sig_origin = (0, 0) if crop_to is not None and tshape.sig != crop_to.shape.sig: tshape = Shape(tuple(tshape.nav) + tuple(crop_to.shape.sig), sig_dims=tshape.sig.dims) sig_origin = crop_to.origin[1:] data = np.ndarray(tshape, dtype=self.dtype) for t in range(num_tiles): tile_slice = Slice( origin=(t * stackheight + self.slice.origin[0], ) + sig_origin, shape=tshape) if crop_to is not None: intersection = tile_slice.intersection_with(crop_to) if intersection.is_null(): continue self.partfile.read_frames(num=stackheight, offset=t * stackheight, out=data, crop_to=crop_to) assert all( [item > 0 for item in tile_slice.shift(self.slice).shape]) assert all( [item >= 0 for item in tile_slice.shift(self.slice).origin]) yield DataTile(data=data, tile_slice=tile_slice)
def test_partition_shape_4(): assert get_partition_shape(Shape((128, 15, 15, 16, 16), sig_dims=2), target_size_items=15 * 512) == (( 1, 2, 15, ))
def get_partitions(self): num_frames = self.shape.nav.size f_per_part = num_frames // self._get_num_partitions() c0 = itertools.count(start=0, step=f_per_part) c1 = itertools.count(start=f_per_part, step=f_per_part) for (start, stop) in zip(c0, c1): if start >= num_frames: break stop = min(stop, num_frames) part_slice = Slice(origin=( start, 0, 0, ), shape=Shape( ((stop - start), ) + tuple(self.shape.sig), sig_dims=self.shape.sig.dims)) yield SERPartition( meta=self._meta, partition_slice=part_slice, reader=SERReader(path=self._path, emipath=self._emipath), start_frame=start, num_frames=stop - start, )
def test_apply_mask_job(default_k2is, lt_ctx): mask = np.ones((1860, 2048)) tileshape = Shape( (16, 930, 16), sig_dims=2, ) tiling_scheme = TilingScheme.make_for_shape( tileshape=tileshape, dataset_shape=default_k2is.shape, ) job = ApplyMasksJob( dataset=default_k2is, mask_factories=[lambda: mask], tiling_scheme=tiling_scheme, ) out = job.get_result_buffer() executor = InlineJobExecutor() for tiles in executor.run_job(job): for tile in tiles: tile.reduce_into_result(out) results = lt_ctx.run(job) assert results[0].shape == (34 * 35,) # there should be _something_ in each result pixel for px in results[0].reshape((-1,)): assert not np.isclose(px, 0)
def test_missing_frames(lt_ctx, raw_data_8x8x8x8_path, io_backend): ds = RawFileDataSet( path=raw_data_8x8x8x8_path, nav_shape=(10, 8), sig_shape=(8, 8), dtype="float32", io_backend=io_backend, ) ds.set_num_cores(4) ds = ds.initialize(lt_ctx.executor) tileshape = Shape((4, ) + tuple(ds.shape.sig), sig_dims=ds.shape.sig.dims) tiling_scheme = TilingScheme.make_for_shape( tileshape=tileshape, dataset_shape=ds.shape, ) for p in ds.get_partitions(): for t in p.get_tiles(tiling_scheme=tiling_scheme): pass assert p._start_frame == 60 assert p._num_frames == 20 assert p.slice.origin == (60, 0, 0) assert p.slice.shape[0] == 20 assert t.tile_slice.origin == (60, 0, 0) assert t.tile_slice.shape[0] == 4
def get_tiles(self, tiling_scheme, dest_dtype="float32", roi=None): shape = Shape((1, ) + tuple(self.shape.sig), sig_dims=self.shape.sig.dims) self.validate_tiling_scheme(tiling_scheme) start = self._start_frame stop = start + self._num_frames if roi is None: indices = np.arange(start, stop) offset = start else: indices = _roi_to_indices(roi, start, stop) offset = np.count_nonzero(roi.reshape((-1, ))[:start]) with fileSER(self._path) as f: for num, idx in enumerate(indices): tile_slice = Slice(origin=(num + offset, 0, 0), shape=shape) data, metadata = f.getDataset(int(idx)) if data.dtype != np.dtype(dest_dtype): data = data.astype(dest_dtype) data = data.reshape(shape) self._preprocess(data, tile_slice) yield DataTile( data, tile_slice=tile_slice, scheme_idx=0, )
def get_partitions(self): ds_shape = Shape(self.shape, sig_dims=self.sig_dims) ds_slice = Slice(origin=[0] * len(self.shape), shape=ds_shape) target_size = self.target_size if target_size is None: if self._compression is None: target_size = 512 * 1024 * 1024 else: target_size = 256 * 1024 * 1024 partition_shape = self.partition_shape( target_size=target_size, dtype=self.dtype, ) + tuple(self.shape.sig) # if the data is chunked in the navigation axes, choose a compatible # partition size (even important for non-compressed data!) chunks = self._chunks if chunks is not None and not _have_contig_chunks(chunks, ds_shape): partition_shape = _partition_shape_for_chunking(chunks, ds_shape) for pslice in ds_slice.subslices(partition_shape): yield H5Partition( meta=self._meta, reader=self.get_reader(), partition_slice=pslice.flatten_nav(self.shape), slice_nd=pslice, io_backend=self.get_io_backend(), chunks=self._chunks, )
def test_for_datatile_2(masks): tile = DataTile( tile_slice=Slice(origin=(0, 0, 0), shape=Shape((2 * 2, 10, 10), sig_dims=2)), data=np.ones((2 * 2, 10, 10)) ) slice_ = masks.get_masks_for_slice(tile.tile_slice) assert slice_.shape == (100, 5)
def test_simple(default_cached_ds): parts = list(default_cached_ds.get_partitions()) p = parts[0] datadir = default_cached_ds._cache_path default_raw = default_cached_ds._source_ds tileshape = Shape((16, ) + tuple(default_raw.shape.sig), sig_dims=default_raw.shape.sig.dims) tiling_scheme = TilingScheme.make_for_shape( tileshape=tileshape, dataset_shape=default_raw.shape, ) print(datadir, os.listdir(datadir)) print(p) t_cache = next(p.get_tiles(tiling_scheme=tiling_scheme)) t_orig = next( next(default_raw.get_partitions()).get_tiles( tiling_scheme=tiling_scheme)) print(t_cache) print(t_orig) assert np.allclose(t_cache.data, t_orig.data) for p in default_cached_ds.get_partitions(): for tile in p.get_tiles(tiling_scheme=tiling_scheme): pass
def __init__(self, tileshape=None, num_partitions=None, data=None, sig_dims=2, check_cast=True, crop_frames=False, tiledelay=None, datashape=None): # For HTTP API testing purposes: Allow to create empty dataset with given shape if data is None: assert datashape is not None data = np.zeros(datashape, dtype=np.float32) if num_partitions is None: num_partitions = psutil.cpu_count(logical=False) if tileshape is None: sig_shape = data.shape[-sig_dims:] target = 2**20 framesize = np.prod(sig_shape) framecount = max(1, min(np.prod(data.shape[:-sig_dims]), int(target / framesize))) tileshape = (framecount, ) + sig_shape assert len(tileshape) == sig_dims + 1 self.data = data self.tileshape = Shape(tileshape, sig_dims=sig_dims) self.num_partitions = num_partitions self.sig_dims = sig_dims self._meta = DataSetMeta( shape=self.shape, raw_dtype=self.data.dtype, ) self._check_cast = check_cast self._crop_frames = crop_frames self._tiledelay = tiledelay
async def test_fd_limit(async_executor): import resource import psutil # set soft limit, throws errors but allows to raise it # again afterwards: proc = psutil.Process() oldlimit = resource.getrlimit(resource.RLIMIT_NOFILE) resource.setrlimit(resource.RLIMIT_NOFILE, (proc.num_fds() + 24, oldlimit[1])) print("fds", proc.num_fds()) try: data = _mk_random(size=(1, 16, 16), dtype='<u2') dataset = MemoryDataSet(data=data, tileshape=(1, 16, 16), num_partitions=1) slice_ = Slice(origin=(0, 0, 0), shape=Shape((1, 16, 16), sig_dims=2)) job = PickFrameJob(dataset=dataset, slice_=slice_) for i in range(32): print(i) print(proc.num_fds()) async for tiles in async_executor.run_job(job, cancel_id="42"): pass finally: resource.setrlimit(resource.RLIMIT_NOFILE, oldlimit)
def _do_initialize(self): self._headers = self._preread_headers() self._files_sorted = list( sorted(self._files(), key=lambda f: f.fields['sequence_first_image'])) try: first_file = self._files_sorted[0] except IndexError: raise DataSetException("no files found") if self._scan_size is None: hdr = read_hdr_file(self._path) self._scan_size = scan_size_from_hdr(hdr) shape = Shape(self._scan_size + first_file.fields['image_size'], sig_dims=self._sig_dims) dtype = first_file.fields['dtype'] meta = DataSetMeta(shape=shape, raw_dtype=dtype, iocaps={"FRAME_CROPS", "MMAP", "FULL_FRAMES"}) if first_file.fields['mib_dtype'] == "r64": meta.iocaps.remove("MMAP") self._meta = meta self._total_filesize = sum( os.stat(path).st_size for path in self._filenames()) self._sequence_start = first_file.fields['sequence_first_image'] self._files_sorted = list( sorted(self._files(), key=lambda f: f.fields['sequence_first_image'])) return self
def _get_tiles_w_copy( self, tiling_scheme, fileset, read_ranges, read_dtype, native_dtype, decoder=None, corrections=None, ): if decoder is None: decoder = DtypeConversionDecoder() decode = decoder.get_decode( native_dtype=np.dtype(native_dtype), read_dtype=np.dtype(read_dtype), ) r_n_d = self._r_n_d = self.get_read_and_decode(decode) native_dtype = decoder.get_native_dtype(native_dtype, read_dtype) mmaps = List() for fh in fileset: mmaps.append(np.frombuffer(fh.raw_mmap(), dtype=np.uint8)) sig_dims = tiling_scheme.shape.sig.dims ds_shape = np.array(tiling_scheme.dataset_shape) largest_slice = sorted([ (np.prod(s_.shape), s_) for _, s_ in tiling_scheme.slices ], key=lambda x: x[0], reverse=True)[0][1] buf_shape = (tiling_scheme.depth,) + tuple(largest_slice.shape) need_clear = decoder.do_clear() with self._buffer_pool.empty(buf_shape, dtype=read_dtype) as out_decoded: out_decoded = out_decoded.reshape((-1,)) slices = read_ranges[0] shape_prods = np.prod(slices[..., 1, :], axis=1) ranges = read_ranges[1] scheme_indices = read_ranges[2] for idx in range(slices.shape[0]): origin = slices[idx, 0] shape = slices[idx, 1] tile_slice = Slice( origin=origin, shape=Shape(shape, sig_dims=sig_dims) ) tile_ranges = ranges[idx] scheme_idx = scheme_indices[idx] # if idx < slices.shape[0] - 1: # self._prefetch_for_tile(fileset, ranges[idx + 1]) # pass out_cut = out_decoded[:shape_prods[idx]].reshape((shape[0], -1)) data = r_n_d( idx, mmaps, sig_dims, tile_ranges, out_cut, native_dtype, do_zero=need_clear, origin=origin, shape=shape, ds_shape=ds_shape, ) data = data.reshape(shape) self.preprocess(data, tile_slice, corrections) yield DataTile( data, tile_slice=tile_slice, scheme_idx=scheme_idx, )
def test_missing_frames(lt_ctx, io_backend): """ there can be some frames missing at the end """ # one full row of additional frames in the data set than the number of files nav_shape = (3, 5) ds = lt_ctx.load( "dm", files=list(sorted(glob(os.path.join(DM_TESTDATA_PATH, '*.dm4')))), nav_shape=nav_shape, io_backend=io_backend, ) ds.set_num_cores(4) tileshape = Shape((1, ) + tuple(ds.shape.sig), sig_dims=ds.shape.sig.dims) tiling_scheme = TilingScheme.make_for_shape( tileshape=tileshape, dataset_shape=ds.shape, ) for p in ds.get_partitions(): for t in p.get_tiles(tiling_scheme=tiling_scheme): pass assert p._start_frame == 12 assert p._num_frames == 3 assert p.slice.origin == (12, 0, 0) assert p.slice.shape[0] == 3 assert t.tile_slice.origin == (9, 0, 0) assert t.tile_slice.shape[0] == 1
def _do_initialize(self): self._filenames = list(sorted(glob.glob(self._pattern()))) self._hdr_info = self._read_hdr_info() self._headers = [ _read_file_header(path) for path in self._files() ] header = self._headers[0] raw_frame_size = header['height'], header['width'] # frms6 frames are folded in a specific way, this is the shape after unfolding: frame_size = 2 * header['height'], header['width'] // 2 assert header['width'] % 2 == 0 hdr = self._get_hdr_info() bin_factor = hdr['readoutmode']['bin'] if bin_factor > 1: frame_size = (frame_size[0] * bin_factor, frame_size[1]) sig_dims = 2 # FIXME: is there a different cameraMode that doesn't output 2D signals? preferred_dtype = np.dtype("<u2") if self._enable_offset_correction: preferred_dtype = np.dtype("float32") self._meta = DataSetMeta( raw_dtype=np.dtype("<u2"), dtype=preferred_dtype, metadata={'raw_frame_size': raw_frame_size}, shape=Shape(tuple(hdr['stemimagesize']) + frame_size, sig_dims=sig_dims), ) self._dark_frame = self._get_dark_frame() self._gain_map = self._get_gain_map() self._total_filesize = sum( os.stat(path).st_size for path in self._files() ) return self
def _read_full_frames(self, crop_to=None, dest_dtype="float32", roi=None): with contextlib.ExitStack() as stack: frame_buf = zeros_aligned((1, 1860, 2048), dtype=dest_dtype) open_sectors = [ stack.enter_context(sector) for sector in self._sectors ] frame_offset = 0 if roi is not None: roi = roi.reshape((-1, )) frame_offset = np.count_nonzero(roi[:self._start_frame]) frames_read = 0 for frame in range(self._start_frame, self._start_frame + self._num_frames): if roi is not None and not roi[frame]: continue origin = frame if roi is not None: origin = frame_offset + frames_read tile_slice = Slice( origin=(origin, 0, 0), shape=Shape(frame_buf.shape, sig_dims=2), ) if crop_to is not None: intersection = tile_slice.intersection_with(crop_to) if intersection.is_null(): continue for s in open_sectors: s.read_full_frame( frame=frame, buf=frame_buf[:, :, s.idx * SECTOR_SIZE[1]:(s.idx + 1) * SECTOR_SIZE[1]]) yield DataTile(data=frame_buf, tile_slice=tile_slice) frames_read += 1
def _get_tiles_fullframe(self, tiling_scheme, dest_dtype="float32", roi=None): # assert len(tiling_scheme) == 1 logger.debug("reading up to frame idx %d for this partition", self._end_idx) pool = self._data_source.pool.get_impl( read_upto_frame=self._end_idx, # chunk_size=11, chunk_size=tiling_scheme.depth, ) to_read = self._end_idx - self._start_idx with pool: while to_read > 0: # logger.debug( # "LivePartition.get_tiles: to_read=%d, start_idx=%d end_idx=%d", # to_read, self._start_idx, self._end_idx, # ) with pool.get_result() as res_wrapped: frames_in_tile = res_wrapped.stop - res_wrapped.start tile_shape = Shape( (frames_in_tile, ) + tuple(tiling_scheme[0].shape), sig_dims=2) tile_slice = Slice( origin=(res_wrapped.start, ) + (0, 0), shape=tile_shape, ) yield DataTile( res_wrapped.buf, tile_slice=tile_slice, scheme_idx=0, ) to_read -= frames_in_tile logger.debug("LivePartition.get_tiles: end of method")
def _get_tiles_with_roi(self, roi, dest_dtype): flat_roi = roi.reshape((-1,)) roi = roi.reshape(self.meta.shape.nav) result_shape = Shape((1,) + tuple(self.meta.shape.sig), sig_dims=self.meta.shape.sig.dims) sig_origin = tuple([0] * self.meta.shape.sig.dims) frames_read = 0 start_at_frame = self.slice.origin[0] frame_offset = np.count_nonzero(flat_roi[:start_at_frame]) indices = _roi_to_nd_indices(roi, self.slice_nd) with self.reader.get_h5ds() as h5ds: for idx in indices: tile_slice = Slice( origin=(frames_read + frame_offset,) + sig_origin, shape=result_shape, ) yield DataTile( h5ds[idx].reshape(result_shape), tile_slice=tile_slice, # there is only a single slice in the tiling scheme, so our # scheme_idx is constant 0 scheme_idx=0, ) frames_read += 1
def get_tiles(self, tiling_scheme, dest_dtype="float32", roi=None): yield from self._get_tiles_fullframe(tiling_scheme, dest_dtype, roi) return # assert len(tiling_scheme) == 1 print(tiling_scheme) pool = self._data_source.pool.get_impl( read_upto_frame=self._end_idx, chunk_size=tiling_scheme.depth, ) to_read = int(self._end_idx - self._start_idx) nav_slices_raw = [(..., ) + slice_.get(sig_only=True) for idx, slice_ in tiling_scheme.slices] with pool: while to_read > 0: with pool.get_result() as res_wrapped: frames_in_tile = res_wrapped.stop - res_wrapped.start for (idx, slice_), nav_slice_raw in zip(tiling_scheme.slices, nav_slices_raw): tile_shape = Shape( (frames_in_tile, ) + tuple(slice_.shape), sig_dims=2) tile_slice = Slice( origin=(res_wrapped.start, ) + tuple(slice_.origin), shape=tile_shape, ) sliced_res = res_wrapped.buf[nav_slice_raw] yield DataTile(sliced_res, tile_slice=tile_slice, scheme_idx=idx) to_read -= frames_in_tile
def test_for_datatile_with_frame_origin(masks): tile = DataTile(tile_slice=Slice(origin=(10, 10, 10, 10), shape=Shape((2, 2, 1, 5), sig_dims=2)), data=np.ones((2, 2, 1, 5))) slice_ = masks.get_masks_for_slice(tile.tile_slice) print(slice_) np.allclose( slice_, np.array([ 1, 0, 1, 0, 10, 1, 0, 1, 0, 11, 1, 0, 1, 0, 12, 1, 0, 1, 0, 13, 1, 0, 1, 0, 14, ]).reshape((5, 5)))
def test_get_signal_only(): s = Slice(origin=(0, 0, 0, 0), shape=Shape((1, 1, 1, 1), sig_dims=2)) assert s.get(sig_only=True) == ( slice(0, 1), slice(0, 1), )
def test_mask_caching_1(): input_masks = [ lambda: np.ones((128, 128)), lambda: np.zeros((128, 128)), ] mask_container = MaskContainer(mask_factories=input_masks, dtype="float32") shape = Shape((16 * 16, 128, 128), sig_dims=2) slice_ = Slice(origin=(0, 0, 0), shape=shape) mask_container.get(slice_) key = (mask_container.dtype, False, True, 'numpy') cache_info = mask_container._get_masks_for_slice[key].cache_info() assert cache_info.hits == 0 assert cache_info.misses == 1 mask_container.get(slice_) cache_info = mask_container._get_masks_for_slice[key].cache_info() assert cache_info.hits == 1 assert cache_info.misses == 1 slice_ = Slice(origin=(1, 0, 0), shape=shape) mask_container.get(slice_) cache_info = mask_container._get_masks_for_slice[key].cache_info() assert cache_info.hits == 2 assert cache_info.misses == 1
def initialize(self, executor): self._header = h = executor.run_function(self._read_header) NY = int(h['NY']) NX = int(h['NX']) DP_SZ = int(h['DP_SZ']) self._image_count = NY * NX if self._nav_shape is None: self._nav_shape = (NY, NX) if self._sig_shape is None: self._sig_shape = (DP_SZ, DP_SZ) elif int(prod(self._sig_shape)) != (DP_SZ * DP_SZ): raise DataSetException("sig_shape must be of size: %s" % (DP_SZ * DP_SZ)) self._nav_shape_product = int(prod(self._nav_shape)) self._sync_offset_info = self.get_sync_offset_info() self._shape = Shape(self._nav_shape + self._sig_shape, sig_dims=len(self._sig_shape)) self._meta = DataSetMeta( shape=self._shape, raw_dtype=np.dtype("u1"), sync_offset=self._sync_offset, image_count=self._image_count, ) self._filesize = executor.run_function(self._get_filesize) return self