def default_frms6(lt_ctx): ds = lt_ctx.load( "frms6", path=FRMS6_TESTDATA_PATH, io_backend=MMapBackend(), ) return ds
def default_blo(lt_ctx): ds = lt_ctx.load( "blo", path=str(BLO_TESTDATA_PATH), io_backend=MMapBackend(), ) return ds
def default_k2is(lt_ctx): ds = lt_ctx.load( "k2is", path=K2IS_TESTDATA_PATH, io_backend=MMapBackend(), ) return ds
def default_empad(lt_ctx): ds = lt_ctx.load( "empad", path=EMPAD_XML, io_backend=MMapBackend(), ) yield ds
def default_dm(lt_ctx): files = list(sorted(glob(os.path.join(DM_TESTDATA_PATH, '*.dm4')))) ds = lt_ctx.load( "dm", files=files, io_backend=MMapBackend(), ) return ds
def medium_raw_float32(medium_raw_file_float32): filename, shape, dtype = medium_raw_file_float32 ds = RawFileDataSet(path=str(filename), nav_shape=shape[:2], dtype=dtype, sig_shape=shape[2:], io_backend=MMapBackend()) ds = ds.initialize(InlineJobExecutor()) yield ds
def default_mib(lt_ctx): nav_shape = (32, 32) ds = lt_ctx.load( "mib", path=MIB_TESTDATA_PATH, nav_shape=nav_shape, io_backend=MMapBackend(), ) ds.set_num_cores(4) return ds
def default_mib_readahead(lt_ctx): nav_shape = (32, 32) ds = lt_ctx.load( "mib", path=MIB_TESTDATA_PATH, nav_shape=nav_shape, io_backend=MMapBackend(enable_readahead_hints=True), ) ds.set_num_cores(4) return ds
def default_tvips(lt_ctx): nav_shape = (10, ) ds = lt_ctx.load( "tvips", path=TVIPS_TESTDATA_PATH, nav_shape=nav_shape, io_backend=MMapBackend(), ) ds.set_num_cores(4) return ds
def test_run_udf_with_io_backend(lt_ctx, default_raw): io_backend = MMapBackend(enable_readahead_hints=True) lt_ctx.load( "raw", path=default_raw._path, io_backend=io_backend, dtype="float32", nav_shape=(16, 16), sig_shape=(128, 128), ) res = lt_ctx.run_udf(dataset=default_raw, udf=SumUDF()) assert np.array(res['intensity']).shape == (128, 128)
def default_seq(lt_ctx): nav_shape = (8, 8) ds = lt_ctx.load( "seq", path=SEQ_TESTDATA_PATH, nav_shape=nav_shape, io_backend=MMapBackend(), ) ds.set_num_cores(4) assert tuple(ds.shape) == (8, 8, 128, 128) return ds
def default_raw(tmpdir_factory, default_raw_data): lt_ctx = lt.Context(executor=InlineJobExecutor()) datadir = tmpdir_factory.mktemp('data') filename = datadir + '/raw-test-default' default_raw_data.tofile(str(filename)) del default_raw_data ds = lt_ctx.load( "raw", path=str(filename), dtype="float32", nav_shape=(16, 16), sig_shape=(128, 128), io_backend=MMapBackend(), ) ds.set_num_cores(2) yield ds
0, # hTemplateFile ) f.close() else: def drop_cache(flist): for fname in flist: with open(fname, "rb") as f: os.posix_fadvise(f.fileno(), 0, 0, os.POSIX_FADV_DONTNEED) def get_testdata_prefixes(): pathstring = os.environ.get('LT_BENCH_PREFIXES') if pathstring is None: dirname = os.path.normpath( os.path.join(os.path.dirname(__file__), 'data')) listing = [os.path.join(dirname, p) for p in os.listdir(dirname)] prefixes = [p for p in listing if os.path.isdir(p)] else: prefixes = pathstring.split(';') for p in prefixes: assert os.path.isdir(p) return prefixes backends_by_name = { "mmap": MMapBackend(), "mmap_readahead": MMapBackend(enable_readahead_hints=True), "buffered": BufferedBackend(), }
# check that the offsets in the fileset are correct: assert f0.num_frames == 20 assert f1.num_frames == 20 assert f0.start_idx == 0 assert f0.end_idx == 20 assert f1.start_idx == 20 assert f1.end_idx == 40 lt_ctx.run_udf(dataset=dm_stack_of_3d, udf=SumUDF()) @pytest.mark.parametrize( "io_backend", ( BufferedBackend(), MMapBackend(), ), ) def test_positive_sync_offset(lt_ctx, io_backend): udf = SumSigUDF() sync_offset = 2 ds = lt_ctx.load( "dm", files=list(sorted(glob(os.path.join(DM_TESTDATA_PATH, '*.dm4')))), nav_shape=(4, 2), io_backend=io_backend, ) result = lt_ctx.run_udf(dataset=ds, udf=udf) result = result['intensity'].raw_data[sync_offset:]
def get_default_io_backend(cls) -> "IOBackend": import platform if platform.system() == "Windows": from libertem.io.dataset.base import BufferedBackend return BufferedBackend() return MMapBackend()