def default_frms6(lt_ctx):
ds = lt_ctx.load(
"frms6",
path=FRMS6_TESTDATA_PATH,
io_backend=MMapBackend(),
)
return ds
def default_blo(lt_ctx):
ds = lt_ctx.load(
"blo",
path=str(BLO_TESTDATA_PATH),
io_backend=MMapBackend(),
)
return ds
def default_k2is(lt_ctx):
ds = lt_ctx.load(
"k2is",
path=K2IS_TESTDATA_PATH,
io_backend=MMapBackend(),
)
return ds
def default_empad(lt_ctx):
ds = lt_ctx.load(
"empad",
path=EMPAD_XML,
io_backend=MMapBackend(),
)
yield ds
def default_dm(lt_ctx):
files = list(sorted(glob(os.path.join(DM_TESTDATA_PATH, '*.dm4'))))
ds = lt_ctx.load(
"dm",
files=files,
io_backend=MMapBackend(),
)
return ds
def medium_raw_float32(medium_raw_file_float32):
filename, shape, dtype = medium_raw_file_float32
ds = RawFileDataSet(path=str(filename),
nav_shape=shape[:2],
dtype=dtype,
sig_shape=shape[2:],
io_backend=MMapBackend())
ds = ds.initialize(InlineJobExecutor())
yield ds
def default_mib(lt_ctx):
nav_shape = (32, 32)
ds = lt_ctx.load(
"mib",
path=MIB_TESTDATA_PATH,
nav_shape=nav_shape,
io_backend=MMapBackend(),
)
ds.set_num_cores(4)
return ds
def default_mib_readahead(lt_ctx):
nav_shape = (32, 32)
ds = lt_ctx.load(
"mib",
path=MIB_TESTDATA_PATH,
nav_shape=nav_shape,
io_backend=MMapBackend(enable_readahead_hints=True),
)
ds.set_num_cores(4)
return ds
def default_tvips(lt_ctx):
nav_shape = (10, )
ds = lt_ctx.load(
"tvips",
path=TVIPS_TESTDATA_PATH,
nav_shape=nav_shape,
io_backend=MMapBackend(),
)
ds.set_num_cores(4)
return ds
def test_run_udf_with_io_backend(lt_ctx, default_raw):
io_backend = MMapBackend(enable_readahead_hints=True)
lt_ctx.load(
"raw",
path=default_raw._path,
io_backend=io_backend,
dtype="float32",
nav_shape=(16, 16),
sig_shape=(128, 128),
)
res = lt_ctx.run_udf(dataset=default_raw, udf=SumUDF())
assert np.array(res['intensity']).shape == (128, 128)
def default_seq(lt_ctx):
nav_shape = (8, 8)
ds = lt_ctx.load(
"seq",
path=SEQ_TESTDATA_PATH,
nav_shape=nav_shape,
io_backend=MMapBackend(),
)
ds.set_num_cores(4)
assert tuple(ds.shape) == (8, 8, 128, 128)
return ds
def default_raw(tmpdir_factory, default_raw_data):
lt_ctx = lt.Context(executor=InlineJobExecutor())
datadir = tmpdir_factory.mktemp('data')
filename = datadir + '/raw-test-default'
default_raw_data.tofile(str(filename))
del default_raw_data
ds = lt_ctx.load(
"raw",
path=str(filename),
dtype="float32",
nav_shape=(16, 16),
sig_shape=(128, 128),
io_backend=MMapBackend(),
)
ds.set_num_cores(2)
yield ds
0, # hTemplateFile
)
f.close()
else:
def drop_cache(flist):
for fname in flist:
with open(fname, "rb") as f:
os.posix_fadvise(f.fileno(), 0, 0, os.POSIX_FADV_DONTNEED)
def get_testdata_prefixes():
pathstring = os.environ.get('LT_BENCH_PREFIXES')
if pathstring is None:
dirname = os.path.normpath(
os.path.join(os.path.dirname(__file__), 'data'))
listing = [os.path.join(dirname, p) for p in os.listdir(dirname)]
prefixes = [p for p in listing if os.path.isdir(p)]
else:
prefixes = pathstring.split(';')
for p in prefixes:
assert os.path.isdir(p)
return prefixes
backends_by_name = {
"mmap": MMapBackend(),
"mmap_readahead": MMapBackend(enable_readahead_hints=True),
"buffered": BufferedBackend(),
}
# check that the offsets in the fileset are correct:
assert f0.num_frames == 20
assert f1.num_frames == 20
assert f0.start_idx == 0
assert f0.end_idx == 20
assert f1.start_idx == 20
assert f1.end_idx == 40
lt_ctx.run_udf(dataset=dm_stack_of_3d, udf=SumUDF())
@pytest.mark.parametrize(
"io_backend",
(
BufferedBackend(),
MMapBackend(),
),
)
def test_positive_sync_offset(lt_ctx, io_backend):
udf = SumSigUDF()
sync_offset = 2
ds = lt_ctx.load(
"dm",
files=list(sorted(glob(os.path.join(DM_TESTDATA_PATH, '*.dm4')))),
nav_shape=(4, 2),
io_backend=io_backend,
)
result = lt_ctx.run_udf(dataset=ds, udf=udf)
result = result['intensity'].raw_data[sync_offset:]
def get_default_io_backend(cls) -> "IOBackend":
import platform
if platform.system() == "Windows":
from libertem.io.dataset.base import BufferedBackend
return BufferedBackend()
return MMapBackend()
