def buffered_k2is(lt_ctx):
buffered = BufferedBackend()
return lt_ctx.load(
"k2is",
path=str(K2IS_TESTDATA_PATH),
io_backend=buffered,
)
def buffered_empad(lt_ctx):
buffered = BufferedBackend()
return lt_ctx.load(
"empad",
path=EMPAD_XML,
io_backend=buffered,
)
def buffered_frms6(lt_ctx):
buffered = BufferedBackend()
return lt_ctx.load(
"frms6",
path=str(FRMS6_TESTDATA_PATH),
io_backend=buffered,
)
def test_custom_io_backend(buffered_raw, tmpdir_factory, lt_ctx,
local_cluster_url):
datadir = tmpdir_factory.mktemp('template_tests')
conn = {'connection': {'type': 'tcp', 'address': local_cluster_url}}
path = buffered_raw._path
dataset_params = {
"type": "RAW",
"params": {
"path": path,
"dtype": "float32",
"nav_shape": [16, 16],
"sig_shape": [128, 128],
"io_backend": BufferedBackend(),
}
}
analysis = [{
"analysisType": 'APPLY_DISK_MASK',
"parameters": {
'shape': 'disk',
'cx': 8,
'cy': 8,
'r': 5,
}
}]
notebook = notebook_generator(conn, dataset_params, analysis, save=True)
notebook = io.StringIO(notebook.getvalue())
nb = nbformat.read(notebook, as_version=4)
ep = ExecutePreprocessor(timeout=600)
# this throws an exception if the `io_backend` parameter is not generated
# properly in the notebook:
ep.preprocess(nb, {"metadata": {"path": datadir}})
def buffered_blo(lt_ctx):
buffered = BufferedBackend()
ds_buffered = lt_ctx.load(
"blo",
path=str(BLO_TESTDATA_PATH),
io_backend=buffered,
)
return ds_buffered
def buffered_dm(lt_ctx):
buffered = BufferedBackend()
files = list(sorted(glob(os.path.join(DM_TESTDATA_PATH, '*.dm4'))))
return lt_ctx.load(
"dm",
files=files,
io_backend=buffered,
)
def buffered_mib(lt_ctx):
buffered = BufferedBackend()
ds = lt_ctx.load(
"mib",
path=MIB_TESTDATA_PATH,
nav_shape=(32, 32),
io_backend=buffered,
)
return ds
def buffered_tvips(lt_ctx):
nav_shape = (10, )
ds = lt_ctx.load(
"tvips",
path=TVIPS_TESTDATA_PATH,
nav_shape=nav_shape,
io_backend=BufferedBackend(),
)
return ds
def buffered_seq(lt_ctx):
nav_shape = (8, 8)
ds = lt_ctx.load(
"seq",
path=SEQ_TESTDATA_PATH,
nav_shape=nav_shape,
io_backend=BufferedBackend(),
)
ds.set_num_cores(4)
return ds
def buffered_raw(tmpdir_factory, default_raw_data):
lt_ctx = lt.Context(executor=InlineJobExecutor())
datadir = tmpdir_factory.mktemp('data')
filename = datadir + '/raw-test-buffered'
default_raw_data.tofile(str(filename))
del default_raw_data
ds = lt_ctx.load(
"raw",
path=str(filename),
dtype="float32",
nav_shape=(16, 16),
sig_shape=(128, 128),
io_backend=BufferedBackend(),
)
yield ds
def _get_default_io_backend(self):
import platform
if platform.system() == "Windows":
from libertem.io.dataset.base import BufferedBackend
return BufferedBackend()
return MMapBackend()
0, # hTemplateFile
)
f.close()
else:
def drop_cache(flist):
for fname in flist:
with open(fname, "rb") as f:
os.posix_fadvise(f.fileno(), 0, 0, os.POSIX_FADV_DONTNEED)
def get_testdata_prefixes():
pathstring = os.environ.get('LT_BENCH_PREFIXES')
if pathstring is None:
dirname = os.path.normpath(
os.path.join(os.path.dirname(__file__), 'data'))
listing = [os.path.join(dirname, p) for p in os.listdir(dirname)]
prefixes = [p for p in listing if os.path.isdir(p)]
else:
prefixes = pathstring.split(';')
for p in prefixes:
assert os.path.isdir(p)
return prefixes
backends_by_name = {
"mmap": MMapBackend(),
"mmap_readahead": MMapBackend(enable_readahead_hints=True),
"buffered": BufferedBackend(),
}
# check that the offsets in the fileset are correct:
assert f0.num_frames == 20
assert f1.num_frames == 20
assert f0.start_idx == 0
assert f0.end_idx == 20
assert f1.start_idx == 20
assert f1.end_idx == 40
lt_ctx.run_udf(dataset=dm_stack_of_3d, udf=SumUDF())
@pytest.mark.parametrize(
"io_backend",
(
BufferedBackend(),
MMapBackend(),
),
)
def test_positive_sync_offset(lt_ctx, io_backend):
udf = SumSigUDF()
sync_offset = 2
ds = lt_ctx.load(
"dm",
files=list(sorted(glob(os.path.join(DM_TESTDATA_PATH, '*.dm4')))),
nav_shape=(4, 2),
io_backend=io_backend,
)
result = lt_ctx.run_udf(dataset=ds, udf=udf)
def buffered_mrc(lt_ctx):
# FIXME: not yet supported
ds = lt_ctx.load("mrc", path=MRC_TESTDATA_PATH, io_backend=BufferedBackend())
return ds