def test_sum(lt_ctx, delayed_ctx):
data = _mk_random((16, 8, 32, 64))
dataset = MemoryDataSet(data=data,
tileshape=(8, 32, 64),
num_partitions=2,
sig_dims=2)
sum_result = lt_ctx.run_udf(udf=SumUDF(), dataset=dataset)
sum_delayed = delayed_ctx.run_udf(udf=SumUDF(), dataset=dataset)
naive_result = data.sum(axis=(0, 1))
assert np.allclose(sum_result['intensity'].data, naive_result)
assert np.allclose(sum_delayed['intensity'].data, naive_result)
def test_patch_corr_empty(lt_ctx):
data = np.ones((13, 17, 19))
excluded_coords = np.array([
(1, 2, 3),
]).astype(np.int64)
excluded_pixels = sparse.COO(coords=excluded_coords, shape=(19, ), data=True)
ds = lt_ctx.load("memory", data=data, sig_dims=1)
udf = SumUDF()
with pytest.raises(RepairValueError):
corr = CorrectionSet(
excluded_pixels=excluded_pixels,
gain=np.ones((19, )),
dark=np.ones((19, ))
)
corr = CorrectionSet(
excluded_pixels=excluded_pixels,
gain=np.ones((19, )),
dark=np.ones((19, )),
allow_empty=True
)
res = lt_ctx.run_udf(
dataset=ds,
udf=udf,
corrections=corr
)
# The value will be unpatched and remain 0 after gain and dark correction are applied
assert np.allclose(res['intensity'], 0)
def test_acquisition_shape(ltl_ctx, merlin_detector_sim, merlin_ds, sig_shape):
triggered = triggered = np.array((False, ))
def trigger(acquisition):
triggered[:] = True
assert acquisition.shape.nav == merlin_ds.shape.nav
host, port = merlin_detector_sim
aq = ltl_ctx.prepare_acquisition('merlin',
trigger=trigger,
nav_shape=(32, 32),
sig_shape=sig_shape,
host=host,
port=port,
drain=False)
try:
udf = SumUDF()
res = ltl_ctx.run_udf(dataset=aq, udf=udf)
ref = ltl_ctx.run_udf(dataset=merlin_ds, udf=udf)
assert np.allclose(res['intensity'], ref['intensity'])
assert sig_shape == tuple(merlin_ds.shape.sig)
except ValueError as e:
assert sig_shape != tuple(merlin_ds.shape.sig)
assert 'received "image_size" header' in e.args[0]
def main():
# Set a plot class for Digital Micrograph
with api.Context(executor=InlineJobExecutor(),
plot_class=GMSLive2DPlot) as ctx:
ds = ctx.load("RAW",
path=r"C:\Users\Dieter\testfile-32-32-32-32-float32.raw",
nav_shape=(32, 32),
sig_shape=(32, 32),
dtype=np.float32)
sum_udf = SumUDF()
ring_udf = ApplyMasksUDF(mask_factories=[
functools.partial(
ring,
centerX=16,
centerY=16,
imageSizeX=32,
imageSizeY=32,
radius=15,
radius_inner=11,
)
])
ctx.run_udf(dataset=ds, udf=[sum_udf, ring_udf], plots=True)
def test_gms_nodisplay(monkeypatch, lt_ctx, default_raw):
udf = SumUDF()
monkeypatch.setitem(sys.modules, 'DigitalMicrograph', mock.MagicMock())
plots = [GMSLive2DPlot(dataset=default_raw, udf=udf)]
with pytest.warns(UserWarning,
match="Plot is not displayed, not plotting."):
lt_ctx.run_udf(dataset=default_raw, udf=udf, plots=plots)
async def test_acquisition_async(ltl_ctx, merlin_detector_sim, merlin_ds):
triggered = triggered = np.array((False, ))
def trigger(acquisition):
triggered[:] = True
assert acquisition.shape.nav == merlin_ds.shape.nav
host, port = merlin_detector_sim
aq = ltl_ctx.prepare_acquisition('merlin',
trigger=trigger,
nav_shape=(32, 32),
host=host,
port=port,
drain=False)
udf = SumUDF()
res = await ltl_ctx.run_udf(dataset=aq, udf=udf, sync=False)
ref = ltl_ctx.run_udf(dataset=merlin_ds, udf=udf)
assert np.allclose(res['intensity'], ref['intensity'])
async for res in ltl_ctx.run_udf_iter(dataset=aq, udf=udf, sync=False):
pass
assert np.allclose(res.buffers[0]['intensity'], ref['intensity'])
def test_acquisition_triggered_control(ltl_ctx, merlin_control_sim,
garbage_sim, merlin_ds):
sim_host, sim_port = garbage_sim
pool = concurrent.futures.ThreadPoolExecutor(1)
trig_res = {0: None}
def trigger(acquisition):
control = MerlinControl(*merlin_control_sim)
def do_scan():
'''
Emulated blocking scan function using the Merlin simulator
'''
print("do_scan()")
with control:
control.cmd('SOFTTRIGGER')
fut = pool.submit(do_scan)
trig_res[0] = fut
aq = ltl_ctx.prepare_acquisition('merlin',
trigger=trigger,
nav_shape=(32, 32),
host=sim_host,
port=sim_port)
udf = SumUDF()
res = ltl_ctx.run_udf(dataset=aq, udf=udf)
assert trig_res[0].result() is None
ref = ltl_ctx.run_udf(dataset=merlin_ds, udf=udf)
assert np.allclose(res['intensity'], ref['intensity'])
def test_patch_pixels_only_excluded_pixels(lt_ctx, default_raw,
default_raw_data):
udf = SumUDF()
excluded_pixels = sparse.COO(np.zeros((128, 128)))
corr = CorrectionSet(excluded_pixels=excluded_pixels)
res = lt_ctx.run_udf(dataset=default_raw, udf=udf, corrections=corr)
assert np.allclose(res['intensity'], np.sum(default_raw_data, axis=(0, 1)))
def prime_numba_cache(ds):
dtypes = (np.float32, None)
for dtype in dtypes:
roi = np.zeros(ds.shape.nav, dtype=bool).reshape((-1,))
roi[0] = 1
from libertem.udf.sum import SumUDF
from libertem.corrections.corrset import CorrectionSet
from libertem.io.dataset.base import Negotiator
udfs = [SumUDF()] # need to have at least one UDF
p = next(ds.get_partitions())
neg = Negotiator()
for corr_dtype in (np.float32, None):
if corr_dtype is not None:
corrections = CorrectionSet(dark=np.zeros(ds.shape.sig, dtype=corr_dtype))
else:
corrections = None
p.set_corrections(corrections)
tiling_scheme = neg.get_scheme(
udfs=udfs,
partition=p,
read_dtype=dtype,
roi=roi,
corrections=corrections,
)
next(p.get_tiles(tiling_scheme=tiling_scheme, roi=roi))
def test_bqp_smoke(monkeypatch, lt_ctx, default_raw):
pytest.importorskip("bqplot")
udf = SumUDF()
monkeypatch.setattr(IPython.display, 'display', mock.MagicMock())
monkeypatch.setattr(lt_ctx, 'plot_class', BQLive2DPlot)
lt_ctx.run_udf(dataset=default_raw, udf=udf, plots=True)
IPython.display.display.assert_called()
def test_mpl_smoke(monkeypatch, lt_ctx, default_raw):
udf = SumUDF()
monkeypatch.setattr(
matplotlib.pyplot, 'subplots',
mock.MagicMock(return_value=(mock.MagicMock(), mock.MagicMock())))
monkeypatch.setattr(lt_ctx, 'plot_class', MPLLive2DPlot)
lt_ctx.run_udf(dataset=default_raw, udf=udf, plots=True)
matplotlib.pyplot.subplots.assert_called()
def test_patch_pixels(lt_ctx, default_raw, default_raw_data):
udf = SumUDF()
# test with empty excluded_pixels array
corr = CorrectionSet(excluded_pixels=np.array([(), ()]).astype(np.int64),
gain=np.ones((128, 128)))
res = lt_ctx.run_udf(dataset=default_raw, udf=udf, corrections=corr)
assert np.allclose(res['intensity'], np.sum(default_raw_data, axis=(0, 1)))
def test_display(lt_ctx, default_raw, use_roi):
if use_roi:
roi = np.random.choice((True, False), size=default_raw.shape.nav)
else:
roi = None
udf = SumUDF()
d = lt_ctx.display(dataset=default_raw, udf=udf, roi=roi)
print(d._repr_html_())
def test_correction_set_dark_one(lt_ctx, default_raw, default_raw_data, gain,
dark):
udf = SumUDF()
corr = CorrectionSet(dark=dark, gain=gain)
res = lt_ctx.run_udf(dataset=default_raw, udf=udf, corrections=corr)
assert np.allclose(res['intensity'],
np.sum(default_raw_data - 1, axis=(0, 1)))
def test_mpl_nodisplay(monkeypatch, lt_ctx, default_raw):
udf = SumUDF()
monkeypatch.setattr(
matplotlib.pyplot, 'subplots',
mock.MagicMock(return_value=(mock.MagicMock(), mock.MagicMock())))
plots = [MPLLive2DPlot(dataset=default_raw, udf=udf)]
with pytest.warns(UserWarning,
match="Plot is not displayed, not plotting."):
lt_ctx.run_udf(dataset=default_raw, udf=udf, plots=plots)
def test_correction_set_zero_gain(lt_ctx, default_raw, gain, dark):
udf = SumUDF()
corr = CorrectionSet(dark=dark, gain=gain)
res = lt_ctx.run_udf(
dataset=default_raw,
udf=udf,
corrections=corr
)
assert np.allclose(res['intensity'], 0)
def test_dist_process(dist_ctx):
shape = (3, 5, 7, 11)
data = da.random.random(shape, chunks=(2, 2, 2, 2))
roi = np.random.choice([True, False], data.shape[:2])
# Dask doesn't do multi-dimensional fancy indexing with booleans,
# unlike NumPy :rolleyes:
ref = data.reshape((np.prod(shape[:2]), *shape[2:]))[roi.flatten()].sum(axis=0).compute()
ds = dist_ctx.load("dask", data, sig_dims=2)
res = dist_ctx.run_udf(dataset=ds, udf=SumUDF(), roi=roi)
assert np.allclose(res['intensity'].raw_data, ref)
def test_run_udf_with_io_backend(lt_ctx, default_raw):
io_backend = MMapBackend(enable_readahead_hints=True)
lt_ctx.load(
"raw",
path=default_raw._path,
io_backend=io_backend,
dtype="float32",
nav_shape=(16, 16),
sig_shape=(128, 128),
)
res = lt_ctx.run_udf(dataset=default_raw, udf=SumUDF())
assert np.array(res['intensity']).shape == (128, 128)
def test_live_plotupdate(lt_ctx, default_raw):
udf = SumUDF()
m = Mock()
m.get_udf.return_value = udf
plots = [m]
lt_ctx.run_udf(dataset=default_raw, udf=udf, plots=plots)
n_part = default_raw.get_num_partitions()
print("Num partitions", n_part)
print("Mock calls", m.new_data.mock_calls)
# Otherwise test is meaningless, no intermediate updates
assert n_part > 1
# Partition updates plus final update
assert len(m.new_data.mock_calls) == n_part + 1
def test_dm_stack_fileset_offsets(dm_stack_of_3d, lt_ctx):
fs = dm_stack_of_3d._get_fileset()
f0, f1 = fs
# check that the offsets in the fileset are correct:
assert f0.num_frames == 20
assert f1.num_frames == 20
assert f0.start_idx == 0
assert f0.end_idx == 20
assert f1.start_idx == 20
assert f1.end_idx == 40
lt_ctx.run_udf(dataset=dm_stack_of_3d, udf=SumUDF())
def test_auto_uses_correct_backend_hdf5(lt_ctx, hdf5):
# H5DataSet currently doesn't support alternative I/O backends
with pytest.raises(ValueError):
ds = lt_ctx.load(
"auto",
path=hdf5.filename,
ds_path="data",
io_backend=FakeBackend(),
)
lt_ctx.run_udf(
dataset=ds,
udf=SumUDF(),
)
def test_load_uses_correct_backend(lt_ctx, default_raw):
with pytest.raises(RuntimeError):
ds = lt_ctx.load(
"raw",
path=default_raw._path,
dtype="float32",
nav_shape=(16, 16),
sig_shape=(128, 128),
io_backend=FakeBackend(),
)
lt_ctx.run_udf(
dataset=ds,
udf=SumUDF(),
)
def test_multiple_clients(local_cluster_url, default_raw):
ex1 = DaskJobExecutor.connect(local_cluster_url)
# this creates a second Client, and even though we are setting `set_as_default=False`,
# this Client is then used by functions like `dd.as_completed`. That is because
# `set_as_default` only sets the dask scheduler config to "dask.distributed", it does
# not affect setting the _client_ as the global default `Client`!
# so any time `as_completed` is called, the `loop` needs to be set correctly, otherwise
# this may result in strange hangs and crashes
DaskJobExecutor.connect(local_cluster_url)
udf = SumUDF()
cx1 = Context(executor=ex1)
cx1.run_udf(dataset=default_raw, udf=udf)
def test_plusequal_without_slice_assignment(lt_ctx, default_raw):
class SigNoSlicePLusEqual(UDF):
def get_result_buffers(self):
return {'sum': self.buffer(kind="sig", dtype="float32")}
def process_frame(self, frame):
self.results.sum += frame
def merge(self, dest, src):
dest.sum += src.sum
bm = SigNoSlicePLusEqual()
assert np.allclose(
lt_ctx.run_udf(dataset=default_raw, udf=bm)['sum'],
lt_ctx.run_udf(dataset=default_raw, udf=SumUDF())['intensity'],
)
def test_patch_corr_odd(lt_ctx_fast):
data = np.ones((13, 17, 19, 23, 29, 31))
excluded_coords = np.array([(2, 5), (2, 5), (2, 5)]).astype(np.int64)
excluded_pixels = sparse.COO(coords=excluded_coords,
shape=(23, 29, 31),
data=True)
ds = lt_ctx_fast.load("memory", data=data, sig_dims=3)
udf = SumUDF()
corr = CorrectionSet(excluded_pixels=excluded_pixels,
gain=np.ones((23, 29, 31)),
dark=np.ones((23, 29, 31)))
res = lt_ctx_fast.run_udf(dataset=ds, udf=udf, corrections=corr)
assert np.allclose(res['intensity'], 0)
def test_multi_udf(lt_ctx, default_raw, progress, plots):
udfs = [NoOpUDF(), SumUDF(), SumSigUDF()]
combined_res = lt_ctx.run_udf(dataset=default_raw,
udf=udfs,
progress=progress,
plots=plots)
ref_res = (
lt_ctx.run_udf(dataset=default_raw, udf=udfs[0]),
lt_ctx.run_udf(dataset=default_raw, udf=udfs[1]),
lt_ctx.run_udf(dataset=default_raw, udf=udfs[2]),
)
assert isinstance(ref_res[0], dict)
for index, res in enumerate(ref_res):
for key in res.keys():
assert np.all(res[key].data == combined_res[index][key].data)
def test_plots(lt_ctx, default_raw, plots):
udfs = [NoOpUDF(), SumUDF(), SumSigUDF()]
if plots is None:
def test_channel(udf_result, damage):
result = udf_result['intensity'].data
if udf_result['intensity'].kind == 'nav':
res_damage = damage
else:
res_damage = True
return (result, res_damage)
real_plots = [
lt_ctx.plot_class(dataset=default_raw,
udf=udfs[1],
channel=test_channel,
title="Test title"),
lt_ctx.plot_class(dataset=default_raw,
udf=udfs[2],
channel=test_channel,
title="Test title")
]
mock_target = real_plots[0]
else:
real_plots = plots
mock_target = lt_ctx.plot_class
with mock.patch.object(mock_target, 'update') as plot_update:
with mock.patch.object(mock_target, 'display') as plot_display:
if plots is None:
real_plots[0].display()
real_plots[1].display()
combined_res = lt_ctx.run_udf(dataset=default_raw,
udf=udfs,
plots=real_plots)
plot_update.assert_called()
plot_display.assert_called()
ref_res = (
lt_ctx.run_udf(dataset=default_raw, udf=udfs[0]),
lt_ctx.run_udf(dataset=default_raw, udf=udfs[1]),
lt_ctx.run_udf(dataset=default_raw, udf=udfs[2]),
)
assert isinstance(ref_res[0], dict)
for index, res in enumerate(ref_res):
for key in res.keys():
assert np.all(res[key].data == combined_res[index][key].data)
def test_roi_dtype(lt_ctx, default_raw, dtype):
roi = np.zeros(default_raw.shape.nav, dtype=dtype)
roi[0, 0] = 1
ref_roi = np.zeros(default_raw.shape.nav, dtype=bool)
ref_roi[0, 0] = True
udf = SumUDF()
if roi.dtype is not np.dtype(bool):
match = f"ROI dtype is {roi.dtype}, expected bool. Attempting cast to bool."
with pytest.warns(UserWarning, match=match):
res = lt_ctx.run_udf(dataset=default_raw, udf=udf, roi=roi)
else:
res = lt_ctx.run_udf(dataset=default_raw, udf=udf, roi=roi)
ref = lt_ctx.run_udf(dataset=default_raw, udf=udf, roi=ref_roi)
assert np.all(res['intensity'].raw_data == ref['intensity'].raw_data)
def test_with_udf(lt_ctx):
ds = lt_ctx.load("ser", path=SER_TESTDATA_PATH)
udf = SumUDF()
udf_results = lt_ctx.run_udf(udf=udf, dataset=ds)
# compare result with a known-working variant:
result = np.zeros((ds.shape.sig))
for p in ds.get_partitions():
tileshape = Shape((1, ) + tuple(ds.shape.sig),
sig_dims=ds.shape.sig.dims)
tiling_scheme = TilingScheme.make_for_shape(
tileshape=tileshape,
dataset_shape=ds.shape,
)
for tile in p.get_tiles(tiling_scheme):
result[tile.tile_slice.get(sig_only=True)] += np.sum(tile.data,
axis=0)
assert np.allclose(udf_results['intensity'].data, result)
def test_acquisition_triggered_garbage(ltl_ctx, merlin_control_sim,
trigger_sim, garbage_sim, merlin_ds):
sim_host, sim_port = garbage_sim
pool = concurrent.futures.ThreadPoolExecutor(1)
trig_res = {0: None}
def trigger(acquisition):
control = MerlinControl(*merlin_control_sim)
with control:
control.cmd('STARTACQUISITION')
tr = TriggerClient(*trigger_sim)
print("Trigger connection:", trigger_sim)
tr.connect()
tr.trigger()
def do_scan():
'''
Emulated blocking scan function using the Merlin simulator
'''
print("do_scan()")
fut = pool.submit(do_scan)
trig_res[0] = fut
tr.close()
aq = ltl_ctx.prepare_acquisition(
'merlin',
trigger=trigger,
nav_shape=(32, 32),
host=sim_host,
port=sim_port,
pool_size=16,
)
udf = SumUDF()
res = ltl_ctx.run_udf(dataset=aq, udf=udf)
assert trig_res[0].result() is None
ref = ltl_ctx.run_udf(dataset=merlin_ds, udf=udf)
assert_allclose(res['intensity'], ref['intensity'])