def test_sum(lt_ctx, delayed_ctx): data = _mk_random((16, 8, 32, 64)) dataset = MemoryDataSet(data=data, tileshape=(8, 32, 64), num_partitions=2, sig_dims=2) sum_result = lt_ctx.run_udf(udf=SumUDF(), dataset=dataset) sum_delayed = delayed_ctx.run_udf(udf=SumUDF(), dataset=dataset) naive_result = data.sum(axis=(0, 1)) assert np.allclose(sum_result['intensity'].data, naive_result) assert np.allclose(sum_delayed['intensity'].data, naive_result)
def test_patch_corr_empty(lt_ctx): data = np.ones((13, 17, 19)) excluded_coords = np.array([ (1, 2, 3), ]).astype(np.int64) excluded_pixels = sparse.COO(coords=excluded_coords, shape=(19, ), data=True) ds = lt_ctx.load("memory", data=data, sig_dims=1) udf = SumUDF() with pytest.raises(RepairValueError): corr = CorrectionSet( excluded_pixels=excluded_pixels, gain=np.ones((19, )), dark=np.ones((19, )) ) corr = CorrectionSet( excluded_pixels=excluded_pixels, gain=np.ones((19, )), dark=np.ones((19, )), allow_empty=True ) res = lt_ctx.run_udf( dataset=ds, udf=udf, corrections=corr ) # The value will be unpatched and remain 0 after gain and dark correction are applied assert np.allclose(res['intensity'], 0)
def test_acquisition_shape(ltl_ctx, merlin_detector_sim, merlin_ds, sig_shape): triggered = triggered = np.array((False, )) def trigger(acquisition): triggered[:] = True assert acquisition.shape.nav == merlin_ds.shape.nav host, port = merlin_detector_sim aq = ltl_ctx.prepare_acquisition('merlin', trigger=trigger, nav_shape=(32, 32), sig_shape=sig_shape, host=host, port=port, drain=False) try: udf = SumUDF() res = ltl_ctx.run_udf(dataset=aq, udf=udf) ref = ltl_ctx.run_udf(dataset=merlin_ds, udf=udf) assert np.allclose(res['intensity'], ref['intensity']) assert sig_shape == tuple(merlin_ds.shape.sig) except ValueError as e: assert sig_shape != tuple(merlin_ds.shape.sig) assert 'received "image_size" header' in e.args[0]
def main(): # Set a plot class for Digital Micrograph with api.Context(executor=InlineJobExecutor(), plot_class=GMSLive2DPlot) as ctx: ds = ctx.load("RAW", path=r"C:\Users\Dieter\testfile-32-32-32-32-float32.raw", nav_shape=(32, 32), sig_shape=(32, 32), dtype=np.float32) sum_udf = SumUDF() ring_udf = ApplyMasksUDF(mask_factories=[ functools.partial( ring, centerX=16, centerY=16, imageSizeX=32, imageSizeY=32, radius=15, radius_inner=11, ) ]) ctx.run_udf(dataset=ds, udf=[sum_udf, ring_udf], plots=True)
def test_gms_nodisplay(monkeypatch, lt_ctx, default_raw): udf = SumUDF() monkeypatch.setitem(sys.modules, 'DigitalMicrograph', mock.MagicMock()) plots = [GMSLive2DPlot(dataset=default_raw, udf=udf)] with pytest.warns(UserWarning, match="Plot is not displayed, not plotting."): lt_ctx.run_udf(dataset=default_raw, udf=udf, plots=plots)
async def test_acquisition_async(ltl_ctx, merlin_detector_sim, merlin_ds): triggered = triggered = np.array((False, )) def trigger(acquisition): triggered[:] = True assert acquisition.shape.nav == merlin_ds.shape.nav host, port = merlin_detector_sim aq = ltl_ctx.prepare_acquisition('merlin', trigger=trigger, nav_shape=(32, 32), host=host, port=port, drain=False) udf = SumUDF() res = await ltl_ctx.run_udf(dataset=aq, udf=udf, sync=False) ref = ltl_ctx.run_udf(dataset=merlin_ds, udf=udf) assert np.allclose(res['intensity'], ref['intensity']) async for res in ltl_ctx.run_udf_iter(dataset=aq, udf=udf, sync=False): pass assert np.allclose(res.buffers[0]['intensity'], ref['intensity'])
def test_acquisition_triggered_control(ltl_ctx, merlin_control_sim, garbage_sim, merlin_ds): sim_host, sim_port = garbage_sim pool = concurrent.futures.ThreadPoolExecutor(1) trig_res = {0: None} def trigger(acquisition): control = MerlinControl(*merlin_control_sim) def do_scan(): ''' Emulated blocking scan function using the Merlin simulator ''' print("do_scan()") with control: control.cmd('SOFTTRIGGER') fut = pool.submit(do_scan) trig_res[0] = fut aq = ltl_ctx.prepare_acquisition('merlin', trigger=trigger, nav_shape=(32, 32), host=sim_host, port=sim_port) udf = SumUDF() res = ltl_ctx.run_udf(dataset=aq, udf=udf) assert trig_res[0].result() is None ref = ltl_ctx.run_udf(dataset=merlin_ds, udf=udf) assert np.allclose(res['intensity'], ref['intensity'])
def test_patch_pixels_only_excluded_pixels(lt_ctx, default_raw, default_raw_data): udf = SumUDF() excluded_pixels = sparse.COO(np.zeros((128, 128))) corr = CorrectionSet(excluded_pixels=excluded_pixels) res = lt_ctx.run_udf(dataset=default_raw, udf=udf, corrections=corr) assert np.allclose(res['intensity'], np.sum(default_raw_data, axis=(0, 1)))
def prime_numba_cache(ds): dtypes = (np.float32, None) for dtype in dtypes: roi = np.zeros(ds.shape.nav, dtype=bool).reshape((-1,)) roi[0] = 1 from libertem.udf.sum import SumUDF from libertem.corrections.corrset import CorrectionSet from libertem.io.dataset.base import Negotiator udfs = [SumUDF()] # need to have at least one UDF p = next(ds.get_partitions()) neg = Negotiator() for corr_dtype in (np.float32, None): if corr_dtype is not None: corrections = CorrectionSet(dark=np.zeros(ds.shape.sig, dtype=corr_dtype)) else: corrections = None p.set_corrections(corrections) tiling_scheme = neg.get_scheme( udfs=udfs, partition=p, read_dtype=dtype, roi=roi, corrections=corrections, ) next(p.get_tiles(tiling_scheme=tiling_scheme, roi=roi))
def test_bqp_smoke(monkeypatch, lt_ctx, default_raw): pytest.importorskip("bqplot") udf = SumUDF() monkeypatch.setattr(IPython.display, 'display', mock.MagicMock()) monkeypatch.setattr(lt_ctx, 'plot_class', BQLive2DPlot) lt_ctx.run_udf(dataset=default_raw, udf=udf, plots=True) IPython.display.display.assert_called()
def test_mpl_smoke(monkeypatch, lt_ctx, default_raw): udf = SumUDF() monkeypatch.setattr( matplotlib.pyplot, 'subplots', mock.MagicMock(return_value=(mock.MagicMock(), mock.MagicMock()))) monkeypatch.setattr(lt_ctx, 'plot_class', MPLLive2DPlot) lt_ctx.run_udf(dataset=default_raw, udf=udf, plots=True) matplotlib.pyplot.subplots.assert_called()
def test_patch_pixels(lt_ctx, default_raw, default_raw_data): udf = SumUDF() # test with empty excluded_pixels array corr = CorrectionSet(excluded_pixels=np.array([(), ()]).astype(np.int64), gain=np.ones((128, 128))) res = lt_ctx.run_udf(dataset=default_raw, udf=udf, corrections=corr) assert np.allclose(res['intensity'], np.sum(default_raw_data, axis=(0, 1)))
def test_display(lt_ctx, default_raw, use_roi): if use_roi: roi = np.random.choice((True, False), size=default_raw.shape.nav) else: roi = None udf = SumUDF() d = lt_ctx.display(dataset=default_raw, udf=udf, roi=roi) print(d._repr_html_())
def test_correction_set_dark_one(lt_ctx, default_raw, default_raw_data, gain, dark): udf = SumUDF() corr = CorrectionSet(dark=dark, gain=gain) res = lt_ctx.run_udf(dataset=default_raw, udf=udf, corrections=corr) assert np.allclose(res['intensity'], np.sum(default_raw_data - 1, axis=(0, 1)))
def test_mpl_nodisplay(monkeypatch, lt_ctx, default_raw): udf = SumUDF() monkeypatch.setattr( matplotlib.pyplot, 'subplots', mock.MagicMock(return_value=(mock.MagicMock(), mock.MagicMock()))) plots = [MPLLive2DPlot(dataset=default_raw, udf=udf)] with pytest.warns(UserWarning, match="Plot is not displayed, not plotting."): lt_ctx.run_udf(dataset=default_raw, udf=udf, plots=plots)
def test_correction_set_zero_gain(lt_ctx, default_raw, gain, dark): udf = SumUDF() corr = CorrectionSet(dark=dark, gain=gain) res = lt_ctx.run_udf( dataset=default_raw, udf=udf, corrections=corr ) assert np.allclose(res['intensity'], 0)
def test_dist_process(dist_ctx): shape = (3, 5, 7, 11) data = da.random.random(shape, chunks=(2, 2, 2, 2)) roi = np.random.choice([True, False], data.shape[:2]) # Dask doesn't do multi-dimensional fancy indexing with booleans, # unlike NumPy :rolleyes: ref = data.reshape((np.prod(shape[:2]), *shape[2:]))[roi.flatten()].sum(axis=0).compute() ds = dist_ctx.load("dask", data, sig_dims=2) res = dist_ctx.run_udf(dataset=ds, udf=SumUDF(), roi=roi) assert np.allclose(res['intensity'].raw_data, ref)
def test_run_udf_with_io_backend(lt_ctx, default_raw): io_backend = MMapBackend(enable_readahead_hints=True) lt_ctx.load( "raw", path=default_raw._path, io_backend=io_backend, dtype="float32", nav_shape=(16, 16), sig_shape=(128, 128), ) res = lt_ctx.run_udf(dataset=default_raw, udf=SumUDF()) assert np.array(res['intensity']).shape == (128, 128)
def test_live_plotupdate(lt_ctx, default_raw): udf = SumUDF() m = Mock() m.get_udf.return_value = udf plots = [m] lt_ctx.run_udf(dataset=default_raw, udf=udf, plots=plots) n_part = default_raw.get_num_partitions() print("Num partitions", n_part) print("Mock calls", m.new_data.mock_calls) # Otherwise test is meaningless, no intermediate updates assert n_part > 1 # Partition updates plus final update assert len(m.new_data.mock_calls) == n_part + 1
def test_dm_stack_fileset_offsets(dm_stack_of_3d, lt_ctx): fs = dm_stack_of_3d._get_fileset() f0, f1 = fs # check that the offsets in the fileset are correct: assert f0.num_frames == 20 assert f1.num_frames == 20 assert f0.start_idx == 0 assert f0.end_idx == 20 assert f1.start_idx == 20 assert f1.end_idx == 40 lt_ctx.run_udf(dataset=dm_stack_of_3d, udf=SumUDF())
def test_auto_uses_correct_backend_hdf5(lt_ctx, hdf5): # H5DataSet currently doesn't support alternative I/O backends with pytest.raises(ValueError): ds = lt_ctx.load( "auto", path=hdf5.filename, ds_path="data", io_backend=FakeBackend(), ) lt_ctx.run_udf( dataset=ds, udf=SumUDF(), )
def test_load_uses_correct_backend(lt_ctx, default_raw): with pytest.raises(RuntimeError): ds = lt_ctx.load( "raw", path=default_raw._path, dtype="float32", nav_shape=(16, 16), sig_shape=(128, 128), io_backend=FakeBackend(), ) lt_ctx.run_udf( dataset=ds, udf=SumUDF(), )
def test_multiple_clients(local_cluster_url, default_raw): ex1 = DaskJobExecutor.connect(local_cluster_url) # this creates a second Client, and even though we are setting `set_as_default=False`, # this Client is then used by functions like `dd.as_completed`. That is because # `set_as_default` only sets the dask scheduler config to "dask.distributed", it does # not affect setting the _client_ as the global default `Client`! # so any time `as_completed` is called, the `loop` needs to be set correctly, otherwise # this may result in strange hangs and crashes DaskJobExecutor.connect(local_cluster_url) udf = SumUDF() cx1 = Context(executor=ex1) cx1.run_udf(dataset=default_raw, udf=udf)
def test_plusequal_without_slice_assignment(lt_ctx, default_raw): class SigNoSlicePLusEqual(UDF): def get_result_buffers(self): return {'sum': self.buffer(kind="sig", dtype="float32")} def process_frame(self, frame): self.results.sum += frame def merge(self, dest, src): dest.sum += src.sum bm = SigNoSlicePLusEqual() assert np.allclose( lt_ctx.run_udf(dataset=default_raw, udf=bm)['sum'], lt_ctx.run_udf(dataset=default_raw, udf=SumUDF())['intensity'], )
def test_patch_corr_odd(lt_ctx_fast): data = np.ones((13, 17, 19, 23, 29, 31)) excluded_coords = np.array([(2, 5), (2, 5), (2, 5)]).astype(np.int64) excluded_pixels = sparse.COO(coords=excluded_coords, shape=(23, 29, 31), data=True) ds = lt_ctx_fast.load("memory", data=data, sig_dims=3) udf = SumUDF() corr = CorrectionSet(excluded_pixels=excluded_pixels, gain=np.ones((23, 29, 31)), dark=np.ones((23, 29, 31))) res = lt_ctx_fast.run_udf(dataset=ds, udf=udf, corrections=corr) assert np.allclose(res['intensity'], 0)
def test_multi_udf(lt_ctx, default_raw, progress, plots): udfs = [NoOpUDF(), SumUDF(), SumSigUDF()] combined_res = lt_ctx.run_udf(dataset=default_raw, udf=udfs, progress=progress, plots=plots) ref_res = ( lt_ctx.run_udf(dataset=default_raw, udf=udfs[0]), lt_ctx.run_udf(dataset=default_raw, udf=udfs[1]), lt_ctx.run_udf(dataset=default_raw, udf=udfs[2]), ) assert isinstance(ref_res[0], dict) for index, res in enumerate(ref_res): for key in res.keys(): assert np.all(res[key].data == combined_res[index][key].data)
def test_plots(lt_ctx, default_raw, plots): udfs = [NoOpUDF(), SumUDF(), SumSigUDF()] if plots is None: def test_channel(udf_result, damage): result = udf_result['intensity'].data if udf_result['intensity'].kind == 'nav': res_damage = damage else: res_damage = True return (result, res_damage) real_plots = [ lt_ctx.plot_class(dataset=default_raw, udf=udfs[1], channel=test_channel, title="Test title"), lt_ctx.plot_class(dataset=default_raw, udf=udfs[2], channel=test_channel, title="Test title") ] mock_target = real_plots[0] else: real_plots = plots mock_target = lt_ctx.plot_class with mock.patch.object(mock_target, 'update') as plot_update: with mock.patch.object(mock_target, 'display') as plot_display: if plots is None: real_plots[0].display() real_plots[1].display() combined_res = lt_ctx.run_udf(dataset=default_raw, udf=udfs, plots=real_plots) plot_update.assert_called() plot_display.assert_called() ref_res = ( lt_ctx.run_udf(dataset=default_raw, udf=udfs[0]), lt_ctx.run_udf(dataset=default_raw, udf=udfs[1]), lt_ctx.run_udf(dataset=default_raw, udf=udfs[2]), ) assert isinstance(ref_res[0], dict) for index, res in enumerate(ref_res): for key in res.keys(): assert np.all(res[key].data == combined_res[index][key].data)
def test_roi_dtype(lt_ctx, default_raw, dtype): roi = np.zeros(default_raw.shape.nav, dtype=dtype) roi[0, 0] = 1 ref_roi = np.zeros(default_raw.shape.nav, dtype=bool) ref_roi[0, 0] = True udf = SumUDF() if roi.dtype is not np.dtype(bool): match = f"ROI dtype is {roi.dtype}, expected bool. Attempting cast to bool." with pytest.warns(UserWarning, match=match): res = lt_ctx.run_udf(dataset=default_raw, udf=udf, roi=roi) else: res = lt_ctx.run_udf(dataset=default_raw, udf=udf, roi=roi) ref = lt_ctx.run_udf(dataset=default_raw, udf=udf, roi=ref_roi) assert np.all(res['intensity'].raw_data == ref['intensity'].raw_data)
def test_with_udf(lt_ctx): ds = lt_ctx.load("ser", path=SER_TESTDATA_PATH) udf = SumUDF() udf_results = lt_ctx.run_udf(udf=udf, dataset=ds) # compare result with a known-working variant: result = np.zeros((ds.shape.sig)) for p in ds.get_partitions(): tileshape = Shape((1, ) + tuple(ds.shape.sig), sig_dims=ds.shape.sig.dims) tiling_scheme = TilingScheme.make_for_shape( tileshape=tileshape, dataset_shape=ds.shape, ) for tile in p.get_tiles(tiling_scheme): result[tile.tile_slice.get(sig_only=True)] += np.sum(tile.data, axis=0) assert np.allclose(udf_results['intensity'].data, result)
def test_acquisition_triggered_garbage(ltl_ctx, merlin_control_sim, trigger_sim, garbage_sim, merlin_ds): sim_host, sim_port = garbage_sim pool = concurrent.futures.ThreadPoolExecutor(1) trig_res = {0: None} def trigger(acquisition): control = MerlinControl(*merlin_control_sim) with control: control.cmd('STARTACQUISITION') tr = TriggerClient(*trigger_sim) print("Trigger connection:", trigger_sim) tr.connect() tr.trigger() def do_scan(): ''' Emulated blocking scan function using the Merlin simulator ''' print("do_scan()") fut = pool.submit(do_scan) trig_res[0] = fut tr.close() aq = ltl_ctx.prepare_acquisition( 'merlin', trigger=trigger, nav_shape=(32, 32), host=sim_host, port=sim_port, pool_size=16, ) udf = SumUDF() res = ltl_ctx.run_udf(dataset=aq, udf=udf) assert trig_res[0].result() is None ref = ltl_ctx.run_udf(dataset=merlin_ds, udf=udf) assert_allclose(res['intensity'], ref['intensity'])