def test_hdf5_apply_masks_1(lt_ctx, hdf5_ds_1, TYPE): mask = _mk_random(size=(16, 16)) with hdf5_ds_1.get_reader().get_h5ds() as h5ds: data = h5ds[:] expected = _naive_mask_apply([mask], data) analysis = lt_ctx.create_mask_analysis(dataset=hdf5_ds_1, factories=[lambda: mask]) analysis.TYPE = TYPE results = lt_ctx.run(analysis) assert np.allclose(results.mask_0.raw_data, expected)
def test_multi_mask_stack_force_scipy_sparse(lt_ctx, TYPE): data = _mk_random(size=(16, 16, 16, 16), dtype="<u2") masks = _mk_random(size=(2, 16, 16)) expected = _naive_mask_apply(masks, data) dataset = MemoryDataSet(data=data, tileshape=(4 * 4, 4, 4), num_partitions=2) analysis = lt_ctx.create_mask_analysis( dataset=dataset, factories=lambda: masks, use_sparse='scipy.sparse', mask_count=2 ) analysis.TYPE = TYPE results = lt_ctx.run(analysis) assert np.allclose( results.mask_0.raw_data, expected[0], ) assert np.allclose( results.mask_1.raw_data, expected[1], )
def test_com_default_params(lt_ctx, TYPE): data = _mk_random(size=(16, 16, 16, 16)) dataset = MemoryDataSet( data=data.astype("<u2"), tileshape=(1, 16, 16), num_partitions=16, sig_dims=2, ) analysis = lt_ctx.create_com_analysis(dataset=dataset, ) analysis.TYPE = TYPE lt_ctx.run(analysis)
def test_sum_with_crop_frames(lt_ctx): data = _mk_random(size=(16, 16, 16, 16), dtype="float32") dataset = MemoryDataSet(data=data, tileshape=(7, 8, 8), num_partitions=2, sig_dims=2) analysis = lt_ctx.create_sum_analysis(dataset=dataset) res = lt_ctx.run(analysis) print(data.shape, res.intensity.raw_data.shape) assert np.allclose(res.intensity.raw_data, np.sum(data, axis=(0, 1)))
def test_get_macrotile(): data = _mk_random(size=(16, 16, 16, 16)) dataset = MemoryDataSet( data=data, tileshape=(16, 16, 16), num_partitions=2, ) p = next(dataset.get_partitions()) mt = p.get_macrotile() assert tuple(mt.shape) == (128, 16, 16)
def test_com_fails_with_non_4d_data_1(lt_ctx): data = _mk_random(size=(16 * 16, 16, 16)) dataset = MemoryDataSet( data=data.astype("<u2"), tileshape=(1, 16, 16), num_partitions=32, ) with pytest.raises(Exception): lt_ctx.create_com_analysis( dataset=dataset, cx=0, cy=0, mask_radius=8 )
def test_sum_endian(lt_ctx): data = _mk_random(size=(16, 16, 16, 16), dtype='>u2') dataset = MemoryDataSet(data=data, tileshape=(8, 16, 16), num_partitions=32) expected = data.sum(axis=(0, 1)) analysis = lt_ctx.create_sum_analysis(dataset=dataset) results = lt_ctx.run(analysis) assert results['intensity'].raw_data.shape == (16, 16) assert np.allclose(results['intensity'].raw_data, expected)
def test_aux_2(lt_ctx): data = _mk_random(size=(16, 16, 16, 16), dtype="float32") aux_data = EchoUDF.aux_data( kind="nav", dtype="float32", extra_shape=(2, ), data=_mk_random(size=(16, 16, 2), dtype="float32"), ) dataset = MemoryDataSet(data=data, tileshape=(7, 16, 16), num_partitions=2, sig_dims=2) echo_udf = EchoUDF(aux=aux_data) res = lt_ctx.run_udf(dataset=dataset, udf=echo_udf) assert 'weighted' in res print(data.shape, res['weighted'].data.shape) assert np.allclose( res['weighted'].raw_data, np.sum(data, axis=(2, 3)).reshape(-1) * aux_data.raw_data[..., 0])
def test_com_linescan(lt_ctx): data = _mk_random(size=(1, 16, 16, 16)) dataset = MemoryDataSet( data=data.astype("<u2"), tileshape=(1, 16, 16), num_partitions=4, ) analysis = lt_ctx.create_com_analysis( dataset=dataset, cx=0, cy=0, mask_radius=8 ) lt_ctx.run(analysis)
def test_all_sparse_analysis(lt_ctx): data = _mk_random(size=(16, 16, 16, 16), dtype="<u2") mask0 = sp.csr_matrix(_mk_random(size=(16, 16))) mask1 = sparse.COO.from_numpy(_mk_random(size=(16, 16))) expected = _naive_mask_apply([mask0, mask1], data) dataset = MemoryDataSet(data=data, tileshape=(4 * 4, 4, 4), num_partitions=2) analysis = lt_ctx.create_mask_analysis( dataset=dataset, factories=[lambda: mask0, lambda: mask1] ) results = lt_ctx.run(analysis) assert np.allclose( results.mask_0.raw_data, expected[0], ) assert np.allclose( results.mask_1.raw_data, expected[1], )
def test_avoid_calculating_masks_on_client(hdf5_ds_1): mask = _mk_random(size=(16, 16)) # We have to start a local cluster so that the masks are # computed in a different process with api.Context() as ctx: analysis = ctx.create_mask_analysis( dataset=hdf5_ds_1, factories=[lambda: mask], mask_count=1, mask_dtype=np.float32 ) job = analysis.get_job() ctx.run(job) assert job.masks._computed_masks is None
def test_avoid_calculating_masks_on_client(hdf5_ds_1): mask = _mk_random(size=(16, 16)) with api.Context() as ctx: analysis = ctx.create_mask_analysis(dataset=hdf5_ds_1, factories=[lambda: mask], mask_count=1, mask_dtype=np.float32) job = analysis.get_job() ctx.run(job) assert job.masks._computed_masks is None
def test_multi_mask_autodtype_complex(lt_ctx, TYPE): data = _mk_random(size=(16, 16, 16, 16), dtype="complex64") masks = _mk_random(size=(2, 16, 16)) expected = _naive_mask_apply(masks, data) dataset = MemoryDataSet(data=data, tileshape=(4 * 4, 4, 4), num_partitions=2) analysis = lt_ctx.create_mask_analysis(dataset=dataset, factories=lambda: masks) analysis.TYPE = TYPE results = lt_ctx.run(analysis) assert results.mask_0_complex.raw_data.dtype.kind == 'c' assert results.mask_0_complex.raw_data.dtype == np.complex64 assert np.allclose( results.mask_0_complex.raw_data, expected[0], ) assert np.allclose( results.mask_1_complex.raw_data, expected[1], )
def test_chunked(lt_ctx, tmpdir_factory, chunks): datadir = tmpdir_factory.mktemp('data') filename = os.path.join(datadir, 'hdf5-test-chunked.h5') data = _mk_random((16, 16, 16, 16), dtype=np.float32) with h5py.File(filename, "w") as f: f.create_dataset("data", data=data, chunks=chunks) ds = lt_ctx.load("hdf5", path=filename) udf = PixelsumUDF() res = lt_ctx.run_udf(udf=udf, dataset=ds)['pixelsum'] assert np.allclose(res, np.sum(data, axis=(2, 3)))
def test_pick_analysis_via_api_3_3d_ds_fail_5(lt_ctx): data = _mk_random(size=(16, 256, 16, 16)) dataset = MemoryDataSet( data=data, tileshape=(1, 16, 16), num_partitions=2, sig_dims=2 ) analysis = PickFrameAnalysis(dataset=dataset, parameters={"x": 7, "y": 8, "z": 11}) with pytest.raises(ValueError): lt_ctx.run(analysis)
def test_roi_all_ones(lt_ctx): data = _mk_random(size=(16, 16, 16, 16), dtype="float32") dataset = MemoryDataSet(data=data, tileshape=(3, 16, 16), num_partitions=16, sig_dims=2) mask = np.ones(data.shape[:2], dtype=bool) pixelsum = PixelsumUDF() res = lt_ctx.run_udf(dataset=dataset, udf=pixelsum, roi=mask) assert 'pixelsum' in res print(data.shape, res['pixelsum'].data.shape) expected = np.sum(data[mask, ...], axis=(-1, -2)) assert np.allclose(res['pixelsum'].raw_data, expected)
def test_start_local(hdf5_ds_1): mask = _mk_random(size=(16, 16)) with hdf5_ds_1.get_reader().get_h5ds() as h5ds: data = h5ds[:] expected = _naive_mask_apply([mask], data) with api.Context() as ctx: analysis = ctx.create_mask_analysis(dataset=hdf5_ds_1, factories=[lambda: mask]) results = ctx.run(analysis) assert np.allclose(results.mask_0.raw_data, expected)
def test_smoke(lt_ctx, progress): """ just check if the analysis runs without throwing exceptions: """ data = _mk_random(size=(16 * 16, 16, 16), dtype="float32") dataset = MemoryDataSet(data=data, tileshape=(1, 16, 16), num_partitions=2, sig_dims=2) match_pattern = common.patterns.RadialGradient(radius=4) udf.correlation.run_blobfinder( ctx=lt_ctx, dataset=dataset, num_peaks=1, match_pattern=match_pattern, progress=progress )