def test_detector_patch_large(num_excluded, benchmark): nav_dims = (8, 8) sig_dims = (1336, 2004) data = gradient_data(nav_dims, sig_dims) exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=num_excluded) damaged_data = data.copy() if exclude is not None: assert exclude.shape[1] == num_excluded damaged_data[(Ellipsis, *exclude)] = 1e24 print("Nav dims: ", nav_dims) print("Sig dims:", sig_dims) print("Exclude: ", exclude) benchmark.pedantic( detector.correct, kwargs=dict( buffer=damaged_data, excluded_pixels=exclude, sig_shape=sig_dims, inplace=False ), warmup_rounds=0, rounds=5, iterations=1, )
def test_detector_patch_too_large(): for i in range(REPEATS): print(f"Loop number {i}") num_nav_dims = np.random.choice([2, 3]) num_sig_dims = 2 nav_dims = tuple(np.random.randint(low=3, high=5, size=num_nav_dims)) sig_dims = tuple( np.random.randint(low=4 * 32, high=1024, size=num_sig_dims)) data = gradient_data(nav_dims, sig_dims) exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=1001) damaged_data = data.copy() damaged_data[(Ellipsis, *exclude)] = 1e24 print("Nav dims: ", nav_dims) print("Sig dims:", sig_dims) print("Exclude: ", exclude) corrected = detector.correct(buffer=damaged_data, excluded_pixels=exclude, sig_shape=sig_dims, inplace=False) _check_result(data=data, corrected=corrected, atol=1e-8, rtol=1e-5)
def test_detector_patch(): for i in range(REPEATS): print(f"Loop number {i}") num_nav_dims = np.random.choice([2, 3]) num_sig_dims = np.random.choice([2, 3]) nav_dims = tuple(np.random.randint(low=8, high=16, size=num_nav_dims)) sig_dims = tuple(np.random.randint(low=8, high=16, size=num_sig_dims)) data = gradient_data(nav_dims, sig_dims) gain_map = np.random.random(sig_dims) + 1 dark_image = np.random.random(sig_dims) exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=3) damaged_data = data.copy() damaged_data /= gain_map damaged_data += dark_image damaged_data[(Ellipsis, *exclude)] = 1e24 print("Nav dims: ", nav_dims) print("Sig dims:", sig_dims) print("Exclude: ", exclude) corrected = detector.correct(buffer=damaged_data, dark_image=dark_image, gain_map=gain_map, excluded_pixels=exclude, inplace=False) _check_result(data=data, corrected=corrected, atol=1e-8, rtol=1e-5)
def test_detector_uint8(): for i in range(REPEATS): print(f"Loop number {i}") num_nav_dims = np.random.choice([1, 2, 3]) num_sig_dims = np.random.choice([1, 2, 3]) nav_dims = tuple(np.random.randint(low=8, high=16, size=num_nav_dims)) sig_dims = tuple(np.random.randint(low=8, high=16, size=num_sig_dims)) data = np.ones(nav_dims + sig_dims, dtype=np.uint8) exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=2) gain_map = np.ones(sig_dims) dark_image = (np.random.random(sig_dims) * 3).astype(np.uint8) # Make sure the dark image is not all zero so that # the damaged data is different from the original # https://github.com/LiberTEM/LiberTEM/issues/910 # This is only necessary for an integer dark image # since for float it would be extremely unlikely # that all values are exactly 0 atleastone = np.random.randint(0, np.prod(sig_dims)) dark_image[np.unravel_index(atleastone, sig_dims)] = 1 damaged_data = data.copy() # We don't do that since it is set to 1 above # damaged_data /= gain_map damaged_data += dark_image damaged_data = damaged_data.astype(np.uint8) print("Nav dims: ", nav_dims) print("Sig dims:", sig_dims) print("Exclude: ", exclude) corrected = detector.correct( buffer=damaged_data, dark_image=dark_image, gain_map=gain_map, excluded_pixels=exclude, inplace=False ) assert corrected.dtype.kind == 'f' _check_result( data=data, corrected=corrected, atol=1e-8, rtol=1e-5 ) # Make sure we didn't do it in place assert not np.allclose(corrected, damaged_data) with pytest.raises(TypeError): detector.correct( buffer=damaged_data, dark_image=dark_image, gain_map=gain_map, excluded_pixels=exclude, inplace=True )
def test_real_correction(self, shared_dist_ctx, large_raw_file, benchmark, gain, dark, num_excluded): filename, shape, dtype = large_raw_file nav_dims = shape[:2] sig_dims = shape[2:] if gain == 'use gain': gain_map = (np.random.random(sig_dims) + 1).astype(np.float64) elif gain == 'no gain': gain_map = None else: raise ValueError if dark == 'use dark': dark_image = np.random.random(sig_dims).astype(np.float64) elif dark == 'no dark': dark_image = None else: raise ValueError if num_excluded > 0: excluded_coords = exclude_pixels(sig_dims=sig_dims, num_excluded=num_excluded) assert excluded_coords.shape[1] == num_excluded exclude = sparse.COO(coords=excluded_coords, shape=sig_dims, data=True) else: exclude = None print("Nav dims: ", nav_dims) print("Sig dims:", sig_dims) corrset = CorrectionSet( dark=dark_image, gain=gain_map, excluded_pixels=exclude, ) udf = NoOpUDF() ds = shared_dist_ctx.load( 'RAW', path=str(filename), scan_size=shape[:2], dtype=dtype, detector_size=shape[2:], ) benchmark.pedantic( shared_dist_ctx.run_udf, kwargs=dict( dataset=ds, udf=udf, corrections=corrset, ), warmup_rounds=0, rounds=5, iterations=1, )
def test_tileshape_adjustment_many(large_raw, lt_ctx): udf = EarlyExitUDF() exclude = sparse.COO(coords=exclude_pixels(sig_dims=tuple( large_raw.shape.sig), num_excluded=1000), shape=tuple(large_raw.shape.sig), data=True) corr = CorrectionSet(excluded_pixels=exclude) with pytest.raises(EarlyExit): lt_ctx.run_udf(dataset=large_raw, udf=udf, corrections=corr)
def test_detector_correction(): for i in range(REPEATS): print(f"Loop number {i}") num_nav_dims = np.random.choice([1, 2, 3]) num_sig_dims = np.random.choice([1, 2, 3]) nav_dims = tuple(np.random.randint(low=1, high=16, size=num_nav_dims)) sig_dims = tuple(np.random.randint(low=1, high=16, size=num_sig_dims)) data = gradient_data(nav_dims, sig_dims) # Test pure gain and offset correction without # patching pixels exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=0) gain_map = np.random.random(sig_dims) + 1 dark_image = np.random.random(sig_dims) damaged_data = data.copy() damaged_data /= gain_map damaged_data += dark_image assert np.allclose((damaged_data - dark_image) * gain_map, data) print("Nav dims: ", nav_dims) print("Sig dims:", sig_dims) print("Exclude: ", exclude) corrected = detector.correct( buffer=damaged_data, dark_image=dark_image, gain_map=gain_map, excluded_pixels=exclude, inplace=False ) _check_result( data=data, corrected=corrected, atol=1e-8, rtol=1e-5 ) # Make sure we didn't do it in place assert not np.allclose(corrected, damaged_data) detector.correct( buffer=damaged_data, dark_image=dark_image, gain_map=gain_map, excluded_pixels=exclude, inplace=True ) # Now damaged_data should be modified and equal to corrected # since it should have been done in place assert np.allclose(corrected, damaged_data)
def test_mask_patch_sparse(): for i in range(REPEATS): print(f"Loop number {i}") num_nav_dims = np.random.choice([1, 2, 3]) num_sig_dims = np.random.choice([2, 3]) nav_dims = tuple(np.random.randint(low=8, high=16, size=num_nav_dims)) sig_dims = tuple(np.random.randint(low=8, high=16, size=num_sig_dims)) # The mask-based correction is performed as float64 since it creates # numerical instabilities otherwise data = gradient_data(nav_dims, sig_dims).astype(np.float64) gain_map = (np.random.random(sig_dims) + 1).astype(np.float64) dark_image = np.random.random(sig_dims).astype(np.float64) exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=3) damaged_data = data.copy() damaged_data /= gain_map damaged_data += dark_image damaged_data[(Ellipsis, *exclude)] = 1e24 print("Nav dims: ", nav_dims) print("Sig dims:", sig_dims) print("Exclude: ", exclude) masks = sparse.DOK(sparse.zeros((20, ) + sig_dims, dtype=np.float64)) indices = [ np.random.randint(low=0, high=s, size=s // 2) for s in (20, ) + sig_dims ] for tup in zip(*indices): masks[tup] = 1 masks = masks.to_coo() data_flat = data.reshape((np.prod(nav_dims), np.prod(sig_dims))) damaged_flat = damaged_data.reshape( (np.prod(nav_dims), np.prod(sig_dims))) correct_dot = sparse.dot(data_flat, masks.reshape((-1, np.prod(sig_dims))).T) corrected_masks = detector.correct_dot_masks(masks, gain_map, exclude) assert is_sparse(corrected_masks) reconstructed_dot =\ sparse.dot(damaged_flat, corrected_masks.reshape((-1, np.prod(sig_dims))).T)\ - sparse.dot(dark_image.flatten(), corrected_masks.reshape((-1, np.prod(sig_dims))).T) _check_result(data=correct_dot, corrected=reconstructed_dot, atol=1e-8, rtol=1e-5)
def test_detector_uint8(): for i in range(10): print(f"Loop number {i}") num_nav_dims = np.random.choice([1, 2, 3]) num_sig_dims = np.random.choice([1, 2, 3]) nav_dims = tuple(np.random.randint(low=8, high=16, size=num_nav_dims)) sig_dims = tuple(np.random.randint(low=8, high=16, size=num_sig_dims)) data = np.ones(nav_dims + sig_dims, dtype=np.uint8) exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=2) gain_map = np.ones(sig_dims) dark_image = (np.random.random(sig_dims) * 3).astype(np.uint8) damaged_data = data.copy() # We don't do that since it is set to 1 above # damaged_data /= gain_map damaged_data += dark_image damaged_data = damaged_data.astype(np.uint8) print("Nav dims: ", nav_dims) print("Sig dims:", sig_dims) print("Exclude: ", exclude) corrected = detector.correct( buffer=damaged_data, dark_image=dark_image, gain_map=gain_map, excluded_pixels=exclude, inplace=False ) assert corrected.dtype.kind == 'f' _check_result( data=data, corrected=corrected, atol=1e-8, rtol=1e-5 ) # Make sure we didn't do it in place assert not np.allclose(corrected, damaged_data) with pytest.raises(TypeError): detector.correct( buffer=damaged_data, dark_image=dark_image, gain_map=gain_map, excluded_pixels=exclude, inplace=True )
def test_mask_correction(): for i in range(REPEATS): print(f"Loop number {i}") num_nav_dims = np.random.choice([1, 2, 3]) num_sig_dims = np.random.choice([2, 3]) nav_dims = tuple(np.random.randint(low=8, high=16, size=num_nav_dims)) sig_dims = tuple(np.random.randint(low=8, high=16, size=num_sig_dims)) # The mask-based correction is performed as float64 since it creates # numerical instabilities otherwise data = gradient_data(nav_dims, sig_dims).astype(np.float64) gain_map = (np.random.random(sig_dims) + 1).astype(np.float64) dark_image = np.random.random(sig_dims).astype(np.float64) exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=0) damaged_data = data.copy() damaged_data /= gain_map damaged_data += dark_image assert np.allclose((damaged_data - dark_image) * gain_map, data) print("Nav dims: ", nav_dims) print("Sig dims:", sig_dims) print("Exclude: ", exclude) masks = (np.random.random((2, ) + sig_dims) - 0.5).astype(np.float64) data_flat = data.reshape((np.prod(nav_dims), np.prod(sig_dims))) damaged_flat = damaged_data.reshape( (np.prod(nav_dims), np.prod(sig_dims))) correct_dot = np.dot(data_flat, masks.reshape((-1, np.prod(sig_dims))).T) corrected_masks = detector.correct_dot_masks(masks, gain_map, exclude) assert not is_sparse(corrected_masks) reconstructed_dot =\ np.dot(damaged_flat, corrected_masks.reshape((-1, np.prod(sig_dims))).T)\ - np.dot(dark_image.flatten(), corrected_masks.reshape((-1, np.prod(sig_dims))).T) _check_result(data=correct_dot, corrected=reconstructed_dot, atol=1e-8, rtol=1e-5)