def test_detector_patch_large(num_excluded, benchmark):
nav_dims = (8, 8)
sig_dims = (1336, 2004)
data = gradient_data(nav_dims, sig_dims)
exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=num_excluded)
damaged_data = data.copy()
if exclude is not None:
assert exclude.shape[1] == num_excluded
damaged_data[(Ellipsis, *exclude)] = 1e24
print("Nav dims: ", nav_dims)
print("Sig dims:", sig_dims)
print("Exclude: ", exclude)
benchmark.pedantic(
detector.correct,
kwargs=dict(
buffer=damaged_data,
excluded_pixels=exclude,
sig_shape=sig_dims,
inplace=False
),
warmup_rounds=0,
rounds=5,
iterations=1,
)
def test_detector_patch_too_large():
for i in range(REPEATS):
print(f"Loop number {i}")
num_nav_dims = np.random.choice([2, 3])
num_sig_dims = 2
nav_dims = tuple(np.random.randint(low=3, high=5, size=num_nav_dims))
sig_dims = tuple(
np.random.randint(low=4 * 32, high=1024, size=num_sig_dims))
data = gradient_data(nav_dims, sig_dims)
exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=1001)
damaged_data = data.copy()
damaged_data[(Ellipsis, *exclude)] = 1e24
print("Nav dims: ", nav_dims)
print("Sig dims:", sig_dims)
print("Exclude: ", exclude)
corrected = detector.correct(buffer=damaged_data,
excluded_pixels=exclude,
sig_shape=sig_dims,
inplace=False)
_check_result(data=data, corrected=corrected, atol=1e-8, rtol=1e-5)
def test_detector_patch():
for i in range(REPEATS):
print(f"Loop number {i}")
num_nav_dims = np.random.choice([2, 3])
num_sig_dims = np.random.choice([2, 3])
nav_dims = tuple(np.random.randint(low=8, high=16, size=num_nav_dims))
sig_dims = tuple(np.random.randint(low=8, high=16, size=num_sig_dims))
data = gradient_data(nav_dims, sig_dims)
gain_map = np.random.random(sig_dims) + 1
dark_image = np.random.random(sig_dims)
exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=3)
damaged_data = data.copy()
damaged_data /= gain_map
damaged_data += dark_image
damaged_data[(Ellipsis, *exclude)] = 1e24
print("Nav dims: ", nav_dims)
print("Sig dims:", sig_dims)
print("Exclude: ", exclude)
corrected = detector.correct(buffer=damaged_data,
dark_image=dark_image,
gain_map=gain_map,
excluded_pixels=exclude,
inplace=False)
_check_result(data=data, corrected=corrected, atol=1e-8, rtol=1e-5)
def test_detector_uint8():
for i in range(REPEATS):
print(f"Loop number {i}")
num_nav_dims = np.random.choice([1, 2, 3])
num_sig_dims = np.random.choice([1, 2, 3])
nav_dims = tuple(np.random.randint(low=8, high=16, size=num_nav_dims))
sig_dims = tuple(np.random.randint(low=8, high=16, size=num_sig_dims))
data = np.ones(nav_dims + sig_dims, dtype=np.uint8)
exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=2)
gain_map = np.ones(sig_dims)
dark_image = (np.random.random(sig_dims) * 3).astype(np.uint8)
# Make sure the dark image is not all zero so that
# the damaged data is different from the original
# https://github.com/LiberTEM/LiberTEM/issues/910
# This is only necessary for an integer dark image
# since for float it would be extremely unlikely
# that all values are exactly 0
atleastone = np.random.randint(0, np.prod(sig_dims))
dark_image[np.unravel_index(atleastone, sig_dims)] = 1
damaged_data = data.copy()
# We don't do that since it is set to 1 above
# damaged_data /= gain_map
damaged_data += dark_image
damaged_data = damaged_data.astype(np.uint8)
print("Nav dims: ", nav_dims)
print("Sig dims:", sig_dims)
print("Exclude: ", exclude)
corrected = detector.correct(
buffer=damaged_data,
dark_image=dark_image,
gain_map=gain_map,
excluded_pixels=exclude,
inplace=False
)
assert corrected.dtype.kind == 'f'
_check_result(
data=data, corrected=corrected,
atol=1e-8, rtol=1e-5
)
# Make sure we didn't do it in place
assert not np.allclose(corrected, damaged_data)
with pytest.raises(TypeError):
detector.correct(
buffer=damaged_data,
dark_image=dark_image,
gain_map=gain_map,
excluded_pixels=exclude,
inplace=True
)
def test_real_correction(self, shared_dist_ctx, large_raw_file, benchmark,
gain, dark, num_excluded):
filename, shape, dtype = large_raw_file
nav_dims = shape[:2]
sig_dims = shape[2:]
if gain == 'use gain':
gain_map = (np.random.random(sig_dims) + 1).astype(np.float64)
elif gain == 'no gain':
gain_map = None
else:
raise ValueError
if dark == 'use dark':
dark_image = np.random.random(sig_dims).astype(np.float64)
elif dark == 'no dark':
dark_image = None
else:
raise ValueError
if num_excluded > 0:
excluded_coords = exclude_pixels(sig_dims=sig_dims,
num_excluded=num_excluded)
assert excluded_coords.shape[1] == num_excluded
exclude = sparse.COO(coords=excluded_coords,
shape=sig_dims,
data=True)
else:
exclude = None
print("Nav dims: ", nav_dims)
print("Sig dims:", sig_dims)
corrset = CorrectionSet(
dark=dark_image,
gain=gain_map,
excluded_pixels=exclude,
)
udf = NoOpUDF()
ds = shared_dist_ctx.load(
'RAW',
path=str(filename),
scan_size=shape[:2],
dtype=dtype,
detector_size=shape[2:],
)
benchmark.pedantic(
shared_dist_ctx.run_udf,
kwargs=dict(
dataset=ds,
udf=udf,
corrections=corrset,
),
warmup_rounds=0,
rounds=5,
iterations=1,
)
def test_tileshape_adjustment_many(large_raw, lt_ctx):
udf = EarlyExitUDF()
exclude = sparse.COO(coords=exclude_pixels(sig_dims=tuple(
large_raw.shape.sig),
num_excluded=1000),
shape=tuple(large_raw.shape.sig),
data=True)
corr = CorrectionSet(excluded_pixels=exclude)
with pytest.raises(EarlyExit):
lt_ctx.run_udf(dataset=large_raw, udf=udf, corrections=corr)
def test_detector_correction():
for i in range(REPEATS):
print(f"Loop number {i}")
num_nav_dims = np.random.choice([1, 2, 3])
num_sig_dims = np.random.choice([1, 2, 3])
nav_dims = tuple(np.random.randint(low=1, high=16, size=num_nav_dims))
sig_dims = tuple(np.random.randint(low=1, high=16, size=num_sig_dims))
data = gradient_data(nav_dims, sig_dims)
# Test pure gain and offset correction without
# patching pixels
exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=0)
gain_map = np.random.random(sig_dims) + 1
dark_image = np.random.random(sig_dims)
damaged_data = data.copy()
damaged_data /= gain_map
damaged_data += dark_image
assert np.allclose((damaged_data - dark_image) * gain_map, data)
print("Nav dims: ", nav_dims)
print("Sig dims:", sig_dims)
print("Exclude: ", exclude)
corrected = detector.correct(
buffer=damaged_data,
dark_image=dark_image,
gain_map=gain_map,
excluded_pixels=exclude,
inplace=False
)
_check_result(
data=data, corrected=corrected,
atol=1e-8, rtol=1e-5
)
# Make sure we didn't do it in place
assert not np.allclose(corrected, damaged_data)
detector.correct(
buffer=damaged_data,
dark_image=dark_image,
gain_map=gain_map,
excluded_pixels=exclude,
inplace=True
)
# Now damaged_data should be modified and equal to corrected
# since it should have been done in place
assert np.allclose(corrected, damaged_data)
def test_mask_patch_sparse():
for i in range(REPEATS):
print(f"Loop number {i}")
num_nav_dims = np.random.choice([1, 2, 3])
num_sig_dims = np.random.choice([2, 3])
nav_dims = tuple(np.random.randint(low=8, high=16, size=num_nav_dims))
sig_dims = tuple(np.random.randint(low=8, high=16, size=num_sig_dims))
# The mask-based correction is performed as float64 since it creates
# numerical instabilities otherwise
data = gradient_data(nav_dims, sig_dims).astype(np.float64)
gain_map = (np.random.random(sig_dims) + 1).astype(np.float64)
dark_image = np.random.random(sig_dims).astype(np.float64)
exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=3)
damaged_data = data.copy()
damaged_data /= gain_map
damaged_data += dark_image
damaged_data[(Ellipsis, *exclude)] = 1e24
print("Nav dims: ", nav_dims)
print("Sig dims:", sig_dims)
print("Exclude: ", exclude)
masks = sparse.DOK(sparse.zeros((20, ) + sig_dims, dtype=np.float64))
indices = [
np.random.randint(low=0, high=s, size=s // 2)
for s in (20, ) + sig_dims
]
for tup in zip(*indices):
masks[tup] = 1
masks = masks.to_coo()
data_flat = data.reshape((np.prod(nav_dims), np.prod(sig_dims)))
damaged_flat = damaged_data.reshape(
(np.prod(nav_dims), np.prod(sig_dims)))
correct_dot = sparse.dot(data_flat,
masks.reshape((-1, np.prod(sig_dims))).T)
corrected_masks = detector.correct_dot_masks(masks, gain_map, exclude)
assert is_sparse(corrected_masks)
reconstructed_dot =\
sparse.dot(damaged_flat, corrected_masks.reshape((-1, np.prod(sig_dims))).T)\
- sparse.dot(dark_image.flatten(), corrected_masks.reshape((-1, np.prod(sig_dims))).T)
_check_result(data=correct_dot,
corrected=reconstructed_dot,
atol=1e-8,
rtol=1e-5)
def test_detector_uint8():
for i in range(10):
print(f"Loop number {i}")
num_nav_dims = np.random.choice([1, 2, 3])
num_sig_dims = np.random.choice([1, 2, 3])
nav_dims = tuple(np.random.randint(low=8, high=16, size=num_nav_dims))
sig_dims = tuple(np.random.randint(low=8, high=16, size=num_sig_dims))
data = np.ones(nav_dims + sig_dims, dtype=np.uint8)
exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=2)
gain_map = np.ones(sig_dims)
dark_image = (np.random.random(sig_dims) * 3).astype(np.uint8)
damaged_data = data.copy()
# We don't do that since it is set to 1 above
# damaged_data /= gain_map
damaged_data += dark_image
damaged_data = damaged_data.astype(np.uint8)
print("Nav dims: ", nav_dims)
print("Sig dims:", sig_dims)
print("Exclude: ", exclude)
corrected = detector.correct(
buffer=damaged_data,
dark_image=dark_image,
gain_map=gain_map,
excluded_pixels=exclude,
inplace=False
)
assert corrected.dtype.kind == 'f'
_check_result(
data=data, corrected=corrected,
atol=1e-8, rtol=1e-5
)
# Make sure we didn't do it in place
assert not np.allclose(corrected, damaged_data)
with pytest.raises(TypeError):
detector.correct(
buffer=damaged_data,
dark_image=dark_image,
gain_map=gain_map,
excluded_pixels=exclude,
inplace=True
)
def test_mask_correction():
for i in range(REPEATS):
print(f"Loop number {i}")
num_nav_dims = np.random.choice([1, 2, 3])
num_sig_dims = np.random.choice([2, 3])
nav_dims = tuple(np.random.randint(low=8, high=16, size=num_nav_dims))
sig_dims = tuple(np.random.randint(low=8, high=16, size=num_sig_dims))
# The mask-based correction is performed as float64 since it creates
# numerical instabilities otherwise
data = gradient_data(nav_dims, sig_dims).astype(np.float64)
gain_map = (np.random.random(sig_dims) + 1).astype(np.float64)
dark_image = np.random.random(sig_dims).astype(np.float64)
exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=0)
damaged_data = data.copy()
damaged_data /= gain_map
damaged_data += dark_image
assert np.allclose((damaged_data - dark_image) * gain_map, data)
print("Nav dims: ", nav_dims)
print("Sig dims:", sig_dims)
print("Exclude: ", exclude)
masks = (np.random.random((2, ) + sig_dims) - 0.5).astype(np.float64)
data_flat = data.reshape((np.prod(nav_dims), np.prod(sig_dims)))
damaged_flat = damaged_data.reshape(
(np.prod(nav_dims), np.prod(sig_dims)))
correct_dot = np.dot(data_flat,
masks.reshape((-1, np.prod(sig_dims))).T)
corrected_masks = detector.correct_dot_masks(masks, gain_map, exclude)
assert not is_sparse(corrected_masks)
reconstructed_dot =\
np.dot(damaged_flat, corrected_masks.reshape((-1, np.prod(sig_dims))).T)\
- np.dot(dark_image.flatten(), corrected_masks.reshape((-1, np.prod(sig_dims))).T)
_check_result(data=correct_dot,
corrected=reconstructed_dot,
atol=1e-8,
rtol=1e-5)
