def apply(self, data: np.ndarray, tile_slice: Slice):
"""
Apply corrections in-place to `data`, cropping the
correction data to the `tile_slice`.
"""
dark_frame = self.get_dark_frame()
gain_map = self.get_gain_map()
excluded_pixels = self.get_excluded_pixels()
if not self.have_corrections():
return
sig_slice = tile_slice.get(sig_only=True)
if dark_frame is not None:
dark_frame = dark_frame[sig_slice]
if gain_map is not None:
gain_map = gain_map[sig_slice]
if excluded_pixels is not None:
excluded_pixels = excluded_pixels[sig_slice]
excluded_pixels = excluded_pixels.coords
correct(
buffer=data,
dark_image=dark_frame,
gain_map=gain_map,
excluded_pixels=excluded_pixels,
inplace=True,
sig_shape=tuple(tile_slice.shape.sig),
)
def apply(self, data: np.ndarray, tile_slice: Slice):
"""
Apply corrections in-place to `data`, cropping the
correction data to the `tile_slice`.
"""
dark_frame = self.get_dark_frame()
gain_map = self.get_gain_map()
if not self.have_corrections():
return
sig_slice = tile_slice.get(sig_only=True)
if dark_frame is not None:
dark_frame = dark_frame[sig_slice]
if gain_map is not None:
gain_map = gain_map[sig_slice]
correct(buffer=data,
dark_image=dark_frame,
gain_map=gain_map,
repair_descriptor=self.repair_descriptor(
tile_slice.discard_nav()),
inplace=True,
sig_shape=tuple(tile_slice.shape.sig),
allow_empty=self._allow_empty)
def test_detector_correction():
for i in range(10):
print(f"Loop number {i}")
num_nav_dims = np.random.choice([1, 2, 3])
num_sig_dims = np.random.choice([1, 2, 3])
nav_dims = tuple(np.random.randint(low=1, high=16, size=num_nav_dims))
sig_dims = tuple(np.random.randint(low=1, high=16, size=num_sig_dims))
data = _make_data(nav_dims, sig_dims)
# Test pure gain and offset correction without
# patching pixels
exclude = _generate_exclude_pixels(sig_dims=sig_dims, num_excluded=0)
gain_map = np.random.random(sig_dims) + 1
dark_image = np.random.random(sig_dims)
damaged_data = data.copy()
damaged_data /= gain_map
damaged_data += dark_image
assert np.allclose((damaged_data - dark_image) * gain_map, data)
print("Nav dims: ", nav_dims)
print("Sig dims:", sig_dims)
print("Exclude: ", exclude)
corrected = detector.correct(
buffer=damaged_data,
dark_image=dark_image,
gain_map=gain_map,
excluded_pixels=exclude,
inplace=False
)
_check_result(
data=data, corrected=corrected,
atol=1e-8, rtol=1e-5
)
# Make sure we didn't do it in place
assert not np.allclose(corrected, damaged_data)
detector.correct(
buffer=damaged_data,
dark_image=dark_image,
gain_map=gain_map,
excluded_pixels=exclude,
inplace=True
)
# Now damaged_data should be modified and equal to corrected
# since it should have been done in place
assert np.allclose(corrected, damaged_data)
def test_detector_uint8():
for i in range(REPEATS):
print(f"Loop number {i}")
num_nav_dims = np.random.choice([1, 2, 3])
num_sig_dims = np.random.choice([1, 2, 3])
nav_dims = tuple(np.random.randint(low=8, high=16, size=num_nav_dims))
sig_dims = tuple(np.random.randint(low=8, high=16, size=num_sig_dims))
data = np.ones(nav_dims + sig_dims, dtype=np.uint8)
exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=2)
gain_map = np.ones(sig_dims)
dark_image = (np.random.random(sig_dims) * 3).astype(np.uint8)
# Make sure the dark image is not all zero so that
# the damaged data is different from the original
# https://github.com/LiberTEM/LiberTEM/issues/910
# This is only necessary for an integer dark image
# since for float it would be extremely unlikely
# that all values are exactly 0
atleastone = np.random.randint(0, np.prod(sig_dims))
dark_image[np.unravel_index(atleastone, sig_dims)] = 1
damaged_data = data.copy()
# We don't do that since it is set to 1 above
# damaged_data /= gain_map
damaged_data += dark_image
damaged_data = damaged_data.astype(np.uint8)
print("Nav dims: ", nav_dims)
print("Sig dims:", sig_dims)
print("Exclude: ", exclude)
corrected = detector.correct(buffer=damaged_data,
dark_image=dark_image,
gain_map=gain_map,
excluded_pixels=exclude,
inplace=False)
assert corrected.dtype.kind == 'f'
_check_result(data=data, corrected=corrected, atol=1e-8, rtol=1e-5)
# Make sure we didn't do it in place
assert not np.allclose(corrected, damaged_data)
with pytest.raises(TypeError):
detector.correct(buffer=damaged_data,
dark_image=dark_image,
gain_map=gain_map,
excluded_pixels=exclude,
inplace=True)
def test_detector_uint8():
for i in range(10):
print(f"Loop number {i}")
num_nav_dims = np.random.choice([1, 2, 3])
num_sig_dims = np.random.choice([1, 2, 3])
nav_dims = tuple(np.random.randint(low=8, high=16, size=num_nav_dims))
sig_dims = tuple(np.random.randint(low=8, high=16, size=num_sig_dims))
data = np.ones(nav_dims + sig_dims, dtype=np.uint8)
exclude = _generate_exclude_pixels(sig_dims=sig_dims, num_excluded=2)
gain_map = np.ones(sig_dims)
dark_image = (np.random.random(sig_dims) * 3).astype(np.uint8)
damaged_data = data.copy()
# We don't do that since it is set to 1 above
# damaged_data /= gain_map
damaged_data += dark_image
damaged_data = damaged_data.astype(np.uint8)
print("Nav dims: ", nav_dims)
print("Sig dims:", sig_dims)
print("Exclude: ", exclude)
corrected = detector.correct(
buffer=damaged_data,
dark_image=dark_image,
gain_map=gain_map,
excluded_pixels=exclude,
inplace=False
)
assert corrected.dtype.kind == 'f'
_check_result(
data=data, corrected=corrected,
atol=1e-8, rtol=1e-5
)
# Make sure we didn't do it in place
assert not np.allclose(corrected, damaged_data)
with pytest.raises(TypeError):
detector.correct(
buffer=damaged_data,
dark_image=dark_image,
gain_map=gain_map,
excluded_pixels=exclude,
inplace=True
)
def preprocess(self, data, tile_slice, decoder):
dark_frame = decoder.get_dark_frame()
gain_map = decoder.get_gain_map()
if dark_frame is None and gain_map is None:
return
if dark_frame is not None:
dark_frame = dark_frame[tile_slice.get(sig_only=True)]
if gain_map is not None:
gain_map = gain_map[tile_slice.get(sig_only=True)]
correct(
buffer=data,
dark_image=dark_frame,
gain_map=gain_map,
excluded_pixels=None,
inplace=True,
)
def test_detector_patch_too_large():
for i in range(REPEATS):
print(f"Loop number {i}")
num_nav_dims = np.random.choice([2, 3])
num_sig_dims = 2
nav_dims = tuple(np.random.randint(low=3, high=5, size=num_nav_dims))
sig_dims = tuple(
np.random.randint(low=4 * 32, high=1024, size=num_sig_dims))
data = gradient_data(nav_dims, sig_dims)
exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=1001)
damaged_data = data.copy()
damaged_data[(Ellipsis, *exclude)] = 1e24
print("Nav dims: ", nav_dims)
print("Sig dims:", sig_dims)
print("Exclude: ", exclude)
corrected = detector.correct(buffer=damaged_data,
excluded_pixels=exclude,
sig_shape=sig_dims,
inplace=False)
_check_result(data=data, corrected=corrected, atol=1e-8, rtol=1e-5)
def test_detector_patch():
for i in range(REPEATS):
print(f"Loop number {i}")
num_nav_dims = np.random.choice([2, 3])
num_sig_dims = np.random.choice([2, 3])
nav_dims = tuple(np.random.randint(low=8, high=16, size=num_nav_dims))
sig_dims = tuple(np.random.randint(low=8, high=16, size=num_sig_dims))
data = gradient_data(nav_dims, sig_dims)
gain_map = np.random.random(sig_dims) + 1
dark_image = np.random.random(sig_dims)
exclude = exclude_pixels(sig_dims=sig_dims, num_excluded=3)
damaged_data = data.copy()
damaged_data /= gain_map
damaged_data += dark_image
damaged_data[(Ellipsis, *exclude)] = 1e24
print("Nav dims: ", nav_dims)
print("Sig dims:", sig_dims)
print("Exclude: ", exclude)
corrected = detector.correct(buffer=damaged_data,
dark_image=dark_image,
gain_map=gain_map,
excluded_pixels=exclude,
inplace=False)
_check_result(data=data, corrected=corrected, atol=1e-8, rtol=1e-5)
def dataset_correction_masks(ds, roi, lt_ctx, exclude=None):
"""
compare correction via sparse mask multiplication w/ correct function
"""
for i in range(1):
shape = (-1, *tuple(ds.shape.sig))
uncorr = CorrectionSet()
data = lt_ctx.run_udf(udf=PickUDF(), dataset=ds, roi=roi, corrections=uncorr)
gain = np.random.random(ds.shape.sig) + 1
dark = np.random.random(ds.shape.sig) - 0.5
if exclude is None:
exclude = [
(np.random.randint(0, s), np.random.randint(0, s))
for s in tuple(ds.shape.sig)
]
exclude_coo = sparse.COO(coords=exclude, data=True, shape=ds.shape.sig)
corrset = CorrectionSet(dark=dark, gain=gain, excluded_pixels=exclude_coo)
def mask_factory():
s = tuple(ds.shape.sig)
return sparse.eye(np.prod(s)).reshape((-1, *s))
# This one casts to float
mask_res = lt_ctx.run_udf(
udf=ApplyMasksUDF(mask_factory),
dataset=ds,
corrections=corrset,
roi=roi,
)
# This one uses native input data
corrected = correct(
buffer=data['intensity'].raw_data.reshape(shape),
dark_image=dark,
gain_map=gain,
excluded_pixels=exclude,
inplace=False
)
print("Exclude: ", exclude)
print(mask_res['intensity'].raw_data.dtype)
print(corrected.dtype)
assert np.allclose(
mask_res['intensity'].raw_data.reshape(shape),
corrected
)
def dataset_correction_verification(ds, roi, lt_ctx, exclude=None):
"""
compare correct function w/ corrected pick
"""
for i in range(1):
shape = (-1, *tuple(ds.shape.sig))
uncorr = CorrectionSet()
data = lt_ctx.run_udf(udf=PickUDF(),
dataset=ds,
roi=roi,
corrections=uncorr)
gain = np.random.random(ds.shape.sig) + 1
dark = np.random.random(ds.shape.sig) - 0.5
if exclude is None:
exclude = [(np.random.randint(0, s), np.random.randint(0, s))
for s in tuple(ds.shape.sig)]
exclude_coo = sparse.COO(coords=exclude, data=True, shape=ds.shape.sig)
corrset = CorrectionSet(dark=dark,
gain=gain,
excluded_pixels=exclude_coo)
# This one uses native input data
pick_res = lt_ctx.run_udf(udf=PickUDF(),
dataset=ds,
corrections=corrset,
roi=roi)
corrected = correct(buffer=data['intensity'].raw_data.reshape(shape),
dark_image=dark,
gain_map=gain,
excluded_pixels=exclude,
inplace=False)
print("Exclude: ", exclude)
print(pick_res['intensity'].raw_data.dtype)
print(corrected.dtype)
assert np.allclose(pick_res['intensity'].raw_data.reshape(shape),
corrected)
def test_detector_patch_overlapping():
for i in range(10):
print(f"Loop number {i}")
num_nav_dims = np.random.choice([2, 3])
num_sig_dims = np.random.choice([2, 3])
nav_dims = tuple(np.random.randint(low=8, high=16, size=num_nav_dims))
sig_dims = tuple(np.random.randint(low=8, high=16, size=num_sig_dims))
# Faithfully reconstruct in a constant dataset
data = np.ones(nav_dims + sig_dims)
gain_map = np.random.random(sig_dims) + 1
dark_image = np.random.random(sig_dims)
# Neighboring excluded pixels
exclude = np.ones((num_sig_dims, 3), dtype=np.int32)
exclude[0, 1] += 1
exclude[1, 2] += 1
damaged_data = data.copy()
damaged_data /= gain_map
damaged_data += dark_image
damaged_data[(Ellipsis, *exclude)] = 1e24
print("Nav dims: ", nav_dims)
print("Sig dims:", sig_dims)
print("Exclude: ", exclude)
corrected = detector.correct(
buffer=damaged_data,
dark_image=dark_image,
gain_map=gain_map,
excluded_pixels=exclude,
inplace=False
)
_check_result(
data=data, corrected=corrected,
atol=1e-8, rtol=1e-5
)
