def _update_pois(self, image_data, assembled):
if assembled.ndim == 2 or image_data.poi_indices is None:
return
n_images = image_data.n_images
out_of_bound_indices = []
# only keep POI in 'images'
for i in image_data.poi_indices:
if i < n_images:
image_data.images[i] = assembled[i].copy()
mask_image_data(image_data.images[i],
image_mask=self._image_mask,
threshold_mask=self._threshold_mask,
keep_nan=True)
else:
out_of_bound_indices.append(i)
if image_data.poi_indices[1] == image_data.poi_indices[0]:
# skip the second one if two POIs have the same index
break
if out_of_bound_indices:
# This is still ProcessingError since it is not fatal and should
# not stop the pipeline.
raise ProcessingError(
f"[Image processor] POI indices {out_of_bound_indices[0]} "
f"is out of bound (0 - {n_images-1}")
def _integrate1d_imp(i):
masked = assembled[i].copy()
mask = np.zeros_like(image_mask)
mask_image_data(masked,
image_mask=image_mask,
threshold_mask=threshold_mask,
out=mask)
return integ1d(masked, integ_points, mask=mask)
def testGeneral(self):
with self.assertRaises(TypeError):
_SimpleImageData([1, 2, 3])
gt_data = np.random.randn(3, 3).astype(np.float32)
img_data = _SimpleImageData.from_array(gt_data)
img_data.threshold_mask = (3, 6)
masked_gt = gt_data.copy()
mask_image_data(masked_gt, threshold_mask=(3, 6))
np.testing.assert_array_almost_equal(masked_gt, img_data.masked)
self.assertEqual(1.0e-3, img_data.pixel_size)
def _compute_fom(self, roi, fom_type, image_mask, threshold_mask):
if roi is None:
return
try:
handler = self._fom_handlers[fom_type]
except KeyError:
raise UnknownParameterError(
f"[ROI][FOM] Unknown FOM type: {fom_type}")
mask_image_data(roi,
image_mask=image_mask,
threshold_mask=threshold_mask)
return handler(roi, axis=(-2, -1))
def testAzimuthalIntegrationPp(self):
proc = self._proc
shape = (4, 128, 64)
image_mask = np.zeros(shape[-2:], dtype=np.bool)
image_mask[::2, ::2] = True
threshold_mask = (0, 0.5)
data, processed = self.data_with_assembled(
1001, shape, image_mask=image_mask, threshold_mask=threshold_mask)
image_on = np.nanmean(data['assembled']['sliced'][::2, ...], axis=0)
mask_on = np.zeros_like(image_mask)
mask_image_data(image_on,
image_mask=image_mask,
threshold_mask=threshold_mask,
out=mask_on)
image_off = np.nanmean(data['assembled']['sliced'][1::2, ...], axis=0)
mask_off = np.zeros_like(image_mask)
mask_image_data(image_off,
image_mask=image_mask,
threshold_mask=threshold_mask,
out=mask_off)
pp = processed.pp
pp.analysis_type = AnalysisType.AZIMUTHAL_INTEG
pp.image_on = image_on
pp.image_off = image_off
pp.on.mask = mask_on
pp.off.mask = mask_off
with patch.object(proc._meta,
'has_analysis',
side_effect=lambda x: True
if x == AnalysisType.AZIMUTHAL_INTEG else False):
proc.process(data)
assert len(pp.y_on) == proc._integ_points
assert all([not np.isnan(v) for v in pp.y_on])
assert len(pp.y_off) == proc._integ_points
assert all([not np.isnan(v) for v in pp.y_off])
assert len(pp.x) == proc._integ_points
assert len(pp.y) == proc._integ_points
assert pp.fom is not None and pp.fom != 0
def _run_mask_image_array(data, mask, data_type, keep_nan=False):
lb, ub = 0.2, 0.8
data = data.astype(data_type)
# mask nan
data_cpp = data.copy()
t0 = time.perf_counter()
mask_image_data(data_cpp, keep_nan=keep_nan)
dt_cpp_raw = time.perf_counter() - t0
data_py = data.copy()
t0 = time.perf_counter()
if not keep_nan:
data_py[np.isnan(data_py)] = 0
dt_py_raw = time.perf_counter() - t0
np.testing.assert_array_equal(data_cpp, data_py)
# mask by threshold
data_cpp = data.copy()
t0 = time.perf_counter()
mask_image_data(data_cpp, threshold_mask=(lb, ub), keep_nan=keep_nan)
dt_cpp_th = time.perf_counter() - t0
data_py = data.copy()
t0 = time.perf_counter()
if not keep_nan:
data_py[np.isnan(data_py) | (data_py > ub) | (data_py < lb)] = 0
else:
data_py[(data_py > ub) | (data_py < lb)] = np.nan
dt_py_th = time.perf_counter() - t0
np.testing.assert_array_equal(data_cpp, data_py)
# mask by both image and threshold
data_cpp = data.copy()
t0 = time.perf_counter()
mask_image_data(data_cpp, image_mask=mask, threshold_mask=(lb, ub), keep_nan=keep_nan)
dt_cpp = time.perf_counter() - t0
data_py = data.copy()
t0 = time.perf_counter()
if not keep_nan:
data_py[np.isnan(data_py) | mask | (data_py > ub) | (data_py < lb)] = 0
else:
data_py[mask | (data_py > ub) | (data_py < lb)] = np.nan
dt_py = time.perf_counter() - t0
np.testing.assert_array_equal(data_cpp, data_py)
print(f"\nmask_image_data (keep_nan = {keep_nan}) with {data_type} - \n"
f"dt (cpp para) raw: {dt_cpp_raw:.4f}, "
f"dt (numpy) raw: {dt_py_raw:.4f}, \n"
f"dt (cpp para) threshold: {dt_cpp_th:.4f}, "
f"dt (numpy) threshold: {dt_py_th:.4f}, \n"
f"dt (cpp para) threshold and image: {dt_cpp:.4f}, "
f"dt (numpy) threshold and image: {dt_py:.4f}")
def _run_mask_image_array(data_type):
def prepare_data():
dt = np.ones((64, 1024, 512), dtype=data_type)
dt[::2, ::2, ::2] = np.nan
return dt
# mask nan
data = prepare_data()
t0 = time.perf_counter()
mask_image_data(data)
dt_cpp_raw = time.perf_counter() - t0
data = prepare_data()
t0 = time.perf_counter()
data[np.isnan(data)] = 0
dt_py_raw = time.perf_counter() - t0
# mask by threshold
data = prepare_data()
t0 = time.perf_counter()
mask_image_data(data, threshold_mask=(2., 3.))
dt_cpp_th = time.perf_counter() - t0
data = prepare_data()
t0 = time.perf_counter()
data[np.isnan(data)] = 0
data[(data > 3) | (data < 2)] = 0
dt_py_th = time.perf_counter() - t0
# mask by both image and threshold
mask = np.ones((1024, 512), dtype=np.bool)
data = prepare_data()
t0 = time.perf_counter()
mask_image_data(data, image_mask=mask, threshold_mask=(2., 3.))
dt_cpp = time.perf_counter() - t0
data = prepare_data()
t0 = time.perf_counter()
data[np.isnan(data)] = 0
data[:, mask] = 0
data[(data > 3) | (data < 2)] = 0
dt_py = time.perf_counter() - t0
print(f"\nmask_image_data with {data_type} - \n"
f"dt (cpp para) raw: {dt_cpp_raw:.4f}, "
f"dt (numpy) raw: {dt_py_raw:.4f}, \n"
f"dt (cpp para) threshold: {dt_cpp_th:.4f}, "
f"dt (numpy) threshold: {dt_py_th:.4f}, \n"
f"dt (cpp para) threshold and image: {dt_cpp:.4f}, "
f"dt (numpy) threshold and image: {dt_py:.4f}")
def testMaskImageData(self):
arr1d = np.ones(2, dtype=np.float32)
arr2d = np.ones((2, 2), dtype=np.float32)
arr3d = np.ones((2, 2, 2), dtype=np.float32)
arr4d = np.ones((2, 2, 2, 2), dtype=np.float32)
# test invalid input
with self.assertRaises(TypeError):
mask_image_data()
with self.assertRaises(TypeError):
mask_image_data(arr1d, threshold_mask=(1, 2))
with self.assertRaises(TypeError):
mask_image_data(arr4d, threshold_mask=(1, 2))
# test inconsistent shape
with self.assertRaises(TypeError):
mask_image_data(arr2d, image_mask=arr3d, threshold_mask=(1, 2))
with self.assertRaises(TypeError):
mask_image_data(arr3d, image_mask=arr2d, threshold_mask=(1, 2))
with self.assertRaises(TypeError):
mask_image_data(arr3d, image_mask=arr3d, threshold_mask=(1, 2))
with self.assertRaises(ValueError):
mask_image_data(arr3d, image_mask=np.ones((3, 2), dtype=bool))
with self.assertRaises(ValueError):
mask_image_data(arr2d, image_mask=np.ones((3, 2), dtype=bool))
# test inconsistent dtype
with self.assertRaises(TypeError):
mask_image_data(arr3d, image_mask=np.ones((2, 2), dtype=int))
# ------------
# single image
# ------------
# threshold mask
img = np.array([[1, 2, np.nan], [3, 4, 5]], dtype=np.float32)
mask_image_data(img, threshold_mask=(2, 3))
np.testing.assert_array_equal(
np.array([[0, 2, 0], [3, 0, 0]], dtype=np.float32), img)
# image mask
img = np.array([[1, np.nan, np.nan], [3, 4, 5]], dtype=np.float32)
img_mask = np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool)
mask_image_data(img, image_mask=img_mask)
np.testing.assert_array_equal(
np.array([[0, 0, 0], [0, 4, 0]], dtype=np.float32), img)
# ------------
# train images
# ------------
# threshold mask
img = np.array(
[[[1, 2, 3], [3, np.nan, 5]], [[1, 2, 3], [3, np.nan, 5]]],
dtype=np.float32)
mask_image_data(img, threshold_mask=(2, 3))
np.testing.assert_array_equal(
np.array([[[0, 2, 3], [3, 0, 0]], [[0, 2, 3], [3, 0, 0]]],
dtype=np.float32), img)
# image mask
img = np.array([[[1, 2, 3], [3, np.nan, np.nan]],
[[1, 2, 3], [3, np.nan, np.nan]]],
dtype=np.float32)
img_mask = np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool)
np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool)
mask_image_data(img, image_mask=img_mask)
np.testing.assert_array_equal(
np.array([[[0, 0, 3], [0, 0, 0]], [[0, 0, 3], [0, 0, 0]]],
dtype=np.float32), img)
def process(self, data):
processed = data['processed']
assembled = data['assembled']['sliced']
pp = processed.pp
pp.reset = self._reset
self._reset = False
pp.mode = self._mode
pp.indices_on = self._indices_on
pp.indices_off = self._indices_off
pp.analysis_type = self.analysis_type
pp.abs_difference = self._abs_difference
tid = processed.tid
# parameters used for processing pump-probe images
image_data = processed.image
image_mask = image_data.image_mask
threshold_mask = image_data.threshold_mask
reference = image_data.reference
n_images = image_data.n_images
xi = processed.pulse.xgm.intensity
digitizer_ch_norm = processed.pulse.digitizer.ch_normalizer
dpi = processed.pulse.digitizer[digitizer_ch_norm].pulse_integral
dropped_indices = processed.pidx.dropped_indices(n_images).tolist()
# pump-probe means
image_on, image_off, xi_on, xi_off, dpi_on, dpi_off, \
curr_indices, curr_means = self._compute_on_off_data(
tid, assembled, xi, dpi, dropped_indices, reference=reference)
# avoid calculating nanmean more than once
if curr_indices == list(range(n_images)):
if len(curr_means) == 1:
images_mean = curr_means[0].copy()
else:
images_mean = nanmean_image_data((image_on, image_off))
else:
if assembled.ndim == 3:
if dropped_indices:
indices = list(set(range(n_images)) - set(dropped_indices))
if not indices:
raise DropAllPulsesError(
f"{tid}: all pulses were dropped")
images_mean = nanmean_image_data(assembled, indices)
else:
# for performance
images_mean = nanmean_image_data(assembled)
else:
# Note: _image is _mean for train-resolved detectors
images_mean = assembled
# apply mask to the averaged images of the train
masked_mean = images_mean.copy()
mask_image_data(masked_mean,
image_mask=image_mask,
threshold_mask=threshold_mask)
processed.image.mean = images_mean
processed.image.masked_mean = masked_mean
# apply mask to the averaged on/off images
# Note: due to the in-place masking, the pump-probe code the the
# rest code are interleaved.
if image_on is not None:
mask_image_data(image_on,
image_mask=image_mask,
threshold_mask=threshold_mask)
mask_image_data(image_off,
image_mask=image_mask,
threshold_mask=threshold_mask)
processed.pp.image_on = image_on
processed.pp.image_off = image_off
processed.pp.on.xgm_intensity = xi_on
processed.pp.off.xgm_intensity = xi_off
processed.pp.on.digitizer_pulse_integral = dpi_on
processed.pp.off.digitizer_pulse_integral = dpi_off
def from_array(cls,
arr,
*,
image_mask=None,
threshold_mask=None,
sliced_indices=None,
poi_indices=None):
"""Construct a self-consistent ImageData."""
if not isinstance(arr, np.ndarray):
raise TypeError(r"Image data must be numpy.ndarray!")
if arr.ndim <= 1 or arr.ndim > 3:
raise ValueError(f"The shape of image data must be (y, x) or "
f"(n_pulses, y, x)!")
image_dtype = config['SOURCE_PROC_IMAGE_DTYPE']
if arr.dtype != image_dtype:
arr = arr.astype(image_dtype)
instance = cls()
if arr.ndim == 3:
n_images = len(arr)
instance.images = [None] * n_images
if poi_indices is None:
poi_indices = [0, 0]
for i in poi_indices:
instance.images[i] = arr[i]
instance.mean = nanmean_image_data(arr)
if sliced_indices is None:
instance.sliced_indices = list(range(n_images))
else:
sliced_indices = list(set(sliced_indices))
n_indices = len(sliced_indices)
if n_indices != n_images:
raise ValueError(
f"Length of sliced indices {sliced_indices} "
f"!= number of images {n_images}")
instance.sliced_indices = sliced_indices
else:
instance.images = [None]
instance.mean = arr
if sliced_indices is not None:
raise ValueError("Train-resolved data does not support "
"'sliced_indices'!")
instance.sliced_indices = [0] # be consistent
if poi_indices is None:
poi_indices = [0, 0]
instance.poi_indices = poi_indices
instance.masked_mean = instance.mean.copy()
if image_mask is None:
image_mask = np.zeros(arr.shape[-2:], dtype=np.bool)
mask = image_mask.copy()
image_with_mask(instance.masked_mean,
mask,
threshold_mask=threshold_mask)
if arr.ndim == 3:
for idx in poi_indices:
mask_image_data(instance.images[idx],
image_mask=image_mask,
threshold_mask=threshold_mask,
keep_nan=True)
instance.mask = mask
instance.image_mask = image_mask
instance.threshold_mask = threshold_mask
return instance
def testMaskImageDataWithOutput(self, keep_nan, mt, dtype):
arr1d = np.ones(2, dtype=dtype)
arr2d = np.ones((2, 2), dtype=dtype)
arr3d = np.ones((2, 2, 2), dtype=dtype)
out = np.ones((2, 2), dtype=bool)
with pytest.raises(TypeError):
mask_image_data(arr1d, keep_nan=keep_nan, out=out)
with pytest.raises(ValueError, match="must be 2D"):
mask_image_data(arr3d, keep_nan=keep_nan, out=out)
with pytest.raises(TypeError):
mask_image_data(arr2d, image_mask=arr3d, keep_nan=keep_nan, out=out)
with pytest.raises(TypeError):
mask_image_data(arr2d, image_mask=arr1d, keep_nan=keep_nan, out=out)
with pytest.raises(ValueError, match="must be bool"):
mask_image_data(arr2d, image_mask=arr1d, keep_nan=keep_nan, out=out.astype(float))
# raw
img = np.array([[1, 2, np.nan], [3, 4, 5]], dtype=dtype)
out = np.zeros((2, 3), dtype=bool)
mask_image_data(img, keep_nan=keep_nan, out=out)
np.testing.assert_array_equal(
np.array([[False, False, True], [False, False, False]], dtype=np.bool), out)
# threshold mask
img = np.array([[1, 2, np.nan], [3, 4, 5]], dtype=dtype)
out = np.zeros((2, 3), dtype=bool)
mask_image_data(img, threshold_mask=(2, 3), keep_nan=keep_nan, out=out)
np.testing.assert_array_equal(
np.array([[mt, 2, mt], [3, mt, mt]], dtype=dtype), img)
np.testing.assert_array_equal(
np.array([[True, False, True], [False, True, True]], dtype=np.bool), out)
# image mask
img = np.array([[1, np.nan, np.nan], [3, 4, 5]], dtype=dtype)
img_mask = np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool)
out = np.zeros((2, 3), dtype=bool)
mask_image_data(img, image_mask=img_mask, keep_nan=keep_nan, out=out)
np.testing.assert_array_equal(np.array([[mt, mt, mt], [mt, 4, mt]], dtype=dtype), img)
np.testing.assert_array_equal(
np.array([[True, True, True], [True, False, True]], dtype=np.bool), out)
# both masks
img = np.array([[1, 2, np.nan], [3, 4, 5]], dtype=dtype)
img_mask = np.array([[1, 0, 0], [1, 0, 0]], dtype=np.bool)
out = np.zeros((2, 3), dtype=bool)
mask_image_data(img, image_mask=img_mask, threshold_mask=(2, 3), keep_nan=keep_nan, out=out)
np.testing.assert_array_equal(
np.array([[mt, 2, mt], [mt, mt, mt]], dtype=dtype), img)
np.testing.assert_array_equal(
np.array([[True, False, True], [True, True, True]], dtype=np.bool), out)
def testMaskImageData(self, keep_nan, mt, dtype):
arr1d = np.ones(2, dtype=dtype)
arr2d = np.ones((2, 2), dtype=dtype)
arr3d = np.ones((2, 2, 2), dtype=dtype)
arr4d = np.ones((2, 2, 2, 2), dtype=dtype)
# test invalid input
with pytest.raises(TypeError):
mask_image_data()
with pytest.raises(TypeError):
mask_image_data(arr1d, threshold_mask=(1, 2), keep_nan=keep_nan)
with pytest.raises(TypeError):
mask_image_data(arr4d, threshold_mask=(1, 2), keep_nan=keep_nan)
# test inconsistent shape
with pytest.raises(TypeError):
mask_image_data(arr2d, image_mask=arr3d, threshold_mask=(1, 2), keep_nan=keep_nan)
with pytest.raises(TypeError):
mask_image_data(arr3d, image_mask=arr2d, threshold_mask=(1, 2), keep_nan=keep_nan)
with pytest.raises(TypeError):
mask_image_data(arr3d, image_mask=arr3d, threshold_mask=(1, 2), keep_nan=keep_nan)
with pytest.raises(ValueError):
mask_image_data(arr3d, image_mask=np.ones((3, 2), dtype=bool), keep_nan=keep_nan)
with pytest.raises(ValueError):
mask_image_data(arr2d, image_mask=np.ones((3, 2), dtype=bool), keep_nan=keep_nan)
# test inconsistent dtype
with pytest.raises(TypeError):
mask_image_data(arr3d, image_mask=np.ones((2, 2), dtype=int), keep_nan=keep_nan)
# ------------
# single image
# ------------
# raw
img = np.array([[1, 2, np.nan], [3, 4, 5]], dtype=dtype)
mask_image_data(img, keep_nan=keep_nan)
np.testing.assert_array_equal(
np.array([[1, 2, mt], [3, 4, 5]], dtype=np.float32), img)
# threshold mask
img = np.array([[1, 2, np.nan], [3, 4, 5]], dtype=dtype)
mask_image_data(img, threshold_mask=(2, 3), keep_nan=keep_nan)
np.testing.assert_array_equal(
np.array([[mt, 2, mt], [3, mt, mt]], dtype=np.float32), img)
# image mask
img = np.array([[1, np.nan, np.nan], [3, 4, 5]], dtype=dtype)
img_mask = np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool)
mask_image_data(img, image_mask=img_mask, keep_nan=keep_nan)
np.testing.assert_array_equal(
np.array([[mt, mt, mt], [mt, 4, mt]], dtype=np.float32), img)
# both masks
img = np.array([[1, np.nan, np.nan], [3, 4, 5]], dtype=dtype)
img_mask = np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool)
mask_image_data(img, image_mask=img_mask, threshold_mask=(2, 3), keep_nan=keep_nan)
np.testing.assert_array_equal(
np.array([[mt, mt, mt], [mt, mt, mt]], dtype=np.float32), img)
# ------------
# train images
# ------------
# raw
img = np.array([[[1, 2, 3], [3, np.nan, 5]],
[[1, 2, 3], [3, np.nan, 5]]], dtype=dtype)
mask_image_data(img, keep_nan=keep_nan)
np.testing.assert_array_equal(np.array([[[1, 2, 3], [3, mt, 5]],
[[1, 2, 3], [3, mt, 5]]], dtype=dtype), img)
# threshold mask
img = np.array([[[1, 2, 3], [3, np.nan, 5]],
[[1, 2, 3], [3, np.nan, 5]]], dtype=dtype)
mask_image_data(img, threshold_mask=(2, 3), keep_nan=keep_nan)
np.testing.assert_array_equal(np.array([[[mt, 2, 3], [3, mt, mt]],
[[mt, 2, 3], [3, mt, mt]]], dtype=dtype), img)
# image mask
img = np.array([[[1, 2, 3], [3, np.nan, np.nan]],
[[1, 2, 3], [3, np.nan, np.nan]]], dtype=dtype)
img_mask = np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool)
np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool)
mask_image_data(img, image_mask=img_mask, keep_nan=keep_nan)
np.testing.assert_array_equal(np.array([[[mt, mt, 3], [mt, mt, mt]],
[[mt, mt, 3], [mt, mt, mt]]], dtype=dtype), img)
# both masks
img = np.array([[[1, 2, 3], [3, np.nan, np.nan]],
[[1, 2, 6], [3, np.nan, np.nan]]], dtype=dtype)
img_mask = np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool)
np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool)
mask_image_data(img, image_mask=img_mask, threshold_mask=(2, 4), keep_nan=keep_nan)
np.testing.assert_array_equal(np.array([[[mt, mt, 3], [mt, mt, mt]],
[[mt, mt, mt], [mt, mt, mt]]], dtype=dtype), img)
def masked(self):
img = self._image.copy()
mask_image_data(img, threshold_mask=self._threshold_mask)
return img
