def _compute_on_off_data(self,
tid,
assembled,
xi,
dpi,
dropped_indices,
*,
reference=None):
curr_indices = []
curr_means = []
image_on, image_off = None, None
xi_on, xi_off = None, None
dpi_on, dpi_off = None, None
mode = self._mode
if mode != PumpProbeMode.UNDEFINED:
self._parse_on_off_indices(assembled.shape)
if assembled.ndim == 3:
self._validate_on_off_indices(assembled.shape[0])
indices_on = list(set(self._indices_on) - set(dropped_indices))
indices_off = list(set(self._indices_off) - set(dropped_indices))
# on and off are not from different trains
if mode in (PumpProbeMode.REFERENCE_AS_OFF,
PumpProbeMode.SAME_TRAIN):
if assembled.ndim == 3:
# pulse resolved
if not indices_on:
raise DropAllPulsesError(
f"{tid}: all on pulses were dropped")
image_on = nanmean_image_data(assembled, indices_on)
curr_indices.extend(indices_on)
curr_means.append(image_on)
else:
image_on = assembled.copy()
if xi is not None:
xi_on = np.mean(xi[indices_on])
if dpi is not None:
dpi_on = np.mean(dpi[indices_on])
if mode == PumpProbeMode.REFERENCE_AS_OFF:
if reference is None:
image_off = np.zeros_like(image_on)
else:
# do not operate on the original reference image
image_off = reference.copy()
if xi is not None:
xi_off = 1.
if dpi is not None:
dpi_off = 1.
else:
# train-resolved data does not have the mode 'SAME_TRAIN'
if not indices_off:
raise DropAllPulsesError(
f"{tid}: all off pulses were dropped")
image_off = nanmean_image_data(assembled, indices_off)
curr_indices.extend(indices_off)
curr_means.append(image_off)
if xi is not None:
xi_off = np.mean(xi[indices_off])
if dpi is not None:
dpi_off = np.mean(dpi[indices_off])
if mode in (PumpProbeMode.EVEN_TRAIN_ON,
PumpProbeMode.ODD_TRAIN_ON):
# on and off are from different trains
if mode == PumpProbeMode.EVEN_TRAIN_ON:
flag = 1
else: # mode == PumpProbeMode.ODD_TRAIN_ON:
flag = 0
if tid % 2 == 1 ^ flag:
if assembled.ndim == 3:
if not indices_on:
raise DropAllPulsesError(
f"{tid}: all on pulses were dropped")
self._prev_unmasked_on = nanmean_image_data(
assembled, indices_on)
curr_indices.extend(indices_on)
curr_means.append(self._prev_unmasked_on)
else:
self._prev_unmasked_on = assembled.copy()
if xi is not None:
self._prev_xi_on = np.mean(xi[indices_on])
if dpi is not None:
self._prev_dpi_on = np.mean(dpi[indices_on])
else:
if self._prev_unmasked_on is not None:
image_on = self._prev_unmasked_on
xi_on = self._prev_xi_on
dpi_on = self._prev_dpi_on
self._prev_unmasked_on = None
# acknowledge off image only if on image has been received
if assembled.ndim == 3:
if not indices_off:
raise DropAllPulsesError(
f"{tid}: all off pulses were dropped")
image_off = nanmean_image_data(
assembled, indices_off)
curr_indices.extend(indices_off)
curr_means.append(image_off)
else:
image_off = assembled.copy()
if xi is not None:
xi_off = np.mean(xi[indices_off])
if dpi is not None:
dpi_off = np.mean(dpi[indices_off])
return (image_on, image_off, xi_on, xi_off, dpi_on, dpi_off,
sorted(curr_indices), curr_means)
def from_array(cls,
arr,
*,
image_mask=None,
threshold_mask=None,
sliced_indices=None,
poi_indices=None):
"""Construct a self-consistent ImageData."""
if not isinstance(arr, np.ndarray):
raise TypeError(r"Image data must be numpy.ndarray!")
if arr.ndim <= 1 or arr.ndim > 3:
raise ValueError(f"The shape of image data must be (y, x) or "
f"(n_pulses, y, x)!")
image_dtype = config['SOURCE_PROC_IMAGE_DTYPE']
if arr.dtype != image_dtype:
arr = arr.astype(image_dtype)
instance = cls()
if arr.ndim == 3:
n_images = len(arr)
instance.images = [None] * n_images
if poi_indices is None:
poi_indices = [0, 0]
for i in poi_indices:
instance.images[i] = arr[i]
instance.mean = nanmean_image_data(arr)
if sliced_indices is None:
instance.sliced_indices = list(range(n_images))
else:
sliced_indices = list(set(sliced_indices))
n_indices = len(sliced_indices)
if n_indices != n_images:
raise ValueError(
f"Length of sliced indices {sliced_indices} "
f"!= number of images {n_images}")
instance.sliced_indices = sliced_indices
else:
instance.images = [None]
instance.mean = arr
if sliced_indices is not None:
raise ValueError("Train-resolved data does not support "
"'sliced_indices'!")
instance.sliced_indices = [0] # be consistent
if poi_indices is None:
poi_indices = [0, 0]
instance.poi_indices = poi_indices
instance.masked_mean = instance.mean.copy()
if image_mask is None:
image_mask = np.zeros(arr.shape[-2:], dtype=np.bool)
mask = image_mask.copy()
image_with_mask(instance.masked_mean,
mask,
threshold_mask=threshold_mask)
if arr.ndim == 3:
for idx in poi_indices:
mask_image_data(instance.images[idx],
image_mask=image_mask,
threshold_mask=threshold_mask,
keep_nan=True)
instance.mask = mask
instance.image_mask = image_mask
instance.threshold_mask = threshold_mask
return instance
def process(self, data):
processed = data['processed']
assembled = data['assembled']['sliced']
pp = processed.pp
pp.reset = self._reset
self._reset = False
pp.mode = self._mode
pp.indices_on = self._indices_on
pp.indices_off = self._indices_off
pp.analysis_type = self.analysis_type
pp.abs_difference = self._abs_difference
tid = processed.tid
# parameters used for processing pump-probe images
image_data = processed.image
image_mask = image_data.image_mask
threshold_mask = image_data.threshold_mask
reference = image_data.reference
n_images = image_data.n_images
xi = processed.pulse.xgm.intensity
digitizer_ch_norm = processed.pulse.digitizer.ch_normalizer
dpi = processed.pulse.digitizer[digitizer_ch_norm].pulse_integral
dropped_indices = processed.pidx.dropped_indices(n_images).tolist()
# pump-probe means
image_on, image_off, xi_on, xi_off, dpi_on, dpi_off, \
curr_indices, curr_means = self._compute_on_off_data(
tid, assembled, xi, dpi, dropped_indices, reference=reference)
# avoid calculating nanmean more than once
if curr_indices == list(range(n_images)):
if len(curr_means) == 1:
images_mean = curr_means[0].copy()
else:
images_mean = nanmean_image_data((image_on, image_off))
else:
if assembled.ndim == 3:
if dropped_indices:
indices = list(set(range(n_images)) - set(dropped_indices))
if not indices:
raise DropAllPulsesError(
f"[Pump-probe] {tid}: all pulses were dropped")
images_mean = nanmean_image_data(assembled, kept=indices)
else:
# for performance
images_mean = nanmean_image_data(assembled)
else:
# Note: _image is _mean for train-resolved detectors
images_mean = assembled
# apply mask to the averaged images of the train
masked_mean = images_mean.copy()
mask = np.zeros_like(image_mask)
mask_image_data(masked_mean,
image_mask=image_mask,
threshold_mask=threshold_mask,
out=mask)
processed.image.mean = images_mean
processed.image.mask = mask
processed.image.masked_mean = masked_mean
# apply mask to the averaged on/off images
# Note: due to the in-place masking, the pump-probe code the the
# rest code are interleaved.
if image_on is not None:
mask_on = np.zeros_like(image_mask)
mask_image_data(image_on,
image_mask=image_mask,
threshold_mask=threshold_mask,
out=mask_on)
mask_off = np.zeros_like(image_mask)
mask_image_data(image_off,
image_mask=image_mask,
threshold_mask=threshold_mask,
out=mask_off)
processed.pp.image_on = image_on
processed.pp.image_off = image_off
processed.pp.on.mask = mask_on
processed.pp.off.mask = mask_off
processed.pp.on.xgm_intensity = xi_on
processed.pp.off.xgm_intensity = xi_off
processed.pp.on.digitizer_pulse_integral = dpi_on
processed.pp.off.digitizer_pulse_integral = dpi_off
def testNanmeanImageData(self):
arr1d = np.ones(2, dtype=np.float32)
arr2d = np.ones((2, 2), dtype=np.float32)
arr3d = np.ones((2, 2, 2), dtype=np.float32)
arr4d = np.ones((2, 2, 2, 2), dtype=np.float32)
# test invalid shapes
with self.assertRaises(TypeError):
nanmean_image_data(arr4d)
with self.assertRaises(TypeError):
nanmean_image_data(arr1d)
# kept is an empty list
with self.assertRaises(ValueError):
nanmean_image_data(arr3d, kept=[])
# test two images have different shapes
with self.assertRaises(ValueError):
nanmean_image_data([arr2d, np.ones((2, 3), dtype=np.float32)])
# test two images have different dtype
with self.assertRaises(TypeError):
nanmean_image_data([arr2d, np.ones((2, 3), dtype=np.float64)])
# invalid input
with self.assertRaises(TypeError):
nanmean_image_data(arr2d, arr2d)
# input is a 2D array
data = np.random.randn(2, 2)
ret = nanmean_image_data(data)
np.testing.assert_array_equal(data, ret)
self.assertIsNot(ret, data)
# input is a 3D array
data = np.array([[[np.nan, 2, np.nan], [ 1, 2, -np.inf]],
[[ 1, -np.inf, np.nan], [np.nan, 3, np.inf]],
[[np.inf, 4, np.nan], [ 1, 4, 1]]], dtype=np.float32)
with np.warnings.catch_warnings():
np.warnings.simplefilter("ignore", category=RuntimeWarning)
# Note that mean of -np.inf, np.inf and 1 are np.nan!!!
expected = np.array([[np.inf, -np.inf, np.nan], [ 1, 3, np.nan]], dtype=np.float32)
np.testing.assert_array_almost_equal(expected, np.nanmean(data, axis=0))
np.testing.assert_array_almost_equal(expected, nanmean_image_data(data))
# test nanmean on the sliced array
np.testing.assert_array_almost_equal(np.nanmean(data[0:3, ...], axis=0),
nanmean_image_data(data, kept=[0, 1, 2]))
np.testing.assert_array_almost_equal(np.nanmean(data[1:2, ...], axis=0),
nanmean_image_data(data, kept=[1]))
np.testing.assert_array_almost_equal(np.nanmean(data[0:3:2, ...], axis=0),
nanmean_image_data(data, kept=[0, 2]))
# input are a list/tuple of two images
img1 = np.array([[1, 1, 2], [np.inf, np.nan, 0]], dtype=np.float32)
img2 = np.array([[np.nan, 0, 4], [2, np.nan, -np.inf]], dtype=np.float32)
expected = np.array([[1., 0.5, 3], [np.inf, np.nan, -np.inf]])
np.testing.assert_array_almost_equal(expected, nanmean_image_data((img1, img2)))
np.testing.assert_array_almost_equal(expected, nanmean_image_data([img1, img2]))
def _update_gain_offset(self):
try:
gain_updated, gain, offset_updated, offset = self._cal_sub.update()
except Exception as e:
raise ImageProcessingError(str(e))
if gain_updated:
if gain is not None:
if gain.dtype != _IMAGE_DTYPE:
gain = gain.astype(_IMAGE_DTYPE)
self._full_gain = gain
logger.info(f"[Image processor] Loaded gain constants with "
f"shape = {gain.shape}")
if gain.ndim == 3:
self._gain = gain[self._gain_cells]
self._gain_mean = nanmean_image_data(self._gain)
else:
# no need to copy
self._gain = gain
self._gain_mean = gain
else:
self._full_gain = None
logger.info(f"[Image processor] Gain constants removed")
self._gain = None
self._gain_mean = None
else:
if self._gain_cells_updated and self._full_gain is not None:
self._gain = self._full_gain[self._gain_cells]
self._gain_mean = nanmean_image_data(self._gain)
if offset_updated:
if offset is not None:
if offset.dtype != _IMAGE_DTYPE:
offset = offset.astype(_IMAGE_DTYPE)
self._full_offset = offset
logger.info(f"[Image processor] Loaded offset constants with "
f"shape = {offset.shape}")
if offset.ndim == 3:
self._offset = offset[self._offset_cells]
self._offset_mean = nanmean_image_data(self._offset)
else:
# no need to copy
self._offset = offset
self._offset_mean = offset
else:
self._full_offset = None
logger.info(f"[Image processor] Offset constants removed")
self._offset = None
self._offset_mean = None
else:
if self._offset_cells_updated:
if self._full_offset is not None:
self._offset = self._full_offset[self._offset_cells]
self._offset_mean = nanmean_image_data(self._offset)
else:
self._offset = None
self._offset_mean = None
