def _update_pois(self, image_data, assembled): if assembled.ndim == 2 or image_data.poi_indices is None: return n_images = image_data.n_images out_of_bound_indices = [] # only keep POI in 'images' for i in image_data.poi_indices: if i < n_images: image_data.images[i] = assembled[i].copy() mask_image_data(image_data.images[i], image_mask=self._image_mask, threshold_mask=self._threshold_mask, keep_nan=True) else: out_of_bound_indices.append(i) if image_data.poi_indices[1] == image_data.poi_indices[0]: # skip the second one if two POIs have the same index break if out_of_bound_indices: # This is still ProcessingError since it is not fatal and should # not stop the pipeline. raise ProcessingError( f"[Image processor] POI indices {out_of_bound_indices[0]} " f"is out of bound (0 - {n_images-1}")
def _integrate1d_imp(i): masked = assembled[i].copy() mask = np.zeros_like(image_mask) mask_image_data(masked, image_mask=image_mask, threshold_mask=threshold_mask, out=mask) return integ1d(masked, integ_points, mask=mask)
def testGeneral(self): with self.assertRaises(TypeError): _SimpleImageData([1, 2, 3]) gt_data = np.random.randn(3, 3).astype(np.float32) img_data = _SimpleImageData.from_array(gt_data) img_data.threshold_mask = (3, 6) masked_gt = gt_data.copy() mask_image_data(masked_gt, threshold_mask=(3, 6)) np.testing.assert_array_almost_equal(masked_gt, img_data.masked) self.assertEqual(1.0e-3, img_data.pixel_size)
def _compute_fom(self, roi, fom_type, image_mask, threshold_mask): if roi is None: return try: handler = self._fom_handlers[fom_type] except KeyError: raise UnknownParameterError( f"[ROI][FOM] Unknown FOM type: {fom_type}") mask_image_data(roi, image_mask=image_mask, threshold_mask=threshold_mask) return handler(roi, axis=(-2, -1))
def testAzimuthalIntegrationPp(self): proc = self._proc shape = (4, 128, 64) image_mask = np.zeros(shape[-2:], dtype=np.bool) image_mask[::2, ::2] = True threshold_mask = (0, 0.5) data, processed = self.data_with_assembled( 1001, shape, image_mask=image_mask, threshold_mask=threshold_mask) image_on = np.nanmean(data['assembled']['sliced'][::2, ...], axis=0) mask_on = np.zeros_like(image_mask) mask_image_data(image_on, image_mask=image_mask, threshold_mask=threshold_mask, out=mask_on) image_off = np.nanmean(data['assembled']['sliced'][1::2, ...], axis=0) mask_off = np.zeros_like(image_mask) mask_image_data(image_off, image_mask=image_mask, threshold_mask=threshold_mask, out=mask_off) pp = processed.pp pp.analysis_type = AnalysisType.AZIMUTHAL_INTEG pp.image_on = image_on pp.image_off = image_off pp.on.mask = mask_on pp.off.mask = mask_off with patch.object(proc._meta, 'has_analysis', side_effect=lambda x: True if x == AnalysisType.AZIMUTHAL_INTEG else False): proc.process(data) assert len(pp.y_on) == proc._integ_points assert all([not np.isnan(v) for v in pp.y_on]) assert len(pp.y_off) == proc._integ_points assert all([not np.isnan(v) for v in pp.y_off]) assert len(pp.x) == proc._integ_points assert len(pp.y) == proc._integ_points assert pp.fom is not None and pp.fom != 0
def _run_mask_image_array(data, mask, data_type, keep_nan=False): lb, ub = 0.2, 0.8 data = data.astype(data_type) # mask nan data_cpp = data.copy() t0 = time.perf_counter() mask_image_data(data_cpp, keep_nan=keep_nan) dt_cpp_raw = time.perf_counter() - t0 data_py = data.copy() t0 = time.perf_counter() if not keep_nan: data_py[np.isnan(data_py)] = 0 dt_py_raw = time.perf_counter() - t0 np.testing.assert_array_equal(data_cpp, data_py) # mask by threshold data_cpp = data.copy() t0 = time.perf_counter() mask_image_data(data_cpp, threshold_mask=(lb, ub), keep_nan=keep_nan) dt_cpp_th = time.perf_counter() - t0 data_py = data.copy() t0 = time.perf_counter() if not keep_nan: data_py[np.isnan(data_py) | (data_py > ub) | (data_py < lb)] = 0 else: data_py[(data_py > ub) | (data_py < lb)] = np.nan dt_py_th = time.perf_counter() - t0 np.testing.assert_array_equal(data_cpp, data_py) # mask by both image and threshold data_cpp = data.copy() t0 = time.perf_counter() mask_image_data(data_cpp, image_mask=mask, threshold_mask=(lb, ub), keep_nan=keep_nan) dt_cpp = time.perf_counter() - t0 data_py = data.copy() t0 = time.perf_counter() if not keep_nan: data_py[np.isnan(data_py) | mask | (data_py > ub) | (data_py < lb)] = 0 else: data_py[mask | (data_py > ub) | (data_py < lb)] = np.nan dt_py = time.perf_counter() - t0 np.testing.assert_array_equal(data_cpp, data_py) print(f"\nmask_image_data (keep_nan = {keep_nan}) with {data_type} - \n" f"dt (cpp para) raw: {dt_cpp_raw:.4f}, " f"dt (numpy) raw: {dt_py_raw:.4f}, \n" f"dt (cpp para) threshold: {dt_cpp_th:.4f}, " f"dt (numpy) threshold: {dt_py_th:.4f}, \n" f"dt (cpp para) threshold and image: {dt_cpp:.4f}, " f"dt (numpy) threshold and image: {dt_py:.4f}")
def _run_mask_image_array(data_type): def prepare_data(): dt = np.ones((64, 1024, 512), dtype=data_type) dt[::2, ::2, ::2] = np.nan return dt # mask nan data = prepare_data() t0 = time.perf_counter() mask_image_data(data) dt_cpp_raw = time.perf_counter() - t0 data = prepare_data() t0 = time.perf_counter() data[np.isnan(data)] = 0 dt_py_raw = time.perf_counter() - t0 # mask by threshold data = prepare_data() t0 = time.perf_counter() mask_image_data(data, threshold_mask=(2., 3.)) dt_cpp_th = time.perf_counter() - t0 data = prepare_data() t0 = time.perf_counter() data[np.isnan(data)] = 0 data[(data > 3) | (data < 2)] = 0 dt_py_th = time.perf_counter() - t0 # mask by both image and threshold mask = np.ones((1024, 512), dtype=np.bool) data = prepare_data() t0 = time.perf_counter() mask_image_data(data, image_mask=mask, threshold_mask=(2., 3.)) dt_cpp = time.perf_counter() - t0 data = prepare_data() t0 = time.perf_counter() data[np.isnan(data)] = 0 data[:, mask] = 0 data[(data > 3) | (data < 2)] = 0 dt_py = time.perf_counter() - t0 print(f"\nmask_image_data with {data_type} - \n" f"dt (cpp para) raw: {dt_cpp_raw:.4f}, " f"dt (numpy) raw: {dt_py_raw:.4f}, \n" f"dt (cpp para) threshold: {dt_cpp_th:.4f}, " f"dt (numpy) threshold: {dt_py_th:.4f}, \n" f"dt (cpp para) threshold and image: {dt_cpp:.4f}, " f"dt (numpy) threshold and image: {dt_py:.4f}")
def testMaskImageData(self): arr1d = np.ones(2, dtype=np.float32) arr2d = np.ones((2, 2), dtype=np.float32) arr3d = np.ones((2, 2, 2), dtype=np.float32) arr4d = np.ones((2, 2, 2, 2), dtype=np.float32) # test invalid input with self.assertRaises(TypeError): mask_image_data() with self.assertRaises(TypeError): mask_image_data(arr1d, threshold_mask=(1, 2)) with self.assertRaises(TypeError): mask_image_data(arr4d, threshold_mask=(1, 2)) # test inconsistent shape with self.assertRaises(TypeError): mask_image_data(arr2d, image_mask=arr3d, threshold_mask=(1, 2)) with self.assertRaises(TypeError): mask_image_data(arr3d, image_mask=arr2d, threshold_mask=(1, 2)) with self.assertRaises(TypeError): mask_image_data(arr3d, image_mask=arr3d, threshold_mask=(1, 2)) with self.assertRaises(ValueError): mask_image_data(arr3d, image_mask=np.ones((3, 2), dtype=bool)) with self.assertRaises(ValueError): mask_image_data(arr2d, image_mask=np.ones((3, 2), dtype=bool)) # test inconsistent dtype with self.assertRaises(TypeError): mask_image_data(arr3d, image_mask=np.ones((2, 2), dtype=int)) # ------------ # single image # ------------ # threshold mask img = np.array([[1, 2, np.nan], [3, 4, 5]], dtype=np.float32) mask_image_data(img, threshold_mask=(2, 3)) np.testing.assert_array_equal( np.array([[0, 2, 0], [3, 0, 0]], dtype=np.float32), img) # image mask img = np.array([[1, np.nan, np.nan], [3, 4, 5]], dtype=np.float32) img_mask = np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool) mask_image_data(img, image_mask=img_mask) np.testing.assert_array_equal( np.array([[0, 0, 0], [0, 4, 0]], dtype=np.float32), img) # ------------ # train images # ------------ # threshold mask img = np.array( [[[1, 2, 3], [3, np.nan, 5]], [[1, 2, 3], [3, np.nan, 5]]], dtype=np.float32) mask_image_data(img, threshold_mask=(2, 3)) np.testing.assert_array_equal( np.array([[[0, 2, 3], [3, 0, 0]], [[0, 2, 3], [3, 0, 0]]], dtype=np.float32), img) # image mask img = np.array([[[1, 2, 3], [3, np.nan, np.nan]], [[1, 2, 3], [3, np.nan, np.nan]]], dtype=np.float32) img_mask = np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool) np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool) mask_image_data(img, image_mask=img_mask) np.testing.assert_array_equal( np.array([[[0, 0, 3], [0, 0, 0]], [[0, 0, 3], [0, 0, 0]]], dtype=np.float32), img)
def process(self, data): processed = data['processed'] assembled = data['assembled']['sliced'] pp = processed.pp pp.reset = self._reset self._reset = False pp.mode = self._mode pp.indices_on = self._indices_on pp.indices_off = self._indices_off pp.analysis_type = self.analysis_type pp.abs_difference = self._abs_difference tid = processed.tid # parameters used for processing pump-probe images image_data = processed.image image_mask = image_data.image_mask threshold_mask = image_data.threshold_mask reference = image_data.reference n_images = image_data.n_images xi = processed.pulse.xgm.intensity digitizer_ch_norm = processed.pulse.digitizer.ch_normalizer dpi = processed.pulse.digitizer[digitizer_ch_norm].pulse_integral dropped_indices = processed.pidx.dropped_indices(n_images).tolist() # pump-probe means image_on, image_off, xi_on, xi_off, dpi_on, dpi_off, \ curr_indices, curr_means = self._compute_on_off_data( tid, assembled, xi, dpi, dropped_indices, reference=reference) # avoid calculating nanmean more than once if curr_indices == list(range(n_images)): if len(curr_means) == 1: images_mean = curr_means[0].copy() else: images_mean = nanmean_image_data((image_on, image_off)) else: if assembled.ndim == 3: if dropped_indices: indices = list(set(range(n_images)) - set(dropped_indices)) if not indices: raise DropAllPulsesError( f"{tid}: all pulses were dropped") images_mean = nanmean_image_data(assembled, indices) else: # for performance images_mean = nanmean_image_data(assembled) else: # Note: _image is _mean for train-resolved detectors images_mean = assembled # apply mask to the averaged images of the train masked_mean = images_mean.copy() mask_image_data(masked_mean, image_mask=image_mask, threshold_mask=threshold_mask) processed.image.mean = images_mean processed.image.masked_mean = masked_mean # apply mask to the averaged on/off images # Note: due to the in-place masking, the pump-probe code the the # rest code are interleaved. if image_on is not None: mask_image_data(image_on, image_mask=image_mask, threshold_mask=threshold_mask) mask_image_data(image_off, image_mask=image_mask, threshold_mask=threshold_mask) processed.pp.image_on = image_on processed.pp.image_off = image_off processed.pp.on.xgm_intensity = xi_on processed.pp.off.xgm_intensity = xi_off processed.pp.on.digitizer_pulse_integral = dpi_on processed.pp.off.digitizer_pulse_integral = dpi_off
def from_array(cls, arr, *, image_mask=None, threshold_mask=None, sliced_indices=None, poi_indices=None): """Construct a self-consistent ImageData.""" if not isinstance(arr, np.ndarray): raise TypeError(r"Image data must be numpy.ndarray!") if arr.ndim <= 1 or arr.ndim > 3: raise ValueError(f"The shape of image data must be (y, x) or " f"(n_pulses, y, x)!") image_dtype = config['SOURCE_PROC_IMAGE_DTYPE'] if arr.dtype != image_dtype: arr = arr.astype(image_dtype) instance = cls() if arr.ndim == 3: n_images = len(arr) instance.images = [None] * n_images if poi_indices is None: poi_indices = [0, 0] for i in poi_indices: instance.images[i] = arr[i] instance.mean = nanmean_image_data(arr) if sliced_indices is None: instance.sliced_indices = list(range(n_images)) else: sliced_indices = list(set(sliced_indices)) n_indices = len(sliced_indices) if n_indices != n_images: raise ValueError( f"Length of sliced indices {sliced_indices} " f"!= number of images {n_images}") instance.sliced_indices = sliced_indices else: instance.images = [None] instance.mean = arr if sliced_indices is not None: raise ValueError("Train-resolved data does not support " "'sliced_indices'!") instance.sliced_indices = [0] # be consistent if poi_indices is None: poi_indices = [0, 0] instance.poi_indices = poi_indices instance.masked_mean = instance.mean.copy() if image_mask is None: image_mask = np.zeros(arr.shape[-2:], dtype=np.bool) mask = image_mask.copy() image_with_mask(instance.masked_mean, mask, threshold_mask=threshold_mask) if arr.ndim == 3: for idx in poi_indices: mask_image_data(instance.images[idx], image_mask=image_mask, threshold_mask=threshold_mask, keep_nan=True) instance.mask = mask instance.image_mask = image_mask instance.threshold_mask = threshold_mask return instance
def testMaskImageDataWithOutput(self, keep_nan, mt, dtype): arr1d = np.ones(2, dtype=dtype) arr2d = np.ones((2, 2), dtype=dtype) arr3d = np.ones((2, 2, 2), dtype=dtype) out = np.ones((2, 2), dtype=bool) with pytest.raises(TypeError): mask_image_data(arr1d, keep_nan=keep_nan, out=out) with pytest.raises(ValueError, match="must be 2D"): mask_image_data(arr3d, keep_nan=keep_nan, out=out) with pytest.raises(TypeError): mask_image_data(arr2d, image_mask=arr3d, keep_nan=keep_nan, out=out) with pytest.raises(TypeError): mask_image_data(arr2d, image_mask=arr1d, keep_nan=keep_nan, out=out) with pytest.raises(ValueError, match="must be bool"): mask_image_data(arr2d, image_mask=arr1d, keep_nan=keep_nan, out=out.astype(float)) # raw img = np.array([[1, 2, np.nan], [3, 4, 5]], dtype=dtype) out = np.zeros((2, 3), dtype=bool) mask_image_data(img, keep_nan=keep_nan, out=out) np.testing.assert_array_equal( np.array([[False, False, True], [False, False, False]], dtype=np.bool), out) # threshold mask img = np.array([[1, 2, np.nan], [3, 4, 5]], dtype=dtype) out = np.zeros((2, 3), dtype=bool) mask_image_data(img, threshold_mask=(2, 3), keep_nan=keep_nan, out=out) np.testing.assert_array_equal( np.array([[mt, 2, mt], [3, mt, mt]], dtype=dtype), img) np.testing.assert_array_equal( np.array([[True, False, True], [False, True, True]], dtype=np.bool), out) # image mask img = np.array([[1, np.nan, np.nan], [3, 4, 5]], dtype=dtype) img_mask = np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool) out = np.zeros((2, 3), dtype=bool) mask_image_data(img, image_mask=img_mask, keep_nan=keep_nan, out=out) np.testing.assert_array_equal(np.array([[mt, mt, mt], [mt, 4, mt]], dtype=dtype), img) np.testing.assert_array_equal( np.array([[True, True, True], [True, False, True]], dtype=np.bool), out) # both masks img = np.array([[1, 2, np.nan], [3, 4, 5]], dtype=dtype) img_mask = np.array([[1, 0, 0], [1, 0, 0]], dtype=np.bool) out = np.zeros((2, 3), dtype=bool) mask_image_data(img, image_mask=img_mask, threshold_mask=(2, 3), keep_nan=keep_nan, out=out) np.testing.assert_array_equal( np.array([[mt, 2, mt], [mt, mt, mt]], dtype=dtype), img) np.testing.assert_array_equal( np.array([[True, False, True], [True, True, True]], dtype=np.bool), out)
def testMaskImageData(self, keep_nan, mt, dtype): arr1d = np.ones(2, dtype=dtype) arr2d = np.ones((2, 2), dtype=dtype) arr3d = np.ones((2, 2, 2), dtype=dtype) arr4d = np.ones((2, 2, 2, 2), dtype=dtype) # test invalid input with pytest.raises(TypeError): mask_image_data() with pytest.raises(TypeError): mask_image_data(arr1d, threshold_mask=(1, 2), keep_nan=keep_nan) with pytest.raises(TypeError): mask_image_data(arr4d, threshold_mask=(1, 2), keep_nan=keep_nan) # test inconsistent shape with pytest.raises(TypeError): mask_image_data(arr2d, image_mask=arr3d, threshold_mask=(1, 2), keep_nan=keep_nan) with pytest.raises(TypeError): mask_image_data(arr3d, image_mask=arr2d, threshold_mask=(1, 2), keep_nan=keep_nan) with pytest.raises(TypeError): mask_image_data(arr3d, image_mask=arr3d, threshold_mask=(1, 2), keep_nan=keep_nan) with pytest.raises(ValueError): mask_image_data(arr3d, image_mask=np.ones((3, 2), dtype=bool), keep_nan=keep_nan) with pytest.raises(ValueError): mask_image_data(arr2d, image_mask=np.ones((3, 2), dtype=bool), keep_nan=keep_nan) # test inconsistent dtype with pytest.raises(TypeError): mask_image_data(arr3d, image_mask=np.ones((2, 2), dtype=int), keep_nan=keep_nan) # ------------ # single image # ------------ # raw img = np.array([[1, 2, np.nan], [3, 4, 5]], dtype=dtype) mask_image_data(img, keep_nan=keep_nan) np.testing.assert_array_equal( np.array([[1, 2, mt], [3, 4, 5]], dtype=np.float32), img) # threshold mask img = np.array([[1, 2, np.nan], [3, 4, 5]], dtype=dtype) mask_image_data(img, threshold_mask=(2, 3), keep_nan=keep_nan) np.testing.assert_array_equal( np.array([[mt, 2, mt], [3, mt, mt]], dtype=np.float32), img) # image mask img = np.array([[1, np.nan, np.nan], [3, 4, 5]], dtype=dtype) img_mask = np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool) mask_image_data(img, image_mask=img_mask, keep_nan=keep_nan) np.testing.assert_array_equal( np.array([[mt, mt, mt], [mt, 4, mt]], dtype=np.float32), img) # both masks img = np.array([[1, np.nan, np.nan], [3, 4, 5]], dtype=dtype) img_mask = np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool) mask_image_data(img, image_mask=img_mask, threshold_mask=(2, 3), keep_nan=keep_nan) np.testing.assert_array_equal( np.array([[mt, mt, mt], [mt, mt, mt]], dtype=np.float32), img) # ------------ # train images # ------------ # raw img = np.array([[[1, 2, 3], [3, np.nan, 5]], [[1, 2, 3], [3, np.nan, 5]]], dtype=dtype) mask_image_data(img, keep_nan=keep_nan) np.testing.assert_array_equal(np.array([[[1, 2, 3], [3, mt, 5]], [[1, 2, 3], [3, mt, 5]]], dtype=dtype), img) # threshold mask img = np.array([[[1, 2, 3], [3, np.nan, 5]], [[1, 2, 3], [3, np.nan, 5]]], dtype=dtype) mask_image_data(img, threshold_mask=(2, 3), keep_nan=keep_nan) np.testing.assert_array_equal(np.array([[[mt, 2, 3], [3, mt, mt]], [[mt, 2, 3], [3, mt, mt]]], dtype=dtype), img) # image mask img = np.array([[[1, 2, 3], [3, np.nan, np.nan]], [[1, 2, 3], [3, np.nan, np.nan]]], dtype=dtype) img_mask = np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool) np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool) mask_image_data(img, image_mask=img_mask, keep_nan=keep_nan) np.testing.assert_array_equal(np.array([[[mt, mt, 3], [mt, mt, mt]], [[mt, mt, 3], [mt, mt, mt]]], dtype=dtype), img) # both masks img = np.array([[[1, 2, 3], [3, np.nan, np.nan]], [[1, 2, 6], [3, np.nan, np.nan]]], dtype=dtype) img_mask = np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool) np.array([[1, 1, 0], [1, 0, 1]], dtype=np.bool) mask_image_data(img, image_mask=img_mask, threshold_mask=(2, 4), keep_nan=keep_nan) np.testing.assert_array_equal(np.array([[[mt, mt, 3], [mt, mt, mt]], [[mt, mt, mt], [mt, mt, mt]]], dtype=dtype), img)
def masked(self): img = self._image.copy() mask_image_data(img, threshold_mask=self._threshold_mask) return img