def test_allowedvalues_correct(self):
self.assertTrue(valid.valid_1d_array(array=self.data,
allowed_values=1))
def load_cdi_data(
scan_number,
setup,
bin_during_loading=False,
flatfield=None,
hotpixels=None,
background=None,
normalize="skip",
debugging=False,
**kwargs,
):
"""
Load forward CDI data and preprocess it.
It applies beam stop correction and an optional photon threshold, normalization
and binning.
:param scan_number: the scan number to load
:param setup: an instance of the class Setup
:param bin_during_loading: True to bin the data during loading (faster)
:param flatfield: the 2D flatfield array
:param hotpixels: the 2D hotpixels array. 1 for a hotpixel, 0 for normal pixels.
:param background: the 2D background array to subtract to the data
:param normalize: 'skip' to skip, 'monitor' to normalize by the default monitor,
'sum_roi' to normalize by the integrated intensity in the region of interest
defined by detector.sum_roi
:param debugging: set to True to see plots
:param kwargs:
- 'photon_threshold': float, photon threshold to apply before binning
- 'frames_pattern': 1D array of int, of length data.shape[0]. If
frames_pattern is 0 at index, the frame at data[index] will be skipped,
if 1 the frame will added to the stack.
:return:
- the 3D data and mask arrays
- frames_logical: array of initial length the number of measured frames.
In case of padding the length changes. A frame whose index is set to 1 means
that it is used, 0 means not used, -1 means padded (added) frame.
- the monitor values used for the intensity normalization
"""
valid.valid_item(bin_during_loading,
allowed_types=bool,
name="bin_during_loading")
# check and load kwargs
valid.valid_kwargs(
kwargs=kwargs,
allowed_kwargs={"photon_threshold", "frames_pattern"},
name="kwargs",
)
photon_threshold = kwargs.get("photon_threshold", 0)
valid.valid_item(
photon_threshold,
allowed_types=Real,
min_included=0,
name="photon_threshold",
)
frames_pattern = kwargs.get("frames_pattern")
valid.valid_1d_array(frames_pattern,
allow_none=True,
allowed_values={0, 1},
name="frames_pattern")
rawdata, rawmask, monitor, frames_logical = setup.loader.load_check_dataset(
scan_number=scan_number,
setup=setup,
frames_pattern=frames_pattern,
bin_during_loading=bin_during_loading,
flatfield=flatfield,
hotpixels=hotpixels,
background=background,
normalize=normalize,
debugging=debugging,
)
#################################
# apply the beamstop correction #
#################################
rawdata = beamstop_correction(data=rawdata,
setup=setup,
debugging=debugging)
#####################################################
# apply an optional photon threshold before binning #
#####################################################
if photon_threshold != 0:
rawmask[rawdata < photon_threshold] = 1
rawdata[rawdata < photon_threshold] = 0
print("Applying photon threshold before binning: < ", photon_threshold)
####################################################################################
# bin data and mask in the detector plane if not already done during loading #
# binning in the stacking dimension is done at the very end of the data processing #
####################################################################################
if not bin_during_loading and ((setup.detector.binning[1] != 1) or
(setup.detector.binning[2] != 1)):
print(
"Binning the data: detector vertical axis by",
setup.detector.binning[1],
", detector horizontal axis by",
setup.detector.binning[2],
)
rawdata = util.bin_data(
rawdata,
(1, setup.detector.binning[1], setup.detector.binning[2]),
debugging=False,
)
rawmask = util.bin_data(
rawmask,
(1, setup.detector.binning[1], setup.detector.binning[2]),
debugging=False,
)
rawmask[np.nonzero(rawmask)] = 1
################################################
# pad the data to the shape defined by the ROI #
################################################
rawdata, rawmask = util.pad_from_roi(
arrays=(rawdata, rawmask),
roi=setup.detector.roi,
binning=setup.detector.binning[1:],
pad_value=(0, 1),
)
return rawdata, rawmask, frames_logical, monitor
def test_allowedvalues_set(self):
self.assertTrue(
valid.valid_1d_array(array=self.data, allowed_values={0, 1}))
def test_allowedvalues_array(self):
self.assertTrue(
valid.valid_1d_array(array=self.data,
allowed_values=np.array([0, 1])))
def test_allowedtypes_integral(self):
self.assertTrue(
valid.valid_1d_array(array=self.data, allowed_types=Integral))
def test_allowedtypes_float(self):
self.assertTrue(
valid.valid_1d_array(array=np.ones(4, dtype=float),
allowed_types=(int, float)))
def test_allow_none_true(self):
self.assertTrue(valid.valid_1d_array(None, allow_none=True))
def test_allowedtypes_none(self):
self.assertTrue(
valid.valid_1d_array(array=self.data, allowed_types=None))
def test_minlength_equal_length(self):
self.assertTrue(valid.valid_1d_array(self.data, min_length=7))
def test_minlength_None(self):
self.assertTrue(valid.valid_1d_array(self.data, min_length=None))
def test_minlength_null(self):
self.assertTrue(valid.valid_1d_array(self.data, min_length=0))
def test_length_null(self):
self.assertTrue(valid.valid_1d_array(np.empty(0), length=0))
def valid_param(key: str, value: Any) -> Tuple[Any, bool]:
"""
Validate a key value pair corresponding to an input parameter.
It will raise an exception if the check fails.
:param key: name of the parameter
:param value: the value of the parameter
:return: a tuple (formatted_value, is_valid). is_valid is True if the key
is valid, False otherwise.
"""
is_valid = True
allowed: Any = None
# convert 'None' to None
if value == "None":
value = None
# convert 'True' to True
if isinstance(value, str) and value.lower() == "true":
value = True
# convert 'False' to False
if isinstance(value, str) and value.lower() == "false":
value = False
# test the booleans first
if key in {
"align_axis",
"align_q",
"apodize",
"bin_during_loading",
"correct_refraction",
"custom_scan",
"debug",
"flag_interact",
"flip_reconstruction",
"get_temperature",
"grey_background",
"invert_phase",
"is_series",
"keep_size",
"mask_zero_event",
"reload_orthogonal",
"reload_previous",
"save",
"save_as_int",
"save_rawdata",
"save_support",
"save_to_mat",
"save_to_npz",
"save_to_vti",
"simulation",
"use_rawdata",
}:
valid.valid_item(value, allowed_types=bool, name=key)
elif key == "absorption":
valid.valid_item(value, allowed_types=Real, min_excluded=0, name=key)
elif key == "actuators":
valid.valid_container(value,
container_types=dict,
allow_none=True,
name=key)
elif key == "apodization_alpha":
valid.valid_container(
value,
container_types=(tuple, list, np.ndarray),
length=3,
item_types=Real,
min_included=0,
name=key,
)
value = np.asarray(value)
elif key == "apodization_mu":
valid.valid_container(
value,
container_types=(tuple, list, np.ndarray),
length=3,
item_types=Real,
min_included=0,
name=key,
)
value = np.asarray(value)
elif key == "apodization_sigma":
valid.valid_container(
value,
container_types=(tuple, list, np.ndarray),
length=3,
item_types=Real,
min_included=0,
name=key,
)
value = np.asarray(value)
elif key == "apodization_window":
allowed = {"blackman", "tukey", "normal"}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "averaging_space":
allowed = {"reciprocal_space", "direct_space"}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "axis_to_align":
valid.valid_container(
value,
container_types=(tuple, list, np.ndarray),
length=3,
item_types=Real,
name=key,
)
value = np.asarray(value)
elif key == "backend":
allowed = {
"Agg", "Qt5Agg", "module://matplotlib_inline.backend_inline"
}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "background_file":
valid.valid_container(value,
container_types=str,
min_length=1,
allow_none=True,
name=key)
elif key == "background_plot":
valid.valid_container(value,
container_types=str,
min_length=1,
name=key)
elif key == "beam_direction":
valid.valid_container(
value,
container_types=(tuple, list, np.ndarray),
length=3,
item_types=Real,
name=key,
)
value = np.asarray(value)
elif key == "beamline":
valid.valid_container(value,
container_types=str,
min_length=1,
name=key)
elif key == "bragg_peak":
valid.valid_container(
value,
container_types=(tuple, list),
item_types=Real,
min_included=0,
length=3,
allow_none=True,
name=key,
)
elif key == "cch1":
valid.valid_item(value, allowed_types=Real, name=key)
elif key == "cch2":
valid.valid_item(value, allowed_types=Real, name=key)
elif key == "center_fft":
allowed = {
"crop_sym_ZYX",
"crop_asym_ZYX",
"pad_asym_Z_crop_sym_YX",
"pad_sym_Z_crop_asym_YX",
"pad_sym_Z",
"pad_asym_Z",
"pad_sym_ZYX",
"pad_asym_ZYX",
"skip",
}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "centering_method":
allowed = {"com", "max", "max_com", "do_nothing"}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "center_roi_x":
valid.valid_item(value, allowed_types=int, allow_none=True, name=key)
elif key == "center_roi_y":
valid.valid_item(value, allowed_types=int, allow_none=True, name=key)
elif key == "colormap":
if value not in ["turbo", "custom"] and value not in cc.cm:
raise ValueError(f"unknow colormap '{value}'")
elif key == "comment":
valid.valid_container(value, container_types=str, name=key)
if value and not value.startswith("_"):
value += "_"
elif key == "config_file":
valid.valid_container(value,
container_types=str,
min_length=1,
name=key)
if not os.path.isfile(value):
raise ValueError(f"The file {value} does not exist")
elif key == "correlation_threshold":
valid.valid_item(value,
allowed_types=Real,
min_included=0,
max_included=1,
name=key)
elif key == "custom_images":
valid.valid_container(
value,
container_types=(tuple, list, np.ndarray),
item_types=int,
min_included=0,
allow_none=True,
name=key,
)
elif key == "custom_monitor":
valid.valid_container(
value,
container_types=(tuple, list, np.ndarray),
item_types=Real,
min_included=0,
allow_none=True,
name=key,
)
elif key == "custom_motors":
valid.valid_container(value,
container_types=dict,
allow_none=True,
name=key)
elif key == "custom_pixelsize":
valid.valid_item(value,
allowed_types=Real,
min_excluded=0,
allow_none=True,
name=key)
elif key == "data_dir":
if value is not None:
valid.valid_container(value,
container_types=str,
min_length=1,
name=key)
if not os.path.isdir(value):
raise ValueError(f"The directory {value} does not exist")
elif key == "data_frame":
allowed = {"detector", "crystal", "laboratory"}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "dirbeam_detector_angles":
valid.valid_container(
value,
container_types=(list, tuple),
item_types=Real,
length=2,
allow_none=True,
name=key,
)
elif key == "direct_beam":
valid.valid_container(
value,
container_types=(list, tuple),
item_types=Real,
length=2,
allow_none=True,
name=key,
)
elif key == "detector":
valid.valid_container(value,
container_types=str,
min_length=1,
name=key)
elif key == "detrot":
valid.valid_item(value, allowed_types=Real, name=key)
elif key == "dispersion":
valid.valid_item(value, allowed_types=Real, min_excluded=0, name=key)
elif key == "energy":
if value is None or isinstance(value, Number):
valid.valid_item(value,
allowed_types=Real,
min_excluded=0,
allow_none=True,
name=key)
else:
valid.valid_container(
value,
container_types=(tuple, list, np.ndarray),
min_length=1,
item_types=Real,
min_excluded=0,
name=key,
)
elif key == "fill_value_mask":
allowed = (0, 1)
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "fix_size":
valid.valid_container(
value,
container_types=(tuple, list),
length=6,
item_types=int,
allow_none=True,
name=key,
)
elif key == "fix_voxel":
valid.valid_item(value,
allowed_types=Real,
min_excluded=0,
allow_none=True,
name=key)
elif key == "flatfield_file":
valid.valid_container(value,
container_types=str,
min_length=1,
allow_none=True,
name=key)
elif key == "frames_pattern":
if value is not None:
value = np.asarray(value)
valid.valid_1d_array(value,
allow_none=False,
allowed_values={0, 1},
name=key)
elif key == "half_width_avg_phase":
valid.valid_item(value, allowed_types=int, min_included=0, name=key)
elif key == "hotpixels_file":
valid.valid_container(value,
container_types=str,
min_length=1,
allow_none=True,
name=key)
elif key == "inplane_angle":
valid.valid_item(value, allowed_types=Real, allow_none=True, name=key)
elif key == "interpolation_method":
allowed = {"xrayutilities", "linearization"}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "isosurface_strain":
valid.valid_item(value, allowed_types=Real, min_excluded=0, name=key)
elif key == "linearity_func":
valid.valid_container(
value,
container_types=(tuple, list, np.ndarray),
length=5,
item_types=Real,
allow_none=True,
name=key,
)
elif key == "median_filter":
allowed = {"median", "interp_isolated", "mask_isolated", "skip"}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "median_filter_order":
valid.valid_item(value, allowed_types=int, min_included=0, name=key)
elif key == "normalize_flux":
allowed = {"monitor", "skip"}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "offset_inplane":
valid.valid_item(value, allowed_types=Real, name=key)
elif key == "offset_method":
allowed = {"com", "mean"}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "optical_path_method":
allowed = {"threshold", "defect"}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "original_size":
valid.valid_container(
value,
container_types=(tuple, list, np.ndarray),
length=3,
item_types=int,
min_excluded=0,
allow_none=True,
name=key,
)
elif key == "outofplane_angle":
valid.valid_item(value, allowed_types=Real, allow_none=True, name=key)
elif key == "output_size":
valid.valid_container(
value,
container_types=(tuple, list, np.ndarray),
length=3,
item_types=int,
min_excluded=0,
allow_none=True,
name=key,
)
elif key == "pad_size":
valid.valid_container(
value,
container_types=(tuple, list),
length=3,
item_types=int,
allow_none=True,
name=key,
)
elif key == "phase_offset":
valid.valid_item(value, allowed_types=Real, allow_none=True, name=key)
elif key == "phase_offset_origin":
valid.valid_item(value, allowed_types=Real, allow_none=True, name=key)
elif key == "phase_ramp_removal":
allowed = {"gradient", "upsampling"}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "phase_range":
valid.valid_item(value, allowed_types=Real, min_excluded=0, name=key)
elif key == "phasing_binning":
valid.valid_container(
value,
container_types=(tuple, list, np.ndarray),
length=3,
item_types=int,
min_excluded=0,
name=key,
)
elif key == "photon_filter":
allowed = {"loading", "postprocessing"}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "photon_threshold":
valid.valid_item(value, allowed_types=Real, min_included=0, name=key)
elif key == "preprocessing_binning":
valid.valid_container(
value,
container_types=(tuple, list, np.ndarray),
length=3,
item_types=int,
min_excluded=0,
name=key,
)
elif key == "reconstruction_file":
valid.valid_container(value,
container_types=str,
min_length=1,
allow_none=True,
name=key)
if value is not None and not os.path.isfile(value):
raise ValueError(f"The file {value} does not exist")
elif key in {"ref_axis_q", "ref_axis"}:
allowed = {"x", "y", "z"}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "reference_spacing":
valid.valid_item(value,
allowed_types=Real,
min_included=0,
allow_none=True,
name=key)
elif key == "reference_temperature":
valid.valid_item(value,
allowed_types=Real,
min_included=0,
allow_none=True,
name=key)
elif key == "reflection":
valid.valid_container(
value,
container_types=(tuple, list, np.ndarray),
length=3,
item_types=Real,
name=key,
)
value = np.asarray(value)
elif key == "rocking_angle":
allowed = {"outofplane", "inplane", "energy"}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "roi_detector":
valid.valid_container(
value,
container_types=(tuple, list, np.ndarray),
length=4,
item_types=int,
allow_none=True,
name=key,
)
elif key == "roll_modes":
valid.valid_container(
value,
container_types=(tuple, list, np.ndarray),
length=3,
item_types=int,
name=key,
)
elif key == "root_folder":
valid.valid_container(value,
container_types=str,
min_length=1,
name=key)
if not os.path.isdir(value):
raise ValueError(f"The directory {value} does not exist")
elif key == "sample_inplane":
valid.valid_container(
value,
container_types=(tuple, list),
length=3,
item_types=Real,
name=key,
)
elif key == "sample_name":
valid.valid_container(value,
container_types=str,
min_length=1,
name=key)
elif key == "sample_offsets":
valid.valid_container(value,
container_types=(tuple, list, np.ndarray),
allow_none=True,
name=key)
elif key == "sample_outofplane":
valid.valid_container(
value,
container_types=(tuple, list),
length=3,
item_types=Real,
name=key,
)
elif key == "save_dir":
valid.valid_container(value,
container_types=str,
min_length=1,
allow_none=True,
name=key)
if isinstance(value, str) and not value.endswith("/"):
value += "/"
elif key == "save_frame":
allowed = {"laboratory", "crystal", "lab_flat_sample"}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "scan":
valid.valid_item(value, allowed_types=int, min_included=0, name=key)
elif key == "scans":
if isinstance(value, Real):
value = (value, )
valid.valid_container(value,
container_types=(tuple, list, np.ndarray),
min_length=1,
name=key)
elif key == "sdd":
valid.valid_item(value,
allowed_types=Real,
min_excluded=0,
allow_none=True,
name=key)
elif key == "sort_method":
allowed = {"mean_amplitude", "variance", "variance/mean", "volume"}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "specfile_name":
valid.valid_container(value,
container_types=str,
allow_none=True,
name=key)
elif key == "strain_method":
allowed = {"default", "defect"}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "strain_range":
valid.valid_item(value, allowed_types=Real, min_excluded=0, name=key)
elif key == "template_imagefile":
valid.valid_container(value,
container_types=str,
min_length=0,
allow_none=True,
name=key)
elif key == "threshold_gradient":
valid.valid_item(value, allowed_types=Real, min_excluded=0, name=key)
elif key == "threshold_unwrap_refraction":
valid.valid_item(value, allowed_types=Real, min_included=0, name=key)
elif key == "tick_direction":
allowed = {"out", "in", "inout"}
if value not in allowed:
raise ParameterError(key, value, allowed)
elif key == "tick_length":
valid.valid_item(value, allowed_types=int, min_included=1, name=key)
elif key == "tick_spacing":
valid.valid_item(value, allowed_types=Real, min_excluded=0, name=key)
elif key == "tick_width":
valid.valid_item(value, allowed_types=int, min_included=1, name=key)
elif key == "tilt_angle":
valid.valid_item(value, allowed_types=Real, allow_none=True, name=key)
elif key == "tiltazimuth":
valid.valid_item(value, allowed_types=Real, name=key)
elif key == "tilt_detector":
valid.valid_item(value, allowed_types=Real, name=key)
else:
# this key is not in the known parameters
is_valid = False
return value, is_valid
