def tc():
test = _Test()
# Predict the reflections in place and put in a reflection manager
ref_predictor = StillsExperimentsPredictor(test.stills_experiments)
ref_predictor(test.reflections)
test.refman = ReflectionManagerFactory.from_parameters_reflections_experiments(
refman_phil_scope.extract(),
test.reflections,
test.stills_experiments,
do_stills=True,
)
test.refman.finalise()
return test
def tc():
test = _Test()
# Predict the reflections in place and put in a reflection manager
ref_predictor = StillsExperimentsPredictor(test.stills_experiments)
ref_predictor(test.reflections)
test.refman = ReflectionManagerFactory.from_parameters_reflections_experiments(
refman_phil_scope.extract(),
test.reflections,
test.stills_experiments,
do_stills=True,
)
test.refman.finalise()
# Build a prediction parameterisation for the stills experiment
test.pred_param = StillsPredictionParameterisation(
test.stills_experiments,
detector_parameterisations=[test.det_param],
beam_parameterisations=[test.s0_param],
xl_orientation_parameterisations=[test.xlo_param],
xl_unit_cell_parameterisations=[test.xluc_param],
)
return test
def test_stills_pred_param(tc):
print("Testing derivatives for StillsPredictionParameterisation")
print("========================================================")
# Build a prediction parameterisation for the stills experiment
pred_param = StillsPredictionParameterisation(
tc.stills_experiments,
detector_parameterisations=[tc.det_param],
beam_parameterisations=[tc.s0_param],
xl_orientation_parameterisations=[tc.xlo_param],
xl_unit_cell_parameterisations=[tc.xluc_param],
)
# Predict the reflections in place. Must do this ahead of calculating
# the analytical gradients so quantities like s1 are correct
ref_predictor = StillsExperimentsPredictor(tc.stills_experiments)
ref_predictor(tc.reflections)
# get analytical gradients
an_grads = pred_param.get_gradients(tc.reflections)
fd_grads = tc.get_fd_gradients(pred_param, ref_predictor)
for i, (an_grad, fd_grad) in enumerate(zip(an_grads, fd_grads)):
# compare FD with analytical calculations
print(f"\nParameter {i}: {fd_grad['name']}")
for name in ["dX_dp", "dY_dp", "dDeltaPsi_dp"]:
print(name)
a = fd_grad[name]
b = an_grad[name]
abs_error = a - b
fns = five_number_summary(abs_error)
print((" summary of absolute errors: %9.6f %9.6f %9.6f " +
"%9.6f %9.6f") % fns)
assert flex.max(flex.abs(abs_error)) < 0.0003
# largest absolute error found to be about 0.00025 for dY/dp of
# Crystal0g_param_3. Reject outlying absolute errors and test again.
iqr = fns[3] - fns[1]
# skip further stats on errors with an iqr of near zero, e.g. dDeltaPsi_dp
# for detector parameters, which are all equal to zero
if iqr < 1.0e-10:
continue
sel1 = abs_error < fns[3] + 1.5 * iqr
sel2 = abs_error > fns[1] - 1.5 * iqr
sel = sel1 & sel2
tst = flex.max_index(flex.abs(abs_error.select(sel)))
tst_val = abs_error.select(sel)[tst]
n_outliers = sel.count(False)
print((" {0} outliers rejected, leaving greatest " +
"absolute error: {1:9.6f}").format(n_outliers, tst_val))
# largest absolute error now 0.000086 for dX/dp of Beam0Mu2
assert abs(tst_val) < 0.00009
# Completely skip parameters with FD gradients all zero (e.g. gradients of
# DeltaPsi for detector parameters)
sel1 = flex.abs(a) < 1.0e-10
if sel1.all_eq(True):
continue
# otherwise calculate normalised errors, by dividing absolute errors by
# the IQR (more stable than relative error calculation)
norm_error = abs_error / iqr
fns = five_number_summary(norm_error)
print((" summary of normalised errors: %9.6f %9.6f %9.6f " +
"%9.6f %9.6f") % fns)
# largest normalised error found to be about 25.7 for dY/dp of
# Crystal0g_param_3.
try:
assert flex.max(flex.abs(norm_error)) < 30
except AssertionError as e:
e.args += (
f"extreme normalised error value: {flex.max(flex.abs(norm_error))}",
)
raise e
# Reject outlying normalised errors and test again
iqr = fns[3] - fns[1]
if iqr > 0.0:
sel1 = norm_error < fns[3] + 1.5 * iqr
sel2 = norm_error > fns[1] - 1.5 * iqr
sel = sel1 & sel2
tst = flex.max_index(flex.abs(norm_error.select(sel)))
tst_val = norm_error.select(sel)[tst]
n_outliers = sel.count(False)
# most outliers found for for dY/dp of Crystal0g_param_3 (which had
# largest errors, so no surprise there).
try:
assert n_outliers < 250
except AssertionError as e:
e.args += (f"too many outliers rejected: {n_outliers}", )
raise e
print(
(" {0} outliers rejected, leaving greatest " +
"normalised error: {1:9.6f}").format(n_outliers, tst_val))
# largest normalied error now about -4. for dX/dp of Detector0Tau1
assert abs(tst_val) < 6, f"should be < 6, not {tst_val}"
def test_check_and_remove():
test = _Test()
# Override the single panel model and parameterisation. This test function
# exercises the code for non-hierarchical multi-panel detectors. The
# hierarchical detector version is tested via test_cspad_refinement.py
multi_panel_detector = Detector()
for x in range(3):
for y in range(3):
new_panel = make_panel_in_array((x, y), test.detector[0])
multi_panel_detector.add_panel(new_panel)
test.detector = multi_panel_detector
test.stills_experiments[0].detector = multi_panel_detector
test.det_param = DetectorParameterisationMultiPanel(multi_panel_detector, test.beam)
# update the generated reflections
test.generate_reflections()
# Predict the reflections in place and put in a reflection manager
ref_predictor = StillsExperimentsPredictor(test.stills_experiments)
ref_predictor(test.reflections)
test.refman = ReflectionManagerFactory.from_parameters_reflections_experiments(
refman_phil_scope.extract(),
test.reflections,
test.stills_experiments,
do_stills=True,
)
test.refman.finalise()
# Build a prediction parameterisation for the stills experiment
test.pred_param = StillsPredictionParameterisation(
test.stills_experiments,
detector_parameterisations=[test.det_param],
beam_parameterisations=[test.s0_param],
xl_orientation_parameterisations=[test.xlo_param],
xl_unit_cell_parameterisations=[test.xluc_param],
)
# A non-hierarchical detector does not have panel groups, thus panels are
# not treated independently wrt which reflections affect their parameters.
# As before, setting 792 reflections as the minimum should leave all
# parameters free, and should not remove any reflections
options = ar_phil_scope.extract()
options.min_nref_per_parameter = 792
ar = AutoReduce(options, pred_param=test.pred_param, reflection_manager=test.refman)
ar.check_and_remove()
det_params = test.pred_param.get_detector_parameterisations()
beam_params = test.pred_param.get_beam_parameterisations()
xl_ori_params = test.pred_param.get_crystal_orientation_parameterisations()
xl_uc_params = test.pred_param.get_crystal_unit_cell_parameterisations()
assert det_params[0].num_free() == 6
assert beam_params[0].num_free() == 3
assert xl_ori_params[0].num_free() == 3
assert xl_uc_params[0].num_free() == 6
assert len(test.refman.get_obs()) == 823
# Setting 793 reflections as the minimum fixes 3 unit cell parameters,
# and removes all those reflections. There are then too few reflections
# for any parameterisation and all will be fixed, leaving no free
# parameters for refinement. This fails within PredictionParameterisation,
# during update so the final 31 reflections are not removed.
options = ar_phil_scope.extract()
options.min_nref_per_parameter = 793
ar = AutoReduce(options, pred_param=test.pred_param, reflection_manager=test.refman)
with pytest.raises(
DialsRefineConfigError, match="There are no free parameters for refinement"
):
ar.check_and_remove()
det_params = test.pred_param.get_detector_parameterisations()
beam_params = test.pred_param.get_beam_parameterisations()
xl_ori_params = test.pred_param.get_crystal_orientation_parameterisations()
xl_uc_params = test.pred_param.get_crystal_unit_cell_parameterisations()
assert det_params[0].num_free() == 0
assert beam_params[0].num_free() == 0
assert xl_ori_params[0].num_free() == 0
assert xl_uc_params[0].num_free() == 0
assert len(test.refman.get_obs()) == 823 - 792
def test(args=[]):
# Python and cctbx imports
from math import pi
from scitbx import matrix
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
# Import for surgery on reflection_tables
from dials.array_family import flex
# Get module to build models using PHIL
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Crystal parameterisations
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator
from dials.algorithms.refinement.prediction.managed_predictors import (
ScansRayPredictor,
StillsExperimentsPredictor,
)
from dials.algorithms.spot_prediction import ray_intersection
from cctbx.sgtbx import space_group, space_group_symbols
#############################
# Setup experimental models #
#############################
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
# build models, with a larger crystal than default in order to get enough
# reflections on the 'still' image
param = """
geometry.parameters.crystal.a.length.range=40 50;
geometry.parameters.crystal.b.length.range=40 50;
geometry.parameters.crystal.c.length.range=40 50;
geometry.parameters.random_seed = 42"""
models = setup_geometry.Extract(master_phil,
cmdline_args=args,
local_overrides=param)
crystal = models.crystal
mydetector = models.detector
mygonio = models.goniometer
mybeam = models.beam
# Build a mock scan for a 1.5 degree wedge. Only used for generating indices near
# the Ewald sphere
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1),
exposure_times=0.1,
oscillation=(0, 1.5),
epochs=list(range(1)),
deg=True,
)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert approx_equal(im_width, 1.5 * pi / 180.0)
# Build experiment lists
stills_experiments = ExperimentList()
stills_experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
crystal=crystal,
imageset=None))
scans_experiments = ExperimentList()
scans_experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
crystal=crystal,
goniometer=mygonio,
scan=myscan,
imageset=None,
))
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
################################
# Apply known parameter shifts #
################################
# rotate crystal (=5 mrad each rotation)
xlo_p_vals = []
p_vals = xlo_param.get_param_vals()
xlo_p_vals.append(p_vals)
new_p_vals = [a + b for a, b in zip(p_vals, [5.0, 5.0, 5.0])]
xlo_param.set_param_vals(new_p_vals)
# change unit cell (=1.0 Angstrom length upsets, 0.5 degree of
# gamma angle)
xluc_p_vals = []
p_vals = xluc_param.get_param_vals()
xluc_p_vals.append(p_vals)
cell_params = crystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [1.0, 1.0, -1.0, 0.0, 0.0, 0.5])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(crystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
# keep track of the target crystal model to compare with refined
from copy import deepcopy
target_crystal = deepcopy(crystal)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere for crystal
resolution = 2.0
index_generator = IndexGenerator(
crystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# Build a ray predictor and predict rays close to the Ewald sphere by using
# the narrow rotation scan
ref_predictor = ScansRayPredictor(scans_experiments, sweep_range)
obs_refs = ref_predictor(indices, experiment_id=0)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Add in flags and ID columns by copying into standard reflection table
tmp = flex.reflection_table.empty_standard(len(obs_refs))
tmp.update(obs_refs)
obs_refs = tmp
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2)
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# Re-predict using the stills reflection predictor
stills_ref_predictor = StillsExperimentsPredictor(stills_experiments)
obs_refs_stills = stills_ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs_stills["xyzobs.mm.value"] = obs_refs_stills["xyzcal.mm"]
###############################
# Undo known parameter shifts #
###############################
xlo_param.set_param_vals(xlo_p_vals[0])
xluc_param.set_param_vals(xluc_p_vals[0])
# make a refiner
from dials.algorithms.refinement.refiner import phil_scope
params = phil_scope.fetch(source=parse("")).extract()
# Change this to get a plot
do_plot = False
if do_plot:
params.refinement.refinery.journal.track_parameter_correlation = True
from dials.algorithms.refinement.refiner import RefinerFactory
# decrease bin_size_fraction to terminate on RMSD convergence
params.refinement.target.bin_size_fraction = 0.01
params.refinement.parameterisation.beam.fix = "all"
params.refinement.parameterisation.detector.fix = "all"
refiner = RefinerFactory.from_parameters_data_experiments(
params, obs_refs_stills, stills_experiments)
# run refinement
history = refiner.run()
# regression tests
assert len(history["rmsd"]) == 9
refined_crystal = refiner.get_experiments()[0].crystal
uc1 = refined_crystal.get_unit_cell()
uc2 = target_crystal.get_unit_cell()
assert uc1.is_similar_to(uc2)
if do_plot:
plt = refiner.parameter_correlation_plot(
len(history["parameter_correlation"]) - 1)
plt.show()
def test_check_and_remove():
test = _Test()
# Override the single panel model and parameterisation. This test function
# exercises the code for non-hierarchical multi-panel detectors. The
# hierarchical detector version is tested via test_cspad_refinement.py
from dxtbx.model import Detector
from dials.algorithms.refinement.parameterisation.detector_parameters import (
DetectorParameterisationMultiPanel, )
from dials.test.algorithms.refinement.test_multi_panel_detector_parameterisation import (
make_panel_in_array, )
multi_panel_detector = Detector()
for x in range(3):
for y in range(3):
new_panel = make_panel_in_array((x, y), test.detector[0])
multi_panel_detector.add_panel(new_panel)
test.detector = multi_panel_detector
test.stills_experiments[0].detector = multi_panel_detector
test.det_param = DetectorParameterisationMultiPanel(
multi_panel_detector, test.beam)
# update the generated reflections
test.generate_reflections()
# Predict the reflections in place and put in a reflection manager
ref_predictor = StillsExperimentsPredictor(test.stills_experiments)
ref_predictor(test.reflections)
test.refman = ReflectionManagerFactory.from_parameters_reflections_experiments(
refman_phil_scope.extract(),
test.reflections,
test.stills_experiments,
do_stills=True,
)
test.refman.finalise()
# A non-hierarchical detector does not have panel groups, thus panels are
# not treated independently wrt which reflections affect their parameters.
# As before, setting 137 reflections as the minimum should leave all
# parameters free, and should not remove any reflections
options = ar_phil_scope.extract()
options.min_nref_per_parameter = 137
ar = AutoReduce(
options,
[test.det_param],
[test.s0_param],
[test.xlo_param],
[test.xluc_param],
gon_params=[],
reflection_manager=test.refman,
)
ar.check_and_remove()
assert ar.det_params[0].num_free() == 6
assert ar.beam_params[0].num_free() == 3
assert ar.xl_ori_params[0].num_free() == 3
assert ar.xl_uc_params[0].num_free() == 6
assert len(ar.reflection_manager.get_obs()) == 823
# Setting reflections as the minimum should fix the detector parameters,
# which removes that parameterisation. Because all reflections are recorded
# on that detector, they will all be removed as well. This then affects all
# other parameterisations, which will be removed.
options = ar_phil_scope.extract()
options.min_nref_per_parameter = 138
ar = AutoReduce(
options,
[test.det_param],
[test.s0_param],
[test.xlo_param],
[test.xluc_param],
gon_params=[],
reflection_manager=test.refman,
)
ar.check_and_remove()
assert not ar.det_params
assert not ar.beam_params
assert not ar.xl_ori_params
assert not ar.xl_uc_params
assert len(ar.reflection_manager.get_obs()) == 0