def test_check_and_fix(tc):
n_det = tc.det_param.num_free()
n_beam = tc.s0_param.num_free()
n_xlo = tc.xlo_param.num_free()
n_xluc = tc.xluc_param.num_free()
# Similar to test_check_and_fail, setting 137 reflections as the minimum
# should leave all parameters free
options = ar_phil_scope.extract()
options.min_nref_per_parameter = 137
ar = AutoReduce(
options,
[tc.det_param],
[tc.s0_param],
[tc.xlo_param],
[tc.xluc_param],
gon_params=[],
reflection_manager=tc.refman,
)
ar.check_and_fix()
assert ar.det_params[0].num_free() == n_det == 6
assert ar.beam_params[0].num_free() == n_beam == 3
assert ar.xl_ori_params[0].num_free() == n_xlo == 3
assert ar.xl_uc_params[0].num_free() == n_xluc == 6
# Setting 138 reflections as the minimum should fix all the detector
# parameters and remove that parameterisation. The crystal unit cell also
# has 6 parameters, but each parameter is considered separately, so the
# critical minimum number of reflections is 138*1 not 138*6 in that case
options = ar_phil_scope.extract()
options.min_nref_per_parameter = 138
ar = AutoReduce(
options,
[tc.det_param],
[tc.s0_param],
[tc.xlo_param],
[tc.xluc_param],
gon_params=[],
reflection_manager=tc.refman,
)
ar.check_and_fix()
assert not ar.det_params
assert ar.xl_uc_params[0].num_free() == n_xluc
assert ar.beam_params[0].num_free() == n_beam
assert ar.xl_ori_params[0].num_free() == n_xlo
def test_check_and_fix(tc):
n_det = tc.det_param.num_free()
n_beam = tc.s0_param.num_free()
n_xlo = tc.xlo_param.num_free()
n_xluc = tc.xluc_param.num_free()
# Similar to test_check_and_fail, setting 792 reflections as the minimum
# should leave all parameters free
options = ar_phil_scope.extract()
options.min_nref_per_parameter = 792
ar = AutoReduce(options,
pred_param=tc.pred_param,
reflection_manager=tc.refman)
ar.check_and_fix()
det_params = tc.pred_param.get_detector_parameterisations()
beam_params = tc.pred_param.get_beam_parameterisations()
xl_ori_params = tc.pred_param.get_crystal_orientation_parameterisations()
xl_uc_params = tc.pred_param.get_crystal_unit_cell_parameterisations()
assert det_params[0].num_free() == n_det == 6
assert beam_params[0].num_free() == n_beam == 3
assert xl_ori_params[0].num_free() == n_xlo == 3
assert xl_uc_params[0].num_free() == n_xluc == 6
# Setting 793 reflections as the minimum should fix crystal unit cell
# parameters g_param_0, g_param_3 and g_param_4
options = ar_phil_scope.extract()
options.min_nref_per_parameter = 793
ar = AutoReduce(options,
pred_param=tc.pred_param,
reflection_manager=tc.refman)
ar.check_and_fix()
det_params = tc.pred_param.get_detector_parameterisations()
beam_params = tc.pred_param.get_beam_parameterisations()
xl_ori_params = tc.pred_param.get_crystal_orientation_parameterisations()
xl_uc_params = tc.pred_param.get_crystal_unit_cell_parameterisations()
assert det_params[0].num_free() == n_det
assert xl_uc_params[0].num_free() == 3
assert xl_uc_params[0].get_param_names() == [
"g_param_1", "g_param_2", "g_param_5"
]
assert beam_params[0].num_free() == n_beam
assert xl_ori_params[0].num_free() == n_xlo
def test_check_and_fail(tc):
# There are 823 reflections
assert len(tc.refman.get_matches()) == 823
# The parameters affecting the smallest number of reflections are
# g_param_0, g_param_3 and g_param_4, all of which have gradients for
# 792 reflections. Setting 792 reflections as the minimum should pass.
options = ar_phil_scope.extract()
options.min_nref_per_parameter = 792
ar = AutoReduce(options, pred_param=tc.pred_param, reflection_manager=tc.refman)
ar.check_and_fail()
# Setting 793 reflections as the minimum should fail
options.min_nref_per_parameter = 793
ar = AutoReduce(options, pred_param=tc.pred_param, reflection_manager=tc.refman)
with pytest.raises(DialsRefineConfigError):
ar.check_and_fail()
def test_check_and_fail(tc):
# There are 823 reflections and the detector parameterisation has 6 free
# parameters
assert len(tc.refman.get_matches()) == 823
assert tc.det_param.num_free() == 6
# Setting 137 reflections as the minimum should pass (137*6<823)
options = ar_phil_scope.extract()
options.min_nref_per_parameter = 137
ar = AutoReduce(
options,
[tc.det_param],
[tc.s0_param],
[tc.xlo_param],
[tc.xluc_param],
gon_params=[],
reflection_manager=tc.refman,
)
ar.check_and_fail()
# Setting 138 reflections as the minimum should fail (138*6>823)
options.min_nref_per_parameter = 138
ar = AutoReduce(
options,
[tc.det_param],
[tc.s0_param],
[tc.xlo_param],
[tc.xluc_param],
gon_params=[],
reflection_manager=tc.refman,
)
with pytest.raises(DialsRefineConfigError):
ar.check_and_fail()
def test_check_and_remove():
test = _Test()
# Override the single panel model and parameterisation. This test function
# exercises the code for non-hierarchical multi-panel detectors. The
# hierarchical detector version is tested via test_cspad_refinement.py
multi_panel_detector = Detector()
for x in range(3):
for y in range(3):
new_panel = make_panel_in_array((x, y), test.detector[0])
multi_panel_detector.add_panel(new_panel)
test.detector = multi_panel_detector
test.stills_experiments[0].detector = multi_panel_detector
test.det_param = DetectorParameterisationMultiPanel(multi_panel_detector, test.beam)
# update the generated reflections
test.generate_reflections()
# Predict the reflections in place and put in a reflection manager
ref_predictor = StillsExperimentsPredictor(test.stills_experiments)
ref_predictor(test.reflections)
test.refman = ReflectionManagerFactory.from_parameters_reflections_experiments(
refman_phil_scope.extract(),
test.reflections,
test.stills_experiments,
do_stills=True,
)
test.refman.finalise()
# Build a prediction parameterisation for the stills experiment
test.pred_param = StillsPredictionParameterisation(
test.stills_experiments,
detector_parameterisations=[test.det_param],
beam_parameterisations=[test.s0_param],
xl_orientation_parameterisations=[test.xlo_param],
xl_unit_cell_parameterisations=[test.xluc_param],
)
# A non-hierarchical detector does not have panel groups, thus panels are
# not treated independently wrt which reflections affect their parameters.
# As before, setting 792 reflections as the minimum should leave all
# parameters free, and should not remove any reflections
options = ar_phil_scope.extract()
options.min_nref_per_parameter = 792
ar = AutoReduce(options, pred_param=test.pred_param, reflection_manager=test.refman)
ar.check_and_remove()
det_params = test.pred_param.get_detector_parameterisations()
beam_params = test.pred_param.get_beam_parameterisations()
xl_ori_params = test.pred_param.get_crystal_orientation_parameterisations()
xl_uc_params = test.pred_param.get_crystal_unit_cell_parameterisations()
assert det_params[0].num_free() == 6
assert beam_params[0].num_free() == 3
assert xl_ori_params[0].num_free() == 3
assert xl_uc_params[0].num_free() == 6
assert len(test.refman.get_obs()) == 823
# Setting 793 reflections as the minimum fixes 3 unit cell parameters,
# and removes all those reflections. There are then too few reflections
# for any parameterisation and all will be fixed, leaving no free
# parameters for refinement. This fails within PredictionParameterisation,
# during update so the final 31 reflections are not removed.
options = ar_phil_scope.extract()
options.min_nref_per_parameter = 793
ar = AutoReduce(options, pred_param=test.pred_param, reflection_manager=test.refman)
with pytest.raises(
DialsRefineConfigError, match="There are no free parameters for refinement"
):
ar.check_and_remove()
det_params = test.pred_param.get_detector_parameterisations()
beam_params = test.pred_param.get_beam_parameterisations()
xl_ori_params = test.pred_param.get_crystal_orientation_parameterisations()
xl_uc_params = test.pred_param.get_crystal_unit_cell_parameterisations()
assert det_params[0].num_free() == 0
assert beam_params[0].num_free() == 0
assert xl_ori_params[0].num_free() == 0
assert xl_uc_params[0].num_free() == 0
assert len(test.refman.get_obs()) == 823 - 792
def _build_components(cls, params, reflections, experiments):
"""low level build"""
# Currently a refinement job can only have one parameterisation of the
# prediction equation. This can either be of the XYDelPsi (stills) type, the
# XYPhi (scans) type or the scan-varying XYPhi type with a varying crystal
# model
single_as_still = params.refinement.parameterisation.treat_single_image_as_still
exps_are_stills = []
for exp in experiments:
if exp.scan is None:
exps_are_stills.append(True)
elif exp.scan.get_num_images() == 1:
if single_as_still:
exps_are_stills.append(True)
elif exp.scan.is_still():
exps_are_stills.append(True)
else:
exps_are_stills.append(False)
else:
if exp.scan.get_oscillation()[1] <= 0.0:
raise DialsRefineConfigError(
"Cannot refine a zero-width scan")
exps_are_stills.append(False)
# check experiment types are consistent
if not all(exps_are_stills[0] == e for e in exps_are_stills):
raise DialsRefineConfigError(
"Cannot refine a mixture of stills and scans")
do_stills = exps_are_stills[0]
# If experiments are stills, ensure scan-varying refinement won't be attempted
if do_stills:
params.refinement.parameterisation.scan_varying = False
# Refiner does not accept scan_varying=Auto. This is a special case for
# doing macrocycles of refinement in dials.refine.
if params.refinement.parameterisation.scan_varying is libtbx.Auto:
params.refinement.parameterisation.scan_varying = False
# Trim scans and calculate reflection block_width if required for scan-varying refinement
if (params.refinement.parameterisation.scan_varying and
params.refinement.parameterisation.trim_scan_to_observations):
experiments = _trim_scans_to_observations(experiments, reflections)
from dials.algorithms.refinement.reflection_manager import BlockCalculator
block_calculator = BlockCalculator(experiments, reflections)
if params.refinement.parameterisation.compose_model_per == "block":
reflections = block_calculator.per_width(
params.refinement.parameterisation.block_width, deg=True)
elif params.refinement.parameterisation.compose_model_per == "image":
reflections = block_calculator.per_image()
logger.debug("\nBuilding reflection manager")
logger.debug("Input reflection list size = %d observations",
len(reflections))
# create reflection manager
refman = ReflectionManagerFactory.from_parameters_reflections_experiments(
params.refinement.reflections, reflections, experiments, do_stills)
logger.debug(
"Number of observations that pass initial inclusion criteria = %d",
refman.get_accepted_refs_size(),
)
sample_size = refman.get_sample_size()
if sample_size > 0:
logger.debug("Working set size = %d observations", sample_size)
logger.debug("Reflection manager built\n")
# configure use of sparse data types
params = cls.config_sparse(params, experiments)
do_sparse = params.refinement.parameterisation.sparse
# create managed reflection predictor
ref_predictor = ExperimentsPredictorFactory.from_experiments(
experiments,
force_stills=do_stills,
spherical_relp=params.refinement.parameterisation.
spherical_relp_model,
)
# Predict for the managed observations, set columns for residuals and set
# the used_in_refinement flag to the predictions
obs = refman.get_obs()
ref_predictor(obs)
x_obs, y_obs, phi_obs = obs["xyzobs.mm.value"].parts()
x_calc, y_calc, phi_calc = obs["xyzcal.mm"].parts()
obs["x_resid"] = x_calc - x_obs
obs["y_resid"] = y_calc - y_obs
obs["phi_resid"] = phi_calc - phi_obs
# determine whether to do basic centroid analysis to automatically
# determine outlier rejection block
if params.refinement.reflections.outlier.block_width is libtbx.Auto:
ca = refman.get_centroid_analyser()
analysis = ca(calc_average_residuals=False,
calc_periodograms=False)
else:
analysis = None
# Now predictions and centroid analysis are available, so we can finalise
# the reflection manager
refman.finalise(analysis)
# Create model parameterisations
logger.debug("Building prediction equation parameterisation")
pred_param = build_prediction_parameterisation(
params.refinement.parameterisation, experiments, refman, do_stills)
# Build a constraints manager, if requested
cmf = ConstraintManagerFactory(params, pred_param)
constraints_manager = cmf()
# Test for parameters that have too little data to refine and act accordingly
autoreduce = AutoReduce(
params.refinement.parameterisation.auto_reduction,
pred_param,
refman,
constraints_manager,
cmf,
)
autoreduce()
# if reduction was done, constraints_manager will have changed
constraints_manager = autoreduce.constraints_manager
# Build a restraints parameterisation (if requested).
# Only unit cell restraints are supported at the moment.
restraints_parameterisation = cls.config_restraints(
params.refinement.parameterisation, pred_param)
# Parameter reporting
logger.debug("Prediction equation parameterisation built")
logger.debug("Parameter order : name mapping")
for i, e in enumerate(pred_param.get_param_names()):
logger.debug("Parameter %03d : %s", i + 1, e)
param_reporter = ParameterReporter(
pred_param.get_detector_parameterisations(),
pred_param.get_beam_parameterisations(),
pred_param.get_crystal_orientation_parameterisations(),
pred_param.get_crystal_unit_cell_parameterisations(),
pred_param.get_goniometer_parameterisations(),
)
# Create target function
logger.debug("Building target function")
target = cls.config_target(
params.refinement.target,
experiments,
refman,
ref_predictor,
pred_param,
restraints_parameterisation,
do_stills,
do_sparse,
)
logger.debug("Target function built")
# create refinery
logger.debug("Building refinement engine")
refinery = cls.config_refinery(params, target, pred_param,
constraints_manager)
logger.debug("Refinement engine built")
nparam = len(pred_param)
ndim = target.dim
nref = len(refman.get_matches())
logger.info(
"There are %s parameters to refine against %s reflections in %s dimensions",
nparam,
nref,
ndim,
)
if not params.refinement.parameterisation.sparse and isinstance(
refinery, AdaptLstbx):
dense_jacobian_gigabytes = (nparam * nref * ndim *
flex.double.element_size()) / 1e9
avail_memory_gigabytes = psutil.virtual_memory().available / 1e9
# Report if the Jacobian requires a large amount of storage
if (dense_jacobian_gigabytes > 0.2 * avail_memory_gigabytes
or dense_jacobian_gigabytes > 0.5):
logger.info(
"Storage of the Jacobian matrix requires %.1f GB",
dense_jacobian_gigabytes,
)
# build refiner interface and return
if params.refinement.parameterisation.scan_varying:
refiner = ScanVaryingRefiner
else:
refiner = Refiner
return refiner(experiments, pred_param, param_reporter, refman, target,
refinery)
def test_check_and_remove():
test = _Test()
# Override the single panel model and parameterisation. This test function
# exercises the code for non-hierarchical multi-panel detectors. The
# hierarchical detector version is tested via test_cspad_refinement.py
from dxtbx.model import Detector
from dials.algorithms.refinement.parameterisation.detector_parameters import (
DetectorParameterisationMultiPanel, )
from dials.test.algorithms.refinement.test_multi_panel_detector_parameterisation import (
make_panel_in_array, )
multi_panel_detector = Detector()
for x in range(3):
for y in range(3):
new_panel = make_panel_in_array((x, y), test.detector[0])
multi_panel_detector.add_panel(new_panel)
test.detector = multi_panel_detector
test.stills_experiments[0].detector = multi_panel_detector
test.det_param = DetectorParameterisationMultiPanel(
multi_panel_detector, test.beam)
# update the generated reflections
test.generate_reflections()
# Predict the reflections in place and put in a reflection manager
ref_predictor = StillsExperimentsPredictor(test.stills_experiments)
ref_predictor(test.reflections)
test.refman = ReflectionManagerFactory.from_parameters_reflections_experiments(
refman_phil_scope.extract(),
test.reflections,
test.stills_experiments,
do_stills=True,
)
test.refman.finalise()
# A non-hierarchical detector does not have panel groups, thus panels are
# not treated independently wrt which reflections affect their parameters.
# As before, setting 137 reflections as the minimum should leave all
# parameters free, and should not remove any reflections
options = ar_phil_scope.extract()
options.min_nref_per_parameter = 137
ar = AutoReduce(
options,
[test.det_param],
[test.s0_param],
[test.xlo_param],
[test.xluc_param],
gon_params=[],
reflection_manager=test.refman,
)
ar.check_and_remove()
assert ar.det_params[0].num_free() == 6
assert ar.beam_params[0].num_free() == 3
assert ar.xl_ori_params[0].num_free() == 3
assert ar.xl_uc_params[0].num_free() == 6
assert len(ar.reflection_manager.get_obs()) == 823
# Setting reflections as the minimum should fix the detector parameters,
# which removes that parameterisation. Because all reflections are recorded
# on that detector, they will all be removed as well. This then affects all
# other parameterisations, which will be removed.
options = ar_phil_scope.extract()
options.min_nref_per_parameter = 138
ar = AutoReduce(
options,
[test.det_param],
[test.s0_param],
[test.xlo_param],
[test.xluc_param],
gon_params=[],
reflection_manager=test.refman,
)
ar.check_and_remove()
assert not ar.det_params
assert not ar.beam_params
assert not ar.xl_ori_params
assert not ar.xl_uc_params
assert len(ar.reflection_manager.get_obs()) == 0