def test_check_and_fix(tc): n_det = tc.det_param.num_free() n_beam = tc.s0_param.num_free() n_xlo = tc.xlo_param.num_free() n_xluc = tc.xluc_param.num_free() # Similar to test_check_and_fail, setting 137 reflections as the minimum # should leave all parameters free options = ar_phil_scope.extract() options.min_nref_per_parameter = 137 ar = AutoReduce( options, [tc.det_param], [tc.s0_param], [tc.xlo_param], [tc.xluc_param], gon_params=[], reflection_manager=tc.refman, ) ar.check_and_fix() assert ar.det_params[0].num_free() == n_det == 6 assert ar.beam_params[0].num_free() == n_beam == 3 assert ar.xl_ori_params[0].num_free() == n_xlo == 3 assert ar.xl_uc_params[0].num_free() == n_xluc == 6 # Setting 138 reflections as the minimum should fix all the detector # parameters and remove that parameterisation. The crystal unit cell also # has 6 parameters, but each parameter is considered separately, so the # critical minimum number of reflections is 138*1 not 138*6 in that case options = ar_phil_scope.extract() options.min_nref_per_parameter = 138 ar = AutoReduce( options, [tc.det_param], [tc.s0_param], [tc.xlo_param], [tc.xluc_param], gon_params=[], reflection_manager=tc.refman, ) ar.check_and_fix() assert not ar.det_params assert ar.xl_uc_params[0].num_free() == n_xluc assert ar.beam_params[0].num_free() == n_beam assert ar.xl_ori_params[0].num_free() == n_xlo
def test_check_and_fix(tc): n_det = tc.det_param.num_free() n_beam = tc.s0_param.num_free() n_xlo = tc.xlo_param.num_free() n_xluc = tc.xluc_param.num_free() # Similar to test_check_and_fail, setting 792 reflections as the minimum # should leave all parameters free options = ar_phil_scope.extract() options.min_nref_per_parameter = 792 ar = AutoReduce(options, pred_param=tc.pred_param, reflection_manager=tc.refman) ar.check_and_fix() det_params = tc.pred_param.get_detector_parameterisations() beam_params = tc.pred_param.get_beam_parameterisations() xl_ori_params = tc.pred_param.get_crystal_orientation_parameterisations() xl_uc_params = tc.pred_param.get_crystal_unit_cell_parameterisations() assert det_params[0].num_free() == n_det == 6 assert beam_params[0].num_free() == n_beam == 3 assert xl_ori_params[0].num_free() == n_xlo == 3 assert xl_uc_params[0].num_free() == n_xluc == 6 # Setting 793 reflections as the minimum should fix crystal unit cell # parameters g_param_0, g_param_3 and g_param_4 options = ar_phil_scope.extract() options.min_nref_per_parameter = 793 ar = AutoReduce(options, pred_param=tc.pred_param, reflection_manager=tc.refman) ar.check_and_fix() det_params = tc.pred_param.get_detector_parameterisations() beam_params = tc.pred_param.get_beam_parameterisations() xl_ori_params = tc.pred_param.get_crystal_orientation_parameterisations() xl_uc_params = tc.pred_param.get_crystal_unit_cell_parameterisations() assert det_params[0].num_free() == n_det assert xl_uc_params[0].num_free() == 3 assert xl_uc_params[0].get_param_names() == [ "g_param_1", "g_param_2", "g_param_5" ] assert beam_params[0].num_free() == n_beam assert xl_ori_params[0].num_free() == n_xlo
def test_check_and_fail(tc): # There are 823 reflections assert len(tc.refman.get_matches()) == 823 # The parameters affecting the smallest number of reflections are # g_param_0, g_param_3 and g_param_4, all of which have gradients for # 792 reflections. Setting 792 reflections as the minimum should pass. options = ar_phil_scope.extract() options.min_nref_per_parameter = 792 ar = AutoReduce(options, pred_param=tc.pred_param, reflection_manager=tc.refman) ar.check_and_fail() # Setting 793 reflections as the minimum should fail options.min_nref_per_parameter = 793 ar = AutoReduce(options, pred_param=tc.pred_param, reflection_manager=tc.refman) with pytest.raises(DialsRefineConfigError): ar.check_and_fail()
def test_check_and_fail(tc): # There are 823 reflections and the detector parameterisation has 6 free # parameters assert len(tc.refman.get_matches()) == 823 assert tc.det_param.num_free() == 6 # Setting 137 reflections as the minimum should pass (137*6<823) options = ar_phil_scope.extract() options.min_nref_per_parameter = 137 ar = AutoReduce( options, [tc.det_param], [tc.s0_param], [tc.xlo_param], [tc.xluc_param], gon_params=[], reflection_manager=tc.refman, ) ar.check_and_fail() # Setting 138 reflections as the minimum should fail (138*6>823) options.min_nref_per_parameter = 138 ar = AutoReduce( options, [tc.det_param], [tc.s0_param], [tc.xlo_param], [tc.xluc_param], gon_params=[], reflection_manager=tc.refman, ) with pytest.raises(DialsRefineConfigError): ar.check_and_fail()
def test_check_and_remove(): test = _Test() # Override the single panel model and parameterisation. This test function # exercises the code for non-hierarchical multi-panel detectors. The # hierarchical detector version is tested via test_cspad_refinement.py multi_panel_detector = Detector() for x in range(3): for y in range(3): new_panel = make_panel_in_array((x, y), test.detector[0]) multi_panel_detector.add_panel(new_panel) test.detector = multi_panel_detector test.stills_experiments[0].detector = multi_panel_detector test.det_param = DetectorParameterisationMultiPanel(multi_panel_detector, test.beam) # update the generated reflections test.generate_reflections() # Predict the reflections in place and put in a reflection manager ref_predictor = StillsExperimentsPredictor(test.stills_experiments) ref_predictor(test.reflections) test.refman = ReflectionManagerFactory.from_parameters_reflections_experiments( refman_phil_scope.extract(), test.reflections, test.stills_experiments, do_stills=True, ) test.refman.finalise() # Build a prediction parameterisation for the stills experiment test.pred_param = StillsPredictionParameterisation( test.stills_experiments, detector_parameterisations=[test.det_param], beam_parameterisations=[test.s0_param], xl_orientation_parameterisations=[test.xlo_param], xl_unit_cell_parameterisations=[test.xluc_param], ) # A non-hierarchical detector does not have panel groups, thus panels are # not treated independently wrt which reflections affect their parameters. # As before, setting 792 reflections as the minimum should leave all # parameters free, and should not remove any reflections options = ar_phil_scope.extract() options.min_nref_per_parameter = 792 ar = AutoReduce(options, pred_param=test.pred_param, reflection_manager=test.refman) ar.check_and_remove() det_params = test.pred_param.get_detector_parameterisations() beam_params = test.pred_param.get_beam_parameterisations() xl_ori_params = test.pred_param.get_crystal_orientation_parameterisations() xl_uc_params = test.pred_param.get_crystal_unit_cell_parameterisations() assert det_params[0].num_free() == 6 assert beam_params[0].num_free() == 3 assert xl_ori_params[0].num_free() == 3 assert xl_uc_params[0].num_free() == 6 assert len(test.refman.get_obs()) == 823 # Setting 793 reflections as the minimum fixes 3 unit cell parameters, # and removes all those reflections. There are then too few reflections # for any parameterisation and all will be fixed, leaving no free # parameters for refinement. This fails within PredictionParameterisation, # during update so the final 31 reflections are not removed. options = ar_phil_scope.extract() options.min_nref_per_parameter = 793 ar = AutoReduce(options, pred_param=test.pred_param, reflection_manager=test.refman) with pytest.raises( DialsRefineConfigError, match="There are no free parameters for refinement" ): ar.check_and_remove() det_params = test.pred_param.get_detector_parameterisations() beam_params = test.pred_param.get_beam_parameterisations() xl_ori_params = test.pred_param.get_crystal_orientation_parameterisations() xl_uc_params = test.pred_param.get_crystal_unit_cell_parameterisations() assert det_params[0].num_free() == 0 assert beam_params[0].num_free() == 0 assert xl_ori_params[0].num_free() == 0 assert xl_uc_params[0].num_free() == 0 assert len(test.refman.get_obs()) == 823 - 792
def _build_components(cls, params, reflections, experiments): """low level build""" # Currently a refinement job can only have one parameterisation of the # prediction equation. This can either be of the XYDelPsi (stills) type, the # XYPhi (scans) type or the scan-varying XYPhi type with a varying crystal # model single_as_still = params.refinement.parameterisation.treat_single_image_as_still exps_are_stills = [] for exp in experiments: if exp.scan is None: exps_are_stills.append(True) elif exp.scan.get_num_images() == 1: if single_as_still: exps_are_stills.append(True) elif exp.scan.is_still(): exps_are_stills.append(True) else: exps_are_stills.append(False) else: if exp.scan.get_oscillation()[1] <= 0.0: raise DialsRefineConfigError( "Cannot refine a zero-width scan") exps_are_stills.append(False) # check experiment types are consistent if not all(exps_are_stills[0] == e for e in exps_are_stills): raise DialsRefineConfigError( "Cannot refine a mixture of stills and scans") do_stills = exps_are_stills[0] # If experiments are stills, ensure scan-varying refinement won't be attempted if do_stills: params.refinement.parameterisation.scan_varying = False # Refiner does not accept scan_varying=Auto. This is a special case for # doing macrocycles of refinement in dials.refine. if params.refinement.parameterisation.scan_varying is libtbx.Auto: params.refinement.parameterisation.scan_varying = False # Trim scans and calculate reflection block_width if required for scan-varying refinement if (params.refinement.parameterisation.scan_varying and params.refinement.parameterisation.trim_scan_to_observations): experiments = _trim_scans_to_observations(experiments, reflections) from dials.algorithms.refinement.reflection_manager import BlockCalculator block_calculator = BlockCalculator(experiments, reflections) if params.refinement.parameterisation.compose_model_per == "block": reflections = block_calculator.per_width( params.refinement.parameterisation.block_width, deg=True) elif params.refinement.parameterisation.compose_model_per == "image": reflections = block_calculator.per_image() logger.debug("\nBuilding reflection manager") logger.debug("Input reflection list size = %d observations", len(reflections)) # create reflection manager refman = ReflectionManagerFactory.from_parameters_reflections_experiments( params.refinement.reflections, reflections, experiments, do_stills) logger.debug( "Number of observations that pass initial inclusion criteria = %d", refman.get_accepted_refs_size(), ) sample_size = refman.get_sample_size() if sample_size > 0: logger.debug("Working set size = %d observations", sample_size) logger.debug("Reflection manager built\n") # configure use of sparse data types params = cls.config_sparse(params, experiments) do_sparse = params.refinement.parameterisation.sparse # create managed reflection predictor ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, force_stills=do_stills, spherical_relp=params.refinement.parameterisation. spherical_relp_model, ) # Predict for the managed observations, set columns for residuals and set # the used_in_refinement flag to the predictions obs = refman.get_obs() ref_predictor(obs) x_obs, y_obs, phi_obs = obs["xyzobs.mm.value"].parts() x_calc, y_calc, phi_calc = obs["xyzcal.mm"].parts() obs["x_resid"] = x_calc - x_obs obs["y_resid"] = y_calc - y_obs obs["phi_resid"] = phi_calc - phi_obs # determine whether to do basic centroid analysis to automatically # determine outlier rejection block if params.refinement.reflections.outlier.block_width is libtbx.Auto: ca = refman.get_centroid_analyser() analysis = ca(calc_average_residuals=False, calc_periodograms=False) else: analysis = None # Now predictions and centroid analysis are available, so we can finalise # the reflection manager refman.finalise(analysis) # Create model parameterisations logger.debug("Building prediction equation parameterisation") pred_param = build_prediction_parameterisation( params.refinement.parameterisation, experiments, refman, do_stills) # Build a constraints manager, if requested cmf = ConstraintManagerFactory(params, pred_param) constraints_manager = cmf() # Test for parameters that have too little data to refine and act accordingly autoreduce = AutoReduce( params.refinement.parameterisation.auto_reduction, pred_param, refman, constraints_manager, cmf, ) autoreduce() # if reduction was done, constraints_manager will have changed constraints_manager = autoreduce.constraints_manager # Build a restraints parameterisation (if requested). # Only unit cell restraints are supported at the moment. restraints_parameterisation = cls.config_restraints( params.refinement.parameterisation, pred_param) # Parameter reporting logger.debug("Prediction equation parameterisation built") logger.debug("Parameter order : name mapping") for i, e in enumerate(pred_param.get_param_names()): logger.debug("Parameter %03d : %s", i + 1, e) param_reporter = ParameterReporter( pred_param.get_detector_parameterisations(), pred_param.get_beam_parameterisations(), pred_param.get_crystal_orientation_parameterisations(), pred_param.get_crystal_unit_cell_parameterisations(), pred_param.get_goniometer_parameterisations(), ) # Create target function logger.debug("Building target function") target = cls.config_target( params.refinement.target, experiments, refman, ref_predictor, pred_param, restraints_parameterisation, do_stills, do_sparse, ) logger.debug("Target function built") # create refinery logger.debug("Building refinement engine") refinery = cls.config_refinery(params, target, pred_param, constraints_manager) logger.debug("Refinement engine built") nparam = len(pred_param) ndim = target.dim nref = len(refman.get_matches()) logger.info( "There are %s parameters to refine against %s reflections in %s dimensions", nparam, nref, ndim, ) if not params.refinement.parameterisation.sparse and isinstance( refinery, AdaptLstbx): dense_jacobian_gigabytes = (nparam * nref * ndim * flex.double.element_size()) / 1e9 avail_memory_gigabytes = psutil.virtual_memory().available / 1e9 # Report if the Jacobian requires a large amount of storage if (dense_jacobian_gigabytes > 0.2 * avail_memory_gigabytes or dense_jacobian_gigabytes > 0.5): logger.info( "Storage of the Jacobian matrix requires %.1f GB", dense_jacobian_gigabytes, ) # build refiner interface and return if params.refinement.parameterisation.scan_varying: refiner = ScanVaryingRefiner else: refiner = Refiner return refiner(experiments, pred_param, param_reporter, refman, target, refinery)
def test_check_and_remove(): test = _Test() # Override the single panel model and parameterisation. This test function # exercises the code for non-hierarchical multi-panel detectors. The # hierarchical detector version is tested via test_cspad_refinement.py from dxtbx.model import Detector from dials.algorithms.refinement.parameterisation.detector_parameters import ( DetectorParameterisationMultiPanel, ) from dials.test.algorithms.refinement.test_multi_panel_detector_parameterisation import ( make_panel_in_array, ) multi_panel_detector = Detector() for x in range(3): for y in range(3): new_panel = make_panel_in_array((x, y), test.detector[0]) multi_panel_detector.add_panel(new_panel) test.detector = multi_panel_detector test.stills_experiments[0].detector = multi_panel_detector test.det_param = DetectorParameterisationMultiPanel( multi_panel_detector, test.beam) # update the generated reflections test.generate_reflections() # Predict the reflections in place and put in a reflection manager ref_predictor = StillsExperimentsPredictor(test.stills_experiments) ref_predictor(test.reflections) test.refman = ReflectionManagerFactory.from_parameters_reflections_experiments( refman_phil_scope.extract(), test.reflections, test.stills_experiments, do_stills=True, ) test.refman.finalise() # A non-hierarchical detector does not have panel groups, thus panels are # not treated independently wrt which reflections affect their parameters. # As before, setting 137 reflections as the minimum should leave all # parameters free, and should not remove any reflections options = ar_phil_scope.extract() options.min_nref_per_parameter = 137 ar = AutoReduce( options, [test.det_param], [test.s0_param], [test.xlo_param], [test.xluc_param], gon_params=[], reflection_manager=test.refman, ) ar.check_and_remove() assert ar.det_params[0].num_free() == 6 assert ar.beam_params[0].num_free() == 3 assert ar.xl_ori_params[0].num_free() == 3 assert ar.xl_uc_params[0].num_free() == 6 assert len(ar.reflection_manager.get_obs()) == 823 # Setting reflections as the minimum should fix the detector parameters, # which removes that parameterisation. Because all reflections are recorded # on that detector, they will all be removed as well. This then affects all # other parameterisations, which will be removed. options = ar_phil_scope.extract() options.min_nref_per_parameter = 138 ar = AutoReduce( options, [test.det_param], [test.s0_param], [test.xlo_param], [test.xluc_param], gon_params=[], reflection_manager=test.refman, ) ar.check_and_remove() assert not ar.det_params assert not ar.beam_params assert not ar.xl_ori_params assert not ar.xl_uc_params assert len(ar.reflection_manager.get_obs()) == 0