def create_refiner(params, reflections, experiments): # Only parameterise the crystal unit cell det_params = None beam_params = None xlo_params = None xluc_params = [] for crystal in experiments.crystals(): exp_ids = experiments.indices(crystal) xluc_params.append( CrystalUnitCellParameterisation(crystal, experiment_ids=exp_ids) ) # Two theta prediction equation parameterisation pred_param = TwoThetaPredictionParameterisation( experiments, det_params, beam_params, xlo_params, xluc_params ) param_reporter = ParameterReporter( det_params, beam_params, xlo_params, xluc_params ) # ReflectionManager, currently without outlier rejection # Note: If not all reflections are used, then the filtering must be # communicated to generate_cif/mmcif() to be included in the CIF file! refman = TwoThetaReflectionManager( reflections, experiments, outlier_detector=None ) # Reflection predictor ref_predictor = TwoThetaExperimentsPredictor(experiments) # Two theta target target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param) # Switch on correlation matrix tracking if a correlation plot is requested journal = None if params.output.correlation_plot.filename is not None: journal = refinery_phil_scope.extract().refinery.journal journal.track_parameter_correlation = True # Minimisation engine - hardcoded to LevMar for now. refinery = Refinery( target=target, prediction_parameterisation=pred_param, log=None, tracking=journal, max_iterations=20, ) # Refiner refiner = Refiner( experiments=experiments, pred_param=pred_param, param_reporter=param_reporter, refman=refman, target=target, refinery=refinery, ) return refiner
def config_parameterisation(params, experiments, refman, do_stills=False): from dials.algorithms.refinement.parameterisation import ( build_prediction_parameterisation, ) pred_param = build_prediction_parameterisation(params, experiments, refman, do_stills) # Parameter reporting from dials.algorithms.refinement.parameterisation.parameter_report import ( ParameterReporter, ) param_reporter = ParameterReporter( pred_param.get_detector_parameterisations(), pred_param.get_beam_parameterisations(), pred_param.get_crystal_orientation_parameterisations(), pred_param.get_crystal_unit_cell_parameterisations(), pred_param.get_goniometer_parameterisations(), ) return pred_param, param_reporter
def test_refinement(dials_regression): """Test a refinement run""" # Get a beam and detector from a experiments. This one has a CS-PAD, but that # is irrelevant data_dir = os.path.join(dials_regression, "refinement_test_data", "hierarchy_test") experiments_path = os.path.join(data_dir, "datablock.json") assert os.path.exists(experiments_path) # load models from dxtbx.model.experiment_list import ExperimentListFactory experiments = ExperimentListFactory.from_serialized_format( experiments_path, check_format=False) im_set = experiments.imagesets()[0] detector = deepcopy(im_set.get_detector()) beam = im_set.get_beam() # Invent a crystal, goniometer and scan for this test from dxtbx.model import Crystal crystal = Crystal((40.0, 0.0, 0.0), (0.0, 40.0, 0.0), (0.0, 0.0, 40.0), space_group_symbol="P1") orig_xl = deepcopy(crystal) from dxtbx.model import GoniometerFactory goniometer = GoniometerFactory.known_axis((1.0, 0.0, 0.0)) # Build a mock scan for a 180 degree sequence from dxtbx.model import ScanFactory sf = ScanFactory() scan = sf.make_scan( image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(1800)), deg=True, ) sequence_range = scan.get_oscillation_range(deg=False) im_width = scan.get_oscillation(deg=False)[1] assert sequence_range == (0.0, pi) assert approx_equal(im_width, 0.1 * pi / 180.0) # Build an experiment list experiments = ExperimentList() experiments.append( Experiment( beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal, imageset=None, )) # simulate some reflections refs, _ = generate_reflections(experiments) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # alpha and beta angles) from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalUnitCellParameterisation, ) xluc_param = CrystalUnitCellParameterisation(crystal) cell_params = crystal.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1, -0.1, 0.0]) ] from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge from scitbx import matrix new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(crystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) # reparameterise the crystal at the perturbed geometry xluc_param = CrystalUnitCellParameterisation(crystal) # Dummy parameterisations for other models beam_param = None xlo_param = None det_param = None # parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.parameter_report import ( ParameterReporter, ) pred_param = TwoThetaPredictionParameterisation(experiments, det_param, beam_param, xlo_param, [xluc_param]) param_reporter = ParameterReporter(det_param, beam_param, xlo_param, [xluc_param]) # reflection manager refman = TwoThetaReflectionManager(refs, experiments, nref_per_degree=20) # reflection predictor ref_predictor = TwoThetaExperimentsPredictor(experiments) # target function target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param) # minimisation engine from dials.algorithms.refinement.engine import ( LevenbergMarquardtIterations as Refinery, ) refinery = Refinery( target=target, prediction_parameterisation=pred_param, log=None, max_iterations=20, ) # Refiner from dials.algorithms.refinement.refiner import Refiner refiner = Refiner( experiments=experiments, pred_param=pred_param, param_reporter=param_reporter, refman=refman, target=target, refinery=refinery, ) refiner.run() # compare crystal with original crystal refined_xl = refiner.get_experiments()[0].crystal # print refined_xl assert refined_xl.is_similar_to(orig_xl, uc_rel_length_tolerance=0.001, uc_abs_angle_tolerance=0.01)
def _build_components(cls, params, reflections, experiments): """low level build""" # Currently a refinement job can only have one parameterisation of the # prediction equation. This can either be of the XYDelPsi (stills) type, the # XYPhi (scans) type or the scan-varying XYPhi type with a varying crystal # model single_as_still = params.refinement.parameterisation.treat_single_image_as_still exps_are_stills = [] for exp in experiments: if exp.scan is None: exps_are_stills.append(True) elif exp.scan.get_num_images() == 1: if single_as_still: exps_are_stills.append(True) elif exp.scan.is_still(): exps_are_stills.append(True) else: exps_are_stills.append(False) else: if exp.scan.get_oscillation()[1] <= 0.0: raise DialsRefineConfigError( "Cannot refine a zero-width scan") exps_are_stills.append(False) # check experiment types are consistent if not all(exps_are_stills[0] == e for e in exps_are_stills): raise DialsRefineConfigError( "Cannot refine a mixture of stills and scans") do_stills = exps_are_stills[0] # If experiments are stills, ensure scan-varying refinement won't be attempted if do_stills: params.refinement.parameterisation.scan_varying = False # Refiner does not accept scan_varying=Auto. This is a special case for # doing macrocycles of refinement in dials.refine. if params.refinement.parameterisation.scan_varying is libtbx.Auto: params.refinement.parameterisation.scan_varying = False # Trim scans and calculate reflection block_width if required for scan-varying refinement if (params.refinement.parameterisation.scan_varying and params.refinement.parameterisation.trim_scan_to_observations): experiments = _trim_scans_to_observations(experiments, reflections) from dials.algorithms.refinement.reflection_manager import BlockCalculator block_calculator = BlockCalculator(experiments, reflections) if params.refinement.parameterisation.compose_model_per == "block": reflections = block_calculator.per_width( params.refinement.parameterisation.block_width, deg=True) elif params.refinement.parameterisation.compose_model_per == "image": reflections = block_calculator.per_image() logger.debug("\nBuilding reflection manager") logger.debug("Input reflection list size = %d observations", len(reflections)) # create reflection manager refman = ReflectionManagerFactory.from_parameters_reflections_experiments( params.refinement.reflections, reflections, experiments, do_stills) logger.debug( "Number of observations that pass initial inclusion criteria = %d", refman.get_accepted_refs_size(), ) sample_size = refman.get_sample_size() if sample_size > 0: logger.debug("Working set size = %d observations", sample_size) logger.debug("Reflection manager built\n") # configure use of sparse data types params = cls.config_sparse(params, experiments) do_sparse = params.refinement.parameterisation.sparse # create managed reflection predictor ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, force_stills=do_stills, spherical_relp=params.refinement.parameterisation. spherical_relp_model, ) # Predict for the managed observations, set columns for residuals and set # the used_in_refinement flag to the predictions obs = refman.get_obs() ref_predictor(obs) x_obs, y_obs, phi_obs = obs["xyzobs.mm.value"].parts() x_calc, y_calc, phi_calc = obs["xyzcal.mm"].parts() obs["x_resid"] = x_calc - x_obs obs["y_resid"] = y_calc - y_obs obs["phi_resid"] = phi_calc - phi_obs # determine whether to do basic centroid analysis to automatically # determine outlier rejection block if params.refinement.reflections.outlier.block_width is libtbx.Auto: ca = refman.get_centroid_analyser() analysis = ca(calc_average_residuals=False, calc_periodograms=False) else: analysis = None # Now predictions and centroid analysis are available, so we can finalise # the reflection manager refman.finalise(analysis) # Create model parameterisations logger.debug("Building prediction equation parameterisation") pred_param = build_prediction_parameterisation( params.refinement.parameterisation, experiments, refman, do_stills) # Build a constraints manager, if requested cmf = ConstraintManagerFactory(params, pred_param) constraints_manager = cmf() # Test for parameters that have too little data to refine and act accordingly autoreduce = AutoReduce( params.refinement.parameterisation.auto_reduction, pred_param, refman, constraints_manager, cmf, ) autoreduce() # if reduction was done, constraints_manager will have changed constraints_manager = autoreduce.constraints_manager # Build a restraints parameterisation (if requested). # Only unit cell restraints are supported at the moment. restraints_parameterisation = cls.config_restraints( params.refinement.parameterisation, pred_param) # Parameter reporting logger.debug("Prediction equation parameterisation built") logger.debug("Parameter order : name mapping") for i, e in enumerate(pred_param.get_param_names()): logger.debug("Parameter %03d : %s", i + 1, e) param_reporter = ParameterReporter( pred_param.get_detector_parameterisations(), pred_param.get_beam_parameterisations(), pred_param.get_crystal_orientation_parameterisations(), pred_param.get_crystal_unit_cell_parameterisations(), pred_param.get_goniometer_parameterisations(), ) # Create target function logger.debug("Building target function") target = cls.config_target( params.refinement.target, experiments, refman, ref_predictor, pred_param, restraints_parameterisation, do_stills, do_sparse, ) logger.debug("Target function built") # create refinery logger.debug("Building refinement engine") refinery = cls.config_refinery(params, target, pred_param, constraints_manager) logger.debug("Refinement engine built") nparam = len(pred_param) ndim = target.dim nref = len(refman.get_matches()) logger.info( "There are %s parameters to refine against %s reflections in %s dimensions", nparam, nref, ndim, ) if not params.refinement.parameterisation.sparse and isinstance( refinery, AdaptLstbx): dense_jacobian_gigabytes = (nparam * nref * ndim * flex.double.element_size()) / 1e9 avail_memory_gigabytes = psutil.virtual_memory().available / 1e9 # Report if the Jacobian requires a large amount of storage if (dense_jacobian_gigabytes > 0.2 * avail_memory_gigabytes or dense_jacobian_gigabytes > 0.5): logger.info( "Storage of the Jacobian matrix requires %.1f GB", dense_jacobian_gigabytes, ) # build refiner interface and return if params.refinement.parameterisation.scan_varying: refiner = ScanVaryingRefiner else: refiner = Refiner return refiner(experiments, pred_param, param_reporter, refman, target, refinery)
def test1(): dials_regression = libtbx.env.find_in_repositories( relative_path="dials_regression", test=os.path.isdir) # use a datablock that contains a CS-PAD detector description data_dir = os.path.join(dials_regression, "refinement_test_data", "hierarchy_test") datablock_path = os.path.join(data_dir, "datablock.json") assert os.path.exists(datablock_path) # load models from dxtbx.datablock import DataBlockFactory datablock = DataBlockFactory.from_serialized_format(datablock_path, check_format=False) im_set = datablock[0].extract_imagesets()[0] from copy import deepcopy detector = deepcopy(im_set.get_detector()) beam = im_set.get_beam() # we'll invent a crystal, goniometer and scan for this test from dxtbx.model import Crystal crystal = Crystal((40., 0., 0.), (0., 40., 0.), (0., 0., 40.), space_group_symbol="P1") from dxtbx.model import GoniometerFactory goniometer = GoniometerFactory.known_axis((1., 0., 0.)) # Build a mock scan for a 180 degree sweep from dxtbx.model import ScanFactory sf = ScanFactory() scan = sf.make_scan(image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=range(1800), deg=True) sweep_range = scan.get_oscillation_range(deg=False) im_width = scan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) from dxtbx.model.experiment_list import ExperimentList, Experiment # Build an experiment list experiments = ExperimentList() experiments.append( Experiment(beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal, imageset=None)) # simulate some reflections refs, ref_predictor = generate_reflections(experiments) # move the detector quadrants apart by 2mm both horizontally and vertically from dials.algorithms.refinement.parameterisation \ import DetectorParameterisationHierarchical det_param = DetectorParameterisationHierarchical(detector, level=1) det_p_vals = det_param.get_param_vals() p_vals = list(det_p_vals) p_vals[1] += 2 p_vals[2] -= 2 p_vals[7] += 2 p_vals[8] += 2 p_vals[13] -= 2 p_vals[14] += 2 p_vals[19] -= 2 p_vals[20] -= 2 det_param.set_param_vals(p_vals) # reparameterise the detector at the new perturbed geometry det_param = DetectorParameterisationHierarchical(detector, level=1) # parameterise other models from dials.algorithms.refinement.parameterisation.beam_parameters import \ BeamParameterisation from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalOrientationParameterisation, CrystalUnitCellParameterisation beam_param = BeamParameterisation(beam, goniometer) xlo_param = CrystalOrientationParameterisation(crystal) xluc_param = CrystalUnitCellParameterisation(crystal) # fix beam beam_param.set_fixed([True] * 3) # fix crystal xluc_param.set_fixed([True] * 6) xlo_param.set_fixed([True] * 3) # parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.prediction_parameters import \ XYPhiPredictionParameterisation from dials.algorithms.refinement.parameterisation.parameter_report import \ ParameterReporter pred_param = XYPhiPredictionParameterisation(experiments, [det_param], [beam_param], [xlo_param], [xluc_param]) param_reporter = ParameterReporter([det_param], [beam_param], [xlo_param], [xluc_param]) # reflection manager and target function from dials.algorithms.refinement.target import \ LeastSquaresPositionalResidualWithRmsdCutoff from dials.algorithms.refinement.reflection_manager import ReflectionManager refman = ReflectionManager(refs, experiments, nref_per_degree=20) # set a very tight rmsd target of 1/10000 of a pixel target = LeastSquaresPositionalResidualWithRmsdCutoff( experiments, ref_predictor, refman, pred_param, restraints_parameterisation=None, frac_binsize_cutoff=0.0001) # minimisation engine from dials.algorithms.refinement.engine \ import LevenbergMarquardtIterations as Refinery refinery = Refinery(target=target, prediction_parameterisation=pred_param, log=None, verbosity=0, max_iterations=20) # Refiner from dials.algorithms.refinement.refiner import Refiner refiner = Refiner(reflections=refs, experiments=experiments, pred_param=pred_param, param_reporter=param_reporter, refman=refman, target=target, refinery=refinery, verbosity=0) history = refiner.run() assert history.reason_for_termination == "RMSD target achieved" #compare detector with original detector orig_det = im_set.get_detector() refined_det = refiner.get_experiments()[0].detector from scitbx import matrix import math for op, rp in zip(orig_det, refined_det): # compare the origin vectors by... o1 = matrix.col(op.get_origin()) o2 = matrix.col(rp.get_origin()) # ...their relative lengths assert approx_equal(math.fabs(o1.length() - o2.length()) / o1.length(), 0, eps=1e-5) # ...the angle between them assert approx_equal(o1.accute_angle(o2), 0, eps=1e-5) print "OK" return
def create_refiner(params, reflections, experiments): from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalUnitCellParameterisation from dials.algorithms.refinement.parameterisation.parameter_report import \ ParameterReporter from dials.algorithms.refinement.two_theta_refiner import ( TwoThetaReflectionManager, TwoThetaTarget, TwoThetaExperimentsPredictor, TwoThetaPredictionParameterisation) verb = params.refinement.verbosity # Only parameterise the crystal unit cell det_params = None beam_params = None xlo_params = None xluc_params = [] for icrystal, crystal in enumerate(experiments.crystals()): exp_ids = experiments.indices(crystal) xluc_params.append( CrystalUnitCellParameterisation(crystal, experiment_ids=exp_ids)) # Two theta prediction equation parameterisation pred_param = TwoThetaPredictionParameterisation( experiments, det_params, beam_params, xlo_params, xluc_params) param_reporter = ParameterReporter(det_params, beam_params, xlo_params, xluc_params) # ReflectionManager, currently without outlier rejection # Note: If not all reflections are used, then the filtering must be # communicated to generate_cif/mmcif() to be included in the CIF file! refman = TwoThetaReflectionManager(reflections, experiments, outlier_detector=None, verbosity=verb) # Reflection predictor ref_predictor = TwoThetaExperimentsPredictor(experiments) # Two theta target target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param) # Do a correlation plot? tpc = params.output.correlation_plot.filename is not None # Minimisation engine - FIXME not many choices exposed yet. Do we want # more cowbell? from dials.algorithms.refinement.engine \ import LevenbergMarquardtIterations as Refinery refinery = Refinery(target=target, prediction_parameterisation=pred_param, log=None, verbosity=verb, track_step=False, track_gradient=False, track_parameter_correlation=tpc, max_iterations=20) # Refiner from dials.algorithms.refinement.refiner import Refiner refiner = Refiner(reflections=reflections, experiments=experiments, pred_param=pred_param, param_reporter=param_reporter, refman=refman, target=target, refinery=refinery, verbosity=verb) return refiner
def create_refiner(params, reflections, experiments): from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalUnitCellParameterisation from dials.algorithms.refinement.parameterisation.parameter_report import \ ParameterReporter from dials.algorithms.refinement.two_theta_refiner import \ TwoThetaReflectionManager, TwoThetaTarget, TwoThetaExperimentsPredictor, \ TwoThetaPredictionParameterisation verb = params.refinement.verbosity # Only parameterise the crystal unit cell det_params = None beam_params = None xlo_params = None xluc_params = [] for icrystal, crystal in enumerate(experiments.crystals()): exp_ids = experiments.indices(crystal) xluc_params.append( CrystalUnitCellParameterisation(crystal, experiment_ids=exp_ids)) # Two theta prediction equation parameterisation pred_param = TwoThetaPredictionParameterisation( experiments, det_params, beam_params, xlo_params, xluc_params) param_reporter = ParameterReporter(det_params, beam_params, xlo_params, xluc_params) # ReflectionManager, currently without outlier rejection # Note: If not all reflections are used, then the filtering must be # communicated to generate_cif/mmcif() to be included in the CIF file! refman = TwoThetaReflectionManager(reflections, experiments, outlier_detector=None, verbosity=verb) # Reflection predictor ref_predictor = TwoThetaExperimentsPredictor(experiments) # Two theta target target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param) # Switch on correlation matrix tracking if a correlation plot is requested journal = None if params.output.correlation_plot.filename is not None: from dials.algorithms.refinement.engine import refinery_phil_scope journal = refinery_phil_scope.extract().refinery.journal journal.track_parameter_correlation = True # Minimisation engine - hardcoded to LevMar for now. from dials.algorithms.refinement.engine \ import LevenbergMarquardtIterations as Refinery refinery = Refinery(target=target, prediction_parameterisation=pred_param, log=None, verbosity=verb, tracking=journal, max_iterations=20) # Refiner from dials.algorithms.refinement.refiner import Refiner refiner = Refiner(reflections=reflections, experiments=experiments, pred_param=pred_param, param_reporter=param_reporter, refman=refman, target=target, refinery=refinery, verbosity=verb) return refiner