def run(args): assert len(args) == 2 expts = load.experiment_list(args[0], check_format=False) refl = flex.reflection_table.from_file(args[1]) params = phil_scope.fetch(source=phil.parse("")).extract() params.refinement.verbosity = 12 # no output by default print("Starting refinement #1") refiner = RefinerFactory.from_parameters_data_experiments( params, refl, expts) refiner.run() print("Finished refinement #1") # configure logging logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") # we should now get logging output from refinement print("Starting refinement #2") refiner = RefinerFactory.from_parameters_data_experiments( params, refl, expts) refiner.run() print("Finished refinement #2") # switch off logging for dials.algorithms.refinement logging.getLogger("dials.algorithms.refinement").setLevel(logging.ERROR) print("Starting refinement #3") refiner = RefinerFactory.from_parameters_data_experiments( params, refl, expts) refiner.run() print("Finished refinement #3")
def test_multi_panel_parameterisations(dials_regression, detector_parameterisation_choice): data_dir = os.path.join(dials_regression, "refinement_test_data", "cspad_refinement") exp_file = os.path.join(data_dir, "cspad_refined_experiments_step6_level2_300.json") ref_file = os.path.join(data_dir, "cspad_reflections_step7_300.pickle") reflections = flex.reflection_table.from_file(ref_file) experiments = ExperimentListFactory.from_json_file(exp_file, check_format=False) # Set refinement parameters params = phil_scope.fetch(source=phil.parse("")).extract() params.refinement.parameterisation.detector.panels = ( detector_parameterisation_choice) # Construct refiner if detector_parameterisation_choice == "single": with pytest.raises(DialsRefineConfigError): # Cannot create a single panel parameterisation for a multi-panel detector RefinerFactory.from_parameters_data_experiments( params, reflections, experiments) else: refiner = RefinerFactory.from_parameters_data_experiments( params, reflections, experiments) assert refiner.experiment_type == "stills"
class DetectorRefiner(object): user_phil = parse(""" refinement{ parameterisation { beam.fix=all crystal.fix=all detector.hierarchy_level=1 sparse=True } target.gradient_calculation_blocksize=100000 reflections.outlier.algorithm=tukey refinery.engine=LBFGScurvs }""") from dials.algorithms.refinement.refiner import phil_scope as refinement_phil working_phil = refinement_phil.fetch(sources=[user_phil]) def __call__(self, experiments, reflections): self.working_phil.show() params = self.working_phil.extract() refiner = RefinerFactory.from_parameters_data_experiments(params, reflections, experiments, verbosity=1) refiner.run() experiments = refiner.get_experiments() print("Detector refinement finished") return experiments
def test_run(dials_regression): from dials.array_family import flex from dials.algorithms.refinement import RefinerFactory from dials.algorithms.refinement.refiner import phil_scope from dxtbx.model.experiment_list import ExperimentListFactory from libtbx import phil data_dir = os.path.join(dials_regression, "refinement_test_data", "dials-423") exp_file = os.path.join(data_dir, "experiments.json") ref_file = os.path.join(data_dir, "subset.pickle") reflections = flex.reflection_table.from_file(ref_file) experiments = ExperimentListFactory.from_json_file(exp_file, check_format=False) """Test that the detector remains similar after refiner construction""" params = phil_scope.fetch(source=phil.parse("")).extract() # disable outlier rejection for speed of refiner construction params.refinement.reflections.outlier.algorithm = "null" refiner = RefinerFactory.from_parameters_data_experiments( params, reflections, experiments) d1 = experiments[0].detector d2 = refiner.get_experiments()[0].detector assert d1.is_similar_to(d2)
def test_constrained_refinement(dials_regression, run_in_tmpdir): """Do constrained refinement, checking that a panel group with no data on it still moves with its partners in the constraint. See https://github.com/dials/dials/issues/990""" bevington = pytest.importorskip("scitbx.examples.bevington") if not hasattr(bevington, "non_linear_ls_eigen_wrapper"): pytest.skip("Skipping test as SparseLevMar engine not available") data_dir = os.path.join(dials_regression, "refinement_test_data", "xfel_metrology") # Load experiments and reflections refl = flex.reflection_table.from_file( os.path.join(data_dir, "benchmark_level2d.pickle") ) expt = ExperimentListFactory.from_json_file( os.path.join(data_dir, "benchmark_level2d.json") ) # There are zero reflections on some panels, so these will only move via constraints for i in [8, 10, 11, 26, 27, 40, 42, 43, 56, 58, 59]: assert (refl["panel"] == i).count(True) == 0 # Get parameters, combining refine.phil with constraints that enforce distances to move in lockstep with open(os.path.join(data_dir, "refine.phil")) as f: refine_phil = phil.parse(f.read()) constraint_phil = phil.parse( """ refinement { parameterisation { detector { fix_list=Tau2,Tau3 constraints { parameter=Dist } } } } """ ) params = refiner_phil_scope.fetch(sources=[refine_phil, constraint_phil]).extract() detector = expt.detectors()[0] initial_distances = [p.get_distance() for p in detector] # Set up a refiner refiner = RefinerFactory.from_parameters_data_experiments(params, refl, expt) refiner.run() detector = refiner.get_experiments().detectors()[0] final_distances = [p.get_distance() for p in detector] # The shifts between initial and final distances should all be equal dist_diff = [a - b for a, b in zip(final_distances, initial_distances)] for d in dist_diff[1:]: assert d == pytest.approx(dist_diff[0])
def test_restrained_refinement_with_fixed_parameterisations( dials_regression, run_in_tmpdir ): # Avoid a regression to https://github.com/dials/dials/issues/1142 by # testing that refinement succeeds when some parameterisations are fixed # by parameter auto reduction code, but restraints are requested for # those parameterisations. # The phil scope from dials.algorithms.refinement.refiner import phil_scope user_phil = phil.parse( """ refinement { parameterisation { auto_reduction { min_nref_per_parameter = 90 action = fail *fix remove } crystal { unit_cell { restraints { tie_to_target { values = 95 95 132 90 90 120 sigmas = 1 1 1 0 0 0 id = 0 1 2 3 4 5 6 7 8 9 } } } } } } """ ) working_phil = phil_scope.fetch(source=user_phil) working_params = working_phil.extract() # use the multi stills test data data_dir = os.path.join(dials_regression, "refinement_test_data", "multi_stills") experiments_path = os.path.join(data_dir, "combined_experiments.json") pickle_path = os.path.join(data_dir, "combined_reflections.pickle") experiments = ExperimentListFactory.from_json_file( experiments_path, check_format=False ) reflections = flex.reflection_table.from_file(pickle_path) refiner = RefinerFactory.from_parameters_data_experiments( working_params, reflections, experiments ) history = refiner.run() rmsd_limits = (0.2044, 0.2220, 0.0063) for a, b in zip(history["rmsd"][-1], rmsd_limits): assert a < b
def refine(self, experiments, reflections, isoform=None): from dials.algorithms.refinement.refiner import phil_scope from libtbx.phil import parse params = phil_scope.fetch(source=parse('')).extract() params.refinement.reflections.weighting_strategy.delpsi_constant = 100000. params.refinement.reflections.weighting_strategy.override = "stills" params.refinement.parameterisation.auto_reduction.action = "fix" #params.refinement.reflections.do_outlier_rejection=True #params.refinement.reflections.iqr_multiplier=0.5 #params.refinement.reflections.minimum_sample_size=50 #params.refinement.reflections.maximum_sample_size=50 #params.refinement.reflections.random_seed=1 if self.horizons_phil.integration.dials_refinement.strategy == "distance": params.refinement.parameterisation.beam.fix = "all" params.refinement.parameterisation.detector.fix_list = [ "Tau1" ] # fix detector rotz, allow distance to refine elif self.horizons_phil.integration.dials_refinement.strategy == "wavelength": params.refinement.parameterisation.beam.fix = "in_spindle_plane,out_spindle_plane" params.refinement.parameterisation.detector.fix_list = [ "Dist", "Tau1" ] # fix detector rotz and distance elif self.horizons_phil.integration.dials_refinement.strategy == "fix": params.refinement.parameterisation.beam.fix = "all" params.refinement.parameterisation.detector.fix_list = [ "Dist", "Tau1" ] # fix detector rotz and distance if isoform is not None: params.refinement.reflections.outlier.algorithm = "null" params.refinement.parameterisation.crystal.fix = "cell" from dials.algorithms.refinement.refiner import RefinerFactory refiner = RefinerFactory.from_parameters_data_experiments(params, reflections, experiments, verbosity=1) history = refiner.run() print(history.keys()) for item in history["rmsd"]: print("%5.2f %5.2f %8.5f" % (item[0], item[1], 180. * item[2] / math.pi)) #for item in history["parameter_vector"]: # print ["%8.5f"%a for a in item] print(refiner.selection_used_for_refinement().count(True), "spots used for refinement") print(refiner.get_experiments()[0].beam) print(refiner.get_experiments()[0].detector) print( "Distance:", -refiner.get_experiments()[0].detector[0].get_beam_centre_lab( refiner.get_experiments()[0].beam.get_s0())[2]) print(refiner.get_experiments()[0].crystal) return refiner
def test_ModelEvaluation(dials_regression, tmpdir): # thaumatin data_dir = os.path.join(dials_regression, "indexing_test_data", "i04_weak_data") pickle_path = os.path.join(data_dir, "full.pickle") sweep_path = os.path.join(data_dir, "experiments_import.json") input_reflections = flex.reflection_table.from_file(pickle_path) input_experiments = load.experiment_list(sweep_path, check_format=False) input_reflections = input_reflections.select( input_reflections["xyzobs.px.value"].parts()[2] < 100) input_reflections.centroid_px_to_mm(input_experiments[0].detector, scan=input_experiments[0].scan) input_reflections.map_centroids_to_reciprocal_space( input_experiments[0].detector, input_experiments[0].beam, goniometer=input_experiments[0].goniometer, ) input_reflections["imageset_id"] = flex.size_t(input_reflections.size(), 0) input_reflections["id"] = flex.int(input_reflections.size(), -1) refine_params = refine_phil.fetch().extract() evaluator = model_evaluation.ModelEvaluation( refinement_params=refine_params) experiments = copy.deepcopy(input_experiments) reflections = copy.deepcopy(input_reflections) experiments[0].crystal = Crystal.from_dict( dict([ ("__id__", "crystal"), ( "real_space_a", (20.007058080503633, 49.721143642677994, 16.636052132572846), ), ( "real_space_b", (-15.182202482876685, 24.93846318493148, -50.7116866438356), ), ( "real_space_c", (-135.23051191036296, 41.14539066294313, 55.41374425160883), ), ("space_group_hall_symbol", " P 1"), ])) assign_indices = AssignIndicesGlobal() assign_indices(reflections, experiments) result = evaluator.evaluate(experiments, reflections) assert result is not None assert result.n_indexed == 7313 assert result.fraction_indexed == pytest.approx(0.341155066244) assert result.rmsds == pytest.approx( (0.10215787570953785, 0.12954412140128563, 0.0010980583382102509))
def run(self): """Test that the detector remains similar after refiner construction""" from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=phil.parse('')).extract() # disable outlier rejection for speed of refiner construction params.refinement.reflections.outlier.algorithm = 'null' refiner = RefinerFactory.from_parameters_data_experiments( params, self._reflections, self._experiments) d1 = self._experiments[0].detector d2 = refiner.get_experiments()[0].detector assert d1.is_similar_to(d2) print "OK" return
class CrystalRefiners(object): user_phil = parse( """ refinement{ parameterisation { beam.fix=all detector.fix=all } reflections.outlier.algorithm=tukey refinery.engine=LBFGScurvs }""" ) from dials.algorithms.refinement.refiner import phil_scope as refinement_phil working_phil = refinement_phil.fetch(sources=[user_phil]) def __call__(self, experiments, reflections): self.working_phil.show() params = self.working_phil.extract() for iexp, exp in enumerate(experiments): print("Refining crystal", iexp) # reflection subset for a single experiment refs = reflections.select(reflections["id"] == iexp) refs["id"] = flex.size_t(len(refs), 0) # experiment list for a single experiment exps = ExperimentList() exps.append(exp) refiner = RefinerFactory.from_parameters_data_experiments( params, refs, exps, verbosity=1 ) # do refinement refiner.run() refined_exps = refiner.get_experiments() # replace this experiment with the refined one experiments[iexp] = refined_exps[0] return experiments
def refine(self, experiments, reflections, isoform=None): from dials.algorithms.refinement.refiner import phil_scope from libtbx.phil import parse params = phil_scope.fetch(source=parse('')).extract() params.refinement.reflections.weighting_strategy.delpsi_constant=100000. params.refinement.reflections.weighting_strategy.override="stills" params.refinement.parameterisation.auto_reduction.action="fix" #params.refinement.reflections.do_outlier_rejection=True #params.refinement.reflections.iqr_multiplier=0.5 #params.refinement.reflections.minimum_sample_size=50 #params.refinement.reflections.maximum_sample_size=50 #params.refinement.reflections.random_seed=1 if self.horizons_phil.integration.dials_refinement.strategy=="distance": params.refinement.parameterisation.beam.fix="all" params.refinement.parameterisation.detector.fix_list=["Tau1"] # fix detector rotz, allow distance to refine elif self.horizons_phil.integration.dials_refinement.strategy=="wavelength": params.refinement.parameterisation.beam.fix="in_spindle_plane,out_spindle_plane" params.refinement.parameterisation.detector.fix_list=["Dist","Tau1"] # fix detector rotz and distance if isoform is not None: params.refinement.reflections.outlier.algorithm="null" params.refinement.parameterisation.crystal.fix="cell" from dials.algorithms.refinement.refiner import RefinerFactory refiner = RefinerFactory.from_parameters_data_experiments(params, reflections, experiments, verbosity=1) history = refiner.run() print history.keys() for item in history["rmsd"]: print "%5.2f %5.2f %8.5f"%(item[0],item[1],180.*item[2]/math.pi) #for item in history["parameter_vector"]: # print ["%8.5f"%a for a in item] print refiner.selection_used_for_refinement().count(True),"spots used for refinement" print refiner.get_experiments()[0].beam print refiner.get_experiments()[0].detector print "Distance:", -refiner.get_experiments()[0].detector[0].get_beam_centre_lab(refiner.get_experiments()[0].beam.get_s0())[2] print refiner.get_experiments()[0].crystal return refiner
def test(args=[]): # Python and cctbx imports from math import pi from scitbx import matrix from scitbx.array_family import flex from libtbx.phil import parse from libtbx.test_utils import approx_equal # Get module to build models using PHIL import dials.test.algorithms.refinement.setup_geometry as setup_geometry # We will set up a mock scan and a mock experiment list from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import ExperimentList, Experiment # Model parameterisations from dials.algorithms.refinement.parameterisation.detector_parameters import \ DetectorParameterisationSinglePanel from dials.algorithms.refinement.parameterisation.beam_parameters import \ BeamParameterisation from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalOrientationParameterisation, CrystalUnitCellParameterisation # Symmetry constrained parameterisation for the unit cell from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import \ symmetrize_reduce_enlarge # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.refinement.prediction import ScansRayPredictor, \ ExperimentsPredictor from dials.algorithms.spot_prediction import ray_intersection from cctbx.sgtbx import space_group, space_group_symbols # Parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.prediction_parameters import \ XYPhiPredictionParameterisation # implicit import # Imports for the target function from dials.algorithms.refinement.target import \ LeastSquaresPositionalResidualWithRmsdCutoff # implicit import ############################# # Setup experimental models # ############################# master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True) models = setup_geometry.Extract( master_phil, cmdline_args=args, local_overrides="geometry.parameters.random_seed = 1") crystal1 = models.crystal models = setup_geometry.Extract( master_phil, cmdline_args=args, local_overrides="geometry.parameters.random_seed = 2") mydetector = models.detector mygonio = models.goniometer crystal2 = models.crystal mybeam = models.beam # Build a mock scan for a 180 degree sweep sf = ScanFactory() myscan = sf.make_scan(image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=range(1800), deg=True) sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) # Build an experiment list experiments = ExperimentList() experiments.append( Experiment(beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=crystal1, imageset=None)) experiments.append( Experiment(beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=crystal2, imageset=None)) assert len(experiments.detectors()) == 1 ########################################################## # Parameterise the models (only for perturbing geometry) # ########################################################## det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xl1o_param = CrystalOrientationParameterisation(crystal1) xl1uc_param = CrystalUnitCellParameterisation(crystal1) xl2o_param = CrystalOrientationParameterisation(crystal2) xl2uc_param = CrystalUnitCellParameterisation(crystal2) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) # Fix crystal parameters #xluc_param.set_fixed([True, True, True, True, True, True]) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ######################################################################## #pred_param = XYPhiPredictionParameterisation(experiments, # [det_param], [s0_param], [xlo_param], [xluc_param]) ################################ # Apply known parameter shifts # ################################ # shift detector by 1.0 mm each translation and 2 mrad each rotation det_p_vals = det_param.get_param_vals() p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2., 2., 2.])] det_param.set_param_vals(p_vals) # shift beam by 2 mrad in free axis s0_p_vals = s0_param.get_param_vals() p_vals = list(s0_p_vals) p_vals[0] += 2. s0_param.set_param_vals(p_vals) # rotate crystal a bit (=2 mrad each rotation) xlo_p_vals = [] for xlo in (xl1o_param, xl2o_param): p_vals = xlo.get_param_vals() xlo_p_vals.append(p_vals) new_p_vals = [a + b for a, b in zip(p_vals, [2., 2., 2.])] xlo.set_param_vals(new_p_vals) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # gamma angle) xluc_p_vals = [] for xluc, xl in ((xl1uc_param, crystal1), ((xl2uc_param, crystal2))): p_vals = xluc.get_param_vals() xluc_p_vals.append(p_vals) cell_params = xl.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(xl.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.e5 for e in S.forward_independent_parameters()]) xluc.set_param_vals(X) ############################# # Generate some reflections # ############################# #print "Reflections will be generated with the following geometry:" #print mybeam #print mydetector #print crystal1 #print crystal2 # All indices in a 2.0 Angstrom sphere for crystal1 resolution = 2.0 index_generator = IndexGenerator( crystal1.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices1 = index_generator.to_array() # All indices in a 2.0 Angstrom sphere for crystal2 resolution = 2.0 index_generator = IndexGenerator( crystal2.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices2 = index_generator.to_array() # Predict rays within the sweep range. Set experiment IDs ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs1 = ray_predictor(indices1, experiment_id=0) obs_refs1['id'] = flex.int(len(obs_refs1), 0) obs_refs2 = ray_predictor(indices1, experiment_id=1) obs_refs2['id'] = flex.int(len(obs_refs2), 1) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs1) obs_refs1 = obs_refs1.select(intersects) intersects = ray_intersection(mydetector, obs_refs2) obs_refs2 = obs_refs2.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs1 = ref_predictor(obs_refs1) obs_refs2 = ref_predictor(obs_refs2) # Set 'observed' centroids from the predicted ones obs_refs1['xyzobs.mm.value'] = obs_refs1['xyzcal.mm'] obs_refs2['xyzobs.mm.value'] = obs_refs2['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs1), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs1), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs1), (im_width / 2.)**2) obs_refs1['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) var_x = flex.double(len(obs_refs2), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs2), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs2), (im_width / 2.)**2) obs_refs2['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) #print "Total number of reflections excited for crystal1", len(obs_refs1) #print "Total number of reflections excited for crystal2", len(obs_refs2) # concatenate reflection lists obs_refs1.extend(obs_refs2) obs_refs = obs_refs1 ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xl1o_param.set_param_vals(xlo_p_vals[0]) xl2o_param.set_param_vals(xlo_p_vals[1]) xl1uc_param.set_param_vals(xluc_p_vals[0]) xl2uc_param.set_param_vals(xluc_p_vals[1]) #print "Initial values of parameters are" #msg = "Parameters: " + "%.5f " * len(pred_param) #print msg % tuple(pred_param.get_param_vals()) #print # make a refiner from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse('')).extract() # in case we want a plot params.refinement.refinery.journal.track_parameter_correlation = True # scan static first from dials.algorithms.refinement.refiner import RefinerFactory refiner = RefinerFactory.from_parameters_data_experiments(params, obs_refs, experiments, verbosity=0) history = refiner.run() # scan varying params.refinement.parameterisation.scan_varying = True refiner = RefinerFactory.from_parameters_data_experiments(params, obs_refs, experiments, verbosity=0) history = refiner.run()
def test4(): '''Test group restraint with multiple crystals, and a stills refiner''' if not libtbx.env.has_module("dials_regression"): print "Skipping test2 in " + __file__ + " as dials_regression not present" return # The phil scope from dials.algorithms.refinement.refiner import phil_scope user_phil = parse(''' refinement { parameterisation { crystal { unit_cell { restraints { tie_to_group { sigmas=1,0,2,0,0,0 apply_to_all=true } } } } } } ''') working_phil = phil_scope.fetch(source=user_phil) working_params = working_phil.extract() dials_regression = libtbx.env.find_in_repositories( relative_path="dials_regression", test=os.path.isdir) # use the multi stills test data data_dir = os.path.join(dials_regression, "refinement_test_data", "multi_stills") experiments_path = os.path.join(data_dir, "combined_experiments.json") pickle_path = os.path.join(data_dir, "combined_reflections.pickle") experiments = ExperimentListFactory.from_json_file(experiments_path, check_format=False) reflections = flex.reflection_table.from_pickle(pickle_path) refiner = RefinerFactory.from_parameters_data_experiments( working_params, reflections, experiments) # hack to extract the objects needed from the Refiner rp = refiner._target._restraints_parameterisation pred_param = refiner._pred_param # get analytical values and gradients vals, grads, weights = rp.get_residuals_gradients_and_weights() # get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.e-7] * len(p_vals) fd_grad = [] for i in range(len(deltas)): val = p_vals[i] p_vals[i] -= deltas[i] / 2. pred_param.set_param_vals(p_vals) rev_state, foo, bar = rp.get_residuals_gradients_and_weights() rev_state = flex.double(rev_state) p_vals[i] += deltas[i] pred_param.set_param_vals(p_vals) fwd_state, foo, bar = rp.get_residuals_gradients_and_weights() fwd_state = flex.double(fwd_state) p_vals[i] = val fd = (fwd_state - rev_state) / deltas[i] fd_grad.append(fd) # for comparison, fd_grad is a list of flex.doubles, each of which corresponds # to the gradients of the residuals wrt to a single parameter. pnames = pred_param.get_param_names() for i, (pname, fd) in enumerate(zip(pnames, fd_grad)): # extract dense column from the sparse matrix an = grads.col(i).as_dense_vector() #print pname #print list(an.round(6)) #print list(fd.round(6)) #print assert approx_equal(an, fd, eps=1e-5) print "OK" return
class CrystalRefiners(object): user_phil = parse(""" refinement{ parameterisation { beam.fix=all detector.fix=all } reflections.outlier.algorithm=tukey refinery.engine=LBFGScurvs }""") from dials.algorithms.refinement.refiner import phil_scope as refinement_phil working_phil = refinement_phil.fetch(sources=[user_phil]) def __call__(self, experiments, reflections): comm = MPI.COMM_WORLD rank = comm.Get_rank() params = self.working_phil.extract() if rank == 0: data = [] size = comm.Get_size() chunk_size = len(experiments) // size remainder = len(experiments) % size pointer = 0 self.working_phil.show() for i in xrange(size): if i < remainder: sel_range = xrange(pointer, pointer + chunk_size + 1) else: sel_range = xrange(pointer, pointer + chunk_size) sel = flex.bool(len(reflections)) for exp_id in sel_range: sel |= reflections["id"] == exp_id if i < remainder: data.append(( range(pointer, pointer + chunk_size + 1), experiments[pointer:pointer + chunk_size + 1], reflections.select(sel), )) pointer += 1 else: data.append(( range(pointer, pointer + chunk_size), experiments[pointer:pointer + chunk_size], reflections.select(sel), )) pointer += chunk_size else: data = None data = comm.scatter(data, root=0) for i, (iexp, exp) in enumerate(zip(data[0], data[1])): print("Refining crystal", iexp) # reflection subset for a single experiment refs = data[2].select(data[2]["id"] == iexp) refs["id"] = flex.size_t(len(refs), 0) # experiment list for a single experiment exps = ExperimentList() exps.append(exp) refiner = RefinerFactory.from_parameters_data_experiments( params, refs, exps, verbosity=1) # do refinement refiner.run() refined_exps = refiner.get_experiments() # replace this experiment with the refined one data[1][i] = refined_exps[0] data = comm.gather(data, root=0) if rank == 0: for chunk in data: for iexp, experiment in zip(chunk[0], chunk[1]): experiments[iexp] = experiment return experiments else: assert data == None
def test(args=[]): ############################# # Setup experimental models # ############################# master_phil = parse( """ include scope dials.tests.algorithms.refinement.geometry_phil include scope dials.tests.algorithms.refinement.minimiser_phil """, process_includes=True, ) models = setup_geometry.Extract( master_phil, cmdline_args=args, local_overrides="geometry.parameters.random_seed = 1", ) crystal1 = models.crystal models = setup_geometry.Extract( master_phil, cmdline_args=args, local_overrides="geometry.parameters.random_seed = 2", ) mydetector = models.detector mygonio = models.goniometer crystal2 = models.crystal mybeam = models.beam # Build a mock scan for an 18 degree sequence sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 180), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(180)), deg=True, ) sequence_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sequence_range == (0.0, pi / 10) assert approx_equal(im_width, 0.1 * pi / 180.0) # Build an experiment list experiments = ExperimentList() experiments.append( Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=crystal1, imageset=None, )) experiments.append( Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=crystal2, imageset=None, )) assert len(experiments.detectors()) == 1 ########################################################## # Parameterise the models (only for perturbing geometry) # ########################################################## det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xl1o_param = CrystalOrientationParameterisation(crystal1) xl1uc_param = CrystalUnitCellParameterisation(crystal1) xl2o_param = CrystalOrientationParameterisation(crystal2) xl2uc_param = CrystalUnitCellParameterisation(crystal2) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) ################################ # Apply known parameter shifts # ################################ # shift detector by 1.0 mm each translation and 2 mrad each rotation det_p_vals = det_param.get_param_vals() p_vals = [ a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0]) ] det_param.set_param_vals(p_vals) # shift beam by 2 mrad in free axis s0_p_vals = s0_param.get_param_vals() p_vals = list(s0_p_vals) p_vals[0] += 2.0 s0_param.set_param_vals(p_vals) # rotate crystal a bit (=2 mrad each rotation) xlo_p_vals = [] for xlo in (xl1o_param, xl2o_param): p_vals = xlo.get_param_vals() xlo_p_vals.append(p_vals) new_p_vals = [a + b for a, b in zip(p_vals, [2.0, 2.0, 2.0])] xlo.set_param_vals(new_p_vals) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # gamma angle) xluc_p_vals = [] for xluc, xl in ((xl1uc_param, crystal1), ((xl2uc_param, crystal2))): p_vals = xluc.get_param_vals() xluc_p_vals.append(p_vals) cell_params = xl.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(xl.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()]) xluc.set_param_vals(X) ############################# # Generate some reflections # ############################# # All indices in a 2.5 Angstrom sphere for crystal1 resolution = 2.5 index_generator = IndexGenerator( crystal1.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices1 = index_generator.to_array() # All indices in a 2.5 Angstrom sphere for crystal2 resolution = 2.5 index_generator = IndexGenerator( crystal2.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices2 = index_generator.to_array() # Predict rays within the sequence range. Set experiment IDs ray_predictor = ScansRayPredictor(experiments, sequence_range) obs_refs1 = ray_predictor(indices1, experiment_id=0) obs_refs1["id"] = flex.int(len(obs_refs1), 0) obs_refs2 = ray_predictor(indices2, experiment_id=1) obs_refs2["id"] = flex.int(len(obs_refs2), 1) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs1) obs_refs1 = obs_refs1.select(intersects) intersects = ray_intersection(mydetector, obs_refs2) obs_refs2 = obs_refs2.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ScansExperimentsPredictor(experiments) obs_refs1 = ref_predictor(obs_refs1) obs_refs2 = ref_predictor(obs_refs2) # Set 'observed' centroids from the predicted ones obs_refs1["xyzobs.mm.value"] = obs_refs1["xyzcal.mm"] obs_refs2["xyzobs.mm.value"] = obs_refs2["xyzcal.mm"] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 18.0 px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs1), (px_size[0] / 2.0)**2) var_y = flex.double(len(obs_refs1), (px_size[1] / 2.0)**2) var_phi = flex.double(len(obs_refs1), (im_width / 2.0)**2) obs_refs1["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) var_x = flex.double(len(obs_refs2), (px_size[0] / 2.0)**2) var_y = flex.double(len(obs_refs2), (px_size[1] / 2.0)**2) var_phi = flex.double(len(obs_refs2), (im_width / 2.0)**2) obs_refs2["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) # concatenate reflection lists obs_refs1.extend(obs_refs2) obs_refs = obs_refs1 ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xl1o_param.set_param_vals(xlo_p_vals[0]) xl2o_param.set_param_vals(xlo_p_vals[1]) xl1uc_param.set_param_vals(xluc_p_vals[0]) xl2uc_param.set_param_vals(xluc_p_vals[1]) # scan static first params = phil_scope.fetch(source=parse("")).extract() refiner = RefinerFactory.from_parameters_data_experiments( params, obs_refs, experiments) refiner.run() # scan varying params.refinement.parameterisation.scan_varying = True refiner = RefinerFactory.from_parameters_data_experiments( params, obs_refs, experiments) refiner.run() # Ensure all models have scan-varying state set # (https://github.com/dials/dials/issues/798) refined_experiments = refiner.get_experiments() sp = [xl.get_num_scan_points() for xl in refined_experiments.crystals()] assert sp.count(181) == 2
def test4(): '''Test group restraint with multiple crystals, and a stills refiner''' if not libtbx.env.has_module("dials_regression"): print "Skipping test2 in " + __file__ + " as dials_regression not present" return # The phil scope from dials.algorithms.refinement.refiner import phil_scope user_phil = parse(''' refinement { parameterisation { crystal { unit_cell { restraints { tie_to_group { sigmas=1,0,2,0,0,0 apply_to_all=true } } } } } } ''') working_phil = phil_scope.fetch(source=user_phil) working_params = working_phil.extract() dials_regression = libtbx.env.find_in_repositories( relative_path="dials_regression", test=os.path.isdir) # use the multi stills test data data_dir = os.path.join(dials_regression, "refinement_test_data", "multi_stills") experiments_path = os.path.join(data_dir, "combined_experiments.json") pickle_path = os.path.join(data_dir, "combined_reflections.pickle") experiments = ExperimentListFactory.from_json_file(experiments_path, check_format=False) reflections = flex.reflection_table.from_pickle(pickle_path) refiner = RefinerFactory.from_parameters_data_experiments(working_params, reflections, experiments) # hack to extract the objects needed from the Refiner rp = refiner._target._restraints_parameterisation pred_param = refiner._pred_param # get analytical values and gradients vals, grads, weights = rp.get_residuals_gradients_and_weights() # get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.e-7] * len(p_vals) fd_grad=[] for i in range(len(deltas)): val = p_vals[i] p_vals[i] -= deltas[i] / 2. pred_param.set_param_vals(p_vals) rev_state, foo, bar = rp.get_residuals_gradients_and_weights() rev_state = flex.double(rev_state) p_vals[i] += deltas[i] pred_param.set_param_vals(p_vals) fwd_state, foo, bar = rp.get_residuals_gradients_and_weights() fwd_state = flex.double(fwd_state) p_vals[i] = val fd = (fwd_state - rev_state) / deltas[i] fd_grad.append(fd) # for comparison, fd_grad is a list of flex.doubles, each of which corresponds # to the gradients of the residuals wrt to a single parameter. pnames = pred_param.get_param_names() for i, (pname, fd) in enumerate(zip(pnames, fd_grad)): # extract dense column from the sparse matrix an = grads.col(i).as_dense_vector() #print pname #print list(an.round(6)) #print list(fd.round(6)) #print assert approx_equal(an, fd, eps=1e-5) print "OK" return
def test_run(dials_regression, tmpdir): expected_unit_cell = uctbx.unit_cell( (11.624, 13.550, 30.103, 89.964, 93.721, 90.132)) expected_rmsds = (0.039, 0.035, 0.002) experiments_old = os.path.join(dials_regression, "indexing_test_data", "phi_scan", "datablock_old.json") experiments_new = os.path.join(dials_regression, "indexing_test_data", "phi_scan", "datablock.json") strong_pickle = os.path.join(dials_regression, "indexing_test_data", "phi_scan", "strong.pickle") from dxtbx.serialize import load imageset_old = load.experiment_list(experiments_old, check_format=False).imagesets()[0] imageset_new = load.experiment_list(experiments_new, check_format=False).imagesets()[0] gonio_old = imageset_old.get_goniometer() gonio_new = imageset_new.get_goniometer() assert gonio_old.get_rotation_axis() == pytest.approx( (0.7497646259807715, -0.5517923303436749, 0.36520984351713554)) assert gonio_old.get_setting_rotation() == pytest.approx( (1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0)) assert gonio_old.get_fixed_rotation() == pytest.approx(( 0.7497646259807748, -0.20997265900532208, -0.6275065641872948, -0.5517923303436731, 0.3250014637526764, -0.7680490041218182, 0.3652098435171313, 0.9221092836691605, 0.12781329809272568, )) assert gonio_new.get_rotation_axis() == pytest.approx( gonio_old.get_rotation_axis()) assert gonio_new.get_rotation_axis_datum() == pytest.approx((1, 0, 0)) assert gonio_new.get_setting_rotation() == pytest.approx(( 0.7497646259807705, -0.20997265900532142, -0.6275065641873, -0.5517923303436786, 0.3250014637526763, -0.768049004121814, 0.3652098435171315, 0.9221092836691607, 0.12781329809272335, )) assert gonio_new.get_fixed_rotation() == pytest.approx( (1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0)) result_old = run_indexing( strong_pickle, experiments_old, tmpdir, extra_args=[], expected_unit_cell=expected_unit_cell, expected_rmsds=expected_rmsds, expected_hall_symbol=" P 1", ) result_new = run_indexing( strong_pickle, experiments_new, tmpdir, extra_args=[], expected_unit_cell=expected_unit_cell, expected_rmsds=expected_rmsds, expected_hall_symbol=" P 1", ) assert result_old.rmsds == pytest.approx(result_new.rmsds, abs=1e-6) assert result_old.experiments[0].crystal.get_unit_cell().parameters( ) == pytest.approx( result_new.experiments[0].crystal.get_unit_cell().parameters(), abs=1e-6) # Now test refinement gradients are correct from dxtbx.model.experiment_list import Experiment, ExperimentList old_exps = ExperimentList([ Experiment( beam=imageset_old.get_beam(), detector=imageset_old.get_detector(), goniometer=gonio_old, scan=imageset_old.get_scan(), crystal=result_old.experiments[0].crystal, imageset=None, ) ]) new_exps = ExperimentList([ Experiment( beam=imageset_new.get_beam(), detector=imageset_new.get_detector(), goniometer=gonio_new, scan=imageset_new.get_scan(), crystal=result_new.experiments[0].crystal, imageset=None, ) ]) from libtbx.phil import parse from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse("")).extract() from dials.algorithms.refinement.refiner import RefinerFactory refiner_old = RefinerFactory.from_parameters_data_experiments( params, result_old.indexed_reflections, old_exps) refiner_new = RefinerFactory.from_parameters_data_experiments( params, result_new.indexed_reflections, new_exps) # Analytical gradients should be approximately the same in either case an_grads_old = refiner_old._pred_param.get_gradients( refiner_old.get_matches()) an_grads_new = refiner_new._pred_param.get_gradients( refiner_new.get_matches()) for g1, g2 in zip(an_grads_old, an_grads_new): assert g1["dX_dp"] == pytest.approx(g2["dX_dp"], abs=1.0e-6) assert g1["dY_dp"] == pytest.approx(g2["dY_dp"], abs=1.0e-6) assert g1["dphi_dp"] == pytest.approx(g2["dphi_dp"], abs=1.0e-6) # Analytical gradients should be approximately equal to finite difference # gradients in either case fd_grads_old = calc_fd_grads(refiner_old) for g1, g2 in zip(fd_grads_old, an_grads_old): assert g1["dX_dp"] == pytest.approx(g2["dX_dp"], abs=5.0e-6) assert g1["dY_dp"] == pytest.approx(g2["dY_dp"], abs=5.0e-6) assert g1["dphi_dp"] == pytest.approx(g2["dphi_dp"], abs=5.0e-6) fd_grads_new = calc_fd_grads(refiner_new) for g1, g2 in zip(fd_grads_new, an_grads_new): assert g1["dX_dp"] == pytest.approx(g2["dX_dp"], abs=5.0e-6) assert g1["dY_dp"] == pytest.approx(g2["dY_dp"], abs=5.0e-6) assert g1["dphi_dp"] == pytest.approx(g2["dphi_dp"], abs=5.0e-6)
s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xl1o_param.set_param_vals(xlo_p_vals[0]) xl2o_param.set_param_vals(xlo_p_vals[1]) xl1uc_param.set_param_vals(xluc_p_vals[0]) xl2uc_param.set_param_vals(xluc_p_vals[1]) #print "Initial values of parameters are" #msg = "Parameters: " + "%.5f " * len(pred_param) #print msg % tuple(pred_param.get_param_vals()) #print # make a refiner from dials.algorithms.refinement.refiner import phil_scope from libtbx.phil import parse params = phil_scope.fetch(source=parse('')).extract() # in case we want a plot params.refinement.refinery.track_parameter_correlation=True # DEBUG scan varying by uncommenting the following line #params.refinement.parameterisation.crystal.scan_varying=True from dials.algorithms.refinement.refiner import RefinerFactory refiner = RefinerFactory.from_parameters_data_experiments(params, obs_refs, experiments, verbosity=0) history = refiner.run() #plt = refiner.parameter_correlation_plot(len(history["parameter_correlation"])-1) #plt.show()
def run(): have_dials_regression = libtbx.env.has_module("dials_regression") if not have_dials_regression: print "Skipped: dials_regression not available" return dials_regression = libtbx.env.find_in_repositories( relative_path="dials_regression", test=os.path.isdir) from dials.test.algorithms.indexing.tst_index import run_one_indexing expected_unit_cell = uctbx.unit_cell( (11.624, 13.550, 30.103, 89.964, 93.721, 90.132)) expected_rmsds = (0.039, 0.035, 0.002) datablock_old = os.path.join( dials_regression, "indexing_test_data/phi_scan/datablock_old.json") datablock_new = os.path.join(dials_regression, "indexing_test_data/phi_scan/datablock.json") strong_pickle = os.path.join(dials_regression, "indexing_test_data/phi_scan/strong.pickle") from dxtbx.serialize import load imageset_old = load.datablock(datablock_old, check_format=False)[0].extract_imagesets()[0] imageset_new = load.datablock(datablock_new, check_format=False)[0].extract_imagesets()[0] gonio_old = imageset_old.get_goniometer() gonio_new = imageset_new.get_goniometer() assert approx_equal( gonio_old.get_rotation_axis(), (0.7497646259807715, -0.5517923303436749, 0.36520984351713554)) assert approx_equal(gonio_old.get_setting_rotation(), (1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0)) assert approx_equal( gonio_old.get_fixed_rotation(), (0.7497646259807748, -0.20997265900532208, -0.6275065641872948, -0.5517923303436731, 0.3250014637526764, -0.7680490041218182, 0.3652098435171313, 0.9221092836691605, 0.12781329809272568)) assert approx_equal(gonio_new.get_rotation_axis(), gonio_old.get_rotation_axis()) assert approx_equal(gonio_new.get_rotation_axis_datum(), (1, 0, 0)) assert approx_equal( gonio_new.get_setting_rotation(), (0.7497646259807705, -0.20997265900532142, -0.6275065641873, -0.5517923303436786, 0.3250014637526763, -0.768049004121814, 0.3652098435171315, 0.9221092836691607, 0.12781329809272335)) assert approx_equal(gonio_new.get_fixed_rotation(), (1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0)) result_old = run_one_indexing( pickle_path=strong_pickle, sweep_path=datablock_old, extra_args=[], expected_unit_cell=expected_unit_cell, expected_rmsds=expected_rmsds, expected_hall_symbol=' P 1', ) result_new = run_one_indexing( pickle_path=strong_pickle, sweep_path=datablock_new, extra_args=[], expected_unit_cell=expected_unit_cell, expected_rmsds=expected_rmsds, expected_hall_symbol=' P 1', ) assert approx_equal(result_old.rmsds, result_new.rmsds) assert approx_equal(result_old.crystal_model.get_unit_cell().parameters(), result_new.crystal_model.get_unit_cell().parameters()) # Now test refinement gradients are correct from dxtbx.model.experiment_list import ExperimentList, Experiment old_exps = ExperimentList([ Experiment(beam=imageset_old.get_beam(), detector=imageset_old.get_detector(), goniometer=gonio_old, scan=imageset_old.get_scan(), crystal=result_old.crystal_model, imageset=None) ]) new_exps = ExperimentList([ Experiment(beam=imageset_new.get_beam(), detector=imageset_new.get_detector(), goniometer=gonio_new, scan=imageset_new.get_scan(), crystal=result_new.crystal_model, imageset=None) ]) from libtbx.phil import parse from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse('')).extract() from dials.algorithms.refinement.refiner import RefinerFactory refiner_old = RefinerFactory.from_parameters_data_experiments( params, result_old.indexed_reflections, old_exps, verbosity=0) refiner_new = RefinerFactory.from_parameters_data_experiments( params, result_new.indexed_reflections, new_exps, verbosity=0) # Analytical gradients should be approximately the same in either case an_grads_old = refiner_old._pred_param.get_gradients( refiner_old.get_matches()) an_grads_new = refiner_new._pred_param.get_gradients( refiner_new.get_matches()) for g1, g2 in zip(an_grads_old, an_grads_new): assert approx_equal(g1["dX_dp"], g2["dX_dp"], eps=1.e-6) assert approx_equal(g1["dY_dp"], g2["dY_dp"], eps=1.e-6) assert approx_equal(g1["dphi_dp"], g2["dphi_dp"], eps=1.e-6) # Analytical gradients should be approximately equal to finite difference # gradients in either case fd_grads_old = calc_fd_grads(refiner_old) for g1, g2 in zip(fd_grads_old, an_grads_old): assert approx_equal(g1["dX_dp"], g2["dX_dp"], eps=5.e-6) assert approx_equal(g1["dY_dp"], g2["dY_dp"], eps=5.e-6) assert approx_equal(g1["dphi_dp"], g2["dphi_dp"], eps=5.e-6) fd_grads_new = calc_fd_grads(refiner_new) for g1, g2 in zip(fd_grads_new, an_grads_new): assert approx_equal(g1["dX_dp"], g2["dX_dp"], eps=5.e-6) assert approx_equal(g1["dY_dp"], g2["dY_dp"], eps=5.e-6) assert approx_equal(g1["dphi_dp"], g2["dphi_dp"], eps=5.e-6)
def test_group_restraint_with_multiple_crystals_and_a_stills_refiner( dials_regression): # The phil scope from dials.algorithms.refinement.refiner import phil_scope user_phil = parse(""" refinement { parameterisation { crystal { unit_cell { restraints { tie_to_group { sigmas=1,0,2,0,0,0 } } } } } } """) working_phil = phil_scope.fetch(source=user_phil) working_params = working_phil.extract() # use the multi stills test data data_dir = os.path.join(dials_regression, "refinement_test_data", "multi_stills") experiments_path = os.path.join(data_dir, "combined_experiments.json") pickle_path = os.path.join(data_dir, "combined_reflections.pickle") experiments = ExperimentListFactory.from_json_file(experiments_path, check_format=False) reflections = flex.reflection_table.from_file(pickle_path) refiner = RefinerFactory.from_parameters_data_experiments( working_params, reflections, experiments) # hack to extract the objects needed from the Refiner rp = refiner._target._restraints_parameterisation pred_param = refiner._pred_param # get analytical values and gradients vals, grads, weights = rp.get_residuals_gradients_and_weights() assert len(vals) == rp.num_residuals # get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.0e-7] * len(p_vals) fd_grad = [] for i, delta in enumerate(deltas): val = p_vals[i] p_vals[i] -= delta / 2.0 pred_param.set_param_vals(p_vals) rev_state, foo, bar = rp.get_residuals_gradients_and_weights() rev_state = flex.double(rev_state) p_vals[i] += delta pred_param.set_param_vals(p_vals) fwd_state, foo, bar = rp.get_residuals_gradients_and_weights() fwd_state = flex.double(fwd_state) p_vals[i] = val fd = (fwd_state - rev_state) / delta fd_grad.append(fd) # for comparison, fd_grad is a list of flex.doubles, each of which corresponds # to the gradients of the residuals wrt to a single parameter. pnames = pred_param.get_param_names() for i, (pname, fd) in enumerate(zip(pnames, fd_grad)): # extract dense column from the sparse matrix an = grads.col(i).as_dense_vector() # print pname # print list(an.round(6)) # print list(fd.round(6)) # print assert an == pytest.approx(fd, abs=1e-5)
stills_ref_predictor = ExperimentsPredictor(stills_experiments) obs_refs_stills = stills_ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs_stills['xyzobs.mm.value'] = obs_refs_stills['xyzcal.mm'] ############################### # Undo known parameter shifts # ############################### xlo_param.set_param_vals(xlo_p_vals[0]) xluc_param.set_param_vals(xluc_p_vals[0]) # make a refiner from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse('')).extract() # Change this to get a plot do_plot = False if do_plot: params.refinement.refinery.journal.track_parameter_correlation = True from dials.algorithms.refinement.refiner import RefinerFactory # decrease bin_size_fraction to terminate on RMSD convergence params.refinement.target.bin_size_fraction = 0.01 params.refinement.parameterisation.beam.fix = "all" params.refinement.parameterisation.detector.fix = "all" refiner = RefinerFactory.from_parameters_data_experiments(params, obs_refs_stills, stills_experiments, verbosity=0)
def test(args=[]): # Python and cctbx imports from math import pi from scitbx import matrix from libtbx.phil import parse from libtbx.test_utils import approx_equal # Import for surgery on reflection_tables from dials.array_family import flex # Get module to build models using PHIL import dials.test.algorithms.refinement.setup_geometry as setup_geometry # We will set up a mock scan and a mock experiment list from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import ExperimentList, Experiment # Crystal parameterisations from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalOrientationParameterisation, CrystalUnitCellParameterisation, ) # Symmetry constrained parameterisation for the unit cell from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.refinement.prediction.managed_predictors import ( ScansRayPredictor, StillsExperimentsPredictor, ) from dials.algorithms.spot_prediction import ray_intersection from cctbx.sgtbx import space_group, space_group_symbols ############################# # Setup experimental models # ############################# master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True, ) # build models, with a larger crystal than default in order to get enough # reflections on the 'still' image param = """ geometry.parameters.crystal.a.length.range=40 50; geometry.parameters.crystal.b.length.range=40 50; geometry.parameters.crystal.c.length.range=40 50; geometry.parameters.random_seed = 42""" models = setup_geometry.Extract(master_phil, cmdline_args=args, local_overrides=param) crystal = models.crystal mydetector = models.detector mygonio = models.goniometer mybeam = models.beam # Build a mock scan for a 1.5 degree wedge. Only used for generating indices near # the Ewald sphere sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 1), exposure_times=0.1, oscillation=(0, 1.5), epochs=list(range(1)), deg=True, ) sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert approx_equal(im_width, 1.5 * pi / 180.0) # Build experiment lists stills_experiments = ExperimentList() stills_experiments.append( Experiment(beam=mybeam, detector=mydetector, crystal=crystal, imageset=None)) scans_experiments = ExperimentList() scans_experiments.append( Experiment( beam=mybeam, detector=mydetector, crystal=crystal, goniometer=mygonio, scan=myscan, imageset=None, )) ########################################################## # Parameterise the models (only for perturbing geometry) # ########################################################## xlo_param = CrystalOrientationParameterisation(crystal) xluc_param = CrystalUnitCellParameterisation(crystal) ################################ # Apply known parameter shifts # ################################ # rotate crystal (=5 mrad each rotation) xlo_p_vals = [] p_vals = xlo_param.get_param_vals() xlo_p_vals.append(p_vals) new_p_vals = [a + b for a, b in zip(p_vals, [5.0, 5.0, 5.0])] xlo_param.set_param_vals(new_p_vals) # change unit cell (=1.0 Angstrom length upsets, 0.5 degree of # gamma angle) xluc_p_vals = [] p_vals = xluc_param.get_param_vals() xluc_p_vals.append(p_vals) cell_params = crystal.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [1.0, 1.0, -1.0, 0.0, 0.0, 0.5]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(crystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) # keep track of the target crystal model to compare with refined from copy import deepcopy target_crystal = deepcopy(crystal) ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere for crystal resolution = 2.0 index_generator = IndexGenerator( crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() # Build a ray predictor and predict rays close to the Ewald sphere by using # the narrow rotation scan ref_predictor = ScansRayPredictor(scans_experiments, sweep_range) obs_refs = ref_predictor(indices, experiment_id=0) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Add in flags and ID columns by copying into standard reflection table tmp = flex.reflection_table.empty_standard(len(obs_refs)) tmp.update(obs_refs) obs_refs = tmp # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180.0 px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2) obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) # Re-predict using the stills reflection predictor stills_ref_predictor = StillsExperimentsPredictor(stills_experiments) obs_refs_stills = stills_ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs_stills["xyzobs.mm.value"] = obs_refs_stills["xyzcal.mm"] ############################### # Undo known parameter shifts # ############################### xlo_param.set_param_vals(xlo_p_vals[0]) xluc_param.set_param_vals(xluc_p_vals[0]) # make a refiner from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse("")).extract() # Change this to get a plot do_plot = False if do_plot: params.refinement.refinery.journal.track_parameter_correlation = True from dials.algorithms.refinement.refiner import RefinerFactory # decrease bin_size_fraction to terminate on RMSD convergence params.refinement.target.bin_size_fraction = 0.01 params.refinement.parameterisation.beam.fix = "all" params.refinement.parameterisation.detector.fix = "all" refiner = RefinerFactory.from_parameters_data_experiments( params, obs_refs_stills, stills_experiments) # run refinement history = refiner.run() # regression tests assert len(history["rmsd"]) == 9 refined_crystal = refiner.get_experiments()[0].crystal uc1 = refined_crystal.get_unit_cell() uc2 = target_crystal.get_unit_cell() assert uc1.is_similar_to(uc2) if do_plot: plt = refiner.parameter_correlation_plot( len(history["parameter_correlation"]) - 1) plt.show()