def create_models(self): # build models, with a larger crystal than default in order to get plenty of # reflections on the 'still' image overrides = """ geometry.parameters.crystal.a.length.range=40 50; geometry.parameters.crystal.b.length.range=40 50; geometry.parameters.crystal.c.length.range=40 50; geometry.parameters.random_seed = 42""" master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True) models = Extract(master_phil, overrides) # keep track of the models self.detector = models.detector self.gonio = models.goniometer self.crystal = models.crystal self.beam = models.beam # Create a stills ExperimentList self.stills_experiments = ExperimentList() self.stills_experiments.append( Experiment(beam=self.beam, detector=self.detector, crystal=self.crystal, imageset=None)) # keep track of the parameterisation of the models self.det_param = DetectorParameterisationSinglePanel(self.detector) self.s0_param = BeamParameterisation(self.beam, self.gonio) self.xlo_param = CrystalOrientationParameterisation(self.crystal) self.xluc_param = CrystalUnitCellParameterisation(self.crystal)
def create_models(self, cmdline_overrides=None): from dxtbx.model import ScanFactory from libtbx.phil import parse from dials.test.algorithms.refinement.setup_geometry import Extract if cmdline_overrides is None: cmdline_overrides = [] overrides = """geometry.parameters.crystal.a.length.range = 10 50 geometry.parameters.crystal.b.length.range = 10 50 geometry.parameters.crystal.c.length.range = 10 50""" master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True, ) # Extract models models = Extract(master_phil, overrides, cmdline_args=cmdline_overrides) self.detector = models.detector self.goniometer = models.goniometer self.crystal = models.crystal self.beam = models.beam # Make a scan of 1-20 * 0.5 deg images sf = ScanFactory() self.scan = sf.make_scan((1, 20), 0.5, (0, 0.5), list(range(20))) # Generate an ExperimentList self.experiments = ExperimentList() self.experiments.append( Experiment( beam=self.beam, detector=self.detector, goniometer=self.goniometer, scan=self.scan, crystal=self.crystal, imageset=None, )) # Create a reflection predictor for the experiments self.ref_predictor = ScansExperimentsPredictor(self.experiments) # Create scan-varying parameterisations of these models, with 3 samples self.det_param = ScanVaryingDetectorParameterisationSinglePanel( self.detector, self.scan.get_array_range(), 3) self.s0_param = ScanVaryingBeamParameterisation( self.beam, self.scan.get_array_range(), 3, self.goniometer) self.xlo_param = ScanVaryingCrystalOrientationParameterisation( self.crystal, self.scan.get_array_range(), 3) self.xluc_param = ScanVaryingCrystalUnitCellParameterisation( self.crystal, self.scan.get_array_range(), 3) self.gon_param = ScanVaryingGoniometerParameterisation( self.goniometer, self.scan.get_array_range(), 3, self.beam)
def create_experiments(image_start=1): # Create models from libtbx.phil import parse overrides = """geometry.parameters.crystal.a.length.range = 10 50 geometry.parameters.crystal.b.length.range = 10 50 geometry.parameters.crystal.c.length.range = 10 50""" master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True, ) from dials.test.algorithms.refinement.setup_geometry import Extract models = Extract(master_phil, overrides) detector = models.detector goniometer = models.goniometer crystal = models.crystal beam = models.beam # Build a mock scan for a 72 degree sequence from dxtbx.model import ScanFactory sf = ScanFactory() scan = sf.make_scan( image_range=(image_start, image_start + 720 - 1), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(720)), deg=True, ) # No matter what image_start is, scan should start at 0.0 and end at 72.0 deg assert scan.get_oscillation_range(deg=True) == (0.0, 72.0) # Create an ExperimentList experiments = ExperimentList() experiments.append( Experiment( beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal, imageset=None, ) ) return experiments
def test(): from cctbx.sgtbx import space_group, space_group_symbols # We will set up a mock scan from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import Experiment, ExperimentList from libtbx.phil import parse from scitbx import matrix from scitbx.array_family import flex from dials.algorithms.refinement.prediction.managed_predictors import ( ScansExperimentsPredictor, ScansRayPredictor, ) # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator # Building experimental models from dials.test.algorithms.refinement.setup_geometry import Extract master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True, ) overrides = """geometry.parameters.crystal.a.length.range = 10 50 geometry.parameters.crystal.b.length.range = 10 50 geometry.parameters.crystal.c.length.range = 10 50""" models = Extract(master_phil, local_overrides=overrides) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator( mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() # Build a mock scan for a 30 degree sequence sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 300), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(300)), deg=True, ) sequence_range = myscan.get_oscillation_range(deg=False) assert sequence_range == pytest.approx((0.0, math.pi / 6.0)) im_width = myscan.get_oscillation(deg=False)[1] assert im_width == pytest.approx(0.1 * math.pi / 180.0) # Create an ExperimentList for ScansRayPredictor experiments = ExperimentList() experiments.append( Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None, )) # Select those that are excited in a 30 degree sequence and get angles ray_predictor = ScansRayPredictor(experiments, sequence_range) obs_refs = ray_predictor(indices) # Set the experiment number obs_refs["id"] = flex.int(len(obs_refs), 0) # Calculate intersections ref_predictor = ScansExperimentsPredictor(experiments) obs_refs = ref_predictor(obs_refs) print("Total number of observations made", len(obs_refs)) s0 = matrix.col(mybeam.get_s0()) spindle = matrix.col(mygonio.get_rotation_axis()) for ref in obs_refs.rows(): # get the s1 vector of this reflection s1 = matrix.col(ref["s1"]) r = s1 - s0 r_orig = r.rotate_around_origin(spindle, -1.0, deg=True) # is it outside the Ewald sphere (i.e. entering)? test = (s0 + r_orig).length() > s0.length() assert ref["entering"] == test
def test_fd_derivatives(): """Test derivatives of the prediction equation""" from libtbx.phil import parse # Import model builder from dials.test.algorithms.refinement.setup_geometry import Extract # Create models overrides = """geometry.parameters.crystal.a.length.range = 10 50 geometry.parameters.crystal.b.length.range = 10 50 geometry.parameters.crystal.c.length.range = 10 50""" master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True, ) models = Extract(master_phil, overrides) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam # Build a mock scan for a 72 degree sequence from dxtbx.model import ScanFactory sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 720), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(720)), deg=True, ) # Create a parameterisation of the crystal unit cell from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalUnitCellParameterisation, ) xluc_param = CrystalUnitCellParameterisation(mycrystal) # Create an ExperimentList experiments = ExperimentList() experiments.append( Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None, )) # Build a prediction parameterisation for two theta prediction pred_param = TwoThetaPredictionParameterisation( experiments, detector_parameterisations=None, beam_parameterisations=None, xl_orientation_parameterisations=None, xl_unit_cell_parameterisations=[xluc_param], ) # Generate some reflections obs_refs, ref_predictor = generate_reflections(experiments) # Build a ReflectionManager with overloads for handling 2theta residuals refman = TwoThetaReflectionManager(obs_refs, experiments, outlier_detector=None) # Build a TwoThetaExperimentsPredictor ref_predictor = TwoThetaExperimentsPredictor(experiments) # Make a target for the least squares 2theta residual target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param) # Keep only reflections that pass inclusion criteria and have predictions reflections = refman.get_matches() # Get analytical gradients an_grads = pred_param.get_gradients(reflections) # Get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.0e-7] * len(p_vals) for i in range(len(deltas)): val = p_vals[i] p_vals[i] -= deltas[i] / 2.0 pred_param.set_param_vals(p_vals) target.predict() reflections = refman.get_matches() rev_state = reflections["2theta_resid"].deep_copy() p_vals[i] += deltas[i] pred_param.set_param_vals(p_vals) target.predict() reflections = refman.get_matches() fwd_state = reflections["2theta_resid"].deep_copy() p_vals[i] = val fd = fwd_state - rev_state fd /= deltas[i] # compare with analytical calculation assert approx_equal(fd, an_grads[i]["d2theta_dp"], eps=1.0e-6) # return to the initial state pred_param.set_param_vals(p_vals)
def test(): # Build models, with a larger crystal than default in order to get plenty of # reflections on the 'still' image overrides = """ geometry.parameters.crystal.a.length.range=40 50; geometry.parameters.crystal.b.length.range=40 50; geometry.parameters.crystal.c.length.range=40 50; geometry.parameters.random_seed = 42""" master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True, ) models = Extract(master_phil, overrides) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam # Build a mock scan for a 3 degree sweep from dxtbx.model import ScanFactory sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 1), exposure_times=0.1, oscillation=(0, 3.0), epochs=list(range(1)), deg=True, ) sweep_range = myscan.get_oscillation_range(deg=False) # Create parameterisations of these models det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) # Create a scans ExperimentList, only for generating reflections experiments = ExperimentList() experiments.append( Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None, )) # Create a stills ExperimentList stills_experiments = ExperimentList() stills_experiments.append( Experiment(beam=mybeam, detector=mydetector, crystal=mycrystal, imageset=None)) # Generate rays - only to work out which hkls are predicted ray_predictor = ScansRayPredictor(experiments, sweep_range) resolution = 2.0 index_generator = IndexGenerator( mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() rays = ray_predictor(indices) # Make a standard reflection_table and copy in the ray data reflections = flex.reflection_table.empty_standard(len(rays)) reflections.update(rays) # Build a standard prediction parameterisation for the stills experiment to do # FD calculation (not used for its analytical gradients) pred_param = StillsPredictionParameterisation( stills_experiments, detector_parameterisations=[det_param], beam_parameterisations=[s0_param], xl_orientation_parameterisations=[xlo_param], xl_unit_cell_parameterisations=[xluc_param], ) # Make a managed SphericalRelpStillsReflectionPredictor reflection predictor # for the first (only) experiment ref_predictor = Predictor(stills_experiments) # Predict these reflections in place. Must do this ahead of calculating # the analytical gradients so quantities like s1 are correct ref_predictor.update() ref_predictor.predict(reflections) # calculate analytical gradients ag = AnalyticalGradients( stills_experiments, detector_parameterisation=det_param, beam_parameterisation=s0_param, xl_orientation_parameterisation=xlo_param, xl_unit_cell_parameterisation=xluc_param, ) an_grads = ag.get_beam_gradients(reflections) an_grads.update(ag.get_crystal_orientation_gradients(reflections)) an_grads.update(ag.get_crystal_unit_cell_gradients(reflections)) # get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.0e-7] * len(p_vals) fd_grads = [] p_names = pred_param.get_param_names() for i, delta in enumerate(deltas): # save parameter value val = p_vals[i] # calc reverse state p_vals[i] -= delta / 2.0 pred_param.set_param_vals(p_vals) ref_predictor.update() ref_predictor.predict(reflections) x, y, _ = reflections["xyzcal.mm"].deep_copy().parts() s1 = reflections["s1"].deep_copy() rev_state = s1 # calc forward state p_vals[i] += delta pred_param.set_param_vals(p_vals) ref_predictor.update() ref_predictor.predict(reflections) x, y, _ = reflections["xyzcal.mm"].deep_copy().parts() s1 = reflections["s1"].deep_copy() fwd_state = s1 # reset parameter to saved value p_vals[i] = val # finite difference - currently for s1 only fd = fwd_state - rev_state inv_delta = 1.0 / delta s1_grads = fd * inv_delta # store gradients fd_grads.append({"name": p_names[i], "ds1": s1_grads}) # return to the initial state pred_param.set_param_vals(p_vals) for i, fd_grad in enumerate(fd_grads): ## compare FD with analytical calculations print("\n\nParameter {0}: {1}".format(i, fd_grad["name"])) print("d[s1]/dp for the first reflection") print("finite diff", fd_grad["ds1"][0]) try: an_grad = an_grads[fd_grad["name"]] except KeyError: continue print("checking analytical vs finite difference gradients for s1") for a, b in zip(fd_grad["ds1"], an_grad["ds1"]): assert a == pytest.approx(b, abs=1e-7)
def test_single_crystal_restraints_gradients(): """Simple test with a single triclinic crystal restrained to a target unit cell""" from dxtbx.model.experiment_list import Experiment, ExperimentList from dials.algorithms.refinement.parameterisation.beam_parameters import ( BeamParameterisation, ) from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalOrientationParameterisation, CrystalUnitCellParameterisation, ) from dials.algorithms.refinement.parameterisation.detector_parameters import ( DetectorParameterisationSinglePanel, ) from dials.algorithms.refinement.parameterisation.prediction_parameters import ( XYPhiPredictionParameterisation, ) from dials.test.algorithms.refinement.setup_geometry import Extract overrides = """geometry.parameters.crystal.a.length.range = 10 50 geometry.parameters.crystal.b.length.range = 10 50 geometry.parameters.crystal.c.length.range = 10 50""" master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True, ) models = Extract(master_phil, overrides) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam # Build a mock scan for a 72 degree sequence from dxtbx.model import ScanFactory sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 720), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(720)), deg=True, ) # Create parameterisations of these models det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) # Create an ExperimentList experiments = ExperimentList() experiments.append( Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None, )) # Build a prediction parameterisation pred_param = XYPhiPredictionParameterisation( experiments, detector_parameterisations=[det_param], beam_parameterisations=[s0_param], xl_orientation_parameterisations=[xlo_param], xl_unit_cell_parameterisations=[xluc_param], ) # Build a restraints parameterisation rp = RestraintsParameterisation( detector_parameterisations=[det_param], beam_parameterisations=[s0_param], xl_orientation_parameterisations=[xlo_param], xl_unit_cell_parameterisations=[xluc_param], ) # make a unit cell target sigma = 1.0 uc = mycrystal.get_unit_cell().parameters() target_uc = [random.gauss(e, sigma) for e in uc] rp.add_restraints_to_target_xl_unit_cell(experiment_id=0, values=target_uc, sigma=[sigma] * 6) # get analytical values and gradients vals, grads, weights = rp.get_residuals_gradients_and_weights() assert len(vals) == rp.num_residuals # get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.0e-7] * len(p_vals) fd_grad = [] for i, delta in enumerate(deltas): val = p_vals[i] p_vals[i] -= delta / 2.0 pred_param.set_param_vals(p_vals) rev_state, foo, bar = rp.get_residuals_gradients_and_weights() rev_state = flex.double(rev_state) p_vals[i] += delta pred_param.set_param_vals(p_vals) fwd_state, foo, bar = rp.get_residuals_gradients_and_weights() fwd_state = flex.double(fwd_state) p_vals[i] = val fd = (fwd_state - rev_state) / delta fd_grad.append(fd) # for comparison, fd_grad is a list of flex.doubles, each of which corresponds # to a column of the sparse matrix grads. for i, fd in enumerate(fd_grad): # extract dense column from the sparse matrix an = grads.col(i).as_dense_vector() assert an == pytest.approx(fd, abs=1e-5)
############################# # Setup experimental models # ############################# args = sys.argv[1:] # make a small cell to speed up calculations overrides = """geometry.parameters.crystal.a.length.range = 10 15 geometry.parameters.crystal.b.length.range = 10 15 geometry.parameters.crystal.c.length.range = 10 15""" master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True) models = Extract(master_phil, overrides, cmdline_args=args) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam # Build a mock scan for a 180 degree sweep of 0.1 degree images sf = ScanFactory() myscan = sf.make_scan(image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=range(1800), deg=True) sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1]
def test(args=[]): # Python and cctbx imports from math import pi import random from scitbx import matrix from scitbx.array_family import flex from libtbx.phil import parse from libtbx.test_utils import approx_equal # Experimental model builder from dials.test.algorithms.refinement.setup_geometry import Extract # We will set up a mock scan and a mock experiment list from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import ExperimentList, Experiment # Model parameterisations from dials.algorithms.refinement.parameterisation.detector_parameters import ( DetectorParameterisationSinglePanel, ) from dials.algorithms.refinement.parameterisation.beam_parameters import ( BeamParameterisation, ) from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalOrientationParameterisation, CrystalUnitCellParameterisation, ) # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection from dials.algorithms.refinement.prediction.managed_predictors import ( ScansRayPredictor, ScansExperimentsPredictor, ) from cctbx.sgtbx import space_group, space_group_symbols # Parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.prediction_parameters import ( XYPhiPredictionParameterisation, ) # Imports for the target function from dials.algorithms.refinement.target import ( LeastSquaresPositionalResidualWithRmsdCutoff, ) from dials.algorithms.refinement.reflection_manager import ReflectionManager # Local functions def random_direction_close_to(vector, sd=0.5): return vector.rotate_around_origin( matrix.col((random.random(), random.random(), random.random())).normalize(), random.gauss(0, sd), deg=True, ) ############################# # Setup experimental models # ############################# # make a small cell to speed up calculations overrides = """geometry.parameters.crystal.a.length.range = 10 15 geometry.parameters.crystal.b.length.range = 10 15 geometry.parameters.crystal.c.length.range = 10 15""" master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True, ) models = Extract(master_phil, overrides, cmdline_args=args) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam # Build a mock scan for a 180 degree sweep of 0.1 degree images sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(1800)), deg=True, ) sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0.0, pi) assert approx_equal(im_width, 0.1 * pi / 180.0) experiments = ExperimentList() experiments.append( Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None, )) ########################### # Parameterise the models # ########################### det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ######################################################################## pred_param = XYPhiPredictionParameterisation(experiments, [det_param], [s0_param], [xlo_param], [xluc_param]) ################################ # Apply known parameter shifts # ################################ # shift detector by 0.2 mm each translation and 2 mrad each rotation det_p_vals = det_param.get_param_vals() p_vals = [ a + b for a, b in zip(det_p_vals, [2.0, 2.0, 2.0, 2.0, 2.0, 2.0]) ] det_param.set_param_vals(p_vals) # shift beam by 2 mrad in one axis s0_p_vals = s0_param.get_param_vals() p_vals = list(s0_p_vals) p_vals[1] += 2.0 s0_param.set_param_vals(p_vals) # rotate crystal a bit (=2 mrad each rotation) xlo_p_vals = xlo_param.get_param_vals() p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])] xlo_param.set_param_vals(p_vals) ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator( mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() # Predict rays within the sweep range ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ScansExperimentsPredictor(experiments) obs_refs["id"] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180.0 px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2) obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) ##################################### # Select reflections for refinement # ##################################### refman = ReflectionManager(obs_refs, experiments) ############################## # Set up the target function # ############################## # Redefine the reflection predictor to use the type expected by the Target class ref_predictor = ScansExperimentsPredictor(experiments) mytarget = LeastSquaresPositionalResidualWithRmsdCutoff( experiments, ref_predictor, refman, pred_param, restraints_parameterisation=None) # get the functional and gradients mytarget.predict() L, dL_dp, curvs = mytarget.compute_functional_gradients_and_curvatures() #################################### # Do FD calculation for comparison # #################################### # function for calculating finite difference gradients of the target function def get_fd_gradients(target, pred_param, deltas): """Calculate centered finite difference gradients for each of the parameters of the target function. "deltas" must be a sequence of the same length as the parameter list, and contains the step size for the difference calculations for each parameter. """ p_vals = pred_param.get_param_vals() assert len(deltas) == len(p_vals) fd_grad = [] fd_curvs = [] for i in range(len(deltas)): val = p_vals[i] p_vals[i] -= deltas[i] / 2.0 pred_param.set_param_vals(p_vals) target.predict() rev_state = target.compute_functional_gradients_and_curvatures() p_vals[i] += deltas[i] pred_param.set_param_vals(p_vals) target.predict() fwd_state = target.compute_functional_gradients_and_curvatures() # finite difference estimation of first derivatives fd_grad.append((fwd_state[0] - rev_state[0]) / deltas[i]) # finite difference estimation of curvatures, using the analytical # first derivatives fd_curvs.append((fwd_state[1][i] - rev_state[1][i]) / deltas[i]) # set parameter back to centred value p_vals[i] = val # return to the initial state pred_param.set_param_vals(p_vals) return fd_grad, fd_curvs # test normalised differences between FD and analytical calculations fdgrads = get_fd_gradients(mytarget, pred_param, [1.0e-7] * len(pred_param)) diffs = [a - b for a, b in zip(dL_dp, fdgrads[0])] norm_diffs = tuple([a / b for a, b in zip(diffs, fdgrads[0])]) for e in norm_diffs: assert abs(e) < 0.001 # check differences less than 0.1% # test normalised differences between FD curvatures and analytical least # squares approximation. We don't expect this to be especially close if curvs: diffs = [a - b for a, b in zip(curvs, fdgrads[1])] norm_diffs = tuple([a / b for a, b in zip(diffs, fdgrads[1])]) for e in norm_diffs: assert abs(e) < 0.1 # check differences less than 10%
def test2(): '''Simple test with two triclinic crystals restrained to a target unit cell''' from math import pi from random import gauss from dials.test.algorithms.refinement.setup_geometry import Extract from dxtbx.model.experiment.experiment_list import ExperimentList, Experiment #### Import model parameterisations from dials.algorithms.refinement.parameterisation.prediction_parameters import \ XYPhiPredictionParameterisation from dials.algorithms.refinement.parameterisation.detector_parameters import \ DetectorParameterisationSinglePanel from dials.algorithms.refinement.parameterisation.beam_parameters import \ BeamParameterisation from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalOrientationParameterisation, \ CrystalUnitCellParameterisation overrides = """geometry.parameters.crystal.a.length.range = 10 50 geometry.parameters.crystal.b.length.range = 10 50 geometry.parameters.crystal.c.length.range = 10 50""" master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True) models = Extract(master_phil, overrides) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal # duplicate the crystal from copy import deepcopy mycrystal2 = deepcopy(mycrystal) mybeam = models.beam # Build a mock scan for a 72 degree sweep sweep_range = (0., pi / 5.) from dxtbx.model.scan import scan_factory sf = scan_factory() myscan = sf.make_scan(image_range=(1, 720), exposure_times=0.1, oscillation=(0, 0.1), epochs=range(720), deg=True) # Create parameterisations of these models det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) xluc_param2 = CrystalUnitCellParameterisation(mycrystal2, experiment_ids=[1]) # Create an ExperimentList with the crystal duplicated experiments = ExperimentList() experiments.append( Experiment(beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None)) experiments.append( Experiment(beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal2, imageset=None)) # Build a prediction parameterisation pred_param = XYPhiPredictionParameterisation( experiments, detector_parameterisations=[det_param], beam_parameterisations=[s0_param], xl_orientation_parameterisations=[xlo_param], xl_unit_cell_parameterisations=[xluc_param, xluc_param2]) # Build a restraints parameterisation rp = RestraintsParameterisation( detector_parameterisations=[det_param], beam_parameterisations=[s0_param], xl_orientation_parameterisations=[xlo_param], xl_unit_cell_parameterisations=[xluc_param, xluc_param2]) # make a unit cell target sigma = 1. uc = mycrystal.get_unit_cell().parameters() target_uc = [gauss(e, sigma) for e in uc] rp.add_restraints_to_target_xl_unit_cell(experiment_id=0, values=target_uc, sigma=[sigma] * 6) rp.add_restraints_to_target_xl_unit_cell(experiment_id=1, values=target_uc, sigma=[sigma] * 6) # get analytical values and gradients vals, grads, weights = rp.get_residuals_gradients_and_weights() # get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.e-7] * len(p_vals) fd_grad = [] for i in range(len(deltas)): val = p_vals[i] p_vals[i] -= deltas[i] / 2. pred_param.set_param_vals(p_vals) rev_state, foo, bar = rp.get_residuals_gradients_and_weights() rev_state = flex.double(rev_state) p_vals[i] += deltas[i] pred_param.set_param_vals(p_vals) fwd_state, foo, bar = rp.get_residuals_gradients_and_weights() fwd_state = flex.double(fwd_state) p_vals[i] = val fd = (fwd_state - rev_state) / deltas[i] fd_grad.append(fd) # for comparison, fd_grad is a list of flex.doubles, each of which corresponds # to a column of the sparse matrix grads. for i, fd in enumerate(fd_grad): # extract dense column from the sparse matrix an = grads.col(i).as_dense_vector() assert approx_equal(an, fd, eps=1e-5) print "OK"
def test(): from cctbx.sgtbx import space_group, space_group_symbols from libtbx.phil import parse from scitbx.array_family import flex ##### Import model builder from dials.test.algorithms.refinement.setup_geometry import Extract ##### Imports for reflection prediction from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection from dxtbx.model.experiment_list import ExperimentList, Experiment from dials.algorithms.refinement.prediction import ScansRayPredictor, \ ExperimentsPredictor #### Import model parameterisations from dials.algorithms.refinement.parameterisation.prediction_parameters import \ XYPhiPredictionParameterisation from dials.algorithms.refinement.parameterisation.detector_parameters import \ DetectorParameterisationSinglePanel from dials.algorithms.refinement.parameterisation.beam_parameters import \ BeamParameterisation from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalOrientationParameterisation, \ CrystalUnitCellParameterisation from dials.algorithms.refinement.parameterisation.goniometer_parameters import \ GoniometerParameterisation #### Create models overrides = """geometry.parameters.crystal.a.length.range = 10 50 geometry.parameters.crystal.b.length.range = 10 50 geometry.parameters.crystal.c.length.range = 10 50""" master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True) models = Extract(master_phil, overrides) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam # Build a mock scan for a 72 degree sweep sweep_range = (0., math.pi/5.) from dxtbx.model import ScanFactory sf = ScanFactory() myscan = sf.make_scan(image_range = (1,720), exposure_times = 0.1, oscillation = (0, 0.1), epochs = range(720), deg = True) #### Create parameterisations of these models det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) gon_param = GoniometerParameterisation(mygonio, mybeam) # Create an ExperimentList experiments = ExperimentList() experiments.append(Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None)) #### Unit tests # Build a prediction parameterisation pred_param = XYPhiPredictionParameterisation(experiments, detector_parameterisations = [det_param], beam_parameterisations = [s0_param], xl_orientation_parameterisations = [xlo_param], xl_unit_cell_parameterisations = [xluc_param], goniometer_parameterisations = [gon_param]) # Generate reflections resolution = 2.0 index_generator = IndexGenerator(mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() # Predict rays within the sweep range ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs['id'] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * math.pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) # use a ReflectionManager to exclude reflections too close to the spindle from dials.algorithms.refinement.reflection_manager import ReflectionManager refman = ReflectionManager(obs_refs, experiments, outlier_detector=None) # Redefine the reflection predictor to use the type expected by the Target class ref_predictor = ExperimentsPredictor(experiments) # make a target to ensure reflections are predicted and refman is finalised from dials.algorithms.refinement.target import \ LeastSquaresPositionalResidualWithRmsdCutoff target = LeastSquaresPositionalResidualWithRmsdCutoff(experiments, ref_predictor, refman, pred_param, restraints_parameterisation=None) # keep only those reflections that pass inclusion criteria and have predictions reflections = refman.get_matches() # get analytical gradients an_grads = pred_param.get_gradients(reflections) # get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.e-7] * len(p_vals) for i, delta in enumerate(deltas): val = p_vals[i] p_vals[i] -= delta / 2. pred_param.set_param_vals(p_vals) ref_predictor(reflections) rev_state = reflections['xyzcal.mm'].deep_copy() p_vals[i] += delta pred_param.set_param_vals(p_vals) ref_predictor(reflections) fwd_state = reflections['xyzcal.mm'].deep_copy() p_vals[i] = val fd = (fwd_state - rev_state) x_grads, y_grads, phi_grads = fd.parts() x_grads /= delta y_grads /= delta phi_grads /= delta # compare with analytical calculation assert x_grads == pytest.approx(an_grads[i]["dX_dp"], abs=5.e-6) assert y_grads == pytest.approx(an_grads[i]["dY_dp"], abs=5.e-6) assert phi_grads == pytest.approx(an_grads[i]["dphi_dp"], abs=5.e-6) # return to the initial state pred_param.set_param_vals(p_vals)
# We will set up a mock scan from dxtbx.model import ScanFactory args = sys.argv[1:] master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True) overrides = """geometry.parameters.crystal.a.length.range = 10 50 geometry.parameters.crystal.b.length.range = 10 50 geometry.parameters.crystal.c.length.range = 10 50""" models = Extract(master_phil, local_overrides=overrides, cmdline_args=args) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator( mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution)