def __init__( self, experiment, mosaicity_parameterisation, fix_mosaic_spread=False, fix_wavelength_spread=False, fix_unit_cell=False, fix_orientation=False, ): """ Initialise the model state """ # Save the crystal model self.experiment = experiment self.crystal = experiment.crystal # The U and P parameterisation self.U_parameterisation = CrystalOrientationParameterisation( self.crystal) self.B_parameterisation = CrystalUnitCellParameterisation(self.crystal) # The M, L and W parameterisations self.M_parameterisation = mosaicity_parameterisation # Set the flags to fix parameters self._is_mosaic_spread_fixed = fix_mosaic_spread self._is_wavelength_spread_fixed = fix_wavelength_spread self._is_unit_cell_fixed = fix_unit_cell self._is_orientation_fixed = fix_orientation
def refine(self, do_print=False): crystal = self.crystal from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalUnitCellParameterisation, CrystalOrientationParameterisation, ) self.cucp = CrystalUnitCellParameterisation(crystal) self.cop = CrystalOrientationParameterisation(crystal) self.zero() # 0-point and deltas values = flex.double(self.cucp.get_param_vals() + self.cop.get_param_vals()) offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] + [0.1, 0.1, 0.1]) initial = crystal.get_unit_cell() self.cells = [] self.best_score = 1e99 initial_score = self.target(values) doohicky = simple_simplex(values, offset, self, 2000) best = doohicky.get_solution() if do_print: print("Initial cell:", initial) print("Final cell: ", crystal.get_unit_cell()) print("Score change", initial_score, self.target(best, do_print=False)) self.best = best
def create_models(self): # build models, with a larger crystal than default in order to get plenty of # reflections on the 'still' image overrides = """ geometry.parameters.crystal.a.length.range=40 50; geometry.parameters.crystal.b.length.range=40 50; geometry.parameters.crystal.c.length.range=40 50; geometry.parameters.random_seed = 42""" master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True) models = Extract(master_phil, overrides) # keep track of the models self.detector = models.detector self.gonio = models.goniometer self.crystal = models.crystal self.beam = models.beam # Create a stills ExperimentList self.stills_experiments = ExperimentList() self.stills_experiments.append( Experiment(beam=self.beam, detector=self.detector, crystal=self.crystal, imageset=None)) # keep track of the parameterisation of the models self.det_param = DetectorParameterisationSinglePanel(self.detector) self.s0_param = BeamParameterisation(self.beam, self.gonio) self.xlo_param = CrystalOrientationParameterisation(self.crystal) self.xluc_param = CrystalUnitCellParameterisation(self.crystal)
def __init__(self, experiment, reflections, parameters): from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalUnitCellParameterisation, ) from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalOrientationParameterisation, ) # Store the input self.experiment = experiment self.reflections = reflections self.parameters = parameters # Get the crystal model and the parameterisation self.crystal = self.experiment.crystal self.cucp = CrystalUnitCellParameterisation(self.crystal) self.cop = CrystalOrientationParameterisation(self.crystal) # Get the current values and generate some offsets values = flex.double(self.cucp.get_param_vals() + self.cop.get_param_vals()) offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] + [0.1, 0.1, 0.1]) # The optimization history self.history = [] # Get the initial cell and initial score initial_cell = self.crystal.get_unit_cell() initial_orientation = self.crystal.get_U() initial_score = self.target(values) # Perform the optimization optimizer = simple_simplex(values, offset, self, 2000) result = optimizer.get_solution() # Print some information fmt = "(%.3f, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f)" print("Initial cell:", initial_cell) print("Final cell: ", self.crystal.get_unit_cell()) print("Initial orientation: ", fmt % tuple(initial_orientation)) print("Final orientation: ", fmt % tuple(self.crystal.get_U())) # Print RMSD Xobs, Yobs, _ = self.reflections["xyzobs.px.value"].parts() Xcal, Ycal, _ = self.reflections["xyzcal.px"].parts() rmsd_x = sqrt(flex.sum((Xcal - Xobs)**2) / len(Xcal)) rmsd_y = sqrt(flex.sum((Ycal - Yobs)**2) / len(Ycal)) print("RMSD X, Y (px): %f, %f" % (rmsd_x, rmsd_y))
def create_refiner(params, reflections, experiments): # Only parameterise the crystal unit cell det_params = None beam_params = None xlo_params = None xluc_params = [] for crystal in experiments.crystals(): exp_ids = experiments.indices(crystal) xluc_params.append( CrystalUnitCellParameterisation(crystal, experiment_ids=exp_ids) ) # Two theta prediction equation parameterisation pred_param = TwoThetaPredictionParameterisation( experiments, det_params, beam_params, xlo_params, xluc_params ) param_reporter = ParameterReporter( det_params, beam_params, xlo_params, xluc_params ) # ReflectionManager, currently without outlier rejection # Note: If not all reflections are used, then the filtering must be # communicated to generate_cif/mmcif() to be included in the CIF file! refman = TwoThetaReflectionManager( reflections, experiments, outlier_detector=None ) # Reflection predictor ref_predictor = TwoThetaExperimentsPredictor(experiments) # Two theta target target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param) # Switch on correlation matrix tracking if a correlation plot is requested journal = None if params.output.correlation_plot.filename is not None: journal = refinery_phil_scope.extract().refinery.journal journal.track_parameter_correlation = True # Minimisation engine - hardcoded to LevMar for now. refinery = Refinery( target=target, prediction_parameterisation=pred_param, log=None, tracking=journal, max_iterations=20, ) # Refiner refiner = Refiner( experiments=experiments, pred_param=pred_param, param_reporter=param_reporter, refman=refman, target=target, refinery=refinery, ) return refiner
def __init__(self, reflection_file, experiment_file): # make a pie data = pickle.load(open(reflection_file, "rb")) print("%d reflections" % data.size()) self.data = data.select(data["intensity.sum.variance"] > 0) i = data["intensity.sum.value"] v = data["intensity.sum.variance"] s = flex.sqrt(v) self.i_s = i / s from dxtbx.model.experiment_list import ExperimentListFactory expt = ExperimentListFactory.from_json_file(experiment_file) crystal = expt.crystals()[0] self.s0 = matrix.col(expt.beams()[0].get_s0()) from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalUnitCellParameterisation, CrystalOrientationParameterisation, ) self.crystal = crystal self.cucp = CrystalUnitCellParameterisation(crystal) self.cop = CrystalOrientationParameterisation(crystal) # 0-point and deltas values = flex.double(self.cucp.get_param_vals() + self.cop.get_param_vals()) offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] + [0.1, 0.1, 0.1]) initial = crystal.get_unit_cell() initial_score = self.target(values) doohicky = simple_simplex(values, offset, self, 2000) best = doohicky.get_solution() print("Initial cell:", initial) print("Final cell: ", crystal.get_unit_cell()) print("Score change", initial_score, self.target(best, do_print=False)) self.best = best
def __init__( self, experiment, mosaicity_parameterisation, wavelength_parameterisation=None, fix_mosaic_spread=False, fix_wavelength_spread=True, fix_unit_cell=False, fix_orientation=False, ): """ Initialise the model state """ # Save the crystal model self.experiment = experiment self.crystal = experiment.crystal # The U and P parameterisation self._U_parameterisation = CrystalOrientationParameterisation( self.crystal) self._B_parameterisation = CrystalUnitCellParameterisation( self.crystal) # The M and L parameterisations self._M_parameterisation = mosaicity_parameterisation self._L_parameterisation = wavelength_parameterisation # Set the flags to fix parameters self._is_mosaic_spread_fixed = fix_mosaic_spread self._is_wavelength_spread_fixed = fix_wavelength_spread self._is_unit_cell_fixed = fix_unit_cell self._is_orientation_fixed = fix_orientation # Check wavelength parameterisation if not self.is_wavelength_spread_fixed: assert self._L_parameterisation is not None
def generate_data(experiments, reflections): from random import seed seed(0) index = randint(0, len(reflections)) h = reflections[index]["miller_index"] s0 = matrix.col(experiments[0].beam.get_s0()) U_param = CrystalOrientationParameterisation(experiments[0].crystal) B_param = CrystalUnitCellParameterisation(experiments[0].crystal) U = matrix.sqr(experiments[0].crystal.get_U()) B = matrix.sqr(experiments[0].crystal.get_B()) r = U * B * matrix.col(h) s2 = s0 + r mobs = (s2 + matrix.col((uniform(0, 1e-3), uniform( 0, 1e-3), uniform(0, 1e-3)))).normalize() * s0.length() b1, b2, b3, b4, b5, b6 = ( uniform(1e-3, 3e-3), uniform(0.0, 1e-3), uniform(1e-3, 3e-3), uniform(0.0, 1e-3), uniform(0.0, 1e-3), uniform(1e-3, 3e-3), ) params = (b1, b2, b3, b4, b5, b6) S_param = SimpleMosaicityParameterisation(params) L_param = (uniform(1e-3, 2e-3), ) W_param = (uniform(1e-3, 2e-3), uniform(0, 1e-3), uniform(1e-3, 2e-3)) ctot = randint(100, 1000) T = matrix.sqr( (uniform(1e-3, 2e-3), 0, uniform(1e-6, 2e-6), uniform(1e-3, 2e-3))) Sobs = T * T.transpose() params = [S_param, U_param, B_param, L_param, W_param] return params, s0, h, ctot, mobs, Sobs
def generate_data(experiments, reflections): from random import seed seed(0) index = randint(0, len(reflections)) h = reflections[index]["miller_index"] s0 = matrix.col(experiments[0].beam.get_s0()) U_param = CrystalOrientationParameterisation(experiments[0].crystal) B_param = CrystalUnitCellParameterisation(experiments[0].crystal) U = matrix.sqr(experiments[0].crystal.get_U()) B = matrix.sqr(experiments[0].crystal.get_B()) r = U * B * matrix.col(h) s2 = s0 + r mobs = (s2 + matrix.col((uniform(0, 1e-3), uniform( 0, 1e-3), uniform(0, 1e-3)))).normalize() * s0.length() mobs = np.array([uniform(0, 1e-3), uniform(0, 1e-3)]) sp = s2.normalize() * s0.length() b1, b2, b3, b4, b5, b6 = ( uniform(1e-3, 3e-3), uniform(0.0, 1e-3), uniform(1e-3, 3e-3), uniform(0.0, 1e-3), uniform(0.0, 1e-3), uniform(1e-3, 3e-3), ) S_param = (b1, b2, b3, b4, b5, b6) L_param = (uniform(1e-3, 2e-3), ) ctot = randint(100, 1000) T = np.array((uniform(1e-3, 2e-3), 0, uniform(1e-6, 2e-6), uniform(1e-3, 2e-3))).reshape(2, 2) Sobs = np.matmul(T, T.T) params = [S_param, U_param, B_param, L_param] return params, s0, sp, h, ctot, mobs, Sobs
assert approx_equal(im_width, 0.1 * pi / 180.) # Build an experiment list experiments = ExperimentList() experiments.append(Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None)) ########################### # Parameterise the models # ########################### det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) # TEMPORARY TESTING HERE from dials.algorithms.refinement.restraints.restraints import SingleUnitCellTie uct = SingleUnitCellTie(xluc_param, [None]*6, [None]*6) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) # Fix crystal parameters #xluc_param.set_fixed([True, True, True, True, True, True]) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ########################################################################
local_overrides=override, verbose=False) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam ########################### # Parameterise the models # ########################### det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ######################################################################## # Build a mock scan for a 180 degree sweep sf = scan_factory() myscan = sf.make_scan(image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=range(1800),
sweep_range = (0., pi / 5.) from dxtbx.model.scan import scan_factory sf = scan_factory() myscan = sf.make_scan(image_range=(1, 720), exposure_times=0.1, oscillation=(0, 0.1), epochs=range(720), deg=True) #### Create parameterisations of these models det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) # Create an ExperimentList experiments = ExperimentList() experiments.append( Experiment(beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None)) #### Unit tests # Build a prediction parameterisation pred_param = XYPhiPredictionParameterisation(
def test(dials_regression): from libtbx.phil import parse from libtbx.test_utils import approx_equal from scitbx import matrix from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalOrientationParameterisation, CrystalUnitCellParameterisation, ) # Get modules to build models and minimiser using PHIL from dials.tests.algorithms.refinement import setup_geometry # Symmetry constrained parameterisation for the unit cell DEG2RAD = math.pi / 180.0 RAD2DEG = 180.0 / math.pi master_phil = parse( """ include scope dials.tests.algorithms.refinement.geometry_phil include scope dials.tests.algorithms.refinement.minimiser_phil """, process_includes=True, ) # make cell more oblique args = [ "a.direction.close_to.sd=5", "b.direction.close_to.sd=5", "c.direction.close_to.sd=5", ] models = setup_geometry.Extract(master_phil, cmdline_args=args) crystal = models.crystal # a hexagonal crystal is a good test case for behaviour of oblique cells do_hexagonal = True if do_hexagonal: from dxtbx.model.experiment_list import ExperimentListFactory experiments = ExperimentListFactory.from_json_file( os.path.join( dials_regression, "refinement_test_data", "multi_stills", "combined_experiments.json", ), check_format=False, ) crystal = experiments[0].crystal # derive finite difference gradients of various quantities wrt each param def check_fd_gradients(parameterisation): mp = parameterisation p_vals = mp.get_param_vals() deltas = [1.0e-7 for p in p_vals] assert len(deltas) == len(p_vals) fd_grad = [] # get matrix to unset rotations of unit cell vectors Ut = matrix.sqr(mp.get_model().get_U()).transpose() for i, delta in enumerate(deltas): val = p_vals[i] p_vals[i] -= delta / 2.0 mp.set_param_vals(p_vals) rev_uc = mp.get_model().get_unit_cell().parameters() rev_vec = mp.get_model().get_real_space_vectors() rev_vec = [Ut * vec for vec in rev_vec] rev_B = matrix.sqr(mp.get_model().get_B()) rev_O = rev_B.transpose().inverse() p_vals[i] += delta mp.set_param_vals(p_vals) fwd_uc = mp.get_model().get_unit_cell().parameters() fwd_vec = mp.get_model().get_real_space_vectors() fwd_vec = [Ut * vec for vec in fwd_vec] fwd_B = matrix.sqr(mp.get_model().get_B()) fwd_O = fwd_B.transpose().inverse() fd_uc = [(f - r) / delta for f, r in zip(fwd_uc, rev_uc)] fd_vec = [(f - r) / delta for f, r in zip(fwd_vec, rev_vec)] fd_B = (fwd_B - rev_B) / delta fd_O = (fwd_O - rev_O) / delta fd_grad.append( { "da_dp": fd_uc[0], "db_dp": fd_uc[1], "dc_dp": fd_uc[2], "daa_dp": fd_uc[3], "dbb_dp": fd_uc[4], "dcc_dp": fd_uc[5], "davec_dp": fd_vec[0], "dbvec_dp": fd_vec[1], "dcvec_dp": fd_vec[2], "dB_dp": fd_B, "dO_dp": fd_O, } ) p_vals[i] = val # return to the initial state mp.set_param_vals(p_vals) return fd_grad assert CrystalOrientationParameterisation(crystal) xluc_param = CrystalUnitCellParameterisation(crystal) from dials.algorithms.refinement.restraints.restraints import SingleUnitCellTie assert SingleUnitCellTie(xluc_param, [0] * 6, [0] * 6) from scitbx.math import angle_derivative_wrt_vectors B = matrix.sqr(crystal.get_B()) O = (B.transpose()).inverse() a, b, c, aa, bb, cc = crystal.get_unit_cell().parameters() aa *= DEG2RAD bb *= DEG2RAD cc *= DEG2RAD Ut = matrix.sqr(crystal.get_U()).transpose() avec, bvec, cvec = [Ut * vec for vec in crystal.get_real_space_vectors()] # calculate d[B^T]/dp dB_dp = xluc_param.get_ds_dp() dBT_dp = [dB.transpose() for dB in dB_dp] # calculate d[O]/dp dO_dp = [-O * dBT * O for dBT in dBT_dp] # function to get analytical derivative of angles wrt vectors def dangle(u, v): return [matrix.col(e) for e in angle_derivative_wrt_vectors(u, v)] dalpha_db, dalpha_dc = dangle(bvec, cvec) dbeta_da, dbeta_dc = dangle(avec, cvec) dgamma_da, dgamma_db = dangle(avec, bvec) # get all FD derivatives fd_grad = check_fd_gradients(xluc_param) # look at each parameter for i, dO in enumerate(dO_dp): # print # print "***** PARAMETER {0} *****".format(i) # print "dB_dp analytical" # print dB_dp[i] # print "dB_dp FD" # print fd_grad[i]['dB_dp'] # print # dB_dp is good. What about dO_dp? # print "O MATRIX" # print "dO_dp analytical" # print dO.round(6) # print "dO_dp FD" # print fd_grad[i]['dO_dp'].round(6) # print assert approx_equal(dO, fd_grad[i]["dO_dp"]) # extract derivatives of each unit cell vector wrt p dav_dp, dbv_dp, dcv_dp = dO.transpose().as_list_of_lists() dav_dp = matrix.col(dav_dp) dbv_dp = matrix.col(dbv_dp) dcv_dp = matrix.col(dcv_dp) # check these are correct vs FD # print "CELL VECTORS" # diff = dav_dp - fd_grad[i]['davec_dp'] # print 2 * diff.length() / (dav_dp.length() + fd_grad[i]['davec_dp'].length()) * 100 # print 'davec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dav_dp.elems) # print 'davec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['davec_dp'].elems) assert approx_equal(dav_dp, fd_grad[i]["davec_dp"]) # diff = dbv_dp - fd_grad[i]['dbvec_dp'] # print 2 * diff.length() / (dbv_dp.length() + fd_grad[i]['dbvec_dp'].length()) * 100 # print 'dbvec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dbv_dp.elems) # print 'dbvec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['dbvec_dp'].elems) assert approx_equal(dbv_dp, fd_grad[i]["dbvec_dp"]) # diff = dcv_dp - fd_grad[i]['dcvec_dp'] # print 2 * diff.length() / (dcv_dp.length() + fd_grad[i]['dcvec_dp'].length()) * 100 # print 'dcvec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dcv_dp.elems) # print 'dcvec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['dcvec_dp'].elems) # print assert approx_equal(dcv_dp, fd_grad[i]["dcvec_dp"]) # print "CELL LENGTHS" da_dp = 1.0 / a * avec.dot(dav_dp) # print "d[a]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(da_dp, fd_grad[i]['da_dp'], i) assert approx_equal(da_dp, fd_grad[i]["da_dp"]) db_dp = 1.0 / b * bvec.dot(dbv_dp) # print "d[b]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(db_dp, fd_grad[i]['db_dp'], i) assert approx_equal(db_dp, fd_grad[i]["db_dp"]) dc_dp = 1.0 / c * cvec.dot(dcv_dp) # print "d[c]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dc_dp, fd_grad[i]['dc_dp'], i) assert approx_equal(dc_dp, fd_grad[i]["dc_dp"]) # print # print "CELL ANGLES" daa_dp = RAD2DEG * (dbv_dp.dot(dalpha_db) + dcv_dp.dot(dalpha_dc)) dbb_dp = RAD2DEG * (dav_dp.dot(dbeta_da) + dcv_dp.dot(dbeta_dc)) dcc_dp = RAD2DEG * (dav_dp.dot(dgamma_da) + dbv_dp.dot(dgamma_db)) # print "d[alpha]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(daa_dp, fd_grad[i]['daa_dp'], i) # print "d[beta]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dbb_dp, fd_grad[i]['dbb_dp'], i) # print "d[gamma]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dcc_dp, fd_grad[i]['dcc_dp'], i) assert approx_equal(daa_dp, fd_grad[i]["daa_dp"]) assert approx_equal(dbb_dp, fd_grad[i]["dbb_dp"]) assert approx_equal(dcc_dp, fd_grad[i]["dcc_dp"])
def test(args=[]): # Python and cctbx imports from math import pi from scitbx import matrix from scitbx.array_family import flex from libtbx.phil import parse from libtbx.test_utils import approx_equal # Get module to build models using PHIL import dials.test.algorithms.refinement.setup_geometry as setup_geometry # We will set up a mock scan and a mock experiment list from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import ExperimentList, Experiment # Model parameterisations from dials.algorithms.refinement.parameterisation.detector_parameters import \ DetectorParameterisationSinglePanel from dials.algorithms.refinement.parameterisation.beam_parameters import \ BeamParameterisation from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalOrientationParameterisation, CrystalUnitCellParameterisation # Symmetry constrained parameterisation for the unit cell from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import \ symmetrize_reduce_enlarge # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.refinement.prediction import ScansRayPredictor, \ ExperimentsPredictor from dials.algorithms.spot_prediction import ray_intersection from cctbx.sgtbx import space_group, space_group_symbols # Parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.prediction_parameters import \ XYPhiPredictionParameterisation # implicit import # Imports for the target function from dials.algorithms.refinement.target import \ LeastSquaresPositionalResidualWithRmsdCutoff # implicit import ############################# # Setup experimental models # ############################# master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True) models = setup_geometry.Extract( master_phil, cmdline_args=args, local_overrides="geometry.parameters.random_seed = 1") crystal1 = models.crystal models = setup_geometry.Extract( master_phil, cmdline_args=args, local_overrides="geometry.parameters.random_seed = 2") mydetector = models.detector mygonio = models.goniometer crystal2 = models.crystal mybeam = models.beam # Build a mock scan for a 180 degree sweep sf = ScanFactory() myscan = sf.make_scan(image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=range(1800), deg=True) sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) # Build an experiment list experiments = ExperimentList() experiments.append( Experiment(beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=crystal1, imageset=None)) experiments.append( Experiment(beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=crystal2, imageset=None)) assert len(experiments.detectors()) == 1 ########################################################## # Parameterise the models (only for perturbing geometry) # ########################################################## det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xl1o_param = CrystalOrientationParameterisation(crystal1) xl1uc_param = CrystalUnitCellParameterisation(crystal1) xl2o_param = CrystalOrientationParameterisation(crystal2) xl2uc_param = CrystalUnitCellParameterisation(crystal2) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) # Fix crystal parameters #xluc_param.set_fixed([True, True, True, True, True, True]) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ######################################################################## #pred_param = XYPhiPredictionParameterisation(experiments, # [det_param], [s0_param], [xlo_param], [xluc_param]) ################################ # Apply known parameter shifts # ################################ # shift detector by 1.0 mm each translation and 2 mrad each rotation det_p_vals = det_param.get_param_vals() p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2., 2., 2.])] det_param.set_param_vals(p_vals) # shift beam by 2 mrad in free axis s0_p_vals = s0_param.get_param_vals() p_vals = list(s0_p_vals) p_vals[0] += 2. s0_param.set_param_vals(p_vals) # rotate crystal a bit (=2 mrad each rotation) xlo_p_vals = [] for xlo in (xl1o_param, xl2o_param): p_vals = xlo.get_param_vals() xlo_p_vals.append(p_vals) new_p_vals = [a + b for a, b in zip(p_vals, [2., 2., 2.])] xlo.set_param_vals(new_p_vals) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # gamma angle) xluc_p_vals = [] for xluc, xl in ((xl1uc_param, crystal1), ((xl2uc_param, crystal2))): p_vals = xluc.get_param_vals() xluc_p_vals.append(p_vals) cell_params = xl.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(xl.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.e5 for e in S.forward_independent_parameters()]) xluc.set_param_vals(X) ############################# # Generate some reflections # ############################# #print "Reflections will be generated with the following geometry:" #print mybeam #print mydetector #print crystal1 #print crystal2 # All indices in a 2.0 Angstrom sphere for crystal1 resolution = 2.0 index_generator = IndexGenerator( crystal1.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices1 = index_generator.to_array() # All indices in a 2.0 Angstrom sphere for crystal2 resolution = 2.0 index_generator = IndexGenerator( crystal2.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices2 = index_generator.to_array() # Predict rays within the sweep range. Set experiment IDs ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs1 = ray_predictor(indices1, experiment_id=0) obs_refs1['id'] = flex.int(len(obs_refs1), 0) obs_refs2 = ray_predictor(indices1, experiment_id=1) obs_refs2['id'] = flex.int(len(obs_refs2), 1) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs1) obs_refs1 = obs_refs1.select(intersects) intersects = ray_intersection(mydetector, obs_refs2) obs_refs2 = obs_refs2.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs1 = ref_predictor(obs_refs1) obs_refs2 = ref_predictor(obs_refs2) # Set 'observed' centroids from the predicted ones obs_refs1['xyzobs.mm.value'] = obs_refs1['xyzcal.mm'] obs_refs2['xyzobs.mm.value'] = obs_refs2['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs1), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs1), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs1), (im_width / 2.)**2) obs_refs1['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) var_x = flex.double(len(obs_refs2), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs2), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs2), (im_width / 2.)**2) obs_refs2['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) #print "Total number of reflections excited for crystal1", len(obs_refs1) #print "Total number of reflections excited for crystal2", len(obs_refs2) # concatenate reflection lists obs_refs1.extend(obs_refs2) obs_refs = obs_refs1 ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xl1o_param.set_param_vals(xlo_p_vals[0]) xl2o_param.set_param_vals(xlo_p_vals[1]) xl1uc_param.set_param_vals(xluc_p_vals[0]) xl2uc_param.set_param_vals(xluc_p_vals[1]) #print "Initial values of parameters are" #msg = "Parameters: " + "%.5f " * len(pred_param) #print msg % tuple(pred_param.get_param_vals()) #print # make a refiner from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse('')).extract() # in case we want a plot params.refinement.refinery.journal.track_parameter_correlation = True # scan static first from dials.algorithms.refinement.refiner import RefinerFactory refiner = RefinerFactory.from_parameters_data_experiments(params, obs_refs, experiments, verbosity=0) history = refiner.run() # scan varying params.refinement.parameterisation.scan_varying = True refiner = RefinerFactory.from_parameters_data_experiments(params, obs_refs, experiments, verbosity=0) history = refiner.run()
def __init__(self, experiment, reflections, mosaicity, params): """ Perform the refinement :param experiment: The experiment :param reflections: The reflections :param mosaicity: The mosaicity :param params: The parameters """ from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalUnitCellParameterisation, ) from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalOrientationParameterisation, ) from dials_scratch.jmp.stills import Model from dials.array_family import flex from scitbx import simplex from math import sqrt # Store the input self.experiment = experiment self.reflections = reflections self.mosaicity = mosaicity self.params = params # Get the data and image mask data = self.experiment.imageset.get_raw_data(0)[0] mask = self.experiment.imageset.get_mask(0)[0] # Initialise the model self.model = Model( beam=self.experiment.beam, detector=self.experiment.detector, crystal=self.experiment.crystal, reflections=self.reflections, image_data=data, image_mask=mask, mosaicity=self.mosaicity, bandpass=0.0, foreground_limit=0.3, background_limit=0.5, num_samples=self.params.refinement.profile.num_samples, ) # Get the crystal model and the parameterisation self.crystal = self.experiment.crystal self.cucp = CrystalUnitCellParameterisation(self.crystal) self.cop = CrystalOrientationParameterisation(self.crystal) # Get the current values and generate some offsets values = flex.double(self.cucp.get_param_vals() + self.cop.get_param_vals()) offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] + [0.1, 0.1, 0.1]) # The optimization history self.history = [] # Get the initial cell and initial score initial_cell = self.crystal.get_unit_cell() initial_score = self.target(values) # Perform the optimization optimizer = simple_simplex(values, offset, self, 2000) result = optimizer.get_solution() print("Initial cell:", initial_cell) print("Final cell: ", self.crystal.get_unit_cell()) # Compute RMSD xcal, ycal, _ = self.model.observed().parts() xobs, yobs, _ = self.model.predicted().parts() rmsd_x = sqrt(flex.sum((xcal - xobs)**2) / len(xcal)) rmsd_y = sqrt(flex.sum((ycal - yobs)**2) / len(ycal)) print("RMSD X, Y (px): %f, %f" % (rmsd_x, rmsd_y))
def tst_use_in_stills_parameterisation_for_crystal(crystal_param=0): # test use of analytical expression in stills prediction parameterisation from scitbx import matrix from math import pi, sqrt, atan2 import random print print "Test use of analytical expressions in stills prediction " + "parameterisation for crystal parameters" # crystal model from dxtbx.model.crystal import crystal_model from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalOrientationParameterisation, CrystalUnitCellParameterisation, ) crystal = crystal_model((20, 0, 0), (0, 30, 0), (0, 0, 40), space_group_symbol="P 1") # random reorientation e = matrix.col((random.random(), random.random(), random.random())).normalize() angle = random.random() * 180 crystal.rotate_around_origin(e, angle) wl = 1.1 s0 = matrix.col((0, 0, 1 / wl)) s0u = s0.normalize() # these are stills, but need a rotation axis for the Reeke algorithm axis = matrix.col((1, 0, 0)) # crystal parameterisations xlop = CrystalOrientationParameterisation(crystal) xlucp = CrystalUnitCellParameterisation(crystal) # Find some reflections close to the Ewald sphere from dials.algorithms.spot_prediction.reeke import reeke_model U = crystal.get_U() B = crystal.get_B() UB = U * B dmin = 4 hkl = reeke_model(UB, UB, axis, s0, dmin, margin=1).generate_indices() # choose first reflection for now, calc quantities relating to q h = matrix.col(hkl[0]) q = UB * h q0 = q.normalize() q_scalar = q.length() qq = q_scalar * q_scalar # calculate the axis of closest rotation e1 = q0.cross(s0u).normalize() # calculate c0, a vector orthogonal to s0u and e1 c0 = s0u.cross(e1).normalize() # calculate q1 q1 = q0.cross(e1).normalize() # calculate DeltaPsi a = 0.5 * qq * wl b = sqrt(qq - a * a) r = -1.0 * a * s0u + b * c0 DeltaPsi = -1.0 * atan2(r.dot(q1), r.dot(q0)) # Checks on the reflection prediction from libtbx.test_utils import approx_equal # 1. check r is on the Ewald sphere s1 = s0 + r assert approx_equal(s1.length(), s0.length()) # 2. check DeltaPsi is correct tst = q.rotate_around_origin(e1, DeltaPsi) assert approx_equal(tst, r) # choose the derivative with respect to a particular parameter. if crystal_param < 3: dU_dp = xlop.get_ds_dp()[crystal_param] dq = dU_dp * B * h else: dB_dp = xlucp.get_ds_dp()[crystal_param - 3] dq = U * dB_dp * h # NKS method of calculating d[q0]/dp q_dot_dq = q.dot(dq) dqq = 2.0 * q_dot_dq dq_scalar = dqq / q_scalar dq0_dp = (q_scalar * dq - (q_dot_dq * q0)) / qq # orthogonal to q0, as expected. print "NKS [q0].(d[q0]/dp) = {0} (should be 0.0)".format(q0.dot(dq0_dp)) # intuitive method of calculating d[q0]/dp, based on the fact that # it must be orthogonal to q0, i.e. in the plane containing q1 and e1 scaled = dq / q.length() dq0_dp = scaled.dot(q1) * q1 + scaled.dot(e1) * e1 # orthogonal to q0, as expected. print "DGW [q0].(d[q0]/dp) = {0} (should be 0.0)".format(q0.dot(dq0_dp)) # So it doesn't matter which method I use to calculate d[q0]/dp, as # both methods give the same results # use the fact that -e1 == q0.cross(q1) to redefine the derivative d[e1]/dp # from Sauter et al. (2014) (A.22) de1_dp = -1.0 * dq0_dp.cross(q1) # this *is* orthogonal to e1, as expected. print "[e1].(d[e1]/dp) = {0} (should be 0.0)".format(e1.dot(de1_dp)) # calculate (d[r]/d[e1])(d[e1]/dp) analytically from scitbx.array_family import flex from dials_refinement_helpers_ext import dRq_de dr_de1 = matrix.sqr(dRq_de(flex.double([DeltaPsi]), flex.vec3_double([e1]), flex.vec3_double([q]))[0]) print "Analytical calculation for (d[r]/d[e1])(d[e1]/dp):" print dr_de1 * de1_dp # now calculate using finite differences. dp = 1.0e-8 del_e1 = de1_dp * dp e1f = e1 + del_e1 * 0.5 rfwd = q.rotate_around_origin(e1f, DeltaPsi) e1r = e1 - del_e1 * 0.5 rrev = q.rotate_around_origin(e1r, DeltaPsi) print "Finite difference estimate for (d[r]/d[e1])(d[e1]/dp):" print (rfwd - rrev) * (1 / dp) print "These are essentially the same :-)"
def test_fd_derivatives(): """Test derivatives of the prediction equation""" from libtbx.phil import parse # Import model builder from dials.test.algorithms.refinement.setup_geometry import Extract # Create models overrides = """geometry.parameters.crystal.a.length.range = 10 50 geometry.parameters.crystal.b.length.range = 10 50 geometry.parameters.crystal.c.length.range = 10 50""" master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True, ) models = Extract(master_phil, overrides) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam # Build a mock scan for a 72 degree sequence from dxtbx.model import ScanFactory sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 720), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(720)), deg=True, ) # Create a parameterisation of the crystal unit cell from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalUnitCellParameterisation, ) xluc_param = CrystalUnitCellParameterisation(mycrystal) # Create an ExperimentList experiments = ExperimentList() experiments.append( Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None, )) # Build a prediction parameterisation for two theta prediction pred_param = TwoThetaPredictionParameterisation( experiments, detector_parameterisations=None, beam_parameterisations=None, xl_orientation_parameterisations=None, xl_unit_cell_parameterisations=[xluc_param], ) # Generate some reflections obs_refs, ref_predictor = generate_reflections(experiments) # Build a ReflectionManager with overloads for handling 2theta residuals refman = TwoThetaReflectionManager(obs_refs, experiments, outlier_detector=None) # Build a TwoThetaExperimentsPredictor ref_predictor = TwoThetaExperimentsPredictor(experiments) # Make a target for the least squares 2theta residual target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param) # Keep only reflections that pass inclusion criteria and have predictions reflections = refman.get_matches() # Get analytical gradients an_grads = pred_param.get_gradients(reflections) # Get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.0e-7] * len(p_vals) for i in range(len(deltas)): val = p_vals[i] p_vals[i] -= deltas[i] / 2.0 pred_param.set_param_vals(p_vals) target.predict() reflections = refman.get_matches() rev_state = reflections["2theta_resid"].deep_copy() p_vals[i] += deltas[i] pred_param.set_param_vals(p_vals) target.predict() reflections = refman.get_matches() fwd_state = reflections["2theta_resid"].deep_copy() p_vals[i] = val fd = fwd_state - rev_state fd /= deltas[i] # compare with analytical calculation assert approx_equal(fd, an_grads[i]["d2theta_dp"], eps=1.0e-6) # return to the initial state pred_param.set_param_vals(p_vals)
def create_refiner(params, reflections, experiments): from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalUnitCellParameterisation from dials.algorithms.refinement.parameterisation.parameter_report import \ ParameterReporter from dials.algorithms.refinement.two_theta_refiner import \ TwoThetaReflectionManager, TwoThetaTarget, TwoThetaExperimentsPredictor, \ TwoThetaPredictionParameterisation verb = params.refinement.verbosity # Only parameterise the crystal unit cell det_params = None beam_params = None xlo_params = None xluc_params = [] for icrystal, crystal in enumerate(experiments.crystals()): exp_ids = experiments.indices(crystal) xluc_params.append( CrystalUnitCellParameterisation(crystal, experiment_ids=exp_ids)) # Two theta prediction equation parameterisation pred_param = TwoThetaPredictionParameterisation( experiments, det_params, beam_params, xlo_params, xluc_params) param_reporter = ParameterReporter(det_params, beam_params, xlo_params, xluc_params) # ReflectionManager, currently without outlier rejection # Note: If not all reflections are used, then the filtering must be # communicated to generate_cif/mmcif() to be included in the CIF file! refman = TwoThetaReflectionManager(reflections, experiments, outlier_detector=None, verbosity=verb) # Reflection predictor ref_predictor = TwoThetaExperimentsPredictor(experiments) # Two theta target target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param) # Switch on correlation matrix tracking if a correlation plot is requested journal = None if params.output.correlation_plot.filename is not None: from dials.algorithms.refinement.engine import refinery_phil_scope journal = refinery_phil_scope.extract().refinery.journal journal.track_parameter_correlation = True # Minimisation engine - hardcoded to LevMar for now. from dials.algorithms.refinement.engine \ import LevenbergMarquardtIterations as Refinery refinery = Refinery(target=target, prediction_parameterisation=pred_param, log=None, verbosity=verb, tracking=journal, max_iterations=20) # Refiner from dials.algorithms.refinement.refiner import Refiner refiner = Refiner(reflections=reflections, experiments=experiments, pred_param=pred_param, param_reporter=param_reporter, refman=refman, target=target, refinery=refinery, verbosity=verb) return refiner
def test(args=[]): ############################# # Setup experimental models # ############################# master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True, ) models = setup_geometry.Extract(master_phil, cmdline_args=args) single_panel_detector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam # Make a 3x3 multi panel detector filling the same space as the existing # single panel detector. Each panel of the multi-panel detector has pixels with # 1/3 the length dimensions of the single panel. multi_panel_detector = Detector() for x in range(3): for y in range(3): new_panel = make_panel_in_array((x, y), single_panel_detector[0]) multi_panel_detector.add_panel(new_panel) # Build a mock scan for a 180 degree sweep sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(1800)), deg=True, ) sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0.0, pi) assert approx_equal(im_width, 0.1 * pi / 180.0) # Build ExperimentLists experiments_single_panel = ExperimentList() experiments_multi_panel = ExperimentList() experiments_single_panel.append( Experiment( beam=mybeam, detector=single_panel_detector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None, )) experiments_multi_panel.append( Experiment( beam=mybeam, detector=multi_panel_detector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None, )) ########################### # Parameterise the models # ########################### det_param = DetectorParameterisationSinglePanel(single_panel_detector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) multi_det_param = DetectorParameterisationMultiPanel( multi_panel_detector, mybeam) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) # Fix crystal parameters # xluc_param.set_fixed([True, True, True, True, True, True]) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ######################################################################## pred_param = XYPhiPredictionParameterisation(experiments_single_panel, [det_param], [s0_param], [xlo_param], [xluc_param]) pred_param2 = XYPhiPredictionParameterisation( experiments_multi_panel, [multi_det_param], [s0_param], [xlo_param], [xluc_param], ) ################################ # Apply known parameter shifts # ################################ # shift detectors by 1.0 mm each translation and 2 mrad each rotation det_p_vals = det_param.get_param_vals() p_vals = [ a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0]) ] det_param.set_param_vals(p_vals) multi_det_p_vals = multi_det_param.get_param_vals() p_vals = [ a + b for a, b in zip(multi_det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0]) ] multi_det_param.set_param_vals(p_vals) # shift beam by 2 mrad in free axis s0_p_vals = s0_param.get_param_vals() p_vals = list(s0_p_vals) p_vals[0] += 2.0 s0_param.set_param_vals(p_vals) # rotate crystal a bit (=2 mrad each rotation) xlo_p_vals = xlo_param.get_param_vals() p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])] xlo_param.set_param_vals(p_vals) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # gamma angle) xluc_p_vals = xluc_param.get_param_vals() cell_params = mycrystal.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(mycrystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator( mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() # for the reflection predictor, it doesn't matter which experiment list is # passed, as the detector is not used ref_predictor = ScansRayPredictor(experiments_single_panel, sweep_range) # get two sets of identical reflections obs_refs = ref_predictor(indices) obs_refs2 = ref_predictor(indices) for r1, r2 in zip(obs_refs, obs_refs2): assert r1["s1"] == r2["s1"] # get the panel intersections sel = ray_intersection(single_panel_detector, obs_refs) obs_refs = obs_refs.select(sel) sel = ray_intersection(multi_panel_detector, obs_refs2) obs_refs2 = obs_refs2.select(sel) assert len(obs_refs) == len(obs_refs2) # Set 'observed' centroids from the predicted ones obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"] obs_refs2["xyzobs.mm.value"] = obs_refs2["xyzcal.mm"] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180.0 px_size = single_panel_detector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2) # set the variances and frame numbers obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) obs_refs2["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) # Add in flags and ID columns by copying into standard reflection tables tmp = flex.reflection_table.empty_standard(len(obs_refs)) tmp.update(obs_refs) obs_refs = tmp tmp = flex.reflection_table.empty_standard(len(obs_refs2)) tmp.update(obs_refs2) obs_refs2 = tmp ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) multi_det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) xluc_param.set_param_vals(xluc_p_vals) ##################################### # Select reflections for refinement # ##################################### refman = ReflectionManager(obs_refs, experiments_single_panel) refman2 = ReflectionManager(obs_refs, experiments_multi_panel) ############################### # Set up the target functions # ############################### mytarget = LeastSquaresPositionalResidualWithRmsdCutoff( experiments_single_panel, ScansExperimentsPredictor(experiments_single_panel), refman, pred_param, restraints_parameterisation=None, ) mytarget2 = LeastSquaresPositionalResidualWithRmsdCutoff( experiments_multi_panel, ScansExperimentsPredictor(experiments_multi_panel), refman2, pred_param2, restraints_parameterisation=None, ) ################################# # Set up the refinement engines # ################################# refiner = setup_minimiser.Extract(master_phil, mytarget, pred_param, cmdline_args=args).refiner refiner2 = setup_minimiser.Extract(master_phil, mytarget2, pred_param2, cmdline_args=args).refiner refiner.run() # reset parameters and run refinement with the multi panel detector s0_param.set_param_vals(s0_p_vals) multi_det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) xluc_param.set_param_vals(xluc_p_vals) refiner2.run() # same number of steps assert refiner.get_num_steps() == refiner2.get_num_steps() # same rmsds for rmsd, rmsd2 in zip(refiner.history["rmsd"], refiner2.history["rmsd"]): assert approx_equal(rmsd, rmsd2) # same parameter values each step for params, params2 in zip(refiner.history["parameter_vector"], refiner.history["parameter_vector"]): assert approx_equal(params, params2)
def create_refiner(params, reflections, experiments): from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalUnitCellParameterisation from dials.algorithms.refinement.parameterisation.parameter_report import \ ParameterReporter from dials.algorithms.refinement.two_theta_refiner import ( TwoThetaReflectionManager, TwoThetaTarget, TwoThetaExperimentsPredictor, TwoThetaPredictionParameterisation) verb = params.refinement.verbosity # Only parameterise the crystal unit cell det_params = None beam_params = None xlo_params = None xluc_params = [] for icrystal, crystal in enumerate(experiments.crystals()): exp_ids = experiments.indices(crystal) xluc_params.append( CrystalUnitCellParameterisation(crystal, experiment_ids=exp_ids)) # Two theta prediction equation parameterisation pred_param = TwoThetaPredictionParameterisation( experiments, det_params, beam_params, xlo_params, xluc_params) param_reporter = ParameterReporter(det_params, beam_params, xlo_params, xluc_params) # ReflectionManager, currently without outlier rejection # Note: If not all reflections are used, then the filtering must be # communicated to generate_cif/mmcif() to be included in the CIF file! refman = TwoThetaReflectionManager(reflections, experiments, outlier_detector=None, verbosity=verb) # Reflection predictor ref_predictor = TwoThetaExperimentsPredictor(experiments) # Two theta target target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param) # Do a correlation plot? tpc = params.output.correlation_plot.filename is not None # Minimisation engine - FIXME not many choices exposed yet. Do we want # more cowbell? from dials.algorithms.refinement.engine \ import LevenbergMarquardtIterations as Refinery refinery = Refinery(target=target, prediction_parameterisation=pred_param, log=None, verbosity=verb, track_step=False, track_gradient=False, track_parameter_correlation=tpc, max_iterations=20) # Refiner from dials.algorithms.refinement.refiner import Refiner refiner = Refiner(reflections=reflections, experiments=experiments, pred_param=pred_param, param_reporter=param_reporter, refman=refman, target=target, refinery=refinery, verbosity=verb) return refiner
def test1(): dials_regression = libtbx.env.find_in_repositories( relative_path="dials_regression", test=os.path.isdir) # use a datablock that contains a CS-PAD detector description data_dir = os.path.join(dials_regression, "refinement_test_data", "hierarchy_test") datablock_path = os.path.join(data_dir, "datablock.json") assert os.path.exists(datablock_path) # load models from dxtbx.datablock import DataBlockFactory datablock = DataBlockFactory.from_serialized_format(datablock_path, check_format=False) im_set = datablock[0].extract_imagesets()[0] from copy import deepcopy detector = deepcopy(im_set.get_detector()) beam = im_set.get_beam() # we'll invent a crystal, goniometer and scan for this test from dxtbx.model import Crystal crystal = Crystal((40., 0., 0.), (0., 40., 0.), (0., 0., 40.), space_group_symbol="P1") from dxtbx.model import GoniometerFactory goniometer = GoniometerFactory.known_axis((1., 0., 0.)) # Build a mock scan for a 180 degree sweep from dxtbx.model import ScanFactory sf = ScanFactory() scan = sf.make_scan(image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=range(1800), deg=True) sweep_range = scan.get_oscillation_range(deg=False) im_width = scan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) from dxtbx.model.experiment_list import ExperimentList, Experiment # Build an experiment list experiments = ExperimentList() experiments.append( Experiment(beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal, imageset=None)) # simulate some reflections refs, ref_predictor = generate_reflections(experiments) # move the detector quadrants apart by 2mm both horizontally and vertically from dials.algorithms.refinement.parameterisation \ import DetectorParameterisationHierarchical det_param = DetectorParameterisationHierarchical(detector, level=1) det_p_vals = det_param.get_param_vals() p_vals = list(det_p_vals) p_vals[1] += 2 p_vals[2] -= 2 p_vals[7] += 2 p_vals[8] += 2 p_vals[13] -= 2 p_vals[14] += 2 p_vals[19] -= 2 p_vals[20] -= 2 det_param.set_param_vals(p_vals) # reparameterise the detector at the new perturbed geometry det_param = DetectorParameterisationHierarchical(detector, level=1) # parameterise other models from dials.algorithms.refinement.parameterisation.beam_parameters import \ BeamParameterisation from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalOrientationParameterisation, CrystalUnitCellParameterisation beam_param = BeamParameterisation(beam, goniometer) xlo_param = CrystalOrientationParameterisation(crystal) xluc_param = CrystalUnitCellParameterisation(crystal) # fix beam beam_param.set_fixed([True] * 3) # fix crystal xluc_param.set_fixed([True] * 6) xlo_param.set_fixed([True] * 3) # parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.prediction_parameters import \ XYPhiPredictionParameterisation from dials.algorithms.refinement.parameterisation.parameter_report import \ ParameterReporter pred_param = XYPhiPredictionParameterisation(experiments, [det_param], [beam_param], [xlo_param], [xluc_param]) param_reporter = ParameterReporter([det_param], [beam_param], [xlo_param], [xluc_param]) # reflection manager and target function from dials.algorithms.refinement.target import \ LeastSquaresPositionalResidualWithRmsdCutoff from dials.algorithms.refinement.reflection_manager import ReflectionManager refman = ReflectionManager(refs, experiments, nref_per_degree=20) # set a very tight rmsd target of 1/10000 of a pixel target = LeastSquaresPositionalResidualWithRmsdCutoff( experiments, ref_predictor, refman, pred_param, restraints_parameterisation=None, frac_binsize_cutoff=0.0001) # minimisation engine from dials.algorithms.refinement.engine \ import LevenbergMarquardtIterations as Refinery refinery = Refinery(target=target, prediction_parameterisation=pred_param, log=None, verbosity=0, max_iterations=20) # Refiner from dials.algorithms.refinement.refiner import Refiner refiner = Refiner(reflections=refs, experiments=experiments, pred_param=pred_param, param_reporter=param_reporter, refman=refman, target=target, refinery=refinery, verbosity=0) history = refiner.run() assert history.reason_for_termination == "RMSD target achieved" #compare detector with original detector orig_det = im_set.get_detector() refined_det = refiner.get_experiments()[0].detector from scitbx import matrix import math for op, rp in zip(orig_det, refined_det): # compare the origin vectors by... o1 = matrix.col(op.get_origin()) o2 = matrix.col(rp.get_origin()) # ...their relative lengths assert approx_equal(math.fabs(o1.length() - o2.length()) / o1.length(), 0, eps=1e-5) # ...the angle between them assert approx_equal(o1.accute_angle(o2), 0, eps=1e-5) print "OK" return
def test(init_test): single_panel_detector = init_test.experiments_single_panel.detectors()[0] multi_panel_detector = init_test.experiments_multi_panel.detectors()[0] beam = init_test.experiments_single_panel.beams()[0] gonio = init_test.experiments_single_panel.goniometers()[0] crystal = init_test.experiments_single_panel.crystals()[0] # Parameterise the models det_param = DetectorParameterisationSinglePanel(single_panel_detector) s0_param = BeamParameterisation(beam, gonio) xlo_param = CrystalOrientationParameterisation(crystal) xluc_param = CrystalUnitCellParameterisation(crystal) multi_det_param = DetectorParameterisationMultiPanel(multi_panel_detector, beam) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) # Link model parameterisations together into a parameterisation of the # prediction equation, first for the single panel detector pred_param = XYPhiPredictionParameterisation( init_test.experiments_single_panel, [det_param], [s0_param], [xlo_param], [xluc_param], ) # ... and now for the multi-panel detector pred_param2 = XYPhiPredictionParameterisation( init_test.experiments_multi_panel, [multi_det_param], [s0_param], [xlo_param], [xluc_param], ) ################################ # Apply known parameter shifts # ################################ # shift detectors by 1.0 mm each translation and 2 mrad each rotation det_p_vals = det_param.get_param_vals() p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])] det_param.set_param_vals(p_vals) multi_det_p_vals = multi_det_param.get_param_vals() p_vals = [a + b for a, b in zip(multi_det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])] multi_det_param.set_param_vals(p_vals) # shift beam by 2 mrad in free axis s0_p_vals = s0_param.get_param_vals() p_vals = list(s0_p_vals) p_vals[0] += 2.0 s0_param.set_param_vals(p_vals) # rotate crystal a bit (=2 mrad each rotation) xlo_p_vals = xlo_param.get_param_vals() p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])] xlo_param.set_param_vals(p_vals) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # gamma angle) xluc_p_vals = xluc_param.get_param_vals() cell_params = crystal.get_unit_cell().parameters() cell_params = [a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(crystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) multi_det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) xluc_param.set_param_vals(xluc_p_vals) ##################################### # Select reflections for refinement # ##################################### refman = ReflectionManager( init_test.observations_single_panel, init_test.experiments_single_panel ) refman2 = ReflectionManager( init_test.observations_multi_panel, init_test.experiments_multi_panel ) ############################### # Set up the target functions # ############################### target = LeastSquaresPositionalResidualWithRmsdCutoff( init_test.experiments_single_panel, ScansExperimentsPredictor(init_test.experiments_single_panel), refman, pred_param, restraints_parameterisation=None, ) target2 = LeastSquaresPositionalResidualWithRmsdCutoff( init_test.experiments_multi_panel, ScansExperimentsPredictor(init_test.experiments_multi_panel), refman2, pred_param2, restraints_parameterisation=None, ) ################################# # Set up the refinement engines # ################################# refiner = setup_minimiser.Extract(master_phil, target, pred_param).refiner refiner2 = setup_minimiser.Extract(master_phil, target2, pred_param2).refiner refiner.run() # reset parameters and run refinement with the multi panel detector s0_param.set_param_vals(s0_p_vals) multi_det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) xluc_param.set_param_vals(xluc_p_vals) refiner2.run() # same number of steps assert refiner.get_num_steps() == refiner2.get_num_steps() # same rmsds for rmsd, rmsd2 in zip(refiner.history["rmsd"], refiner2.history["rmsd"]): assert approx_equal(rmsd, rmsd2) # same parameter values each step for params, params2 in zip( refiner.history["parameter_vector"], refiner.history["parameter_vector"] ): assert approx_equal(params, params2)
def test1(): dials_regression = libtbx.env.find_in_repositories( relative_path="dials_regression", test=os.path.isdir) # use a datablock that contains a CS-PAD detector description data_dir = os.path.join(dials_regression, "refinement_test_data", "hierarchy_test") datablock_path = os.path.join(data_dir, "datablock.json") assert os.path.exists(datablock_path) # load models from dxtbx.datablock import DataBlockFactory datablock = DataBlockFactory.from_serialized_format(datablock_path, check_format=False) im_set = datablock[0].extract_imagesets()[0] from copy import deepcopy detector = deepcopy(im_set.get_detector()) beam = im_set.get_beam() # we'll invent a crystal, goniometer and scan for this test from dxtbx.model.crystal import crystal_model crystal = crystal_model((40.,0.,0.) ,(0.,40.,0.), (0.,0.,40.), space_group_symbol = "P1") from dxtbx.model.experiment import goniometer_factory goniometer = goniometer_factory.known_axis((1., 0., 0.)) # Build a mock scan for a 180 degree sweep from dxtbx.model.scan import scan_factory sf = scan_factory() scan = sf.make_scan(image_range = (1,1800), exposure_times = 0.1, oscillation = (0, 0.1), epochs = range(1800), deg = True) sweep_range = scan.get_oscillation_range(deg=False) im_width = scan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) from dxtbx.model.experiment.experiment_list import ExperimentList, Experiment # Build an experiment list experiments = ExperimentList() experiments.append(Experiment( beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal, imageset=None)) # simulate some reflections refs, ref_predictor = generate_reflections(experiments) # move the detector quadrants apart by 2mm both horizontally and vertically from dials.algorithms.refinement.parameterisation \ import DetectorParameterisationHierarchical det_param = DetectorParameterisationHierarchical(detector, level=1) det_p_vals = det_param.get_param_vals() p_vals = list(det_p_vals) p_vals[1] += 2 p_vals[2] -= 2 p_vals[7] += 2 p_vals[8] += 2 p_vals[13] -= 2 p_vals[14] += 2 p_vals[19] -= 2 p_vals[20] -= 2 det_param.set_param_vals(p_vals) # reparameterise the detector at the new perturbed geometry det_param = DetectorParameterisationHierarchical(detector, level=1) # parameterise other models from dials.algorithms.refinement.parameterisation.beam_parameters import \ BeamParameterisation from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalOrientationParameterisation, CrystalUnitCellParameterisation beam_param = BeamParameterisation(beam, goniometer) xlo_param = CrystalOrientationParameterisation(crystal) xluc_param = CrystalUnitCellParameterisation(crystal) # fix beam beam_param.set_fixed([True]*3) # fix crystal xluc_param.set_fixed([True]*6) xlo_param.set_fixed([True]*3) # parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.prediction_parameters import \ XYPhiPredictionParameterisation from dials.algorithms.refinement.parameterisation.parameter_report import \ ParameterReporter pred_param = XYPhiPredictionParameterisation(experiments, [det_param], [beam_param], [xlo_param], [xluc_param]) param_reporter = ParameterReporter([det_param], [beam_param], [xlo_param], [xluc_param]) # reflection manager and target function from dials.algorithms.refinement.target import \ LeastSquaresPositionalResidualWithRmsdCutoff from dials.algorithms.refinement.reflection_manager import ReflectionManager refman = ReflectionManager(refs, experiments, nref_per_degree=20) # set a very tight rmsd target of 1/10000 of a pixel target = LeastSquaresPositionalResidualWithRmsdCutoff(experiments, ref_predictor, refman, pred_param, restraints_parameterisation=None, frac_binsize_cutoff=0.0001) # minimisation engine from dials.algorithms.refinement.engine \ import LevenbergMarquardtIterations as Refinery refinery = Refinery(target = target, prediction_parameterisation = pred_param, log = None, verbosity = 0, track_step = False, track_gradient = False, track_parameter_correlation = False, max_iterations = 20) # Refiner from dials.algorithms.refinement.refiner import Refiner refiner = Refiner(reflections=refs, experiments=experiments, pred_param=pred_param, param_reporter=param_reporter, refman=refman, target=target, refinery=refinery, verbosity=0) history = refiner.run() assert history.reason_for_termination == "RMSD target achieved" #compare detector with original detector orig_det = im_set.get_detector() refined_det = refiner.get_experiments()[0].detector from scitbx import matrix import math for op, rp in zip(orig_det, refined_det): # compare the origin vectors by... o1 = matrix.col(op.get_origin()) o2 = matrix.col(rp.get_origin()) # ...their relative lengths assert approx_equal( math.fabs(o1.length() - o2.length()) / o1.length(), 0, eps=1e-5) # ...the angle between them assert approx_equal(o1.accute_angle(o2), 0, eps=1e-5) print "OK" return
def test_single_crystal_restraints_gradients(): """Simple test with a single triclinic crystal restrained to a target unit cell""" from dxtbx.model.experiment_list import Experiment, ExperimentList from dials.algorithms.refinement.parameterisation.beam_parameters import ( BeamParameterisation, ) from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalOrientationParameterisation, CrystalUnitCellParameterisation, ) from dials.algorithms.refinement.parameterisation.detector_parameters import ( DetectorParameterisationSinglePanel, ) from dials.algorithms.refinement.parameterisation.prediction_parameters import ( XYPhiPredictionParameterisation, ) from dials.test.algorithms.refinement.setup_geometry import Extract overrides = """geometry.parameters.crystal.a.length.range = 10 50 geometry.parameters.crystal.b.length.range = 10 50 geometry.parameters.crystal.c.length.range = 10 50""" master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True, ) models = Extract(master_phil, overrides) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam # Build a mock scan for a 72 degree sequence from dxtbx.model import ScanFactory sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 720), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(720)), deg=True, ) # Create parameterisations of these models det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) # Create an ExperimentList experiments = ExperimentList() experiments.append( Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None, )) # Build a prediction parameterisation pred_param = XYPhiPredictionParameterisation( experiments, detector_parameterisations=[det_param], beam_parameterisations=[s0_param], xl_orientation_parameterisations=[xlo_param], xl_unit_cell_parameterisations=[xluc_param], ) # Build a restraints parameterisation rp = RestraintsParameterisation( detector_parameterisations=[det_param], beam_parameterisations=[s0_param], xl_orientation_parameterisations=[xlo_param], xl_unit_cell_parameterisations=[xluc_param], ) # make a unit cell target sigma = 1.0 uc = mycrystal.get_unit_cell().parameters() target_uc = [random.gauss(e, sigma) for e in uc] rp.add_restraints_to_target_xl_unit_cell(experiment_id=0, values=target_uc, sigma=[sigma] * 6) # get analytical values and gradients vals, grads, weights = rp.get_residuals_gradients_and_weights() assert len(vals) == rp.num_residuals # get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.0e-7] * len(p_vals) fd_grad = [] for i, delta in enumerate(deltas): val = p_vals[i] p_vals[i] -= delta / 2.0 pred_param.set_param_vals(p_vals) rev_state, foo, bar = rp.get_residuals_gradients_and_weights() rev_state = flex.double(rev_state) p_vals[i] += delta pred_param.set_param_vals(p_vals) fwd_state, foo, bar = rp.get_residuals_gradients_and_weights() fwd_state = flex.double(fwd_state) p_vals[i] = val fd = (fwd_state - rev_state) / delta fd_grad.append(fd) # for comparison, fd_grad is a list of flex.doubles, each of which corresponds # to a column of the sparse matrix grads. for i, fd in enumerate(fd_grad): # extract dense column from the sparse matrix an = grads.col(i).as_dense_vector() assert an == pytest.approx(fd, abs=1e-5)
def test_refinement(): '''Test a refinement run''' dials_regression = libtbx.env.find_in_repositories( relative_path="dials_regression", test=os.path.isdir) # Get a beam and detector from a datablock. This one has a CS-PAD, but that # is irrelevant data_dir = os.path.join(dials_regression, "refinement_test_data", "hierarchy_test") datablock_path = os.path.join(data_dir, "datablock.json") assert os.path.exists(datablock_path) # load models from dxtbx.datablock import DataBlockFactory datablock = DataBlockFactory.from_serialized_format(datablock_path, check_format=False) im_set = datablock[0].extract_imagesets()[0] from copy import deepcopy detector = deepcopy(im_set.get_detector()) beam = im_set.get_beam() # Invent a crystal, goniometer and scan for this test from dxtbx.model.crystal import crystal_model crystal = crystal_model((40.,0.,0.) ,(0.,40.,0.), (0.,0.,40.), space_group_symbol = "P1") orig_xl = deepcopy(crystal) from dxtbx.model.experiment import goniometer_factory goniometer = goniometer_factory.known_axis((1., 0., 0.)) # Build a mock scan for a 180 degree sweep from dxtbx.model.scan import scan_factory sf = scan_factory() scan = sf.make_scan(image_range = (1,1800), exposure_times = 0.1, oscillation = (0, 0.1), epochs = range(1800), deg = True) sweep_range = scan.get_oscillation_range(deg=False) im_width = scan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) from dxtbx.model.experiment.experiment_list import ExperimentList, Experiment # Build an experiment list experiments = ExperimentList() experiments.append(Experiment( beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal, imageset=None)) # simulate some reflections refs, _ = generate_reflections(experiments) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # alpha and beta angles) from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalUnitCellParameterisation xluc_param = CrystalUnitCellParameterisation(crystal) xluc_p_vals = xluc_param.get_param_vals() cell_params = crystal.get_unit_cell().parameters() cell_params = [a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1, -0.1, 0.0])] from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import \ symmetrize_reduce_enlarge from scitbx import matrix new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(crystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) # reparameterise the crystal at the perturbed geometry xluc_param = CrystalUnitCellParameterisation(crystal) # Dummy parameterisations for other models beam_param = None xlo_param = None det_param = None # parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.parameter_report import \ ParameterReporter pred_param = TwoThetaPredictionParameterisation(experiments, det_param, beam_param, xlo_param, [xluc_param]) param_reporter = ParameterReporter(det_param, beam_param, xlo_param, [xluc_param]) # reflection manager refman = TwoThetaReflectionManager(refs, experiments, nref_per_degree=20, verbosity=2) # reflection predictor ref_predictor = TwoThetaExperimentsPredictor(experiments) # target function target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param) # minimisation engine from dials.algorithms.refinement.engine \ import LevenbergMarquardtIterations as Refinery refinery = Refinery(target = target, prediction_parameterisation = pred_param, log = None, verbosity = 0, track_step = False, track_gradient = False, track_parameter_correlation = False, max_iterations = 20) # Refiner from dials.algorithms.refinement.refiner import Refiner refiner = Refiner(reflections=refs, experiments=experiments, pred_param=pred_param, param_reporter=param_reporter, refman=refman, target=target, refinery=refinery, verbosity=1) history = refiner.run() # compare crystal with original crystal refined_xl = refiner.get_experiments()[0].crystal #print refined_xl assert refined_xl.is_similar_to(orig_xl, uc_rel_length_tolerance=0.001, uc_abs_angle_tolerance=0.01) #print "Unit cell esds:" #print refined_xl.get_cell_parameter_sd() return
def test(): import random import textwrap from cctbx.uctbx import unit_cell from libtbx.test_utils import approx_equal def random_direction_close_to(vector): return vector.rotate_around_origin( matrix.col((random.random(), random.random(), random.random())).normalize(), random.gauss(0, 1.0), deg=True, ) # make a random P1 crystal and parameterise it a = random.uniform(10, 50) * random_direction_close_to( matrix.col((1, 0, 0))) b = random.uniform(10, 50) * random_direction_close_to( matrix.col((0, 1, 0))) c = random.uniform(10, 50) * random_direction_close_to( matrix.col((0, 0, 1))) xl = Crystal(a, b, c, space_group_symbol="P 1") xl_op = CrystalOrientationParameterisation(xl) xl_ucp = CrystalUnitCellParameterisation(xl) null_mat = matrix.sqr((0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0)) # compare analytical and finite difference derivatives an_ds_dp = xl_op.get_ds_dp() fd_ds_dp = get_fd_gradients(xl_op, [1.0e-6 * pi / 180] * 3) for e, f in zip(an_ds_dp, fd_ds_dp): assert approx_equal((e - f), null_mat, eps=1.0e-6) an_ds_dp = xl_ucp.get_ds_dp() fd_ds_dp = get_fd_gradients(xl_ucp, [1.0e-7] * xl_ucp.num_free()) for e, f in zip(an_ds_dp, fd_ds_dp): assert approx_equal((e - f), null_mat, eps=1.0e-6) # random initial orientations with a random parameter shift at each attempts = 100 for i in range(attempts): # make a random P1 crystal and parameterise it a = random.uniform(10, 50) * random_direction_close_to( matrix.col((1, 0, 0))) b = random.uniform(10, 50) * random_direction_close_to( matrix.col((0, 1, 0))) c = random.uniform(10, 50) * random_direction_close_to( matrix.col((0, 0, 1))) xl = Crystal(a, b, c, space_group_symbol="P 1") xl_op = CrystalOrientationParameterisation(xl) xl_uc = CrystalUnitCellParameterisation(xl) # apply a random parameter shift to the orientation p_vals = xl_op.get_param_vals() p_vals = random_param_shift( p_vals, [1000 * pi / 9, 1000 * pi / 9, 1000 * pi / 9]) xl_op.set_param_vals(p_vals) # compare analytical and finite difference derivatives xl_op_an_ds_dp = xl_op.get_ds_dp() xl_op_fd_ds_dp = get_fd_gradients(xl_op, [1.0e-5 * pi / 180] * 3) # apply a random parameter shift to the unit cell. We have to # do this in a way that is respectful to metrical constraints, # so don't modify the parameters directly; modify the cell # constants and extract the new parameters cell_params = xl.get_unit_cell().parameters() cell_params = random_param_shift(cell_params, [1.0] * 6) new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(xl.get_space_group()) S.set_orientation(orientation=newB) X = S.forward_independent_parameters() xl_uc.set_param_vals(X) xl_uc_an_ds_dp = xl_ucp.get_ds_dp() # now doing finite differences about each parameter in turn xl_uc_fd_ds_dp = get_fd_gradients(xl_ucp, [1.0e-7] * xl_ucp.num_free()) for j in range(3): assert approx_equal((xl_op_fd_ds_dp[j] - xl_op_an_ds_dp[j]), null_mat, eps=1.0e-6), textwrap.dedent("""\ Failure in try {i} failure for parameter number {j} of the orientation parameterisation with fd_ds_dp = {fd} and an_ds_dp = {an} so that difference fd_ds_dp - an_ds_dp = {diff} """).format( i=i, j=j, fd=xl_op_fd_ds_dp[j], an=xl_op_an_ds_dp[j], diff=xl_op_fd_ds_dp[j] - xl_op_an_ds_dp[j], ) for j in range(xl_ucp.num_free()): assert approx_equal((xl_uc_fd_ds_dp[j] - xl_uc_an_ds_dp[j]), null_mat, eps=1.0e-6), textwrap.dedent("""\ Failure in try {i} failure for parameter number {j} of the unit cell parameterisation with fd_ds_dp = {fd} and an_ds_dp = {an} so that difference fd_ds_dp - an_ds_dp = {diff} """).format( i=i, j=j, fd=xl_uc_fd_ds_dp[j], an=xl_uc_an_ds_dp[j], diff=xl_uc_fd_ds_dp[j] - xl_uc_an_ds_dp[j], )
def test(): # Build models, with a larger crystal than default in order to get plenty of # reflections on the 'still' image overrides = """ geometry.parameters.crystal.a.length.range=40 50; geometry.parameters.crystal.b.length.range=40 50; geometry.parameters.crystal.c.length.range=40 50; geometry.parameters.random_seed = 42""" master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True, ) models = Extract(master_phil, overrides) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam # Build a mock scan for a 3 degree sweep from dxtbx.model import ScanFactory sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 1), exposure_times=0.1, oscillation=(0, 3.0), epochs=list(range(1)), deg=True, ) sweep_range = myscan.get_oscillation_range(deg=False) # Create parameterisations of these models det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) # Create a scans ExperimentList, only for generating reflections experiments = ExperimentList() experiments.append( Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None, )) # Create a stills ExperimentList stills_experiments = ExperimentList() stills_experiments.append( Experiment(beam=mybeam, detector=mydetector, crystal=mycrystal, imageset=None)) # Generate rays - only to work out which hkls are predicted ray_predictor = ScansRayPredictor(experiments, sweep_range) resolution = 2.0 index_generator = IndexGenerator( mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() rays = ray_predictor(indices) # Make a standard reflection_table and copy in the ray data reflections = flex.reflection_table.empty_standard(len(rays)) reflections.update(rays) # Build a standard prediction parameterisation for the stills experiment to do # FD calculation (not used for its analytical gradients) pred_param = StillsPredictionParameterisation( stills_experiments, detector_parameterisations=[det_param], beam_parameterisations=[s0_param], xl_orientation_parameterisations=[xlo_param], xl_unit_cell_parameterisations=[xluc_param], ) # Make a managed SphericalRelpStillsReflectionPredictor reflection predictor # for the first (only) experiment ref_predictor = Predictor(stills_experiments) # Predict these reflections in place. Must do this ahead of calculating # the analytical gradients so quantities like s1 are correct ref_predictor.update() ref_predictor.predict(reflections) # calculate analytical gradients ag = AnalyticalGradients( stills_experiments, detector_parameterisation=det_param, beam_parameterisation=s0_param, xl_orientation_parameterisation=xlo_param, xl_unit_cell_parameterisation=xluc_param, ) an_grads = ag.get_beam_gradients(reflections) an_grads.update(ag.get_crystal_orientation_gradients(reflections)) an_grads.update(ag.get_crystal_unit_cell_gradients(reflections)) # get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.0e-7] * len(p_vals) fd_grads = [] p_names = pred_param.get_param_names() for i, delta in enumerate(deltas): # save parameter value val = p_vals[i] # calc reverse state p_vals[i] -= delta / 2.0 pred_param.set_param_vals(p_vals) ref_predictor.update() ref_predictor.predict(reflections) x, y, _ = reflections["xyzcal.mm"].deep_copy().parts() s1 = reflections["s1"].deep_copy() rev_state = s1 # calc forward state p_vals[i] += delta pred_param.set_param_vals(p_vals) ref_predictor.update() ref_predictor.predict(reflections) x, y, _ = reflections["xyzcal.mm"].deep_copy().parts() s1 = reflections["s1"].deep_copy() fwd_state = s1 # reset parameter to saved value p_vals[i] = val # finite difference - currently for s1 only fd = fwd_state - rev_state inv_delta = 1.0 / delta s1_grads = fd * inv_delta # store gradients fd_grads.append({"name": p_names[i], "ds1": s1_grads}) # return to the initial state pred_param.set_param_vals(p_vals) for i, fd_grad in enumerate(fd_grads): ## compare FD with analytical calculations print("\n\nParameter {0}: {1}".format(i, fd_grad["name"])) print("d[s1]/dp for the first reflection") print("finite diff", fd_grad["ds1"][0]) try: an_grad = an_grads[fd_grad["name"]] except KeyError: continue print("checking analytical vs finite difference gradients for s1") for a, b in zip(fd_grad["ds1"], an_grad["ds1"]): assert a == pytest.approx(b, abs=1e-7)
def test(args=[]): # Python and cctbx imports from math import pi from scitbx import matrix from libtbx.phil import parse from libtbx.test_utils import approx_equal # Import for surgery on reflection_tables from dials.array_family import flex # Get module to build models using PHIL import dials.test.algorithms.refinement.setup_geometry as setup_geometry # We will set up a mock scan and a mock experiment list from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import ExperimentList, Experiment # Crystal parameterisations from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalOrientationParameterisation, CrystalUnitCellParameterisation, ) # Symmetry constrained parameterisation for the unit cell from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.refinement.prediction.managed_predictors import ( ScansRayPredictor, StillsExperimentsPredictor, ) from dials.algorithms.spot_prediction import ray_intersection from cctbx.sgtbx import space_group, space_group_symbols ############################# # Setup experimental models # ############################# master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True, ) # build models, with a larger crystal than default in order to get enough # reflections on the 'still' image param = """ geometry.parameters.crystal.a.length.range=40 50; geometry.parameters.crystal.b.length.range=40 50; geometry.parameters.crystal.c.length.range=40 50; geometry.parameters.random_seed = 42""" models = setup_geometry.Extract(master_phil, cmdline_args=args, local_overrides=param) crystal = models.crystal mydetector = models.detector mygonio = models.goniometer mybeam = models.beam # Build a mock scan for a 1.5 degree wedge. Only used for generating indices near # the Ewald sphere sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 1), exposure_times=0.1, oscillation=(0, 1.5), epochs=list(range(1)), deg=True, ) sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert approx_equal(im_width, 1.5 * pi / 180.0) # Build experiment lists stills_experiments = ExperimentList() stills_experiments.append( Experiment(beam=mybeam, detector=mydetector, crystal=crystal, imageset=None)) scans_experiments = ExperimentList() scans_experiments.append( Experiment( beam=mybeam, detector=mydetector, crystal=crystal, goniometer=mygonio, scan=myscan, imageset=None, )) ########################################################## # Parameterise the models (only for perturbing geometry) # ########################################################## xlo_param = CrystalOrientationParameterisation(crystal) xluc_param = CrystalUnitCellParameterisation(crystal) ################################ # Apply known parameter shifts # ################################ # rotate crystal (=5 mrad each rotation) xlo_p_vals = [] p_vals = xlo_param.get_param_vals() xlo_p_vals.append(p_vals) new_p_vals = [a + b for a, b in zip(p_vals, [5.0, 5.0, 5.0])] xlo_param.set_param_vals(new_p_vals) # change unit cell (=1.0 Angstrom length upsets, 0.5 degree of # gamma angle) xluc_p_vals = [] p_vals = xluc_param.get_param_vals() xluc_p_vals.append(p_vals) cell_params = crystal.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [1.0, 1.0, -1.0, 0.0, 0.0, 0.5]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(crystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) # keep track of the target crystal model to compare with refined from copy import deepcopy target_crystal = deepcopy(crystal) ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere for crystal resolution = 2.0 index_generator = IndexGenerator( crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() # Build a ray predictor and predict rays close to the Ewald sphere by using # the narrow rotation scan ref_predictor = ScansRayPredictor(scans_experiments, sweep_range) obs_refs = ref_predictor(indices, experiment_id=0) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Add in flags and ID columns by copying into standard reflection table tmp = flex.reflection_table.empty_standard(len(obs_refs)) tmp.update(obs_refs) obs_refs = tmp # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180.0 px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2) obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) # Re-predict using the stills reflection predictor stills_ref_predictor = StillsExperimentsPredictor(stills_experiments) obs_refs_stills = stills_ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs_stills["xyzobs.mm.value"] = obs_refs_stills["xyzcal.mm"] ############################### # Undo known parameter shifts # ############################### xlo_param.set_param_vals(xlo_p_vals[0]) xluc_param.set_param_vals(xluc_p_vals[0]) # make a refiner from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse("")).extract() # Change this to get a plot do_plot = False if do_plot: params.refinement.refinery.journal.track_parameter_correlation = True from dials.algorithms.refinement.refiner import RefinerFactory # decrease bin_size_fraction to terminate on RMSD convergence params.refinement.target.bin_size_fraction = 0.01 params.refinement.parameterisation.beam.fix = "all" params.refinement.parameterisation.detector.fix = "all" refiner = RefinerFactory.from_parameters_data_experiments( params, obs_refs_stills, stills_experiments) # run refinement history = refiner.run() # regression tests assert len(history["rmsd"]) == 9 refined_crystal = refiner.get_experiments()[0].crystal uc1 = refined_crystal.get_unit_cell() uc2 = target_crystal.get_unit_cell() assert uc1.is_similar_to(uc2) if do_plot: plt = refiner.parameter_correlation_plot( len(history["parameter_correlation"]) - 1) plt.show()
beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=crystal1, imageset=None)) experiments.append(Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=crystal2, imageset=None)) assert len(experiments.detectors()) == 1 ########################################################## # Parameterise the models (only for perturbing geometry) # ########################################################## det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xl1o_param = CrystalOrientationParameterisation(crystal1) xl1uc_param = CrystalUnitCellParameterisation(crystal1) xl2o_param = CrystalOrientationParameterisation(crystal2) xl2uc_param = CrystalUnitCellParameterisation(crystal2) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) # Fix crystal parameters #xluc_param.set_fixed([True, True, True, True, True, True]) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ######################################################################## #pred_param = XYPhiPredictionParameterisation(experiments,
def crystal_parameters(): from dxtbx.model.experiment.experiment_list import ExperimentListFactory from dials.array_family import flex from scitbx import matrix experiments = ExperimentListFactory.from_json_file("experiments.json") reflections = flex.reflection_table.from_pickle("reflections.pickle") beam = experiments[0].beam detector = experiments[0].detector goniometer = experiments[0].goniometer scan = experiments[0].scan crystal = experiments[0].crystal from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalUnitCellParameterisation, ) parameters = CrystalUnitCellParameterisation(crystal) print("") print("Parameters") for param in parameters.get_params(): print(param.name, param.value) print("") print("db/dp") for db_dp in parameters.get_ds_dp(): print(db_dp.round(5)) print("") print("B matrix") B = crystal.get_B().round(7) print(B) print(B.transpose() * B) # from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge # temp = symmetrize_reduce_enlarge(crystal.get_space_group()) # temp.set_orientation(crystal.get_B()) # independent = temp.forward_independent_parameters() # print independent # new_mm = temp.enlarge(independent) # temp.Bconverter.validate_and_setG(new_mm) # B = temp.Bconverter.back_as_orientation() # print matrix.sqr(B.reciprocal_matrix()).round(5) # from cctbx import sgtbx # from rstbx.symmetry.constraints import AGconvert # constraints = sgtbx.tensor_rank_2_constraints(space_group=crystal.get_space_group(), reciprocal_space=True) # params = constraints.all_params(independent_params=tuple(p.value for p in # parameters.get_params())) # from cctbx.crystal_orientation import crystal_orientation # converter = AGconvert() # converter.forward(crystal_orientation(crystal.get_B(), False)) # converter.validate_and_setG(params) # orientation = converter.back_as_orientation() # print orientation # print "" print("SUM(p * dp/dp)") MAT = [ p.value * db_dp for p, db_dp in zip(parameters.get_params(), parameters.get_ds_dp()) ] COV = sum(MAT[1:], MAT[0]) print(2 * COV.round(7)) print((2 * COV - crystal.get_B()).round(7))
class ModelState(object): """ A class to keep track of the model state """ def __init__( self, experiment, mosaicity_parameterisation, wavelength_parameterisation=None, fix_mosaic_spread=False, fix_wavelength_spread=True, fix_unit_cell=False, fix_orientation=False, ): """ Initialise the model state """ # Save the crystal model self.experiment = experiment self.crystal = experiment.crystal # The U and P parameterisation self._U_parameterisation = CrystalOrientationParameterisation( self.crystal) self._B_parameterisation = CrystalUnitCellParameterisation( self.crystal) # The M and L parameterisations self._M_parameterisation = mosaicity_parameterisation self._L_parameterisation = wavelength_parameterisation # Set the flags to fix parameters self._is_mosaic_spread_fixed = fix_mosaic_spread self._is_wavelength_spread_fixed = fix_wavelength_spread self._is_unit_cell_fixed = fix_unit_cell self._is_orientation_fixed = fix_orientation # Check wavelength parameterisation if not self.is_wavelength_spread_fixed: assert self._L_parameterisation is not None @property def is_orientation_fixed(self) -> bool: return self._is_orientation_fixed @property def is_unit_cell_fixed(self) -> bool: return self._is_unit_cell_fixed @property def is_mosaic_spread_fixed(self) -> bool: return self._is_mosaic_spread_fixed @property def is_mosaic_spread_angular(self) -> bool: return self._M_parameterisation.is_angular() @property def is_wavelength_spread_fixed(self) -> bool: return self._is_wavelength_spread_fixed @property def unit_cell(self): return self.crystal.get_unit_cell() @property def U_matrix(self) -> np.array: return np.array([self.crystal.get_U()], dtype=np.float64).reshape(3, 3) @property def B_matrix(self) -> np.array: return np.array([self.crystal.get_B()], dtype=np.float64).reshape(3, 3) @property def A_matrix(self) -> np.array: return np.array([self.crystal.get_A()], dtype=np.float64).reshape(3, 3) @property def mosaicity_covariance_matrix(self) -> np.array: return self._M_parameterisation.sigma() @property def wavelength_spread(self) -> flex.double: if self._L_parameterisation is not None: return self._L_parameterisation.sigma() return flex.double() @property def U_params(self) -> np.array: """Get the parameters of the orientation parameterisation""" return np.array(self._U_parameterisation.get_param_vals(), dtype=np.float64) @U_params.setter def U_params(self, params) -> None: self._U_parameterisation.set_param_vals(tuple( float(i) for i in params)) @property def B_params(self) -> np.array: """Get the parameters of the orientation parameterisation""" return np.array(self._B_parameterisation.get_param_vals(), dtype=np.float64) @B_params.setter def B_params(self, params) -> None: self._B_parameterisation.set_param_vals(tuple( float(i) for i in params)) @property def M_params(self) -> np.array: "Parameters of the mosaicity parameterisation" return self._M_parameterisation.parameters @M_params.setter def M_params(self, params) -> None: self._M_parameterisation.parameters = params @property def L_params(self) -> np.array: "Parameters of the Lambda (wavelength) parameterisation" if self._L_parameterisation is not None: return np.array(self._L_parameterisation.parameters, dtype=np.float64) return np.array([]) @L_params.setter def L_params(self, params: flex.double) -> None: if self._L_parameterisation is not None: self._L_parameterisation.parameters = params @property def dU_dp(self) -> np.array: """ Get the first derivatives of U w.r.t its parameters """ ds_dp = self._U_parameterisation.get_ds_dp() n = len(ds_dp) s = np.array([ds_dp[i] for i in range(n)], dtype=np.float64).reshape(n, 3, 3) return s @property def dB_dp(self) -> np.array: """ Get the first derivatives of B w.r.t its parameters """ ds_dp = self._B_parameterisation.get_ds_dp() n = len(ds_dp) s = np.array([ds_dp[i] for i in range(n)], dtype=np.float64).reshape(n, 3, 3) return s @property def dM_dp(self) -> np.array: """ Get the first derivatives of M w.r.t its parameters """ return self._M_parameterisation.first_derivatives() @property def dL_dp(self) -> flex.double: """ Get the first derivatives of L w.r.t its parameters """ if self._L_parameterisation is not None: return self._L_parameterisation.first_derivatives() return flex.double() @property def active_parameters(self) -> np.array: """ The active parameters in order: U, B, M, L, W """ active_params = [] if not self.is_orientation_fixed: active_params.append(self.U_params) if not self.is_unit_cell_fixed: active_params.append(self.B_params) if not self.is_mosaic_spread_fixed: active_params.append(self.M_params) if not self.is_wavelength_spread_fixed: active_params.append(self.L_params) active_params = np.concatenate(active_params) assert len(active_params) > 0 return active_params @active_parameters.setter def active_parameters(self, params) -> None: """ Set the active parameters in order: U, B, M, L, W """ if not self.is_orientation_fixed: n_U_params = len(self.U_params) temp = params[:n_U_params] params = params[n_U_params:] self.U_params = temp if not self.is_unit_cell_fixed: n_B_params = len(self.B_params) temp = params[:n_B_params] params = params[n_B_params:] self.B_params = temp if not self.is_mosaic_spread_fixed: n_M_params = self.M_params.size temp = params[:n_M_params] params = params[n_M_params:] self.M_params = np.array(temp) if not self.is_wavelength_spread_fixed: n_L_params = self.L_params.size temp = params[:n_L_params] self.L_params = temp @property def parameter_labels(self) -> List[str]: """ Get the parameter labels """ labels = [] if not self.is_orientation_fixed: labels += [f"Crystal_U_{i}" for i in range(self.U_params.size)] if not self.is_unit_cell_fixed: labels += [f"Crystal_B_{i}" for i in range(self.B_params.size)] if not self.is_mosaic_spread_fixed: labels += [f"Mosaicity_{i}" for i in range(self.M_params.size)] if not self.is_wavelength_spread_fixed: labels.append("Wavelength_Spread") assert len(labels) > 0 return labels
detector=mydetector, goniometer=mygonio, scan=myscan, crystal=crystal2, imageset=None)) assert len(experiments.detectors()) == 1 ########################################################## # Parameterise the models (only for perturbing geometry) # ########################################################## det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xl1o_param = CrystalOrientationParameterisation(crystal1) xl1uc_param = CrystalUnitCellParameterisation(crystal1) xl2o_param = CrystalOrientationParameterisation(crystal2) xl2uc_param = CrystalUnitCellParameterisation(crystal2) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) # Fix crystal parameters #xluc_param.set_fixed([True, True, True, True, True, True]) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ######################################################################## #pred_param = XYPhiPredictionParameterisation(experiments,
def test_refinement(dials_regression): """Test a refinement run""" # Get a beam and detector from a experiments. This one has a CS-PAD, but that # is irrelevant data_dir = os.path.join(dials_regression, "refinement_test_data", "hierarchy_test") experiments_path = os.path.join(data_dir, "datablock.json") assert os.path.exists(experiments_path) # load models from dxtbx.model.experiment_list import ExperimentListFactory experiments = ExperimentListFactory.from_serialized_format( experiments_path, check_format=False) im_set = experiments.imagesets()[0] detector = deepcopy(im_set.get_detector()) beam = im_set.get_beam() # Invent a crystal, goniometer and scan for this test from dxtbx.model import Crystal crystal = Crystal((40.0, 0.0, 0.0), (0.0, 40.0, 0.0), (0.0, 0.0, 40.0), space_group_symbol="P1") orig_xl = deepcopy(crystal) from dxtbx.model import GoniometerFactory goniometer = GoniometerFactory.known_axis((1.0, 0.0, 0.0)) # Build a mock scan for a 180 degree sequence from dxtbx.model import ScanFactory sf = ScanFactory() scan = sf.make_scan( image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(1800)), deg=True, ) sequence_range = scan.get_oscillation_range(deg=False) im_width = scan.get_oscillation(deg=False)[1] assert sequence_range == (0.0, pi) assert approx_equal(im_width, 0.1 * pi / 180.0) # Build an experiment list experiments = ExperimentList() experiments.append( Experiment( beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal, imageset=None, )) # simulate some reflections refs, _ = generate_reflections(experiments) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # alpha and beta angles) from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalUnitCellParameterisation, ) xluc_param = CrystalUnitCellParameterisation(crystal) cell_params = crystal.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1, -0.1, 0.0]) ] from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge from scitbx import matrix new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(crystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) # reparameterise the crystal at the perturbed geometry xluc_param = CrystalUnitCellParameterisation(crystal) # Dummy parameterisations for other models beam_param = None xlo_param = None det_param = None # parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.parameter_report import ( ParameterReporter, ) pred_param = TwoThetaPredictionParameterisation(experiments, det_param, beam_param, xlo_param, [xluc_param]) param_reporter = ParameterReporter(det_param, beam_param, xlo_param, [xluc_param]) # reflection manager refman = TwoThetaReflectionManager(refs, experiments, nref_per_degree=20) # reflection predictor ref_predictor = TwoThetaExperimentsPredictor(experiments) # target function target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param) # minimisation engine from dials.algorithms.refinement.engine import ( LevenbergMarquardtIterations as Refinery, ) refinery = Refinery( target=target, prediction_parameterisation=pred_param, log=None, max_iterations=20, ) # Refiner from dials.algorithms.refinement.refiner import Refiner refiner = Refiner( experiments=experiments, pred_param=pred_param, param_reporter=param_reporter, refman=refman, target=target, refinery=refinery, ) refiner.run() # compare crystal with original crystal refined_xl = refiner.get_experiments()[0].crystal # print refined_xl assert refined_xl.is_similar_to(orig_xl, uc_rel_length_tolerance=0.001, uc_abs_angle_tolerance=0.01)
def test(): # Python and cctbx imports from math import pi from scitbx import matrix from scitbx.array_family import flex from libtbx.phil import parse from libtbx.test_utils import approx_equal # Get modules to build models and minimiser using PHIL import dials.test.algorithms.refinement.setup_geometry as setup_geometry import dials.test.algorithms.refinement.setup_minimiser as setup_minimiser # We will set up a mock scan and a mock experiment list from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import ExperimentList, Experiment # Model parameterisations from dials.algorithms.refinement.parameterisation.detector_parameters import ( DetectorParameterisationSinglePanel, ) from dials.algorithms.refinement.parameterisation.beam_parameters import ( BeamParameterisation, ) from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalOrientationParameterisation, CrystalUnitCellParameterisation, ) # Symmetry constrained parameterisation for the unit cell from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection from dials.algorithms.refinement.prediction.managed_predictors import ( ScansRayPredictor, ScansExperimentsPredictor, ) from cctbx.sgtbx import space_group, space_group_symbols # Parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.prediction_parameters import ( XYPhiPredictionParameterisation, ) # Imports for the target function from dials.algorithms.refinement.target import ( LeastSquaresPositionalResidualWithRmsdCutoff, ) from dials.algorithms.refinement.reflection_manager import ReflectionManager ############################# # Setup experimental models # ############################# override = """geometry.parameters { beam.wavelength.random=False beam.wavelength.value=1.0 beam.direction.inclination.random=False crystal.a.length.random=False crystal.a.length.value=12.0 crystal.a.direction.method=exactly crystal.a.direction.exactly.direction=1.0 0.002 -0.004 crystal.b.length.random=False crystal.b.length.value=14.0 crystal.b.direction.method=exactly crystal.b.direction.exactly.direction=-0.002 1.0 0.002 crystal.c.length.random=False crystal.c.length.value=13.0 crystal.c.direction.method=exactly crystal.c.direction.exactly.direction=0.002 -0.004 1.0 detector.directions.method=exactly detector.directions.exactly.dir1=0.99 0.002 -0.004 detector.directions.exactly.norm=0.002 -0.001 0.99 detector.centre.method=exactly detector.centre.exactly.value=1.0 -0.5 199.0 }""" master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True, ) models = setup_geometry.Extract(master_phil, local_overrides=override, verbose=False) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam ########################### # Parameterise the models # ########################### det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ######################################################################## # Build a mock scan for a 180 degree sweep sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(1800)), deg=True, ) # Build an ExperimentList experiments = ExperimentList() experiments.append( Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None, )) # Create the PredictionParameterisation pred_param = XYPhiPredictionParameterisation(experiments, [det_param], [s0_param], [xlo_param], [xluc_param]) ################################ # Apply known parameter shifts # ################################ # shift detector by 1.0 mm each translation and 4 mrad each rotation det_p_vals = det_param.get_param_vals() p_vals = [ a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 4.0, 4.0, 4.0]) ] det_param.set_param_vals(p_vals) # shift beam by 4 mrad in free axis s0_p_vals = s0_param.get_param_vals() p_vals = list(s0_p_vals) p_vals[0] += 4.0 s0_param.set_param_vals(p_vals) # rotate crystal a bit (=3 mrad each rotation) xlo_p_vals = xlo_param.get_param_vals() p_vals = [a + b for a, b in zip(xlo_p_vals, [3.0, 3.0, 3.0])] xlo_param.set_param_vals(p_vals) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # alpha and beta angles) xluc_p_vals = xluc_param.get_param_vals() cell_params = mycrystal.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1, -0.1, 0.0]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(mycrystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator( mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0.0, pi) assert approx_equal(im_width, 0.1 * pi / 180.0) # Predict rays within the sweep range ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ScansExperimentsPredictor(experiments) obs_refs["id"] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180.0 px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2) obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) # The total number of observations should be 1128 assert len(obs_refs) == 1128 ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) xluc_param.set_param_vals(xluc_p_vals) ##################################### # Select reflections for refinement # ##################################### refman = ReflectionManager(obs_refs, experiments, outlier_detector=None, close_to_spindle_cutoff=0.1) ############################## # Set up the target function # ############################## # The current 'achieved' criterion compares RMSD against 1/3 the pixel size and # 1/3 the image width in radians. For the simulated data, these are just made up mytarget = LeastSquaresPositionalResidualWithRmsdCutoff( experiments, ref_predictor, refman, pred_param, restraints_parameterisation=None) ###################################### # Set up the LSTBX refinement engine # ###################################### overrides = """minimiser.parameters.engine=GaussNewton minimiser.parameters.logfile=None""" refiner = setup_minimiser.Extract(master_phil, mytarget, pred_param, local_overrides=overrides).refiner refiner.run() assert mytarget.achieved() assert refiner.get_num_steps() == 1 assert approx_equal( mytarget.rmsds(), (0.00508252354876, 0.00420954552156, 8.97303428289e-05)) ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) xluc_param.set_param_vals(xluc_p_vals) ###################################################### # Set up the LBFGS with curvatures refinement engine # ###################################################### overrides = """minimiser.parameters.engine=LBFGScurvs minimiser.parameters.logfile=None""" refiner = setup_minimiser.Extract(master_phil, mytarget, pred_param, local_overrides=overrides).refiner refiner.run() assert mytarget.achieved() assert refiner.get_num_steps() == 9 assert approx_equal(mytarget.rmsds(), (0.0558857700305, 0.0333446685335, 0.000347402754278))
'dbvec_dp': fd_vec[1], 'dcvec_dp': fd_vec[2], 'dB_dp': fd_B, 'dO_dp': fd_O }) p_vals[i] = val # return to the initial state mp.set_param_vals(p_vals) return fd_grad xlo_param = CrystalOrientationParameterisation(crystal) xluc_param = CrystalUnitCellParameterisation(crystal) from dials.algorithms.refinement.restraints.restraints import SingleUnitCellTie uct = SingleUnitCellTie(xluc_param, [None] * 6, [None] * 6) from scitbx.math import angle_derivative_wrt_vectors B = matrix.sqr(crystal.get_B()) O = (B.transpose()).inverse() a, b, c, aa, bb, cc = crystal.get_unit_cell().parameters() aa *= DEG2RAD bb *= DEG2RAD cc *= DEG2RAD Ut = matrix.sqr(crystal.get_U()).transpose() avec, bvec, cvec = [Ut * vec for vec in crystal.get_real_space_vectors()]
'dcc_dp':fd_uc[5], 'davec_dp':fd_vec[0], 'dbvec_dp':fd_vec[1], 'dcvec_dp':fd_vec[2], 'dB_dp':fd_B, 'dO_dp':fd_O}) p_vals[i] = val # return to the initial state mp.set_param_vals(p_vals) return fd_grad xlo_param = CrystalOrientationParameterisation(crystal) xluc_param = CrystalUnitCellParameterisation(crystal) from dials.algorithms.refinement.restraints.restraints import SingleUnitCellTie uct = SingleUnitCellTie(xluc_param, [None]*6, [None]*6) from dials.algorithms.refinement.refinement_helpers import \ AngleDerivativeWrtVectorElts B = crystal.get_B() O = (B.transpose()).inverse() a, b, c, aa, bb, cc = crystal.get_unit_cell().parameters() aa *= DEG2RAD bb *= DEG2RAD cc *= DEG2RAD Ut = crystal.get_U().transpose() avec, bvec, cvec = [Ut * vec for vec in crystal.get_real_space_vectors()]