def e_refine(params, experiments, reflections, graph_verbose=False): # Stills-specific parameters we always want assert params.refinement.reflections.outlier.algorithm in ( None, "null", ), "Cannot index, set refinement.reflections.outlier.algorithm=null" # we do our own outlier rejection from dials.algorithms.refinement.refiner import RefinerFactory refiner = RefinerFactory.from_parameters_data_experiments( params, reflections, experiments ) refiner.run() ref_sel = refiner.selection_used_for_refinement() assert ref_sel.count(True) == len(reflections) if not graph_verbose: return refiner RR = refiner.predict_for_reflection_table(reflections) plot_displacements(reflections, RR, experiments) return refiner
def refine(self, experiments, reflections, isoform=None): from dials.algorithms.refinement.refiner import phil_scope from libtbx.phil import parse params = phil_scope.fetch(source=parse('')).extract() params.refinement.reflections.weighting_strategy.delpsi_constant = 100000. params.refinement.reflections.weighting_strategy.override = "stills" params.refinement.parameterisation.auto_reduction.action = "fix" #params.refinement.reflections.do_outlier_rejection=True #params.refinement.reflections.iqr_multiplier=0.5 #params.refinement.reflections.minimum_sample_size=50 #params.refinement.reflections.maximum_sample_size=50 #params.refinement.reflections.random_seed=1 if self.horizons_phil.integration.dials_refinement.strategy == "distance": params.refinement.parameterisation.beam.fix = "all" params.refinement.parameterisation.detector.fix_list = [ "Tau1" ] # fix detector rotz, allow distance to refine elif self.horizons_phil.integration.dials_refinement.strategy == "wavelength": params.refinement.parameterisation.beam.fix = "in_spindle_plane,out_spindle_plane" params.refinement.parameterisation.detector.fix_list = [ "Dist", "Tau1" ] # fix detector rotz and distance elif self.horizons_phil.integration.dials_refinement.strategy == "fix": params.refinement.parameterisation.beam.fix = "all" params.refinement.parameterisation.detector.fix_list = [ "Dist", "Tau1" ] # fix detector rotz and distance if isoform is not None: params.refinement.reflections.outlier.algorithm = "null" params.refinement.parameterisation.crystal.fix = "cell" from dials.algorithms.refinement.refiner import RefinerFactory refiner = RefinerFactory.from_parameters_data_experiments(params, reflections, experiments, verbosity=1) history = refiner.run() print(history.keys()) for item in history["rmsd"]: print("%5.2f %5.2f %8.5f" % (item[0], item[1], 180. * item[2] / math.pi)) #for item in history["parameter_vector"]: # print ["%8.5f"%a for a in item] print(refiner.selection_used_for_refinement().count(True), "spots used for refinement") print(refiner.get_experiments()[0].beam) print(refiner.get_experiments()[0].detector) print( "Distance:", -refiner.get_experiments()[0].detector[0].get_beam_centre_lab( refiner.get_experiments()[0].beam.get_s0())[2]) print(refiner.get_experiments()[0].crystal) return refiner
def e_refine(params, experiments, reflections, graph_verbose=False): # Stills-specific parameters we always want assert params.refinement.reflections.outlier.algorithm in (None, "null"), \ "Cannot index, set refinement.reflections.outlier.algorithm=null" # we do our own outlier rejection from dials.algorithms.refinement.refiner import RefinerFactory refiner = RefinerFactory.from_parameters_data_experiments(params, reflections, experiments, verbosity=1) history = refiner.run() ref_sel = refiner.selection_used_for_refinement() assert ref_sel.count(True) == len(reflections) if not graph_verbose: return refiner RR = refiner.predict_for_reflection_table(reflections) plot_displacements(reflections, RR, experiments) return refiner
def refine(self, experiments, reflections, isoform=None): from dials.algorithms.refinement.refiner import phil_scope from libtbx.phil import parse params = phil_scope.fetch(source=parse('')).extract() params.refinement.reflections.weighting_strategy.delpsi_constant=100000. params.refinement.reflections.weighting_strategy.override="stills" params.refinement.parameterisation.auto_reduction.action="fix" #params.refinement.reflections.do_outlier_rejection=True #params.refinement.reflections.iqr_multiplier=0.5 #params.refinement.reflections.minimum_sample_size=50 #params.refinement.reflections.maximum_sample_size=50 #params.refinement.reflections.random_seed=1 if self.horizons_phil.integration.dials_refinement.strategy=="distance": params.refinement.parameterisation.beam.fix="all" params.refinement.parameterisation.detector.fix_list=["Tau1"] # fix detector rotz, allow distance to refine elif self.horizons_phil.integration.dials_refinement.strategy=="wavelength": params.refinement.parameterisation.beam.fix="in_spindle_plane,out_spindle_plane" params.refinement.parameterisation.detector.fix_list=["Dist","Tau1"] # fix detector rotz and distance if isoform is not None: params.refinement.reflections.outlier.algorithm="null" params.refinement.parameterisation.crystal.fix="cell" from dials.algorithms.refinement.refiner import RefinerFactory refiner = RefinerFactory.from_parameters_data_experiments(params, reflections, experiments, verbosity=1) history = refiner.run() print history.keys() for item in history["rmsd"]: print "%5.2f %5.2f %8.5f"%(item[0],item[1],180.*item[2]/math.pi) #for item in history["parameter_vector"]: # print ["%8.5f"%a for a in item] print refiner.selection_used_for_refinement().count(True),"spots used for refinement" print refiner.get_experiments()[0].beam print refiner.get_experiments()[0].detector print "Distance:", -refiner.get_experiments()[0].detector[0].get_beam_centre_lab(refiner.get_experiments()[0].beam.get_s0())[2] print refiner.get_experiments()[0].crystal return refiner
def test_run(dials_regression, tmpdir): expected_unit_cell = uctbx.unit_cell( (11.624, 13.550, 30.103, 89.964, 93.721, 90.132)) expected_rmsds = (0.039, 0.035, 0.002) experiments_old = os.path.join(dials_regression, "indexing_test_data", "phi_scan", "datablock_old.json") experiments_new = os.path.join(dials_regression, "indexing_test_data", "phi_scan", "datablock.json") strong_pickle = os.path.join(dials_regression, "indexing_test_data", "phi_scan", "strong.pickle") from dxtbx.serialize import load imageset_old = load.experiment_list(experiments_old, check_format=False).imagesets()[0] imageset_new = load.experiment_list(experiments_new, check_format=False).imagesets()[0] gonio_old = imageset_old.get_goniometer() gonio_new = imageset_new.get_goniometer() assert gonio_old.get_rotation_axis() == pytest.approx( (0.7497646259807715, -0.5517923303436749, 0.36520984351713554)) assert gonio_old.get_setting_rotation() == pytest.approx( (1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0)) assert gonio_old.get_fixed_rotation() == pytest.approx(( 0.7497646259807748, -0.20997265900532208, -0.6275065641872948, -0.5517923303436731, 0.3250014637526764, -0.7680490041218182, 0.3652098435171313, 0.9221092836691605, 0.12781329809272568, )) assert gonio_new.get_rotation_axis() == pytest.approx( gonio_old.get_rotation_axis()) assert gonio_new.get_rotation_axis_datum() == pytest.approx((1, 0, 0)) assert gonio_new.get_setting_rotation() == pytest.approx(( 0.7497646259807705, -0.20997265900532142, -0.6275065641873, -0.5517923303436786, 0.3250014637526763, -0.768049004121814, 0.3652098435171315, 0.9221092836691607, 0.12781329809272335, )) assert gonio_new.get_fixed_rotation() == pytest.approx( (1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0)) result_old = run_indexing( strong_pickle, experiments_old, tmpdir, extra_args=[], expected_unit_cell=expected_unit_cell, expected_rmsds=expected_rmsds, expected_hall_symbol=" P 1", ) result_new = run_indexing( strong_pickle, experiments_new, tmpdir, extra_args=[], expected_unit_cell=expected_unit_cell, expected_rmsds=expected_rmsds, expected_hall_symbol=" P 1", ) assert result_old.rmsds == pytest.approx(result_new.rmsds, abs=1e-6) assert result_old.experiments[0].crystal.get_unit_cell().parameters( ) == pytest.approx( result_new.experiments[0].crystal.get_unit_cell().parameters(), abs=1e-6) # Now test refinement gradients are correct from dxtbx.model.experiment_list import Experiment, ExperimentList old_exps = ExperimentList([ Experiment( beam=imageset_old.get_beam(), detector=imageset_old.get_detector(), goniometer=gonio_old, scan=imageset_old.get_scan(), crystal=result_old.experiments[0].crystal, imageset=None, ) ]) new_exps = ExperimentList([ Experiment( beam=imageset_new.get_beam(), detector=imageset_new.get_detector(), goniometer=gonio_new, scan=imageset_new.get_scan(), crystal=result_new.experiments[0].crystal, imageset=None, ) ]) from libtbx.phil import parse from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse("")).extract() from dials.algorithms.refinement.refiner import RefinerFactory refiner_old = RefinerFactory.from_parameters_data_experiments( params, result_old.indexed_reflections, old_exps) refiner_new = RefinerFactory.from_parameters_data_experiments( params, result_new.indexed_reflections, new_exps) # Analytical gradients should be approximately the same in either case an_grads_old = refiner_old._pred_param.get_gradients( refiner_old.get_matches()) an_grads_new = refiner_new._pred_param.get_gradients( refiner_new.get_matches()) for g1, g2 in zip(an_grads_old, an_grads_new): assert g1["dX_dp"] == pytest.approx(g2["dX_dp"], abs=1.0e-6) assert g1["dY_dp"] == pytest.approx(g2["dY_dp"], abs=1.0e-6) assert g1["dphi_dp"] == pytest.approx(g2["dphi_dp"], abs=1.0e-6) # Analytical gradients should be approximately equal to finite difference # gradients in either case fd_grads_old = calc_fd_grads(refiner_old) for g1, g2 in zip(fd_grads_old, an_grads_old): assert g1["dX_dp"] == pytest.approx(g2["dX_dp"], abs=5.0e-6) assert g1["dY_dp"] == pytest.approx(g2["dY_dp"], abs=5.0e-6) assert g1["dphi_dp"] == pytest.approx(g2["dphi_dp"], abs=5.0e-6) fd_grads_new = calc_fd_grads(refiner_new) for g1, g2 in zip(fd_grads_new, an_grads_new): assert g1["dX_dp"] == pytest.approx(g2["dX_dp"], abs=5.0e-6) assert g1["dY_dp"] == pytest.approx(g2["dY_dp"], abs=5.0e-6) assert g1["dphi_dp"] == pytest.approx(g2["dphi_dp"], abs=5.0e-6)
#print "Initial values of parameters are" #msg = "Parameters: " + "%.5f " * len(pred_param) #print msg % tuple(pred_param.get_param_vals()) #print # make a refiner from dials.algorithms.refinement.refiner import phil_scope from libtbx.phil import parse params = phil_scope.fetch(source=parse('')).extract() # in case we want a plot params.refinement.refinery.track_parameter_correlation=True # DEBUG scan varying by uncommenting the following line #params.refinement.parameterisation.crystal.scan_varying=True from dials.algorithms.refinement.refiner import RefinerFactory refiner = RefinerFactory.from_parameters_data_experiments(params, obs_refs, experiments, verbosity=0) history = refiner.run() #plt = refiner.parameter_correlation_plot(len(history["parameter_correlation"])-1) #plt.show() #print "Refinement has completed with the following geometry:" #expts = refiner.get_experiments() #for beam in expts.beams(): print beam #for detector in expts.detectors(): print detector #for crystal in expts.crystals(): print crystal print "OK"
def run(): have_dials_regression = libtbx.env.has_module("dials_regression") if not have_dials_regression: print "Skipped: dials_regression not available" return dials_regression = libtbx.env.find_in_repositories( relative_path="dials_regression", test=os.path.isdir) from dials.test.algorithms.indexing.tst_index import run_one_indexing expected_unit_cell = uctbx.unit_cell( (11.624, 13.550, 30.103, 89.964, 93.721, 90.132)) expected_rmsds = (0.039, 0.035, 0.002) datablock_old = os.path.join( dials_regression, "indexing_test_data/phi_scan/datablock_old.json") datablock_new = os.path.join(dials_regression, "indexing_test_data/phi_scan/datablock.json") strong_pickle = os.path.join(dials_regression, "indexing_test_data/phi_scan/strong.pickle") from dxtbx.serialize import load imageset_old = load.datablock(datablock_old, check_format=False)[0].extract_imagesets()[0] imageset_new = load.datablock(datablock_new, check_format=False)[0].extract_imagesets()[0] gonio_old = imageset_old.get_goniometer() gonio_new = imageset_new.get_goniometer() assert approx_equal( gonio_old.get_rotation_axis(), (0.7497646259807715, -0.5517923303436749, 0.36520984351713554)) assert approx_equal(gonio_old.get_setting_rotation(), (1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0)) assert approx_equal( gonio_old.get_fixed_rotation(), (0.7497646259807748, -0.20997265900532208, -0.6275065641872948, -0.5517923303436731, 0.3250014637526764, -0.7680490041218182, 0.3652098435171313, 0.9221092836691605, 0.12781329809272568)) assert approx_equal(gonio_new.get_rotation_axis(), gonio_old.get_rotation_axis()) assert approx_equal(gonio_new.get_rotation_axis_datum(), (1, 0, 0)) assert approx_equal( gonio_new.get_setting_rotation(), (0.7497646259807705, -0.20997265900532142, -0.6275065641873, -0.5517923303436786, 0.3250014637526763, -0.768049004121814, 0.3652098435171315, 0.9221092836691607, 0.12781329809272335)) assert approx_equal(gonio_new.get_fixed_rotation(), (1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0)) result_old = run_one_indexing( pickle_path=strong_pickle, sweep_path=datablock_old, extra_args=[], expected_unit_cell=expected_unit_cell, expected_rmsds=expected_rmsds, expected_hall_symbol=' P 1', ) result_new = run_one_indexing( pickle_path=strong_pickle, sweep_path=datablock_new, extra_args=[], expected_unit_cell=expected_unit_cell, expected_rmsds=expected_rmsds, expected_hall_symbol=' P 1', ) assert approx_equal(result_old.rmsds, result_new.rmsds) assert approx_equal(result_old.crystal_model.get_unit_cell().parameters(), result_new.crystal_model.get_unit_cell().parameters()) # Now test refinement gradients are correct from dxtbx.model.experiment_list import ExperimentList, Experiment old_exps = ExperimentList([ Experiment(beam=imageset_old.get_beam(), detector=imageset_old.get_detector(), goniometer=gonio_old, scan=imageset_old.get_scan(), crystal=result_old.crystal_model, imageset=None) ]) new_exps = ExperimentList([ Experiment(beam=imageset_new.get_beam(), detector=imageset_new.get_detector(), goniometer=gonio_new, scan=imageset_new.get_scan(), crystal=result_new.crystal_model, imageset=None) ]) from libtbx.phil import parse from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse('')).extract() from dials.algorithms.refinement.refiner import RefinerFactory refiner_old = RefinerFactory.from_parameters_data_experiments( params, result_old.indexed_reflections, old_exps, verbosity=0) refiner_new = RefinerFactory.from_parameters_data_experiments( params, result_new.indexed_reflections, new_exps, verbosity=0) # Analytical gradients should be approximately the same in either case an_grads_old = refiner_old._pred_param.get_gradients( refiner_old.get_matches()) an_grads_new = refiner_new._pred_param.get_gradients( refiner_new.get_matches()) for g1, g2 in zip(an_grads_old, an_grads_new): assert approx_equal(g1["dX_dp"], g2["dX_dp"], eps=1.e-6) assert approx_equal(g1["dY_dp"], g2["dY_dp"], eps=1.e-6) assert approx_equal(g1["dphi_dp"], g2["dphi_dp"], eps=1.e-6) # Analytical gradients should be approximately equal to finite difference # gradients in either case fd_grads_old = calc_fd_grads(refiner_old) for g1, g2 in zip(fd_grads_old, an_grads_old): assert approx_equal(g1["dX_dp"], g2["dX_dp"], eps=5.e-6) assert approx_equal(g1["dY_dp"], g2["dY_dp"], eps=5.e-6) assert approx_equal(g1["dphi_dp"], g2["dphi_dp"], eps=5.e-6) fd_grads_new = calc_fd_grads(refiner_new) for g1, g2 in zip(fd_grads_new, an_grads_new): assert approx_equal(g1["dX_dp"], g2["dX_dp"], eps=5.e-6) assert approx_equal(g1["dY_dp"], g2["dY_dp"], eps=5.e-6) assert approx_equal(g1["dphi_dp"], g2["dphi_dp"], eps=5.e-6)
# make a refiner from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse('')).extract() # Change this to get a plot do_plot = False if do_plot: params.refinement.refinery.journal.track_parameter_correlation = True from dials.algorithms.refinement.refiner import RefinerFactory # decrease bin_size_fraction to terminate on RMSD convergence params.refinement.target.bin_size_fraction = 0.01 params.refinement.parameterisation.beam.fix = "all" params.refinement.parameterisation.detector.fix = "all" refiner = RefinerFactory.from_parameters_data_experiments(params, obs_refs_stills, stills_experiments, verbosity=0) # run refinement history = refiner.run() # regression tests assert len(history["rmsd"]) == 9 refined_crystal = refiner.get_experiments()[0].crystal uc1 = refined_crystal.get_unit_cell() uc2 = target_crystal.get_unit_cell() assert uc1.is_similar_to(uc2) if do_plot: plt = refiner.parameter_correlation_plot(
def test(args=[]): # Python and cctbx imports from math import pi from scitbx import matrix from scitbx.array_family import flex from libtbx.phil import parse from libtbx.test_utils import approx_equal # Get module to build models using PHIL import dials.test.algorithms.refinement.setup_geometry as setup_geometry # We will set up a mock scan and a mock experiment list from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import ExperimentList, Experiment # Model parameterisations from dials.algorithms.refinement.parameterisation.detector_parameters import \ DetectorParameterisationSinglePanel from dials.algorithms.refinement.parameterisation.beam_parameters import \ BeamParameterisation from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalOrientationParameterisation, CrystalUnitCellParameterisation # Symmetry constrained parameterisation for the unit cell from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import \ symmetrize_reduce_enlarge # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.refinement.prediction import ScansRayPredictor, \ ExperimentsPredictor from dials.algorithms.spot_prediction import ray_intersection from cctbx.sgtbx import space_group, space_group_symbols # Parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.prediction_parameters import \ XYPhiPredictionParameterisation # implicit import # Imports for the target function from dials.algorithms.refinement.target import \ LeastSquaresPositionalResidualWithRmsdCutoff # implicit import ############################# # Setup experimental models # ############################# master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True) models = setup_geometry.Extract( master_phil, cmdline_args=args, local_overrides="geometry.parameters.random_seed = 1") crystal1 = models.crystal models = setup_geometry.Extract( master_phil, cmdline_args=args, local_overrides="geometry.parameters.random_seed = 2") mydetector = models.detector mygonio = models.goniometer crystal2 = models.crystal mybeam = models.beam # Build a mock scan for a 180 degree sweep sf = ScanFactory() myscan = sf.make_scan(image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=range(1800), deg=True) sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) # Build an experiment list experiments = ExperimentList() experiments.append( Experiment(beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=crystal1, imageset=None)) experiments.append( Experiment(beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=crystal2, imageset=None)) assert len(experiments.detectors()) == 1 ########################################################## # Parameterise the models (only for perturbing geometry) # ########################################################## det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xl1o_param = CrystalOrientationParameterisation(crystal1) xl1uc_param = CrystalUnitCellParameterisation(crystal1) xl2o_param = CrystalOrientationParameterisation(crystal2) xl2uc_param = CrystalUnitCellParameterisation(crystal2) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) # Fix crystal parameters #xluc_param.set_fixed([True, True, True, True, True, True]) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ######################################################################## #pred_param = XYPhiPredictionParameterisation(experiments, # [det_param], [s0_param], [xlo_param], [xluc_param]) ################################ # Apply known parameter shifts # ################################ # shift detector by 1.0 mm each translation and 2 mrad each rotation det_p_vals = det_param.get_param_vals() p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2., 2., 2.])] det_param.set_param_vals(p_vals) # shift beam by 2 mrad in free axis s0_p_vals = s0_param.get_param_vals() p_vals = list(s0_p_vals) p_vals[0] += 2. s0_param.set_param_vals(p_vals) # rotate crystal a bit (=2 mrad each rotation) xlo_p_vals = [] for xlo in (xl1o_param, xl2o_param): p_vals = xlo.get_param_vals() xlo_p_vals.append(p_vals) new_p_vals = [a + b for a, b in zip(p_vals, [2., 2., 2.])] xlo.set_param_vals(new_p_vals) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # gamma angle) xluc_p_vals = [] for xluc, xl in ((xl1uc_param, crystal1), ((xl2uc_param, crystal2))): p_vals = xluc.get_param_vals() xluc_p_vals.append(p_vals) cell_params = xl.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(xl.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.e5 for e in S.forward_independent_parameters()]) xluc.set_param_vals(X) ############################# # Generate some reflections # ############################# #print "Reflections will be generated with the following geometry:" #print mybeam #print mydetector #print crystal1 #print crystal2 # All indices in a 2.0 Angstrom sphere for crystal1 resolution = 2.0 index_generator = IndexGenerator( crystal1.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices1 = index_generator.to_array() # All indices in a 2.0 Angstrom sphere for crystal2 resolution = 2.0 index_generator = IndexGenerator( crystal2.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices2 = index_generator.to_array() # Predict rays within the sweep range. Set experiment IDs ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs1 = ray_predictor(indices1, experiment_id=0) obs_refs1['id'] = flex.int(len(obs_refs1), 0) obs_refs2 = ray_predictor(indices1, experiment_id=1) obs_refs2['id'] = flex.int(len(obs_refs2), 1) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs1) obs_refs1 = obs_refs1.select(intersects) intersects = ray_intersection(mydetector, obs_refs2) obs_refs2 = obs_refs2.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs1 = ref_predictor(obs_refs1) obs_refs2 = ref_predictor(obs_refs2) # Set 'observed' centroids from the predicted ones obs_refs1['xyzobs.mm.value'] = obs_refs1['xyzcal.mm'] obs_refs2['xyzobs.mm.value'] = obs_refs2['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs1), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs1), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs1), (im_width / 2.)**2) obs_refs1['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) var_x = flex.double(len(obs_refs2), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs2), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs2), (im_width / 2.)**2) obs_refs2['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) #print "Total number of reflections excited for crystal1", len(obs_refs1) #print "Total number of reflections excited for crystal2", len(obs_refs2) # concatenate reflection lists obs_refs1.extend(obs_refs2) obs_refs = obs_refs1 ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xl1o_param.set_param_vals(xlo_p_vals[0]) xl2o_param.set_param_vals(xlo_p_vals[1]) xl1uc_param.set_param_vals(xluc_p_vals[0]) xl2uc_param.set_param_vals(xluc_p_vals[1]) #print "Initial values of parameters are" #msg = "Parameters: " + "%.5f " * len(pred_param) #print msg % tuple(pred_param.get_param_vals()) #print # make a refiner from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse('')).extract() # in case we want a plot params.refinement.refinery.journal.track_parameter_correlation = True # scan static first from dials.algorithms.refinement.refiner import RefinerFactory refiner = RefinerFactory.from_parameters_data_experiments(params, obs_refs, experiments, verbosity=0) history = refiner.run() # scan varying params.refinement.parameterisation.scan_varying = True refiner = RefinerFactory.from_parameters_data_experiments(params, obs_refs, experiments, verbosity=0) history = refiner.run()
def test(args=[]): # Python and cctbx imports from math import pi from scitbx import matrix from libtbx.phil import parse from libtbx.test_utils import approx_equal # Import for surgery on reflection_tables from dials.array_family import flex # Get module to build models using PHIL import dials.test.algorithms.refinement.setup_geometry as setup_geometry # We will set up a mock scan and a mock experiment list from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import ExperimentList, Experiment # Crystal parameterisations from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalOrientationParameterisation, CrystalUnitCellParameterisation, ) # Symmetry constrained parameterisation for the unit cell from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.refinement.prediction.managed_predictors import ( ScansRayPredictor, StillsExperimentsPredictor, ) from dials.algorithms.spot_prediction import ray_intersection from cctbx.sgtbx import space_group, space_group_symbols ############################# # Setup experimental models # ############################# master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True, ) # build models, with a larger crystal than default in order to get enough # reflections on the 'still' image param = """ geometry.parameters.crystal.a.length.range=40 50; geometry.parameters.crystal.b.length.range=40 50; geometry.parameters.crystal.c.length.range=40 50; geometry.parameters.random_seed = 42""" models = setup_geometry.Extract(master_phil, cmdline_args=args, local_overrides=param) crystal = models.crystal mydetector = models.detector mygonio = models.goniometer mybeam = models.beam # Build a mock scan for a 1.5 degree wedge. Only used for generating indices near # the Ewald sphere sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 1), exposure_times=0.1, oscillation=(0, 1.5), epochs=list(range(1)), deg=True, ) sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert approx_equal(im_width, 1.5 * pi / 180.0) # Build experiment lists stills_experiments = ExperimentList() stills_experiments.append( Experiment(beam=mybeam, detector=mydetector, crystal=crystal, imageset=None)) scans_experiments = ExperimentList() scans_experiments.append( Experiment( beam=mybeam, detector=mydetector, crystal=crystal, goniometer=mygonio, scan=myscan, imageset=None, )) ########################################################## # Parameterise the models (only for perturbing geometry) # ########################################################## xlo_param = CrystalOrientationParameterisation(crystal) xluc_param = CrystalUnitCellParameterisation(crystal) ################################ # Apply known parameter shifts # ################################ # rotate crystal (=5 mrad each rotation) xlo_p_vals = [] p_vals = xlo_param.get_param_vals() xlo_p_vals.append(p_vals) new_p_vals = [a + b for a, b in zip(p_vals, [5.0, 5.0, 5.0])] xlo_param.set_param_vals(new_p_vals) # change unit cell (=1.0 Angstrom length upsets, 0.5 degree of # gamma angle) xluc_p_vals = [] p_vals = xluc_param.get_param_vals() xluc_p_vals.append(p_vals) cell_params = crystal.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [1.0, 1.0, -1.0, 0.0, 0.0, 0.5]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(crystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) # keep track of the target crystal model to compare with refined from copy import deepcopy target_crystal = deepcopy(crystal) ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere for crystal resolution = 2.0 index_generator = IndexGenerator( crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() # Build a ray predictor and predict rays close to the Ewald sphere by using # the narrow rotation scan ref_predictor = ScansRayPredictor(scans_experiments, sweep_range) obs_refs = ref_predictor(indices, experiment_id=0) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Add in flags and ID columns by copying into standard reflection table tmp = flex.reflection_table.empty_standard(len(obs_refs)) tmp.update(obs_refs) obs_refs = tmp # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180.0 px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2) obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) # Re-predict using the stills reflection predictor stills_ref_predictor = StillsExperimentsPredictor(stills_experiments) obs_refs_stills = stills_ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs_stills["xyzobs.mm.value"] = obs_refs_stills["xyzcal.mm"] ############################### # Undo known parameter shifts # ############################### xlo_param.set_param_vals(xlo_p_vals[0]) xluc_param.set_param_vals(xluc_p_vals[0]) # make a refiner from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse("")).extract() # Change this to get a plot do_plot = False if do_plot: params.refinement.refinery.journal.track_parameter_correlation = True from dials.algorithms.refinement.refiner import RefinerFactory # decrease bin_size_fraction to terminate on RMSD convergence params.refinement.target.bin_size_fraction = 0.01 params.refinement.parameterisation.beam.fix = "all" params.refinement.parameterisation.detector.fix = "all" refiner = RefinerFactory.from_parameters_data_experiments( params, obs_refs_stills, stills_experiments) # run refinement history = refiner.run() # regression tests assert len(history["rmsd"]) == 9 refined_crystal = refiner.get_experiments()[0].crystal uc1 = refined_crystal.get_unit_cell() uc2 = target_crystal.get_unit_cell() assert uc1.is_similar_to(uc2) if do_plot: plt = refiner.parameter_correlation_plot( len(history["parameter_correlation"]) - 1) plt.show()
def test(args=[]): ############################# # Setup experimental models # ############################# master_phil = parse( """ include scope dials.tests.algorithms.refinement.geometry_phil include scope dials.tests.algorithms.refinement.minimiser_phil """, process_includes=True, ) models = setup_geometry.Extract( master_phil, cmdline_args=args, local_overrides="geometry.parameters.random_seed = 1", ) crystal1 = models.crystal models = setup_geometry.Extract( master_phil, cmdline_args=args, local_overrides="geometry.parameters.random_seed = 2", ) mydetector = models.detector mygonio = models.goniometer crystal2 = models.crystal mybeam = models.beam # Build a mock scan for an 18 degree sequence sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 180), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(180)), deg=True, ) sequence_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sequence_range == (0.0, pi / 10) assert approx_equal(im_width, 0.1 * pi / 180.0) # Build an experiment list experiments = ExperimentList() experiments.append( Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=crystal1, imageset=None, )) experiments.append( Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=crystal2, imageset=None, )) assert len(experiments.detectors()) == 1 ########################################################## # Parameterise the models (only for perturbing geometry) # ########################################################## det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xl1o_param = CrystalOrientationParameterisation(crystal1) xl1uc_param = CrystalUnitCellParameterisation(crystal1) xl2o_param = CrystalOrientationParameterisation(crystal2) xl2uc_param = CrystalUnitCellParameterisation(crystal2) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) ################################ # Apply known parameter shifts # ################################ # shift detector by 1.0 mm each translation and 2 mrad each rotation det_p_vals = det_param.get_param_vals() p_vals = [ a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0]) ] det_param.set_param_vals(p_vals) # shift beam by 2 mrad in free axis s0_p_vals = s0_param.get_param_vals() p_vals = list(s0_p_vals) p_vals[0] += 2.0 s0_param.set_param_vals(p_vals) # rotate crystal a bit (=2 mrad each rotation) xlo_p_vals = [] for xlo in (xl1o_param, xl2o_param): p_vals = xlo.get_param_vals() xlo_p_vals.append(p_vals) new_p_vals = [a + b for a, b in zip(p_vals, [2.0, 2.0, 2.0])] xlo.set_param_vals(new_p_vals) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # gamma angle) xluc_p_vals = [] for xluc, xl in ((xl1uc_param, crystal1), ((xl2uc_param, crystal2))): p_vals = xluc.get_param_vals() xluc_p_vals.append(p_vals) cell_params = xl.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(xl.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()]) xluc.set_param_vals(X) ############################# # Generate some reflections # ############################# # All indices in a 2.5 Angstrom sphere for crystal1 resolution = 2.5 index_generator = IndexGenerator( crystal1.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices1 = index_generator.to_array() # All indices in a 2.5 Angstrom sphere for crystal2 resolution = 2.5 index_generator = IndexGenerator( crystal2.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices2 = index_generator.to_array() # Predict rays within the sequence range. Set experiment IDs ray_predictor = ScansRayPredictor(experiments, sequence_range) obs_refs1 = ray_predictor(indices1, experiment_id=0) obs_refs1["id"] = flex.int(len(obs_refs1), 0) obs_refs2 = ray_predictor(indices2, experiment_id=1) obs_refs2["id"] = flex.int(len(obs_refs2), 1) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs1) obs_refs1 = obs_refs1.select(intersects) intersects = ray_intersection(mydetector, obs_refs2) obs_refs2 = obs_refs2.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ScansExperimentsPredictor(experiments) obs_refs1 = ref_predictor(obs_refs1) obs_refs2 = ref_predictor(obs_refs2) # Set 'observed' centroids from the predicted ones obs_refs1["xyzobs.mm.value"] = obs_refs1["xyzcal.mm"] obs_refs2["xyzobs.mm.value"] = obs_refs2["xyzcal.mm"] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 18.0 px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs1), (px_size[0] / 2.0)**2) var_y = flex.double(len(obs_refs1), (px_size[1] / 2.0)**2) var_phi = flex.double(len(obs_refs1), (im_width / 2.0)**2) obs_refs1["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) var_x = flex.double(len(obs_refs2), (px_size[0] / 2.0)**2) var_y = flex.double(len(obs_refs2), (px_size[1] / 2.0)**2) var_phi = flex.double(len(obs_refs2), (im_width / 2.0)**2) obs_refs2["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) # concatenate reflection lists obs_refs1.extend(obs_refs2) obs_refs = obs_refs1 ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xl1o_param.set_param_vals(xlo_p_vals[0]) xl2o_param.set_param_vals(xlo_p_vals[1]) xl1uc_param.set_param_vals(xluc_p_vals[0]) xl2uc_param.set_param_vals(xluc_p_vals[1]) # scan static first params = phil_scope.fetch(source=parse("")).extract() refiner = RefinerFactory.from_parameters_data_experiments( params, obs_refs, experiments) refiner.run() # scan varying params.refinement.parameterisation.scan_varying = True refiner = RefinerFactory.from_parameters_data_experiments( params, obs_refs, experiments) refiner.run() # Ensure all models have scan-varying state set # (https://github.com/dials/dials/issues/798) refined_experiments = refiner.get_experiments() sp = [xl.get_num_scan_points() for xl in refined_experiments.crystals()] assert sp.count(181) == 2