def ref_gen_static(experiments): """Generate some reflections using the static predictor""" beam = experiments[0].beam crystal = experiments[0].crystal goniometer = experiments[0].goniometer detector = experiments[0].detector scan = experiments[0].scan # All indices to the detector max resolution dmin = detector.get_max_resolution(beam.get_s0()) index_generator = IndexGenerator(crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), dmin) indices = index_generator.to_array() # Predict rays within the sweep range sweep_range = scan.get_oscillation_range(deg=False) ray_predictor = ScansRayPredictor(experiments, sweep_range) refs = ray_predictor.predict(indices) # Take only those rays that intersect the detector intersects = ray_intersection(detector, refs) refs = refs.select(intersects) # Make a reflection predictor and re-predict for these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) refs['id'] = flex.int(len(refs), 0) refs = ref_predictor.predict(refs) return refs
def ref_gen_static(experiments): """Generate some reflections using the static predictor""" beam = experiments[0].beam crystal = experiments[0].crystal goniometer = experiments[0].goniometer detector = experiments[0].detector scan = experiments[0].scan # All indices to the detector max resolution dmin = detector.get_max_resolution(beam.get_s0()) index_generator = IndexGenerator( crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), dmin) indices = index_generator.to_array() # Predict rays within the sweep range sweep_range = scan.get_oscillation_range(deg=False) ray_predictor = ScansRayPredictor(experiments, sweep_range) refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(detector, refs) refs = refs.select(intersects) # Make a reflection predictor and re-predict for these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) refs['id'] = flex.int(len(refs), 0) refs = ref_predictor(refs) return refs
def generate_reflections(experiments): from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.refinement.prediction import \ ScansRayPredictor, ExperimentsPredictor from dials.algorithms.spot_prediction import ray_intersection from cctbx.sgtbx import space_group, space_group_symbols from scitbx.array_family import flex detector = experiments[0].detector crystal = experiments[0].crystal # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator(crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() # Predict rays within the sweep range scan = experiments[0].scan sweep_range = scan.get_oscillation_range(deg=False) ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor.predict(indices) # Take only those rays that intersect the detector intersects = ray_intersection(detector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs['id'] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor.predict(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = detector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) return obs_refs, ref_predictor
def run_spherical_relp_stills_pred_param(self, verbose=True): if verbose: print 'Testing derivatives for SphericalRelpStillsPredictionParameterisation' print '=====================================================================' # Build a prediction parameterisation for the stills experiment pred_param = SphericalRelpStillsPredictionParameterisation( self.stills_experiments, detector_parameterisations = [self.det_param], beam_parameterisations = [self.s0_param], xl_orientation_parameterisations = [self.xlo_param], xl_unit_cell_parameterisations = [self.xluc_param]) # Predict the reflections in place. Must do this ahead of calculating # the analytical gradients so quantities like s1 are correct from dials.algorithms.refinement.prediction import ExperimentsPredictor ref_predictor = ExperimentsPredictor(self.stills_experiments, spherical_relp=True) ref_predictor.update() ref_predictor.predict(self.reflections) # get analytical gradients an_grads = pred_param.get_gradients(self.reflections) fd_grads = self.get_fd_gradients(pred_param, ref_predictor) # compare FD with analytical calculations for i, (an_grad, fd_grad) in enumerate(zip(an_grads, fd_grads)): if verbose: print "\nParameter {0}: {1}". format(i, fd_grad['name']) for idx, name in enumerate(["dX_dp", "dY_dp", "dDeltaPsi_dp"]): if verbose: print name for a, b in zip(an_grad[name], fd_grad[name]): if name == 'dDeltaPsi_dp': # DeltaPsi errors are much worse than X, Y errors! # FIXME, look into this further assert approx_equal(a,b, eps=5e-3) else: assert approx_equal(a,b, eps=5e-6) if verbose: print "OK" if verbose: print
def create_models(self, cmdline_overrides=None): from dials.test.algorithms.refinement.setup_geometry import Extract from dxtbx.model import ScanFactory from libtbx.phil import parse if cmdline_overrides is None: cmdline_overrides = [] overrides = """geometry.parameters.crystal.a.length.range = 10 50 geometry.parameters.crystal.b.length.range = 10 50 geometry.parameters.crystal.c.length.range = 10 50""" master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True) # Extract models models = Extract(master_phil, overrides, cmdline_args=cmdline_overrides) self.detector = models.detector self.goniometer = models.goniometer self.crystal = models.crystal self.beam = models.beam # Make a scan of 1-360 * 0.5 deg images sf = ScanFactory() self.scan = sf.make_scan((1, 360), 0.5, (0, 0.5), range(360)) # Generate an ExperimentList self.experiments = ExperimentList() self.experiments.append( Experiment(beam=self.beam, detector=self.detector, goniometer=self.goniometer, scan=self.scan, crystal=self.crystal, imageset=None)) # Create a reflection predictor for the experiments self.ref_predictor = ExperimentsPredictor(self.experiments) # Create scan-varying parameterisations of these models, with 5 samples self.det_param = ScanVaryingDetectorParameterisationSinglePanel( self.detector, self.scan.get_array_range(), 5) self.s0_param = ScanVaryingBeamParameterisation( self.beam, self.scan.get_array_range(), 5, self.goniometer) self.xlo_param = ScanVaryingCrystalOrientationParameterisation( self.crystal, self.scan.get_array_range(), 5) self.xluc_param = ScanVaryingCrystalUnitCellParameterisation( self.crystal, self.scan.get_array_range(), 5) self.gon_param = ScanVaryingGoniometerParameterisation( self.goniometer, self.scan.get_array_range(), 5, self.beam)
def generate_reflections(experiments): from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.refinement.prediction import \ ScansRayPredictor, ExperimentsPredictor from dials.algorithms.spot_prediction import ray_intersection from cctbx.sgtbx import space_group, space_group_symbols from scitbx.array_family import flex detector = experiments[0].detector crystal = experiments[0].crystal # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator( crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() # Predict rays within the sweep range scan = experiments[0].scan sweep_range = scan.get_oscillation_range(deg=False) ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(detector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs['id'] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = detector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) return obs_refs, ref_predictor
def test_spherical_relp_stills_pred_param(tc): print( 'Testing derivatives for SphericalRelpStillsPredictionParameterisation' ) print( '=====================================================================' ) # Build a prediction parameterisation for the stills experiment pred_param = SphericalRelpStillsPredictionParameterisation( tc.stills_experiments, detector_parameterisations=[tc.det_param], beam_parameterisations=[tc.s0_param], xl_orientation_parameterisations=[tc.xlo_param], xl_unit_cell_parameterisations=[tc.xluc_param]) # Predict the reflections in place. Must do this ahead of calculating # the analytical gradients so quantities like s1 are correct from dials.algorithms.refinement.prediction import ExperimentsPredictor ref_predictor = ExperimentsPredictor(tc.stills_experiments, spherical_relp=True) ref_predictor(tc.reflections) # get analytical gradients an_grads = pred_param.get_gradients(tc.reflections) fd_grads = tc.get_fd_gradients(pred_param, ref_predictor) # compare FD with analytical calculations for i, (an_grad, fd_grad) in enumerate(zip(an_grads, fd_grads)): print("\nParameter {0}: {1}".format(i, fd_grad['name'])) for name in ["dX_dp", "dY_dp", "dDeltaPsi_dp"]: print(name) for a, b in zip(an_grad[name], fd_grad[name]): if name == 'dDeltaPsi_dp': # DeltaPsi errors are much worse than X, Y errors! # FIXME, look into this further assert a == pytest.approx(b, abs=5e-3) else: assert a == pytest.approx(b, abs=5e-6) print("OK")
def run_spherical_relp_stills_pred_param(self, verbose=True): if verbose: print 'Testing derivatives for SphericalRelpStillsPredictionParameterisation' print '=====================================================================' # Build a prediction parameterisation for the stills experiment pred_param = SphericalRelpStillsPredictionParameterisation( self.stills_experiments, detector_parameterisations=[self.det_param], beam_parameterisations=[self.s0_param], xl_orientation_parameterisations=[self.xlo_param], xl_unit_cell_parameterisations=[self.xluc_param]) # Predict the reflections in place. Must do this ahead of calculating # the analytical gradients so quantities like s1 are correct from dials.algorithms.refinement.prediction import ExperimentsPredictor ref_predictor = ExperimentsPredictor(self.stills_experiments, spherical_relp=True) ref_predictor(self.reflections) # get analytical gradients an_grads = pred_param.get_gradients(self.reflections) fd_grads = self.get_fd_gradients(pred_param, ref_predictor) # compare FD with analytical calculations for i, (an_grad, fd_grad) in enumerate(zip(an_grads, fd_grads)): if verbose: print "\nParameter {0}: {1}".format(i, fd_grad['name']) for idx, name in enumerate(["dX_dp", "dY_dp", "dDeltaPsi_dp"]): if verbose: print name for a, b in zip(an_grad[name], fd_grad[name]): if name == 'dDeltaPsi_dp': # DeltaPsi errors are much worse than X, Y errors! # FIXME, look into this further assert approx_equal(a, b, eps=5e-3) else: assert approx_equal(a, b, eps=5e-6) if verbose: print "OK" if verbose: print
def create_indexed(self, experiments, filename): print("Simulating indexed observations for {0}".format(filename)) # Extract the experiment exp = experiments[0] # print "Experiment scan range is {0},{1}".format(*exp.scan.get_oscillation_range(deg=True)) # Copy essential columns of the original reflections into a new table # (imageset_id is required for reciprocal_lattice_viewer) obs_refs = flex.reflection_table() cols = [ "id", "imageset_id", "miller_index", "panel", "s1", "flags", "entering", "xyzobs.mm.value", "xyzobs.mm.variance", "xyzcal.mm", "xyzobs.px.value", "xyzobs.px.variance", "xyzcal.px", ] for k in cols: if k in self.original_reflections.keys(): obs_refs[k] = self.original_reflections[k] x_obs, y_obs, phi_obs = obs_refs["xyzobs.mm.value"].parts() # print "Original observation scan range is {0},{1}".format( # flex.min(phi_obs) * RAD_TO_DEG, # flex.max(phi_obs) * RAD_TO_DEG) # Reset panel number to zero prior to prediction obs_refs["panel"] = obs_refs["panel"] * 0 # Predict new centroid positions from dials.algorithms.refinement.prediction import ExperimentsPredictor predictor = ExperimentsPredictor(experiments) predictor(obs_refs) # Get vectors to add as errors to the predicted centroid positions to form # new 'observations' shift_px = flex.vec3_double(self.shiftX_px, self.shiftY_px, self.shiftZ_px) px_size_mm = exp.detector[0].get_pixel_size() image_width_rad = exp.scan.get_oscillation(deg=False)[1] shift = flex.vec3_double( px_size_mm[0] * self.shiftX_px, px_size_mm[1] * self.shiftY_px, image_width_rad * self.shiftZ_px, ) obs_refs["xyzobs.px.value"] = obs_refs["xyzcal.px"] + shift_px obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"] + shift x, y, z = obs_refs["xyzobs.mm.value"].parts() # print "Simulated observation scan range is {0},{1}".format( # flex.min(z) * RAD_TO_DEG, # flex.max(z) * RAD_TO_DEG) # Keep original variance estimates from spot-finding for centroids in # pixels/images, but optionally rescale for centroids in mm/rad. # Note that an overall scale factor for the variance is irrelevant to # refinement. What matters are differences in the relative scale between # the detector space and rotation parts. if self.params.recalculate_centroid_variances: var_x_px, var_y_px, var_z_px = obs_refs[ "xyzobs.px.variance"].parts() var_x_mm = var_x_px * px_size_mm[0]**2 var_y_mm = var_y_px * px_size_mm[1]**2 var_z_rd = var_z_px * image_width_rad**2 obs_refs["xyzobs.mm.variance"] = flex.vec3_double( var_x_mm, var_y_mm, var_z_rd) # Reset observed s1 vectors from dials.algorithms.refinement.refinement_helpers import set_obs_s1 obs_refs["s1"] = flex.vec3_double(len(obs_refs)) set_obs_s1(obs_refs, experiments) return obs_refs
def test(args=[]): # Python and cctbx imports from math import pi from scitbx import matrix from libtbx.phil import parse from libtbx.test_utils import approx_equal # Import for surgery on reflection_tables from dials.array_family import flex # Get module to build models using PHIL import dials.test.algorithms.refinement.setup_geometry as setup_geometry # We will set up a mock scan and a mock experiment list from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import ExperimentList, Experiment # Crystal parameterisations from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalOrientationParameterisation, CrystalUnitCellParameterisation # Symmetry constrained parameterisation for the unit cell from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import \ symmetrize_reduce_enlarge # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.refinement.prediction import ScansRayPredictor, \ ExperimentsPredictor from dials.algorithms.spot_prediction import ray_intersection from cctbx.sgtbx import space_group, space_group_symbols ############################# # Setup experimental models # ############################# master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True) # build models, with a larger crystal than default in order to get enough # reflections on the 'still' image param = """ geometry.parameters.crystal.a.length.range=40 50; geometry.parameters.crystal.b.length.range=40 50; geometry.parameters.crystal.c.length.range=40 50; geometry.parameters.random_seed = 42""" models = setup_geometry.Extract(master_phil, cmdline_args=args, local_overrides=param) crystal = models.crystal mydetector = models.detector mygonio = models.goniometer mybeam = models.beam # Build a mock scan for a 1.5 degree wedge. Only used for generating indices near # the Ewald sphere sf = ScanFactory() myscan = sf.make_scan(image_range=(1, 1), exposure_times=0.1, oscillation=(0, 1.5), epochs=range(1), deg=True) sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert approx_equal(im_width, 1.5 * pi / 180.) # Build experiment lists stills_experiments = ExperimentList() stills_experiments.append( Experiment(beam=mybeam, detector=mydetector, crystal=crystal, imageset=None)) scans_experiments = ExperimentList() scans_experiments.append( Experiment(beam=mybeam, detector=mydetector, crystal=crystal, goniometer=mygonio, scan=myscan, imageset=None)) ########################################################## # Parameterise the models (only for perturbing geometry) # ########################################################## xlo_param = CrystalOrientationParameterisation(crystal) xluc_param = CrystalUnitCellParameterisation(crystal) ################################ # Apply known parameter shifts # ################################ # rotate crystal (=5 mrad each rotation) xlo_p_vals = [] p_vals = xlo_param.get_param_vals() xlo_p_vals.append(p_vals) new_p_vals = [a + b for a, b in zip(p_vals, [5., 5., 5.])] xlo_param.set_param_vals(new_p_vals) # change unit cell (=1.0 Angstrom length upsets, 0.5 degree of # gamma angle) xluc_p_vals = [] p_vals = xluc_param.get_param_vals() xluc_p_vals.append(p_vals) cell_params = crystal.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [1.0, 1.0, -1.0, 0.0, 0.0, 0.5]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(crystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) # keep track of the target crystal model to compare with refined from copy import deepcopy target_crystal = deepcopy(crystal) ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere for crystal resolution = 2.0 index_generator = IndexGenerator( crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() # Build a ray predictor and predict rays close to the Ewald sphere by using # the narrow rotation scan ref_predictor = ScansRayPredictor(scans_experiments, sweep_range) obs_refs = ref_predictor(indices, experiment_id=0) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Add in flags and ID columns by copying into standard reflection table tmp = flex.reflection_table.empty_standard(len(obs_refs)) tmp.update(obs_refs) obs_refs = tmp # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) # Re-predict using the stills reflection predictor stills_ref_predictor = ExperimentsPredictor(stills_experiments) obs_refs_stills = stills_ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs_stills['xyzobs.mm.value'] = obs_refs_stills['xyzcal.mm'] ############################### # Undo known parameter shifts # ############################### xlo_param.set_param_vals(xlo_p_vals[0]) xluc_param.set_param_vals(xluc_p_vals[0]) # make a refiner from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse('')).extract() # Change this to get a plot do_plot = False if do_plot: params.refinement.refinery.journal.track_parameter_correlation = True from dials.algorithms.refinement.refiner import RefinerFactory # decrease bin_size_fraction to terminate on RMSD convergence params.refinement.target.bin_size_fraction = 0.01 params.refinement.parameterisation.beam.fix = "all" params.refinement.parameterisation.detector.fix = "all" refiner = RefinerFactory.from_parameters_data_experiments( params, obs_refs_stills, stills_experiments, verbosity=0) # run refinement history = refiner.run() # regression tests assert len(history["rmsd"]) == 9 refined_crystal = refiner.get_experiments()[0].crystal uc1 = refined_crystal.get_unit_cell() uc2 = target_crystal.get_unit_cell() assert uc1.is_similar_to(uc2) if do_plot: plt = refiner.parameter_correlation_plot( len(history["parameter_correlation"]) - 1) plt.show()
def run_stills_pred_param(self, verbose = False): if verbose: print 'Testing derivatives for StillsPredictionParameterisation' print '========================================================' # Build a prediction parameterisation for the stills experiment pred_param = StillsPredictionParameterisation(self.stills_experiments, detector_parameterisations = [self.det_param], beam_parameterisations = [self.s0_param], xl_orientation_parameterisations = [self.xlo_param], xl_unit_cell_parameterisations = [self.xluc_param]) # Predict the reflections in place. Must do this ahead of calculating # the analytical gradients so quantities like s1 are correct from dials.algorithms.refinement.prediction import ExperimentsPredictor ref_predictor = ExperimentsPredictor(self.stills_experiments) ref_predictor.update() ref_predictor.predict(self.reflections) # get analytical gradients an_grads = pred_param.get_gradients(self.reflections) fd_grads = self.get_fd_gradients(pred_param, ref_predictor) for i, (an_grad, fd_grad) in enumerate(zip(an_grads, fd_grads)): # compare FD with analytical calculations if verbose: print "\nParameter {0}: {1}". format(i, fd_grad['name']) for idx, name in enumerate(["dX_dp", "dY_dp", "dDeltaPsi_dp"]): if verbose: print name a = fd_grad[name] b = an_grad[name] abs_error = a - b denom = a + b fns = five_number_summary(abs_error) if verbose: print (" summary of absolute errors: %9.6f %9.6f %9.6f " + \ "%9.6f %9.6f") % fns assert flex.max(flex.abs(abs_error)) < 0.0003 # largest absolute error found to be about 0.00025 for dY/dp of # Crystal0g_param_3. Reject outlying absolute errors and test again. iqr = fns[3] - fns[1] # skip further stats on errors with an iqr of near zero, e.g. dDeltaPsi_dp # for detector parameters, which are all equal to zero if iqr < 1.e-10: continue sel1 = abs_error < fns[3] + 1.5 * iqr sel2 = abs_error > fns[1] - 1.5 * iqr sel = sel1 & sel2 tst = flex.max_index(flex.abs(abs_error.select(sel))) tst_val = abs_error.select(sel)[tst] n_outliers = sel.count(False) if verbose: print (" {0} outliers rejected, leaving greatest " + \ "absolute error: {1:9.6f}").format(n_outliers, tst_val) # largest absolute error now 0.000086 for dX/dp of Beam0Mu2 assert abs(tst_val) < 0.00009 # Completely skip parameters with FD gradients all zero (e.g. gradients of # DeltaPsi for detector parameters) sel1 = flex.abs(a) < 1.e-10 if sel1.all_eq(True): continue # otherwise calculate normalised errors, by dividing absolute errors by # the IQR (more stable than relative error calculation) norm_error = abs_error / iqr fns = five_number_summary(norm_error) if verbose: print (" summary of normalised errors: %9.6f %9.6f %9.6f " + \ "%9.6f %9.6f") % fns # largest normalised error found to be about 25.7 for dY/dp of # Crystal0g_param_3. try: assert flex.max(flex.abs(norm_error)) < 30 except AssertionError as e: e.args += ("extreme normalised error value: {0}".format( flex.max(flex.abs(norm_error))),) raise e # Reject outlying normalised errors and test again iqr = fns[3] - fns[1] if iqr > 0.: sel1 = norm_error < fns[3] + 1.5 * iqr sel2 = norm_error > fns[1] - 1.5 * iqr sel = sel1 & sel2 tst = flex.max_index(flex.abs(norm_error.select(sel))) tst_val = norm_error.select(sel)[tst] n_outliers = sel.count(False) # most outliers found for for dY/dp of Crystal0g_param_3 (which had # largest errors, so no surprise there). try: assert n_outliers < 250 except AssertionError as e: e.args += ("too many outliers rejected: {0}".format(n_outliers),) raise e if verbose: print (" {0} outliers rejected, leaving greatest " + \ "normalised error: {1:9.6f}").format(n_outliers, tst_val) # largest normalied error now about -4. for dX/dp of Detector0Tau1 assert abs(tst_val) < 4.5 if verbose: print return
def run_stills_pred_param(self, verbose=False): if verbose: print 'Testing derivatives for StillsPredictionParameterisation' print '========================================================' # Build a prediction parameterisation for the stills experiment pred_param = StillsPredictionParameterisation( self.stills_experiments, detector_parameterisations=[self.det_param], beam_parameterisations=[self.s0_param], xl_orientation_parameterisations=[self.xlo_param], xl_unit_cell_parameterisations=[self.xluc_param]) # Predict the reflections in place. Must do this ahead of calculating # the analytical gradients so quantities like s1 are correct from dials.algorithms.refinement.prediction import ExperimentsPredictor ref_predictor = ExperimentsPredictor(self.stills_experiments) ref_predictor(self.reflections) # get analytical gradients an_grads = pred_param.get_gradients(self.reflections) fd_grads = self.get_fd_gradients(pred_param, ref_predictor) for i, (an_grad, fd_grad) in enumerate(zip(an_grads, fd_grads)): # compare FD with analytical calculations if verbose: print "\nParameter {0}: {1}".format(i, fd_grad['name']) for idx, name in enumerate(["dX_dp", "dY_dp", "dDeltaPsi_dp"]): if verbose: print name a = fd_grad[name] b = an_grad[name] abs_error = a - b denom = a + b fns = five_number_summary(abs_error) if verbose: print (" summary of absolute errors: %9.6f %9.6f %9.6f " + \ "%9.6f %9.6f") % fns assert flex.max(flex.abs(abs_error)) < 0.0003 # largest absolute error found to be about 0.00025 for dY/dp of # Crystal0g_param_3. Reject outlying absolute errors and test again. iqr = fns[3] - fns[1] # skip further stats on errors with an iqr of near zero, e.g. dDeltaPsi_dp # for detector parameters, which are all equal to zero if iqr < 1.e-10: continue sel1 = abs_error < fns[3] + 1.5 * iqr sel2 = abs_error > fns[1] - 1.5 * iqr sel = sel1 & sel2 tst = flex.max_index(flex.abs(abs_error.select(sel))) tst_val = abs_error.select(sel)[tst] n_outliers = sel.count(False) if verbose: print (" {0} outliers rejected, leaving greatest " + \ "absolute error: {1:9.6f}").format(n_outliers, tst_val) # largest absolute error now 0.000086 for dX/dp of Beam0Mu2 assert abs(tst_val) < 0.00009 # Completely skip parameters with FD gradients all zero (e.g. gradients of # DeltaPsi for detector parameters) sel1 = flex.abs(a) < 1.e-10 if sel1.all_eq(True): continue # otherwise calculate normalised errors, by dividing absolute errors by # the IQR (more stable than relative error calculation) norm_error = abs_error / iqr fns = five_number_summary(norm_error) if verbose: print (" summary of normalised errors: %9.6f %9.6f %9.6f " + \ "%9.6f %9.6f") % fns # largest normalised error found to be about 25.7 for dY/dp of # Crystal0g_param_3. try: assert flex.max(flex.abs(norm_error)) < 30 except AssertionError as e: e.args += ("extreme normalised error value: {0}".format( flex.max(flex.abs(norm_error))), ) raise e # Reject outlying normalised errors and test again iqr = fns[3] - fns[1] if iqr > 0.: sel1 = norm_error < fns[3] + 1.5 * iqr sel2 = norm_error > fns[1] - 1.5 * iqr sel = sel1 & sel2 tst = flex.max_index(flex.abs(norm_error.select(sel))) tst_val = norm_error.select(sel)[tst] n_outliers = sel.count(False) # most outliers found for for dY/dp of Crystal0g_param_3 (which had # largest errors, so no surprise there). try: assert n_outliers < 250 except AssertionError as e: e.args += ("too many outliers rejected: {0}".format( n_outliers), ) raise e if verbose: print (" {0} outliers rejected, leaving greatest " + \ "normalised error: {1:9.6f}").format(n_outliers, tst_val) # largest normalied error now about -4. for dX/dp of Detector0Tau1 assert abs( tst_val) < 4.5, 'should be about 4 not %s' % tst_val if verbose: print return
# Predict rays within the sweep range. Set experiment IDs ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs1 = ray_predictor.predict(indices1, experiment_id=0) obs_refs1['id'] = flex.int(len(obs_refs1), 0) obs_refs2 = ray_predictor.predict(indices1, experiment_id=1) obs_refs2['id'] = flex.int(len(obs_refs2), 1) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs1) obs_refs1 = obs_refs1.select(intersects) intersects = ray_intersection(mydetector, obs_refs2) obs_refs2 = obs_refs2.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs1 = ref_predictor.predict(obs_refs1) obs_refs2 = ref_predictor.predict(obs_refs2) # Set 'observed' centroids from the predicted ones obs_refs1['xyzobs.mm.value'] = obs_refs1['xyzcal.mm'] obs_refs2['xyzobs.mm.value'] = obs_refs2['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs1), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs1), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs1), (im_width / 2.)**2) obs_refs1['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) var_x = flex.double(len(obs_refs2), (px_size[0] / 2.)**2)
def choose_best_orientation_matrix(self, candidate_orientation_matrices): from dxtbx.model.experiment_list import Experiment, ExperimentList import copy logger.info('*' * 80) logger.info('Selecting the best orientation matrix') logger.info('*' * 80) from libtbx import group_args class candidate_info(group_args): pass candidates = [] params = copy.deepcopy(self.all_params) n_cand = len(candidate_orientation_matrices) for icm,cm in enumerate(candidate_orientation_matrices): sel = ((self.reflections['id'] == -1)) #(1/self.reflections['rlp'].norms() > self.d_min)) refl = self.reflections.select(sel) experiments = self.experiment_list_for_crystal(cm) self.index_reflections(experiments, refl) indexed = refl.select(refl['id'] >= 0) indexed = indexed.select(indexed.get_flags(indexed.flags.indexed)) if params.indexing.stills.refine_all_candidates: try: print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d initial outlier identification"%(icm, n_cand) acceptance_flags = self.identify_outliers(params, experiments, indexed) #create a new "indexed" list with outliers thrown out: indexed = indexed.select(acceptance_flags) print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d refinement before outlier rejection"%(icm, n_cand) R = e_refine(params = params, experiments=experiments, reflections=indexed, graph_verbose=False) ref_experiments = R.get_experiments() # try to improve the outcome with a second round of outlier rejection post-initial refinement: acceptance_flags = self.identify_outliers(params, ref_experiments, indexed) # insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection nv0 = nave_parameters(params = params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False) crystal_model_nv0 = nv0() acceptance_flags_nv0 = nv0.nv_acceptance_flags indexed = indexed.select(acceptance_flags & acceptance_flags_nv0) print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d after positional and delta-psi outlier rejection"%(icm, n_cand) R = e_refine(params = params, experiments=ref_experiments, reflections=indexed, graph_verbose=False) ref_experiments = R.get_experiments() nv = nave_parameters(params = params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False) crystal_model = nv() # Drop candidates that after refinement can no longer be converted to the known target space group if self.params.known_symmetry.space_group is not None: target_space_group = self.target_symmetry_primitive.space_group() new_crystal, cb_op_to_primitive = self.apply_symmetry(crystal_model, target_space_group) if new_crystal is None: print "P1 refinement yielded model diverged from target, candidate %d/%d"%(icm, n_cand) continue rmsd, _ = calc_2D_rmsd_and_displacements(R.predict_for_reflection_table(indexed)) except Exception, e: print "Couldn't refine candiate %d/%d, %s"%(icm, n_cand, str(e)) else: print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d done"%(icm, n_cand) candidates.append(candidate_info(crystal = crystal_model, green_curve_area = nv.green_curve_area, ewald_proximal_volume = nv.ewald_proximal_volume(), n_indexed = len(indexed), rmsd = rmsd, indexed = indexed, experiments = ref_experiments)) else: from dials.algorithms.refinement.prediction import ExperimentsPredictor ref_predictor = ExperimentsPredictor(experiments, force_stills=True, spherical_relp=params.refinement.parameterisation.spherical_relp_model) rmsd, _ = calc_2D_rmsd_and_displacements(ref_predictor(indexed)) candidates.append(candidate_info(crystal = cm, n_indexed = len(indexed), rmsd = rmsd, indexed = indexed, experiments = experiments))
def test2(): """Test on simulated data""" # Get models for reflection prediction import dials.test.algorithms.refinement.setup_geometry as setup_geometry from libtbx.phil import parse overrides = """geometry.parameters.crystal.a.length.value = 77 geometry.parameters.crystal.b.length.value = 77 geometry.parameters.crystal.c.length.value = 37""" master_phil = parse( """ include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True, ) from dxtbx.model import Crystal models = setup_geometry.Extract(master_phil) crystal = Crystal( real_space_a=(2.62783398111729, -63.387215823567125, -45.751375737456975), real_space_b=(15.246640559660356, -44.48254330406616, 62.50501032727026), real_space_c=(-76.67246874451074, -11.01804131886244, 10.861322446352226), space_group_symbol="I 2 3", ) detector = models.detector goniometer = models.goniometer beam = models.beam # Build a mock scan for a 180 degree sweep from dxtbx.model import ScanFactory sf = ScanFactory() scan = sf.make_scan( image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=range(1800), deg=True, ) # Build an experiment list from dxtbx.model.experiment_list import ExperimentList, Experiment experiments = ExperimentList() experiments.append( Experiment( beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal, imageset=None, )) # Generate all indices in a 1.5 Angstrom sphere from dials.algorithms.spot_prediction import IndexGenerator from cctbx.sgtbx import space_group, space_group_symbols resolution = 1.5 index_generator = IndexGenerator( crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() # Predict rays within the sweep range from dials.algorithms.refinement.prediction import ScansRayPredictor sweep_range = scan.get_oscillation_range(deg=False) ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector from dials.algorithms.spot_prediction import ray_intersection intersects = ray_intersection(detector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns from dials.algorithms.refinement.prediction import ExperimentsPredictor ref_predictor = ExperimentsPredictor(experiments) obs_refs["id"] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor(obs_refs) # Copy 'observed' centroids from the predicted ones, applying sinusoidal # offsets obs_x, obs_y, obs_z = obs_refs["xyzcal.mm"].parts() # obs_z is in range (0, pi). Calculate offsets for phi at twice that # frequency im_width = scan.get_oscillation(deg=False)[1] z_off = flex.sin(2 * obs_z) * im_width obs_z += z_off # Calculate offsets for x pixel_size = detector[0].get_pixel_size() x_off = flex.sin(20 * obs_z) * pixel_size[0] # Calculate offsets for y with a phase-shifted sine wave from math import pi y_off = flex.sin(4 * obs_z + pi / 6) * pixel_size[1] # Incorporate the offsets into the 'observed' centroids obs_z += z_off obs_x += x_off obs_y += y_off obs_refs["xyzobs.mm.value"] = flex.vec3_double(obs_x, obs_y, obs_z) # Now do centroid analysis of the residuals results = CentroidAnalyser(obs_refs, debug=True)() # FIXME this test shows that the suggested interval width heuristic is not # yet robust. This simulation function seems a useful direction to proceed # in though raise RuntimeError("test2 failed") print("OK") return
def test(args=[]): # Python and cctbx imports from math import pi import random from scitbx import matrix from scitbx.array_family import flex from libtbx.phil import parse from libtbx.test_utils import approx_equal # Experimental model builder from dials.test.algorithms.refinement.setup_geometry import Extract # We will set up a mock scan and a mock experiment list from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import ExperimentList, Experiment # Model parameterisations from dials.algorithms.refinement.parameterisation.detector_parameters import \ DetectorParameterisationSinglePanel from dials.algorithms.refinement.parameterisation.beam_parameters import \ BeamParameterisation from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalOrientationParameterisation, CrystalUnitCellParameterisation # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection from dials.algorithms.refinement.prediction import ScansRayPredictor, \ ExperimentsPredictor from cctbx.sgtbx import space_group, space_group_symbols # Parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.prediction_parameters import \ XYPhiPredictionParameterisation # Imports for the target function from dials.algorithms.refinement.target import \ LeastSquaresPositionalResidualWithRmsdCutoff from dials.algorithms.refinement.reflection_manager import ReflectionManager # Local functions def random_direction_close_to(vector, sd = 0.5): return vector.rotate_around_origin(matrix.col( (random.random(), random.random(), random.random())).normalize(), random.gauss(0, sd), deg = True) ############################# # Setup experimental models # ############################# # make a small cell to speed up calculations overrides = """geometry.parameters.crystal.a.length.range = 10 15 geometry.parameters.crystal.b.length.range = 10 15 geometry.parameters.crystal.c.length.range = 10 15""" master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True) models = Extract(master_phil, overrides, cmdline_args = args) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam # Build a mock scan for a 180 degree sweep of 0.1 degree images sf = ScanFactory() myscan = sf.make_scan(image_range = (1,1800), exposure_times = 0.1, oscillation = (0, 0.1), epochs = range(1800), deg = True) sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) experiments = ExperimentList() experiments.append(Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None)) ########################### # Parameterise the models # ########################### det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ######################################################################## pred_param = XYPhiPredictionParameterisation(experiments, [det_param], [s0_param], [xlo_param], [xluc_param]) ################################ # Apply known parameter shifts # ################################ # shift detector by 0.2 mm each translation and 2 mrad each rotation det_p_vals = det_param.get_param_vals() p_vals = [a + b for a, b in zip(det_p_vals, [2.0, 2.0, 2.0, 2.0, 2.0, 2.0])] det_param.set_param_vals(p_vals) # shift beam by 2 mrad in one axis s0_p_vals = s0_param.get_param_vals() p_vals = list(s0_p_vals) p_vals[1] += 2.0 s0_param.set_param_vals(p_vals) # rotate crystal a bit (=2 mrad each rotation) xlo_p_vals = xlo_param.get_param_vals() p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])] xlo_param.set_param_vals(p_vals) ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator(mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() # Predict rays within the sweep range ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs['id'] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) ##################################### # Select reflections for refinement # ##################################### refman = ReflectionManager(obs_refs, experiments) ############################## # Set up the target function # ############################## # Redefine the reflection predictor to use the type expected by the Target class ref_predictor = ExperimentsPredictor(experiments) mytarget = LeastSquaresPositionalResidualWithRmsdCutoff( experiments, ref_predictor, refman, pred_param, restraints_parameterisation=None) # get the functional and gradients mytarget.predict() L, dL_dp, curvs = mytarget.compute_functional_gradients_and_curvatures() #################################### # Do FD calculation for comparison # #################################### # function for calculating finite difference gradients of the target function def get_fd_gradients(target, pred_param, deltas): """Calculate centered finite difference gradients for each of the parameters of the target function. "deltas" must be a sequence of the same length as the parameter list, and contains the step size for the difference calculations for each parameter. """ p_vals = pred_param.get_param_vals() assert len(deltas) == len(p_vals) fd_grad = [] fd_curvs = [] for i in range(len(deltas)): val = p_vals[i] p_vals[i] -= deltas[i] / 2. pred_param.set_param_vals(p_vals) target.predict() rev_state = target.compute_functional_gradients_and_curvatures() p_vals[i] += deltas[i] pred_param.set_param_vals(p_vals) target.predict() fwd_state = target.compute_functional_gradients_and_curvatures() # finite difference estimation of first derivatives fd_grad.append((fwd_state[0] - rev_state[0]) / deltas[i]) # finite difference estimation of curvatures, using the analytical # first derivatives fd_curvs.append((fwd_state[1][i] - rev_state[1][i]) / deltas[i]) # set parameter back to centred value p_vals[i] = val # return to the initial state pred_param.set_param_vals(p_vals) return fd_grad, fd_curvs # test normalised differences between FD and analytical calculations fdgrads = get_fd_gradients(mytarget, pred_param, [1.e-7] * len(pred_param)) diffs = [a - b for a, b in zip(dL_dp, fdgrads[0])] norm_diffs = tuple([a / b for a, b in zip(diffs, fdgrads[0])]) for e in norm_diffs: assert abs(e) < 0.001 # check differences less than 0.1% # test normalised differences between FD curvatures and analytical least # squares approximation. We don't expect this to be especially close if curvs: diffs = [a - b for a, b in zip(curvs, fdgrads[1])] norm_diffs = tuple([a / b for a, b in zip(diffs, fdgrads[1])]) for e in norm_diffs: assert abs(e) < 0.1 # check differences less than 10%
sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) # Predict rays within the sweep range ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor.predict(indices) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs['id'] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor.predict(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) # The total number of observations should be 1128
def test(): from libtbx.phil import parse from scitbx import matrix from scitbx.array_family import flex # Building experimental models from dials.test.algorithms.refinement.setup_geometry import Extract from dxtbx.model.experiment_list import ExperimentList, Experiment # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.refinement.prediction import ScansRayPredictor, \ ExperimentsPredictor from cctbx.sgtbx import space_group, space_group_symbols # We will set up a mock scan from dxtbx.model import ScanFactory master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True) overrides = """geometry.parameters.crystal.a.length.range = 10 50 geometry.parameters.crystal.b.length.range = 10 50 geometry.parameters.crystal.c.length.range = 10 50""" models = Extract(master_phil, local_overrides=overrides) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator( mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() # Build a mock scan for a 30 degree sweep sf = ScanFactory() myscan = sf.make_scan(image_range=(1, 300), exposure_times=0.1, oscillation=(0, 0.1), epochs=range(300), deg=True) sweep_range = myscan.get_oscillation_range(deg=False) assert sweep_range == pytest.approx((0., math.pi / 6.)) im_width = myscan.get_oscillation(deg=False)[1] assert im_width == pytest.approx(0.1 * math.pi / 180.) # Create an ExperimentList for ScansRayPredictor experiments = ExperimentList() experiments.append( Experiment(beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None)) # Select those that are excited in a 30 degree sweep and get angles UB = mycrystal.get_A() ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor(indices) # Set the experiment number obs_refs['id'] = flex.int(len(obs_refs), 0) # Calculate intersections ref_predictor = ExperimentsPredictor(experiments) obs_refs = ref_predictor(obs_refs) print("Total number of observations made", len(obs_refs)) mypanel = mydetector[0] s0 = matrix.col(mybeam.get_s0()) spindle = matrix.col(mygonio.get_rotation_axis()) for ref in obs_refs: # get the s1 vector of this reflection s1 = matrix.col(ref['s1']) r = s1 - s0 r_orig = r.rotate_around_origin(spindle, -1., deg=True) # is it outside the Ewald sphere (i.e. entering)? test = (s0 + r_orig).length() > s0.length() assert ref['entering'] == test
def test(args=[]): ############################# # Setup experimental models # ############################# master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True) models = setup_geometry.Extract(master_phil, cmdline_args=args) single_panel_detector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam # Make a 3x3 multi panel detector filling the same space as the existing # single panel detector. Each panel of the multi-panel detector has pixels with # 1/3 the length dimensions of the single panel. multi_panel_detector = Detector() for x in range(3): for y in range(3): new_panel = make_panel_in_array((x, y), single_panel_detector[0]) multi_panel_detector.add_panel(new_panel) # Build a mock scan for a 180 degree sweep sf = ScanFactory() myscan = sf.make_scan(image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=range(1800), deg=True) sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) # Build ExperimentLists experiments_single_panel = ExperimentList() experiments_multi_panel = ExperimentList() experiments_single_panel.append( Experiment(beam=mybeam, detector=single_panel_detector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None)) experiments_multi_panel.append( Experiment(beam=mybeam, detector=multi_panel_detector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None)) ########################### # Parameterise the models # ########################### det_param = DetectorParameterisationSinglePanel(single_panel_detector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) multi_det_param = DetectorParameterisationMultiPanel( multi_panel_detector, mybeam) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) # Fix crystal parameters #xluc_param.set_fixed([True, True, True, True, True, True]) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ######################################################################## pred_param = XYPhiPredictionParameterisation(experiments_single_panel, [det_param], [s0_param], [xlo_param], [xluc_param]) pred_param2 = XYPhiPredictionParameterisation(experiments_multi_panel, [multi_det_param], [s0_param], [xlo_param], [xluc_param]) ################################ # Apply known parameter shifts # ################################ # shift detectors by 1.0 mm each translation and 2 mrad each rotation det_p_vals = det_param.get_param_vals() p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2., 2., 2.])] det_param.set_param_vals(p_vals) multi_det_p_vals = multi_det_param.get_param_vals() p_vals = [ a + b for a, b in zip(multi_det_p_vals, [1.0, 1.0, 1.0, 2., 2., 2.]) ] multi_det_param.set_param_vals(p_vals) # shift beam by 2 mrad in free axis s0_p_vals = s0_param.get_param_vals() p_vals = list(s0_p_vals) p_vals[0] += 2. s0_param.set_param_vals(p_vals) # rotate crystal a bit (=2 mrad each rotation) xlo_p_vals = xlo_param.get_param_vals() p_vals = [a + b for a, b in zip(xlo_p_vals, [2., 2., 2.])] xlo_param.set_param_vals(p_vals) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # gamma angle) xluc_p_vals = xluc_param.get_param_vals() cell_params = mycrystal.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(mycrystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator( mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() # for the reflection predictor, it doesn't matter which experiment list is # passed, as the detector is not used ref_predictor = ScansRayPredictor(experiments_single_panel, sweep_range) # get two sets of identical reflections obs_refs = ref_predictor(indices) obs_refs2 = ref_predictor(indices) for r1, r2 in zip(obs_refs, obs_refs2): assert r1['s1'] == r2['s1'] # get the panel intersections sel = ray_intersection(single_panel_detector, obs_refs) obs_refs = obs_refs.select(sel) sel = ray_intersection(multi_panel_detector, obs_refs2) obs_refs2 = obs_refs2.select(sel) assert len(obs_refs) == len(obs_refs2) # Set 'observed' centroids from the predicted ones obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] obs_refs2['xyzobs.mm.value'] = obs_refs2['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = single_panel_detector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) # set the variances and frame numbers obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) obs_refs2['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) # Add in flags and ID columns by copying into standard reflection tables tmp = flex.reflection_table.empty_standard(len(obs_refs)) tmp.update(obs_refs) obs_refs = tmp tmp = flex.reflection_table.empty_standard(len(obs_refs2)) tmp.update(obs_refs2) obs_refs2 = tmp ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) multi_det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) xluc_param.set_param_vals(xluc_p_vals) ##################################### # Select reflections for refinement # ##################################### refman = ReflectionManager(obs_refs, experiments_single_panel) refman2 = ReflectionManager(obs_refs, experiments_multi_panel) ############################### # Set up the target functions # ############################### mytarget = LeastSquaresPositionalResidualWithRmsdCutoff( experiments_single_panel, ExperimentsPredictor(experiments_single_panel), refman, pred_param, restraints_parameterisation=None) mytarget2 = LeastSquaresPositionalResidualWithRmsdCutoff( experiments_multi_panel, ExperimentsPredictor(experiments_multi_panel), refman2, pred_param2, restraints_parameterisation=None) ################################# # Set up the refinement engines # ################################# refiner = setup_minimiser.Extract(master_phil, mytarget, pred_param, cmdline_args=args).refiner refiner2 = setup_minimiser.Extract(master_phil, mytarget2, pred_param2, cmdline_args=args).refiner refiner.run() # reset parameters and run refinement with the multi panel detector s0_param.set_param_vals(s0_p_vals) multi_det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) xluc_param.set_param_vals(xluc_p_vals) refiner2.run() # same number of steps assert refiner.get_num_steps() == refiner2.get_num_steps() # same rmsds for rmsd, rmsd2 in zip(refiner.history["rmsd"], refiner2.history["rmsd"]): assert approx_equal(rmsd, rmsd2) # same parameter values each step for params, params2 in zip(refiner.history["parameter_vector"], refiner.history["parameter_vector"]): assert approx_equal(params, params2)
def test(): from cctbx.sgtbx import space_group, space_group_symbols from libtbx.phil import parse from scitbx.array_family import flex ##### Import model builder from dials.test.algorithms.refinement.setup_geometry import Extract ##### Imports for reflection prediction from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection from dxtbx.model.experiment_list import ExperimentList, Experiment from dials.algorithms.refinement.prediction import ScansRayPredictor, \ ExperimentsPredictor #### Import model parameterisations from dials.algorithms.refinement.parameterisation.prediction_parameters import \ XYPhiPredictionParameterisation from dials.algorithms.refinement.parameterisation.detector_parameters import \ DetectorParameterisationSinglePanel from dials.algorithms.refinement.parameterisation.beam_parameters import \ BeamParameterisation from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalOrientationParameterisation, \ CrystalUnitCellParameterisation from dials.algorithms.refinement.parameterisation.goniometer_parameters import \ GoniometerParameterisation #### Create models overrides = """geometry.parameters.crystal.a.length.range = 10 50 geometry.parameters.crystal.b.length.range = 10 50 geometry.parameters.crystal.c.length.range = 10 50""" master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True) models = Extract(master_phil, overrides) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam # Build a mock scan for a 72 degree sweep sweep_range = (0., math.pi/5.) from dxtbx.model import ScanFactory sf = ScanFactory() myscan = sf.make_scan(image_range = (1,720), exposure_times = 0.1, oscillation = (0, 0.1), epochs = range(720), deg = True) #### Create parameterisations of these models det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) gon_param = GoniometerParameterisation(mygonio, mybeam) # Create an ExperimentList experiments = ExperimentList() experiments.append(Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None)) #### Unit tests # Build a prediction parameterisation pred_param = XYPhiPredictionParameterisation(experiments, detector_parameterisations = [det_param], beam_parameterisations = [s0_param], xl_orientation_parameterisations = [xlo_param], xl_unit_cell_parameterisations = [xluc_param], goniometer_parameterisations = [gon_param]) # Generate reflections resolution = 2.0 index_generator = IndexGenerator(mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() # Predict rays within the sweep range ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs['id'] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * math.pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) # use a ReflectionManager to exclude reflections too close to the spindle from dials.algorithms.refinement.reflection_manager import ReflectionManager refman = ReflectionManager(obs_refs, experiments, outlier_detector=None) # Redefine the reflection predictor to use the type expected by the Target class ref_predictor = ExperimentsPredictor(experiments) # make a target to ensure reflections are predicted and refman is finalised from dials.algorithms.refinement.target import \ LeastSquaresPositionalResidualWithRmsdCutoff target = LeastSquaresPositionalResidualWithRmsdCutoff(experiments, ref_predictor, refman, pred_param, restraints_parameterisation=None) # keep only those reflections that pass inclusion criteria and have predictions reflections = refman.get_matches() # get analytical gradients an_grads = pred_param.get_gradients(reflections) # get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.e-7] * len(p_vals) for i, delta in enumerate(deltas): val = p_vals[i] p_vals[i] -= delta / 2. pred_param.set_param_vals(p_vals) ref_predictor(reflections) rev_state = reflections['xyzcal.mm'].deep_copy() p_vals[i] += delta pred_param.set_param_vals(p_vals) ref_predictor(reflections) fwd_state = reflections['xyzcal.mm'].deep_copy() p_vals[i] = val fd = (fwd_state - rev_state) x_grads, y_grads, phi_grads = fd.parts() x_grads /= delta y_grads /= delta phi_grads /= delta # compare with analytical calculation assert x_grads == pytest.approx(an_grads[i]["dX_dp"], abs=5.e-6) assert y_grads == pytest.approx(an_grads[i]["dY_dp"], abs=5.e-6) assert phi_grads == pytest.approx(an_grads[i]["dphi_dp"], abs=5.e-6) # return to the initial state pred_param.set_param_vals(p_vals)
# Add in flags and ID columns by copying into standard reflection table tmp = flex.reflection_table.empty_standard(len(obs_refs)) tmp.update(obs_refs) obs_refs = tmp # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) # Re-predict using the stills reflection predictor stills_ref_predictor = ExperimentsPredictor(stills_experiments) stills_ref_predictor.update() obs_refs_stills = stills_ref_predictor.predict(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs_stills['xyzobs.mm.value'] = obs_refs_stills['xyzcal.mm'] ############################### # Undo known parameter shifts # ############################### xlo_param.set_param_vals(xlo_p_vals[0]) xluc_param.set_param_vals(xluc_p_vals[0]) # make a refiner from dials.algorithms.refinement.refiner import phil_scope
def test(args=[]): # Python and cctbx imports from math import pi from scitbx import matrix from scitbx.array_family import flex from libtbx.phil import parse from libtbx.test_utils import approx_equal # Get module to build models using PHIL import dials.test.algorithms.refinement.setup_geometry as setup_geometry # We will set up a mock scan and a mock experiment list from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import ExperimentList, Experiment # Model parameterisations from dials.algorithms.refinement.parameterisation.detector_parameters import \ DetectorParameterisationSinglePanel from dials.algorithms.refinement.parameterisation.beam_parameters import \ BeamParameterisation from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalOrientationParameterisation, CrystalUnitCellParameterisation # Symmetry constrained parameterisation for the unit cell from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import \ symmetrize_reduce_enlarge # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.refinement.prediction import ScansRayPredictor, \ ExperimentsPredictor from dials.algorithms.spot_prediction import ray_intersection from cctbx.sgtbx import space_group, space_group_symbols # Parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.prediction_parameters import \ XYPhiPredictionParameterisation # implicit import # Imports for the target function from dials.algorithms.refinement.target import \ LeastSquaresPositionalResidualWithRmsdCutoff # implicit import ############################# # Setup experimental models # ############################# master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True) models = setup_geometry.Extract( master_phil, cmdline_args=args, local_overrides="geometry.parameters.random_seed = 1") crystal1 = models.crystal models = setup_geometry.Extract( master_phil, cmdline_args=args, local_overrides="geometry.parameters.random_seed = 2") mydetector = models.detector mygonio = models.goniometer crystal2 = models.crystal mybeam = models.beam # Build a mock scan for a 180 degree sweep sf = ScanFactory() myscan = sf.make_scan(image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=range(1800), deg=True) sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) # Build an experiment list experiments = ExperimentList() experiments.append( Experiment(beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=crystal1, imageset=None)) experiments.append( Experiment(beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=crystal2, imageset=None)) assert len(experiments.detectors()) == 1 ########################################################## # Parameterise the models (only for perturbing geometry) # ########################################################## det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xl1o_param = CrystalOrientationParameterisation(crystal1) xl1uc_param = CrystalUnitCellParameterisation(crystal1) xl2o_param = CrystalOrientationParameterisation(crystal2) xl2uc_param = CrystalUnitCellParameterisation(crystal2) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) # Fix crystal parameters #xluc_param.set_fixed([True, True, True, True, True, True]) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ######################################################################## #pred_param = XYPhiPredictionParameterisation(experiments, # [det_param], [s0_param], [xlo_param], [xluc_param]) ################################ # Apply known parameter shifts # ################################ # shift detector by 1.0 mm each translation and 2 mrad each rotation det_p_vals = det_param.get_param_vals() p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2., 2., 2.])] det_param.set_param_vals(p_vals) # shift beam by 2 mrad in free axis s0_p_vals = s0_param.get_param_vals() p_vals = list(s0_p_vals) p_vals[0] += 2. s0_param.set_param_vals(p_vals) # rotate crystal a bit (=2 mrad each rotation) xlo_p_vals = [] for xlo in (xl1o_param, xl2o_param): p_vals = xlo.get_param_vals() xlo_p_vals.append(p_vals) new_p_vals = [a + b for a, b in zip(p_vals, [2., 2., 2.])] xlo.set_param_vals(new_p_vals) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # gamma angle) xluc_p_vals = [] for xluc, xl in ((xl1uc_param, crystal1), ((xl2uc_param, crystal2))): p_vals = xluc.get_param_vals() xluc_p_vals.append(p_vals) cell_params = xl.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(xl.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.e5 for e in S.forward_independent_parameters()]) xluc.set_param_vals(X) ############################# # Generate some reflections # ############################# #print "Reflections will be generated with the following geometry:" #print mybeam #print mydetector #print crystal1 #print crystal2 # All indices in a 2.0 Angstrom sphere for crystal1 resolution = 2.0 index_generator = IndexGenerator( crystal1.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices1 = index_generator.to_array() # All indices in a 2.0 Angstrom sphere for crystal2 resolution = 2.0 index_generator = IndexGenerator( crystal2.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices2 = index_generator.to_array() # Predict rays within the sweep range. Set experiment IDs ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs1 = ray_predictor(indices1, experiment_id=0) obs_refs1['id'] = flex.int(len(obs_refs1), 0) obs_refs2 = ray_predictor(indices1, experiment_id=1) obs_refs2['id'] = flex.int(len(obs_refs2), 1) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs1) obs_refs1 = obs_refs1.select(intersects) intersects = ray_intersection(mydetector, obs_refs2) obs_refs2 = obs_refs2.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs1 = ref_predictor(obs_refs1) obs_refs2 = ref_predictor(obs_refs2) # Set 'observed' centroids from the predicted ones obs_refs1['xyzobs.mm.value'] = obs_refs1['xyzcal.mm'] obs_refs2['xyzobs.mm.value'] = obs_refs2['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs1), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs1), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs1), (im_width / 2.)**2) obs_refs1['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) var_x = flex.double(len(obs_refs2), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs2), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs2), (im_width / 2.)**2) obs_refs2['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) #print "Total number of reflections excited for crystal1", len(obs_refs1) #print "Total number of reflections excited for crystal2", len(obs_refs2) # concatenate reflection lists obs_refs1.extend(obs_refs2) obs_refs = obs_refs1 ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xl1o_param.set_param_vals(xlo_p_vals[0]) xl2o_param.set_param_vals(xlo_p_vals[1]) xl1uc_param.set_param_vals(xluc_p_vals[0]) xl2uc_param.set_param_vals(xluc_p_vals[1]) #print "Initial values of parameters are" #msg = "Parameters: " + "%.5f " * len(pred_param) #print msg % tuple(pred_param.get_param_vals()) #print # make a refiner from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse('')).extract() # in case we want a plot params.refinement.refinery.journal.track_parameter_correlation = True # scan static first from dials.algorithms.refinement.refiner import RefinerFactory refiner = RefinerFactory.from_parameters_data_experiments(params, obs_refs, experiments, verbosity=0) history = refiner.run() # scan varying params.refinement.parameterisation.scan_varying = True refiner = RefinerFactory.from_parameters_data_experiments(params, obs_refs, experiments, verbosity=0) history = refiner.run()
def test(): # Python and cctbx imports from math import pi from scitbx import matrix from scitbx.array_family import flex from libtbx.phil import parse from libtbx.test_utils import approx_equal # Get modules to build models and minimiser using PHIL import dials.test.algorithms.refinement.setup_geometry as setup_geometry import dials.test.algorithms.refinement.setup_minimiser as setup_minimiser # We will set up a mock scan and a mock experiment list from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import ExperimentList, Experiment # Model parameterisations from dials.algorithms.refinement.parameterisation.detector_parameters import \ DetectorParameterisationSinglePanel from dials.algorithms.refinement.parameterisation.beam_parameters import \ BeamParameterisation from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalOrientationParameterisation, CrystalUnitCellParameterisation # Symmetry constrained parameterisation for the unit cell from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import \ symmetrize_reduce_enlarge # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection from dials.algorithms.refinement.prediction import ScansRayPredictor, \ ExperimentsPredictor from cctbx.sgtbx import space_group, space_group_symbols # Parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.prediction_parameters import \ XYPhiPredictionParameterisation # Imports for the target function from dials.algorithms.refinement.target import \ LeastSquaresPositionalResidualWithRmsdCutoff from dials.algorithms.refinement.reflection_manager import ReflectionManager ############################# # Setup experimental models # ############################# override = """geometry.parameters { beam.wavelength.random=False beam.wavelength.value=1.0 beam.direction.inclination.random=False crystal.a.length.random=False crystal.a.length.value=12.0 crystal.a.direction.method=exactly crystal.a.direction.exactly.direction=1.0 0.002 -0.004 crystal.b.length.random=False crystal.b.length.value=14.0 crystal.b.direction.method=exactly crystal.b.direction.exactly.direction=-0.002 1.0 0.002 crystal.c.length.random=False crystal.c.length.value=13.0 crystal.c.direction.method=exactly crystal.c.direction.exactly.direction=0.002 -0.004 1.0 detector.directions.method=exactly detector.directions.exactly.dir1=0.99 0.002 -0.004 detector.directions.exactly.norm=0.002 -0.001 0.99 detector.centre.method=exactly detector.centre.exactly.value=1.0 -0.5 199.0 }""" master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True) models = setup_geometry.Extract(master_phil, local_overrides=override, verbose=False) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam ########################### # Parameterise the models # ########################### det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ######################################################################## # Build a mock scan for a 180 degree sweep sf = ScanFactory() myscan = sf.make_scan(image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=range(1800), deg=True) # Build an ExperimentList experiments = ExperimentList() experiments.append( Experiment(beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None)) # Create the PredictionParameterisation pred_param = XYPhiPredictionParameterisation(experiments, [det_param], [s0_param], [xlo_param], [xluc_param]) ################################ # Apply known parameter shifts # ################################ # shift detector by 1.0 mm each translation and 4 mrad each rotation det_p_vals = det_param.get_param_vals() p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 4., 4., 4.])] det_param.set_param_vals(p_vals) # shift beam by 4 mrad in free axis s0_p_vals = s0_param.get_param_vals() p_vals = list(s0_p_vals) p_vals[0] += 4. s0_param.set_param_vals(p_vals) # rotate crystal a bit (=3 mrad each rotation) xlo_p_vals = xlo_param.get_param_vals() p_vals = [a + b for a, b in zip(xlo_p_vals, [3., 3., 3.])] xlo_param.set_param_vals(p_vals) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # alpha and beta angles) xluc_p_vals = xluc_param.get_param_vals() cell_params = mycrystal.get_unit_cell().parameters() cell_params = [ a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1, -0.1, 0.0]) ] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(mycrystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator( mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) # Predict rays within the sweep range ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs['id'] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) # The total number of observations should be 1128 assert len(obs_refs) == 1128 ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) xluc_param.set_param_vals(xluc_p_vals) ##################################### # Select reflections for refinement # ##################################### refman = ReflectionManager(obs_refs, experiments, outlier_detector=None, close_to_spindle_cutoff=0.1) ############################## # Set up the target function # ############################## # The current 'achieved' criterion compares RMSD against 1/3 the pixel size and # 1/3 the image width in radians. For the simulated data, these are just made up mytarget = LeastSquaresPositionalResidualWithRmsdCutoff( experiments, ref_predictor, refman, pred_param, restraints_parameterisation=None) ###################################### # Set up the LSTBX refinement engine # ###################################### overrides = """minimiser.parameters.engine=GaussNewton minimiser.parameters.verbosity=0 minimiser.parameters.logfile=None""" refiner = setup_minimiser.Extract(master_phil, mytarget, pred_param, local_overrides=overrides).refiner refiner.run() assert mytarget.achieved() assert refiner.get_num_steps() == 1 assert approx_equal( mytarget.rmsds(), (0.00508252354876, 0.00420954552156, 8.97303428289e-05)) ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) xluc_param.set_param_vals(xluc_p_vals) ###################################################### # Set up the LBFGS with curvatures refinement engine # ###################################################### overrides = """minimiser.parameters.engine=LBFGScurvs minimiser.parameters.verbosity=0 minimiser.parameters.logfile=None""" refiner = setup_minimiser.Extract(master_phil, mytarget, pred_param, local_overrides=overrides).refiner refiner.run() assert mytarget.achieved() assert refiner.get_num_steps() == 9 assert approx_equal(mytarget.rmsds(), (0.0558857700305, 0.0333446685335, 0.000347402754278))
# Add in flags and ID columns by copying into standard reflection table tmp = flex.reflection_table.empty_standard(len(obs_refs)) tmp.update(obs_refs) obs_refs = tmp # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) # Re-predict using the stills reflection predictor stills_ref_predictor = ExperimentsPredictor(stills_experiments) obs_refs_stills = stills_ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs_stills['xyzobs.mm.value'] = obs_refs_stills['xyzcal.mm'] ############################### # Undo known parameter shifts # ############################### xlo_param.set_param_vals(xlo_p_vals[0]) xluc_param.set_param_vals(xluc_p_vals[0]) # make a refiner from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse('')).extract()
index_generator = IndexGenerator(mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() # Predict rays within the sweep range ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor.predict(indices) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs['id'] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor.predict(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) # set the flex random seed to an 'uninteresting' number
def config_target(params, experiments, refman, do_stills): """Given a set of parameters, configure a factory to build a target function Params: params The input parameters Returns: The target factory instance """ # Shorten parameter paths options = params.refinement.target sparse = params.refinement.parameterisation.sparse if options.rmsd_cutoff == "fraction_of_bin_size": absolute_cutoffs = None elif options.rmsd_cutoff == "absolute": absolute_cutoffs = options.absolute_cutoffs else: raise RuntimeError("Target function rmsd_cutoff option" + options.rmsd_cutoff + " not recognised") # all experiments have the same (or no) goniometer goniometer = experiments[0].goniometer for e in experiments: assert e.goniometer is goniometer # build managed reflection predictors from dials.algorithms.refinement.prediction import ExperimentsPredictor ref_predictor = ExperimentsPredictor(experiments, do_stills) # Determine whether the target is in X, Y, Phi space or just X, Y. if do_stills: if sparse: targ = LeastSquaresStillsDetectorSparse else: targ = LeastSquaresStillsDetector else: raise NotImplementedError("currently only for stills") #if sparse: # raise NotImplementedError("currently only for stills") # #from dials.algorithms.refinement.target \ # # import LeastSquaresPositionalResidualWithRmsdCutoffSparse as targ #else: # raise NotImplementedError("currently only for stills") # #from dials.algorithms.refinement.target \ # # import LeastSquaresPositionalResidualWithRmsdCutoff as targ # Here we pass in None for prediction_parameterisation and # restraints_parameterisation, as these will be linked to the object later target = targ(experiments=experiments, reflection_predictor=ref_predictor, ref_man=refman, prediction_parameterisation=None, restraints_parameterisation=None, frac_binsize_cutoff=options.bin_size_fraction, absolute_cutoffs=absolute_cutoffs, gradient_calculation_blocksize=options. gradient_calculation_blocksize) return target
xlo_param.set_param_vals(xlo_p_vals) xluc_param.set_param_vals(xluc_p_vals) ##################################### # Select reflections for refinement # ##################################### refman = ReflectionManager(obs_refs, experiments_single_panel) refman2 = ReflectionManager(obs_refs, experiments_multi_panel) ############################### # Set up the target functions # ############################### mytarget = LeastSquaresPositionalResidualWithRmsdCutoff( experiments_single_panel, ExperimentsPredictor(experiments_single_panel), refman, pred_param, restraints_parameterisation=None) mytarget2 = LeastSquaresPositionalResidualWithRmsdCutoff( experiments_multi_panel, ExperimentsPredictor(experiments_multi_panel), refman2, pred_param2, restraints_parameterisation=None) ################################# # Set up the refinement engines # ################################# refiner = setup_minimiser.Extract(master_phil, mytarget, pred_param, cmdline_args = args).refiner refiner2 = setup_minimiser.Extract(master_phil, mytarget2,
sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) # Predict rays within the sweep range ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs['id'] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) # The total number of observations should be 1128
def generate_reflections(experiments, xyzvar=(0.0, 0.0, 0.0)): """Generate synthetic reflection centroids using the supplied experiments, with normally-distributed errors applied the variances in xyzvar""" # check input if [e >= 0.0 for e in xyzvar].count(False) > 0: msg = "negative variance requested in " + str(xyzvar) + "!" raise RuntimeError(msg) refs = [] for iexp, exp in enumerate(experiments): info("Generating reflections for experiment {0}".format(iexp)) # All indices in a 1.5 Angstrom sphere resolution = 1.5 index_generator = IndexGenerator( exp.crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() # Predict rays within the sweep range ray_predictor = ScansRayPredictor( experiments, exp.scan.get_oscillation_range(deg=False) ) obs_refs = ray_predictor.predict(indices, experiment_id=iexp) info("Total number of reflections excited: {0}".format(len(obs_refs))) # Take only those rays that intersect the detector intersects = ray_intersection(exp.detector, obs_refs) obs_refs = obs_refs.select(intersects) obs_refs["id"] = flex.size_t(len(obs_refs), iexp) refs.append(obs_refs) info("Total number of impacts: {0}".format(len(obs_refs))) # Concatenate reflections obs_refs = reduce(lambda x, y: x.extend(y), refs) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs = ref_predictor.predict(obs_refs) # calculate (uncorrelated) errors to offset the centroids # this is safe as elts of xyzvar are already tested to be > 0 sigX, sigY, sigZ = [sqrt(e) for e in xyzvar] shift = [ (random.gauss(0, sigX), random.gauss(0, sigY), random.gauss(0, sigZ)) for _ in xrange(len(obs_refs)) ] shift = flex.vec3_double(shift) # Set 'observed' centroids from the predicted ones obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"] + shift # Store variances for the centroid positions of the simulated data. If errors # are zero, invent some variances if tuple(xyzvar) == (0.0, 0.0, 0.0): im_width = exp.scan.get_oscillation()[1] * pi / 180.0 px_size = exp.detector[0].get_pixel_size() xyzvar = ( (px_size[0] / 2.0) ** 2, (px_size[1] / 2.0) ** 2, (im_width / 2.0) ** 2, ) var_x = flex.double(len(obs_refs), xyzvar[0]) var_y = flex.double(len(obs_refs), xyzvar[1]) var_phi = flex.double(len(obs_refs), xyzvar[2]) obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) info("Total number of observations made: {0}".format(len(obs_refs))) return obs_refs
def choose_best_orientation_matrix(self, candidate_orientation_matrices): from dxtbx.model.experiment_list import Experiment, ExperimentList import copy logger.info('*' * 80) logger.info('Selecting the best orientation matrix') logger.info('*' * 80) from libtbx import group_args class candidate_info(group_args): pass candidates = [] params = copy.deepcopy(self.all_params) n_cand = len(candidate_orientation_matrices) for icm, cm in enumerate(candidate_orientation_matrices): # Index reflections in P1 sel = ((self.reflections['id'] == -1)) refl = self.reflections.select(sel) experiments = self.experiment_list_for_crystal(cm) self.index_reflections(experiments, refl) indexed = refl.select(refl['id'] >= 0) indexed = indexed.select(indexed.get_flags(indexed.flags.indexed)) # If target symmetry supplied, try to apply it. Then, apply the change of basis to the reflections # indexed in P1 to the target setting if self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None: target_space_group = self.target_symmetry_primitive.space_group( ) new_crystal, cb_op_to_primitive = self.apply_symmetry( cm, target_space_group) if new_crystal is None: print( "Cannot convert to target symmetry, candidate %d/%d" % (icm, n_cand)) continue new_crystal = new_crystal.change_basis( self.cb_op_primitive_inp) cm = candidate_orientation_matrices[icm] = new_crystal experiments = self.experiment_list_for_crystal(cm) if not cb_op_to_primitive.is_identity_op(): indexed['miller_index'] = cb_op_to_primitive.apply( indexed['miller_index']) if self.cb_op_primitive_inp is not None: indexed['miller_index'] = self.cb_op_primitive_inp.apply( indexed['miller_index']) if params.indexing.stills.refine_all_candidates: try: print( "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d initial outlier identification" % (icm, n_cand)) acceptance_flags = self.identify_outliers( params, experiments, indexed) #create a new "indexed" list with outliers thrown out: indexed = indexed.select(acceptance_flags) print( "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d refinement before outlier rejection" % (icm, n_cand)) R = e_refine(params=params, experiments=experiments, reflections=indexed, graph_verbose=False) ref_experiments = R.get_experiments() # try to improve the outcome with a second round of outlier rejection post-initial refinement: acceptance_flags = self.identify_outliers( params, ref_experiments, indexed) # insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection nv0 = nave_parameters(params=params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False) crystal_model_nv0 = nv0() acceptance_flags_nv0 = nv0.nv_acceptance_flags indexed = indexed.select(acceptance_flags & acceptance_flags_nv0) print( "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d after positional and delta-psi outlier rejection" % (icm, n_cand)) R = e_refine(params=params, experiments=ref_experiments, reflections=indexed, graph_verbose=False) ref_experiments = R.get_experiments() nv = nave_parameters(params=params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False) crystal_model = nv() # Drop candidates that after refinement can no longer be converted to the known target space group if not self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None: target_space_group = self.target_symmetry_primitive.space_group( ) new_crystal, cb_op_to_primitive = self.apply_symmetry( crystal_model, target_space_group) if new_crystal is None: print( "P1 refinement yielded model diverged from target, candidate %d/%d" % (icm, n_cand)) continue rmsd, _ = calc_2D_rmsd_and_displacements( R.predict_for_reflection_table(indexed)) except Exception as e: print("Couldn't refine candiate %d/%d, %s" % (icm, n_cand, str(e))) else: print( "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d done" % (icm, n_cand)) candidates.append( candidate_info( crystal=crystal_model, green_curve_area=nv.green_curve_area, ewald_proximal_volume=nv.ewald_proximal_volume(), n_indexed=len(indexed), rmsd=rmsd, indexed=indexed, experiments=ref_experiments)) else: from dials.algorithms.refinement.prediction import ExperimentsPredictor ref_predictor = ExperimentsPredictor( experiments, force_stills=True, spherical_relp=params.refinement.parameterisation. spherical_relp_model) rmsd, _ = calc_2D_rmsd_and_displacements( ref_predictor(indexed)) candidates.append( candidate_info(crystal=cm, n_indexed=len(indexed), rmsd=rmsd, indexed=indexed, experiments=experiments)) if len(candidates) == 0: raise Sorry("No suitable indexing solution found") print("**** ALL CANDIDATES:") for i, XX in enumerate(candidates): print("\n****Candidate %d" % i, XX) cc = XX.crystal if hasattr(cc, 'get_half_mosaicity_deg'): print(" half mosaicity %5.2f deg." % (cc.get_half_mosaicity_deg())) print(" domain size %.0f Ang." % (cc.get_domain_size_ang())) print("\n**** BEST CANDIDATE:") results = flex.double([c.rmsd for c in candidates]) best = candidates[flex.min_index(results)] print(best) if params.indexing.stills.refine_all_candidates: if best.rmsd > params.indexing.stills.rmsd_min_px: raise Sorry("RMSD too high, %f" % best.rmsd) if best.ewald_proximal_volume > params.indexing.stills.ewald_proximal_volume_max: raise Sorry("Ewald proximity volume too high, %f" % best.ewald_proximal_volume) if len(candidates) > 1: for i in xrange(len(candidates)): if i == flex.min_index(results): continue if best.ewald_proximal_volume > candidates[ i].ewald_proximal_volume: print( "Couldn't figure out which candidate is best; picked the one with the best RMSD." ) best.indexed['entering'] = flex.bool(best.n_indexed, False) return best.crystal, best.n_indexed
def generate_reflections(experiments, xyzvar=(0., 0., 0.)): '''Generate synthetic reflection centroids using the supplied experiments, with normally-distributed errors applied the variances in xyzvar''' # check input if [e >= 0. for e in xyzvar].count(False) > 0: msg = "negative variance requested in " + str(xyzvar) + "!" raise RuntimeError(msg) refs = [] for iexp, exp in enumerate(experiments): info("Generating reflections for experiment {0}".format(iexp)) # All indices in a 1.5 Angstrom sphere resolution = 1.5 index_generator = IndexGenerator(exp.crystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution) indices = index_generator.to_array() # Predict rays within the sweep range ray_predictor = ScansRayPredictor(experiments, exp.scan.get_oscillation_range(deg=False)) obs_refs = ray_predictor.predict(indices, experiment_id=iexp) info("Total number of reflections excited: {0}".format(len(obs_refs))) # Take only those rays that intersect the detector intersects = ray_intersection(exp.detector, obs_refs) obs_refs = obs_refs.select(intersects) obs_refs['id'] = flex.size_t(len(obs_refs), iexp) refs.append(obs_refs) info("Total number of impacts: {0}".format(len(obs_refs))) # Concatenate reflections obs_refs = reduce(lambda x, y: x.extend(y), refs) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs = ref_predictor.predict(obs_refs) # calculate (uncorrelated) errors to offset the centroids # this is safe as elts of xyzvar are already tested to be > 0 sigX, sigY, sigZ = [sqrt(e) for e in xyzvar] shift = [(random.gauss(0, sigX), random.gauss(0, sigY), random.gauss(0, sigZ)) for _ in xrange(len(obs_refs))] shift = flex.vec3_double(shift) # Set 'observed' centroids from the predicted ones obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] + shift # Store variances for the centroid positions of the simulated data. If errors # are zero, invent some variances if tuple(xyzvar) == (0., 0., 0.): im_width = exp.scan.get_oscillation()[1] * pi / 180. px_size = exp.detector[0].get_pixel_size() xyzvar = ((px_size[0] / 2.)**2, (px_size[1] / 2.)**2, (im_width / 2.)**2) var_x = flex.double(len(obs_refs), xyzvar[0]) var_y = flex.double(len(obs_refs), xyzvar[1]) var_phi = flex.double(len(obs_refs), xyzvar[2]) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) info("Total number of observations made: {0}".format(len(obs_refs))) return obs_refs