def choose_best_orientation_matrix(self, candidate_orientation_matrices): logger.info("*" * 80) logger.info("Selecting the best orientation matrix") logger.info("*" * 80) class CandidateInfo(libtbx.group_args): pass candidates = [] params = copy.deepcopy(self.all_params) for icm, cm in enumerate(candidate_orientation_matrices): if icm >= self.params.basis_vector_combinations.max_refine: break # Index reflections in P1 sel = self.reflections["id"] == -1 refl = self.reflections.select(sel) experiments = self.experiment_list_for_crystal(cm) self.index_reflections(experiments, refl) indexed = refl.select(refl["id"] >= 0) indexed = indexed.select(indexed.get_flags(indexed.flags.indexed)) # If target symmetry supplied, try to apply it. Then, apply the change of basis to the reflections # indexed in P1 to the target setting if ( self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None ): new_crystal, cb_op_to_primitive = self._symmetry_handler.apply_symmetry( cm ) if new_crystal is None: logger.info("Cannot convert to target symmetry, candidate %d", icm) continue new_crystal = new_crystal.change_basis( self._symmetry_handler.cb_op_primitive_inp ) cm = new_crystal experiments = self.experiment_list_for_crystal(cm) if not cb_op_to_primitive.is_identity_op(): indexed["miller_index"] = cb_op_to_primitive.apply( indexed["miller_index"] ) if self._symmetry_handler.cb_op_primitive_inp is not None: indexed[ "miller_index" ] = self._symmetry_handler.cb_op_primitive_inp.apply( indexed["miller_index"] ) if params.indexing.stills.refine_all_candidates: try: logger.info( "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d initial outlier identification", icm, ) acceptance_flags = self.identify_outliers( params, experiments, indexed ) # create a new "indexed" list with outliers thrown out: indexed = indexed.select(acceptance_flags) logger.info( "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d refinement before outlier rejection", icm, ) R = e_refine( params=params, experiments=experiments, reflections=indexed, graph_verbose=False, ) ref_experiments = R.get_experiments() # try to improve the outcome with a second round of outlier rejection post-initial refinement: acceptance_flags = self.identify_outliers( params, ref_experiments, indexed ) # insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection nv0 = NaveParameters( params=params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False, ) nv0() acceptance_flags_nv0 = nv0.nv_acceptance_flags indexed = indexed.select(acceptance_flags & acceptance_flags_nv0) logger.info( "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d after positional and delta-psi outlier rejection", icm, ) R = e_refine( params=params, experiments=ref_experiments, reflections=indexed, graph_verbose=False, ) ref_experiments = R.get_experiments() nv = NaveParameters( params=params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False, ) crystal_model = nv() assert ( len(crystal_model) == 1 ), "$$$ stills_indexer::choose_best_orientation_matrix, Only one crystal at this stage" crystal_model = crystal_model[0] # Drop candidates that after refinement can no longer be converted to the known target space group if ( not self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None ): ( new_crystal, cb_op_to_primitive, ) = self._symmetry_handler.apply_symmetry(crystal_model) if new_crystal is None: logger.info( "P1 refinement yielded model diverged from target, candidate %d", icm, ) continue rmsd, _ = calc_2D_rmsd_and_displacements( R.predict_for_reflection_table(indexed) ) except Exception as e: logger.info( "Couldn't refine candidate %d, %s: %s", icm, e.__class__.__name__, str(e), ) else: logger.info( "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d done", icm, ) candidates.append( CandidateInfo( crystal=crystal_model, green_curve_area=nv.green_curve_area, ewald_proximal_volume=nv.ewald_proximal_volume(), n_indexed=len(indexed), rmsd=rmsd, indexed=indexed, experiments=ref_experiments, ) ) else: from dials.algorithms.refinement.prediction.managed_predictors import ( ExperimentsPredictorFactory, ) ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, force_stills=True, spherical_relp=params.refinement.parameterisation.spherical_relp_model, ) rmsd, _ = calc_2D_rmsd_and_displacements(ref_predictor(indexed)) candidates.append( CandidateInfo( crystal=cm, n_indexed=len(indexed), rmsd=rmsd, indexed=indexed, experiments=experiments, ) ) if len(candidates) == 0: raise DialsIndexError("No suitable indexing solution found") logger.info("**** ALL CANDIDATES:") for i, XX in enumerate(candidates): logger.info("\n****Candidate %d %s", i, XX) cc = XX.crystal if hasattr(cc, "get_half_mosaicity_deg"): logger.info( " half mosaicity %5.2f deg.", (cc.get_half_mosaicity_deg()) ) logger.info(" domain size %.0f Ang.", (cc.get_domain_size_ang())) logger.info("\n**** BEST CANDIDATE:") results = flex.double([c.rmsd for c in candidates]) best = candidates[flex.min_index(results)] logger.info(best) if params.indexing.stills.refine_all_candidates: if best.rmsd > params.indexing.stills.rmsd_min_px: raise DialsIndexError("RMSD too high, %f" % best.rmsd) if len(candidates) > 1: for i in range(len(candidates)): if i == flex.min_index(results): continue if best.ewald_proximal_volume > candidates[i].ewald_proximal_volume: logger.info( "Couldn't figure out which candidate is best; picked the one with the best RMSD." ) best.indexed["entering"] = flex.bool(best.n_indexed, False) return best.crystal, best.n_indexed
def index(self): # most of this is the same as dials.algorithms.indexing.indexer.indexer_base.index(), with some stills # specific modifications (don't re-index after choose best orientation matrix, but use the indexing from # choose best orientation matrix, also don't use macrocycles) of refinement after indexing. # 2017 update: do accept multiple lattices per shot experiments = ExperimentList() while True: self.d_min = self.params.refinement_protocol.d_min_start max_lattices = self.params.multiple_lattice_search.max_lattices if max_lattices is not None and len(experiments) >= max_lattices: break if len(experiments) > 0: cutoff_fraction = ( self.params.multiple_lattice_search.recycle_unindexed_reflections_cutoff ) d_spacings = 1 / self.reflections["rlp"].norms() d_min_indexed = flex.min(d_spacings.select(self.indexed_reflections)) min_reflections_for_indexing = cutoff_fraction * len( self.reflections.select(d_spacings > d_min_indexed) ) crystal_ids = self.reflections.select(d_spacings > d_min_indexed)["id"] if (crystal_ids == -1).count(True) < min_reflections_for_indexing: logger.info( "Finish searching for more lattices: %i unindexed reflections remaining." % (min_reflections_for_indexing) ) break n_lattices_previous_cycle = len(experiments) # index multiple lattices per shot if len(experiments) == 0: new = self.find_lattices() generate_experiment_identifiers(new) experiments.extend(new) if len(experiments) == 0: raise DialsIndexError("No suitable lattice could be found.") else: try: new = self.find_lattices() generate_experiment_identifiers(new) experiments.extend(new) except Exception as e: logger.info("Indexing remaining reflections failed") logger.debug( "Indexing remaining reflections failed, exception:\n" + str(e) ) # reset reflection lattice flags # the lattice a given reflection belongs to: a value of -1 indicates # that a reflection doesn't belong to any lattice so far self.reflections["id"] = flex.int(len(self.reflections), -1) self.index_reflections(experiments, self.reflections) if len(experiments) == n_lattices_previous_cycle: # no more lattices found break if ( not self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None ): self._apply_symmetry_post_indexing( experiments, self.reflections, n_lattices_previous_cycle ) # discard nearly overlapping lattices on the same shot if self._check_have_similar_crystal_models(experiments): break self.indexed_reflections = self.reflections["id"] > -1 if self.d_min is None: sel = self.reflections["id"] <= -1 else: sel = flex.bool(len(self.reflections), False) lengths = 1 / self.reflections["rlp"].norms() isel = (lengths >= self.d_min).iselection() sel.set_selected(isel, True) sel.set_selected(self.reflections["id"] > -1, False) self.unindexed_reflections = self.reflections.select(sel) reflections_for_refinement = self.reflections.select( self.indexed_reflections ) if len(self.params.stills.isoforms) > 0: logger.info("") logger.info("#" * 80) logger.info("Starting refinement") logger.info("#" * 80) logger.info("") isoform_experiments = ExperimentList() isoform_reflections = flex.reflection_table() # Note, changes to params after initial indexing. Cannot use tie to target when fixing the unit cell. self.all_params.refinement.reflections.outlier.algorithm = "null" self.all_params.refinement.parameterisation.crystal.fix = "cell" self.all_params.refinement.parameterisation.crystal.unit_cell.restraints.tie_to_target = ( [] ) for expt_id, experiment in enumerate(experiments): reflections = reflections_for_refinement.select( reflections_for_refinement["id"] == expt_id ) reflections["id"] = flex.int(len(reflections), 0) refiners = [] for isoform in self.params.stills.isoforms: iso_experiment = copy.deepcopy(experiment) crystal = iso_experiment.crystal if ( isoform.lookup_symbol != crystal.get_space_group().type().lookup_symbol() ): logger.info( "Crystal isoform lookup_symbol %s does not match isoform %s lookup_symbol %s" % ( crystal.get_space_group().type().lookup_symbol(), isoform.name, isoform.lookup_symbol, ) ) continue crystal.set_B(isoform.cell.fractionalization_matrix()) logger.info("Refining isoform %s" % isoform.name) refiners.append( e_refine( params=self.all_params, experiments=ExperimentList([iso_experiment]), reflections=reflections, graph_verbose=False, ) ) if len(refiners) == 0: raise DialsIndexError( "No isoforms had a lookup symbol that matched" ) positional_rmsds = [ math.sqrt(P.rmsds()[0] ** 2 + P.rmsds()[1] ** 2) for P in refiners ] logger.info( "Positional rmsds for all isoforms:" + str(positional_rmsds) ) minrmsd_mm = min(positional_rmsds) minindex = positional_rmsds.index(minrmsd_mm) logger.info( "The smallest rmsd is %5.1f um from isoform %s" % ( 1000.0 * minrmsd_mm, self.params.stills.isoforms[minindex].name, ) ) if self.params.stills.isoforms[minindex].rmsd_target_mm is not None: logger.info( "Asserting %f < %f" % ( minrmsd_mm, self.params.stills.isoforms[minindex].rmsd_target_mm, ) ) assert ( minrmsd_mm < self.params.stills.isoforms[minindex].rmsd_target_mm ) logger.info( "Acceptable rmsd for isoform %s." % (self.params.stills.isoforms[minindex].name) ) if len(self.params.stills.isoforms) == 2: logger.info( "Rmsd gain over the other isoform %5.1f um." % (1000.0 * abs(positional_rmsds[0] - positional_rmsds[1])) ) R = refiners[minindex] # Now one last check to see if direct beam is out of bounds if self.params.stills.isoforms[minindex].beam_restraint is not None: from scitbx import matrix refined_beam = matrix.col( R.get_experiments()[0] .detector[0] .get_beam_centre_lab(experiments[0].beam.get_s0())[0:2] ) known_beam = matrix.col( self.params.stills.isoforms[minindex].beam_restraint ) logger.info( "Asserting difference in refined beam center and expected beam center %f < %f" % ( (refined_beam - known_beam).length(), self.params.stills.isoforms[minindex].rmsd_target_mm, ) ) assert ( (refined_beam - known_beam).length() < self.params.stills.isoforms[minindex].rmsd_target_mm ) # future--circle of confusion could be given as a separate length in mm instead of reusing rmsd_target experiment = R.get_experiments()[0] experiment.crystal.identified_isoform = self.params.stills.isoforms[ minindex ].name isoform_experiments.append(experiment) reflections["id"] = flex.int(len(reflections), expt_id) isoform_reflections.extend(reflections) experiments = isoform_experiments reflections_for_refinement = isoform_reflections if self.params.refinement_protocol.mode == "repredict_only": from dials.algorithms.indexing.nave_parameters import NaveParameters from dials.algorithms.refinement.prediction.managed_predictors import ( ExperimentsPredictorFactory, ) refined_experiments, refined_reflections = ( experiments, reflections_for_refinement, ) ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, force_stills=True, spherical_relp=self.all_params.refinement.parameterisation.spherical_relp_model, ) ref_predictor(refined_reflections) refined_reflections["delpsical2"] = ( refined_reflections["delpsical.rad"] ** 2 ) for expt_id in range(len(refined_experiments)): refls = refined_reflections.select( refined_reflections["id"] == expt_id ) nv = NaveParameters( params=self.all_params, experiments=refined_experiments[expt_id : expt_id + 1], reflections=refls, refinery=None, graph_verbose=False, ) experiments[expt_id].crystal = nv() ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, force_stills=True, spherical_relp=self.all_params.refinement.parameterisation.spherical_relp_model, ) ref_predictor(refined_reflections) elif self.params.refinement_protocol.mode is None: refined_experiments, refined_reflections = ( experiments, reflections_for_refinement, ) else: try: refined_experiments, refined_reflections = self.refine( experiments, reflections_for_refinement ) except Exception as e: s = str(e) if len(experiments) == 1: raise DialsIndexRefineError(e.message) logger.info("Refinement failed:") logger.info(s) del experiments[-1] break self._unit_cell_volume_sanity_check(experiments, refined_experiments) self.refined_reflections = refined_reflections.select( refined_reflections["id"] > -1 ) for i, expt in enumerate(self.experiments): ref_sel = self.refined_reflections.select( self.refined_reflections["imageset_id"] == i ) ref_sel = ref_sel.select(ref_sel["id"] >= 0) for i_expt in set(ref_sel["id"]): refined_expt = refined_experiments[i_expt] expt.detector = refined_expt.detector expt.beam = refined_expt.beam expt.goniometer = refined_expt.goniometer expt.scan = refined_expt.scan refined_expt.imageset = expt.imageset if not ( self.all_params.refinement.parameterisation.beam.fix == "all" and self.all_params.refinement.parameterisation.detector.fix == "all" ): # Experimental geometry may have changed - re-map centroids to # reciprocal space self.reflections.map_centroids_to_reciprocal_space(self.experiments) # update for next cycle experiments = refined_experiments self.refined_experiments = refined_experiments if self.refined_experiments is None: raise DialsIndexRefineError("None of the experiments could refine.") # discard experiments with zero reflections after refinement id_set = set(self.refined_reflections["id"]) if len(id_set) < len(self.refined_experiments): filtered_refined_reflections = flex.reflection_table() for i in range(len(self.refined_experiments)): if i not in id_set: del self.refined_experiments[i] for old, new in zip(sorted(id_set), range(len(id_set))): subset = self.refined_reflections.select( self.refined_reflections["id"] == old ) subset["id"] = flex.int(len(subset), new) filtered_refined_reflections.extend(subset) self.refined_reflections = filtered_refined_reflections if len(self.refined_experiments) > 1: from dials.algorithms.indexing.compare_orientation_matrices import ( rotation_matrix_differences, ) logger.info( rotation_matrix_differences(self.refined_experiments.crystals()) ) logger.info("Final refined crystal models:") for i, crystal_model in enumerate(self.refined_experiments.crystals()): n_indexed = 0 for _ in experiments.where(crystal=crystal_model): n_indexed += (self.reflections["id"] == i).count(True) logger.info("model %i (%i reflections):" % (i + 1, n_indexed)) logger.info(crystal_model) if ( "xyzcal.mm" in self.refined_reflections ): # won't be there if refine_all_candidates = False and no isoforms self._xyzcal_mm_to_px(self.experiments, self.refined_reflections)
def index(self): # most of this is the same as dials.algorithms.indexing.indexer.indexer_base.index(), with some stills # specific modifications (don't re-index after choose best orientation matrix, but use the indexing from # choose best orientation matrix, also don't use macrocycles) of refinement after indexing. # 2017 update: do accept multiple lattices per shot if self.params.refinement_protocol.n_macro_cycles > 1: raise Sorry("For stills, please set refinement_protocol.n_macro_cycles = 1") experiments = ExperimentList() had_refinement_error = False have_similar_crystal_models = False # This enters into a long series of refinements and checks while True: self.d_min = self.params.refinement_protocol.d_min_start if had_refinement_error or have_similar_crystal_models: break max_lattices = self.params.multiple_lattice_search.max_lattices if max_lattices is not None and len(experiments) >= max_lattices: break if len(experiments) > 0: cutoff_fraction = \ self.params.multiple_lattice_search.recycle_unindexed_reflections_cutoff d_spacings = 1/self.reflections['rlp'].norms() d_min_indexed = flex.min(d_spacings.select(self.indexed_reflections)) min_reflections_for_indexing = \ cutoff_fraction * len(self.reflections.select(d_spacings > d_min_indexed)) crystal_ids = self.reflections.select(d_spacings > d_min_indexed)['id'] if (crystal_ids == -1).count(True) < min_reflections_for_indexing: logger.info("Finish searching for more lattices: %i unindexed reflections remaining." %( min_reflections_for_indexing)) break n_lattices_previous_cycle = len(experiments) # indexing starts around here if len(experiments) == 0: experiments.extend(self.find_lattices()) # try to find a good crystal model.. if len(experiments) == 0: raise Sorry("No suitable lattice could be found.") else: print("\n\n Do you really want to go here? Dont go here!\n\n") exit() try: new = self.find_lattices() experiments.extend(new) except Exception as e: logger.info("Indexing remaining reflections failed") logger.debug("Indexing remaining reflections failed, exception:\n" + str(e)) # reset reflection lattice flags # the lattice a given reflection belongs to: a value of -1 indicates # that a reflection doesn't belong to any lattice so far self.reflections['id'] = flex.int(len(self.reflections), -1) self.index_reflections(experiments, self.reflections) if len(experiments) == n_lattices_previous_cycle: # no more lattices found break ######################################################### if not self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None: # now apply the space group symmetry only after the first indexing # need to make sure that the symmetrized orientation is similar to the P1 model target_space_group = self.target_symmetry_primitive.space_group() for i_cryst, cryst in enumerate(experiments.crystals()): if i_cryst >= n_lattices_previous_cycle: new_cryst, cb_op_to_primitive = self.apply_symmetry( cryst, target_space_group) if self.cb_op_primitive_inp is not None: new_cryst = new_cryst.change_basis(self.cb_op_primitive_inp) logger.info(new_cryst.get_space_group().info()) cryst.update(new_cryst) cryst.set_space_group( self.params.known_symmetry.space_group.group()) for i_expt, expt in enumerate(experiments): if expt.crystal is not cryst: continue if not cb_op_to_primitive.is_identity_op(): miller_indices = self.reflections['miller_index'].select( self.reflections['id'] == i_expt) miller_indices = cb_op_to_primitive.apply(miller_indices) self.reflections['miller_index'].set_selected( self.reflections['id'] == i_expt, miller_indices) if self.cb_op_primitive_inp is not None: miller_indices = self.reflections['miller_index'].select( self.reflections['id'] == i_expt) miller_indices = self.cb_op_primitive_inp.apply(miller_indices) self.reflections['miller_index'].set_selected( self.reflections['id'] == i_expt, miller_indices) ########################################################### # discard nearly overlapping lattices on the same shot if len(experiments) > 1: from dials.algorithms.indexing.compare_orientation_matrices \ import difference_rotation_matrix_axis_angle cryst_b = experiments.crystals()[-1] have_similar_crystal_models = False for i_a, cryst_a in enumerate(experiments.crystals()[:-1]): R_ab, axis, angle, cb_op_ab = \ difference_rotation_matrix_axis_angle(cryst_a, cryst_b) min_angle = self.params.multiple_lattice_search.minimum_angular_separation if abs(angle) < min_angle: # degrees logger.info("Crystal models too similar, rejecting crystal %i:" %( len(experiments))) logger.info("Rotation matrix to transform crystal %i to crystal %i" %( i_a+1, len(experiments))) logger.info(R_ab) logger.info("Rotation of %.3f degrees" %angle + " about axis (%.3f, %.3f, %.3f)" %axis) #show_rotation_matrix_differences([cryst_a, cryst_b]) have_similar_crystal_models = True del experiments[-1] break if have_similar_crystal_models: break self.indexed_reflections = (self.reflections['id'] > -1) if self.d_min is None: sel = self.reflections['id'] <= -1 else: sel = flex.bool(len(self.reflections), False) lengths = 1/self.reflections['rlp'].norms() isel = (lengths >= self.d_min).iselection() sel.set_selected(isel, True) sel.set_selected(self.reflections['id'] > -1, False) self.unindexed_reflections = self.reflections.select(sel) reflections_for_refinement = self.reflections.select( self.indexed_reflections) if len(self.params.stills.isoforms) > 0: logger.info("") logger.info("#" * 80) logger.info("Starting refinement") logger.info("#" * 80) logger.info("") import copy isoform_experiments = ExperimentList() isoform_reflections = flex.reflection_table() # Note, changes to params after initial indexing. Cannot use tie to target when fixing the unit cell. self.all_params.refinement.reflections.outlier.algorithm = "null" self.all_params.refinement.parameterisation.crystal.fix = "cell" self.all_params.refinement.parameterisation.crystal.unit_cell.restraints.tie_to_target = [] for expt_id, experiment in enumerate(experiments): reflections = reflections_for_refinement.select(reflections_for_refinement['id'] == expt_id) reflections['id'] = flex.int(len(reflections),0) refiners = [] for isoform in self.params.stills.isoforms: iso_experiment = copy.deepcopy(experiment) crystal = iso_experiment.crystal if isoform.lookup_symbol != crystal.get_space_group().type().lookup_symbol(): logger.info("Crystal isoform lookup_symbol %s does not match isoform %s lookup_symbol %s"%(crystal.get_space_group().type().lookup_symbol(), isoform.name, isoform.lookup_symbol)) continue crystal.set_B(isoform.cell.fractionalization_matrix()) logger.info("Refining isoform %s"%isoform.name) refiners.append(e_refine(params=self.all_params, experiments=ExperimentList([iso_experiment]), reflections=reflections, graph_verbose=False)) if len(refiners) == 0: raise Sorry("No isoforms had a lookup symbol that matched") positional_rmsds = [math.sqrt(P.rmsds()[0] ** 2 + P.rmsds()[1] ** 2) for P in refiners] logger.info("Positional rmsds for all isoforms:" + str(positional_rmsds)) minrmsd_mm = min(positional_rmsds) minindex = positional_rmsds.index(minrmsd_mm) logger.info("The smallest rmsd is %5.1f um from isoform %s" % ( 1000. * minrmsd_mm, self.params.stills.isoforms[minindex].name)) if self.params.stills.isoforms[minindex].rmsd_target_mm is not None: logger.info("Asserting %f < %f"%(minrmsd_mm, self.params.stills.isoforms[minindex].rmsd_target_mm)) assert minrmsd_mm < self.params.stills.isoforms[minindex].rmsd_target_mm logger.info("Acceptable rmsd for isoform %s." % (self.params.stills.isoforms[minindex].name)) if len(self.params.stills.isoforms) == 2: logger.info("Rmsd gain over the other isoform %5.1f um." % ( 1000. * abs(positional_rmsds[0] - positional_rmsds[1]))) R = refiners[minindex] # Now one last check to see if direct beam is out of bounds if self.params.stills.isoforms[minindex].beam_restraint is not None: from scitbx import matrix refined_beam = matrix.col( R.get_experiments()[0].detector[0].get_beam_centre_lab(experiments[0].beam.get_s0())[0:2]) known_beam = matrix.col(self.params.stills.isoforms[minindex].beam_restraint) logger.info("Asserting difference in refined beam center and expected beam center %f < %f"%((refined_beam - known_beam).length(), self.params.stills.isoforms[ minindex].rmsd_target_mm)) assert (refined_beam - known_beam).length() < self.params.stills.isoforms[minindex].rmsd_target_mm # future--circle of confusion could be given as a separate length in mm instead of reusing rmsd_target experiment = R.get_experiments()[0] experiment.crystal.identified_isoform = self.params.stills.isoforms[minindex].name isoform_experiments.append(experiment) reflections['id'] = flex.int(len(reflections),expt_id) isoform_reflections.extend(reflections) experiments = isoform_experiments reflections_for_refinement = isoform_reflections if self.params.refinement_protocol.mode == 'repredict_only': from dials.algorithms.indexing.nave_parameters \ import nave_parameters from dials.algorithms.refinement.prediction.managed_predictors \ import ExperimentsPredictorFactory # TODO: need to check here if an experiment does not have any refined reflections # as nv will fail miserably if len(refined_reflections)==0 refined_experiments, refined_reflections = experiments, reflections_for_refinement ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, force_stills=True, spherical_relp=self.all_params.refinement.parameterisation.spherical_relp_model) ref_predictor(refined_reflections) refined_reflections['delpsical2'] = refined_reflections['delpsical.rad']**2 for expt_id in range(len(refined_experiments)): refls = refined_reflections.select(refined_reflections['id'] == expt_id) nv = nave_parameters(params=self.all_params, experiments=refined_experiments[expt_id:expt_id+1], reflections=refls, refinery=None, graph_verbose=False) experiments[expt_id].crystal = nv() ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, force_stills=True, spherical_relp=self.all_params.refinement.parameterisation.spherical_relp_model) ref_predictor(refined_reflections) elif self.params.refinement_protocol.mode == 'ignore': refined_experiments, refined_reflections = experiments, reflections_for_refinement else: try: refined_experiments, refined_reflections = self.refine( experiments, reflections_for_refinement) except Exception as e: s = str(e) if len(experiments) == 1: raise Sorry(e) had_refinement_error = True logger.info("Refinement failed:") logger.info(s) del experiments[-1] break # sanity check for unrealistic unit cell volume increase during refinement # usually this indicates too many parameters are being refined given the # number of observations provided. if not self.params.refinement_protocol.disable_unit_cell_volume_sanity_check: for orig_expt, refined_expt in zip(experiments, refined_experiments): uc1 = orig_expt.crystal.get_unit_cell() uc2 = refined_expt.crystal.get_unit_cell() volume_change = abs(uc1.volume()-uc2.volume())/uc1.volume() cutoff = 0.5 if volume_change > cutoff: msg = "\n".join(( "Unrealistic unit cell volume increase during refinement of %.1f%%.", "Please try refining fewer parameters, either by enforcing symmetry", "constraints (space_group=) and/or disabling experimental geometry", "refinement (detector.fix=all and beam.fix=all). To disable this", "sanity check set disable_unit_cell_volume_sanity_check=True.")) %( 100*volume_change) raise Sorry(msg) self.refined_reflections = refined_reflections.select( refined_reflections['id'] > -1) for i, imageset in enumerate(self.imagesets): ref_sel = self.refined_reflections.select( self.refined_reflections['imageset_id'] == i) ref_sel = ref_sel.select(ref_sel['id'] >= 0) for i_expt in set(ref_sel['id']): expt = refined_experiments[i_expt] imageset.set_detector(expt.detector) imageset.set_beam(expt.beam) imageset.set_goniometer(expt.goniometer) imageset.set_scan(expt.scan) expt.imageset = imageset if not (self.all_params.refinement.parameterisation.beam.fix == 'all' and self.all_params.refinement.parameterisation.detector.fix == 'all'): # Experimental geometry may have changed - re-map centroids to # reciprocal space spots_mm = self.reflections self.reflections = flex.reflection_table() for i, imageset in enumerate(self.imagesets): spots_sel = spots_mm.select(spots_mm['imageset_id'] == i) self.map_centroids_to_reciprocal_space( spots_sel, imageset.get_detector(), imageset.get_beam(), imageset.get_goniometer()) self.reflections.extend(spots_sel) # update for next cycle experiments = refined_experiments self.refined_experiments = refined_experiments ################################################### # end of the long indexing/refinement while loop... ################################################### if not 'refined_experiments' in locals(): raise Sorry("None of the experiments could refine.") # discard experiments with zero reflections after refinement id_set = set(self.refined_reflections['id']) if len(id_set) < len(self.refined_experiments): filtered_refined_reflections = flex.reflection_table() for i in xrange(len(self.refined_experiments)): if i not in id_set: del self.refined_experiments[i] for old, new in zip(sorted(id_set), range(len(id_set))): subset = self.refined_reflections.select(self.refined_reflections['id'] == old) subset['id'] = flex.int(len(subset), new) filtered_refined_reflections.extend(subset) self.refined_reflections = filtered_refined_reflections if len(self.refined_experiments) > 1: from dials.algorithms.indexing.compare_orientation_matrices \ import show_rotation_matrix_differences show_rotation_matrix_differences( self.refined_experiments.crystals(), out=info_handle) logger.info("Final refined crystal models:") for i, crystal_model in enumerate(self.refined_experiments.crystals()): n_indexed = 0 for i_expt in experiments.where(crystal=crystal_model): n_indexed += (self.reflections['id'] == i).count(True) logger.info("model %i (%i reflections):" %(i+1, n_indexed)) logger.info(crystal_model) if 'xyzcal.mm' in self.refined_reflections: # won't be there if refine_all_candidates = False and no isoforms self.refined_reflections['xyzcal.px'] = flex.vec3_double( len(self.refined_reflections)) for i, imageset in enumerate(self.imagesets): imgset_sel = self.refined_reflections['imageset_id'] == i # set xyzcal.px field in self.refined_reflections refined_reflections = self.refined_reflections.select(imgset_sel) panel_numbers = flex.size_t(refined_reflections['panel']) xyzcal_mm = refined_reflections['xyzcal.mm'] x_mm, y_mm, z_rad = xyzcal_mm.parts() xy_cal_mm = flex.vec2_double(x_mm, y_mm) xy_cal_px = flex.vec2_double(len(xy_cal_mm)) for i_panel in range(len(imageset.get_detector())): panel = imageset.get_detector()[i_panel] sel = (panel_numbers == i_panel) isel = sel.iselection() ref_panel = refined_reflections.select(panel_numbers == i_panel) xy_cal_px.set_selected( sel, panel.millimeter_to_pixel(xy_cal_mm.select(sel))) x_px, y_px = xy_cal_px.parts() scan = imageset.get_scan() if scan is not None: z_px = scan.get_array_index_from_angle(z_rad, deg=False) else: # must be a still image, z centroid not meaningful z_px = z_rad xyzcal_px = flex.vec3_double(x_px, y_px, z_px) self.refined_reflections['xyzcal.px'].set_selected(imgset_sel, xyzcal_px)
def choose_best_orientation_matrix(self, candidate_orientation_matrices): from dxtbx.model.experiment_list import Experiment, ExperimentList import copy logger.info('*' * 80) logger.info('Selecting the best orientation matrix') logger.info('*' * 80) from libtbx import group_args class candidate_info(group_args): pass candidates = [] params = copy.deepcopy(self.all_params) n_cand = len(candidate_orientation_matrices) for icm,cm in enumerate(candidate_orientation_matrices): # Index reflections in P1 print (cm) print("\n\n REFINING CANDIDATE HERE\n\n",) sel = ((self.reflections['id'] == -1)) refl = self.reflections.select(sel) experiments = self.experiment_list_for_crystal(cm) # NOTE: important method call, decides what # reflections are indexed and which are not self.index_reflections(experiments, refl) indexed = refl.select(refl['id'] >= 0) indexed = indexed.select(indexed.get_flags(indexed.flags.indexed)) # If target symmetry supplied, try to apply it. Then, apply the change of basis to the reflections # indexed in P1 to the target setting if self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None: target_space_group = self.target_symmetry_primitive.space_group() new_crystal, cb_op_to_primitive = self.apply_symmetry(cm, target_space_group) if new_crystal is None: print("Cannot convert to target symmetry, candidate %d/%d"%(icm, n_cand)) continue new_crystal = new_crystal.change_basis(self.cb_op_primitive_inp) cm = candidate_orientation_matrices[icm] = new_crystal experiments = self.experiment_list_for_crystal(cm) if not cb_op_to_primitive.is_identity_op(): indexed['miller_index'] = cb_op_to_primitive.apply(indexed['miller_index']) if self.cb_op_primitive_inp is not None: indexed['miller_index'] = self.cb_op_primitive_inp.apply(indexed['miller_index']) if params.indexing.stills.refine_all_candidates: try: print("$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d initial outlier identification"%(icm, n_cand)) acceptance_flags = self.identify_outliers(params, experiments, indexed) #create a new "indexed" list with outliers thrown out: indexed = indexed.select(acceptance_flags) indexed = indexed print("$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d refinement before outlier rejection"%(icm, n_cand)) R = e_refine(params = params, experiments=experiments, reflections=indexed, graph_verbose=False) ref_experiments = R.get_experiments() # try to improve the outcome with a second round of outlier rejection post-initial refinement: acceptance_flags = self.identify_outliers(params, ref_experiments, indexed) # insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection nv0 = nave_parameters(params = params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False) crystal_model_nv0 = nv0() acceptance_flags_nv0 = nv0.nv_acceptance_flags indexed = indexed.select(acceptance_flags & acceptance_flags_nv0) print("$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d after positional and delta-psi outlier rejection"%(icm, n_cand)) R = e_refine(params = params, experiments=ref_experiments, reflections=indexed, graph_verbose=False) ref_experiments = R.get_experiments() nv = nave_parameters(params = params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False) crystal_model = nv() # Drop candidates that after refinement can no longer be converted to the known target space group if not self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None: target_space_group = self.target_symmetry_primitive.space_group() new_crystal, cb_op_to_primitive = self.apply_symmetry(crystal_model, target_space_group) if new_crystal is None: print("P1 refinement yielded model diverged from target, candidate %d/%d"%(icm, n_cand)) continue rmsd, _ = calc_2D_rmsd_and_displacements(R.predict_for_reflection_table(indexed)) except Exception as e: print("Couldn't refine candiate %d/%d, %s"%(icm, n_cand, str(e))) else: print("$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d done"%(icm, n_cand)) candidates.append(candidate_info(crystal = crystal_model, green_curve_area = nv.green_curve_area, ewald_proximal_volume = nv.ewald_proximal_volume(), n_indexed = len(indexed), rmsd = rmsd, indexed = indexed, experiments = ref_experiments)) else: from dials.algorithms.refinement.prediction.managed_predictors import ExperimentsPredictorFactory ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, force_stills=True, spherical_relp=params.refinement.parameterisation.spherical_relp_model) rmsd, _ = calc_2D_rmsd_and_displacements(ref_predictor(indexed)) candidates.append(candidate_info(crystal = cm, n_indexed = len(indexed), rmsd = rmsd, indexed = indexed, experiments = experiments)) if len(candidates) == 0: raise Sorry("No suitable indexing solution found") print("**** ALL CANDIDATES:") for i,XX in enumerate(candidates): print("\n****Candidate %d"%i,XX) cc = XX.crystal if hasattr(cc, 'get_half_mosaicity_deg'): print(" half mosaicity %5.2f deg."%(cc.get_half_mosaicity_deg())) print(" domain size %.0f Ang."%(cc.get_domain_size_ang())) print("\n**** BEST CANDIDATE:") results = flex.double([c.rmsd for c in candidates]) best = candidates[flex.min_index(results)] print(best) self.best_rmsd = best.rmsd if params.indexing.stills.refine_all_candidates: if best.rmsd > params.indexing.stills.rmsd_min_px: raise Sorry ("RMSD too high, %f" %best.rmsd) if best.ewald_proximal_volume > params.indexing.stills.ewald_proximal_volume_max: raise Sorry ("Ewald proximity volume too high, %f"%best.ewald_proximal_volume) if len(candidates) > 1: for i in xrange(len(candidates)): if i == flex.min_index(results): continue if best.ewald_proximal_volume > candidates[i].ewald_proximal_volume: print("Couldn't figure out which candidate is best; picked the one with the best RMSD.") best.indexed['entering'] = flex.bool(best.n_indexed, False) return best.crystal, best.n_indexed
def sequence_to_stills(experiments, reflections, params): assert len(reflections) == 1 reflections = reflections[0] new_experiments = ExperimentList() new_reflections = flex.reflection_table() # This is the subset needed to integrate for key in [ "id", "imageset_id", "shoebox", "bbox", "intensity.sum.value", "intensity.sum.variance", "entering", "flags", "miller_index", "panel", "xyzobs.px.value", "xyzobs.px.variance", ]: if key in reflections: new_reflections[key] = type(reflections[key])() elif key == "imageset_id": assert len(experiments.imagesets()) == 1 reflections["imageset_id"] = flex.int(len(reflections), 0) new_reflections["imageset_id"] = flex.int() elif key == "entering": reflections["entering"] = flex.bool(len(reflections), False) new_reflections["entering"] = flex.bool() else: raise RuntimeError( "Expected key not found in reflection table: %s" % key) for expt_id, experiment in enumerate(experiments): # Get the goniometr setting matrix goniometer_setting_matrix = matrix.sqr( experiment.goniometer.get_setting_rotation()) goniometer_axis = matrix.col(experiment.goniometer.get_rotation_axis()) step = experiment.scan.get_oscillation()[1] refls = reflections.select(reflections["id"] == expt_id) _, _, _, _, z1, z2 = refls["bbox"].parts() # Create an experiment for each scanpoint for i_scan_point in range(*experiment.scan.get_array_range()): if params.max_scan_points and i_scan_point >= params.max_scan_points: break # The A matrix is the goniometer setting matrix for this scan point # times the scan varying A matrix at this scan point. Note, the # goniometer setting matrix for scan point zero will be the identity # matrix and represents the beginning of the oscillation. # For stills, the A matrix needs to be positioned in the midpoint of an # oscillation step. Hence, here the goniometer setting matrixis rotated # by a further half oscillation step. A = (goniometer_axis.axis_and_angle_as_r3_rotation_matrix( angle=experiment.scan.get_angle_from_array_index(i_scan_point) + (step / 2), deg=True, ) * goniometer_setting_matrix * matrix.sqr( experiment.crystal.get_A_at_scan_point(i_scan_point))) crystal = MosaicCrystalSauter2014(experiment.crystal) crystal.set_A(A) # Copy in mosaic parameters if available if params.output.domain_size_ang is None and hasattr( experiment.crystal, "get_domain_size_ang"): crystal.set_domain_size_ang( experiment.crystal.get_domain_size_ang()) elif params.output.domain_size_ang is not None: crystal.set_domain_size_ang(params.output.domain_size_ang) if params.output.half_mosaicity_deg is None and hasattr( experiment.crystal, "get_half_mosaicity_deg"): crystal.set_half_mosaicity_deg( experiment.crystal.get_half_mosaicity_deg()) elif params.output.half_mosaicity_deg is not None: crystal.set_half_mosaicity_deg( params.output.half_mosaicity_deg) new_experiment = Experiment( detector=experiment.detector, beam=experiment.beam, crystal=crystal, imageset=experiment.imageset.as_imageset() [i_scan_point:i_scan_point + 1], ) new_experiments.append(new_experiment) # Each reflection in a 3D shoebox can be found on multiple images. # Slice the reflections such that any reflection on this scan point # is included with this image new_id = len(new_experiments) - 1 subrefls = refls.select((i_scan_point >= z1) & (i_scan_point < z2)) for refl in subrefls.rows(): assert i_scan_point in range(*refl["bbox"][4:6]) new_sb = Shoebox() start = i_scan_point - refl["bbox"][4] # z1 new_sb.data = refl["shoebox"].data[start:start + 1, :, :] new_sb.background = refl["shoebox"].background[start:start + 1, :, :] new_sb.mask = refl["shoebox"].mask[start:start + 1, :, :] intensity = new_sb.summed_intensity() new_sb.bbox = tuple( list(refl["bbox"])[0:4] + [0, 1]) # keep the original shoebox but reset the z values new_sb.panel = refl["panel"] new_refl = {} new_refl["id"] = new_refl["imageset_id"] = new_id new_refl["shoebox"] = new_sb new_refl["bbox"] = new_sb.bbox new_refl["intensity.sum.value"] = intensity.observed.value new_refl[ "intensity.sum.variance"] = intensity.observed.variance for key in ["entering", "flags", "miller_index", "panel"]: new_refl[key] = refl[key] centroid = new_sb.centroid_foreground_minus_background() new_refl["xyzobs.px.value"] = centroid.px.position new_refl["xyzobs.px.variance"] = centroid.px.variance new_reflections.append({}) for key in new_refl: new_reflections[key][-1] = new_refl[key] # Re-predict using the reflection slices and the stills predictors ref_predictor = ExperimentsPredictorFactory.from_experiments( new_experiments, force_stills=new_experiments.all_stills()) new_reflections = ref_predictor(new_reflections) return (new_experiments, new_reflections)
def index(self): experiments = ExperimentList() had_refinement_error = False have_similar_crystal_models = False while True: if had_refinement_error or have_similar_crystal_models: break max_lattices = self.params.multiple_lattice_search.max_lattices if max_lattices is not None and len(experiments) >= max_lattices: break if len(experiments) > 0: cutoff_fraction = (self.params.multiple_lattice_search. recycle_unindexed_reflections_cutoff) d_spacings = 1 / self.reflections["rlp"].norms() d_min_indexed = flex.min( d_spacings.select(self.indexed_reflections)) min_reflections_for_indexing = cutoff_fraction * len( self.reflections.select(d_spacings > d_min_indexed)) crystal_ids = self.reflections.select( d_spacings > d_min_indexed)["id"] if (crystal_ids == -1).count(True) < min_reflections_for_indexing: logger.info( "Finish searching for more lattices: %i unindexed reflections remaining." % ((crystal_ids == -1).count(True))) break n_lattices_previous_cycle = len(experiments) if self.d_min is None: self.d_min = self.params.refinement_protocol.d_min_start if len(experiments) == 0: new_expts = self.find_lattices() generate_experiment_identifiers(new_expts) experiments.extend(new_expts) else: try: new = self.find_lattices() generate_experiment_identifiers(new) experiments.extend(new) except DialsIndexError: logger.info("Indexing remaining reflections failed") if self.params.refinement_protocol.d_min_step is libtbx.Auto: n_cycles = self.params.refinement_protocol.n_macro_cycles if self.d_min is None or n_cycles == 1: self.params.refinement_protocol.d_min_step = 0 else: d_spacings = 1 / self.reflections["rlp"].norms() d_min_all = flex.min(d_spacings) self.params.refinement_protocol.d_min_step = ( self.d_min - d_min_all) / (n_cycles - 1) logger.info("Using d_min_step %.1f" % self.params.refinement_protocol.d_min_step) if len(experiments) == 0: raise DialsIndexError("No suitable lattice could be found.") elif len(experiments) == n_lattices_previous_cycle: # no more lattices found break for i_cycle in range( self.params.refinement_protocol.n_macro_cycles): if (i_cycle > 0 and self.d_min is not None and self.params.refinement_protocol.d_min_step > 0): d_min = self.d_min - self.params.refinement_protocol.d_min_step d_min = max(d_min, 0) if self.params.refinement_protocol.d_min_final is not None: d_min = max( d_min, self.params.refinement_protocol.d_min_final) if d_min >= 0: self.d_min = d_min logger.info("Increasing resolution to %.2f Angstrom" % d_min) # reset reflection lattice flags # the lattice a given reflection belongs to: a value of -1 indicates # that a reflection doesn't belong to any lattice so far self.reflections["id"] = flex.int(len(self.reflections), -1) self.index_reflections(experiments, self.reflections) if i_cycle == 0 and self.params.known_symmetry.space_group is not None: self._apply_symmetry_post_indexing( experiments, self.reflections, n_lattices_previous_cycle) logger.info("\nIndexed crystal models:") self.show_experiments(experiments, self.reflections, d_min=self.d_min) if self._check_have_similar_crystal_models(experiments): have_similar_crystal_models = True break logger.info("") logger.info("#" * 80) logger.info("Starting refinement (macro-cycle %i)" % (i_cycle + 1)) logger.info("#" * 80) logger.info("") self.indexed_reflections = self.reflections["id"] > -1 sel = flex.bool(len(self.reflections), False) lengths = 1 / self.reflections["rlp"].norms() if self.d_min is not None: isel = (lengths <= self.d_min).iselection() sel.set_selected(isel, True) sel.set_selected(self.reflections["id"] == -1, True) self.reflections.unset_flags(sel, self.reflections.flags.indexed) self.unindexed_reflections = self.reflections.select(sel) reflections_for_refinement = self.reflections.select( self.indexed_reflections) if self.params.refinement_protocol.mode == "repredict_only": refined_experiments, refined_reflections = ( experiments, reflections_for_refinement, ) from dials.algorithms.refinement.prediction.managed_predictors import ( ExperimentsPredictorFactory, ) ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, spherical_relp=self.all_params.refinement. parameterisation.spherical_relp_model, ) ref_predictor(refined_reflections) else: try: refined_experiments, refined_reflections = self.refine( experiments, reflections_for_refinement) except (DialsRefineConfigError, DialsRefineRuntimeError) as e: if len(experiments) == 1: raise DialsIndexRefineError(str(e)) had_refinement_error = True logger.info("Refinement failed:") logger.info(e) del experiments[-1] # remove experiment id from the reflections associated # with this deleted experiment - indexed flag removed # below last = len(experiments) sel = refined_reflections["id"] == last logger.info("Removing %d reflections with id %d" % (sel.count(True), last)) refined_reflections["id"].set_selected(sel, -1) break self._unit_cell_volume_sanity_check(experiments, refined_experiments) self.refined_reflections = refined_reflections self.refined_reflections.unset_flags( self.refined_reflections["id"] < 0, self.refined_reflections.flags.indexed, ) for i, expt in enumerate(self.experiments): ref_sel = self.refined_reflections.select( self.refined_reflections["imageset_id"] == i) ref_sel = ref_sel.select(ref_sel["id"] >= 0) for i_expt in set(ref_sel["id"]): refined_expt = refined_experiments[i_expt] expt.detector = refined_expt.detector expt.beam = refined_expt.beam expt.goniometer = refined_expt.goniometer expt.scan = refined_expt.scan refined_expt.imageset = expt.imageset if not (self.all_params.refinement.parameterisation.beam.fix == "all" and self.all_params.refinement. parameterisation.detector.fix == "all"): # Experimental geometry may have changed - re-map centroids to # reciprocal space self.reflections.map_centroids_to_reciprocal_space( self.experiments) # update for next cycle experiments = refined_experiments self.refined_experiments = refined_experiments logger.info("\nRefined crystal models:") self.show_experiments(self.refined_experiments, self.reflections, d_min=self.d_min) if (i_cycle >= 2 and self.d_min == self.params.refinement_protocol.d_min_final): logger.info( "Target d_min_final reached: finished with refinement") break if self.refined_experiments is None: raise DialsIndexRefineError( "None of the experiments could refine.") if len(self.refined_experiments) > 1: from dials.algorithms.indexing.compare_orientation_matrices import ( rotation_matrix_differences, ) logger.info( rotation_matrix_differences( self.refined_experiments.crystals())) self._xyzcal_mm_to_px(self.refined_experiments, self.refined_reflections)
def _build_components(cls, params, reflections, experiments): """low level build""" # Currently a refinement job can only have one parameterisation of the # prediction equation. This can either be of the XYDelPsi (stills) type, the # XYPhi (scans) type or the scan-varying XYPhi type with a varying crystal # model single_as_still = params.refinement.parameterisation.treat_single_image_as_still exps_are_stills = [] for exp in experiments: if exp.scan is None: exps_are_stills.append(True) elif exp.scan.get_num_images() == 1: if single_as_still: exps_are_stills.append(True) elif exp.scan.is_still(): exps_are_stills.append(True) else: exps_are_stills.append(False) else: if exp.scan.get_oscillation()[1] <= 0.0: raise DialsRefineConfigError( "Cannot refine a zero-width scan") exps_are_stills.append(False) # check experiment types are consistent if not all(exps_are_stills[0] == e for e in exps_are_stills): raise DialsRefineConfigError( "Cannot refine a mixture of stills and scans") do_stills = exps_are_stills[0] # If experiments are stills, ensure scan-varying refinement won't be attempted if do_stills: params.refinement.parameterisation.scan_varying = False # Refiner does not accept scan_varying=Auto. This is a special case for # doing macrocycles of refinement in dials.refine. if params.refinement.parameterisation.scan_varying is libtbx.Auto: params.refinement.parameterisation.scan_varying = False # Trim scans and calculate reflection block_width if required for scan-varying refinement if (params.refinement.parameterisation.scan_varying and params.refinement.parameterisation.trim_scan_to_observations): experiments = _trim_scans_to_observations(experiments, reflections) from dials.algorithms.refinement.reflection_manager import BlockCalculator block_calculator = BlockCalculator(experiments, reflections) if params.refinement.parameterisation.compose_model_per == "block": reflections = block_calculator.per_width( params.refinement.parameterisation.block_width, deg=True) elif params.refinement.parameterisation.compose_model_per == "image": reflections = block_calculator.per_image() logger.debug("\nBuilding reflection manager") logger.debug("Input reflection list size = %d observations", len(reflections)) # create reflection manager refman = ReflectionManagerFactory.from_parameters_reflections_experiments( params.refinement.reflections, reflections, experiments, do_stills) logger.debug( "Number of observations that pass initial inclusion criteria = %d", refman.get_accepted_refs_size(), ) sample_size = refman.get_sample_size() if sample_size > 0: logger.debug("Working set size = %d observations", sample_size) logger.debug("Reflection manager built\n") # configure use of sparse data types params = cls.config_sparse(params, experiments) do_sparse = params.refinement.parameterisation.sparse # create managed reflection predictor ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, force_stills=do_stills, spherical_relp=params.refinement.parameterisation. spherical_relp_model, ) # Predict for the managed observations, set columns for residuals and set # the used_in_refinement flag to the predictions obs = refman.get_obs() ref_predictor(obs) x_obs, y_obs, phi_obs = obs["xyzobs.mm.value"].parts() x_calc, y_calc, phi_calc = obs["xyzcal.mm"].parts() obs["x_resid"] = x_calc - x_obs obs["y_resid"] = y_calc - y_obs obs["phi_resid"] = phi_calc - phi_obs # determine whether to do basic centroid analysis to automatically # determine outlier rejection block if params.refinement.reflections.outlier.block_width is libtbx.Auto: ca = refman.get_centroid_analyser() analysis = ca(calc_average_residuals=False, calc_periodograms=False) else: analysis = None # Now predictions and centroid analysis are available, so we can finalise # the reflection manager refman.finalise(analysis) # Create model parameterisations logger.debug("Building prediction equation parameterisation") pred_param = build_prediction_parameterisation( params.refinement.parameterisation, experiments, refman, do_stills) # Build a constraints manager, if requested cmf = ConstraintManagerFactory(params, pred_param) constraints_manager = cmf() # Test for parameters that have too little data to refine and act accordingly autoreduce = AutoReduce( params.refinement.parameterisation.auto_reduction, pred_param, refman, constraints_manager, cmf, ) autoreduce() # if reduction was done, constraints_manager will have changed constraints_manager = autoreduce.constraints_manager # Build a restraints parameterisation (if requested). # Only unit cell restraints are supported at the moment. restraints_parameterisation = cls.config_restraints( params.refinement.parameterisation, pred_param) # Parameter reporting logger.debug("Prediction equation parameterisation built") logger.debug("Parameter order : name mapping") for i, e in enumerate(pred_param.get_param_names()): logger.debug("Parameter %03d : %s", i + 1, e) param_reporter = ParameterReporter( pred_param.get_detector_parameterisations(), pred_param.get_beam_parameterisations(), pred_param.get_crystal_orientation_parameterisations(), pred_param.get_crystal_unit_cell_parameterisations(), pred_param.get_goniometer_parameterisations(), ) # Create target function logger.debug("Building target function") target = cls.config_target( params.refinement.target, experiments, refman, ref_predictor, pred_param, restraints_parameterisation, do_stills, do_sparse, ) logger.debug("Target function built") # create refinery logger.debug("Building refinement engine") refinery = cls.config_refinery(params, target, pred_param, constraints_manager) logger.debug("Refinement engine built") nparam = len(pred_param) ndim = target.dim nref = len(refman.get_matches()) logger.info( "There are %s parameters to refine against %s reflections in %s dimensions", nparam, nref, ndim, ) if not params.refinement.parameterisation.sparse and isinstance( refinery, AdaptLstbx): dense_jacobian_gigabytes = (nparam * nref * ndim * flex.double.element_size()) / 1e9 avail_memory_gigabytes = psutil.virtual_memory().available / 1e9 # Report if the Jacobian requires a large amount of storage if (dense_jacobian_gigabytes > 0.2 * avail_memory_gigabytes or dense_jacobian_gigabytes > 0.5): logger.info( "Storage of the Jacobian matrix requires %.1f GB", dense_jacobian_gigabytes, ) # build refiner interface and return if params.refinement.parameterisation.scan_varying: refiner = ScanVaryingRefiner else: refiner = Refiner return refiner(experiments, pred_param, param_reporter, refman, target, refinery)
def calculate_fractional_hkl_from_Ainverse_q(self, reflections, experiments, debug=False): ''' Calculate hkl_frac = A^-1*q. Will also calculate the integer hkl values ''' assert len( experiments.crystals()) == 1, 'Should have only one crystal model' #self.map_centroids_to_reciprocal_space(observed, ) from scitbx.matrix import sqr Ainverse = flex.mat3_double( len(reflections), sqr(experiments.crystals()[0].get_A()).inverse()) q = reflections['rlp'] hkl_frac = Ainverse * q hkl = hkl_frac.iround() reflections['miller_index'] = flex.miller_index( len(reflections), (0, 0, 0)) reflections['fractional_miller_index'] = flex.vec3_double( len(reflections), (0.0, 0.0, 0.0)) reflections['miller_index'] = flex.miller_index(list(hkl)) reflections['fractional_miller_index'] = hkl_frac reflections.set_flags(reflections['miller_index'] != (0, 0, 0), reflections.flags.indexed) reflections['id'].set_selected(flex.size_t(range(len(reflections))), 0) # Add predicted reflections from dials.algorithms.refinement.prediction.managed_predictors import ExperimentsPredictorFactory ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, force_stills=experiments.all_stills()) reflections = ref_predictor(reflections) reflections['id'].set_selected(flex.size_t(range(len(reflections))), -1) def _apply_symmetry_post_indexing(self, experiments, reflections, n_lattices_previous_cycle): # Taken from DIALS post Aug_Refactor # now apply the space group symmetry only after the first indexing # need to make sure that the symmetrized orientation is similar to the P1 model for cryst in experiments.crystals()[n_lattices_previous_cycle:]: new_cryst, cb_op_to_primitive = self._symmetry_handler.apply_symmetry( cryst) if self._symmetry_handler.cb_op_primitive_inp is not None: new_cryst = new_cryst.change_basis( self._symmetry_handler.cb_op_primitive_inp) cryst.update(new_cryst) cryst.set_space_group( self.params.known_symmetry.space_group.group()) for i_expt, expt in enumerate(experiments): if expt.crystal is not cryst: continue if not cb_op_to_primitive.is_identity_op(): miller_indices = reflections["miller_index"].select( reflections["id"] == i_expt) miller_indices = cb_op_to_primitive.apply( miller_indices) reflections["miller_index"].set_selected( reflections["id"] == i_expt, miller_indices) if self._symmetry_handler.cb_op_primitive_inp is not None: miller_indices = reflections["miller_index"].select( reflections["id"] == i_expt) miller_indices = self._symmetry_handler.cb_op_primitive_inp.apply( miller_indices) reflections["miller_index"].set_selected( reflections["id"] == i_expt, miller_indices) # IOTA from scitbx.matrix import sqr hklfrac = flex.mat3_double( len(miller_indices), sqr(cryst.get_A()).inverse()) * self.reflections[ 'rlp'].select(self.reflections['id'] == i_expt) self.reflections[ 'fractional_miller_index'].set_selected( self.reflections['id'] == i_expt, hklfrac)
def plot_residual_vectors(experiment_list_files, reflection_table_files, params): # Read in an experiment list and reflection table for fjson, fpickle in zip(experiment_list_files, reflection_table_files): experiments = ExperimentListFactory.from_json_file(fjson, check_format=False) reflections = load(fpickle) ref_predictor = ExperimentsPredictorFactory.from_experiments(experiments, force_stills=experiments.all_stills()) reflections = ref_predictor(reflections) for ii,expt in enumerate(experiments): correction_vectors_provisional = [] indexed_pairs_provisional = [] cbf_now = experiments[ii].imageset.get_image_identifier(0).split('/')[-1] FWMOSAICITY=2*expt.crystal.get_half_mosaicity_deg() DOMAIN_SZ_ANG=expt.crystal.get_domain_size_ang() refl_now = reflections.select(reflections['id'] == ii) this_setting_matched_indices = refl_now["miller_index"] hkllist = refl_now['miller_index'] for j,item in enumerate(this_setting_matched_indices): # Every miller index has a prediction. So assuming this_setting_index same as j this_setting_index = hkllist.first_index(item) Match = dict(spot=j,pred=this_setting_index) indexed_pairs_provisional.append(Match) for refl in refl_now: vector = col((refl['xyzobs.px.value'][0]-refl['xyzcal.px'][0], refl['xyzobs.px.value'][1]-refl['xyzcal.px'][1])) correction_vectors_provisional.append(vector) if params.show_plot and params.show_residual_scatter_plot: from matplotlib import pyplot as plt fig = plt.figure() for cv in correction_vectors_provisional: plt.plot([cv[1]],[-cv[0]],"r.") plt.axes().set_aspect("equal") PX = refl_now["xyzobs.px.value"] if params.show_plot and params.show_residual_map_plot: from matplotlib import pyplot as plt fig = plt.figure() for match,cv in zip(indexed_pairs_provisional,correction_vectors_provisional): # First plot Observed spot position #plt.plot([PX[match["spot"]][1]],[-PX[match["spot"]][0]],"r*", markersize=10) # Now plot Calculated spot position #plt.plot([refl_now['xyzcal.px'][match["pred"]][1]],[-refl_now['xyzcal.px'][match["pred"]][0]],"g.", markersize=20) #plt.plot([PX[match["spot"]][1], PX[match["spot"]][1] + 10.*cv[1]], # [-PX[match["spot"]][0], -PX[match["spot"]][0] - 10.*cv[0]],'r-', linewidth=1, solid_capstyle="butt") plt.arrow(PX[match["spot"]][1], -PX[match["spot"]][0], 20.*cv[1], -20.*cv[0], width=0.5) if True: # Uses the calculated spot position from rstbx.apps.stills.util import residual_map_special_deltapsi_add_on temp_expt_list = ExperimentList() temp_expt_list.append(expt) residual_map_special_deltapsi_add_on( reflections = refl_now, matches = indexed_pairs_provisional, experiments=temp_expt_list, hkllist = hkllist, predicted = refl_now['xyzcal.mm'], plot=plt, eta_deg=FWMOSAICITY, deff=DOMAIN_SZ_ANG, markersize=20 ) plt.xlim([0,expt.detector[0].get_image_size()[1]]) plt.ylim([-expt.detector[0].get_image_size()[0],0]) plt.title('%s'%cbf_now) # plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))))) plt.axes().set_aspect("equal") if params.show_plot and (params.show_residual_scatter_plot or params.show_residual_map_plot): plt.show() plt.close()