def index(self): # most of this is the same as dials.algorithms.indexing.indexer.indexer_base.index(), with some stills # specific modifications (don't re-index after choose best orientation matrix, but use the indexing from # choose best orientation matrix, also don't use macrocycles) of refinement after indexing. # 2017 update: do accept multiple lattices per shot experiments = ExperimentList() while True: self.d_min = self.params.refinement_protocol.d_min_start max_lattices = self.params.multiple_lattice_search.max_lattices if max_lattices is not None and len(experiments) >= max_lattices: break if len(experiments) > 0: cutoff_fraction = ( self.params.multiple_lattice_search.recycle_unindexed_reflections_cutoff ) d_spacings = 1 / self.reflections["rlp"].norms() d_min_indexed = flex.min(d_spacings.select(self.indexed_reflections)) min_reflections_for_indexing = cutoff_fraction * len( self.reflections.select(d_spacings > d_min_indexed) ) crystal_ids = self.reflections.select(d_spacings > d_min_indexed)["id"] if (crystal_ids == -1).count(True) < min_reflections_for_indexing: logger.info( "Finish searching for more lattices: %i unindexed reflections remaining." % (min_reflections_for_indexing) ) break n_lattices_previous_cycle = len(experiments) # index multiple lattices per shot if len(experiments) == 0: new = self.find_lattices() generate_experiment_identifiers(new) experiments.extend(new) if len(experiments) == 0: raise DialsIndexError("No suitable lattice could be found.") else: try: new = self.find_lattices() generate_experiment_identifiers(new) experiments.extend(new) except Exception as e: logger.info("Indexing remaining reflections failed") logger.debug( "Indexing remaining reflections failed, exception:\n" + str(e) ) # reset reflection lattice flags # the lattice a given reflection belongs to: a value of -1 indicates # that a reflection doesn't belong to any lattice so far self.reflections["id"] = flex.int(len(self.reflections), -1) self.index_reflections(experiments, self.reflections) if len(experiments) == n_lattices_previous_cycle: # no more lattices found break if ( not self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None ): self._apply_symmetry_post_indexing( experiments, self.reflections, n_lattices_previous_cycle ) # discard nearly overlapping lattices on the same shot if self._check_have_similar_crystal_models(experiments): break self.indexed_reflections = self.reflections["id"] > -1 if self.d_min is None: sel = self.reflections["id"] <= -1 else: sel = flex.bool(len(self.reflections), False) lengths = 1 / self.reflections["rlp"].norms() isel = (lengths >= self.d_min).iselection() sel.set_selected(isel, True) sel.set_selected(self.reflections["id"] > -1, False) self.unindexed_reflections = self.reflections.select(sel) reflections_for_refinement = self.reflections.select( self.indexed_reflections ) if len(self.params.stills.isoforms) > 0: logger.info("") logger.info("#" * 80) logger.info("Starting refinement") logger.info("#" * 80) logger.info("") isoform_experiments = ExperimentList() isoform_reflections = flex.reflection_table() # Note, changes to params after initial indexing. Cannot use tie to target when fixing the unit cell. self.all_params.refinement.reflections.outlier.algorithm = "null" self.all_params.refinement.parameterisation.crystal.fix = "cell" self.all_params.refinement.parameterisation.crystal.unit_cell.restraints.tie_to_target = ( [] ) for expt_id, experiment in enumerate(experiments): reflections = reflections_for_refinement.select( reflections_for_refinement["id"] == expt_id ) reflections["id"] = flex.int(len(reflections), 0) refiners = [] for isoform in self.params.stills.isoforms: iso_experiment = copy.deepcopy(experiment) crystal = iso_experiment.crystal if ( isoform.lookup_symbol != crystal.get_space_group().type().lookup_symbol() ): logger.info( "Crystal isoform lookup_symbol %s does not match isoform %s lookup_symbol %s" % ( crystal.get_space_group().type().lookup_symbol(), isoform.name, isoform.lookup_symbol, ) ) continue crystal.set_B(isoform.cell.fractionalization_matrix()) logger.info("Refining isoform %s" % isoform.name) refiners.append( e_refine( params=self.all_params, experiments=ExperimentList([iso_experiment]), reflections=reflections, graph_verbose=False, ) ) if len(refiners) == 0: raise DialsIndexError( "No isoforms had a lookup symbol that matched" ) positional_rmsds = [ math.sqrt(P.rmsds()[0] ** 2 + P.rmsds()[1] ** 2) for P in refiners ] logger.info( "Positional rmsds for all isoforms:" + str(positional_rmsds) ) minrmsd_mm = min(positional_rmsds) minindex = positional_rmsds.index(minrmsd_mm) logger.info( "The smallest rmsd is %5.1f um from isoform %s" % ( 1000.0 * minrmsd_mm, self.params.stills.isoforms[minindex].name, ) ) if self.params.stills.isoforms[minindex].rmsd_target_mm is not None: logger.info( "Asserting %f < %f" % ( minrmsd_mm, self.params.stills.isoforms[minindex].rmsd_target_mm, ) ) assert ( minrmsd_mm < self.params.stills.isoforms[minindex].rmsd_target_mm ) logger.info( "Acceptable rmsd for isoform %s." % (self.params.stills.isoforms[minindex].name) ) if len(self.params.stills.isoforms) == 2: logger.info( "Rmsd gain over the other isoform %5.1f um." % (1000.0 * abs(positional_rmsds[0] - positional_rmsds[1])) ) R = refiners[minindex] # Now one last check to see if direct beam is out of bounds if self.params.stills.isoforms[minindex].beam_restraint is not None: from scitbx import matrix refined_beam = matrix.col( R.get_experiments()[0] .detector[0] .get_beam_centre_lab(experiments[0].beam.get_s0())[0:2] ) known_beam = matrix.col( self.params.stills.isoforms[minindex].beam_restraint ) logger.info( "Asserting difference in refined beam center and expected beam center %f < %f" % ( (refined_beam - known_beam).length(), self.params.stills.isoforms[minindex].rmsd_target_mm, ) ) assert ( (refined_beam - known_beam).length() < self.params.stills.isoforms[minindex].rmsd_target_mm ) # future--circle of confusion could be given as a separate length in mm instead of reusing rmsd_target experiment = R.get_experiments()[0] experiment.crystal.identified_isoform = self.params.stills.isoforms[ minindex ].name isoform_experiments.append(experiment) reflections["id"] = flex.int(len(reflections), expt_id) isoform_reflections.extend(reflections) experiments = isoform_experiments reflections_for_refinement = isoform_reflections if self.params.refinement_protocol.mode == "repredict_only": from dials.algorithms.indexing.nave_parameters import NaveParameters from dials.algorithms.refinement.prediction.managed_predictors import ( ExperimentsPredictorFactory, ) refined_experiments, refined_reflections = ( experiments, reflections_for_refinement, ) ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, force_stills=True, spherical_relp=self.all_params.refinement.parameterisation.spherical_relp_model, ) ref_predictor(refined_reflections) refined_reflections["delpsical2"] = ( refined_reflections["delpsical.rad"] ** 2 ) for expt_id in range(len(refined_experiments)): refls = refined_reflections.select( refined_reflections["id"] == expt_id ) nv = NaveParameters( params=self.all_params, experiments=refined_experiments[expt_id : expt_id + 1], reflections=refls, refinery=None, graph_verbose=False, ) experiments[expt_id].crystal = nv() ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, force_stills=True, spherical_relp=self.all_params.refinement.parameterisation.spherical_relp_model, ) ref_predictor(refined_reflections) elif self.params.refinement_protocol.mode is None: refined_experiments, refined_reflections = ( experiments, reflections_for_refinement, ) else: try: refined_experiments, refined_reflections = self.refine( experiments, reflections_for_refinement ) except Exception as e: s = str(e) if len(experiments) == 1: raise DialsIndexRefineError(e.message) logger.info("Refinement failed:") logger.info(s) del experiments[-1] break self._unit_cell_volume_sanity_check(experiments, refined_experiments) self.refined_reflections = refined_reflections.select( refined_reflections["id"] > -1 ) for i, expt in enumerate(self.experiments): ref_sel = self.refined_reflections.select( self.refined_reflections["imageset_id"] == i ) ref_sel = ref_sel.select(ref_sel["id"] >= 0) for i_expt in set(ref_sel["id"]): refined_expt = refined_experiments[i_expt] expt.detector = refined_expt.detector expt.beam = refined_expt.beam expt.goniometer = refined_expt.goniometer expt.scan = refined_expt.scan refined_expt.imageset = expt.imageset if not ( self.all_params.refinement.parameterisation.beam.fix == "all" and self.all_params.refinement.parameterisation.detector.fix == "all" ): # Experimental geometry may have changed - re-map centroids to # reciprocal space self.reflections.map_centroids_to_reciprocal_space(self.experiments) # update for next cycle experiments = refined_experiments self.refined_experiments = refined_experiments if self.refined_experiments is None: raise DialsIndexRefineError("None of the experiments could refine.") # discard experiments with zero reflections after refinement id_set = set(self.refined_reflections["id"]) if len(id_set) < len(self.refined_experiments): filtered_refined_reflections = flex.reflection_table() for i in range(len(self.refined_experiments)): if i not in id_set: del self.refined_experiments[i] for old, new in zip(sorted(id_set), range(len(id_set))): subset = self.refined_reflections.select( self.refined_reflections["id"] == old ) subset["id"] = flex.int(len(subset), new) filtered_refined_reflections.extend(subset) self.refined_reflections = filtered_refined_reflections if len(self.refined_experiments) > 1: from dials.algorithms.indexing.compare_orientation_matrices import ( rotation_matrix_differences, ) logger.info( rotation_matrix_differences(self.refined_experiments.crystals()) ) logger.info("Final refined crystal models:") for i, crystal_model in enumerate(self.refined_experiments.crystals()): n_indexed = 0 for _ in experiments.where(crystal=crystal_model): n_indexed += (self.reflections["id"] == i).count(True) logger.info("model %i (%i reflections):" % (i + 1, n_indexed)) logger.info(crystal_model) if ( "xyzcal.mm" in self.refined_reflections ): # won't be there if refine_all_candidates = False and no isoforms self._xyzcal_mm_to_px(self.experiments, self.refined_reflections)
def index(self): experiments = ExperimentList() had_refinement_error = False have_similar_crystal_models = False while True: if had_refinement_error or have_similar_crystal_models: break max_lattices = self.params.multiple_lattice_search.max_lattices if max_lattices is not None and len(experiments) >= max_lattices: break if len(experiments) > 0: cutoff_fraction = (self.params.multiple_lattice_search. recycle_unindexed_reflections_cutoff) d_spacings = 1 / self.reflections["rlp"].norms() d_min_indexed = flex.min( d_spacings.select(self.indexed_reflections)) min_reflections_for_indexing = cutoff_fraction * len( self.reflections.select(d_spacings > d_min_indexed)) crystal_ids = self.reflections.select( d_spacings > d_min_indexed)["id"] if (crystal_ids == -1).count(True) < min_reflections_for_indexing: logger.info( "Finish searching for more lattices: %i unindexed reflections remaining." % ((crystal_ids == -1).count(True))) break n_lattices_previous_cycle = len(experiments) if self.d_min is None: self.d_min = self.params.refinement_protocol.d_min_start if len(experiments) == 0: new_expts = self.find_lattices() generate_experiment_identifiers(new_expts) experiments.extend(new_expts) else: try: new = self.find_lattices() generate_experiment_identifiers(new) experiments.extend(new) except DialsIndexError: logger.info("Indexing remaining reflections failed") if self.params.refinement_protocol.d_min_step is libtbx.Auto: n_cycles = self.params.refinement_protocol.n_macro_cycles if self.d_min is None or n_cycles == 1: self.params.refinement_protocol.d_min_step = 0 else: d_spacings = 1 / self.reflections["rlp"].norms() d_min_all = flex.min(d_spacings) self.params.refinement_protocol.d_min_step = ( self.d_min - d_min_all) / (n_cycles - 1) logger.info("Using d_min_step %.1f" % self.params.refinement_protocol.d_min_step) if len(experiments) == 0: raise DialsIndexError("No suitable lattice could be found.") elif len(experiments) == n_lattices_previous_cycle: # no more lattices found break for i_cycle in range( self.params.refinement_protocol.n_macro_cycles): if (i_cycle > 0 and self.d_min is not None and self.params.refinement_protocol.d_min_step > 0): d_min = self.d_min - self.params.refinement_protocol.d_min_step d_min = max(d_min, 0) if self.params.refinement_protocol.d_min_final is not None: d_min = max( d_min, self.params.refinement_protocol.d_min_final) if d_min >= 0: self.d_min = d_min logger.info("Increasing resolution to %.2f Angstrom" % d_min) # reset reflection lattice flags # the lattice a given reflection belongs to: a value of -1 indicates # that a reflection doesn't belong to any lattice so far self.reflections["id"] = flex.int(len(self.reflections), -1) self.index_reflections(experiments, self.reflections) if i_cycle == 0 and self.params.known_symmetry.space_group is not None: self._apply_symmetry_post_indexing( experiments, self.reflections, n_lattices_previous_cycle) logger.info("\nIndexed crystal models:") self.show_experiments(experiments, self.reflections, d_min=self.d_min) if self._check_have_similar_crystal_models(experiments): have_similar_crystal_models = True break logger.info("") logger.info("#" * 80) logger.info("Starting refinement (macro-cycle %i)" % (i_cycle + 1)) logger.info("#" * 80) logger.info("") self.indexed_reflections = self.reflections["id"] > -1 sel = flex.bool(len(self.reflections), False) lengths = 1 / self.reflections["rlp"].norms() if self.d_min is not None: isel = (lengths <= self.d_min).iselection() sel.set_selected(isel, True) sel.set_selected(self.reflections["id"] == -1, True) self.reflections.unset_flags(sel, self.reflections.flags.indexed) self.unindexed_reflections = self.reflections.select(sel) reflections_for_refinement = self.reflections.select( self.indexed_reflections) if self.params.refinement_protocol.mode == "repredict_only": refined_experiments, refined_reflections = ( experiments, reflections_for_refinement, ) from dials.algorithms.refinement.prediction.managed_predictors import ( ExperimentsPredictorFactory, ) ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, spherical_relp=self.all_params.refinement. parameterisation.spherical_relp_model, ) ref_predictor(refined_reflections) else: try: refined_experiments, refined_reflections = self.refine( experiments, reflections_for_refinement) except (DialsRefineConfigError, DialsRefineRuntimeError) as e: if len(experiments) == 1: raise DialsIndexRefineError(str(e)) had_refinement_error = True logger.info("Refinement failed:") logger.info(e) del experiments[-1] # remove experiment id from the reflections associated # with this deleted experiment - indexed flag removed # below last = len(experiments) sel = refined_reflections["id"] == last logger.info("Removing %d reflections with id %d" % (sel.count(True), last)) refined_reflections["id"].set_selected(sel, -1) break self._unit_cell_volume_sanity_check(experiments, refined_experiments) self.refined_reflections = refined_reflections self.refined_reflections.unset_flags( self.refined_reflections["id"] < 0, self.refined_reflections.flags.indexed, ) for i, expt in enumerate(self.experiments): ref_sel = self.refined_reflections.select( self.refined_reflections["imageset_id"] == i) ref_sel = ref_sel.select(ref_sel["id"] >= 0) for i_expt in set(ref_sel["id"]): refined_expt = refined_experiments[i_expt] expt.detector = refined_expt.detector expt.beam = refined_expt.beam expt.goniometer = refined_expt.goniometer expt.scan = refined_expt.scan refined_expt.imageset = expt.imageset if not (self.all_params.refinement.parameterisation.beam.fix == "all" and self.all_params.refinement. parameterisation.detector.fix == "all"): # Experimental geometry may have changed - re-map centroids to # reciprocal space self.reflections.map_centroids_to_reciprocal_space( self.experiments) # update for next cycle experiments = refined_experiments self.refined_experiments = refined_experiments logger.info("\nRefined crystal models:") self.show_experiments(self.refined_experiments, self.reflections, d_min=self.d_min) if (i_cycle >= 2 and self.d_min == self.params.refinement_protocol.d_min_final): logger.info( "Target d_min_final reached: finished with refinement") break if self.refined_experiments is None: raise DialsIndexRefineError( "None of the experiments could refine.") if len(self.refined_experiments) > 1: from dials.algorithms.indexing.compare_orientation_matrices import ( rotation_matrix_differences, ) logger.info( rotation_matrix_differences( self.refined_experiments.crystals())) self._xyzcal_mm_to_px(self.refined_experiments, self.refined_reflections)