def setup_indexing(self): reflections_input = self.reflections self.reflections = flex.reflection_table() for i, expt in enumerate(self.experiments): if "imageset_id" not in reflections_input: reflections_input["imageset_id"] = reflections_input["id"] refl = reflections_input.select( reflections_input["imageset_id"] == i) refl.centroid_px_to_mm(expt.detector, expt.scan) self.reflections.extend(refl) if len(self.reflections) == 0: raise DialsIndexError("No reflections left to index!") self.reflections = self._map_centroids_to_reciprocal_space( self.experiments, self.reflections) self.reflections.calculate_entering_flags(self.experiments) self.find_max_cell() if self.params.sigma_phi_deg is not None: var_x, var_y, _ = self.reflections["xyzobs.mm.variance"].parts() var_phi_rad = flex.double( var_x.size(), (math.pi / 180 * self.params.sigma_phi_deg)**2) self.reflections["xyzobs.mm.variance"] = flex.vec3_double( var_x, var_y, var_phi_rad) if self.params.debug: self._debug_write_reciprocal_lattice_points_as_pdb() self.reflections["id"] = flex.int(len(self.reflections), -1)
def __init__(self, target_symmetry_primitive, max_lattices, params=None, *args, **kwargs): """Construct a LowResSpotMatch object. Args: target_symmetry_primitive (cctbx.crystal.symmetry): The target crystal symmetry and unit cell max_lattices (int): The maximum number of lattice models to find """ super().__init__(params=params, *args, **kwargs) self._target_symmetry_primitive = target_symmetry_primitive self._max_lattices = max_lattices if target_symmetry_primitive is None: raise DialsIndexError( "Target unit cell and space group must be provided for low_res_spot_match" ) # Set reciprocal space orthogonalisation matrix uc = self._target_symmetry_primitive.unit_cell() self.Bmat = matrix.sqr(uc.fractionalization_matrix()).transpose()
def _unit_cell_volume_sanity_check(self, original_experiments, refined_experiments): # sanity check for unrealistic unit cell volume increase during refinement # usually this indicates too many parameters are being refined given the # number of observations provided. if not self.params.refinement_protocol.disable_unit_cell_volume_sanity_check: for orig_expt, refined_expt in zip( original_experiments, refined_experiments ): uc1 = orig_expt.crystal.get_unit_cell() uc2 = refined_expt.crystal.get_unit_cell() volume_change = abs(uc1.volume() - uc2.volume()) / uc1.volume() cutoff = 0.5 if volume_change > cutoff: msg = ( "\n".join( ( "Unrealistic unit cell volume increase during refinement of %.1f%%.", "Please try refining fewer parameters, either by enforcing symmetry", "constraints (space_group=) and/or disabling experimental geometry", "refinement (detector.fix=all and beam.fix=all). To disable this", "sanity check set disable_unit_cell_volume_sanity_check=True.", ) ) % (100 * volume_change) ) raise DialsIndexError(msg)
def setup_indexing(self): if len(self.reflections) == 0: raise DialsIndexError("No reflections left to index!") if "imageset_id" not in self.reflections: self.reflections["imageset_id"] = self.reflections["id"] self.reflections.centroid_px_to_mm(self.experiments) self.reflections.map_centroids_to_reciprocal_space(self.experiments) self.reflections.calculate_entering_flags(self.experiments) self.find_max_cell() if self.params.sigma_phi_deg is not None: var_x, var_y, _ = self.reflections["xyzobs.mm.variance"].parts() var_phi_rad = flex.double( var_x.size(), (math.pi / 180 * self.params.sigma_phi_deg) ** 2 ) self.reflections["xyzobs.mm.variance"] = flex.vec3_double( var_x, var_y, var_phi_rad ) if self.params.debug: self._debug_write_reciprocal_lattice_points_as_pdb() self.reflections["id"] = flex.int(len(self.reflections), -1)
def _setup_target_unit_cell_and_space_group(self, target_unit_cell, target_space_group): target_bravais_t = bravais_lattice( group=target_space_group.info().reference_setting().group()) best_subgroup = None best_angular_difference = 1e8 space_groups = [target_space_group] if target_space_group.conventional_centring_type_symbol() != "P": space_groups.append(sgtbx.space_group()) for target in space_groups: cs = crystal.symmetry( unit_cell=target_unit_cell, space_group=target, assert_is_compatible_unit_cell=False, ) target_best_cell = cs.best_cell().unit_cell() subgroups = lattice_symmetry.metric_subgroups(cs, max_delta=0.1) for subgroup in subgroups.result_groups: bravais_t = bravais_lattice( group=subgroup["ref_subsym"].space_group()) if bravais_t == target_bravais_t: # allow for the cell to be given as best cell, reference setting # primitive settings, or minimum cell best_subsym = subgroup["best_subsym"] ref_subsym = best_subsym.as_reference_setting() if not (best_subsym.unit_cell().is_similar_to( target_unit_cell) or ref_subsym.unit_cell( ).is_similar_to(target_unit_cell) or ref_subsym.primitive_setting().unit_cell( ).is_similar_to(target_unit_cell) or best_subsym.primitive_setting().unit_cell( ).is_similar_to(target_unit_cell) or best_subsym.minimum_cell().unit_cell( ).is_similar_to(target_unit_cell.minimum_cell()) or best_subsym.unit_cell().is_similar_to( target_best_cell)): continue if subgroup[ "max_angular_difference"] < best_angular_difference: best_subgroup = subgroup best_angular_difference = subgroup[ "max_angular_difference"] if best_subgroup is None: raise DialsIndexError("Unit cell incompatible with space group") cb_op_inp_best = best_subgroup["cb_op_inp_best"] best_subsym = best_subgroup["best_subsym"] cb_op_best_ref = best_subsym.change_of_basis_op_to_reference_setting() self.cb_op_inp_ref = cb_op_best_ref * cb_op_inp_best self.target_symmetry_reference_setting = crystal.symmetry( unit_cell=target_unit_cell.change_basis(self.cb_op_inp_ref), space_group=target_space_group.info().as_reference_setting().group( ), )
def find_max_cell( reflections, max_cell_multiplier=1.3, step_size=45, nearest_neighbor_percentile=None, histogram_binning="linear", nn_per_bin=5, max_height_fraction=0.25, filter_ice=True, filter_overlaps=True, overlaps_border=0, ): logger.debug("Finding suitable max_cell based on %i reflections" % len(reflections)) # Exclude potential ice-ring spots from nearest neighbour analysis if needed if filter_ice: ice_sel = ice_rings_selection(reflections) reflections = reflections.select(~ice_sel) logger.debug("Rejecting %i reflections at ice ring resolution" % ice_sel.count(True)) # need bounding box in reflections to find overlaps; this is not there if # spots are from XDS (for example) if filter_overlaps and "bbox" in reflections: overlap_sel = flex.bool(len(reflections), False) overlaps = reflections.find_overlaps(border=overlaps_border) for item in overlaps.edges(): i0 = overlaps.source(item) i1 = overlaps.target(item) overlap_sel[i0] = True overlap_sel[i1] = True logger.debug("Rejecting %i overlapping bounding boxes" % overlap_sel.count(True)) reflections = reflections.select(~overlap_sel) logger.debug("%i reflections remain for max_cell identification" % len(reflections)) if not len(reflections): raise DialsIndexError( "Too few spots remaining for nearest neighbour analysis (%d)" % len(reflections)) NN = NeighborAnalysis( reflections, step_size=step_size, max_height_fraction=max_height_fraction, tolerance=max_cell_multiplier, percentile=nearest_neighbor_percentile, histogram_binning=histogram_binning, nn_per_bin=nn_per_bin, ) return NN
def __init__(self, max_cell, target_unit_cell, params=None, *args, **kwargs): """Construct a real_space_grid_search object. Args: max_cell (float): An estimate of the maximum cell dimension of the primitive cell. target_unit_cell (cctbx.uctbx.unit_cell): The target unit cell. """ super(RealSpaceGridSearch, self).__init__( max_cell, params=params, *args, **kwargs ) if target_unit_cell is None: raise DialsIndexError( "Target unit cell must be provided for real_space_grid_search" ) self._target_unit_cell = target_unit_cell
def _find_peaks(self, grid_real, d_min): grid_real_binary = grid_real.deep_copy() rmsd = math.sqrt( flex.mean( flex.pow2(grid_real_binary.as_1d() - flex.mean(grid_real_binary.as_1d())))) grid_real_binary.set_selected( grid_real_binary < (self._params.rmsd_cutoff) * rmsd, 0) grid_real_binary.as_1d().set_selected(grid_real_binary.as_1d() > 0, 1) grid_real_binary = grid_real_binary.iround() from cctbx import masks # real space FFT grid dimensions cell_lengths = [self._n_points * d_min / 2 for i in range(3)] self._fft_cell = uctbx.unit_cell(cell_lengths + [90] * 3) flood_fill = masks.flood_fill(grid_real_binary, self._fft_cell) if flood_fill.n_voids() < 4: # Require at least peak at origin and one peak for each basis vector raise DialsIndexError( "Indexing failed: fft3d peak search failed to find sufficient number of peaks." ) # the peak at the origin might have a significantly larger volume than the # rest so exclude any anomalously large peaks from determining minimum volume from scitbx.math import five_number_summary outliers = flex.bool(flood_fill.n_voids(), False) grid_points_per_void = flood_fill.grid_points_per_void() min_x, q1_x, med_x, q3_x, max_x = five_number_summary( grid_points_per_void) iqr_multiplier = 5 iqr_x = q3_x - q1_x cut_x = iqr_multiplier * iqr_x outliers.set_selected( grid_points_per_void.as_double() > (q3_x + cut_x), True) # print q3_x + cut_x, outliers.count(True) isel = (grid_points_per_void > int( self._params.peak_volume_cutoff * flex.max(grid_points_per_void.select(~outliers)))).iselection() sites = flood_fill.centres_of_mass_frac().select(isel) volumes = flood_fill.grid_points_per_void().select(isel) return sites, volumes
def index(self): # most of this is the same as dials.algorithms.indexing.indexer.indexer_base.index(), with some stills # specific modifications (don't re-index after choose best orientation matrix, but use the indexing from # choose best orientation matrix, also don't use macrocycles) of refinement after indexing. # 2017 update: do accept multiple lattices per shot experiments = ExperimentList() while True: self.d_min = self.params.refinement_protocol.d_min_start max_lattices = self.params.multiple_lattice_search.max_lattices if max_lattices is not None and len(experiments) >= max_lattices: break if len(experiments) > 0: cutoff_fraction = ( self.params.multiple_lattice_search.recycle_unindexed_reflections_cutoff ) d_spacings = 1 / self.reflections["rlp"].norms() d_min_indexed = flex.min(d_spacings.select(self.indexed_reflections)) min_reflections_for_indexing = cutoff_fraction * len( self.reflections.select(d_spacings > d_min_indexed) ) crystal_ids = self.reflections.select(d_spacings > d_min_indexed)["id"] if (crystal_ids == -1).count(True) < min_reflections_for_indexing: logger.info( "Finish searching for more lattices: %i unindexed reflections remaining." % (min_reflections_for_indexing) ) break n_lattices_previous_cycle = len(experiments) # index multiple lattices per shot if len(experiments) == 0: new = self.find_lattices() generate_experiment_identifiers(new) experiments.extend(new) if len(experiments) == 0: raise DialsIndexError("No suitable lattice could be found.") else: try: new = self.find_lattices() generate_experiment_identifiers(new) experiments.extend(new) except Exception as e: logger.info("Indexing remaining reflections failed") logger.debug( "Indexing remaining reflections failed, exception:\n" + str(e) ) # reset reflection lattice flags # the lattice a given reflection belongs to: a value of -1 indicates # that a reflection doesn't belong to any lattice so far self.reflections["id"] = flex.int(len(self.reflections), -1) self.index_reflections(experiments, self.reflections) if len(experiments) == n_lattices_previous_cycle: # no more lattices found break if ( not self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None ): self._apply_symmetry_post_indexing( experiments, self.reflections, n_lattices_previous_cycle ) # discard nearly overlapping lattices on the same shot if self._check_have_similar_crystal_models(experiments): break self.indexed_reflections = self.reflections["id"] > -1 if self.d_min is None: sel = self.reflections["id"] <= -1 else: sel = flex.bool(len(self.reflections), False) lengths = 1 / self.reflections["rlp"].norms() isel = (lengths >= self.d_min).iselection() sel.set_selected(isel, True) sel.set_selected(self.reflections["id"] > -1, False) self.unindexed_reflections = self.reflections.select(sel) reflections_for_refinement = self.reflections.select( self.indexed_reflections ) if len(self.params.stills.isoforms) > 0: logger.info("") logger.info("#" * 80) logger.info("Starting refinement") logger.info("#" * 80) logger.info("") isoform_experiments = ExperimentList() isoform_reflections = flex.reflection_table() # Note, changes to params after initial indexing. Cannot use tie to target when fixing the unit cell. self.all_params.refinement.reflections.outlier.algorithm = "null" self.all_params.refinement.parameterisation.crystal.fix = "cell" self.all_params.refinement.parameterisation.crystal.unit_cell.restraints.tie_to_target = ( [] ) for expt_id, experiment in enumerate(experiments): reflections = reflections_for_refinement.select( reflections_for_refinement["id"] == expt_id ) reflections["id"] = flex.int(len(reflections), 0) refiners = [] for isoform in self.params.stills.isoforms: iso_experiment = copy.deepcopy(experiment) crystal = iso_experiment.crystal if ( isoform.lookup_symbol != crystal.get_space_group().type().lookup_symbol() ): logger.info( "Crystal isoform lookup_symbol %s does not match isoform %s lookup_symbol %s" % ( crystal.get_space_group().type().lookup_symbol(), isoform.name, isoform.lookup_symbol, ) ) continue crystal.set_B(isoform.cell.fractionalization_matrix()) logger.info("Refining isoform %s" % isoform.name) refiners.append( e_refine( params=self.all_params, experiments=ExperimentList([iso_experiment]), reflections=reflections, graph_verbose=False, ) ) if len(refiners) == 0: raise DialsIndexError( "No isoforms had a lookup symbol that matched" ) positional_rmsds = [ math.sqrt(P.rmsds()[0] ** 2 + P.rmsds()[1] ** 2) for P in refiners ] logger.info( "Positional rmsds for all isoforms:" + str(positional_rmsds) ) minrmsd_mm = min(positional_rmsds) minindex = positional_rmsds.index(minrmsd_mm) logger.info( "The smallest rmsd is %5.1f um from isoform %s" % ( 1000.0 * minrmsd_mm, self.params.stills.isoforms[minindex].name, ) ) if self.params.stills.isoforms[minindex].rmsd_target_mm is not None: logger.info( "Asserting %f < %f" % ( minrmsd_mm, self.params.stills.isoforms[minindex].rmsd_target_mm, ) ) assert ( minrmsd_mm < self.params.stills.isoforms[minindex].rmsd_target_mm ) logger.info( "Acceptable rmsd for isoform %s." % (self.params.stills.isoforms[minindex].name) ) if len(self.params.stills.isoforms) == 2: logger.info( "Rmsd gain over the other isoform %5.1f um." % (1000.0 * abs(positional_rmsds[0] - positional_rmsds[1])) ) R = refiners[minindex] # Now one last check to see if direct beam is out of bounds if self.params.stills.isoforms[minindex].beam_restraint is not None: from scitbx import matrix refined_beam = matrix.col( R.get_experiments()[0] .detector[0] .get_beam_centre_lab(experiments[0].beam.get_s0())[0:2] ) known_beam = matrix.col( self.params.stills.isoforms[minindex].beam_restraint ) logger.info( "Asserting difference in refined beam center and expected beam center %f < %f" % ( (refined_beam - known_beam).length(), self.params.stills.isoforms[minindex].rmsd_target_mm, ) ) assert ( (refined_beam - known_beam).length() < self.params.stills.isoforms[minindex].rmsd_target_mm ) # future--circle of confusion could be given as a separate length in mm instead of reusing rmsd_target experiment = R.get_experiments()[0] experiment.crystal.identified_isoform = self.params.stills.isoforms[ minindex ].name isoform_experiments.append(experiment) reflections["id"] = flex.int(len(reflections), expt_id) isoform_reflections.extend(reflections) experiments = isoform_experiments reflections_for_refinement = isoform_reflections if self.params.refinement_protocol.mode == "repredict_only": from dials.algorithms.indexing.nave_parameters import NaveParameters from dials.algorithms.refinement.prediction.managed_predictors import ( ExperimentsPredictorFactory, ) refined_experiments, refined_reflections = ( experiments, reflections_for_refinement, ) ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, force_stills=True, spherical_relp=self.all_params.refinement.parameterisation.spherical_relp_model, ) ref_predictor(refined_reflections) refined_reflections["delpsical2"] = ( refined_reflections["delpsical.rad"] ** 2 ) for expt_id in range(len(refined_experiments)): refls = refined_reflections.select( refined_reflections["id"] == expt_id ) nv = NaveParameters( params=self.all_params, experiments=refined_experiments[expt_id : expt_id + 1], reflections=refls, refinery=None, graph_verbose=False, ) experiments[expt_id].crystal = nv() ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, force_stills=True, spherical_relp=self.all_params.refinement.parameterisation.spherical_relp_model, ) ref_predictor(refined_reflections) elif self.params.refinement_protocol.mode is None: refined_experiments, refined_reflections = ( experiments, reflections_for_refinement, ) else: try: refined_experiments, refined_reflections = self.refine( experiments, reflections_for_refinement ) except Exception as e: s = str(e) if len(experiments) == 1: raise DialsIndexRefineError(e.message) logger.info("Refinement failed:") logger.info(s) del experiments[-1] break self._unit_cell_volume_sanity_check(experiments, refined_experiments) self.refined_reflections = refined_reflections.select( refined_reflections["id"] > -1 ) for i, expt in enumerate(self.experiments): ref_sel = self.refined_reflections.select( self.refined_reflections["imageset_id"] == i ) ref_sel = ref_sel.select(ref_sel["id"] >= 0) for i_expt in set(ref_sel["id"]): refined_expt = refined_experiments[i_expt] expt.detector = refined_expt.detector expt.beam = refined_expt.beam expt.goniometer = refined_expt.goniometer expt.scan = refined_expt.scan refined_expt.imageset = expt.imageset if not ( self.all_params.refinement.parameterisation.beam.fix == "all" and self.all_params.refinement.parameterisation.detector.fix == "all" ): # Experimental geometry may have changed - re-map centroids to # reciprocal space self.reflections.map_centroids_to_reciprocal_space(self.experiments) # update for next cycle experiments = refined_experiments self.refined_experiments = refined_experiments if self.refined_experiments is None: raise DialsIndexRefineError("None of the experiments could refine.") # discard experiments with zero reflections after refinement id_set = set(self.refined_reflections["id"]) if len(id_set) < len(self.refined_experiments): filtered_refined_reflections = flex.reflection_table() for i in range(len(self.refined_experiments)): if i not in id_set: del self.refined_experiments[i] for old, new in zip(sorted(id_set), range(len(id_set))): subset = self.refined_reflections.select( self.refined_reflections["id"] == old ) subset["id"] = flex.int(len(subset), new) filtered_refined_reflections.extend(subset) self.refined_reflections = filtered_refined_reflections if len(self.refined_experiments) > 1: from dials.algorithms.indexing.compare_orientation_matrices import ( rotation_matrix_differences, ) logger.info( rotation_matrix_differences(self.refined_experiments.crystals()) ) logger.info("Final refined crystal models:") for i, crystal_model in enumerate(self.refined_experiments.crystals()): n_indexed = 0 for _ in experiments.where(crystal=crystal_model): n_indexed += (self.reflections["id"] == i).count(True) logger.info("model %i (%i reflections):" % (i + 1, n_indexed)) logger.info(crystal_model) if ( "xyzcal.mm" in self.refined_reflections ): # won't be there if refine_all_candidates = False and no isoforms self._xyzcal_mm_to_px(self.experiments, self.refined_reflections)
def choose_best_orientation_matrix(self, candidate_orientation_matrices): logger.info("*" * 80) logger.info("Selecting the best orientation matrix") logger.info("*" * 80) class CandidateInfo(libtbx.group_args): pass candidates = [] params = copy.deepcopy(self.all_params) for icm, cm in enumerate(candidate_orientation_matrices): if icm >= self.params.basis_vector_combinations.max_refine: break # Index reflections in P1 sel = self.reflections["id"] == -1 refl = self.reflections.select(sel) experiments = self.experiment_list_for_crystal(cm) self.index_reflections(experiments, refl) indexed = refl.select(refl["id"] >= 0) indexed = indexed.select(indexed.get_flags(indexed.flags.indexed)) # If target symmetry supplied, try to apply it. Then, apply the change of basis to the reflections # indexed in P1 to the target setting if ( self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None ): new_crystal, cb_op_to_primitive = self._symmetry_handler.apply_symmetry( cm ) if new_crystal is None: logger.info("Cannot convert to target symmetry, candidate %d", icm) continue new_crystal = new_crystal.change_basis( self._symmetry_handler.cb_op_primitive_inp ) cm = new_crystal experiments = self.experiment_list_for_crystal(cm) if not cb_op_to_primitive.is_identity_op(): indexed["miller_index"] = cb_op_to_primitive.apply( indexed["miller_index"] ) if self._symmetry_handler.cb_op_primitive_inp is not None: indexed[ "miller_index" ] = self._symmetry_handler.cb_op_primitive_inp.apply( indexed["miller_index"] ) if params.indexing.stills.refine_all_candidates: try: logger.info( "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d initial outlier identification", icm, ) acceptance_flags = self.identify_outliers( params, experiments, indexed ) # create a new "indexed" list with outliers thrown out: indexed = indexed.select(acceptance_flags) logger.info( "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d refinement before outlier rejection", icm, ) R = e_refine( params=params, experiments=experiments, reflections=indexed, graph_verbose=False, ) ref_experiments = R.get_experiments() # try to improve the outcome with a second round of outlier rejection post-initial refinement: acceptance_flags = self.identify_outliers( params, ref_experiments, indexed ) # insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection nv0 = NaveParameters( params=params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False, ) nv0() acceptance_flags_nv0 = nv0.nv_acceptance_flags indexed = indexed.select(acceptance_flags & acceptance_flags_nv0) logger.info( "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d after positional and delta-psi outlier rejection", icm, ) R = e_refine( params=params, experiments=ref_experiments, reflections=indexed, graph_verbose=False, ) ref_experiments = R.get_experiments() nv = NaveParameters( params=params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False, ) crystal_model = nv() assert ( len(crystal_model) == 1 ), "$$$ stills_indexer::choose_best_orientation_matrix, Only one crystal at this stage" crystal_model = crystal_model[0] # Drop candidates that after refinement can no longer be converted to the known target space group if ( not self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None ): ( new_crystal, cb_op_to_primitive, ) = self._symmetry_handler.apply_symmetry(crystal_model) if new_crystal is None: logger.info( "P1 refinement yielded model diverged from target, candidate %d", icm, ) continue rmsd, _ = calc_2D_rmsd_and_displacements( R.predict_for_reflection_table(indexed) ) except Exception as e: logger.info( "Couldn't refine candidate %d, %s: %s", icm, e.__class__.__name__, str(e), ) else: logger.info( "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d done", icm, ) candidates.append( CandidateInfo( crystal=crystal_model, green_curve_area=nv.green_curve_area, ewald_proximal_volume=nv.ewald_proximal_volume(), n_indexed=len(indexed), rmsd=rmsd, indexed=indexed, experiments=ref_experiments, ) ) else: from dials.algorithms.refinement.prediction.managed_predictors import ( ExperimentsPredictorFactory, ) ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, force_stills=True, spherical_relp=params.refinement.parameterisation.spherical_relp_model, ) rmsd, _ = calc_2D_rmsd_and_displacements(ref_predictor(indexed)) candidates.append( CandidateInfo( crystal=cm, n_indexed=len(indexed), rmsd=rmsd, indexed=indexed, experiments=experiments, ) ) if len(candidates) == 0: raise DialsIndexError("No suitable indexing solution found") logger.info("**** ALL CANDIDATES:") for i, XX in enumerate(candidates): logger.info("\n****Candidate %d %s", i, XX) cc = XX.crystal if hasattr(cc, "get_half_mosaicity_deg"): logger.info( " half mosaicity %5.2f deg.", (cc.get_half_mosaicity_deg()) ) logger.info(" domain size %.0f Ang.", (cc.get_domain_size_ang())) logger.info("\n**** BEST CANDIDATE:") results = flex.double([c.rmsd for c in candidates]) best = candidates[flex.min_index(results)] logger.info(best) if params.indexing.stills.refine_all_candidates: if best.rmsd > params.indexing.stills.rmsd_min_px: raise DialsIndexError("RMSD too high, %f" % best.rmsd) if len(candidates) > 1: for i in range(len(candidates)): if i == flex.min_index(results): continue if best.ewald_proximal_volume > candidates[i].ewald_proximal_volume: logger.info( "Couldn't figure out which candidate is best; picked the one with the best RMSD." ) best.indexed["entering"] = flex.bool(best.n_indexed, False) return best.crystal, best.n_indexed
def index(self): experiments = ExperimentList() had_refinement_error = False have_similar_crystal_models = False while True: if had_refinement_error or have_similar_crystal_models: break max_lattices = self.params.multiple_lattice_search.max_lattices if max_lattices is not None and len(experiments) >= max_lattices: break if len(experiments) > 0: cutoff_fraction = (self.params.multiple_lattice_search. recycle_unindexed_reflections_cutoff) d_spacings = 1 / self.reflections["rlp"].norms() d_min_indexed = flex.min( d_spacings.select(self.indexed_reflections)) min_reflections_for_indexing = cutoff_fraction * len( self.reflections.select(d_spacings > d_min_indexed)) crystal_ids = self.reflections.select( d_spacings > d_min_indexed)["id"] if (crystal_ids == -1).count(True) < min_reflections_for_indexing: logger.info( "Finish searching for more lattices: %i unindexed reflections remaining." % ((crystal_ids == -1).count(True))) break n_lattices_previous_cycle = len(experiments) if self.d_min is None: self.d_min = self.params.refinement_protocol.d_min_start if len(experiments) == 0: new_expts = self.find_lattices() generate_experiment_identifiers(new_expts) experiments.extend(new_expts) else: try: new = self.find_lattices() generate_experiment_identifiers(new) experiments.extend(new) except DialsIndexError: logger.info("Indexing remaining reflections failed") if self.params.refinement_protocol.d_min_step is libtbx.Auto: n_cycles = self.params.refinement_protocol.n_macro_cycles if self.d_min is None or n_cycles == 1: self.params.refinement_protocol.d_min_step = 0 else: d_spacings = 1 / self.reflections["rlp"].norms() d_min_all = flex.min(d_spacings) self.params.refinement_protocol.d_min_step = ( self.d_min - d_min_all) / (n_cycles - 1) logger.info("Using d_min_step %.1f" % self.params.refinement_protocol.d_min_step) if len(experiments) == 0: raise DialsIndexError("No suitable lattice could be found.") elif len(experiments) == n_lattices_previous_cycle: # no more lattices found break for i_cycle in range( self.params.refinement_protocol.n_macro_cycles): if (i_cycle > 0 and self.d_min is not None and self.params.refinement_protocol.d_min_step > 0): d_min = self.d_min - self.params.refinement_protocol.d_min_step d_min = max(d_min, 0) if self.params.refinement_protocol.d_min_final is not None: d_min = max( d_min, self.params.refinement_protocol.d_min_final) if d_min >= 0: self.d_min = d_min logger.info("Increasing resolution to %.2f Angstrom" % d_min) # reset reflection lattice flags # the lattice a given reflection belongs to: a value of -1 indicates # that a reflection doesn't belong to any lattice so far self.reflections["id"] = flex.int(len(self.reflections), -1) self.index_reflections(experiments, self.reflections) if i_cycle == 0 and self.params.known_symmetry.space_group is not None: self._apply_symmetry_post_indexing( experiments, self.reflections, n_lattices_previous_cycle) logger.info("\nIndexed crystal models:") self.show_experiments(experiments, self.reflections, d_min=self.d_min) if self._check_have_similar_crystal_models(experiments): have_similar_crystal_models = True break logger.info("") logger.info("#" * 80) logger.info("Starting refinement (macro-cycle %i)" % (i_cycle + 1)) logger.info("#" * 80) logger.info("") self.indexed_reflections = self.reflections["id"] > -1 sel = flex.bool(len(self.reflections), False) lengths = 1 / self.reflections["rlp"].norms() if self.d_min is not None: isel = (lengths <= self.d_min).iselection() sel.set_selected(isel, True) sel.set_selected(self.reflections["id"] == -1, True) self.reflections.unset_flags(sel, self.reflections.flags.indexed) self.unindexed_reflections = self.reflections.select(sel) reflections_for_refinement = self.reflections.select( self.indexed_reflections) if self.params.refinement_protocol.mode == "repredict_only": refined_experiments, refined_reflections = ( experiments, reflections_for_refinement, ) from dials.algorithms.refinement.prediction.managed_predictors import ( ExperimentsPredictorFactory, ) ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, spherical_relp=self.all_params.refinement. parameterisation.spherical_relp_model, ) ref_predictor(refined_reflections) else: try: refined_experiments, refined_reflections = self.refine( experiments, reflections_for_refinement) except (DialsRefineConfigError, DialsRefineRuntimeError) as e: if len(experiments) == 1: raise DialsIndexRefineError(str(e)) had_refinement_error = True logger.info("Refinement failed:") logger.info(e) del experiments[-1] # remove experiment id from the reflections associated # with this deleted experiment - indexed flag removed # below last = len(experiments) sel = refined_reflections["id"] == last logger.info("Removing %d reflections with id %d" % (sel.count(True), last)) refined_reflections["id"].set_selected(sel, -1) break self._unit_cell_volume_sanity_check(experiments, refined_experiments) self.refined_reflections = refined_reflections self.refined_reflections.unset_flags( self.refined_reflections["id"] < 0, self.refined_reflections.flags.indexed, ) for i, expt in enumerate(self.experiments): ref_sel = self.refined_reflections.select( self.refined_reflections["imageset_id"] == i) ref_sel = ref_sel.select(ref_sel["id"] >= 0) for i_expt in set(ref_sel["id"]): refined_expt = refined_experiments[i_expt] expt.detector = refined_expt.detector expt.beam = refined_expt.beam expt.goniometer = refined_expt.goniometer expt.scan = refined_expt.scan refined_expt.imageset = expt.imageset if not (self.all_params.refinement.parameterisation.beam.fix == "all" and self.all_params.refinement. parameterisation.detector.fix == "all"): # Experimental geometry may have changed - re-map centroids to # reciprocal space self.reflections.map_centroids_to_reciprocal_space( self.experiments) # update for next cycle experiments = refined_experiments self.refined_experiments = refined_experiments logger.info("\nRefined crystal models:") self.show_experiments(self.refined_experiments, self.reflections, d_min=self.d_min) if (i_cycle >= 2 and self.d_min == self.params.refinement_protocol.d_min_final): logger.info( "Target d_min_final reached: finished with refinement") break if self.refined_experiments is None: raise DialsIndexRefineError( "None of the experiments could refine.") if len(self.refined_experiments) > 1: from dials.algorithms.indexing.compare_orientation_matrices import ( rotation_matrix_differences, ) logger.info( rotation_matrix_differences( self.refined_experiments.crystals())) self._xyzcal_mm_to_px(self.refined_experiments, self.refined_reflections)
def __call__(self): """Determine optimal mosaicity and domain size model (monochromatic)""" if self.refinery is None: RR = self.reflections else: RR = self.refinery.predict_for_reflection_table(self.reflections) all_crystals = [] self.nv_acceptance_flags = flex.bool(len(self.reflections["id"])) from dxtbx.model import MosaicCrystalSauter2014 for iid, experiment in enumerate(self.experiments): excursion_rad = RR["delpsical.rad"].select(RR["id"] == iid) delta_psi_deg = excursion_rad * 180.0 / math.pi logger.info("") logger.info("%s %s", flex.max(delta_psi_deg), flex.min(delta_psi_deg)) mean_excursion = flex.mean(delta_psi_deg) logger.info( "The mean excursion is %7.3f degrees, r.m.s.d %7.3f", mean_excursion, math.sqrt(flex.mean(RR["delpsical2"].select(RR["id"] == iid))), ) crystal = MosaicCrystalSauter2014(self.experiments[iid].crystal) self.experiments[iid].crystal = crystal beam = self.experiments[iid].beam miller_indices = self.reflections["miller_index"].select( self.reflections["id"] == iid) # FIXME XXX revise this formula so as to use a different wavelength potentially for each reflection two_thetas = crystal.get_unit_cell().two_theta( miller_indices, beam.get_wavelength(), deg=True) dspacings = crystal.get_unit_cell().d(miller_indices) # First -- try to get a reasonable envelope for the observed excursions. # minimum of three regions; maximum of 50 measurements in each bin logger.info("fitting parameters on %d spots", len(excursion_rad)) n_bins = min(max(3, len(excursion_rad) // 25), 50) bin_sz = len(excursion_rad) // n_bins logger.info("nbins %s bin_sz %s", n_bins, bin_sz) order = flex.sort_permutation(two_thetas) two_thetas_env = flex.double() dspacings_env = flex.double() excursion_rads_env = flex.double() for x in range(0, n_bins): subset = order[x * bin_sz:(x + 1) * bin_sz] two_thetas_env.append(flex.mean(two_thetas.select(subset))) dspacings_env.append(flex.mean(dspacings.select(subset))) excursion_rads_env.append( flex.max(flex.abs(excursion_rad.select(subset)))) # Second -- parameter fit # solve the normal equations sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env) sum_inv_u = flex.sum(dspacings_env) sum_te_u = flex.sum(dspacings_env * excursion_rads_env) sum_te = flex.sum(excursion_rads_env) Normal_Mat = sqr( (sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env))) Vector = col((sum_te_u, sum_te)) solution = Normal_Mat.inverse() * Vector s_ang = 1.0 / (2 * solution[0]) logger.info("Best LSQ fit Scheerer domain size is %9.2f ang", s_ang) k_degrees = solution[1] * 180.0 / math.pi logger.info( "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f", 2 * k_degrees, k_degrees, ) from xfel.mono_simulation.max_like import minimizer # coerce the estimates to be positive for max-likelihood lower_limit_domain_size = ( math.pow(crystal.get_unit_cell().volume(), 1.0 / 3.0) * 3 ) # params.refinement.domain_size_lower_limit d_estimate = max(s_ang, lower_limit_domain_size) M = minimizer( d_i=dspacings, psi_i=excursion_rad, eta_rad=abs(2.0 * solution[1]), Deff=d_estimate, ) logger.info( "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots", M.x[1] * 180.0 / math.pi, 2.0 / M.x[0], len(two_thetas), ) tan_phi_rad_ML = dspacings / (2.0 / M.x[0]) tan_phi_deg_ML = tan_phi_rad_ML * 180.0 / math.pi tan_outer_deg_ML = tan_phi_deg_ML + 0.5 * M.x[1] * 180.0 / math.pi # Only set the flags for those reflections that were indexed for this lattice self.nv_acceptance_flags.set_selected( self.reflections["id"] == iid, flex.abs(delta_psi_deg) < tan_outer_deg_ML, ) if ( self.graph_verbose ): # params.refinement.mosaic.enable_AD14F7B: # Excursion vs resolution fit AD1TF7B_MAX2T = 30.0 AD1TF7B_MAXDP = 1.0 from matplotlib import pyplot as plt plt.plot(two_thetas, delta_psi_deg, "bo") minplot = flex.min(two_thetas) plt.plot([0, minplot], [mean_excursion, mean_excursion], "k-") LR = flex.linear_regression(two_thetas, delta_psi_deg) model_y = LR.slope() * two_thetas + LR.y_intercept() plt.plot(two_thetas, model_y, "k-") plt.title( "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" % (M.x[1] * 180.0 / math.pi, 2.0 / M.x[0], len(two_thetas))) plt.plot(two_thetas, tan_phi_deg_ML, "r.") plt.plot(two_thetas, -tan_phi_deg_ML, "r.") plt.plot(two_thetas, tan_outer_deg_ML, "g.") plt.plot(two_thetas, -tan_outer_deg_ML, "g.") plt.xlim([0, AD1TF7B_MAX2T]) plt.ylim([-AD1TF7B_MAXDP, AD1TF7B_MAXDP]) plt.show() plt.close() from xfel.mono_simulation.util import green_curve_area self.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML) logger.info("The green curve area is %s", self.green_curve_area) crystal.set_half_mosaicity_deg(M.x[1] * 180.0 / (2.0 * math.pi)) crystal.set_domain_size_ang(2.0 / M.x[0]) self._ML_full_mosaicity_rad = M.x[1] self._ML_domain_size_ang = 2.0 / M.x[0] # params.refinement.mosaic.model_expansion_factor """The expansion factor should be initially set to 1, then expanded so that the # reflections matched becomes as close as possible to # of observed reflections input, in the last integration call. Determine this by inspecting the output log file interactively. Do not exceed the bare minimum threshold needed. The intention is to find an optimal value, global for a given dataset.""" model_expansion_factor = 1.4 crystal.set_half_mosaicity_deg(crystal.get_half_mosaicity_deg() * model_expansion_factor) crystal.set_domain_size_ang(crystal.get_domain_size_ang() / model_expansion_factor) if (self.ewald_proximal_volume(iid) > self.params.indexing.stills.ewald_proximal_volume_max): raise DialsIndexError("Ewald proximity volume too high, %f" % self.ewald_proximal_volume(iid)) all_crystals.append(crystal) return all_crystals
def __call__(self, reflections, experiments, d_min=None): from scitbx import matrix from libtbx.math_utils import nearest_integer as nint reciprocal_lattice_points = reflections["rlp"] if "miller_index" not in reflections: reflections["miller_index"] = flex.miller_index(len(reflections)) if d_min is not None: d_spacings = 1 / reciprocal_lattice_points.norms() inside_resolution_limit = d_spacings > d_min else: inside_resolution_limit = flex.bool( reciprocal_lattice_points.size(), True) sel = inside_resolution_limit & (reflections["id"] == -1) isel = sel.iselection() rlps = reciprocal_lattice_points.select(isel) refs = reflections.select(isel) phi = refs["xyzobs.mm.value"].parts()[2] if len(rlps) <= self._nearest_neighbours: raise DialsIndexError( "index_assignment.local.nearest_neighbour must be smaller than the number of accepted reflections (%d)" % len(rlps)) UB_matrices = flex.mat3_double( [cm.get_A() for cm in experiments.crystals()]) result = ext.AssignIndicesLocal( rlps, phi, UB_matrices, epsilon=self._epsilon, delta=self._delta, l_min=self._l_min, nearest_neighbours=self._nearest_neighbours, ) miller_indices = result.miller_indices() crystal_ids = result.crystal_ids() hkl = miller_indices.as_vec3_double().iround() assert miller_indices.select(crystal_ids < 0).all_eq((0, 0, 0)) for i_cryst in set(crystal_ids): if i_cryst < 0: continue A = matrix.sqr(experiments[i_cryst].crystal.get_A()) A_inv = A.inverse() cryst_sel = crystal_ids == i_cryst rlp_sel = rlps.select(cryst_sel) hkl_sel = hkl.select(cryst_sel).as_vec3_double() d_sel = 1 / rlp_sel.norms() d_perm = flex.sort_permutation(d_sel, reverse=True) hf_0 = A_inv * rlp_sel[d_perm[0]] h_0 = matrix.col([nint(j) for j in hf_0.elems]) offset = h_0 - matrix.col(hkl_sel[d_perm[0]]) # print "offset:", offset.elems h = hkl_sel + flex.vec3_double(hkl_sel.size(), offset.elems) refs["miller_index"].set_selected( cryst_sel, flex.miller_index(list(h.iround()))) refs["id"].set_selected(cryst_sel, i_cryst) crystal_ids.set_selected(crystal_ids < 0, -1) refs["id"] = crystal_ids refs["miller_index"].set_selected(crystal_ids < 0, (0, 0, 0)) reflections["miller_index"].set_selected(isel, refs["miller_index"]) reflections["id"].set_selected(isel, refs["id"]) reflections.set_flags(reflections["miller_index"] != (0, 0, 0), reflections.flags.indexed)