def setup_indexing(self):
reflections_input = self.reflections
self.reflections = flex.reflection_table()
for i, expt in enumerate(self.experiments):
if "imageset_id" not in reflections_input:
reflections_input["imageset_id"] = reflections_input["id"]
refl = reflections_input.select(
reflections_input["imageset_id"] == i)
refl.centroid_px_to_mm(expt.detector, expt.scan)
self.reflections.extend(refl)
if len(self.reflections) == 0:
raise DialsIndexError("No reflections left to index!")
self.reflections = self._map_centroids_to_reciprocal_space(
self.experiments, self.reflections)
self.reflections.calculate_entering_flags(self.experiments)
self.find_max_cell()
if self.params.sigma_phi_deg is not None:
var_x, var_y, _ = self.reflections["xyzobs.mm.variance"].parts()
var_phi_rad = flex.double(
var_x.size(), (math.pi / 180 * self.params.sigma_phi_deg)**2)
self.reflections["xyzobs.mm.variance"] = flex.vec3_double(
var_x, var_y, var_phi_rad)
if self.params.debug:
self._debug_write_reciprocal_lattice_points_as_pdb()
self.reflections["id"] = flex.int(len(self.reflections), -1)
def __init__(self,
target_symmetry_primitive,
max_lattices,
params=None,
*args,
**kwargs):
"""Construct a LowResSpotMatch object.
Args:
target_symmetry_primitive (cctbx.crystal.symmetry): The target
crystal symmetry and unit cell
max_lattices (int): The maximum number of lattice models to find
"""
super().__init__(params=params, *args, **kwargs)
self._target_symmetry_primitive = target_symmetry_primitive
self._max_lattices = max_lattices
if target_symmetry_primitive is None:
raise DialsIndexError(
"Target unit cell and space group must be provided for low_res_spot_match"
)
# Set reciprocal space orthogonalisation matrix
uc = self._target_symmetry_primitive.unit_cell()
self.Bmat = matrix.sqr(uc.fractionalization_matrix()).transpose()
def _unit_cell_volume_sanity_check(self, original_experiments, refined_experiments):
# sanity check for unrealistic unit cell volume increase during refinement
# usually this indicates too many parameters are being refined given the
# number of observations provided.
if not self.params.refinement_protocol.disable_unit_cell_volume_sanity_check:
for orig_expt, refined_expt in zip(
original_experiments, refined_experiments
):
uc1 = orig_expt.crystal.get_unit_cell()
uc2 = refined_expt.crystal.get_unit_cell()
volume_change = abs(uc1.volume() - uc2.volume()) / uc1.volume()
cutoff = 0.5
if volume_change > cutoff:
msg = (
"\n".join(
(
"Unrealistic unit cell volume increase during refinement of %.1f%%.",
"Please try refining fewer parameters, either by enforcing symmetry",
"constraints (space_group=) and/or disabling experimental geometry",
"refinement (detector.fix=all and beam.fix=all). To disable this",
"sanity check set disable_unit_cell_volume_sanity_check=True.",
)
)
% (100 * volume_change)
)
raise DialsIndexError(msg)
def setup_indexing(self):
if len(self.reflections) == 0:
raise DialsIndexError("No reflections left to index!")
if "imageset_id" not in self.reflections:
self.reflections["imageset_id"] = self.reflections["id"]
self.reflections.centroid_px_to_mm(self.experiments)
self.reflections.map_centroids_to_reciprocal_space(self.experiments)
self.reflections.calculate_entering_flags(self.experiments)
self.find_max_cell()
if self.params.sigma_phi_deg is not None:
var_x, var_y, _ = self.reflections["xyzobs.mm.variance"].parts()
var_phi_rad = flex.double(
var_x.size(), (math.pi / 180 * self.params.sigma_phi_deg) ** 2
)
self.reflections["xyzobs.mm.variance"] = flex.vec3_double(
var_x, var_y, var_phi_rad
)
if self.params.debug:
self._debug_write_reciprocal_lattice_points_as_pdb()
self.reflections["id"] = flex.int(len(self.reflections), -1)
def _setup_target_unit_cell_and_space_group(self, target_unit_cell,
target_space_group):
target_bravais_t = bravais_lattice(
group=target_space_group.info().reference_setting().group())
best_subgroup = None
best_angular_difference = 1e8
space_groups = [target_space_group]
if target_space_group.conventional_centring_type_symbol() != "P":
space_groups.append(sgtbx.space_group())
for target in space_groups:
cs = crystal.symmetry(
unit_cell=target_unit_cell,
space_group=target,
assert_is_compatible_unit_cell=False,
)
target_best_cell = cs.best_cell().unit_cell()
subgroups = lattice_symmetry.metric_subgroups(cs, max_delta=0.1)
for subgroup in subgroups.result_groups:
bravais_t = bravais_lattice(
group=subgroup["ref_subsym"].space_group())
if bravais_t == target_bravais_t:
# allow for the cell to be given as best cell, reference setting
# primitive settings, or minimum cell
best_subsym = subgroup["best_subsym"]
ref_subsym = best_subsym.as_reference_setting()
if not (best_subsym.unit_cell().is_similar_to(
target_unit_cell) or ref_subsym.unit_cell(
).is_similar_to(target_unit_cell)
or ref_subsym.primitive_setting().unit_cell(
).is_similar_to(target_unit_cell)
or best_subsym.primitive_setting().unit_cell(
).is_similar_to(target_unit_cell)
or best_subsym.minimum_cell().unit_cell(
).is_similar_to(target_unit_cell.minimum_cell())
or best_subsym.unit_cell().is_similar_to(
target_best_cell)):
continue
if subgroup[
"max_angular_difference"] < best_angular_difference:
best_subgroup = subgroup
best_angular_difference = subgroup[
"max_angular_difference"]
if best_subgroup is None:
raise DialsIndexError("Unit cell incompatible with space group")
cb_op_inp_best = best_subgroup["cb_op_inp_best"]
best_subsym = best_subgroup["best_subsym"]
cb_op_best_ref = best_subsym.change_of_basis_op_to_reference_setting()
self.cb_op_inp_ref = cb_op_best_ref * cb_op_inp_best
self.target_symmetry_reference_setting = crystal.symmetry(
unit_cell=target_unit_cell.change_basis(self.cb_op_inp_ref),
space_group=target_space_group.info().as_reference_setting().group(
),
)
def find_max_cell(
reflections,
max_cell_multiplier=1.3,
step_size=45,
nearest_neighbor_percentile=None,
histogram_binning="linear",
nn_per_bin=5,
max_height_fraction=0.25,
filter_ice=True,
filter_overlaps=True,
overlaps_border=0,
):
logger.debug("Finding suitable max_cell based on %i reflections" %
len(reflections))
# Exclude potential ice-ring spots from nearest neighbour analysis if needed
if filter_ice:
ice_sel = ice_rings_selection(reflections)
reflections = reflections.select(~ice_sel)
logger.debug("Rejecting %i reflections at ice ring resolution" %
ice_sel.count(True))
# need bounding box in reflections to find overlaps; this is not there if
# spots are from XDS (for example)
if filter_overlaps and "bbox" in reflections:
overlap_sel = flex.bool(len(reflections), False)
overlaps = reflections.find_overlaps(border=overlaps_border)
for item in overlaps.edges():
i0 = overlaps.source(item)
i1 = overlaps.target(item)
overlap_sel[i0] = True
overlap_sel[i1] = True
logger.debug("Rejecting %i overlapping bounding boxes" %
overlap_sel.count(True))
reflections = reflections.select(~overlap_sel)
logger.debug("%i reflections remain for max_cell identification" %
len(reflections))
if not len(reflections):
raise DialsIndexError(
"Too few spots remaining for nearest neighbour analysis (%d)" %
len(reflections))
NN = NeighborAnalysis(
reflections,
step_size=step_size,
max_height_fraction=max_height_fraction,
tolerance=max_cell_multiplier,
percentile=nearest_neighbor_percentile,
histogram_binning=histogram_binning,
nn_per_bin=nn_per_bin,
)
return NN
def __init__(self, max_cell, target_unit_cell, params=None, *args, **kwargs):
"""Construct a real_space_grid_search object.
Args:
max_cell (float): An estimate of the maximum cell dimension of the primitive
cell.
target_unit_cell (cctbx.uctbx.unit_cell): The target unit cell.
"""
super(RealSpaceGridSearch, self).__init__(
max_cell, params=params, *args, **kwargs
)
if target_unit_cell is None:
raise DialsIndexError(
"Target unit cell must be provided for real_space_grid_search"
)
self._target_unit_cell = target_unit_cell
def _find_peaks(self, grid_real, d_min):
grid_real_binary = grid_real.deep_copy()
rmsd = math.sqrt(
flex.mean(
flex.pow2(grid_real_binary.as_1d() -
flex.mean(grid_real_binary.as_1d()))))
grid_real_binary.set_selected(
grid_real_binary < (self._params.rmsd_cutoff) * rmsd, 0)
grid_real_binary.as_1d().set_selected(grid_real_binary.as_1d() > 0, 1)
grid_real_binary = grid_real_binary.iround()
from cctbx import masks
# real space FFT grid dimensions
cell_lengths = [self._n_points * d_min / 2 for i in range(3)]
self._fft_cell = uctbx.unit_cell(cell_lengths + [90] * 3)
flood_fill = masks.flood_fill(grid_real_binary, self._fft_cell)
if flood_fill.n_voids() < 4:
# Require at least peak at origin and one peak for each basis vector
raise DialsIndexError(
"Indexing failed: fft3d peak search failed to find sufficient number of peaks."
)
# the peak at the origin might have a significantly larger volume than the
# rest so exclude any anomalously large peaks from determining minimum volume
from scitbx.math import five_number_summary
outliers = flex.bool(flood_fill.n_voids(), False)
grid_points_per_void = flood_fill.grid_points_per_void()
min_x, q1_x, med_x, q3_x, max_x = five_number_summary(
grid_points_per_void)
iqr_multiplier = 5
iqr_x = q3_x - q1_x
cut_x = iqr_multiplier * iqr_x
outliers.set_selected(
grid_points_per_void.as_double() > (q3_x + cut_x), True)
# print q3_x + cut_x, outliers.count(True)
isel = (grid_points_per_void > int(
self._params.peak_volume_cutoff *
flex.max(grid_points_per_void.select(~outliers)))).iselection()
sites = flood_fill.centres_of_mass_frac().select(isel)
volumes = flood_fill.grid_points_per_void().select(isel)
return sites, volumes
def index(self):
# most of this is the same as dials.algorithms.indexing.indexer.indexer_base.index(), with some stills
# specific modifications (don't re-index after choose best orientation matrix, but use the indexing from
# choose best orientation matrix, also don't use macrocycles) of refinement after indexing.
# 2017 update: do accept multiple lattices per shot
experiments = ExperimentList()
while True:
self.d_min = self.params.refinement_protocol.d_min_start
max_lattices = self.params.multiple_lattice_search.max_lattices
if max_lattices is not None and len(experiments) >= max_lattices:
break
if len(experiments) > 0:
cutoff_fraction = (
self.params.multiple_lattice_search.recycle_unindexed_reflections_cutoff
)
d_spacings = 1 / self.reflections["rlp"].norms()
d_min_indexed = flex.min(d_spacings.select(self.indexed_reflections))
min_reflections_for_indexing = cutoff_fraction * len(
self.reflections.select(d_spacings > d_min_indexed)
)
crystal_ids = self.reflections.select(d_spacings > d_min_indexed)["id"]
if (crystal_ids == -1).count(True) < min_reflections_for_indexing:
logger.info(
"Finish searching for more lattices: %i unindexed reflections remaining."
% (min_reflections_for_indexing)
)
break
n_lattices_previous_cycle = len(experiments)
# index multiple lattices per shot
if len(experiments) == 0:
new = self.find_lattices()
generate_experiment_identifiers(new)
experiments.extend(new)
if len(experiments) == 0:
raise DialsIndexError("No suitable lattice could be found.")
else:
try:
new = self.find_lattices()
generate_experiment_identifiers(new)
experiments.extend(new)
except Exception as e:
logger.info("Indexing remaining reflections failed")
logger.debug(
"Indexing remaining reflections failed, exception:\n" + str(e)
)
# reset reflection lattice flags
# the lattice a given reflection belongs to: a value of -1 indicates
# that a reflection doesn't belong to any lattice so far
self.reflections["id"] = flex.int(len(self.reflections), -1)
self.index_reflections(experiments, self.reflections)
if len(experiments) == n_lattices_previous_cycle:
# no more lattices found
break
if (
not self.params.stills.refine_candidates_with_known_symmetry
and self.params.known_symmetry.space_group is not None
):
self._apply_symmetry_post_indexing(
experiments, self.reflections, n_lattices_previous_cycle
)
# discard nearly overlapping lattices on the same shot
if self._check_have_similar_crystal_models(experiments):
break
self.indexed_reflections = self.reflections["id"] > -1
if self.d_min is None:
sel = self.reflections["id"] <= -1
else:
sel = flex.bool(len(self.reflections), False)
lengths = 1 / self.reflections["rlp"].norms()
isel = (lengths >= self.d_min).iselection()
sel.set_selected(isel, True)
sel.set_selected(self.reflections["id"] > -1, False)
self.unindexed_reflections = self.reflections.select(sel)
reflections_for_refinement = self.reflections.select(
self.indexed_reflections
)
if len(self.params.stills.isoforms) > 0:
logger.info("")
logger.info("#" * 80)
logger.info("Starting refinement")
logger.info("#" * 80)
logger.info("")
isoform_experiments = ExperimentList()
isoform_reflections = flex.reflection_table()
# Note, changes to params after initial indexing. Cannot use tie to target when fixing the unit cell.
self.all_params.refinement.reflections.outlier.algorithm = "null"
self.all_params.refinement.parameterisation.crystal.fix = "cell"
self.all_params.refinement.parameterisation.crystal.unit_cell.restraints.tie_to_target = (
[]
)
for expt_id, experiment in enumerate(experiments):
reflections = reflections_for_refinement.select(
reflections_for_refinement["id"] == expt_id
)
reflections["id"] = flex.int(len(reflections), 0)
refiners = []
for isoform in self.params.stills.isoforms:
iso_experiment = copy.deepcopy(experiment)
crystal = iso_experiment.crystal
if (
isoform.lookup_symbol
!= crystal.get_space_group().type().lookup_symbol()
):
logger.info(
"Crystal isoform lookup_symbol %s does not match isoform %s lookup_symbol %s"
% (
crystal.get_space_group().type().lookup_symbol(),
isoform.name,
isoform.lookup_symbol,
)
)
continue
crystal.set_B(isoform.cell.fractionalization_matrix())
logger.info("Refining isoform %s" % isoform.name)
refiners.append(
e_refine(
params=self.all_params,
experiments=ExperimentList([iso_experiment]),
reflections=reflections,
graph_verbose=False,
)
)
if len(refiners) == 0:
raise DialsIndexError(
"No isoforms had a lookup symbol that matched"
)
positional_rmsds = [
math.sqrt(P.rmsds()[0] ** 2 + P.rmsds()[1] ** 2)
for P in refiners
]
logger.info(
"Positional rmsds for all isoforms:" + str(positional_rmsds)
)
minrmsd_mm = min(positional_rmsds)
minindex = positional_rmsds.index(minrmsd_mm)
logger.info(
"The smallest rmsd is %5.1f um from isoform %s"
% (
1000.0 * minrmsd_mm,
self.params.stills.isoforms[minindex].name,
)
)
if self.params.stills.isoforms[minindex].rmsd_target_mm is not None:
logger.info(
"Asserting %f < %f"
% (
minrmsd_mm,
self.params.stills.isoforms[minindex].rmsd_target_mm,
)
)
assert (
minrmsd_mm
< self.params.stills.isoforms[minindex].rmsd_target_mm
)
logger.info(
"Acceptable rmsd for isoform %s."
% (self.params.stills.isoforms[minindex].name)
)
if len(self.params.stills.isoforms) == 2:
logger.info(
"Rmsd gain over the other isoform %5.1f um."
% (1000.0 * abs(positional_rmsds[0] - positional_rmsds[1]))
)
R = refiners[minindex]
# Now one last check to see if direct beam is out of bounds
if self.params.stills.isoforms[minindex].beam_restraint is not None:
from scitbx import matrix
refined_beam = matrix.col(
R.get_experiments()[0]
.detector[0]
.get_beam_centre_lab(experiments[0].beam.get_s0())[0:2]
)
known_beam = matrix.col(
self.params.stills.isoforms[minindex].beam_restraint
)
logger.info(
"Asserting difference in refined beam center and expected beam center %f < %f"
% (
(refined_beam - known_beam).length(),
self.params.stills.isoforms[minindex].rmsd_target_mm,
)
)
assert (
(refined_beam - known_beam).length()
< self.params.stills.isoforms[minindex].rmsd_target_mm
)
# future--circle of confusion could be given as a separate length in mm instead of reusing rmsd_target
experiment = R.get_experiments()[0]
experiment.crystal.identified_isoform = self.params.stills.isoforms[
minindex
].name
isoform_experiments.append(experiment)
reflections["id"] = flex.int(len(reflections), expt_id)
isoform_reflections.extend(reflections)
experiments = isoform_experiments
reflections_for_refinement = isoform_reflections
if self.params.refinement_protocol.mode == "repredict_only":
from dials.algorithms.indexing.nave_parameters import NaveParameters
from dials.algorithms.refinement.prediction.managed_predictors import (
ExperimentsPredictorFactory,
)
refined_experiments, refined_reflections = (
experiments,
reflections_for_refinement,
)
ref_predictor = ExperimentsPredictorFactory.from_experiments(
experiments,
force_stills=True,
spherical_relp=self.all_params.refinement.parameterisation.spherical_relp_model,
)
ref_predictor(refined_reflections)
refined_reflections["delpsical2"] = (
refined_reflections["delpsical.rad"] ** 2
)
for expt_id in range(len(refined_experiments)):
refls = refined_reflections.select(
refined_reflections["id"] == expt_id
)
nv = NaveParameters(
params=self.all_params,
experiments=refined_experiments[expt_id : expt_id + 1],
reflections=refls,
refinery=None,
graph_verbose=False,
)
experiments[expt_id].crystal = nv()
ref_predictor = ExperimentsPredictorFactory.from_experiments(
experiments,
force_stills=True,
spherical_relp=self.all_params.refinement.parameterisation.spherical_relp_model,
)
ref_predictor(refined_reflections)
elif self.params.refinement_protocol.mode is None:
refined_experiments, refined_reflections = (
experiments,
reflections_for_refinement,
)
else:
try:
refined_experiments, refined_reflections = self.refine(
experiments, reflections_for_refinement
)
except Exception as e:
s = str(e)
if len(experiments) == 1:
raise DialsIndexRefineError(e.message)
logger.info("Refinement failed:")
logger.info(s)
del experiments[-1]
break
self._unit_cell_volume_sanity_check(experiments, refined_experiments)
self.refined_reflections = refined_reflections.select(
refined_reflections["id"] > -1
)
for i, expt in enumerate(self.experiments):
ref_sel = self.refined_reflections.select(
self.refined_reflections["imageset_id"] == i
)
ref_sel = ref_sel.select(ref_sel["id"] >= 0)
for i_expt in set(ref_sel["id"]):
refined_expt = refined_experiments[i_expt]
expt.detector = refined_expt.detector
expt.beam = refined_expt.beam
expt.goniometer = refined_expt.goniometer
expt.scan = refined_expt.scan
refined_expt.imageset = expt.imageset
if not (
self.all_params.refinement.parameterisation.beam.fix == "all"
and self.all_params.refinement.parameterisation.detector.fix == "all"
):
# Experimental geometry may have changed - re-map centroids to
# reciprocal space
self.reflections.map_centroids_to_reciprocal_space(self.experiments)
# update for next cycle
experiments = refined_experiments
self.refined_experiments = refined_experiments
if self.refined_experiments is None:
raise DialsIndexRefineError("None of the experiments could refine.")
# discard experiments with zero reflections after refinement
id_set = set(self.refined_reflections["id"])
if len(id_set) < len(self.refined_experiments):
filtered_refined_reflections = flex.reflection_table()
for i in range(len(self.refined_experiments)):
if i not in id_set:
del self.refined_experiments[i]
for old, new in zip(sorted(id_set), range(len(id_set))):
subset = self.refined_reflections.select(
self.refined_reflections["id"] == old
)
subset["id"] = flex.int(len(subset), new)
filtered_refined_reflections.extend(subset)
self.refined_reflections = filtered_refined_reflections
if len(self.refined_experiments) > 1:
from dials.algorithms.indexing.compare_orientation_matrices import (
rotation_matrix_differences,
)
logger.info(
rotation_matrix_differences(self.refined_experiments.crystals())
)
logger.info("Final refined crystal models:")
for i, crystal_model in enumerate(self.refined_experiments.crystals()):
n_indexed = 0
for _ in experiments.where(crystal=crystal_model):
n_indexed += (self.reflections["id"] == i).count(True)
logger.info("model %i (%i reflections):" % (i + 1, n_indexed))
logger.info(crystal_model)
if (
"xyzcal.mm" in self.refined_reflections
): # won't be there if refine_all_candidates = False and no isoforms
self._xyzcal_mm_to_px(self.experiments, self.refined_reflections)
def choose_best_orientation_matrix(self, candidate_orientation_matrices):
logger.info("*" * 80)
logger.info("Selecting the best orientation matrix")
logger.info("*" * 80)
class CandidateInfo(libtbx.group_args):
pass
candidates = []
params = copy.deepcopy(self.all_params)
for icm, cm in enumerate(candidate_orientation_matrices):
if icm >= self.params.basis_vector_combinations.max_refine:
break
# Index reflections in P1
sel = self.reflections["id"] == -1
refl = self.reflections.select(sel)
experiments = self.experiment_list_for_crystal(cm)
self.index_reflections(experiments, refl)
indexed = refl.select(refl["id"] >= 0)
indexed = indexed.select(indexed.get_flags(indexed.flags.indexed))
# If target symmetry supplied, try to apply it. Then, apply the change of basis to the reflections
# indexed in P1 to the target setting
if (
self.params.stills.refine_candidates_with_known_symmetry
and self.params.known_symmetry.space_group is not None
):
new_crystal, cb_op_to_primitive = self._symmetry_handler.apply_symmetry(
cm
)
if new_crystal is None:
logger.info("Cannot convert to target symmetry, candidate %d", icm)
continue
new_crystal = new_crystal.change_basis(
self._symmetry_handler.cb_op_primitive_inp
)
cm = new_crystal
experiments = self.experiment_list_for_crystal(cm)
if not cb_op_to_primitive.is_identity_op():
indexed["miller_index"] = cb_op_to_primitive.apply(
indexed["miller_index"]
)
if self._symmetry_handler.cb_op_primitive_inp is not None:
indexed[
"miller_index"
] = self._symmetry_handler.cb_op_primitive_inp.apply(
indexed["miller_index"]
)
if params.indexing.stills.refine_all_candidates:
try:
logger.info(
"$$$ stills_indexer::choose_best_orientation_matrix, candidate %d initial outlier identification",
icm,
)
acceptance_flags = self.identify_outliers(
params, experiments, indexed
)
# create a new "indexed" list with outliers thrown out:
indexed = indexed.select(acceptance_flags)
logger.info(
"$$$ stills_indexer::choose_best_orientation_matrix, candidate %d refinement before outlier rejection",
icm,
)
R = e_refine(
params=params,
experiments=experiments,
reflections=indexed,
graph_verbose=False,
)
ref_experiments = R.get_experiments()
# try to improve the outcome with a second round of outlier rejection post-initial refinement:
acceptance_flags = self.identify_outliers(
params, ref_experiments, indexed
)
# insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection
nv0 = NaveParameters(
params=params,
experiments=ref_experiments,
reflections=indexed,
refinery=R,
graph_verbose=False,
)
nv0()
acceptance_flags_nv0 = nv0.nv_acceptance_flags
indexed = indexed.select(acceptance_flags & acceptance_flags_nv0)
logger.info(
"$$$ stills_indexer::choose_best_orientation_matrix, candidate %d after positional and delta-psi outlier rejection",
icm,
)
R = e_refine(
params=params,
experiments=ref_experiments,
reflections=indexed,
graph_verbose=False,
)
ref_experiments = R.get_experiments()
nv = NaveParameters(
params=params,
experiments=ref_experiments,
reflections=indexed,
refinery=R,
graph_verbose=False,
)
crystal_model = nv()
assert (
len(crystal_model) == 1
), "$$$ stills_indexer::choose_best_orientation_matrix, Only one crystal at this stage"
crystal_model = crystal_model[0]
# Drop candidates that after refinement can no longer be converted to the known target space group
if (
not self.params.stills.refine_candidates_with_known_symmetry
and self.params.known_symmetry.space_group is not None
):
(
new_crystal,
cb_op_to_primitive,
) = self._symmetry_handler.apply_symmetry(crystal_model)
if new_crystal is None:
logger.info(
"P1 refinement yielded model diverged from target, candidate %d",
icm,
)
continue
rmsd, _ = calc_2D_rmsd_and_displacements(
R.predict_for_reflection_table(indexed)
)
except Exception as e:
logger.info(
"Couldn't refine candidate %d, %s: %s",
icm,
e.__class__.__name__,
str(e),
)
else:
logger.info(
"$$$ stills_indexer::choose_best_orientation_matrix, candidate %d done",
icm,
)
candidates.append(
CandidateInfo(
crystal=crystal_model,
green_curve_area=nv.green_curve_area,
ewald_proximal_volume=nv.ewald_proximal_volume(),
n_indexed=len(indexed),
rmsd=rmsd,
indexed=indexed,
experiments=ref_experiments,
)
)
else:
from dials.algorithms.refinement.prediction.managed_predictors import (
ExperimentsPredictorFactory,
)
ref_predictor = ExperimentsPredictorFactory.from_experiments(
experiments,
force_stills=True,
spherical_relp=params.refinement.parameterisation.spherical_relp_model,
)
rmsd, _ = calc_2D_rmsd_and_displacements(ref_predictor(indexed))
candidates.append(
CandidateInfo(
crystal=cm,
n_indexed=len(indexed),
rmsd=rmsd,
indexed=indexed,
experiments=experiments,
)
)
if len(candidates) == 0:
raise DialsIndexError("No suitable indexing solution found")
logger.info("**** ALL CANDIDATES:")
for i, XX in enumerate(candidates):
logger.info("\n****Candidate %d %s", i, XX)
cc = XX.crystal
if hasattr(cc, "get_half_mosaicity_deg"):
logger.info(
" half mosaicity %5.2f deg.", (cc.get_half_mosaicity_deg())
)
logger.info(" domain size %.0f Ang.", (cc.get_domain_size_ang()))
logger.info("\n**** BEST CANDIDATE:")
results = flex.double([c.rmsd for c in candidates])
best = candidates[flex.min_index(results)]
logger.info(best)
if params.indexing.stills.refine_all_candidates:
if best.rmsd > params.indexing.stills.rmsd_min_px:
raise DialsIndexError("RMSD too high, %f" % best.rmsd)
if len(candidates) > 1:
for i in range(len(candidates)):
if i == flex.min_index(results):
continue
if best.ewald_proximal_volume > candidates[i].ewald_proximal_volume:
logger.info(
"Couldn't figure out which candidate is best; picked the one with the best RMSD."
)
best.indexed["entering"] = flex.bool(best.n_indexed, False)
return best.crystal, best.n_indexed
def index(self):
experiments = ExperimentList()
had_refinement_error = False
have_similar_crystal_models = False
while True:
if had_refinement_error or have_similar_crystal_models:
break
max_lattices = self.params.multiple_lattice_search.max_lattices
if max_lattices is not None and len(experiments) >= max_lattices:
break
if len(experiments) > 0:
cutoff_fraction = (self.params.multiple_lattice_search.
recycle_unindexed_reflections_cutoff)
d_spacings = 1 / self.reflections["rlp"].norms()
d_min_indexed = flex.min(
d_spacings.select(self.indexed_reflections))
min_reflections_for_indexing = cutoff_fraction * len(
self.reflections.select(d_spacings > d_min_indexed))
crystal_ids = self.reflections.select(
d_spacings > d_min_indexed)["id"]
if (crystal_ids
== -1).count(True) < min_reflections_for_indexing:
logger.info(
"Finish searching for more lattices: %i unindexed reflections remaining."
% ((crystal_ids == -1).count(True)))
break
n_lattices_previous_cycle = len(experiments)
if self.d_min is None:
self.d_min = self.params.refinement_protocol.d_min_start
if len(experiments) == 0:
new_expts = self.find_lattices()
generate_experiment_identifiers(new_expts)
experiments.extend(new_expts)
else:
try:
new = self.find_lattices()
generate_experiment_identifiers(new)
experiments.extend(new)
except DialsIndexError:
logger.info("Indexing remaining reflections failed")
if self.params.refinement_protocol.d_min_step is libtbx.Auto:
n_cycles = self.params.refinement_protocol.n_macro_cycles
if self.d_min is None or n_cycles == 1:
self.params.refinement_protocol.d_min_step = 0
else:
d_spacings = 1 / self.reflections["rlp"].norms()
d_min_all = flex.min(d_spacings)
self.params.refinement_protocol.d_min_step = (
self.d_min - d_min_all) / (n_cycles - 1)
logger.info("Using d_min_step %.1f" %
self.params.refinement_protocol.d_min_step)
if len(experiments) == 0:
raise DialsIndexError("No suitable lattice could be found.")
elif len(experiments) == n_lattices_previous_cycle:
# no more lattices found
break
for i_cycle in range(
self.params.refinement_protocol.n_macro_cycles):
if (i_cycle > 0 and self.d_min is not None
and self.params.refinement_protocol.d_min_step > 0):
d_min = self.d_min - self.params.refinement_protocol.d_min_step
d_min = max(d_min, 0)
if self.params.refinement_protocol.d_min_final is not None:
d_min = max(
d_min, self.params.refinement_protocol.d_min_final)
if d_min >= 0:
self.d_min = d_min
logger.info("Increasing resolution to %.2f Angstrom" %
d_min)
# reset reflection lattice flags
# the lattice a given reflection belongs to: a value of -1 indicates
# that a reflection doesn't belong to any lattice so far
self.reflections["id"] = flex.int(len(self.reflections), -1)
self.index_reflections(experiments, self.reflections)
if i_cycle == 0 and self.params.known_symmetry.space_group is not None:
self._apply_symmetry_post_indexing(
experiments, self.reflections,
n_lattices_previous_cycle)
logger.info("\nIndexed crystal models:")
self.show_experiments(experiments,
self.reflections,
d_min=self.d_min)
if self._check_have_similar_crystal_models(experiments):
have_similar_crystal_models = True
break
logger.info("")
logger.info("#" * 80)
logger.info("Starting refinement (macro-cycle %i)" %
(i_cycle + 1))
logger.info("#" * 80)
logger.info("")
self.indexed_reflections = self.reflections["id"] > -1
sel = flex.bool(len(self.reflections), False)
lengths = 1 / self.reflections["rlp"].norms()
if self.d_min is not None:
isel = (lengths <= self.d_min).iselection()
sel.set_selected(isel, True)
sel.set_selected(self.reflections["id"] == -1, True)
self.reflections.unset_flags(sel,
self.reflections.flags.indexed)
self.unindexed_reflections = self.reflections.select(sel)
reflections_for_refinement = self.reflections.select(
self.indexed_reflections)
if self.params.refinement_protocol.mode == "repredict_only":
refined_experiments, refined_reflections = (
experiments,
reflections_for_refinement,
)
from dials.algorithms.refinement.prediction.managed_predictors import (
ExperimentsPredictorFactory, )
ref_predictor = ExperimentsPredictorFactory.from_experiments(
experiments,
spherical_relp=self.all_params.refinement.
parameterisation.spherical_relp_model,
)
ref_predictor(refined_reflections)
else:
try:
refined_experiments, refined_reflections = self.refine(
experiments, reflections_for_refinement)
except (DialsRefineConfigError,
DialsRefineRuntimeError) as e:
if len(experiments) == 1:
raise DialsIndexRefineError(str(e))
had_refinement_error = True
logger.info("Refinement failed:")
logger.info(e)
del experiments[-1]
# remove experiment id from the reflections associated
# with this deleted experiment - indexed flag removed
# below
last = len(experiments)
sel = refined_reflections["id"] == last
logger.info("Removing %d reflections with id %d" %
(sel.count(True), last))
refined_reflections["id"].set_selected(sel, -1)
break
self._unit_cell_volume_sanity_check(experiments,
refined_experiments)
self.refined_reflections = refined_reflections
self.refined_reflections.unset_flags(
self.refined_reflections["id"] < 0,
self.refined_reflections.flags.indexed,
)
for i, expt in enumerate(self.experiments):
ref_sel = self.refined_reflections.select(
self.refined_reflections["imageset_id"] == i)
ref_sel = ref_sel.select(ref_sel["id"] >= 0)
for i_expt in set(ref_sel["id"]):
refined_expt = refined_experiments[i_expt]
expt.detector = refined_expt.detector
expt.beam = refined_expt.beam
expt.goniometer = refined_expt.goniometer
expt.scan = refined_expt.scan
refined_expt.imageset = expt.imageset
if not (self.all_params.refinement.parameterisation.beam.fix
== "all" and self.all_params.refinement.
parameterisation.detector.fix == "all"):
# Experimental geometry may have changed - re-map centroids to
# reciprocal space
self.reflections.map_centroids_to_reciprocal_space(
self.experiments)
# update for next cycle
experiments = refined_experiments
self.refined_experiments = refined_experiments
logger.info("\nRefined crystal models:")
self.show_experiments(self.refined_experiments,
self.reflections,
d_min=self.d_min)
if (i_cycle >= 2 and self.d_min
== self.params.refinement_protocol.d_min_final):
logger.info(
"Target d_min_final reached: finished with refinement")
break
if self.refined_experiments is None:
raise DialsIndexRefineError(
"None of the experiments could refine.")
if len(self.refined_experiments) > 1:
from dials.algorithms.indexing.compare_orientation_matrices import (
rotation_matrix_differences, )
logger.info(
rotation_matrix_differences(
self.refined_experiments.crystals()))
self._xyzcal_mm_to_px(self.refined_experiments,
self.refined_reflections)
def __call__(self):
"""Determine optimal mosaicity and domain size model (monochromatic)"""
if self.refinery is None:
RR = self.reflections
else:
RR = self.refinery.predict_for_reflection_table(self.reflections)
all_crystals = []
self.nv_acceptance_flags = flex.bool(len(self.reflections["id"]))
from dxtbx.model import MosaicCrystalSauter2014
for iid, experiment in enumerate(self.experiments):
excursion_rad = RR["delpsical.rad"].select(RR["id"] == iid)
delta_psi_deg = excursion_rad * 180.0 / math.pi
logger.info("")
logger.info("%s %s", flex.max(delta_psi_deg),
flex.min(delta_psi_deg))
mean_excursion = flex.mean(delta_psi_deg)
logger.info(
"The mean excursion is %7.3f degrees, r.m.s.d %7.3f",
mean_excursion,
math.sqrt(flex.mean(RR["delpsical2"].select(RR["id"] == iid))),
)
crystal = MosaicCrystalSauter2014(self.experiments[iid].crystal)
self.experiments[iid].crystal = crystal
beam = self.experiments[iid].beam
miller_indices = self.reflections["miller_index"].select(
self.reflections["id"] == iid)
# FIXME XXX revise this formula so as to use a different wavelength potentially for each reflection
two_thetas = crystal.get_unit_cell().two_theta(
miller_indices, beam.get_wavelength(), deg=True)
dspacings = crystal.get_unit_cell().d(miller_indices)
# First -- try to get a reasonable envelope for the observed excursions.
# minimum of three regions; maximum of 50 measurements in each bin
logger.info("fitting parameters on %d spots", len(excursion_rad))
n_bins = min(max(3, len(excursion_rad) // 25), 50)
bin_sz = len(excursion_rad) // n_bins
logger.info("nbins %s bin_sz %s", n_bins, bin_sz)
order = flex.sort_permutation(two_thetas)
two_thetas_env = flex.double()
dspacings_env = flex.double()
excursion_rads_env = flex.double()
for x in range(0, n_bins):
subset = order[x * bin_sz:(x + 1) * bin_sz]
two_thetas_env.append(flex.mean(two_thetas.select(subset)))
dspacings_env.append(flex.mean(dspacings.select(subset)))
excursion_rads_env.append(
flex.max(flex.abs(excursion_rad.select(subset))))
# Second -- parameter fit
# solve the normal equations
sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env)
sum_inv_u = flex.sum(dspacings_env)
sum_te_u = flex.sum(dspacings_env * excursion_rads_env)
sum_te = flex.sum(excursion_rads_env)
Normal_Mat = sqr(
(sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env)))
Vector = col((sum_te_u, sum_te))
solution = Normal_Mat.inverse() * Vector
s_ang = 1.0 / (2 * solution[0])
logger.info("Best LSQ fit Scheerer domain size is %9.2f ang",
s_ang)
k_degrees = solution[1] * 180.0 / math.pi
logger.info(
"The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f",
2 * k_degrees,
k_degrees,
)
from xfel.mono_simulation.max_like import minimizer
# coerce the estimates to be positive for max-likelihood
lower_limit_domain_size = (
math.pow(crystal.get_unit_cell().volume(), 1.0 / 3.0) * 3
) # params.refinement.domain_size_lower_limit
d_estimate = max(s_ang, lower_limit_domain_size)
M = minimizer(
d_i=dspacings,
psi_i=excursion_rad,
eta_rad=abs(2.0 * solution[1]),
Deff=d_estimate,
)
logger.info(
"ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots",
M.x[1] * 180.0 / math.pi,
2.0 / M.x[0],
len(two_thetas),
)
tan_phi_rad_ML = dspacings / (2.0 / M.x[0])
tan_phi_deg_ML = tan_phi_rad_ML * 180.0 / math.pi
tan_outer_deg_ML = tan_phi_deg_ML + 0.5 * M.x[1] * 180.0 / math.pi
# Only set the flags for those reflections that were indexed for this lattice
self.nv_acceptance_flags.set_selected(
self.reflections["id"] == iid,
flex.abs(delta_psi_deg) < tan_outer_deg_ML,
)
if (
self.graph_verbose
): # params.refinement.mosaic.enable_AD14F7B: # Excursion vs resolution fit
AD1TF7B_MAX2T = 30.0
AD1TF7B_MAXDP = 1.0
from matplotlib import pyplot as plt
plt.plot(two_thetas, delta_psi_deg, "bo")
minplot = flex.min(two_thetas)
plt.plot([0, minplot], [mean_excursion, mean_excursion], "k-")
LR = flex.linear_regression(two_thetas, delta_psi_deg)
model_y = LR.slope() * two_thetas + LR.y_intercept()
plt.plot(two_thetas, model_y, "k-")
plt.title(
"ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" %
(M.x[1] * 180.0 / math.pi, 2.0 / M.x[0], len(two_thetas)))
plt.plot(two_thetas, tan_phi_deg_ML, "r.")
plt.plot(two_thetas, -tan_phi_deg_ML, "r.")
plt.plot(two_thetas, tan_outer_deg_ML, "g.")
plt.plot(two_thetas, -tan_outer_deg_ML, "g.")
plt.xlim([0, AD1TF7B_MAX2T])
plt.ylim([-AD1TF7B_MAXDP, AD1TF7B_MAXDP])
plt.show()
plt.close()
from xfel.mono_simulation.util import green_curve_area
self.green_curve_area = green_curve_area(two_thetas,
tan_outer_deg_ML)
logger.info("The green curve area is %s", self.green_curve_area)
crystal.set_half_mosaicity_deg(M.x[1] * 180.0 / (2.0 * math.pi))
crystal.set_domain_size_ang(2.0 / M.x[0])
self._ML_full_mosaicity_rad = M.x[1]
self._ML_domain_size_ang = 2.0 / M.x[0]
# params.refinement.mosaic.model_expansion_factor
"""The expansion factor should be initially set to 1, then expanded so that the # reflections matched becomes
as close as possible to # of observed reflections input, in the last integration call. Determine this by
inspecting the output log file interactively. Do not exceed the bare minimum threshold needed.
The intention is to find an optimal value, global for a given dataset."""
model_expansion_factor = 1.4
crystal.set_half_mosaicity_deg(crystal.get_half_mosaicity_deg() *
model_expansion_factor)
crystal.set_domain_size_ang(crystal.get_domain_size_ang() /
model_expansion_factor)
if (self.ewald_proximal_volume(iid) >
self.params.indexing.stills.ewald_proximal_volume_max):
raise DialsIndexError("Ewald proximity volume too high, %f" %
self.ewald_proximal_volume(iid))
all_crystals.append(crystal)
return all_crystals
def __call__(self, reflections, experiments, d_min=None):
from scitbx import matrix
from libtbx.math_utils import nearest_integer as nint
reciprocal_lattice_points = reflections["rlp"]
if "miller_index" not in reflections:
reflections["miller_index"] = flex.miller_index(len(reflections))
if d_min is not None:
d_spacings = 1 / reciprocal_lattice_points.norms()
inside_resolution_limit = d_spacings > d_min
else:
inside_resolution_limit = flex.bool(
reciprocal_lattice_points.size(), True)
sel = inside_resolution_limit & (reflections["id"] == -1)
isel = sel.iselection()
rlps = reciprocal_lattice_points.select(isel)
refs = reflections.select(isel)
phi = refs["xyzobs.mm.value"].parts()[2]
if len(rlps) <= self._nearest_neighbours:
raise DialsIndexError(
"index_assignment.local.nearest_neighbour must be smaller than the number of accepted reflections (%d)"
% len(rlps))
UB_matrices = flex.mat3_double(
[cm.get_A() for cm in experiments.crystals()])
result = ext.AssignIndicesLocal(
rlps,
phi,
UB_matrices,
epsilon=self._epsilon,
delta=self._delta,
l_min=self._l_min,
nearest_neighbours=self._nearest_neighbours,
)
miller_indices = result.miller_indices()
crystal_ids = result.crystal_ids()
hkl = miller_indices.as_vec3_double().iround()
assert miller_indices.select(crystal_ids < 0).all_eq((0, 0, 0))
for i_cryst in set(crystal_ids):
if i_cryst < 0:
continue
A = matrix.sqr(experiments[i_cryst].crystal.get_A())
A_inv = A.inverse()
cryst_sel = crystal_ids == i_cryst
rlp_sel = rlps.select(cryst_sel)
hkl_sel = hkl.select(cryst_sel).as_vec3_double()
d_sel = 1 / rlp_sel.norms()
d_perm = flex.sort_permutation(d_sel, reverse=True)
hf_0 = A_inv * rlp_sel[d_perm[0]]
h_0 = matrix.col([nint(j) for j in hf_0.elems])
offset = h_0 - matrix.col(hkl_sel[d_perm[0]])
# print "offset:", offset.elems
h = hkl_sel + flex.vec3_double(hkl_sel.size(), offset.elems)
refs["miller_index"].set_selected(
cryst_sel, flex.miller_index(list(h.iround())))
refs["id"].set_selected(cryst_sel, i_cryst)
crystal_ids.set_selected(crystal_ids < 0, -1)
refs["id"] = crystal_ids
refs["miller_index"].set_selected(crystal_ids < 0, (0, 0, 0))
reflections["miller_index"].set_selected(isel, refs["miller_index"])
reflections["id"].set_selected(isel, refs["id"])
reflections.set_flags(reflections["miller_index"] != (0, 0, 0),
reflections.flags.indexed)