def index(self):
# most of this is the same as dials.algorithms.indexing.indexer.indexer_base.index(), with some stills
# specific modifications (don't re-index after choose best orientation matrix, but use the indexing from
# choose best orientation matrix, also don't use macrocycles) of refinement after indexing.
# 2017 update: do accept multiple lattices per shot
experiments = ExperimentList()
while True:
self.d_min = self.params.refinement_protocol.d_min_start
max_lattices = self.params.multiple_lattice_search.max_lattices
if max_lattices is not None and len(experiments) >= max_lattices:
break
if len(experiments) > 0:
cutoff_fraction = (
self.params.multiple_lattice_search.recycle_unindexed_reflections_cutoff
)
d_spacings = 1 / self.reflections["rlp"].norms()
d_min_indexed = flex.min(d_spacings.select(self.indexed_reflections))
min_reflections_for_indexing = cutoff_fraction * len(
self.reflections.select(d_spacings > d_min_indexed)
)
crystal_ids = self.reflections.select(d_spacings > d_min_indexed)["id"]
if (crystal_ids == -1).count(True) < min_reflections_for_indexing:
logger.info(
"Finish searching for more lattices: %i unindexed reflections remaining."
% (min_reflections_for_indexing)
)
break
n_lattices_previous_cycle = len(experiments)
# index multiple lattices per shot
if len(experiments) == 0:
new = self.find_lattices()
generate_experiment_identifiers(new)
experiments.extend(new)
if len(experiments) == 0:
raise DialsIndexError("No suitable lattice could be found.")
else:
try:
new = self.find_lattices()
generate_experiment_identifiers(new)
experiments.extend(new)
except Exception as e:
logger.info("Indexing remaining reflections failed")
logger.debug(
"Indexing remaining reflections failed, exception:\n" + str(e)
)
# reset reflection lattice flags
# the lattice a given reflection belongs to: a value of -1 indicates
# that a reflection doesn't belong to any lattice so far
self.reflections["id"] = flex.int(len(self.reflections), -1)
self.index_reflections(experiments, self.reflections)
if len(experiments) == n_lattices_previous_cycle:
# no more lattices found
break
if (
not self.params.stills.refine_candidates_with_known_symmetry
and self.params.known_symmetry.space_group is not None
):
self._apply_symmetry_post_indexing(
experiments, self.reflections, n_lattices_previous_cycle
)
# discard nearly overlapping lattices on the same shot
if self._check_have_similar_crystal_models(experiments):
break
self.indexed_reflections = self.reflections["id"] > -1
if self.d_min is None:
sel = self.reflections["id"] <= -1
else:
sel = flex.bool(len(self.reflections), False)
lengths = 1 / self.reflections["rlp"].norms()
isel = (lengths >= self.d_min).iselection()
sel.set_selected(isel, True)
sel.set_selected(self.reflections["id"] > -1, False)
self.unindexed_reflections = self.reflections.select(sel)
reflections_for_refinement = self.reflections.select(
self.indexed_reflections
)
if len(self.params.stills.isoforms) > 0:
logger.info("")
logger.info("#" * 80)
logger.info("Starting refinement")
logger.info("#" * 80)
logger.info("")
isoform_experiments = ExperimentList()
isoform_reflections = flex.reflection_table()
# Note, changes to params after initial indexing. Cannot use tie to target when fixing the unit cell.
self.all_params.refinement.reflections.outlier.algorithm = "null"
self.all_params.refinement.parameterisation.crystal.fix = "cell"
self.all_params.refinement.parameterisation.crystal.unit_cell.restraints.tie_to_target = (
[]
)
for expt_id, experiment in enumerate(experiments):
reflections = reflections_for_refinement.select(
reflections_for_refinement["id"] == expt_id
)
reflections["id"] = flex.int(len(reflections), 0)
refiners = []
for isoform in self.params.stills.isoforms:
iso_experiment = copy.deepcopy(experiment)
crystal = iso_experiment.crystal
if (
isoform.lookup_symbol
!= crystal.get_space_group().type().lookup_symbol()
):
logger.info(
"Crystal isoform lookup_symbol %s does not match isoform %s lookup_symbol %s"
% (
crystal.get_space_group().type().lookup_symbol(),
isoform.name,
isoform.lookup_symbol,
)
)
continue
crystal.set_B(isoform.cell.fractionalization_matrix())
logger.info("Refining isoform %s" % isoform.name)
refiners.append(
e_refine(
params=self.all_params,
experiments=ExperimentList([iso_experiment]),
reflections=reflections,
graph_verbose=False,
)
)
if len(refiners) == 0:
raise DialsIndexError(
"No isoforms had a lookup symbol that matched"
)
positional_rmsds = [
math.sqrt(P.rmsds()[0] ** 2 + P.rmsds()[1] ** 2)
for P in refiners
]
logger.info(
"Positional rmsds for all isoforms:" + str(positional_rmsds)
)
minrmsd_mm = min(positional_rmsds)
minindex = positional_rmsds.index(minrmsd_mm)
logger.info(
"The smallest rmsd is %5.1f um from isoform %s"
% (
1000.0 * minrmsd_mm,
self.params.stills.isoforms[minindex].name,
)
)
if self.params.stills.isoforms[minindex].rmsd_target_mm is not None:
logger.info(
"Asserting %f < %f"
% (
minrmsd_mm,
self.params.stills.isoforms[minindex].rmsd_target_mm,
)
)
assert (
minrmsd_mm
< self.params.stills.isoforms[minindex].rmsd_target_mm
)
logger.info(
"Acceptable rmsd for isoform %s."
% (self.params.stills.isoforms[minindex].name)
)
if len(self.params.stills.isoforms) == 2:
logger.info(
"Rmsd gain over the other isoform %5.1f um."
% (1000.0 * abs(positional_rmsds[0] - positional_rmsds[1]))
)
R = refiners[minindex]
# Now one last check to see if direct beam is out of bounds
if self.params.stills.isoforms[minindex].beam_restraint is not None:
from scitbx import matrix
refined_beam = matrix.col(
R.get_experiments()[0]
.detector[0]
.get_beam_centre_lab(experiments[0].beam.get_s0())[0:2]
)
known_beam = matrix.col(
self.params.stills.isoforms[minindex].beam_restraint
)
logger.info(
"Asserting difference in refined beam center and expected beam center %f < %f"
% (
(refined_beam - known_beam).length(),
self.params.stills.isoforms[minindex].rmsd_target_mm,
)
)
assert (
(refined_beam - known_beam).length()
< self.params.stills.isoforms[minindex].rmsd_target_mm
)
# future--circle of confusion could be given as a separate length in mm instead of reusing rmsd_target
experiment = R.get_experiments()[0]
experiment.crystal.identified_isoform = self.params.stills.isoforms[
minindex
].name
isoform_experiments.append(experiment)
reflections["id"] = flex.int(len(reflections), expt_id)
isoform_reflections.extend(reflections)
experiments = isoform_experiments
reflections_for_refinement = isoform_reflections
if self.params.refinement_protocol.mode == "repredict_only":
from dials.algorithms.indexing.nave_parameters import NaveParameters
from dials.algorithms.refinement.prediction.managed_predictors import (
ExperimentsPredictorFactory,
)
refined_experiments, refined_reflections = (
experiments,
reflections_for_refinement,
)
ref_predictor = ExperimentsPredictorFactory.from_experiments(
experiments,
force_stills=True,
spherical_relp=self.all_params.refinement.parameterisation.spherical_relp_model,
)
ref_predictor(refined_reflections)
refined_reflections["delpsical2"] = (
refined_reflections["delpsical.rad"] ** 2
)
for expt_id in range(len(refined_experiments)):
refls = refined_reflections.select(
refined_reflections["id"] == expt_id
)
nv = NaveParameters(
params=self.all_params,
experiments=refined_experiments[expt_id : expt_id + 1],
reflections=refls,
refinery=None,
graph_verbose=False,
)
experiments[expt_id].crystal = nv()
ref_predictor = ExperimentsPredictorFactory.from_experiments(
experiments,
force_stills=True,
spherical_relp=self.all_params.refinement.parameterisation.spherical_relp_model,
)
ref_predictor(refined_reflections)
elif self.params.refinement_protocol.mode is None:
refined_experiments, refined_reflections = (
experiments,
reflections_for_refinement,
)
else:
try:
refined_experiments, refined_reflections = self.refine(
experiments, reflections_for_refinement
)
except Exception as e:
s = str(e)
if len(experiments) == 1:
raise DialsIndexRefineError(e.message)
logger.info("Refinement failed:")
logger.info(s)
del experiments[-1]
break
self._unit_cell_volume_sanity_check(experiments, refined_experiments)
self.refined_reflections = refined_reflections.select(
refined_reflections["id"] > -1
)
for i, expt in enumerate(self.experiments):
ref_sel = self.refined_reflections.select(
self.refined_reflections["imageset_id"] == i
)
ref_sel = ref_sel.select(ref_sel["id"] >= 0)
for i_expt in set(ref_sel["id"]):
refined_expt = refined_experiments[i_expt]
expt.detector = refined_expt.detector
expt.beam = refined_expt.beam
expt.goniometer = refined_expt.goniometer
expt.scan = refined_expt.scan
refined_expt.imageset = expt.imageset
if not (
self.all_params.refinement.parameterisation.beam.fix == "all"
and self.all_params.refinement.parameterisation.detector.fix == "all"
):
# Experimental geometry may have changed - re-map centroids to
# reciprocal space
self.reflections.map_centroids_to_reciprocal_space(self.experiments)
# update for next cycle
experiments = refined_experiments
self.refined_experiments = refined_experiments
if self.refined_experiments is None:
raise DialsIndexRefineError("None of the experiments could refine.")
# discard experiments with zero reflections after refinement
id_set = set(self.refined_reflections["id"])
if len(id_set) < len(self.refined_experiments):
filtered_refined_reflections = flex.reflection_table()
for i in range(len(self.refined_experiments)):
if i not in id_set:
del self.refined_experiments[i]
for old, new in zip(sorted(id_set), range(len(id_set))):
subset = self.refined_reflections.select(
self.refined_reflections["id"] == old
)
subset["id"] = flex.int(len(subset), new)
filtered_refined_reflections.extend(subset)
self.refined_reflections = filtered_refined_reflections
if len(self.refined_experiments) > 1:
from dials.algorithms.indexing.compare_orientation_matrices import (
rotation_matrix_differences,
)
logger.info(
rotation_matrix_differences(self.refined_experiments.crystals())
)
logger.info("Final refined crystal models:")
for i, crystal_model in enumerate(self.refined_experiments.crystals()):
n_indexed = 0
for _ in experiments.where(crystal=crystal_model):
n_indexed += (self.reflections["id"] == i).count(True)
logger.info("model %i (%i reflections):" % (i + 1, n_indexed))
logger.info(crystal_model)
if (
"xyzcal.mm" in self.refined_reflections
): # won't be there if refine_all_candidates = False and no isoforms
self._xyzcal_mm_to_px(self.experiments, self.refined_reflections)
def index(self):
experiments = ExperimentList()
had_refinement_error = False
have_similar_crystal_models = False
while True:
if had_refinement_error or have_similar_crystal_models:
break
max_lattices = self.params.multiple_lattice_search.max_lattices
if max_lattices is not None and len(experiments) >= max_lattices:
break
if len(experiments) > 0:
cutoff_fraction = (self.params.multiple_lattice_search.
recycle_unindexed_reflections_cutoff)
d_spacings = 1 / self.reflections["rlp"].norms()
d_min_indexed = flex.min(
d_spacings.select(self.indexed_reflections))
min_reflections_for_indexing = cutoff_fraction * len(
self.reflections.select(d_spacings > d_min_indexed))
crystal_ids = self.reflections.select(
d_spacings > d_min_indexed)["id"]
if (crystal_ids
== -1).count(True) < min_reflections_for_indexing:
logger.info(
"Finish searching for more lattices: %i unindexed reflections remaining."
% ((crystal_ids == -1).count(True)))
break
n_lattices_previous_cycle = len(experiments)
if self.d_min is None:
self.d_min = self.params.refinement_protocol.d_min_start
if len(experiments) == 0:
new_expts = self.find_lattices()
generate_experiment_identifiers(new_expts)
experiments.extend(new_expts)
else:
try:
new = self.find_lattices()
generate_experiment_identifiers(new)
experiments.extend(new)
except DialsIndexError:
logger.info("Indexing remaining reflections failed")
if self.params.refinement_protocol.d_min_step is libtbx.Auto:
n_cycles = self.params.refinement_protocol.n_macro_cycles
if self.d_min is None or n_cycles == 1:
self.params.refinement_protocol.d_min_step = 0
else:
d_spacings = 1 / self.reflections["rlp"].norms()
d_min_all = flex.min(d_spacings)
self.params.refinement_protocol.d_min_step = (
self.d_min - d_min_all) / (n_cycles - 1)
logger.info("Using d_min_step %.1f" %
self.params.refinement_protocol.d_min_step)
if len(experiments) == 0:
raise DialsIndexError("No suitable lattice could be found.")
elif len(experiments) == n_lattices_previous_cycle:
# no more lattices found
break
for i_cycle in range(
self.params.refinement_protocol.n_macro_cycles):
if (i_cycle > 0 and self.d_min is not None
and self.params.refinement_protocol.d_min_step > 0):
d_min = self.d_min - self.params.refinement_protocol.d_min_step
d_min = max(d_min, 0)
if self.params.refinement_protocol.d_min_final is not None:
d_min = max(
d_min, self.params.refinement_protocol.d_min_final)
if d_min >= 0:
self.d_min = d_min
logger.info("Increasing resolution to %.2f Angstrom" %
d_min)
# reset reflection lattice flags
# the lattice a given reflection belongs to: a value of -1 indicates
# that a reflection doesn't belong to any lattice so far
self.reflections["id"] = flex.int(len(self.reflections), -1)
self.index_reflections(experiments, self.reflections)
if i_cycle == 0 and self.params.known_symmetry.space_group is not None:
self._apply_symmetry_post_indexing(
experiments, self.reflections,
n_lattices_previous_cycle)
logger.info("\nIndexed crystal models:")
self.show_experiments(experiments,
self.reflections,
d_min=self.d_min)
if self._check_have_similar_crystal_models(experiments):
have_similar_crystal_models = True
break
logger.info("")
logger.info("#" * 80)
logger.info("Starting refinement (macro-cycle %i)" %
(i_cycle + 1))
logger.info("#" * 80)
logger.info("")
self.indexed_reflections = self.reflections["id"] > -1
sel = flex.bool(len(self.reflections), False)
lengths = 1 / self.reflections["rlp"].norms()
if self.d_min is not None:
isel = (lengths <= self.d_min).iselection()
sel.set_selected(isel, True)
sel.set_selected(self.reflections["id"] == -1, True)
self.reflections.unset_flags(sel,
self.reflections.flags.indexed)
self.unindexed_reflections = self.reflections.select(sel)
reflections_for_refinement = self.reflections.select(
self.indexed_reflections)
if self.params.refinement_protocol.mode == "repredict_only":
refined_experiments, refined_reflections = (
experiments,
reflections_for_refinement,
)
from dials.algorithms.refinement.prediction.managed_predictors import (
ExperimentsPredictorFactory, )
ref_predictor = ExperimentsPredictorFactory.from_experiments(
experiments,
spherical_relp=self.all_params.refinement.
parameterisation.spherical_relp_model,
)
ref_predictor(refined_reflections)
else:
try:
refined_experiments, refined_reflections = self.refine(
experiments, reflections_for_refinement)
except (DialsRefineConfigError,
DialsRefineRuntimeError) as e:
if len(experiments) == 1:
raise DialsIndexRefineError(str(e))
had_refinement_error = True
logger.info("Refinement failed:")
logger.info(e)
del experiments[-1]
# remove experiment id from the reflections associated
# with this deleted experiment - indexed flag removed
# below
last = len(experiments)
sel = refined_reflections["id"] == last
logger.info("Removing %d reflections with id %d" %
(sel.count(True), last))
refined_reflections["id"].set_selected(sel, -1)
break
self._unit_cell_volume_sanity_check(experiments,
refined_experiments)
self.refined_reflections = refined_reflections
self.refined_reflections.unset_flags(
self.refined_reflections["id"] < 0,
self.refined_reflections.flags.indexed,
)
for i, expt in enumerate(self.experiments):
ref_sel = self.refined_reflections.select(
self.refined_reflections["imageset_id"] == i)
ref_sel = ref_sel.select(ref_sel["id"] >= 0)
for i_expt in set(ref_sel["id"]):
refined_expt = refined_experiments[i_expt]
expt.detector = refined_expt.detector
expt.beam = refined_expt.beam
expt.goniometer = refined_expt.goniometer
expt.scan = refined_expt.scan
refined_expt.imageset = expt.imageset
if not (self.all_params.refinement.parameterisation.beam.fix
== "all" and self.all_params.refinement.
parameterisation.detector.fix == "all"):
# Experimental geometry may have changed - re-map centroids to
# reciprocal space
self.reflections.map_centroids_to_reciprocal_space(
self.experiments)
# update for next cycle
experiments = refined_experiments
self.refined_experiments = refined_experiments
logger.info("\nRefined crystal models:")
self.show_experiments(self.refined_experiments,
self.reflections,
d_min=self.d_min)
if (i_cycle >= 2 and self.d_min
== self.params.refinement_protocol.d_min_final):
logger.info(
"Target d_min_final reached: finished with refinement")
break
if self.refined_experiments is None:
raise DialsIndexRefineError(
"None of the experiments could refine.")
if len(self.refined_experiments) > 1:
from dials.algorithms.indexing.compare_orientation_matrices import (
rotation_matrix_differences, )
logger.info(
rotation_matrix_differences(
self.refined_experiments.crystals()))
self._xyzcal_mm_to_px(self.refined_experiments,
self.refined_reflections)
