def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
import libtbx.load_env
usage = "%s [options] experiments.json" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) <= 1:
parser.print_help()
return
hkl = flex.miller_index(params.hkl)
from dials.algorithms.indexing.compare_orientation_matrices import \
show_rotation_matrix_differences
crystals = []
for experiment in experiments:
crystal = experiment.crystal
if params.space_group is not None:
crystal.set_space_group(params.space_group.group())
crystals.append(crystal)
show_rotation_matrix_differences(crystals, miller_indices=hkl, comparison=params.comparison)
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
from libtbx.utils import Sorry
import libtbx.load_env
usage = "%s [options] experiments.json" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) <= 1:
parser.print_help()
return
hkl = flex.miller_index(params.hkl)
from dials.algorithms.indexing.compare_orientation_matrices import \
show_rotation_matrix_differences
show_rotation_matrix_differences(experiments.crystals(),
miller_indices=hkl)
def index(self):
# most of this is the same as dials.algorithms.indexing.indexer.indexer_base.index(), with some stills
# specific modifications (don't re-index after choose best orientation matrix, but use the indexing from
# choose best orientation matrix, also don't use macrocycles) of refinement after indexing.
# 2017 update: do accept multiple lattices per shot
if self.params.refinement_protocol.n_macro_cycles > 1:
raise Sorry(
"For stills, please set refinement_protocol.n_macro_cycles = 1"
)
experiments = ExperimentList()
had_refinement_error = False
have_similar_crystal_models = False
while True:
self.d_min = self.params.refinement_protocol.d_min_start
if had_refinement_error or have_similar_crystal_models:
break
max_lattices = self.params.multiple_lattice_search.max_lattices
if max_lattices is not None and len(experiments) >= max_lattices:
break
if len(experiments) > 0:
cutoff_fraction = \
self.params.multiple_lattice_search.recycle_unindexed_reflections_cutoff
d_spacings = 1 / self.reflections['rlp'].norms()
d_min_indexed = flex.min(
d_spacings.select(self.indexed_reflections))
min_reflections_for_indexing = \
cutoff_fraction * len(self.reflections.select(d_spacings > d_min_indexed))
crystal_ids = self.reflections.select(
d_spacings > d_min_indexed)['id']
if (crystal_ids
== -1).count(True) < min_reflections_for_indexing:
logger.info(
"Finish searching for more lattices: %i unindexed reflections remaining."
% (min_reflections_for_indexing))
break
n_lattices_previous_cycle = len(experiments)
# index multiple lattices per shot
if len(experiments) == 0:
experiments.extend(self.find_lattices())
if len(experiments) == 0:
raise Sorry("No suitable lattice could be found.")
else:
try:
new = self.find_lattices()
experiments.extend(new)
except Exception as e:
logger.info("Indexing remaining reflections failed")
logger.debug(
"Indexing remaining reflections failed, exception:\n" +
str(e))
# reset reflection lattice flags
# the lattice a given reflection belongs to: a value of -1 indicates
# that a reflection doesn't belong to any lattice so far
self.reflections['id'] = flex.int(len(self.reflections), -1)
self.index_reflections(experiments, self.reflections)
if len(experiments) == n_lattices_previous_cycle:
# no more lattices found
break
if not self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None:
# now apply the space group symmetry only after the first indexing
# need to make sure that the symmetrized orientation is similar to the P1 model
target_space_group = self.target_symmetry_primitive.space_group(
)
for i_cryst, cryst in enumerate(experiments.crystals()):
if i_cryst >= n_lattices_previous_cycle:
new_cryst, cb_op_to_primitive = self.apply_symmetry(
cryst, target_space_group)
if self.cb_op_primitive_inp is not None:
new_cryst = new_cryst.change_basis(
self.cb_op_primitive_inp)
logger.info(new_cryst.get_space_group().info())
cryst.update(new_cryst)
cryst.set_space_group(
self.params.known_symmetry.space_group.group())
for i_expt, expt in enumerate(experiments):
if expt.crystal is not cryst:
continue
if not cb_op_to_primitive.is_identity_op():
miller_indices = self.reflections[
'miller_index'].select(
self.reflections['id'] == i_expt)
miller_indices = cb_op_to_primitive.apply(
miller_indices)
self.reflections['miller_index'].set_selected(
self.reflections['id'] == i_expt,
miller_indices)
if self.cb_op_primitive_inp is not None:
miller_indices = self.reflections[
'miller_index'].select(
self.reflections['id'] == i_expt)
miller_indices = self.cb_op_primitive_inp.apply(
miller_indices)
self.reflections['miller_index'].set_selected(
self.reflections['id'] == i_expt,
miller_indices)
# discard nearly overlapping lattices on the same shot
if len(experiments) > 1:
from dials.algorithms.indexing.compare_orientation_matrices \
import difference_rotation_matrix_axis_angle
cryst_b = experiments.crystals()[-1]
have_similar_crystal_models = False
for i_a, cryst_a in enumerate(experiments.crystals()[:-1]):
R_ab, axis, angle, cb_op_ab = \
difference_rotation_matrix_axis_angle(cryst_a, cryst_b)
min_angle = self.params.multiple_lattice_search.minimum_angular_separation
if abs(angle) < min_angle: # degrees
logger.info(
"Crystal models too similar, rejecting crystal %i:"
% (len(experiments)))
logger.info(
"Rotation matrix to transform crystal %i to crystal %i"
% (i_a + 1, len(experiments)))
logger.info(R_ab)
logger.info("Rotation of %.3f degrees" % angle +
" about axis (%.3f, %.3f, %.3f)" % axis)
#show_rotation_matrix_differences([cryst_a, cryst_b])
have_similar_crystal_models = True
del experiments[-1]
break
if have_similar_crystal_models:
break
self.indexed_reflections = (self.reflections['id'] > -1)
if self.d_min is None:
sel = self.reflections['id'] <= -1
else:
sel = flex.bool(len(self.reflections), False)
lengths = 1 / self.reflections['rlp'].norms()
isel = (lengths >= self.d_min).iselection()
sel.set_selected(isel, True)
sel.set_selected(self.reflections['id'] > -1, False)
self.unindexed_reflections = self.reflections.select(sel)
reflections_for_refinement = self.reflections.select(
self.indexed_reflections)
if len(self.params.stills.isoforms) > 0:
logger.info("")
logger.info("#" * 80)
logger.info("Starting refinement")
logger.info("#" * 80)
logger.info("")
import copy
isoform_experiments = ExperimentList()
isoform_reflections = flex.reflection_table()
# Note, changes to params after initial indexing. Cannot use tie to target when fixing the unit cell.
self.all_params.refinement.reflections.outlier.algorithm = "null"
self.all_params.refinement.parameterisation.crystal.fix = "cell"
self.all_params.refinement.parameterisation.crystal.unit_cell.restraints.tie_to_target = []
for expt_id, experiment in enumerate(experiments):
reflections = reflections_for_refinement.select(
reflections_for_refinement['id'] == expt_id)
reflections['id'] = flex.int(len(reflections), 0)
refiners = []
for isoform in self.params.stills.isoforms:
iso_experiment = copy.deepcopy(experiment)
crystal = iso_experiment.crystal
if isoform.lookup_symbol != crystal.get_space_group(
).type().lookup_symbol():
logger.info(
"Crystal isoform lookup_symbol %s does not match isoform %s lookup_symbol %s"
% (crystal.get_space_group().type(
).lookup_symbol(), isoform.name,
isoform.lookup_symbol))
continue
crystal.set_B(isoform.cell.fractionalization_matrix())
logger.info("Refining isoform %s" % isoform.name)
refiners.append(
e_refine(params=self.all_params,
experiments=ExperimentList(
[iso_experiment]),
reflections=reflections,
graph_verbose=False))
if len(refiners) == 0:
raise Sorry(
"No isoforms had a lookup symbol that matched")
positional_rmsds = [
math.sqrt(P.rmsds()[0]**2 + P.rmsds()[1]**2)
for P in refiners
]
logger.info("Positional rmsds for all isoforms:" +
str(positional_rmsds))
minrmsd_mm = min(positional_rmsds)
minindex = positional_rmsds.index(minrmsd_mm)
logger.info(
"The smallest rmsd is %5.1f um from isoform %s" %
(1000. * minrmsd_mm,
self.params.stills.isoforms[minindex].name))
if self.params.stills.isoforms[
minindex].rmsd_target_mm is not None:
logger.info("Asserting %f < %f" %
(minrmsd_mm, self.params.stills.
isoforms[minindex].rmsd_target_mm))
assert minrmsd_mm < self.params.stills.isoforms[
minindex].rmsd_target_mm
logger.info("Acceptable rmsd for isoform %s." %
(self.params.stills.isoforms[minindex].name))
if len(self.params.stills.isoforms) == 2:
logger.info(
"Rmsd gain over the other isoform %5.1f um." %
(1000. *
abs(positional_rmsds[0] - positional_rmsds[1])))
R = refiners[minindex]
# Now one last check to see if direct beam is out of bounds
if self.params.stills.isoforms[
minindex].beam_restraint is not None:
from scitbx import matrix
refined_beam = matrix.col(
R.get_experiments()
[0].detector[0].get_beam_centre_lab(
experiments[0].beam.get_s0())[0:2])
known_beam = matrix.col(
self.params.stills.isoforms[minindex].
beam_restraint)
logger.info(
"Asserting difference in refined beam center and expected beam center %f < %f"
%
((refined_beam - known_beam).length(), self.params.
stills.isoforms[minindex].rmsd_target_mm))
assert (refined_beam - known_beam
).length() < self.params.stills.isoforms[
minindex].rmsd_target_mm
# future--circle of confusion could be given as a separate length in mm instead of reusing rmsd_target
experiment = R.get_experiments()[0]
experiment.crystal.identified_isoform = self.params.stills.isoforms[
minindex].name
isoform_experiments.append(experiment)
reflections['id'] = flex.int(len(reflections), expt_id)
isoform_reflections.extend(reflections)
experiments = isoform_experiments
reflections_for_refinement = isoform_reflections
try:
refined_experiments, refined_reflections = self.refine(
experiments, reflections_for_refinement)
except Exception as e:
s = str(e)
if len(experiments) == 1:
raise Sorry(e)
had_refinement_error = True
logger.info("Refinement failed:")
logger.info(s)
del experiments[-1]
break
# sanity check for unrealistic unit cell volume increase during refinement
# usually this indicates too many parameters are being refined given the
# number of observations provided.
if not self.params.refinement_protocol.disable_unit_cell_volume_sanity_check:
for orig_expt, refined_expt in zip(experiments,
refined_experiments):
uc1 = orig_expt.crystal.get_unit_cell()
uc2 = refined_expt.crystal.get_unit_cell()
volume_change = abs(uc1.volume() -
uc2.volume()) / uc1.volume()
cutoff = 0.5
if volume_change > cutoff:
msg = "\n".join((
"Unrealistic unit cell volume increase during refinement of %.1f%%.",
"Please try refining fewer parameters, either by enforcing symmetry",
"constraints (space_group=) and/or disabling experimental geometry",
"refinement (detector.fix=all and beam.fix=all). To disable this",
"sanity check set disable_unit_cell_volume_sanity_check=True."
)) % (100 * volume_change)
raise Sorry(msg)
self.refined_reflections = refined_reflections.select(
refined_reflections['id'] > -1)
for i, imageset in enumerate(self.imagesets):
ref_sel = self.refined_reflections.select(
self.refined_reflections['imageset_id'] == i)
ref_sel = ref_sel.select(ref_sel['id'] >= 0)
for i_expt in set(ref_sel['id']):
expt = refined_experiments[i_expt]
imageset.set_detector(expt.detector)
imageset.set_beam(expt.beam)
imageset.set_goniometer(expt.goniometer)
imageset.set_scan(expt.scan)
expt.imageset = imageset
if not (self.all_params.refinement.parameterisation.beam.fix
== 'all' and
self.all_params.refinement.parameterisation.detector.fix
== 'all'):
# Experimental geometry may have changed - re-map centroids to
# reciprocal space
spots_mm = self.reflections
self.reflections = flex.reflection_table()
for i, imageset in enumerate(self.imagesets):
spots_sel = spots_mm.select(spots_mm['imageset_id'] == i)
self.map_centroids_to_reciprocal_space(
spots_sel, imageset.get_detector(),
imageset.get_beam(), imageset.get_goniometer())
self.reflections.extend(spots_sel)
# update for next cycle
experiments = refined_experiments
self.refined_experiments = refined_experiments
if not 'refined_experiments' in locals():
raise Sorry("None of the experiments could refine.")
# discard experiments with zero reflections after refinement
id_set = set(self.refined_reflections['id'])
if len(id_set) < len(self.refined_experiments):
filtered_refined_reflections = flex.reflection_table()
for i in xrange(len(self.refined_experiments)):
if i not in id_set:
del self.refined_experiments[i]
for old, new in zip(sorted(id_set), range(len(id_set))):
subset = self.refined_reflections.select(
self.refined_reflections['id'] == old)
subset['id'] = flex.int(len(subset), new)
filtered_refined_reflections.extend(subset)
self.refined_reflections = filtered_refined_reflections
if len(self.refined_experiments) > 1:
from dials.algorithms.indexing.compare_orientation_matrices \
import show_rotation_matrix_differences
show_rotation_matrix_differences(
self.refined_experiments.crystals(), out=info_handle)
logger.info("Final refined crystal models:")
for i, crystal_model in enumerate(self.refined_experiments.crystals()):
n_indexed = 0
for i_expt in experiments.where(crystal=crystal_model):
n_indexed += (self.reflections['id'] == i).count(True)
logger.info("model %i (%i reflections):" % (i + 1, n_indexed))
logger.info(crystal_model)
if 'xyzcal.mm' in self.refined_reflections: # won't be there if refine_all_candidates = False and no isoforms
self.refined_reflections['xyzcal.px'] = flex.vec3_double(
len(self.refined_reflections))
for i, imageset in enumerate(self.imagesets):
imgset_sel = self.refined_reflections['imageset_id'] == i
# set xyzcal.px field in self.refined_reflections
refined_reflections = self.refined_reflections.select(
imgset_sel)
panel_numbers = flex.size_t(refined_reflections['panel'])
xyzcal_mm = refined_reflections['xyzcal.mm']
x_mm, y_mm, z_rad = xyzcal_mm.parts()
xy_cal_mm = flex.vec2_double(x_mm, y_mm)
xy_cal_px = flex.vec2_double(len(xy_cal_mm))
for i_panel in range(len(imageset.get_detector())):
panel = imageset.get_detector()[i_panel]
sel = (panel_numbers == i_panel)
isel = sel.iselection()
ref_panel = refined_reflections.select(
panel_numbers == i_panel)
xy_cal_px.set_selected(
sel, panel.millimeter_to_pixel(xy_cal_mm.select(sel)))
x_px, y_px = xy_cal_px.parts()
scan = imageset.get_scan()
if scan is not None:
z_px = scan.get_array_index_from_angle(z_rad, deg=False)
else:
# must be a still image, z centroid not meaningful
z_px = z_rad
xyzcal_px = flex.vec3_double(x_px, y_px, z_px)
self.refined_reflections['xyzcal.px'].set_selected(
imgset_sel, xyzcal_px)
self.map_centroids_to_reciprocal_space(
spots_sel, imageset.get_detector(), imageset.get_beam(),
imageset.get_goniometer())
self.reflections.extend(spots_sel)
# update for next cycle
experiments = refined_experiments
self.refined_experiments = refined_experiments
if not 'refined_experiments' in locals():
raise Sorry("None of the experiments could refine.")
if len(self.refined_experiments) > 1:
from dials.algorithms.indexing.compare_orientation_matrices \
import show_rotation_matrix_differences
show_rotation_matrix_differences(
self.refined_experiments.crystals(), out=info_handle)
info("Final refined crystal models:")
for i, crystal_model in enumerate(self.refined_experiments.crystals()):
n_indexed = 0
for i_expt in experiments.where(crystal=crystal_model):
n_indexed += (self.reflections['id'] == i).count(True)
info("model %i (%i reflections):" %(i+1, n_indexed))
info(crystal_model)
self.refined_reflections['xyzcal.px'] = flex.vec3_double(
len(self.refined_reflections))
for i, imageset in enumerate(self.imagesets):
imgset_sel = self.refined_reflections['imageset_id'] == i
# set xyzcal.px field in self.refined_reflections
refined_reflections = self.refined_reflections.select(imgset_sel)
self.map_centroids_to_reciprocal_space(
spots_sel, imageset.get_detector(),
imageset.get_beam(), imageset.get_goniometer())
self.reflections.extend(spots_sel)
# update for next cycle
experiments = refined_experiments
self.refined_experiments = refined_experiments
if not 'refined_experiments' in locals():
raise Sorry("None of the experiments could refine.")
if len(self.refined_experiments) > 1:
from dials.algorithms.indexing.compare_orientation_matrices \
import show_rotation_matrix_differences
show_rotation_matrix_differences(
self.refined_experiments.crystals(), out=info_handle)
logger.info("Final refined crystal models:")
for i, crystal_model in enumerate(self.refined_experiments.crystals()):
n_indexed = 0
for i_expt in experiments.where(crystal=crystal_model):
n_indexed += (self.reflections['id'] == i).count(True)
logger.info("model %i (%i reflections):" % (i + 1, n_indexed))
logger.info(crystal_model)
if 'xyzcal.mm' in self.refined_reflections: # won't be there if refine_all_candidates = False and no isoforms
self.refined_reflections['xyzcal.px'] = flex.vec3_double(
len(self.refined_reflections))
for i, imageset in enumerate(self.imagesets):
imgset_sel = self.refined_reflections['imageset_id'] == i
# set xyzcal.px field in self.refined_reflections
