def exercise():
from dials.algorithms.indexing import compare_orientation_matrices
from dxtbx.model import Crystal
from cctbx import sgtbx
from scitbx import matrix
from scitbx.math import euler_angles as euler
# try and see if we can get back the original rotation matrix and euler angles
real_space_a = matrix.col((10, 0, 0))
real_space_b = matrix.col((0, 10, 10))
real_space_c = matrix.col((0, 0, 10))
euler_angles = (1.3, 5.6, 7.8)
R = matrix.sqr(
euler.xyz_matrix(euler_angles[0], euler_angles[1], euler_angles[2]))
crystal_a = Crystal(real_space_a,
real_space_b,
real_space_c,
space_group=sgtbx.space_group('P 1'))
crystal_b = Crystal(R * real_space_a,
R * real_space_b,
R * real_space_c,
space_group=sgtbx.space_group('P 1'))
assert approx_equal(
matrix.sqr(crystal_b.get_U()) *
matrix.sqr(crystal_a.get_U()).transpose(), R)
best_R_ab, best_axis, best_angle, best_cb_op = \
compare_orientation_matrices.difference_rotation_matrix_axis_angle(
crystal_a,
crystal_b)
best_euler_angles = euler.xyz_angles(best_R_ab)
assert approx_equal(best_euler_angles, euler_angles)
assert best_cb_op.is_identity_op()
assert approx_equal(best_R_ab, R)
# now see if we can deconvolute the original euler angles after applying
# a change of basis to one of the crystals
crystal_a = Crystal(real_space_a,
real_space_b,
real_space_c,
space_group=sgtbx.space_group('I 2 3'))
crystal_b = Crystal(R * real_space_a,
R * real_space_b,
R * real_space_c,
space_group=sgtbx.space_group('I 2 3'))
cb_op = sgtbx.change_of_basis_op('z,x,y')
crystal_b = crystal_b.change_basis(cb_op)
best_R_ab, best_axis, best_angle, best_cb_op = \
compare_orientation_matrices.difference_rotation_matrix_axis_angle(
crystal_a,
crystal_b)
best_euler_angles = euler.xyz_angles(best_R_ab)
assert approx_equal(best_euler_angles, euler_angles)
assert best_cb_op.c() == cb_op.inverse().c()
assert approx_equal(best_R_ab, R)
def exercise():
from dials.algorithms.indexing import compare_orientation_matrices
from dxtbx.model.crystal import crystal_model
from cctbx import sgtbx
from scitbx import matrix
from scitbx.math import euler_angles as euler
# try and see if we can get back the original rotation matrix and euler angles
real_space_a = matrix.col((10,0,0))
real_space_b = matrix.col((0,10,10))
real_space_c = matrix.col((0,0,10))
euler_angles = (1.3, 5.6, 7.8)
R = matrix.sqr(
euler.xyz_matrix(euler_angles[0], euler_angles[1], euler_angles[2]))
crystal_a = crystal_model(real_space_a,
real_space_b,
real_space_c,
space_group=sgtbx.space_group('P 1'))
crystal_b = crystal_model(R * real_space_a,
R * real_space_b,
R * real_space_c,
space_group=sgtbx.space_group('P 1'))
assert approx_equal(crystal_b.get_U() * crystal_a.get_U().transpose(), R)
best_R_ab, best_axis, best_angle, best_cb_op = \
compare_orientation_matrices.difference_rotation_matrix_axis_angle(
crystal_a,
crystal_b)
best_euler_angles = euler.xyz_angles(best_R_ab)
assert approx_equal(best_euler_angles, euler_angles)
assert best_cb_op.is_identity_op()
assert approx_equal(best_R_ab, R)
# now see if we can deconvolute the original euler angles after applying
# a change of basis to one of the crystals
crystal_a = crystal_model(real_space_a,
real_space_b,
real_space_c,
space_group=sgtbx.space_group('I 2 3'))
crystal_b = crystal_model(R * real_space_a,
R * real_space_b,
R * real_space_c,
space_group=sgtbx.space_group('I 2 3'))
cb_op = sgtbx.change_of_basis_op('z,x,y')
crystal_b = crystal_b.change_basis(cb_op)
best_R_ab, best_axis, best_angle, best_cb_op = \
compare_orientation_matrices.difference_rotation_matrix_axis_angle(
crystal_a,
crystal_b)
best_euler_angles = euler.xyz_angles(best_R_ab)
assert approx_equal(best_euler_angles, euler_angles)
assert best_cb_op.c() == cb_op.inverse().c()
assert approx_equal(best_R_ab, R)
def test_compare_orientation_matrices():
# try and see if we can get back the original rotation matrix and euler angles
real_space_a = matrix.col((10, 0, 0))
real_space_b = matrix.col((0, 10, 10))
real_space_c = matrix.col((0, 0, 10))
euler_angles = (1.3, 5.6, 7.8)
R = matrix.sqr(
euler.xyz_matrix(euler_angles[0], euler_angles[1], euler_angles[2]))
crystal_a = Crystal(real_space_a,
real_space_b,
real_space_c,
space_group=sgtbx.space_group('P 1'))
crystal_b = Crystal(R * real_space_a,
R * real_space_b,
R * real_space_c,
space_group=sgtbx.space_group('P 1'))
assert (matrix.sqr(crystal_b.get_U()) *
matrix.sqr(crystal_a.get_U()).transpose()).elems == pytest.approx(
R.elems)
best_R_ab, best_axis, best_angle, best_cb_op = \
compare_orientation_matrices.difference_rotation_matrix_axis_angle(
crystal_a,
crystal_b)
best_euler_angles = euler.xyz_angles(best_R_ab)
assert best_euler_angles == pytest.approx(euler_angles)
assert best_cb_op.is_identity_op()
assert best_R_ab.elems == pytest.approx(R.elems)
# now see if we can deconvolute the original euler angles after applying
# a change of basis to one of the crystals
crystal_a = Crystal(real_space_a,
real_space_b,
real_space_c,
space_group=sgtbx.space_group('I 2 3'))
crystal_b = Crystal(R * real_space_a,
R * real_space_b,
R * real_space_c,
space_group=sgtbx.space_group('I 2 3'))
cb_op = sgtbx.change_of_basis_op('z,x,y')
crystal_b = crystal_b.change_basis(cb_op)
best_R_ab, best_axis, best_angle, best_cb_op = \
compare_orientation_matrices.difference_rotation_matrix_axis_angle(
crystal_a,
crystal_b)
best_euler_angles = euler.xyz_angles(best_R_ab)
assert best_euler_angles == pytest.approx(euler_angles)
assert best_cb_op.c() == cb_op.inverse().c()
assert best_R_ab.elems == pytest.approx(R.elems)
def _check_have_similar_crystal_models(self, experiments):
"""
Checks for similar crystal models.
Checks whether the most recently added crystal model is similar to previously
found crystal models, and if so, deletes the last crystal model from the
experiment list.
"""
have_similar_crystal_models = False
cryst_b = experiments.crystals()[-1]
for i_a, cryst_a in enumerate(experiments.crystals()[:-1]):
R_ab, axis, angle, cb_op_ab = difference_rotation_matrix_axis_angle(
cryst_a, cryst_b)
min_angle = self.params.multiple_lattice_search.minimum_angular_separation
if abs(angle) < min_angle: # degrees
logger.info(
"Crystal models too similar, rejecting crystal %i:" %
(len(experiments)))
logger.info(
"Rotation matrix to transform crystal %i to crystal %i" %
(i_a + 1, len(experiments)))
logger.info(R_ab)
logger.info("Rotation of %.3f degrees" % angle +
" about axis (%.3f, %.3f, %.3f)" % axis)
have_similar_crystal_models = True
del experiments[-1]
break
return have_similar_crystal_models
def compare_orientation_matrices(image):
serialno = image.lstrip("noiseimage_").rstrip(".img")
exp1 = ExperimentListFactory.from_json_file("experiments_" + serialno +
".json")
exp2 = ExperimentListFactory.from_json_file("indexed.expt")
R_ij, axis, angle, cb_op = difference_rotation_matrix_axis_angle(
exp1[0].crystal, exp2[0].crystal)
return abs(angle)
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
import libtbx.load_env
usage = "%s [options] experiments.json" % libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=False,
epilog=help_message,
)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) <= 1:
parser.print_help()
return
from dials.algorithms.indexing.compare_orientation_matrices import (
difference_rotation_matrix_axis_angle, )
crystals = []
for experiment in experiments:
crystal = experiment.crystal
if params.space_group is not None:
crystal.set_space_group(params.space_group.group())
crystals.append(crystal)
angles = flex.double()
import math
padding = int(math.ceil(math.log10(len(experiments))))
output_template = "%s%%0%ii.json" % (params.output.prefix, padding)
prev_expt = experiments[0]
for i in range(1, len(experiments)):
R_ij, axis, angle, cb_op = difference_rotation_matrix_axis_angle(
prev_expt.crystal, experiments[i].crystal)
angles.append(angle)
# print i, angle
if abs(angle) > params.max_deviation:
continue
experiments[i].crystal = experiments[i].crystal.change_basis(cb_op)
prev_expt = experiments[i]
from dxtbx.serialize import dump
dump.experiment_list(experiments[i:i + 1], output_template % i)
from matplotlib import pyplot
n, bins, patches = pyplot.hist(angles.as_numpy_array(), 100)
pyplot.show()
def filter_similar_orientations(crystal_models,
other_crystal_models,
minimum_angular_separation=5):
for cryst in crystal_models:
orientation_too_similar = False
for i_a, cryst_a in enumerate(other_crystal_models):
R_ab, axis, angle, cb_op_ab = difference_rotation_matrix_axis_angle(
cryst_a, cryst)
if abs(angle) < minimum_angular_separation: # degrees
orientation_too_similar = True
break
if orientation_too_similar:
logger.debug("skipping crystal: too similar to other crystals")
continue
yield cryst
def filter_doubled_cell(solutions):
accepted_solutions = []
for i1, s1 in enumerate(solutions):
doubled_cell = False
for (m1, m2, m3) in (
(2, 1, 1),
(1, 2, 1),
(1, 1, 2),
(2, 2, 1),
(2, 1, 2),
(1, 2, 2),
(2, 2, 2),
):
if doubled_cell:
break
a, b, c = (matrix.col(v)
for v in s1.crystal.get_real_space_vectors())
new_cryst = Crystal(
real_space_a=1 / m1 * a,
real_space_b=1 / m2 * b,
real_space_c=1 / m3 * c,
space_group=s1.crystal.get_space_group(),
)
new_unit_cell = new_cryst.get_unit_cell()
for s2 in solutions:
if s2 is s1:
continue
if new_unit_cell.is_similar_to(s2.crystal.get_unit_cell(),
relative_length_tolerance=0.05):
R, axis, angle, cb = difference_rotation_matrix_axis_angle(
new_cryst, s2.crystal)
if (angle < 1) and (s1.n_indexed < (1.1 * s2.n_indexed)):
doubled_cell = True
break
if not doubled_cell:
accepted_solutions.append(s1)
return accepted_solutions
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
# The script usage
usage = "dials.stereographic_projection [options] [param.phil] experiments.json"
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=False,
epilog=help_message,
)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if not experiments:
parser.print_help()
return
if not params.hkl and params.hkl_limit is None:
sys.exit("Please provide hkl or hkl_limit parameters.")
if params.hkl is not None and len(params.hkl):
miller_indices = flex.miller_index(params.hkl)
elif params.hkl_limit is not None:
limit = params.hkl_limit
miller_indices = flex.miller_index()
for h in range(-limit, limit + 1):
for k in range(-limit, limit + 1):
for l in range(-limit, limit + 1):
if (h, k, l) == (0, 0, 0):
continue
miller_indices.append((h, k, l))
crystals = experiments.crystals()
symmetry = crystal.symmetry(
unit_cell=crystals[0].get_unit_cell(), space_group=crystals[0].get_space_group()
)
miller_set = miller.set(symmetry, miller_indices)
d_spacings = miller_set.d_spacings()
if params.eliminate_sys_absent:
d_spacings = d_spacings.eliminate_sys_absent()
if params.expand_to_p1:
d_spacings = d_spacings.as_non_anomalous_array().expand_to_p1()
d_spacings = d_spacings.generate_bijvoet_mates()
miller_indices = d_spacings.indices()
# find the greatest common factor (divisor) between miller indices
miller_indices_unique = flex.miller_index()
for hkl in miller_indices:
gcd = gcd_list(hkl)
if gcd > 1:
miller_indices_unique.append(tuple(int(h / gcd) for h in hkl))
elif gcd < 1:
pass
else:
miller_indices_unique.append(hkl)
miller_indices = miller_indices_unique
miller_indices = flex.miller_index(list(set(miller_indices)))
ref_crystal = crystals[0]
U = matrix.sqr(ref_crystal.get_U())
B = matrix.sqr(ref_crystal.get_B())
R = matrix.identity(3)
if params.frame == "laboratory":
reference_poles = reference_poles_perpendicular_to_beam(
experiments[0].beam, experiments[0].goniometer
)
if params.use_starting_angle:
rotation_axis = matrix.col(experiments[0].goniometer.get_rotation_axis())
R = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
experiments[0].scan.get_oscillation()[0], deg=True
)
elif params.phi_angle != 0:
rotation_axis = matrix.col(experiments[0].goniometer.get_rotation_axis())
R = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
params.phi_angle, deg=True
)
else:
if params.plane_normal is not None:
plane_normal = params.plane_normal
else:
plane_normal = (0, 0, 1)
reference_poles = reference_poles_crystal(
ref_crystal, plane_normal=plane_normal
)
if params.frame == "crystal":
U = matrix.identity(3)
reciprocal_space_points = list(R * U * B) * miller_indices.as_vec3_double()
projections_ref = stereographic_projection(reciprocal_space_points, reference_poles)
projections_all = [projections_ref]
if experiments:
from dials.algorithms.indexing.compare_orientation_matrices import (
difference_rotation_matrix_axis_angle,
)
for expt in experiments[1:]:
cryst = expt.crystal
if params.frame == "crystal":
R_ij, axis, angle, cb_op = difference_rotation_matrix_axis_angle(
ref_crystal, cryst
)
U = R_ij
elif params.use_starting_angle:
if params.use_starting_angle:
rotation_axis = matrix.col(expt.goniometer.get_rotation_axis())
R = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
expt.scan.get_oscillation()[0], deg=True
)
else:
U = matrix.sqr(cryst.get_U())
reciprocal_space_points = (
list(R * U * matrix.sqr(cryst.get_B()))
* miller_indices.as_vec3_double()
)
projections = stereographic_projection(
reciprocal_space_points, reference_poles
)
projections_all.append(projections)
if params.save_coordinates:
with open("projections.txt", "w") as f:
f.write("crystal h k l x y" + os.linesep)
for i_cryst, projections in enumerate(projections_all):
for hkl, proj in zip(miller_indices, projections):
f.write("%i " % (i_cryst + 1))
f.write("%i %i %i " % hkl)
f.write(("%f %f" + os.linesep) % proj)
if params.plot.show or params.plot.filename:
epochs = None
if params.plot.colour_map is not None:
if experiments[0].scan is not None:
epochs = [expt.scan.get_epochs()[0] for expt in experiments]
else:
epochs = [i for i, expt in enumerate(experiments)]
plot_projections(
projections_all,
filename=params.plot.filename,
show=params.plot.show,
colours=params.plot.colours,
marker_size=params.plot.marker_size,
font_size=params.plot.font_size,
gridsize=params.plot.gridsize,
label_indices=params.plot.label_indices,
epochs=epochs,
colour_map=params.plot.colour_map,
)
if params.json.filename:
projections_as_json(projections_all, filename=params.json.filename)
def get_uc_consensus(experiments_list,
show_plot=False,
save_plot=False,
return_only_first_indexed_model=False,
finalize_method='reindex_with_known_crystal_models',
clustering_params=None):
'''
Uses the Rodriguez Laio 2014 method to do a hierarchical clustering of the crystal models and
then vote for the highest consensus crystal mode. Input needs to be a list of experiments object.
Clustering code taken from github.com/cctbx-xfel/cluster_regression
Clustering is first done first based on unit cell dimensions. Then for each of the clusters identified,
a further clustering is done based on orientational matrix A
'''
if return_only_first_indexed_model:
return [experiments_list[0].crystals()[0]], None
cells = []
from xfel.clustering.singleframe import CellOnlyFrame
# Flag for testing Lysozyme data from NKS.Make sure cluster_regression repository is present and configured
# Program will exit after plots are displayed if this flag is true
test_nks = False
if clustering_params is None:
clustering_params = clustering_iota_scope
if test_nks:
from cctbx import crystal
import libtbx.load_env
cluster_regression = libtbx.env.find_in_repositories(
relative_path="cluster_regression", test=os.path.isdir)
file_name = os.path.join(cluster_regression, 'examples',
'lysozyme1341.txt')
for line in open(file_name, "r").xreadlines():
tokens = line.strip().split()
unit_cell = tuple(float(x) for x in tokens[0:6])
space_group_symbol = tokens[6]
crystal_symmetry = crystal.symmetry(
unit_cell=unit_cell, space_group_symbol=space_group_symbol)
cells.append(CellOnlyFrame(crystal_symmetry))
else:
clustered_experiments_list = flex.int()
for experiment in experiments_list:
if len(experiment.crystals()) > 1:
print('IOTA:Should have only one crystal model')
crystal_symmetry = experiment.crystals()[0].get_crystal_symmetry()
cells.append(CellOnlyFrame(crystal_symmetry))
# Maintain a list which is meaningless right now that will finally contain the
# final clustering results
clustered_experiments_list.append(-1)
MM = [c.mm for c in cells] # metrical matrices
MM_double = flex.double()
for i in range(len(MM)):
Tup = MM[i]
for j in range(6):
MM_double.append(Tup[j])
print('There are %d cells' % len(MM))
coord_x = flex.double([c.uc[0] for c in cells])
coord_y = flex.double([c.uc[1] for c in cells])
if show_plot or save_plot:
import matplotlib
if not show_plot:
matplotlib.use('Agg')
import matplotlib.pyplot as plt
plt.plot([c.uc[0] for c in cells], [c.uc[1] for c in cells],
"k.",
markersize=3.)
plt.axes().set_aspect("equal")
if save_plot:
plot_name = 'uc_cluster.png'
plt.savefig(plot_name,
size_inches=(10, 10),
dpi=300,
bbox_inches='tight')
if show_plot:
plt.show()
print('Now constructing a Dij matrix: Starting Unit Cell clustering')
NN = len(MM)
from cctbx.uctbx.determine_unit_cell import NCDist_flatten
Dij = NCDist_flatten(MM_double)
from scitbx.math import five_number_summary
d_c = clustering_params.d_c #five_number_summary(list(Dij))[1]
d_c = estimate_d_c(Dij)
#d_c = flex.mean_and_variance(Dij.as_1d()).unweighted_sample_standard_deviation()
print('d_c = ', d_c)
if len(cells) < 5:
return [experiments_list[0].crystals()[0]], None
CM = clustering_manager(
Dij=Dij,
d_c=d_c,
max_percentile_rho=clustering_params.max_percentile_rho_uc,
Z_delta=clustering_params.Z_delta,
strategy='strategy_3')
n_cluster = 1 + flex.max(CM.cluster_id_final)
print(len(cells), ' datapoints have been analyzed')
print('%d CLUSTERS' % n_cluster)
for i in range(n_cluster):
item = flex.first_index(CM.cluster_id_maxima, i)
print('Cluster %d central Unit cell = %d' % (i, item))
cells[item].crystal_symmetry.show_summary()
# More plots for debugging
appcolors = [
'b', 'r', '#ff7f0e', '#2ca02c', '#9467bd', '#8c564b', '#e377c2',
'#7f7f7f', '#bcbd22', '#17becf'
]
if show_plot:
# Decision graph
import matplotlib.pyplot as plt
plt.plot(CM.rho, CM.delta, "r.", markersize=3.)
for x in range(NN):
if CM.cluster_id_maxima[x] >= 0:
plt.plot([CM.rho[x]], [CM.delta[x]], "ro")
plt.show()
if show_plot:
import matplotlib.pyplot as plt
colors = [appcolors[i % 10] for i in CM.cluster_id_full]
plt.scatter(coord_x,
coord_y,
marker='o',
color=colors,
linewidth=0.4,
edgecolor='k')
for i in range(n_cluster):
item = flex.first_index(CM.cluster_id_maxima, i)
plt.plot([cells[item].uc[0]], cells[item].uc[1], 'y.')
plt.axes().set_aspect("equal")
plt.show()
if test_nks:
exit()
# Now look at each unit cell cluster for orientational clustering
# idea is to cluster the orientational component in each of the unit cell clusters
#
do_orientational_clustering = not return_only_first_indexed_model # temporary.
dxtbx_crystal_models = []
if do_orientational_clustering:
print('IOTA: Starting orientational clustering')
Dij_ori = {} # dictionary to store Dij for each cluster
uc_experiments_list = {
} # dictionary to store experiments_lists for each cluster
from collections import Counter
uc_cluster_count = Counter(list(CM.cluster_id_final))
# instantiate the Dij_ori flat 1-d array
# Put all experiments list from same uc cluster together
if True:
from scitbx.matrix import sqr
from cctbx_orientation_ext import crystal_orientation
crystal_orientation_list = []
all_A = []
for i in range(len(experiments_list)):
crystal_orientation_list.append(
crystal_orientation(
experiments_list[i].crystals()[0].get_A(), True))
#exit()
A_direct = sqr(crystal_orientation_list[i].reciprocal_matrix()
).transpose().inverse()
all_A.append(A_direct[0])
#print ("Direct A matrix 1st element = %12.6f %12.6f %12.6f"%(A_direct[0], A_direct[1], A_direct[2]))
# exit()
CM_mapping = {}
for i in range(len(experiments_list)):
if CM.cluster_id_full[i] not in uc_experiments_list:
uc_experiments_list[CM.cluster_id_full[i]] = []
CM_mapping[CM.cluster_id_full[i]] = []
uc_experiments_list[CM.cluster_id_full[i]].append(
experiments_list[i])
# Maintain mapping between original experiments_list and uc_exeriments_list
# Mapping: key> index_in_experiments_list | value> cluster_id, index_in_uc_cluster
CM_mapping[CM.cluster_id_full[i]].append(
(i, len(uc_experiments_list[CM.cluster_id_full[i]]) - 1))
for cluster in uc_cluster_count:
# Make sure there are atleast a minimum number of samples in the cluster
if uc_cluster_count[cluster] < clustering_params.min_datapts:
continue
Dij_ori[cluster] = flex.double(
[[0.0] * uc_cluster_count[cluster]] *
uc_cluster_count[cluster])
# Now populate the Dij_ori array
N_samples_in_cluster = len(uc_experiments_list[cluster])
for i in range(N_samples_in_cluster - 1):
for j in range(i + 1, N_samples_in_cluster):
dij_ori = get_dij_ori(
uc_experiments_list[cluster][i].crystals()[0],
uc_experiments_list[cluster][j].crystals()[0])
A_direct_i = sqr(
uc_experiments_list[cluster][i].crystals()
[0].get_A()).transpose().inverse()
A_direct_j = sqr(
uc_experiments_list[cluster][j].crystals()
[0].get_A()).transpose().inverse()
#print ("Direct A matrix 1st element = %12.6f %12.6f %12.6f %12.6f %12.6f %12.6f %12.6f"%(dij_ori, A_direct_i[0], A_direct_j[0], A_direct_i[1],A_direct_j[1], A_direct_i[2], A_direct_j[2] ))
Dij_ori[cluster][N_samples_in_cluster * i + j] = dij_ori
Dij_ori[cluster][N_samples_in_cluster * j + i] = dij_ori
# Now do the orientational cluster analysis
d_c_ori = clustering_params.d_c_ori # 0.13
from exafel_project.ADSE13_25.clustering.plot_with_dimensional_embedding import plot_with_dimensional_embedding
#plot_with_dimensional_embedding(1-Dij_ori[1]/flex.max(Dij_ori[1]), show_plot=True)
A_matrices = []
for cluster in Dij_ori:
#if cluster == 2:
# CM_ori = clustering_manager(Dij=Dij_ori[cluster], d_c=d_c_ori, max_percentile_rho=0.85, debug=True)
d_c_ori = estimate_d_c(Dij_ori[cluster])
#else:
#d_c_ori=flex.mean_and_variance(Dij_ori[cluster].as_1d()).unweighted_sample_standard_deviation()
print('d_c_ori=', d_c_ori)
CM_ori = clustering_manager(
Dij=Dij_ori[cluster],
d_c=d_c_ori,
max_percentile_rho=clustering_params.max_percentile_rho_ori,
Z_delta=clustering_params.Z_delta,
strategy='strategy_3')
n_cluster_ori = 1 + flex.max(CM_ori.cluster_id_final)
#from IPython import embed; embed(); exit()
for i in range(n_cluster_ori):
if len([zz for zz in CM_ori.cluster_id_final if zz == i
]) < clustering_params.min_datapts:
continue
item = flex.first_index(CM_ori.cluster_id_maxima, i)
dxtbx_crystal_model = uc_experiments_list[cluster][
item].crystals()[0]
dxtbx_crystal_models.append(dxtbx_crystal_model)
# Map the orientational clusters to the original experiments_list indices
# This should be the final list of clusters!
for j, ori_cluster_id in enumerate(CM_ori.cluster_id_final):
if ori_cluster_id == i:
xx, yy = CM_mapping[cluster][j]
clustered_experiments_list[xx] = len(
dxtbx_crystal_models) - 1
from scitbx.matrix import sqr
from cctbx_orientation_ext import crystal_orientation
crystal_orientation = crystal_orientation(
dxtbx_crystal_model.get_A(), True)
A_direct = sqr(crystal_orientation.reciprocal_matrix()
).transpose().inverse()
A_matrices.append(A_direct)
print(
"IOTA: Direct A matrix 1st element of orientational cluster %d = %12.6f"
% (i, A_direct[0]))
print(A_direct)
if show_plot:
# Decision graph
stretch_plot_factor = 1.05 # (1+fraction of limits by which xlim,ylim should be set)
import matplotlib.pyplot as plt
plt.plot(CM_ori.rho, CM_ori.delta, "r.", markersize=3.)
for x in range(len(list(CM_ori.cluster_id_final))):
if CM_ori.cluster_id_maxima[x] >= 0:
plt.plot([CM_ori.rho[x]], [CM_ori.delta[x]], "ro")
#exit()
plt.xlim([-10, stretch_plot_factor * flex.max(CM_ori.rho)])
plt.ylim([-10, stretch_plot_factor * flex.max(CM_ori.delta)])
plt.show()
# FIXME Still to be worked out what exactly should be returned
#if return_only_first_indexed_model:
# return [experiments_list[0].crystals()[0]], clustered_experiments_list
# Make sure the crystal models are not too close to each other
# FIXME should be a PHIL
#from IPython import embed; embed(); exit()
min_angle = 5.0 # taken from indexer.py
close_models_list = []
# Not used really; other fixes have been made to code to figure out outliers
# Still keeping this in case it it useful later on.
if len(dxtbx_crystal_models) > 10000:
from dials.algorithms.indexing.compare_orientation_matrices import difference_rotation_matrix_axis_angle
from cctbx_orientation_ext import crystal_orientation
from dxtbx.model import Crystal
for i_a in range(0, len(dxtbx_crystal_models) - 1):
for i_b in range(i_a + 1, len(dxtbx_crystal_models)):
cryst_a = dxtbx_crystal_models[i_a]
cryst_b = dxtbx_crystal_models[i_b]
cryst_a_ori = crystal_orientation(cryst_a.get_A(), True)
cryst_b_ori = crystal_orientation(cryst_b.get_A(), True)
try:
best_similarity_transform = cryst_b_ori.best_similarity_transformation(
other=cryst_a_ori,
fractional_length_tolerance=20.00,
unimodular_generator_range=1)
cryst_b_ori_best = cryst_b_ori.change_basis(
best_similarity_transform)
except Exception as e:
cryst_b_ori_best = cryst_b_ori
# FIXME hardcoded space group for myoglobin LS49
cryst_b_best = Crystal(cryst_b_ori_best.direct_matrix()[0:3],
cryst_b_ori_best.direct_matrix()[3:6],
cryst_b_ori_best.direct_matrix()[6:9],
'P 1 21 1')
R_ab, axis, angle, cb_op_ab = difference_rotation_matrix_axis_angle(
cryst_a, cryst_b_best)
# FIXME
if abs(angle) < min_angle: # degrees
close_models_list.append((i_a, i_b))
# Now prune the dxtbx_crystal_models list
unique_experiments_list = flex.int(range(len(dxtbx_crystal_models)))
for close_models in close_models_list:
i_a, i_b = close_models
if dxtbx_crystal_models[i_a] is not None and dxtbx_crystal_models[
i_b] is not None:
dxtbx_crystal_models[i_b] = None
unique_experiments_list[i_b] = i_a
clustered_experiments_list.set_selected(
clustered_experiments_list == i_b, i_a)
counter = -1
for ii, model in enumerate(dxtbx_crystal_models):
if model is not None:
counter += 1
clustered_experiments_list.set_selected(
clustered_experiments_list == unique_experiments_list[ii],
counter)
dxtbx_crystal_models = [
x for x in dxtbx_crystal_models if x is not None
]
#from IPython import embed; embed(); exit()
if len(dxtbx_crystal_models) > 0:
return dxtbx_crystal_models, list(clustered_experiments_list)
else:
# If nothing works, atleast return the 1st crystal model that was found
return [experiments_list[0].crystals()[0]], None
def test_compare_orientation_matrices():
# try and see if we can get back the original rotation matrix and euler angles
real_space_a = matrix.col((10, 0, 0))
real_space_b = matrix.col((0, 10, 10))
real_space_c = matrix.col((0, 0, 10))
euler_angles = (1.3, 5.6, 7.8)
R = matrix.sqr(
euler.xyz_matrix(euler_angles[0], euler_angles[1], euler_angles[2]))
crystal_a = Crystal(real_space_a,
real_space_b,
real_space_c,
space_group=sgtbx.space_group("P 1"))
crystal_b = Crystal(
R * real_space_a,
R * real_space_b,
R * real_space_c,
space_group=sgtbx.space_group("P 1"),
)
assert (matrix.sqr(crystal_b.get_U()) *
matrix.sqr(crystal_a.get_U()).transpose()).elems == pytest.approx(
R.elems)
(
best_R_ab,
best_axis,
best_angle,
best_cb_op,
) = compare_orientation_matrices.difference_rotation_matrix_axis_angle(
crystal_a, crystal_b)
best_euler_angles = euler.xyz_angles(best_R_ab)
assert best_euler_angles == pytest.approx(euler_angles)
assert best_cb_op.is_identity_op()
assert best_R_ab.elems == pytest.approx(R.elems)
# now see if we can deconvolute the original euler angles after applying
# a change of basis to one of the crystals
crystal_a = Crystal(real_space_a,
real_space_b,
real_space_c,
space_group=sgtbx.space_group("I 2 3"))
cb_op = sgtbx.change_of_basis_op("z,x,y")
crystal_b = Crystal(
R * real_space_a,
R * real_space_b,
R * real_space_c,
space_group=sgtbx.space_group("I 2 3"),
).change_basis(cb_op)
(
best_R_ab,
best_axis,
best_angle,
best_cb_op,
) = compare_orientation_matrices.difference_rotation_matrix_axis_angle(
crystal_a, crystal_b)
best_euler_angles = euler.xyz_angles(best_R_ab)
assert best_euler_angles == pytest.approx(euler_angles)
assert best_cb_op.c() == cb_op.inverse().c()
assert best_R_ab.elems == pytest.approx(R.elems)
crystal_c = crystal_b.change_basis(sgtbx.change_of_basis_op("-y,-z,x"))
assert crystal_c != crystal_b
s = compare_orientation_matrices.rotation_matrix_differences(
[crystal_a, crystal_b, crystal_c], comparison="pairwise")
s = "\n".join(s.splitlines()[:-1]).replace("-0.000", "0.000")
print(s)
assert (s == """\
Change of basis op: b,c,a
Rotation matrix to transform crystal 1 to crystal 2:
{{0.986, -0.135, 0.098},
{0.138, 0.990, -0.023},
{-0.094, 0.036, 0.995}}
Rotation of 9.738 degrees about axis (0.172, 0.565, 0.807)
Change of basis op: -a,-b,c
Rotation matrix to transform crystal 1 to crystal 3:
{{0.986, -0.135, 0.098},
{0.138, 0.990, -0.023},
{-0.094, 0.036, 0.995}}
Rotation of 9.738 degrees about axis (0.172, 0.565, 0.807)
Change of basis op: c,-a,-b
Rotation matrix to transform crystal 2 to crystal 3:
{{1.000, 0.000, 0.000},
{0.000, 1.000, 0.000},
{0.000, 0.000, 1.000}}""")
s = compare_orientation_matrices.rotation_matrix_differences(
[crystal_a, crystal_b, crystal_c], comparison="sequential")
s = "\n".join(s.splitlines()[:-1]).replace("-0.000", "0.000")
print(s)
assert (s == """\
Change of basis op: b,c,a
Rotation matrix to transform crystal 1 to crystal 2:
{{0.986, -0.135, 0.098},
{0.138, 0.990, -0.023},
{-0.094, 0.036, 0.995}}
Rotation of 9.738 degrees about axis (0.172, 0.565, 0.807)
Change of basis op: c,-a,-b
Rotation matrix to transform crystal 2 to crystal 3:
{{1.000, 0.000, 0.000},
{0.000, 1.000, 0.000},
{0.000, 0.000, 1.000}}""")
s = compare_orientation_matrices.rotation_matrix_differences(
(crystal_a, crystal_b), miller_indices=((1, 0, 0), (1, 1, 0)))
assert (s == """\
Change of basis op: b,c,a
Rotation matrix to transform crystal 1 to crystal 2:
{{0.986, -0.135, 0.098},
{0.138, 0.990, -0.023},
{-0.094, 0.036, 0.995}}
Rotation of 9.738 degrees about axis (0.172, 0.565, 0.807)
(1,0,0): 15.26 deg
(1,1,0): 9.12 deg
""")
def test_indexed_hkl():
'''tests the uniqueness of hkl values associated with each experiment for
100 simulated randomly oriented thermolysin diffraction images indexed using
two color indexer'''
flex.set_random_seed(42)
known_symmetry = crystal.symmetry("78,78,37,90,90,90", "P43212")
detector = detector_factory.simple('SENSOR_UNKNOWN', 125, (97.075, 97.075),
'+x', '-y', (0.11, 0.11), (1765, 1765))
wavelength1 = 12398 / 7400 #wavelength for 2 color experiment in Angstroms
wavelength2 = 12398 / 7500 #wavelength for 2 color experiment in Angstroms
beam1 = beam_factory.simple_directional((0, 0, 1), wavelength1)
beam2 = beam_factory.simple_directional((0, 0, 1), wavelength2)
a_basis = []
b_basis = []
c_basis = []
# refiner resets random number seed so in order to get the same 100 images
#generated each time the random seed is set
# the implementation is as follows
# gets simulated images
sims = [merge_close_spots.merge_close_spots() for i in range(2)]
for data in sims:
A = data.A
A_inv = A.inverse()
a = col(A_inv[:3])
b = col(A_inv[3:6])
c = col(A_inv[6:])
crystinp = Crystal(a, b, c, space_group=known_symmetry.space_group())
a_basis.append(a)
b_basis.append(b)
c_basis.append(c)
res = data.two_color_sim
info = data.spot_proximity(res)
refl = info[0]
result = index(refl, detector, known_symmetry, [beam1, beam2])
cm = result.refined_experiments.crystals()[0]
R, best_axis, best_angle, change_of_basis = difference_rotation_matrix_axis_angle(
crystal_a=cm, crystal_b=crystinp)
# cmd_line = command_line.argument_interpreter(master_params=master_phil_scope)
# working_phil = cmd_line.process_and_fetch(args=[])
params = master_phil_scope.extract()
params.refinement.parameterisation.beam.fix = "all"
params.refinement.parameterisation.detector.fix = "all"
params.indexing.known_symmetry.space_group = known_symmetry.space_group_info(
)
params.refinement.verbosity = 3
params.indexing.refinement_protocol.d_min_start = 3
params.indexing.refinement_protocol.n_macro_cycles = 1
params.indexing.known_symmetry.unit_cell = known_symmetry.unit_cell()
params.indexing.multiple_lattice_search.max_lattices = 1
params.indexing.debug = True
params.indexing.known_symmetry.absolute_angle_tolerance = 5.0
params.indexing.known_symmetry.relative_length_tolerance = 0.3
params.indexing.stills.rmsd_min_px = 3.5
expts = copy.deepcopy(result.refined_experiments)
expts.crystals()[0].change_basis(change_of_basis)
reflections_exp0 = result.refined_reflections.select(
result.refined_reflections['id'] == 0)
reflections_exp1 = result.refined_reflections.select(
result.refined_reflections['id'] == 1)
assert len(reflections_exp0['miller_index']) == len(
set(reflections_exp0['miller_index']))
assert len(reflections_exp1['miller_index']) == len(
set(reflections_exp1['miller_index']))
print "OK"
def run(args):
import libtbx.load_env
from libtbx.utils import Sorry
usage = "%s [options] experiments.json indexed.pickle" % libtbx.env.dispatcher_name
parser = OptionParser(usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0 and len(reflections) == 0:
parser.print_help()
return
if params.change_of_basis_op is None:
raise Sorry("Please provide a change_of_basis_op.")
reference_crystal = None
if params.reference is not None:
from dxtbx.serialize import load
reference_experiments = load.experiment_list(params.reference,
check_format=False)
assert len(reference_experiments.crystals()) == 1
reference_crystal = reference_experiments.crystals()[0]
if len(experiments) and params.change_of_basis_op is libtbx.Auto:
if reference_crystal is not None:
if len(experiments.crystals()) > 1:
raise Sorry("Only one crystal can be processed at a time")
from dials.algorithms.indexing.compare_orientation_matrices \
import difference_rotation_matrix_axis_angle
cryst = experiments.crystals()[0]
R, axis, angle, change_of_basis_op = difference_rotation_matrix_axis_angle(
cryst, reference_crystal)
print("Change of basis op: %s" % change_of_basis_op)
print("Rotation matrix to transform input crystal to reference::")
print(R.mathematica_form(format="%.3f", one_row_per_line=True))
print("Rotation of %.3f degrees" % angle,
"about axis (%.3f, %.3f, %.3f)" % axis)
elif len(reflections):
assert len(reflections) == 1
# always re-map reflections to reciprocal space
from dials.algorithms.indexing import indexer
refl_copy = flex.reflection_table()
for i, imageset in enumerate(experiments.imagesets()):
if 'imageset_id' in reflections[0]:
sel = (reflections[0]['imageset_id'] == i)
else:
sel = (reflections[0]['id'] == i)
refl = indexer.indexer_base.map_spots_pixel_to_mm_rad(
reflections[0].select(sel), imageset.get_detector(),
imageset.get_scan())
indexer.indexer_base.map_centroids_to_reciprocal_space(
refl, imageset.get_detector(), imageset.get_beam(),
imageset.get_goniometer())
refl_copy.extend(refl)
# index the reflection list using the input experiments list
refl_copy['id'] = flex.int(len(refl_copy), -1)
from dials.algorithms.indexing import index_reflections
index_reflections(refl_copy, experiments, tolerance=0.2)
hkl_expt = refl_copy['miller_index']
hkl_input = reflections[0]['miller_index']
change_of_basis_op = derive_change_of_basis_op(hkl_input, hkl_expt)
# reset experiments list since we don't want to reindex this
experiments = []
else:
change_of_basis_op = sgtbx.change_of_basis_op(
params.change_of_basis_op)
if len(experiments):
for crystal in experiments.crystals():
cryst_orig = copy.deepcopy(crystal)
cryst_reindexed = cryst_orig.change_basis(change_of_basis_op)
if params.space_group is not None:
a, b, c = cryst_reindexed.get_real_space_vectors()
cryst_reindexed = Crystal(
a, b, c, space_group=params.space_group.group())
crystal.update(cryst_reindexed)
print("Old crystal:")
print(cryst_orig)
print()
print("New crystal:")
print(cryst_reindexed)
print()
print("Saving reindexed experimental models to %s" %
params.output.experiments)
dump.experiment_list(experiments, params.output.experiments)
if len(reflections):
assert (len(reflections) == 1)
reflections = reflections[0]
miller_indices = reflections['miller_index']
if params.hkl_offset is not None:
h, k, l = miller_indices.as_vec3_double().parts()
h += params.hkl_offset[0]
k += params.hkl_offset[1]
l += params.hkl_offset[2]
miller_indices = flex.miller_index(h.iround(), k.iround(),
l.iround())
non_integral_indices = change_of_basis_op.apply_results_in_non_integral_indices(
miller_indices)
if non_integral_indices.size() > 0:
print(
"Removing %i/%i reflections (change of basis results in non-integral indices)"
% (non_integral_indices.size(), miller_indices.size()))
sel = flex.bool(miller_indices.size(), True)
sel.set_selected(non_integral_indices, False)
miller_indices_reindexed = change_of_basis_op.apply(
miller_indices.select(sel))
reflections['miller_index'].set_selected(sel, miller_indices_reindexed)
reflections['miller_index'].set_selected(~sel, (0, 0, 0))
print("Saving reindexed reflections to %s" % params.output.reflections)
easy_pickle.dump(params.output.reflections, reflections)
def get_uc_consensus(experiments_list,
show_plot=False,
return_only_first_indexed_model=False,
finalize_method=None,
clustering_params=None):
'''
Uses the Rodriguez Laio 2014 method to do a clustering of the unit cells and then vote for the highest
consensus unit cell. Input needs to be a list of experiments object.
Clustering code taken from github.com/cctbx-xfel/cluster_regression
Returns an experiment object with crystal unit cell from the cluster with the most points
'''
if return_only_first_indexed_model:
return [experiments_list[0].crystals()[0]], None
cells = []
from xfel.clustering.singleframe import CellOnlyFrame
save_plot = False
# Flag for testing Lysozyme data from NKS.Make sure cluster_regression repository is present and configured
# Program will exit after plots are displayed if this flag is true
test_nks = False
if test_nks:
from cctbx import crystal
import libtbx.load_env
cluster_regression = libtbx.env.find_in_repositories(
relative_path="cluster_regression", test=os.path.isdir)
file_name = os.path.join(cluster_regression, 'examples',
'lysozyme1341.txt')
for line in open(file_name, "r").xreadlines():
tokens = line.strip().split()
unit_cell = tuple(float(x) for x in tokens[0:6])
space_group_symbol = tokens[6]
crystal_symmetry = crystal.symmetry(
unit_cell=unit_cell, space_group_symbol=space_group_symbol)
cells.append(CellOnlyFrame(crystal_symmetry))
else:
for experiment in experiments_list:
if len(experiment.crystals()) > 1:
print('IOTA:Should have only one crystal model')
crystal_symmetry = experiment.crystals()[0].get_crystal_symmetry()
cells.append(CellOnlyFrame(crystal_symmetry))
MM = [c.mm for c in cells] # metrical matrices
MM_double = flex.double()
for i in range(len(MM)):
Tup = MM[i]
for j in range(6):
MM_double.append(Tup[j])
print('There are %d cells' % len(MM))
coord_x = flex.double([c.uc[0] for c in cells])
coord_y = flex.double([c.uc[1] for c in cells])
if show_plot or save_plot:
import matplotlib
if not show_plot:
matplotlib.use('Agg')
import matplotlib.pyplot as plt
#from IPython import embed; embed(); exit()
plt.plot([c.uc[0] for c in cells], [c.uc[1] for c in cells],
"k.",
markersize=3.)
plt.axes().set_aspect("equal")
if save_plot:
plot_name = 'uc_cluster.png'
plt.savefig(plot_name,
size_inches=(10, 10),
dpi=300,
bbox_inches='tight')
if show_plot:
plt.show()
print('Now constructing a Dij matrix: Starting Unit Cell clustering')
NN = len(MM)
from cctbx.uctbx.determine_unit_cell import NCDist_flatten
Dij = NCDist_flatten(MM_double)
d_c = flex.mean_and_variance(
Dij.as_1d()).unweighted_sample_standard_deviation() #6.13
#FIXME should be a PHIL param
if len(cells) < 5:
return [experiments_list[0].crystals()[0]], None
CM = clustering_manager(Dij=Dij, d_c=d_c, max_percentile_rho=0.95)
n_cluster = 1 + flex.max(CM.cluster_id_final)
print(len(cells), ' datapoints have been analyzed')
print('%d CLUSTERS' % n_cluster)
for i in range(n_cluster):
item = flex.first_index(CM.cluster_id_maxima, i)
print('Cluster %d central Unit cell = %d' % (i, item))
cells[item].crystal_symmetry.show_summary()
# More plots for debugging
appcolors = [
'b', 'r', '#ff7f0e', '#2ca02c', '#9467bd', '#8c564b', '#e377c2',
'#7f7f7f', '#bcbd22', '#17becf'
]
if show_plot:
# Decision graph
import matplotlib.pyplot as plt
plt.plot(CM.rho, CM.delta, "r.", markersize=3.)
for x in range(NN):
if CM.cluster_id_maxima[x] >= 0:
plt.plot([CM.rho[x]], [CM.delta[x]], "ro")
plt.show()
if show_plot:
import matplotlib.pyplot as plt
colors = [appcolors[i % 10] for i in CM.cluster_id_full]
plt.scatter(coord_x,
coord_y,
marker='o',
color=colors,
linewidth=0.4,
edgecolor='k')
for i in range(n_cluster):
item = flex.first_index(CM.cluster_id_maxima, i)
plt.plot([cells[item].uc[0]], cells[item].uc[1], 'y.')
plt.axes().set_aspect("equal")
plt.show()
if test_nks:
exit()
# Now look at each unit cell cluster for orientational clustering
# idea is to cluster the orientational component in each of the unit cell clusters
#
do_orientational_clustering = not return_only_first_indexed_model # temporary.
dxtbx_crystal_models = []
if do_orientational_clustering:
print('IOTA: Starting orientational clustering')
Dij_ori = {} # dictionary to store Dij for each cluster
uc_experiments_list = {
} # dictionary to store experiments_lists for each cluster
from collections import Counter
uc_cluster_count = Counter(list(CM.cluster_id_final))
# instantiate the Dij_ori flat 1-d array
# Put all experiments list from same uc cluster together
if True:
from scitbx.matrix import sqr
from cctbx_orientation_ext import crystal_orientation
#crystal_orientation_list = []
#for i in range(len(experiments_list)):
# crystal_orientation_list.append(crystal_orientation(experiments_list[i].crystals()[0].get_A(), True))
#from IPython import embed; embed(); exit()
#A_direct = sqr(crystal_orientation_list[i].reciprocal_matrix()).transpose().inverse()
#print ("Direct A matrix 1st element = %12.6f"%A_direct[0])
for i in range(len(experiments_list)):
if CM.cluster_id_full[i] not in uc_experiments_list:
uc_experiments_list[CM.cluster_id_full[i]] = []
uc_experiments_list[CM.cluster_id_full[i]].append(
experiments_list[i])
for cluster in uc_cluster_count:
# Make sure there are atleast a minimum number of samples in the cluster
if uc_cluster_count[cluster] < 5:
continue
Dij_ori[cluster] = flex.double(
[[0.0] * uc_cluster_count[cluster]] *
uc_cluster_count[cluster])
# Now populate the Dij_ori array
N_samples_in_cluster = len(uc_experiments_list[cluster])
for i in range(N_samples_in_cluster - 1):
for j in range(i + 1, N_samples_in_cluster):
dij_ori = get_dij_ori(
uc_experiments_list[cluster][i].crystals()[0],
uc_experiments_list[cluster][j].crystals()[0])
Dij_ori[cluster][N_samples_in_cluster * i + j] = dij_ori
Dij_ori[cluster][N_samples_in_cluster * j + i] = dij_ori
# Now do the orientational cluster analysis
#from IPython import embed; embed(); exit()
d_c_ori = 0.13
from exafel_project.ADSE13_25.clustering.plot_with_dimensional_embedding import plot_with_dimensional_embedding
#plot_with_dimensional_embedding(1-Dij_ori[1]/flex.max(Dij_ori[1]), show_plot=True)
for cluster in Dij_ori:
d_c_ori = flex.mean_and_variance(Dij_ori[cluster].as_1d(
)).unweighted_sample_standard_deviation()
CM_ori = clustering_manager(Dij=Dij_ori[cluster],
d_c=d_c_ori,
max_percentile_rho=0.85)
n_cluster_ori = 1 + flex.max(CM_ori.cluster_id_final)
#from IPython import embed; embed()
#FIXME should be a PHIL param
for i in range(n_cluster_ori):
if len([zz for zz in CM_ori.cluster_id_final if zz == i]) < 5:
continue
item = flex.first_index(CM_ori.cluster_id_maxima, i)
dxtbx_crystal_model = uc_experiments_list[cluster][
item].crystals()[0]
dxtbx_crystal_models.append(dxtbx_crystal_model)
from scitbx.matrix import sqr
from cctbx_orientation_ext import crystal_orientation
crystal_orientation = crystal_orientation(
dxtbx_crystal_model.get_A(), True)
A_direct = sqr(crystal_orientation.reciprocal_matrix()
).transpose().inverse()
print(
"IOTA: Direct A matrix 1st element of orientational cluster %d = %12.6f"
% (i, A_direct[0]))
if show_plot:
# Decision graph
stretch_plot_factor = 1.05 # (1+fraction of limits by which xlim,ylim should be set)
import matplotlib.pyplot as plt
plt.plot(CM_ori.rho, CM_ori.delta, "r.", markersize=3.)
for x in range(len(list(CM_ori.cluster_id_final))):
if CM_ori.cluster_id_maxima[x] >= 0:
plt.plot([CM_ori.rho[x]], [CM_ori.delta[x]], "ro")
#from IPython import embed; embed(); exit()
plt.xlim([-10, stretch_plot_factor * flex.max(CM_ori.rho)])
plt.ylim([-10, stretch_plot_factor * flex.max(CM_ori.delta)])
plt.show()
# Make sure the crystal models are not too close to each other
# FIXME should be a PHIL
min_angle = 5.0 # taken from indexer.py
close_models_list = []
if len(dxtbx_crystal_models) > 1:
from dials.algorithms.indexing.compare_orientation_matrices import difference_rotation_matrix_axis_angle
for i_a in range(0, len(dxtbx_crystal_models) - 1):
for i_b in range(i_a, len(dxtbx_crystal_models)):
cryst_a = dxtbx_crystal_models[i_a]
cryst_b = dxtbx_crystal_models[i_b]
R_ab, axis, angle, cb_op_ab = difference_rotation_matrix_axis_angle(
cryst_a, cryst_b)
# FIXME
if abs(angle) < min_angle: # degrees
close_models_list.append((i_a, i_b))
# Now prune the dxtbx_crystal_models list
for close_models in close_models_list:
i_a, i_b = close_models
if dxtbx_crystal_models[i_a] is not None and dxtbx_crystal_models[
i_b] is not None:
dxtbx_crystal_models[i_a] = None
dxtbx_crystal_models = [x for x in dxtbx_crystal_models if x is not None]
if len(dxtbx_crystal_models) > 0:
return dxtbx_crystal_models, None
else:
# If nothing works, atleast return the 1st crystal model that was found
return [experiments_list[0].crystals()[0]], None
def run(args):
import libtbx.load_env
from dials.util import Sorry
usage = "dials.reindex [options] indexed.expt indexed.refl"
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message,
)
params, options = parser.parse_args(show_diff_phil=True)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0 and len(reflections) == 0:
parser.print_help()
return
if params.change_of_basis_op is None:
raise Sorry("Please provide a change_of_basis_op.")
reference_crystal = None
if params.reference.experiments is not None:
from dxtbx.serialize import load
reference_experiments = load.experiment_list(
params.reference.experiments, check_format=False)
assert len(reference_experiments.crystals()) == 1
reference_crystal = reference_experiments.crystals()[0]
if params.reference.reflections is not None:
# First check that we have everything as expected for the reference reindexing
# Currently only supports reindexing one dataset at a time
if params.reference.experiments is None:
raise Sorry(
"""For reindexing against a reference dataset, a reference
experiments file must also be specified with the option: reference= """)
if not os.path.exists(params.reference.reflections):
raise Sorry("Could not locate reference dataset reflection file")
if len(experiments) != 1 or len(reflections) != 1:
raise Sorry(
"Only one dataset can be reindexed to a reference at a time")
reference_reflections = flex.reflection_table().from_file(
params.reference.reflections)
test_reflections = reflections[0]
if (reference_crystal.get_space_group().type().number() !=
experiments.crystals()[0].get_space_group().type().number()):
raise Sorry("Space group of input does not match reference")
# Set some flags to allow filtering, if wanting to reindex against
# reference with data that has not yet been through integration
if (test_reflections.get_flags(
test_reflections.flags.integrated_sum).count(True) == 0):
assert (
"intensity.sum.value"
in test_reflections), "No 'intensity.sum.value' in reflections"
test_reflections.set_flags(
flex.bool(test_reflections.size(), True),
test_reflections.flags.integrated_sum,
)
if (reference_reflections.get_flags(
reference_reflections.flags.integrated_sum).count(True) == 0):
assert ("intensity.sum.value" in test_reflections
), "No 'intensity.sum.value in reference reflections"
reference_reflections.set_flags(
flex.bool(reference_reflections.size(), True),
reference_reflections.flags.integrated_sum,
)
# Make miller array of the two datasets
try:
test_miller_set = filtered_arrays_from_experiments_reflections(
experiments, [test_reflections])[0]
except ValueError:
raise Sorry(
"No reflections remain after filtering the test dataset")
try:
reference_miller_set = filtered_arrays_from_experiments_reflections(
reference_experiments, [reference_reflections])[0]
except ValueError:
raise Sorry(
"No reflections remain after filtering the reference dataset")
from dials.algorithms.symmetry.reindex_to_reference import (
determine_reindex_operator_against_reference, )
change_of_basis_op = determine_reindex_operator_against_reference(
test_miller_set, reference_miller_set)
elif len(experiments) and params.change_of_basis_op is libtbx.Auto:
if reference_crystal is not None:
if len(experiments.crystals()) > 1:
raise Sorry("Only one crystal can be processed at a time")
from dials.algorithms.indexing.compare_orientation_matrices import (
difference_rotation_matrix_axis_angle, )
cryst = experiments.crystals()[0]
R, axis, angle, change_of_basis_op = difference_rotation_matrix_axis_angle(
cryst, reference_crystal)
print("Change of basis op: %s" % change_of_basis_op)
print("Rotation matrix to transform input crystal to reference::")
print(R.mathematica_form(format="%.3f", one_row_per_line=True))
print(
"Rotation of %.3f degrees" % angle,
"about axis (%.3f, %.3f, %.3f)" % axis,
)
elif len(reflections):
assert len(reflections) == 1
# always re-map reflections to reciprocal space
refl_copy = flex.reflection_table()
for i, imageset in enumerate(experiments.imagesets()):
if "imageset_id" in reflections[0]:
sel = reflections[0]["imageset_id"] == i
else:
sel = reflections[0]["id"] == i
refl = reflections[0].select(sel)
refl.centroid_px_to_mm(imageset.get_detector(),
imageset.get_scan())
refl.map_centroids_to_reciprocal_space(
imageset.get_detector(),
imageset.get_beam(),
imageset.get_goniometer(),
)
refl_copy.extend(refl)
# index the reflection list using the input experiments list
refl_copy["id"] = flex.int(len(refl_copy), -1)
index = AssignIndicesGlobal(tolerance=0.2)
index(refl_copy, experiments)
hkl_expt = refl_copy["miller_index"]
hkl_input = reflections[0]["miller_index"]
change_of_basis_op = derive_change_of_basis_op(hkl_input, hkl_expt)
# reset experiments list since we don't want to reindex this
experiments = []
else:
change_of_basis_op = sgtbx.change_of_basis_op(
params.change_of_basis_op)
if len(experiments):
for crystal in experiments.crystals():
cryst_orig = copy.deepcopy(crystal)
cryst_reindexed = cryst_orig.change_basis(change_of_basis_op)
if params.space_group is not None:
a, b, c = cryst_reindexed.get_real_space_vectors()
A_varying = [
cryst_reindexed.get_A_at_scan_point(i)
for i in range(cryst_reindexed.num_scan_points)
]
cryst_reindexed = Crystal(
a, b, c, space_group=params.space_group.group())
cryst_reindexed.set_A_at_scan_points(A_varying)
crystal.update(cryst_reindexed)
print("Old crystal:")
print(cryst_orig)
print()
print("New crystal:")
print(cryst_reindexed)
print()
print("Saving reindexed experimental models to %s" %
params.output.experiments)
experiments.as_file(params.output.experiments)
if len(reflections):
assert len(reflections) == 1
reflections = reflections[0]
miller_indices = reflections["miller_index"]
if params.hkl_offset is not None:
h, k, l = miller_indices.as_vec3_double().parts()
h += params.hkl_offset[0]
k += params.hkl_offset[1]
l += params.hkl_offset[2]
miller_indices = flex.miller_index(h.iround(), k.iround(),
l.iround())
non_integral_indices = change_of_basis_op.apply_results_in_non_integral_indices(
miller_indices)
if non_integral_indices.size() > 0:
print(
"Removing %i/%i reflections (change of basis results in non-integral indices)"
% (non_integral_indices.size(), miller_indices.size()))
sel = flex.bool(miller_indices.size(), True)
sel.set_selected(non_integral_indices, False)
miller_indices_reindexed = change_of_basis_op.apply(
miller_indices.select(sel))
reflections["miller_index"].set_selected(sel, miller_indices_reindexed)
reflections["miller_index"].set_selected(~sel, (0, 0, 0))
print("Saving reindexed reflections to %s" % params.output.reflections)
easy_pickle.dump(params.output.reflections, reflections)
def run():
random_seed = 35
flex.set_random_seed(random_seed)
random.seed(random_seed)
data = merge_close_spots.merge_close_spots()
# unit cell and space group for lysozyme
known_symmetry = crystal.symmetry("78,78,37,90,90,90", "P43212")
sim = data.two_color_sim
merged_spot_info = data.spot_proximity(sim)
merged_refl = merged_spot_info[0]
detector = data.detector
beams = data.beams
# to make sure input crystal and indexed crystal model are the same
# orientation before using the refiner
A = sim['input_orientation']
A_inv = A.inverse()
a = A_inv[:3]
b = A_inv[3:6]
c = A_inv[6:]
crystinp = Crystal(a, b, c, space_group=known_symmetry.space_group())
result = index(merged_refl, detector, known_symmetry, beams)
print("RESULTS ARE IN")
cm = result.refined_experiments.crystals()[0]
R, best_axis, best_angle, change_of_basis = difference_rotation_matrix_axis_angle(
crystal_a=cm, crystal_b=crystinp)
euler_angles = euler.xyz_angles(R)
print "input crystal: %s" % crystinp
print "Indexed crystal: %s" % cm
print "rotation of: %s" % R
print "euler angles:", euler_angles
print "change of basis:", change_of_basis.as_hkl()
# cmd_line = command_line.argument_interpreter(master_params=master_phil_scope)
# working_phil = cmd_line.process_and_fetch(args=[])
params = master_phil_scope.extract()
params.refinement.parameterisation.beam.fix = "all"
params.refinement.parameterisation.detector.fix = "all"
params.indexing.known_symmetry.space_group = known_symmetry.space_group_info(
)
params.refinement.verbosity = 3
params.indexing.refinement_protocol.d_min_start = 3
params.indexing.refinement_protocol.n_macro_cycles = 1
params.indexing.known_symmetry.unit_cell = known_symmetry.unit_cell()
params.indexing.multiple_lattice_search.max_lattices = 1
params.indexing.debug = True
params.indexing.known_symmetry.absolute_angle_tolerance = 5.0
params.indexing.known_symmetry.relative_length_tolerance = 0.3
expts = copy.deepcopy(result.refined_experiments)
expts.crystals()[0].change_basis(change_of_basis)
print(expts.crystals()[0])
refined = refine(params, result.reflections, expts, verbosity=1)
print(refined[0].get_experiments().crystals()[0])
# from annlib_ext import AnnAdaptor
# ann = AnnAdaptor(merged_refl['xyzobs.px.value'].as_double(), dim=3, k=1)
# ann.query(result.reflections['xyzobs.px.value'].as_double()+1e-6)
indices_sim = change_of_basis.apply(merged_refl['set_miller_index'])
id_sim = merged_refl['set_id']
# only get those that refined:
idx = align_merged_and_refined_refl(merged_refl,
result.refined_reflections)
indices_sim = flex.miller_index([indices_sim[i] for i in idx])
id_sim = flex.int([id_sim[i] for i in idx])
indices_result = result.refined_reflections['miller_index']
id_result = result.refined_reflections['id']
correct_ind = (indices_sim == indices_result)
wrong_wavelength = (id_sim != id_result) & (id_sim != 2)
wrong_index = (indices_sim != indices_result)
correct_wavelength = (id_sim == id_result) | (id_sim == 2)
correct = correct_ind & correct_wavelength
print "Correct index and wavelength: %i/%i" % (correct.count(True),
len(correct))
print "Correct index but wrong wavelength: %i/%i" % (
wrong_wavelength.count(True), len(wrong_wavelength))
print "Wrong index but correct wavelength: %i/%i" % (
wrong_index.count(True), len(correct_wavelength))
def run(args):
import libtbx.load_env
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
# The script usage
usage = "usage: %s [options] [param.phil] experiments.json" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if not experiments:
parser.print_help()
return
if not params.hkl and params.hkl_limit is None:
from libtbx.utils import Sorry
raise Sorry("Please provide hkl or hkl_limit parameters.")
if params.hkl is not None and len(params.hkl):
miller_indices = flex.miller_index(params.hkl)
elif params.hkl_limit is not None:
limit = params.hkl_limit
miller_indices = flex.miller_index()
for h in range(-limit, limit+1):
for k in range(-limit, limit+1):
for l in range(-limit, limit+1):
if (h,k,l) == (0,0,0): continue
miller_indices.append((h,k,l))
crystals = experiments.crystals()
symmetry = crystal.symmetry(
unit_cell=crystals[0].get_unit_cell(),
space_group=crystals[0].get_space_group())
miller_set = miller.set(symmetry, miller_indices)
d_spacings = miller_set.d_spacings()
if params.eliminate_sys_absent:
d_spacings = d_spacings.eliminate_sys_absent()
if params.expand_to_p1:
d_spacings = d_spacings.as_non_anomalous_array().expand_to_p1()
d_spacings = d_spacings.generate_bijvoet_mates()
miller_indices = d_spacings.indices()
# find the greatest common factor (divisor) between miller indices
miller_indices_unique = flex.miller_index()
for hkl in miller_indices:
gcd = gcd_list(hkl)
if gcd > 1:
miller_indices_unique.append(tuple(int(h/gcd) for h in hkl))
elif gcd < 1:
pass
else:
miller_indices_unique.append(hkl)
miller_indices = miller_indices_unique
miller_indices = flex.miller_index(list(set(miller_indices)))
ref_crystal = crystals[0]
A = ref_crystal.get_A()
U = ref_crystal.get_U()
B = ref_crystal.get_B()
R = matrix.identity(3)
if params.frame == 'laboratory':
reference_poles = reference_poles_perpendicular_to_beam(
experiments[0].beam, experiments[0].goniometer)
if params.use_starting_angle:
rotation_axis = matrix.col(
experiments[0].goniometer.get_rotation_axis())
R = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
experiments[0].scan.get_oscillation()[0], deg=True)
elif params.phi_angle != 0:
rotation_axis = matrix.col(
experiments[0].goniometer.get_rotation_axis())
R = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
params.phi_angle, deg=True)
else:
if params.plane_normal is not None:
plane_normal = params.plane_normal
else:
plane_normal = (0,0,1)
reference_poles = reference_poles_crystal(
ref_crystal, plane_normal=plane_normal)
if params.frame == 'crystal':
U = matrix.identity(3)
reciprocal_space_points = list(R * U * B) * miller_indices.as_vec3_double()
projections_ref = stereographic_projection(
reciprocal_space_points, reference_poles)
projections_all = [projections_ref]
if experiments:
from dials.algorithms.indexing.compare_orientation_matrices import \
difference_rotation_matrix_axis_angle
for expt in experiments[1:]:
cryst = expt.crystal
if params.frame == 'crystal':
R_ij, axis, angle, cb_op = difference_rotation_matrix_axis_angle(
ref_crystal, cryst)
U = R_ij
elif params.use_starting_angle:
if params.use_starting_angle:
rotation_axis = matrix.col(
expt.goniometer.get_rotation_axis())
R = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
expt.scan.get_oscillation()[0], deg=True)
else:
U = cryst.get_U()
reciprocal_space_points = list(R * U * cryst.get_B()) * miller_indices.as_vec3_double()
projections = stereographic_projection(
reciprocal_space_points, reference_poles)
projections_all.append(projections)
if params.save_coordinates:
with open('projections.txt', 'wb') as f:
print >> f, "crystal h k l x y"
for i_cryst, projections in enumerate(projections_all):
for hkl, proj in zip(miller_indices, projections):
print >> f, "%i" %(i_cryst+1),
print >> f, "%i %i %i" %hkl,
print >> f, "%f %f" %proj
if params.plot.show or params.plot.filename:
plot_projections(
projections_all, filename=params.plot.filename, show=params.plot.show,
colours=params.plot.colours, marker_size=params.plot.marker_size,
font_size=params.plot.font_size, label_indices=params.plot.label_indices)
def index(self, provided_experiments=None, debug=False):
''' This step does 1. find_lattices (via a method like fft1d)
2. Assign hkl indices (through index_reflections)
3. Housekeeping like apply_symmetry, discard too similar models
'''
experiments = ExperimentList()
have_similar_crystal_models = False
self.d_min = self.params.refinement_protocol.d_min_start
# Find lattices i.e the basis vectors & unit cell params
# Possible to index multiple lattices ??
while True:
max_lattices = self.params.multiple_lattice_search.max_lattices
if max_lattices is not None and len(experiments) >= max_lattices:
break
n_lattices_previous_cycle = len(experiments)
if len(experiments) == 0:
experiments.extend(self.find_lattices())
else:
try:
new = self.find_lattices()
experiments.extend(new)
except Sorry:
print('Indexing remaining reflections failed')
break
if len(experiments) == 0:
raise Sorry("No suitable lattice could be found.")
# Initialize id values as -1 since no indexing has been done yet
self.reflections['id'] = flex.int(len(self.reflections), -1)
# Now index reflections
self.index_reflections(experiments, self.reflections, debug=debug)
# Housekeeping. Apply symmetry
self._apply_symmetry_post_indexing(experiments, self.reflections,
n_lattices_previous_cycle)
# Aug_Refactor :: probably unnecessary to remove stuff below but still doing so to adapt to new style
# Never mind, might keep it to have IOTA stuff working
'''
target_space_group = self.target_symmetry_primitive.space_group()
for i_cryst, cryst in enumerate(experiments.crystals()):
if i_cryst >= n_lattices_previous_cycle:
new_cryst, cb_op_to_primitive = self.apply_symmetry(
cryst, target_space_group)
if provided_experiments is None:
if self.cb_op_primitive_inp is not None:
new_cryst = new_cryst.change_basis(self.cb_op_primitive_inp)
logger.info(new_cryst.get_space_group().info())
cryst.update(new_cryst)
cryst.set_space_group(
self.params.known_symmetry.space_group.group())
for i_expt, expt in enumerate(experiments):
if expt.crystal is not cryst:
continue
if not cb_op_to_primitive.is_identity_op():
miller_indices = self.reflections['miller_index'].select(
self.reflections['id'] == i_expt)
if provided_experiments is None:
miller_indices = cb_op_to_primitive.apply(miller_indices)
self.reflections['miller_index'].set_selected(
self.reflections['id'] == i_expt, miller_indices)
if self.cb_op_primitive_inp is not None:
miller_indices = self.reflections['miller_index'].select(
self.reflections['id'] == i_expt)
if provided_experiments is None:
miller_indices = self.cb_op_primitive_inp.apply(miller_indices)
self.reflections['miller_index'].set_selected(
self.reflections['id'] == i_expt, miller_indices)
# IOTA
from scitbx.matrix import sqr
hklfrac=flex.mat3_double(len(miller_indices), sqr(cryst.get_A()).inverse())*self.reflections['rlp'].select(self.reflections['id']==i_expt)
self.reflections['fractional_miller_index'].set_selected(self.reflections['id']==i_expt, hklfrac)
'''
logger.info("\nIndexed crystal models:")
self.show_experiments(experiments, self.reflections, d_min=self.d_min)
# Discard nearly overlapping lattices
# difference_rotation_matrix_axis_angle function is there still in DIALS 2.0 so no need to change anything below
if len(experiments) > 1:
from dials.algorithms.indexing.compare_orientation_matrices \
import difference_rotation_matrix_axis_angle
cryst_b = experiments.crystals()[-1]
have_similar_crystal_models = False
for i_a, cryst_a in enumerate(experiments.crystals()[:-1]):
R_ab, axis, angle, cb_op_ab = \
difference_rotation_matrix_axis_angle(cryst_a, cryst_b)
min_angle = self.params.multiple_lattice_search.minimum_angular_separation
if abs(angle) < min_angle: # degrees
logger.info(
"Crystal models too similar, rejecting crystal %i:" %
(len(experiments)))
logger.info(
"Rotation matrix to transform crystal %i to crystal %i"
% (i_a + 1, len(experiments)))
logger.info(R_ab)
logger.info("Rotation of %.3f degrees" % angle +
" about axis (%.3f, %.3f, %.3f)" % axis)
have_similar_crystal_models = True
del experiments[-1]
break
self.indexed_reflections = (self.reflections['id'] > -1)
self.experiments = experiments
print("Opened %d / %d indexing results" % (i_f, len(fnames)))
refls_good.append( f)
Els_good.append(El_f)
dumps_good.append(dump_f)
idx_cryst_mdl = utils.open_flex(dump_f)["crystalAB"]
cryst_good.append(idx_cryst_mdl)
Ex = ExperimentListFactory.from_json_file(El_f, check_format=False)
img_path = Ex[0].imageset.get_path(0)
h5 = h5py.File(img_path, "r")
Atruth = h5["crystalA"][()]
truth_cryst = deepcopy(idx_cryst_mdl)
truth_cryst.set_A(Atruth)
out = difference_rotation_matrix_axis_angle(idx_cryst_mdl, truth_cryst)
all_rots.append(out[2])
from IPython import embed
embed()
"""
Els_good
len( refls_good)
len( Els_good)
import dxtbx
ff
E = ff[0]
E.imageset
def run(args=None):
from dials.util.options import OptionParser, flatten_experiments
usage = "dials.goniometer_calibration [options] models.expt"
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=False,
epilog=help_message,
)
params, options = parser.parse_args(args, show_diff_phil=True)
if not params.use_space_group_from_experiments and params.space_group is None:
parser.print_help()
return
experiments = flatten_experiments(params.input.experiments)
if len(experiments) <= 1:
parser.print_help()
return
from dials.algorithms.indexing.compare_orientation_matrices import (
difference_rotation_matrix_axis_angle, )
for experiment in experiments:
crystal = experiment.crystal
gonio = experiment.goniometer
assert len(experiments) == (len(gonio.get_axes()) + 1)
scan = experiment.scan
fixed_rotation = matrix.sqr(gonio.get_fixed_rotation())
setting_rotation = matrix.sqr(gonio.get_setting_rotation())
rotation_axis = matrix.col(gonio.get_rotation_axis_datum())
rotation_matrix = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
scan.get_oscillation()[0], deg=True)
U = matrix.sqr(crystal.get_U())
U = setting_rotation * rotation_matrix * fixed_rotation * U
crystal.set_U(U)
if params.space_group is not None:
crystal.set_space_group(params.space_group.group())
rows = []
from rstbx.cftbx.coordinate_frame_helpers import align_reference_frame
R_to_mosflm = align_reference_frame(
experiments[0].beam.get_s0(),
(1.0, 0.0, 0.0),
experiments[0].goniometer.get_rotation_axis(),
(0.0, 0.0, 1.0),
)
axes = []
angles = []
for i in range(len(experiments) - 1):
target_angle = experiments[i + 1].goniometer.get_angles()[i]
if i == experiments[i].goniometer.get_scan_axis():
# rotation axis is canonical in our coordinate system
axis = experiments[i].goniometer.get_axes()[i]
angle = target_angle
else:
R_ij, axis, angle, cb_op = difference_rotation_matrix_axis_angle(
experiments[i].crystal,
experiments[i + 1].crystal,
target_angle=target_angle,
)
gonio = experiments[i + 1].goniometer
axis_names = gonio.get_names()
axes.append(axis)
angles.append(angle)
depends_on = "."
if i + 1 < len(axis_names):
depends_on = axis_names[i + 1]
rows.insert(
0,
(
axis_names[i],
"rotation",
"goniometer",
depends_on,
f"{axis[0]:.4f}",
f"{axis[1]:.4f}",
f"{axis[2]:.4f}",
".",
".",
".",
),
)
axis_names = experiments[0].goniometer.get_names()
print("Goniometer axes and angles (ImgCIF coordinate system):")
for axis, angle, name in zip(axes, angles, axis_names):
print(
f"{name}: ",
f"rotation of {angle:.3f} degrees",
"about axis (%.5f,%.5f,%.5f)" % axis,
)
print()
print("Goniometer axes and angles (MOSFLM coordinate system):")
for axis, angle, name in zip(axes, angles, axis_names):
print(
f"{name}: ",
f"rotation of {angle:.3f} degrees",
"about axis (%.5f,%.5f,%.5f)" %
(R_to_mosflm * matrix.col(axis)).elems,
)
print()
print("ImgCIF _axis loop template:")
from iotbx import cif
loop = cif.model.loop(header=[
"_axis.id",
"_axis.type",
"_axis.equipment",
"_axis.depends_on",
"_axis.vector[1]",
"_axis.vector[2]",
"_axis.vector[3]",
"_axis.offset[1]",
"_axis.offset[2]",
"_axis.offset[3]",
])
for row in rows:
loop.add_row(row)
print(loop)
if params.output.xoalign is not None:
axes_mosflm = [(R_to_mosflm * matrix.col(axis)).elems for axis in axes]
write_xoalign_config(params.output.xoalign, reversed(axes_mosflm),
reversed(axis_names))
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
from libtbx.utils import Sorry
import libtbx.load_env
usage = "%s [options] experiments.json" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) <= 1:
parser.print_help()
return
from dials.algorithms.indexing.compare_orientation_matrices import \
difference_rotation_matrix_axis_angle
from scitbx import matrix
crystals = []
for experiment in experiments:
crystal = experiment.crystal
gonio = experiment.goniometer
assert len(experiments) == (len(gonio.get_axes())+1)
scan = experiment.scan
fixed_rotation = matrix.sqr(gonio.get_fixed_rotation())
setting_rotation = matrix.sqr(gonio.get_setting_rotation())
rotation_axis = matrix.col(gonio.get_rotation_axis_datum())
rotation_matrix = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
scan.get_oscillation()[0], deg=True)
U = matrix.sqr(crystal.get_U())
U = setting_rotation * rotation_matrix * fixed_rotation * U
crystal.set_U(U)
if params.space_group is not None:
crystal.set_space_group(params.space_group.group())
rows = []
from rstbx.cftbx.coordinate_frame_helpers import align_reference_frame
from scitbx import matrix
R_to_mosflm = align_reference_frame(
experiments[0].beam.get_s0(), (1.0, 0.0, 0.0),
experiments[0].goniometer.get_rotation_axis(), (0.0, 0.0, 1.0))
axes = []
angles = []
for i in range(len(experiments) - 1):
target_angle = experiments[i+1].goniometer.get_angles()[i]
R_ij, axis, angle, cb_op = difference_rotation_matrix_axis_angle(
experiments[i].crystal, experiments[i+1].crystal, target_angle=target_angle)
gonio = experiments[i+1].goniometer
axis_names = gonio.get_names()
axes.append(axis)
angles.append(angle)
depends_on = '.'
if i+1 < len(axis_names):
depends_on = axis_names[i+1]
rows.insert(0, (
axis_names[i], 'rotation', 'goniometer', depends_on,
'%.4f' %axis[0], '%.4f' %axis[1], '%.4f' %axis[2], '.', '.', '.'))
axis_names = experiments[0].goniometer.get_names()
print "Goniometer axes and angles (ImgCIF coordinate system):"
for axis, angle, name in zip(axes, angles, axis_names):
print "%s: " %name, "rotation of %.3f degrees" %angle, "about axis (%.5f, %.5f, %.5f)" %axis
print
print "Goniometer axes and angles (MOSFLM coordinate system):"
for axis, angle, name in zip(axes, angles, axis_names):
print "%s: " %name, "rotation of %.3f degrees" %angle, "about axis (%.5f, %.5f, %.5f)" %(
R_to_mosflm * matrix.col(axis)).elems
print
print "ImgCIF _axis loop template:"
from iotbx import cif
loop = cif.model.loop(
header=['_axis.id', '_axis.type', '_axis.equipment', '_axis.depends_on',
'_axis.vector[1]', '_axis.vector[2]', '_axis.vector[3]',
'_axis.offset[1]', '_axis.offset[2]', '_axis.offset[3]'])
for row in reversed(rows):
loop.add_row(row)
print loop
if params.output.xoalign is not None:
write_xoalign_config(params.output.xoalign, axes, axis_names)
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
import libtbx.load_env
usage = "%s [options] experiments.json" % libtbx.env.dispatcher_name
parser = OptionParser(usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) <= 1:
parser.print_help()
return
from dials.algorithms.indexing.compare_orientation_matrices import \
difference_rotation_matrix_axis_angle
from scitbx import matrix
crystals = []
for experiment in experiments:
crystal = experiment.crystal
gonio = experiment.goniometer
assert len(experiments) == (len(gonio.get_axes()) + 1)
scan = experiment.scan
fixed_rotation = matrix.sqr(gonio.get_fixed_rotation())
setting_rotation = matrix.sqr(gonio.get_setting_rotation())
rotation_axis = matrix.col(gonio.get_rotation_axis_datum())
rotation_matrix = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
scan.get_oscillation()[0], deg=True)
U = matrix.sqr(crystal.get_U())
U = setting_rotation * rotation_matrix * fixed_rotation * U
crystal.set_U(U)
if params.space_group is not None:
crystal.set_space_group(params.space_group.group())
rows = []
from rstbx.cftbx.coordinate_frame_helpers import align_reference_frame
from scitbx import matrix
R_to_mosflm = align_reference_frame(
experiments[0].beam.get_s0(), (1.0, 0.0, 0.0),
experiments[0].goniometer.get_rotation_axis(), (0.0, 0.0, 1.0))
axes = []
angles = []
for i in range(len(experiments) - 1):
target_angle = experiments[i + 1].goniometer.get_angles()[i]
if i == experiments[i].goniometer.get_scan_axis():
# rotation axis is canonical in our coordinate system
axis = experiments[i].goniometer.get_axes()[i]
angle = target_angle
else:
R_ij, axis, angle, cb_op = difference_rotation_matrix_axis_angle(
experiments[i].crystal,
experiments[i + 1].crystal,
target_angle=target_angle)
gonio = experiments[i + 1].goniometer
axis_names = gonio.get_names()
axes.append(axis)
angles.append(angle)
depends_on = '.'
if i + 1 < len(axis_names):
depends_on = axis_names[i + 1]
rows.insert(
0, (axis_names[i], 'rotation', 'goniometer', depends_on, '%.4f' %
axis[0], '%.4f' % axis[1], '%.4f' % axis[2], '.', '.', '.'))
axis_names = experiments[0].goniometer.get_names()
print "Goniometer axes and angles (ImgCIF coordinate system):"
for axis, angle, name in zip(axes, angles, axis_names):
print "%s: " % name, "rotation of %.3f degrees" % angle, "about axis (%.5f,%.5f,%.5f)" % axis
print
print "Goniometer axes and angles (MOSFLM coordinate system):"
for axis, angle, name in zip(axes, angles, axis_names):
print "%s: " % name, "rotation of %.3f degrees" % angle, "about axis (%.5f,%.5f,%.5f)" % (
R_to_mosflm * matrix.col(axis)).elems
print
print "ImgCIF _axis loop template:"
from iotbx import cif
loop = cif.model.loop(header=[
'_axis.id', '_axis.type', '_axis.equipment', '_axis.depends_on',
'_axis.vector[1]', '_axis.vector[2]', '_axis.vector[3]',
'_axis.offset[1]', '_axis.offset[2]', '_axis.offset[3]'
])
for row in rows:
loop.add_row(row)
print loop
if params.output.xoalign is not None:
axes_mosflm = [(R_to_mosflm * matrix.col(axis)).elems for axis in axes]
write_xoalign_config(params.output.xoalign, reversed(axes_mosflm),
reversed(axis_names))
def run(args):
import libtbx.load_env
from libtbx.utils import Sorry
usage = "%s [options] experiments.json indexed.pickle" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0 and len(reflections) == 0:
parser.print_help()
return
elif len(experiments.crystals()) > 1:
raise Sorry("Only one crystal can be processed at a time")
if params.change_of_basis_op is None:
raise Sorry("Please provide a change_of_basis_op.")
reference_crystal = None
if params.reference is not None:
from dxtbx.serialize import load
reference_experiments = load.experiment_list(
params.reference, check_format=False)
assert len(reference_experiments.crystals()) == 1
reference_crystal = reference_experiments.crystals()[0]
if len(experiments) and params.change_of_basis_op is libtbx.Auto:
if reference_crystal is not None:
from dials.algorithms.indexing.compare_orientation_matrices \
import difference_rotation_matrix_axis_angle
cryst = experiments.crystals()[0]
R, axis, angle, change_of_basis_op = difference_rotation_matrix_axis_angle(
cryst, reference_crystal)
print "Change of basis op: %s" %change_of_basis_op
print "Rotation matrix to transform input crystal to reference::"
print R.mathematica_form(format="%.3f", one_row_per_line=True)
print "Rotation of %.3f degrees" %angle, "about axis (%.3f, %.3f, %.3f)" %axis
elif len(reflections):
assert len(reflections) == 1
# always re-map reflections to reciprocal space
from dials.algorithms.indexing import indexer
refl_copy = flex.reflection_table()
for i, imageset in enumerate(experiments.imagesets()):
if 'imageset_id' in reflections[0]:
sel = (reflections[0]['imageset_id'] == i)
else:
sel = (reflections[0]['id'] == i)
refl = indexer.indexer_base.map_spots_pixel_to_mm_rad(
reflections[0].select(sel),
imageset.get_detector(), imageset.get_scan())
indexer.indexer_base.map_centroids_to_reciprocal_space(
refl, imageset.get_detector(), imageset.get_beam(),
imageset.get_goniometer())
refl_copy.extend(refl)
# index the reflection list using the input experiments list
refl_copy['id'] = flex.int(len(refl_copy), -1)
from dials.algorithms.indexing import index_reflections
index_reflections(refl_copy, experiments, tolerance=0.2)
hkl_expt = refl_copy['miller_index']
hkl_input = reflections[0]['miller_index']
change_of_basis_op = derive_change_of_basis_op(hkl_input, hkl_expt)
# reset experiments list since we don't want to reindex this
experiments = []
else:
change_of_basis_op = sgtbx.change_of_basis_op(params.change_of_basis_op)
if len(experiments):
experiment = experiments[0]
cryst_orig = copy.deepcopy(experiment.crystal)
cryst_reindexed = cryst_orig.change_basis(change_of_basis_op)
if params.space_group is not None:
a, b, c = cryst_reindexed.get_real_space_vectors()
cryst_reindexed = crystal_model(
a, b, c, space_group=params.space_group.group())
experiment.crystal.update(cryst_reindexed)
print "Old crystal:"
print cryst_orig
print
print "New crystal:"
print cryst_reindexed
print
print "Saving reindexed experimental models to %s" %params.output.experiments
dump.experiment_list(experiments, params.output.experiments)
if len(reflections):
assert(len(reflections) == 1)
reflections = reflections[0]
miller_indices = reflections['miller_index']
if params.hkl_offset is not None:
h,k,l = miller_indices.as_vec3_double().parts()
h += params.hkl_offset[0]
k += params.hkl_offset[1]
l += params.hkl_offset[2]
miller_indices = flex.miller_index(h.iround(), k.iround(), l.iround())
non_integral_indices = change_of_basis_op.apply_results_in_non_integral_indices(miller_indices)
if non_integral_indices.size() > 0:
print "Removing %i/%i reflections (change of basis results in non-integral indices)" %(
non_integral_indices.size(), miller_indices.size())
sel = flex.bool(miller_indices.size(), True)
sel.set_selected(non_integral_indices, False)
miller_indices_reindexed = change_of_basis_op.apply(
miller_indices.select(sel))
reflections['miller_index'].set_selected(sel, miller_indices_reindexed)
reflections['miller_index'].set_selected(~sel, (0,0,0))
print "Saving reindexed reflections to %s" %params.output.reflections
easy_pickle.dump(params.output.reflections, reflections)
def index(self):
# most of this is the same as dials.algorithms.indexing.indexer.indexer_base.index(), with some stills
# specific modifications (don't re-index after choose best orientation matrix, but use the indexing from
# choose best orientation matrix, also don't use macrocycles) of refinement after indexing.
# 2017 update: do accept multiple lattices per shot
if self.params.refinement_protocol.n_macro_cycles > 1:
raise Sorry(
"For stills, please set refinement_protocol.n_macro_cycles = 1"
)
experiments = ExperimentList()
had_refinement_error = False
have_similar_crystal_models = False
while True:
self.d_min = self.params.refinement_protocol.d_min_start
if had_refinement_error or have_similar_crystal_models:
break
max_lattices = self.params.multiple_lattice_search.max_lattices
if max_lattices is not None and len(experiments) >= max_lattices:
break
if len(experiments) > 0:
cutoff_fraction = \
self.params.multiple_lattice_search.recycle_unindexed_reflections_cutoff
d_spacings = 1 / self.reflections['rlp'].norms()
d_min_indexed = flex.min(
d_spacings.select(self.indexed_reflections))
min_reflections_for_indexing = \
cutoff_fraction * len(self.reflections.select(d_spacings > d_min_indexed))
crystal_ids = self.reflections.select(
d_spacings > d_min_indexed)['id']
if (crystal_ids
== -1).count(True) < min_reflections_for_indexing:
logger.info(
"Finish searching for more lattices: %i unindexed reflections remaining."
% (min_reflections_for_indexing))
break
n_lattices_previous_cycle = len(experiments)
# index multiple lattices per shot
if len(experiments) == 0:
experiments.extend(self.find_lattices())
if len(experiments) == 0:
raise Sorry("No suitable lattice could be found.")
else:
try:
new = self.find_lattices()
experiments.extend(new)
except Exception as e:
logger.info("Indexing remaining reflections failed")
logger.debug(
"Indexing remaining reflections failed, exception:\n" +
str(e))
# reset reflection lattice flags
# the lattice a given reflection belongs to: a value of -1 indicates
# that a reflection doesn't belong to any lattice so far
self.reflections['id'] = flex.int(len(self.reflections), -1)
self.index_reflections(experiments, self.reflections)
if len(experiments) == n_lattices_previous_cycle:
# no more lattices found
break
if not self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None:
# now apply the space group symmetry only after the first indexing
# need to make sure that the symmetrized orientation is similar to the P1 model
target_space_group = self.target_symmetry_primitive.space_group(
)
for i_cryst, cryst in enumerate(experiments.crystals()):
if i_cryst >= n_lattices_previous_cycle:
new_cryst, cb_op_to_primitive = self.apply_symmetry(
cryst, target_space_group)
if self.cb_op_primitive_inp is not None:
new_cryst = new_cryst.change_basis(
self.cb_op_primitive_inp)
logger.info(new_cryst.get_space_group().info())
cryst.update(new_cryst)
cryst.set_space_group(
self.params.known_symmetry.space_group.group())
for i_expt, expt in enumerate(experiments):
if expt.crystal is not cryst:
continue
if not cb_op_to_primitive.is_identity_op():
miller_indices = self.reflections[
'miller_index'].select(
self.reflections['id'] == i_expt)
miller_indices = cb_op_to_primitive.apply(
miller_indices)
self.reflections['miller_index'].set_selected(
self.reflections['id'] == i_expt,
miller_indices)
if self.cb_op_primitive_inp is not None:
miller_indices = self.reflections[
'miller_index'].select(
self.reflections['id'] == i_expt)
miller_indices = self.cb_op_primitive_inp.apply(
miller_indices)
self.reflections['miller_index'].set_selected(
self.reflections['id'] == i_expt,
miller_indices)
# discard nearly overlapping lattices on the same shot
if len(experiments) > 1:
from dials.algorithms.indexing.compare_orientation_matrices \
import difference_rotation_matrix_axis_angle
cryst_b = experiments.crystals()[-1]
have_similar_crystal_models = False
for i_a, cryst_a in enumerate(experiments.crystals()[:-1]):
R_ab, axis, angle, cb_op_ab = \
difference_rotation_matrix_axis_angle(cryst_a, cryst_b)
min_angle = self.params.multiple_lattice_search.minimum_angular_separation
if abs(angle) < min_angle: # degrees
logger.info(
"Crystal models too similar, rejecting crystal %i:"
% (len(experiments)))
logger.info(
"Rotation matrix to transform crystal %i to crystal %i"
% (i_a + 1, len(experiments)))
logger.info(R_ab)
logger.info("Rotation of %.3f degrees" % angle +
" about axis (%.3f, %.3f, %.3f)" % axis)
#show_rotation_matrix_differences([cryst_a, cryst_b])
have_similar_crystal_models = True
del experiments[-1]
break
if have_similar_crystal_models:
break
self.indexed_reflections = (self.reflections['id'] > -1)
if self.d_min is None:
sel = self.reflections['id'] <= -1
else:
sel = flex.bool(len(self.reflections), False)
lengths = 1 / self.reflections['rlp'].norms()
isel = (lengths >= self.d_min).iselection()
sel.set_selected(isel, True)
sel.set_selected(self.reflections['id'] > -1, False)
self.unindexed_reflections = self.reflections.select(sel)
reflections_for_refinement = self.reflections.select(
self.indexed_reflections)
if len(self.params.stills.isoforms) > 0:
logger.info("")
logger.info("#" * 80)
logger.info("Starting refinement")
logger.info("#" * 80)
logger.info("")
import copy
isoform_experiments = ExperimentList()
isoform_reflections = flex.reflection_table()
# Note, changes to params after initial indexing. Cannot use tie to target when fixing the unit cell.
self.all_params.refinement.reflections.outlier.algorithm = "null"
self.all_params.refinement.parameterisation.crystal.fix = "cell"
self.all_params.refinement.parameterisation.crystal.unit_cell.restraints.tie_to_target = []
for expt_id, experiment in enumerate(experiments):
reflections = reflections_for_refinement.select(
reflections_for_refinement['id'] == expt_id)
reflections['id'] = flex.int(len(reflections), 0)
refiners = []
for isoform in self.params.stills.isoforms:
iso_experiment = copy.deepcopy(experiment)
crystal = iso_experiment.crystal
if isoform.lookup_symbol != crystal.get_space_group(
).type().lookup_symbol():
logger.info(
"Crystal isoform lookup_symbol %s does not match isoform %s lookup_symbol %s"
% (crystal.get_space_group().type(
).lookup_symbol(), isoform.name,
isoform.lookup_symbol))
continue
crystal.set_B(isoform.cell.fractionalization_matrix())
logger.info("Refining isoform %s" % isoform.name)
refiners.append(
e_refine(params=self.all_params,
experiments=ExperimentList(
[iso_experiment]),
reflections=reflections,
graph_verbose=False))
if len(refiners) == 0:
raise Sorry(
"No isoforms had a lookup symbol that matched")
positional_rmsds = [
math.sqrt(P.rmsds()[0]**2 + P.rmsds()[1]**2)
for P in refiners
]
logger.info("Positional rmsds for all isoforms:" +
str(positional_rmsds))
minrmsd_mm = min(positional_rmsds)
minindex = positional_rmsds.index(minrmsd_mm)
logger.info(
"The smallest rmsd is %5.1f um from isoform %s" %
(1000. * minrmsd_mm,
self.params.stills.isoforms[minindex].name))
if self.params.stills.isoforms[
minindex].rmsd_target_mm is not None:
logger.info("Asserting %f < %f" %
(minrmsd_mm, self.params.stills.
isoforms[minindex].rmsd_target_mm))
assert minrmsd_mm < self.params.stills.isoforms[
minindex].rmsd_target_mm
logger.info("Acceptable rmsd for isoform %s." %
(self.params.stills.isoforms[minindex].name))
if len(self.params.stills.isoforms) == 2:
logger.info(
"Rmsd gain over the other isoform %5.1f um." %
(1000. *
abs(positional_rmsds[0] - positional_rmsds[1])))
R = refiners[minindex]
# Now one last check to see if direct beam is out of bounds
if self.params.stills.isoforms[
minindex].beam_restraint is not None:
from scitbx import matrix
refined_beam = matrix.col(
R.get_experiments()
[0].detector[0].get_beam_centre_lab(
experiments[0].beam.get_s0())[0:2])
known_beam = matrix.col(
self.params.stills.isoforms[minindex].
beam_restraint)
logger.info(
"Asserting difference in refined beam center and expected beam center %f < %f"
%
((refined_beam - known_beam).length(), self.params.
stills.isoforms[minindex].rmsd_target_mm))
assert (refined_beam - known_beam
).length() < self.params.stills.isoforms[
minindex].rmsd_target_mm
# future--circle of confusion could be given as a separate length in mm instead of reusing rmsd_target
experiment = R.get_experiments()[0]
experiment.crystal.identified_isoform = self.params.stills.isoforms[
minindex].name
isoform_experiments.append(experiment)
reflections['id'] = flex.int(len(reflections), expt_id)
isoform_reflections.extend(reflections)
experiments = isoform_experiments
reflections_for_refinement = isoform_reflections
try:
refined_experiments, refined_reflections = self.refine(
experiments, reflections_for_refinement)
except Exception as e:
s = str(e)
if len(experiments) == 1:
raise Sorry(e)
had_refinement_error = True
logger.info("Refinement failed:")
logger.info(s)
del experiments[-1]
break
# sanity check for unrealistic unit cell volume increase during refinement
# usually this indicates too many parameters are being refined given the
# number of observations provided.
if not self.params.refinement_protocol.disable_unit_cell_volume_sanity_check:
for orig_expt, refined_expt in zip(experiments,
refined_experiments):
uc1 = orig_expt.crystal.get_unit_cell()
uc2 = refined_expt.crystal.get_unit_cell()
volume_change = abs(uc1.volume() -
uc2.volume()) / uc1.volume()
cutoff = 0.5
if volume_change > cutoff:
msg = "\n".join((
"Unrealistic unit cell volume increase during refinement of %.1f%%.",
"Please try refining fewer parameters, either by enforcing symmetry",
"constraints (space_group=) and/or disabling experimental geometry",
"refinement (detector.fix=all and beam.fix=all). To disable this",
"sanity check set disable_unit_cell_volume_sanity_check=True."
)) % (100 * volume_change)
raise Sorry(msg)
self.refined_reflections = refined_reflections.select(
refined_reflections['id'] > -1)
for i, imageset in enumerate(self.imagesets):
ref_sel = self.refined_reflections.select(
self.refined_reflections['imageset_id'] == i)
ref_sel = ref_sel.select(ref_sel['id'] >= 0)
for i_expt in set(ref_sel['id']):
expt = refined_experiments[i_expt]
imageset.set_detector(expt.detector)
imageset.set_beam(expt.beam)
imageset.set_goniometer(expt.goniometer)
imageset.set_scan(expt.scan)
expt.imageset = imageset
if not (self.all_params.refinement.parameterisation.beam.fix
== 'all' and
self.all_params.refinement.parameterisation.detector.fix
== 'all'):
# Experimental geometry may have changed - re-map centroids to
# reciprocal space
spots_mm = self.reflections
self.reflections = flex.reflection_table()
for i, imageset in enumerate(self.imagesets):
spots_sel = spots_mm.select(spots_mm['imageset_id'] == i)
self.map_centroids_to_reciprocal_space(
spots_sel, imageset.get_detector(),
imageset.get_beam(), imageset.get_goniometer())
self.reflections.extend(spots_sel)
# update for next cycle
experiments = refined_experiments
self.refined_experiments = refined_experiments
if not 'refined_experiments' in locals():
raise Sorry("None of the experiments could refine.")
# discard experiments with zero reflections after refinement
id_set = set(self.refined_reflections['id'])
if len(id_set) < len(self.refined_experiments):
filtered_refined_reflections = flex.reflection_table()
for i in xrange(len(self.refined_experiments)):
if i not in id_set:
del self.refined_experiments[i]
for old, new in zip(sorted(id_set), range(len(id_set))):
subset = self.refined_reflections.select(
self.refined_reflections['id'] == old)
subset['id'] = flex.int(len(subset), new)
filtered_refined_reflections.extend(subset)
self.refined_reflections = filtered_refined_reflections
if len(self.refined_experiments) > 1:
from dials.algorithms.indexing.compare_orientation_matrices \
import show_rotation_matrix_differences
show_rotation_matrix_differences(
self.refined_experiments.crystals(), out=info_handle)
logger.info("Final refined crystal models:")
for i, crystal_model in enumerate(self.refined_experiments.crystals()):
n_indexed = 0
for i_expt in experiments.where(crystal=crystal_model):
n_indexed += (self.reflections['id'] == i).count(True)
logger.info("model %i (%i reflections):" % (i + 1, n_indexed))
logger.info(crystal_model)
if 'xyzcal.mm' in self.refined_reflections: # won't be there if refine_all_candidates = False and no isoforms
self.refined_reflections['xyzcal.px'] = flex.vec3_double(
len(self.refined_reflections))
for i, imageset in enumerate(self.imagesets):
imgset_sel = self.refined_reflections['imageset_id'] == i
# set xyzcal.px field in self.refined_reflections
refined_reflections = self.refined_reflections.select(
imgset_sel)
panel_numbers = flex.size_t(refined_reflections['panel'])
xyzcal_mm = refined_reflections['xyzcal.mm']
x_mm, y_mm, z_rad = xyzcal_mm.parts()
xy_cal_mm = flex.vec2_double(x_mm, y_mm)
xy_cal_px = flex.vec2_double(len(xy_cal_mm))
for i_panel in range(len(imageset.get_detector())):
panel = imageset.get_detector()[i_panel]
sel = (panel_numbers == i_panel)
isel = sel.iselection()
ref_panel = refined_reflections.select(
panel_numbers == i_panel)
xy_cal_px.set_selected(
sel, panel.millimeter_to_pixel(xy_cal_mm.select(sel)))
x_px, y_px = xy_cal_px.parts()
scan = imageset.get_scan()
if scan is not None:
z_px = scan.get_array_index_from_angle(z_rad, deg=False)
else:
# must be a still image, z centroid not meaningful
z_px = z_rad
xyzcal_px = flex.vec3_double(x_px, y_px, z_px)
self.refined_reflections['xyzcal.px'].set_selected(
imgset_sel, xyzcal_px)
def run(args=None):
import libtbx.load_env
from dials.util import Sorry
usage = "dials.reindex [options] indexed.expt indexed.refl"
parser = ArgumentParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message,
)
params, options = parser.parse_args(args, show_diff_phil=True)
reflections, experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments
)
if len(experiments) == 0 and len(reflections) == 0:
parser.print_help()
return
if params.change_of_basis_op is None:
raise Sorry("Please provide a change_of_basis_op.")
reference_crystal = None
if params.reference.experiments is not None:
from dxtbx.serialize import load
reference_experiments = load.experiment_list(
params.reference.experiments, check_format=False
)
if len(reference_experiments.crystals()) == 1:
reference_crystal = reference_experiments.crystals()[0]
else:
# first check sg all same
sgs = [
expt.crystal.get_space_group().type().number() for expt in experiments
]
if len(set(sgs)) > 1:
raise Sorry(
"""The reference experiments have different space groups:
space group numbers found: %s
Please reanalyse the data so that space groups are consistent,
(consider using dials.reindex, dials.symmetry or dials.cosym)"""
% ", ".join(map(str, set(sgs)))
)
reference_crystal = reference_experiments.crystals()[0]
reference_crystal.unit_cell = determine_best_unit_cell(
reference_experiments
)
if params.reference.reflections is not None:
# First check that we have everything as expected for the reference reindexing
if params.reference.experiments is None:
raise Sorry(
"""For reindexing against a reference dataset, a reference
experiments file must also be specified with the option: reference.experiments= """
)
if not os.path.exists(params.reference.reflections):
raise Sorry("Could not locate reference dataset reflection file")
reference_reflections = flex.reflection_table().from_file(
params.reference.reflections
)
test_reflections = reflections[0]
if (
reference_crystal.get_space_group().type().number()
!= experiments.crystals()[0].get_space_group().type().number()
):
raise Sorry("Space group of input does not match reference")
# Set some flags to allow filtering, if wanting to reindex against
# reference with data that has not yet been through integration
if (
test_reflections.get_flags(test_reflections.flags.integrated_sum).count(
True
)
== 0
):
assert (
"intensity.sum.value" in test_reflections
), "No 'intensity.sum.value' in reflections"
test_reflections.set_flags(
flex.bool(test_reflections.size(), True),
test_reflections.flags.integrated_sum,
)
if (
reference_reflections.get_flags(
reference_reflections.flags.integrated_sum
).count(True)
== 0
):
assert (
"intensity.sum.value" in test_reflections
), "No 'intensity.sum.value in reference reflections"
reference_reflections.set_flags(
flex.bool(reference_reflections.size(), True),
reference_reflections.flags.integrated_sum,
)
# Make miller array of the two datasets
try:
test_miller_set = filtered_arrays_from_experiments_reflections(
experiments, [test_reflections]
)[0]
except ValueError:
raise Sorry("No reflections remain after filtering the test dataset")
try:
reference_miller_set = filtered_arrays_from_experiments_reflections(
reference_experiments, [reference_reflections]
)[0]
except ValueError:
raise Sorry("No reflections remain after filtering the reference dataset")
from dials.algorithms.symmetry.reindex_to_reference import (
determine_reindex_operator_against_reference,
)
change_of_basis_op = determine_reindex_operator_against_reference(
test_miller_set, reference_miller_set
)
elif len(experiments) and params.change_of_basis_op is libtbx.Auto:
if reference_crystal is not None:
if len(experiments.crystals()) > 1:
raise Sorry("Only one crystal can be processed at a time")
from dials.algorithms.indexing.compare_orientation_matrices import (
difference_rotation_matrix_axis_angle,
)
cryst = experiments.crystals()[0]
R, axis, angle, change_of_basis_op = difference_rotation_matrix_axis_angle(
cryst, reference_crystal
)
print(f"Change of basis op: {change_of_basis_op}")
print("Rotation matrix to transform input crystal to reference::")
print(R.mathematica_form(format="%.3f", one_row_per_line=True))
print(
f"Rotation of {angle:.3f} degrees",
"about axis (%.3f, %.3f, %.3f)" % axis,
)
elif len(reflections):
assert len(reflections) == 1
# always re-map reflections to reciprocal space
refl = reflections.deep_copy()
refl.centroid_px_to_mm(experiments)
refl.map_centroids_to_reciprocal_space(experiments)
# index the reflection list using the input experiments list
refl["id"] = flex.int(len(refl), -1)
index = AssignIndicesGlobal(tolerance=0.2)
index(refl, experiments)
hkl_expt = refl["miller_index"]
hkl_input = reflections[0]["miller_index"]
change_of_basis_op = derive_change_of_basis_op(hkl_input, hkl_expt)
# reset experiments list since we don't want to reindex this
experiments = []
else:
change_of_basis_op = sgtbx.change_of_basis_op(params.change_of_basis_op)
if len(experiments):
space_group = params.space_group
if space_group is not None:
space_group = space_group.group()
try:
experiments = reindex_experiments(
experiments, change_of_basis_op, space_group=space_group
)
except RuntimeError as e:
# Only catch specific errors here
if "Unsuitable value for rational rotation matrix." in str(e):
original_message = str(e).split(":")[-1].strip()
sys.exit(f"Error: {original_message} Is your change_of_basis_op valid?")
raise
print(f"Saving reindexed experimental models to {params.output.experiments}")
experiments.as_file(params.output.experiments)
if len(reflections):
assert len(reflections) == 1
reflections = reflections[0]
miller_indices = reflections["miller_index"]
if params.hkl_offset is not None:
h, k, l = miller_indices.as_vec3_double().parts()
h += params.hkl_offset[0]
k += params.hkl_offset[1]
l += params.hkl_offset[2]
miller_indices = flex.miller_index(h.iround(), k.iround(), l.iround())
non_integral_indices = change_of_basis_op.apply_results_in_non_integral_indices(
miller_indices
)
if non_integral_indices.size() > 0:
print(
"Removing %i/%i reflections (change of basis results in non-integral indices)"
% (non_integral_indices.size(), miller_indices.size())
)
sel = flex.bool(miller_indices.size(), True)
sel.set_selected(non_integral_indices, False)
miller_indices_reindexed = change_of_basis_op.apply(miller_indices.select(sel))
reflections["miller_index"].set_selected(sel, miller_indices_reindexed)
reflections["miller_index"].set_selected(~sel, (0, 0, 0))
print(f"Saving reindexed reflections to {params.output.reflections}")
reflections.as_file(params.output.reflections)
