def test(dials_regression):
from iotbx.xds import xparm, integrate_hkl
from dials.util import ioutil
from dials.algorithms.spot_prediction import IndexGenerator
import numpy
from rstbx.cftbx.coordinate_frame_converter import \
coordinate_frame_converter
from scitbx import matrix
# The XDS files to read from
integrate_filename = os.path.join(dials_regression, 'data', 'sim_mx',
'INTEGRATE.HKL')
gxparm_filename = os.path.join(dials_regression, 'data', 'sim_mx',
'GXPARM.XDS')
# Read the XDS files
integrate_handle = integrate_hkl.reader()
integrate_handle.read_file(integrate_filename)
gxparm_handle = xparm.reader()
gxparm_handle.read_file(gxparm_filename)
# Get the parameters we need from the GXPARM file
d_min = 1.6
space_group_type = ioutil.get_space_group_type_from_xparm(gxparm_handle)
cfc = coordinate_frame_converter(gxparm_filename)
a_vec = cfc.get('real_space_a')
b_vec = cfc.get('real_space_b')
c_vec = cfc.get('real_space_c')
unit_cell = cfc.get_unit_cell()
UB = matrix.sqr(a_vec + b_vec + c_vec).inverse()
ub_matrix = UB
# Generate the indices
index_generator = IndexGenerator(unit_cell, space_group_type, d_min)
miller_indices = index_generator.to_array()
# Get individual generated hkl
gen_h = [hkl[0] for hkl in miller_indices]
gen_k = [hkl[1] for hkl in miller_indices]
gen_l = [hkl[2] for hkl in miller_indices]
# Get individual xds generated hkl
xds_h = [hkl[0] for hkl in integrate_handle.hkl]
xds_k = [hkl[1] for hkl in integrate_handle.hkl]
xds_l = [hkl[2] for hkl in integrate_handle.hkl]
# Get min/max generated hkl
min_gen_h, max_gen_h = numpy.min(gen_h), numpy.max(gen_h)
min_gen_k, max_gen_k = numpy.min(gen_k), numpy.max(gen_k)
min_gen_l, max_gen_l = numpy.min(gen_l), numpy.max(gen_l)
# Get min/max xds generated hkl
min_xds_h, max_xds_h = numpy.min(xds_h), numpy.max(xds_h)
min_xds_k, max_xds_k = numpy.min(xds_k), numpy.max(xds_k)
min_xds_l, max_xds_l = numpy.min(xds_l), numpy.max(xds_l)
# Ensure we have the whole xds range in the generated set
assert min_gen_h <= min_xds_h and max_gen_h >= max_xds_h
assert min_gen_k <= min_xds_k and max_gen_k >= max_xds_k
assert min_gen_l <= min_xds_l and max_gen_l >= max_xds_l
def test(dials_regression):
import numpy as np
from iotbx.xds import integrate_hkl, xparm
from rstbx.cftbx.coordinate_frame_converter import coordinate_frame_converter
from dials.algorithms.spot_prediction import IndexGenerator
from dials.util import ioutil
# The XDS files to read from
integrate_filename = os.path.join(dials_regression, "data", "sim_mx",
"INTEGRATE.HKL")
gxparm_filename = os.path.join(dials_regression, "data", "sim_mx",
"GXPARM.XDS")
# Read the XDS files
integrate_handle = integrate_hkl.reader()
integrate_handle.read_file(integrate_filename)
gxparm_handle = xparm.reader()
gxparm_handle.read_file(gxparm_filename)
# Get the parameters we need from the GXPARM file
d_min = 1.6
space_group_type = ioutil.get_space_group_type_from_xparm(gxparm_handle)
cfc = coordinate_frame_converter(gxparm_filename)
unit_cell = cfc.get_unit_cell()
# Generate the indices
index_generator = IndexGenerator(unit_cell, space_group_type, d_min)
miller_indices = index_generator.to_array()
# Get individual generated hkl
gen_h = [hkl[0] for hkl in miller_indices]
gen_k = [hkl[1] for hkl in miller_indices]
gen_l = [hkl[2] for hkl in miller_indices]
# Get individual xds generated hkl
xds_h = [hkl[0] for hkl in integrate_handle.hkl]
xds_k = [hkl[1] for hkl in integrate_handle.hkl]
xds_l = [hkl[2] for hkl in integrate_handle.hkl]
# Get min/max generated hkl
min_gen_h, max_gen_h = np.min(gen_h), np.max(gen_h)
min_gen_k, max_gen_k = np.min(gen_k), np.max(gen_k)
min_gen_l, max_gen_l = np.min(gen_l), np.max(gen_l)
# Get min/max xds generated hkl
min_xds_h, max_xds_h = np.min(xds_h), np.max(xds_h)
min_xds_k, max_xds_k = np.min(xds_k), np.max(xds_k)
min_xds_l, max_xds_l = np.min(xds_l), np.max(xds_l)
# Ensure we have the whole xds range in the generated set
assert min_gen_h <= min_xds_h and max_gen_h >= max_xds_h
assert min_gen_k <= min_xds_k and max_gen_k >= max_xds_k
assert min_gen_l <= min_xds_l and max_gen_l >= max_xds_l
def run():
from iotbx.xds import xparm, integrate_hkl
from dials.util import ioutil
from dials.algorithms.spot_prediction import IndexGenerator
from os.path import realpath, dirname, join
import numpy
from rstbx.cftbx.coordinate_frame_converter import \
coordinate_frame_converter
from scitbx import matrix
# The XDS files to read from
test_path = dirname(dirname(dirname(realpath(__file__))))
integrate_filename = join(test_path, 'data/sim_mx/INTEGRATE.HKL')
gxparm_filename = join(test_path, 'data/sim_mx/GXPARM.XDS')
# Read the XDS files
integrate_handle = integrate_hkl.reader()
integrate_handle.read_file(integrate_filename)
gxparm_handle = xparm.reader()
gxparm_handle.read_file(gxparm_filename)
# Get the parameters we need from the GXPARM file
d_min = 1.6
space_group_type = ioutil.get_space_group_type_from_xparm(gxparm_handle)
cfc = coordinate_frame_converter(gxparm_filename)
a_vec = cfc.get('real_space_a')
b_vec = cfc.get('real_space_b')
c_vec = cfc.get('real_space_c')
unit_cell = cfc.get_unit_cell()
UB = matrix.sqr(a_vec + b_vec + c_vec).inverse()
ub_matrix = UB
# Generate the indices
index_generator = IndexGenerator(unit_cell, space_group_type, d_min)
miller_indices = index_generator.to_array()
# Get individual generated hkl
gen_h = [hkl[0] for hkl in miller_indices]
gen_k = [hkl[1] for hkl in miller_indices]
gen_l = [hkl[2] for hkl in miller_indices]
# Get individual xds generated hkl
xds_h = [hkl[0] for hkl in integrate_handle.hkl]
xds_k = [hkl[0] for hkl in integrate_handle.hkl]
xds_l = [hkl[0] for hkl in integrate_handle.hkl]
# Get min/max generated hkl
min_gen_h, max_gen_h = numpy.min(gen_h), numpy.max(gen_h)
min_gen_k, max_gen_k = numpy.min(gen_k), numpy.max(gen_k)
min_gen_l, max_gen_l = numpy.min(gen_l), numpy.max(gen_l)
# Get min/max xds generated hkl
min_xds_h, max_xds_h = numpy.min(xds_h), numpy.max(xds_h)
min_xds_k, max_xds_k = numpy.min(xds_k), numpy.max(xds_k)
min_xds_l, max_xds_l = numpy.min(xds_l), numpy.max(xds_l)
# Ensure we have the whole xds range in the generated set
assert(min_gen_h <= min_xds_h and max_gen_h >= max_xds_h)
assert(min_gen_k <= min_xds_k and max_gen_k >= max_xds_k)
assert(min_gen_l <= min_xds_l and max_gen_l >= max_xds_l)
# Test Passed
print "OK"
def __init__(self):
from dials.algorithms.spot_prediction import IndexGenerator
from dials.algorithms.spot_prediction import ScanStaticRayPredictor
from dials.algorithms.spot_prediction import ray_intersection
from iotbx.xds import xparm, integrate_hkl
from dials.util import ioutil
from math import ceil
import dxtbx
from rstbx.cftbx.coordinate_frame_converter import \
coordinate_frame_converter
from scitbx import matrix
# The XDS files to read from
integrate_filename = join(dials_regression,
'data/sim_mx/INTEGRATE.HKL')
gxparm_filename = join(dials_regression, 'data/sim_mx/GXPARM.XDS')
# Read the XDS files
self.integrate_handle = integrate_hkl.reader()
self.integrate_handle.read_file(integrate_filename)
self.gxparm_handle = xparm.reader()
self.gxparm_handle.read_file(gxparm_filename)
# Get the parameters we need from the GXPARM file
models = dxtbx.load(gxparm_filename)
self.beam = models.get_beam()
self.gonio = models.get_goniometer()
self.detector = models.get_detector()
self.scan = models.get_scan()
assert (len(self.detector) == 1)
#print self.detector
# Get crystal parameters
self.space_group_type = ioutil.get_space_group_type_from_xparm(
self.gxparm_handle)
cfc = coordinate_frame_converter(gxparm_filename)
a_vec = cfc.get('real_space_a')
b_vec = cfc.get('real_space_b')
c_vec = cfc.get('real_space_c')
self.unit_cell = cfc.get_unit_cell()
self.ub_matrix = matrix.sqr(a_vec + b_vec + c_vec).inverse()
# Get the minimum resolution in the integrate file
self.d_min = self.detector[0].get_max_resolution_at_corners(
self.beam.get_s0())
# Get the number of frames from the max z value
xcal, ycal, zcal = zip(*self.integrate_handle.xyzcal)
self.scan.set_image_range(
(self.scan.get_image_range()[0],
self.scan.get_image_range()[0] + int(ceil(max(zcal)))))
# Create the index generator
generate_indices = IndexGenerator(self.unit_cell,
self.space_group_type, self.d_min)
s0 = self.beam.get_s0()
m2 = self.gonio.get_rotation_axis()
fixed_rotation = self.gonio.get_fixed_rotation()
setting_rotation = self.gonio.get_setting_rotation()
UB = self.ub_matrix
dphi = self.scan.get_oscillation_range(deg=False)
# Create the ray predictor
self.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation,
setting_rotation, dphi)
# Predict the spot locations
self.reflections = self.predict_rays(generate_indices.to_array(), UB)
# Calculate the intersection of the detector and reflection frames
success = ray_intersection(self.detector, self.reflections)
self.reflections.select(success)
def __init__(self, dials_regression):
import dxtbx
from iotbx.xds import integrate_hkl, xparm
from rstbx.cftbx.coordinate_frame_converter import coordinate_frame_converter
from dials.algorithms.spot_prediction import (
IndexGenerator,
ScanStaticRayPredictor,
)
from dials.util import ioutil
# The XDS files to read from
integrate_filename = os.path.join(dials_regression, "data", "sim_mx",
"INTEGRATE.HKL")
gxparm_filename = os.path.join(dials_regression, "data", "sim_mx",
"GXPARM.XDS")
# Read the XDS files
self.integrate_handle = integrate_hkl.reader()
self.integrate_handle.read_file(integrate_filename)
self.gxparm_handle = xparm.reader()
self.gxparm_handle.read_file(gxparm_filename)
# Get the parameters we need from the GXPARM file
models = dxtbx.load(gxparm_filename)
self.beam = models.get_beam()
self.gonio = models.get_goniometer()
self.detector = models.get_detector()
self.scan = models.get_scan()
# Get crystal parameters
self.space_group_type = ioutil.get_space_group_type_from_xparm(
self.gxparm_handle)
cfc = coordinate_frame_converter(gxparm_filename)
a_vec = cfc.get("real_space_a")
b_vec = cfc.get("real_space_b")
c_vec = cfc.get("real_space_c")
self.unit_cell = cfc.get_unit_cell()
self.ub_matrix = matrix.sqr(a_vec + b_vec + c_vec).inverse()
# Get the minimum resolution in the integrate file
d = [self.unit_cell.d(h) for h in self.integrate_handle.hkl]
self.d_min = min(d)
# extend the resolution shell by epsilon>0
# to account for rounding artifacts on 32-bit platforms
self.d_min = self.d_min - 1e-15
# Get the number of frames from the max z value
xcal, ycal, zcal = zip(*self.integrate_handle.xyzcal)
self.scan.set_image_range((
self.scan.get_image_range()[0],
self.scan.get_image_range()[0] + int(math.ceil(max(zcal))),
))
# Print stuff
# print self.beam
# print self.gonio
# print self.detector
# print self.scan
# Create the index generator
self.generate_indices = IndexGenerator(self.unit_cell,
self.space_group_type,
self.d_min)
s0 = self.beam.get_s0()
m2 = self.gonio.get_rotation_axis()
fixed_rotation = self.gonio.get_fixed_rotation()
setting_rotation = self.gonio.get_setting_rotation()
UB = self.ub_matrix
dphi = self.scan.get_oscillation_range(deg=False)
# Create the ray predictor
self.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation,
setting_rotation, dphi)
# Predict the spot locations
self.reflections = self.predict_rays(self.generate_indices.to_array(),
UB)
def _integrate_finish(self):
'''Finish off the integration by running correct.'''
# first run the postrefinement etc with spacegroup P1
# and the current unit cell - this will be used to
# obtain a benchmark rmsd in pixels / phi and also
# cell deviations (this is working towards spotting bad
# indexing solutions) - only do this if we have no
# reindex matrix... and no postrefined cell...
p1_deviations = None
# fix for bug # 3264 -
# if we have not run integration with refined parameters, make it so...
# erm? shouldn't this therefore return if this is the principle, or
# set the flag after we have tested the lattice?
if not self._xds_data_files.has_key('GXPARM.XDS') and \
PhilIndex.params.xds.integrate.reintegrate:
Debug.write(
'Resetting integrater, to ensure refined orientation is used')
self.set_integrater_done(False)
if not self.get_integrater_reindex_matrix() and not self._intgr_cell \
and not Flags.get_no_lattice_test() and \
not self.get_integrater_sweep().get_user_lattice():
correct = self.Correct()
correct.set_data_range(self._intgr_wedge[0],
self._intgr_wedge[1])
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# FIXME should this be using the correctly transformed
# cell or are the results ok without it?!
correct.set_spacegroup_number(1)
correct.set_cell(self._intgr_refiner_cell)
correct.run()
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file('%s %s %s %s CORRECT' % \
(pname, xname, dname, sweep),
os.path.join(
self.get_working_directory(),
'CORRECT.LP'))
FileHandler.record_more_data_file(
'%s %s %s %s CORRECT' % (pname, xname, dname, sweep),
os.path.join(self.get_working_directory(), 'XDS_ASCII.HKL'))
cell = correct.get_result('cell')
cell_esd = correct.get_result('cell_esd')
Debug.write('Postrefinement in P1 results:')
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(cell))
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(cell_esd))
Debug.write('Deviations: %.2f pixels %.2f degrees' % \
(correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi')))
p1_deviations = (correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi'))
# next run the postrefinement etc with the given
# cell / lattice - this will be the assumed result...
correct = self.Correct()
correct.set_data_range(self._intgr_wedge[0],
self._intgr_wedge[1])
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# BUG # 2695 probably comes from here - need to check...
# if the pointless interface comes back with a different
# crystal setting then the unit cell stored in self._intgr_cell
# needs to be set to None...
if self.get_integrater_spacegroup_number():
correct.set_spacegroup_number(
self.get_integrater_spacegroup_number())
if not self._intgr_cell:
raise RuntimeError, 'no unit cell to recycle'
correct.set_cell(self._intgr_cell)
# BUG # 3113 - new version of XDS will try and figure the
# best spacegroup out from the intensities (and get it wrong!)
# unless we set the spacegroup and cell explicitly
if not self.get_integrater_spacegroup_number():
cell = self._intgr_refiner_cell
lattice = self._intgr_refiner.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup_number(lattice)
# this should not prevent the postrefinement from
# working correctly, else what is above would not
# work correctly (the postrefinement test)
correct.set_spacegroup_number(spacegroup_number)
correct.set_cell(cell)
Debug.write('Setting spacegroup to: %d' % spacegroup_number)
Debug.write(
'Setting cell to: %.2f %.2f %.2f %.2f %.2f %.2f' % tuple(cell))
if self.get_integrater_reindex_matrix():
# bug! if the lattice is not primitive the values in this
# reindex matrix need to be multiplied by a constant which
# depends on the Bravais lattice centering.
lattice = self._intgr_refiner.get_refiner_lattice()
import scitbx.matrix
matrix = self.get_integrater_reindex_matrix()
matrix = scitbx.matrix.sqr(matrix).transpose().elems
matrix = r_to_rt(matrix)
if lattice[1] == 'P':
mult = 1
elif lattice[1] == 'C' or lattice[1] == 'I':
mult = 2
elif lattice[1] == 'R':
mult = 3
elif lattice[1] == 'F':
mult = 4
else:
raise RuntimeError, 'unknown multiplier for lattice %s' % \
lattice
Debug.write('REIDX multiplier for lattice %s: %d' % \
(lattice, mult))
mult_matrix = [mult * m for m in matrix]
Debug.write('REIDX set to %d %d %d %d %d %d %d %d %d %d %d %d' % \
tuple(mult_matrix))
correct.set_reindex_matrix(mult_matrix)
correct.run()
# erm. just to be sure
if self.get_integrater_reindex_matrix() and \
correct.get_reindex_used():
raise RuntimeError, 'Reindex panic!'
# get the reindex operation used, which may be useful if none was
# set but XDS decided to apply one, e.g. #419.
if not self.get_integrater_reindex_matrix() and \
correct.get_reindex_used():
# convert this reindex operation to h, k, l form: n.b. this
# will involve dividing through by the lattice centring multiplier
matrix = rt_to_r(correct.get_reindex_used())
import scitbx.matrix
matrix = scitbx.matrix.sqr(matrix).transpose().elems
lattice = self._intgr_refiner.get_refiner_lattice()
if lattice[1] == 'P':
mult = 1.0
elif lattice[1] == 'C' or lattice[1] == 'I':
mult = 2.0
elif lattice[1] == 'R':
mult = 3.0
elif lattice[1] == 'F':
mult = 4.0
matrix = [m / mult for m in matrix]
reindex_op = mat_to_symop(matrix)
# assign this to self: will this reset?! make for a leaky
# abstraction and just assign this...
# self.set_integrater_reindex_operator(reindex)
self._intgr_reindex_operator = reindex_op
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file('%s %s %s %s CORRECT' % \
(pname, xname, dname, sweep),
os.path.join(self.get_working_directory(),
'CORRECT.LP'))
# should get some interesting stuff from the XDS correct file
# here, for instance the resolution range to use in integration
# (which should be fed back if not fast) and so on...
self._intgr_corrected_hklout = os.path.join(self.get_working_directory(),
'XDS_ASCII.HKL')
# also record the batch range - needed for the analysis of the
# radiation damage in chef...
self._intgr_batches_out = (self._intgr_wedge[0],
self._intgr_wedge[1])
# FIXME perhaps I should also feedback the GXPARM file here??
for file in ['GXPARM.XDS']:
self._xds_data_files[file] = correct.get_output_data_file(file)
# record the postrefined cell parameters
self._intgr_cell = correct.get_result('cell')
self._intgr_n_ref = correct.get_result('n_ref')
Debug.write('Postrefinement in "correct" spacegroup results:')
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(correct.get_result('cell')))
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(correct.get_result('cell_esd')))
Debug.write('Deviations: %.2f pixels %.2f degrees' % \
(correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi')))
Debug.write('Error correction parameters: A=%.3f B=%.3f' % \
correct.get_result('sdcorrection'))
# compute misorientation of axes
xparm_file = os.path.join(self.get_working_directory(), 'GXPARM.XDS')
from iotbx.xds import xparm
handle = xparm.reader()
handle.read_file(xparm_file)
rotn = handle.rotation_axis
beam = handle.beam_vector
dot = sum([rotn[j] * beam[j] for j in range(3)])
r = math.sqrt(sum([rotn[j] * rotn[j] for j in range(3)]))
b = math.sqrt(sum([beam[j] * beam[j] for j in range(3)]))
rtod = 180.0 / math.pi
angle = rtod * math.fabs(0.5 * math.pi - math.acos(dot / (r * b)))
Debug.write('Axis misalignment %.2f degrees' % angle)
correct_deviations = (correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi'))
if p1_deviations:
# compare and reject if both > 50% higher - though adding a little
# flexibility - 0.5 pixel / osc width slack.
pixel = p1_deviations[0]
phi = math.sqrt(0.05 * 0.05 + \
p1_deviations[1] * p1_deviations[1])
threshold = Flags.get_rejection_threshold()
Debug.write('RMSD ratio: %.2f' % (correct_deviations[0] / pixel))
Debug.write('RMSPhi ratio: %.2f' % (correct_deviations[1] / phi))
if correct_deviations[0] / pixel > threshold and \
correct_deviations[1] / phi > threshold:
Chatter.write(
'Eliminating this indexing solution as postrefinement')
Chatter.write(
'deviations rather high relative to triclinic')
raise BadLatticeError, \
'high relative deviations in postrefinement'
if not Flags.get_quick() and Flags.get_remove():
# check for alien reflections and perhaps recycle - removing them
if len(correct.get_remove()) > 0:
correct_remove = correct.get_remove()
current_remove = []
final_remove = []
# first ensure that there are no duplicate entries...
if os.path.exists(os.path.join(
self.get_working_directory(),
'REMOVE.HKL')):
for line in open(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'), 'r').readlines():
h, k, l = map(int, line.split()[:3])
z = float(line.split()[3])
if not (h, k, l, z) in current_remove:
current_remove.append((h, k, l, z))
for c in correct_remove:
if c in current_remove:
continue
final_remove.append(c)
Debug.write(
'%d alien reflections are already removed' % \
(len(correct_remove) - len(final_remove)))
else:
# we want to remove all of the new dodgy reflections
final_remove = correct_remove
remove_hkl = open(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'), 'w')
z_min = Flags.get_z_min()
rejected = 0
# write in the old reflections
for remove in current_remove:
z = remove[3]
if z >= z_min:
remove_hkl.write('%d %d %d %f\n' % remove)
else:
rejected += 1
Debug.write('Wrote %d old reflections to REMOVE.HKL' % \
(len(current_remove) - rejected))
Debug.write('Rejected %d as z < %f' % \
(rejected, z_min))
# and the new reflections
rejected = 0
used = 0
for remove in final_remove:
z = remove[3]
if z >= z_min:
used += 1
remove_hkl.write('%d %d %d %f\n' % remove)
else:
rejected += 1
Debug.write('Wrote %d new reflections to REMOVE.HKL' % \
(len(final_remove) - rejected))
Debug.write('Rejected %d as z < %f' % \
(rejected, z_min))
remove_hkl.close()
# we want to rerun the finishing step so...
# unless we have added no new reflections... or unless we
# have not confirmed the point group (see SCI-398)
if used and self.get_integrater_reindex_matrix():
self.set_integrater_finish_done(False)
else:
Debug.write(
'Going quickly so not removing %d outlier reflections...' % \
len(correct.get_remove()))
# Convert INTEGRATE.HKL to MTZ format and reapply any reindexing operations
# spacegroup changes to allow use with CCP4 / Aimless for scaling
integrate_hkl = os.path.join(
self.get_working_directory(), 'INTEGRATE.HKL')
hklout = os.path.splitext(integrate_hkl)[0] + ".mtz"
self._factory.set_working_directory(self.get_working_directory())
pointless = self._factory.Pointless()
pointless.set_xdsin(integrate_hkl)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
integrate_mtz = hklout
if self.get_integrater_reindex_operator() or \
self.get_integrater_spacegroup_number():
Debug.write('Reindexing things to MTZ')
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
if self.get_integrater_reindex_operator():
reindex.set_operator(self.get_integrater_reindex_operator())
if self.get_integrater_spacegroup_number():
reindex.set_spacegroup(self.get_integrater_spacegroup_number())
hklout = '%s_reindex.mtz' % os.path.splitext(integrate_mtz)[0]
reindex.set_hklin(integrate_mtz)
reindex.set_hklout(hklout)
reindex.reindex()
integrate_mtz = hklout
return integrate_mtz
def test(dials_regression, run_in_tmpdir):
import dxtbx
from iotbx.xds import integrate_hkl, xparm
from rstbx.cftbx.coordinate_frame_converter import coordinate_frame_converter
from dials.algorithms.spot_prediction import RotationAngles
# The XDS files to read from
integrate_filename = os.path.join(dials_regression,
"data/sim_mx/INTEGRATE.HKL")
gxparm_filename = os.path.join(dials_regression, "data/sim_mx/GXPARM.XDS")
# Read the XDS files
integrate_handle = integrate_hkl.reader()
integrate_handle.read_file(integrate_filename)
gxparm_handle = xparm.reader()
gxparm_handle.read_file(gxparm_filename)
# Get the parameters we need from the GXPARM file
models = dxtbx.load(gxparm_filename)
beam = models.get_beam()
gonio = models.get_goniometer()
scan = models.get_scan()
# Get the crystal parameters
cfc = coordinate_frame_converter(gxparm_filename)
a_vec = cfc.get("real_space_a")
b_vec = cfc.get("real_space_b")
c_vec = cfc.get("real_space_c")
UB = matrix.sqr(a_vec + b_vec + c_vec).inverse()
ub_matrix = UB
# Get the number of frames from the max z value
xcal, ycal, zcal = zip(*integrate_handle.xyzcal)
num_frames = int(math.ceil(max(zcal)))
scan.set_image_range((scan.get_image_range()[0],
scan.get_image_range()[0] + num_frames - 1))
# Create the rotation angle object
ra = RotationAngles(beam.get_s0(), gonio.get_rotation_axis())
# Setup the matrices
ub = matrix.sqr(ub_matrix)
s0 = matrix.col(beam.get_s0())
m2 = matrix.col(gonio.get_rotation_axis())
# For all the miller indices
for h in integrate_handle.hkl:
h = matrix.col(h)
# Calculate the angles
angles = ra(h, ub)
# For all the angles
for phi in angles:
r = m2.axis_and_angle_as_r3_rotation_matrix(angle=phi)
pstar = r * ub * h
s1 = s0 + pstar
assert s1.length() == pytest.approx(s0.length(), abs=1e-7)
# Create a dict of lists of xy for each hkl
gen_phi = {}
for h in integrate_handle.hkl:
# Calculate the angles
angles = ra(h, ub)
gen_phi[h] = angles
# for phi in angles:
# try:
# a = gen_phi[h]
# a.append(phi)
# gen_phi[h] = a
# except KeyError:
# gen_phi[h] = [phi]
# For each hkl in the xds file
for hkl, xyz in zip(integrate_handle.hkl, integrate_handle.xyzcal):
# Calculate the XDS phi value
xds_phi = (scan.get_oscillation(deg=False)[0] +
xyz[2] * scan.get_oscillation(deg=False)[1])
# Select the nearest xy to use if there are 2
my_phi = gen_phi[hkl]
if len(my_phi) == 2:
my_phi0 = my_phi[0]
my_phi1 = my_phi[1]
diff0 = abs(xds_phi - my_phi0)
diff1 = abs(xds_phi - my_phi1)
if diff0 < diff1:
my_phi = my_phi0
else:
my_phi = my_phi1
else:
my_phi = my_phi[0]
# Check the Phi values are the same
assert xds_phi == pytest.approx(my_phi, abs=0.1)
def __init__(self):
from dials.algorithms.spot_prediction import IndexGenerator
from dials.algorithms.spot_prediction import ScanStaticRayPredictor
from iotbx.xds import xparm, integrate_hkl
from dials.util import ioutil
from math import ceil
import dxtbx
from rstbx.cftbx.coordinate_frame_converter import \
coordinate_frame_converter
from scitbx import matrix
# The XDS files to read from
integrate_filename = join(dials_regression, 'data/sim_mx/INTEGRATE.HKL')
gxparm_filename = join(dials_regression, 'data/sim_mx/GXPARM.XDS')
# Read the XDS files
self.integrate_handle = integrate_hkl.reader()
self.integrate_handle.read_file(integrate_filename)
self.gxparm_handle = xparm.reader()
self.gxparm_handle.read_file(gxparm_filename)
# Get the parameters we need from the GXPARM file
models = dxtbx.load(gxparm_filename)
self.beam = models.get_beam()
self.gonio = models.get_goniometer()
self.detector = models.get_detector()
self.scan = models.get_scan()
# Get crystal parameters
self.space_group_type = ioutil.get_space_group_type_from_xparm(
self.gxparm_handle)
cfc = coordinate_frame_converter(gxparm_filename)
a_vec = cfc.get('real_space_a')
b_vec = cfc.get('real_space_b')
c_vec = cfc.get('real_space_c')
self.unit_cell = cfc.get_unit_cell()
self.ub_matrix = matrix.sqr(a_vec + b_vec + c_vec).inverse()
# Get the minimum resolution in the integrate file
d = [self.unit_cell.d(h) for h in self.integrate_handle.hkl]
self.d_min = min(d)
# extend the resolution shell by epsilon>0
# to account for rounding artifacts on 32-bit platforms
self.d_min = self.d_min - 1e-15
# Get the number of frames from the max z value
xcal, ycal, zcal = zip(*self.integrate_handle.xyzcal)
self.scan.set_image_range((self.scan.get_image_range()[0],
self.scan.get_image_range()[0] +
int(ceil(max(zcal)))))
# Print stuff
# print self.beam
# print self.gonio
# print self.detector
# print self.scan
# Create the index generator
self.generate_indices = IndexGenerator(self.unit_cell,
self.space_group_type, self.d_min)
s0 = self.beam.get_s0()
m2 = self.gonio.get_rotation_axis()
fixed_rotation = self.gonio.get_fixed_rotation()
setting_rotation = self.gonio.get_setting_rotation()
UB = self.ub_matrix
dphi = self.scan.get_oscillation_range(deg=False)
# Create the ray predictor
self.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation,
setting_rotation, dphi)
# Predict the spot locations
self.reflections = self.predict_rays(
self.generate_indices.to_array(), UB)
def import_xds_xparm(xparm_file):
'''Read an XDS XPARM file, transform the parameters contained therein
into the standard coordinate frame, record this as a dictionary.'''
from iotbx.xds import xparm
handle = xparm.reader()
handle.read_file(xparm_file)
# first determine the rotation R from the XDS coordinate frame used in
# the processing to the central (i.e. imgCIF) coordinate frame. N.B.
# if the scan was e.g. a PHI scan the resulting frame could well come out
# a little odd...
axis = handle.rotation_axis
beam = handle.beam_vector
x, y = handle.detector_x_axis, handle.detector_y_axis
# XDS defines the beam vector as s0 rather than from sample -> source.
B = - matrix.col(beam).normalize()
A = matrix.col(axis).normalize()
X = matrix.col(x).normalize()
Y = matrix.col(y).normalize()
N = X.cross(Y)
_X = matrix.col([1, 0, 0])
_Y = matrix.col([0, 1, 0])
_Z = matrix.col([0, 0, 1])
R = align_reference_frame(A, _X, B, _Z)
# now transform contents of the XPARM file to the form which we want to
# return...
nx, ny = handle.detector_size
px, py = handle.pixel_size
distance = handle.detector_distance
ox, oy = handle.detector_origin
a = handle.unit_cell_a_axis
b = handle.unit_cell_b_axis
c = handle.unit_cell_c_axis
# Need to subtract 0.5 because XDS seems to do centroids in fortran coords
ox = ox - 0.5
oy = oy - 0.5
detector_origin = R * (distance * N - ox * px * X - oy * py * Y)
detector_fast = R * X
detector_slow = R * Y
rotation_axis = R * A
sample_to_source = R * B
wavelength = handle.wavelength
real_space_a = R * matrix.col(a)
real_space_b = R * matrix.col(b)
real_space_c = R * matrix.col(c)
space_group_number = handle.space_group
starting_angle = handle.starting_angle
oscillation_range = handle.oscillation_range
starting_frame = handle.starting_frame
panel_offset = None
panel_size = None
panel_origins = None
panel_fast_axes = None
panel_slow_axes = None
if handle.num_segments > 1:
# Now, for each detector segment the following two lines of information
# are provided.
# The 5 numbers of this line, iseg x1 x2 y1 y2, define the pixel numbers
# IX,IY belonging to segment #iseg as x1<=IX<=x2, y1<=IY<=y2.
# The 9 numbers of this line, ORGXS ORGYS FS EDS(:,1) EDS(:,2), describe
# origin and orientation of segment #iseg with respect to the detector
# coordinate system.
panel_offset = []
panel_size = []
panel_origins = []
panel_fast_axes = []
panel_slow_axes = []
for segment, orientation, in zip(handle.segments, handle.orientation):
iseg, x1, x2, y1, y2 = segment
# XDS panel limits inclusive range
panel_offset.append((x1-1, y1-1))
panel_size.append((x2-x1+1, y2-y1+1))
panel_fast = matrix.col(orientation[3:6])
panel_slow = matrix.col(orientation[6:9])
# local basis vectors
fl = matrix.col(panel_fast)
sl = matrix.col(panel_slow)
nl = fl.cross(sl)
orgxs, orgys, fs = orientation[:3]
panel_origin = R * (- (orgxs - x1 + 1) * px * fl \
- (orgys - y1 + 1) * py * sl \
+ fs * nl ) \
+ detector_origin
# detector to laboratory transformation
ED = matrix.sqr(list(X) + list(Y) + list(N))
panel_normal = (R * panel_fast).cross(R * panel_slow)
panel_origins.append(panel_origin)
panel_fast_axes.append(R * ED * panel_fast)
panel_slow_axes.append(R * ED * panel_slow)
return coordinate_frame_information(
detector_origin, detector_fast, detector_slow, (nx, ny), (px, py),
rotation_axis, sample_to_source, wavelength,
real_space_a, real_space_b, real_space_c, space_group_number,
None, None, starting_angle, oscillation_range, starting_frame,
original_rotation=R,
panel_offset=panel_offset,
panel_size=panel_size,
panel_origin=panel_origins,
panel_fast=panel_fast_axes,
panel_slow=panel_slow_axes)
def run():
if not have_dials_regression:
print "Skipping test: dials_regression not available."
return
from scitbx import matrix
from iotbx.xds import xparm, integrate_hkl
from dials.util import ioutil
from math import ceil
from dials.algorithms.spot_prediction import RotationAngles
import dxtbx
from rstbx.cftbx.coordinate_frame_converter import \
coordinate_frame_converter
# The XDS files to read from
integrate_filename = join(dials_regression, 'data/sim_mx/INTEGRATE.HKL')
gxparm_filename = join(dials_regression, 'data/sim_mx/GXPARM.XDS')
# Read the XDS files
integrate_handle = integrate_hkl.reader()
integrate_handle.read_file(integrate_filename)
gxparm_handle = xparm.reader()
gxparm_handle.read_file(gxparm_filename)
# Get the parameters we need from the GXPARM file
models = dxtbx.load(gxparm_filename)
beam = models.get_beam()
gonio = models.get_goniometer()
detector = models.get_detector()
scan = models.get_scan()
# Get the crystal parameters
space_group_type = ioutil.get_space_group_type_from_xparm(gxparm_handle)
cfc = coordinate_frame_converter(gxparm_filename)
a_vec = cfc.get('real_space_a')
b_vec = cfc.get('real_space_b')
c_vec = cfc.get('real_space_c')
unit_cell = cfc.get_unit_cell()
UB = matrix.sqr(a_vec + b_vec + c_vec).inverse()
ub_matrix = UB
# Get the minimum resolution in the integrate file
d = [unit_cell.d(h) for h in integrate_handle.hkl]
d_min = min(d)
# Get the number of frames from the max z value
xcal, ycal, zcal = zip(*integrate_handle.xyzcal)
num_frames = int(ceil(max(zcal)))
scan.set_image_range((scan.get_image_range()[0],
scan.get_image_range()[0] + num_frames - 1))
# Create the rotation angle object
ra = RotationAngles(beam.get_s0(), gonio.get_rotation_axis())
# Setup the matrices
ub = matrix.sqr(ub_matrix)
s0 = matrix.col(beam.get_s0())
m2 = matrix.col(gonio.get_rotation_axis())
# For all the miller indices
for h in integrate_handle.hkl:
h = matrix.col(h)
# Calculate the angles
angles = ra(h, ub)
# For all the angles
for phi in angles:
r = m2.axis_and_angle_as_r3_rotation_matrix(angle=phi)
pstar = r * ub * h
s1 = s0 + pstar
assert(abs(s1.length() - s0.length()) < 1e-7)
print "OK"
# Create a dict of lists of xy for each hkl
gen_phi = {}
for h in integrate_handle.hkl:
# Calculate the angles
angles = ra(h, ub)
gen_phi[h] = angles
# for phi in angles:
# try:
# a = gen_phi[h]
# a.append(phi)
# gen_phi[h] = a
# except KeyError:
# gen_phi[h] = [phi]
# For each hkl in the xds file
for hkl, xyz in zip(integrate_handle.hkl,
integrate_handle.xyzcal):
# Calculate the XDS phi value
xds_phi = scan.get_oscillation(deg=False)[0] + \
xyz[2]*scan.get_oscillation(deg=False)[1]
# Select the nearest xy to use if there are 2
my_phi = gen_phi[hkl]
if len(my_phi) == 2:
my_phi0 = my_phi[0]
my_phi1 = my_phi[1]
diff0 = abs(xds_phi - my_phi0)
diff1 = abs(xds_phi - my_phi1)
if diff0 < diff1:
my_phi = my_phi0
else:
my_phi = my_phi1
else:
my_phi = my_phi[0]
# Check the Phi values are the same
assert(abs(xds_phi - my_phi) < 0.1)
# Test Passed
print "OK"
def _integrate_finish(self):
"""Finish off the integration by running correct."""
# first run the postrefinement etc with spacegroup P1
# and the current unit cell - this will be used to
# obtain a benchmark rmsd in pixels / phi and also
# cell deviations (this is working towards spotting bad
# indexing solutions) - only do this if we have no
# reindex matrix... and no postrefined cell...
p1_deviations = None
# fix for bug # 3264 -
# if we have not run integration with refined parameters, make it so...
# erm? shouldn't this therefore return if this is the principle, or
# set the flag after we have tested the lattice?
if ("GXPARM.XDS" not in self._xds_data_files
and PhilIndex.params.xds.integrate.reintegrate):
logger.debug(
"Resetting integrater, to ensure refined orientation is used")
self.set_integrater_done(False)
if (not self.get_integrater_reindex_matrix() and not self._intgr_cell
and PhilIndex.params.xia2.settings.lattice_rejection
and not self.get_integrater_sweep().get_user_lattice()):
correct = self.Correct()
correct.set_data_range(
self._intgr_wedge[0] + self.get_frame_offset(),
self._intgr_wedge[1] + self.get_frame_offset(),
)
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# FIXME should this be using the correctly transformed
# cell or are the results ok without it?!
correct.set_spacegroup_number(1)
correct.set_cell(self._intgr_refiner_cell)
correct.run()
cell = correct.get_result("cell")
cell_esd = correct.get_result("cell_esd")
logger.debug("Postrefinement in P1 results:")
logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" % tuple(cell))
logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
tuple(cell_esd))
logger.debug("Deviations: %.2f pixels %.2f degrees" %
(correct.get_result("rmsd_pixel"),
correct.get_result("rmsd_phi")))
p1_deviations = (
correct.get_result("rmsd_pixel"),
correct.get_result("rmsd_phi"),
)
# next run the postrefinement etc with the given
# cell / lattice - this will be the assumed result...
integrate_hkl = os.path.join(self.get_working_directory(),
"INTEGRATE.HKL")
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
from dxtbx.serialize import load
from dials.algorithms.shadowing.filter import filter_shadowed_reflections
experiments_json = xparm_xds_to_experiments_json(
os.path.join(self.get_working_directory(), "XPARM.XDS"),
self.get_working_directory(),
)
experiments = load.experiment_list(experiments_json,
check_format=True)
imageset = experiments[0].imageset
masker = (imageset.get_format_class().get_instance(
imageset.paths()[0]).get_masker())
if masker is not None:
integrate_filename = integrate_hkl_to_reflection_file(
integrate_hkl, experiments_json,
self.get_working_directory())
reflections = flex.reflection_table.from_file(
integrate_filename)
t0 = time.time()
sel = filter_shadowed_reflections(experiments, reflections)
shadowed = reflections.select(sel)
t1 = time.time()
logger.debug("Filtered %i reflections in %.1f seconds" %
(sel.count(True), t1 - t0))
filter_hkl = os.path.join(self.get_working_directory(),
"FILTER.HKL")
with open(filter_hkl, "wb") as f:
detector = experiments[0].detector
for ref in shadowed:
p = detector[ref["panel"]]
ox, oy = p.get_raw_image_offset()
h, k, l = ref["miller_index"]
x, y, z = ref["xyzcal.px"]
dx, dy, dz = (2, 2, 2)
print(
"%i %i %i %.1f %.1f %.1f %.1f %.1f %.1f" %
(h, k, l, x + ox, y + oy, z, dx, dy, dz),
file=f,
)
t2 = time.time()
logger.debug("Written FILTER.HKL in %.1f seconds" % (t2 - t1))
correct = self.Correct()
correct.set_data_range(
self._intgr_wedge[0] + self.get_frame_offset(),
self._intgr_wedge[1] + self.get_frame_offset(),
)
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# BUG # 2695 probably comes from here - need to check...
# if the pointless interface comes back with a different
# crystal setting then the unit cell stored in self._intgr_cell
# needs to be set to None...
if self.get_integrater_spacegroup_number():
correct.set_spacegroup_number(
self.get_integrater_spacegroup_number())
if not self._intgr_cell:
raise RuntimeError("no unit cell to recycle")
correct.set_cell(self._intgr_cell)
# BUG # 3113 - new version of XDS will try and figure the
# best spacegroup out from the intensities (and get it wrong!)
# unless we set the spacegroup and cell explicitly
if not self.get_integrater_spacegroup_number():
cell = self._intgr_refiner_cell
lattice = self._intgr_refiner.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup_number(lattice)
# this should not prevent the postrefinement from
# working correctly, else what is above would not
# work correctly (the postrefinement test)
correct.set_spacegroup_number(spacegroup_number)
correct.set_cell(cell)
logger.debug("Setting spacegroup to: %d" % spacegroup_number)
logger.debug("Setting cell to: %.2f %.2f %.2f %.2f %.2f %.2f" %
tuple(cell))
if self.get_integrater_reindex_matrix():
# bug! if the lattice is not primitive the values in this
# reindex matrix need to be multiplied by a constant which
# depends on the Bravais lattice centering.
lattice = self._intgr_refiner.get_refiner_lattice()
matrix = self.get_integrater_reindex_matrix()
matrix = scitbx.matrix.sqr(matrix).transpose().elems
matrix = r_to_rt(matrix)
if lattice[1] == "P":
mult = 1
elif lattice[1] == "C" or lattice[1] == "I":
mult = 2
elif lattice[1] == "R":
mult = 3
elif lattice[1] == "F":
mult = 4
else:
raise RuntimeError("unknown multiplier for lattice %s" %
lattice)
logger.debug("REIDX multiplier for lattice %s: %d" %
(lattice, mult))
mult_matrix = [mult * m for m in matrix]
logger.debug("REIDX set to %d %d %d %d %d %d %d %d %d %d %d %d" %
tuple(mult_matrix))
correct.set_reindex_matrix(mult_matrix)
correct.run()
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file(
f"{pname} {xname} {dname} {sweep} CORRECT",
os.path.join(self.get_working_directory(), "CORRECT.LP"),
)
FileHandler.record_more_data_file(
f"{pname} {xname} {dname} {sweep} CORRECT",
os.path.join(self.get_working_directory(), "XDS_ASCII.HKL"),
)
# erm. just to be sure
if self.get_integrater_reindex_matrix() and correct.get_reindex_used():
raise RuntimeError("Reindex panic!")
# get the reindex operation used, which may be useful if none was
# set but XDS decided to apply one, e.g. #419.
if not self.get_integrater_reindex_matrix(
) and correct.get_reindex_used():
# convert this reindex operation to h, k, l form: n.b. this
# will involve dividing through by the lattice centring multiplier
matrix = rt_to_r(correct.get_reindex_used())
matrix = scitbx.matrix.sqr(matrix).transpose().elems
lattice = self._intgr_refiner.get_refiner_lattice()
if lattice[1] == "P":
mult = 1.0
elif lattice[1] == "C" or lattice[1] == "I":
mult = 2.0
elif lattice[1] == "R":
mult = 3.0
elif lattice[1] == "F":
mult = 4.0
matrix = [m / mult for m in matrix]
reindex_op = mat_to_symop(matrix)
# assign this to self: will this reset?! make for a leaky
# abstraction and just assign this...
# self.set_integrater_reindex_operator(reindex)
self._intgr_reindex_operator = reindex_op
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file(
f"{pname} {xname} {dname} {sweep} CORRECT",
os.path.join(self.get_working_directory(), "CORRECT.LP"),
)
# should get some interesting stuff from the XDS correct file
# here, for instance the resolution range to use in integration
# (which should be fed back if not fast) and so on...
self._intgr_corrected_hklout = os.path.join(
self.get_working_directory(), "XDS_ASCII.HKL")
# also record the batch range - needed for the analysis of the
# radiation damage in chef...
self._intgr_batches_out = (self._intgr_wedge[0], self._intgr_wedge[1])
# FIXME perhaps I should also feedback the GXPARM file here??
for file in ["GXPARM.XDS"]:
self._xds_data_files[file] = correct.get_output_data_file(file)
# record the postrefined cell parameters
self._intgr_cell = correct.get_result("cell")
self._intgr_n_ref = correct.get_result("n_ref")
logger.debug('Postrefinement in "correct" spacegroup results:')
logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
tuple(correct.get_result("cell")))
logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
tuple(correct.get_result("cell_esd")))
logger.debug(
"Deviations: %.2f pixels %.2f degrees" %
(correct.get_result("rmsd_pixel"), correct.get_result("rmsd_phi")))
logger.debug("Error correction parameters: A=%.3f B=%.3f" %
correct.get_result("sdcorrection"))
# compute misorientation of axes
xparm_file = os.path.join(self.get_working_directory(), "GXPARM.XDS")
handle = xparm.reader()
handle.read_file(xparm_file)
rotn = handle.rotation_axis
beam = handle.beam_vector
dot = sum(rotn[j] * beam[j] for j in range(3))
r = math.sqrt(sum(rotn[j] * rotn[j] for j in range(3)))
b = math.sqrt(sum(beam[j] * beam[j] for j in range(3)))
rtod = 180.0 / math.pi
angle = rtod * math.fabs(0.5 * math.pi - math.acos(dot / (r * b)))
logger.debug("Axis misalignment %.2f degrees" % angle)
correct_deviations = (
correct.get_result("rmsd_pixel"),
correct.get_result("rmsd_phi"),
)
if p1_deviations:
# compare and reject if both > 50% higher - though adding a little
# flexibility - 0.5 pixel / osc width slack.
pixel = p1_deviations[0]
phi = math.sqrt(0.05 * 0.05 + p1_deviations[1] * p1_deviations[1])
threshold = PhilIndex.params.xia2.settings.lattice_rejection_threshold
logger.debug("RMSD ratio: %.2f" % (correct_deviations[0] / pixel))
logger.debug("RMSPhi ratio: %.2f" % (correct_deviations[1] / phi))
if (correct_deviations[0] / pixel > threshold
and correct_deviations[1] / phi > threshold):
logger.info(
"Eliminating this indexing solution as postrefinement")
logger.info("deviations rather high relative to triclinic")
raise BadLatticeError(
"high relative deviations in postrefinement")
if (not PhilIndex.params.dials.fast_mode
and not PhilIndex.params.xds.keep_outliers):
# check for alien reflections and perhaps recycle - removing them
correct_remove = correct.get_remove()
if correct_remove:
current_remove = set()
final_remove = []
# first ensure that there are no duplicate entries...
if os.path.exists(
os.path.join(self.get_working_directory(),
"REMOVE.HKL")):
with open(
os.path.join(self.get_working_directory(),
"REMOVE.HKL")) as fh:
for line in fh.readlines():
h, k, l = list(map(int, line.split()[:3]))
z = float(line.split()[3])
if (h, k, l, z) not in current_remove:
current_remove.add((h, k, l, z))
for c in correct_remove:
if c in current_remove:
continue
final_remove.append(c)
logger.debug("%d alien reflections are already removed" %
(len(correct_remove) - len(final_remove)))
else:
# we want to remove all of the new dodgy reflections
final_remove = correct_remove
z_min = PhilIndex.params.xds.z_min
rejected = 0
with open(
os.path.join(self.get_working_directory(),
"REMOVE.HKL"), "w") as remove_hkl:
# write in the old reflections
for remove in current_remove:
z = remove[3]
if z >= z_min:
remove_hkl.write("%d %d %d %f\n" % remove)
else:
rejected += 1
logger.debug("Wrote %d old reflections to REMOVE.HKL" %
(len(current_remove) - rejected))
logger.debug("Rejected %d as z < %f" % (rejected, z_min))
# and the new reflections
rejected = 0
used = 0
for remove in final_remove:
z = remove[3]
if z >= z_min:
used += 1
remove_hkl.write("%d %d %d %f\n" % remove)
else:
rejected += 1
logger.debug("Wrote %d new reflections to REMOVE.HKL" %
(len(final_remove) - rejected))
logger.debug("Rejected %d as z < %f" % (rejected, z_min))
# we want to rerun the finishing step so...
# unless we have added no new reflections... or unless we
# have not confirmed the point group (see SCI-398)
if used and self.get_integrater_reindex_matrix():
self.set_integrater_finish_done(False)
else:
logger.debug(
"Going quickly so not removing %d outlier reflections..." %
len(correct.get_remove()))
# Convert INTEGRATE.HKL to MTZ format and reapply any reindexing operations
# spacegroup changes to allow use with CCP4 / Aimless for scaling
hklout = os.path.splitext(integrate_hkl)[0] + ".mtz"
self._factory.set_working_directory(self.get_working_directory())
pointless = self._factory.Pointless()
pointless.set_xdsin(integrate_hkl)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
integrate_mtz = hklout
if (self.get_integrater_reindex_operator()
or self.get_integrater_spacegroup_number()):
logger.debug("Reindexing things to MTZ")
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
if self.get_integrater_reindex_operator():
reindex.set_operator(self.get_integrater_reindex_operator())
if self.get_integrater_spacegroup_number():
reindex.set_spacegroup(self.get_integrater_spacegroup_number())
hklout = "%s_reindex.mtz" % os.path.splitext(integrate_mtz)[0]
reindex.set_hklin(integrate_mtz)
reindex.set_hklout(hklout)
reindex.reindex()
integrate_mtz = hklout
experiments_json = xparm_xds_to_experiments_json(
self._xds_data_files["GXPARM.XDS"], self.get_working_directory())
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(f"{pname} {xname} {dname} {sweep}",
experiments_json)
FileHandler.record_more_data_file(
f"{pname} {xname} {dname} {sweep} INTEGRATE", integrate_mtz)
self._intgr_experiments_filename = experiments_json
return integrate_mtz
def run(self):
from iotbx.xds import xparm
import os
import libtbx.load_env
from libtbx.test_utils import open_tmp_file
iotbx_dir = libtbx.env.dist_path('iotbx')
filename = os.path.join(iotbx_dir, 'xds', 'tests', 'XPARM.XDS')
handle = xparm.reader()
assert handle.find_version(filename) == 1
handle.read_file(filename)
print('OK')
filename = os.path.join(iotbx_dir, 'xds', 'tests', 'NEW_XPARM.XDS')
handle = xparm.reader()
assert handle.find_version(filename) == 2
handle.read_file(filename)
print('OK')
f = open_tmp_file(suffix='XPARM.XDS', mode='wb')
f.close()
writer = xparm.writer(
handle.starting_frame,
handle.starting_angle,
handle.oscillation_range,
handle.rotation_axis,
handle.wavelength,
handle.beam_vector,
handle.space_group,
handle.unit_cell,
handle.unit_cell_a_axis,
handle.unit_cell_b_axis,
handle.unit_cell_c_axis,
handle.num_segments,
handle.detector_size,
handle.pixel_size,
handle.detector_origin,
handle.detector_distance,
handle.detector_x_axis,
handle.detector_y_axis,
handle.detector_normal,
handle.segments,
handle.orientation)
writer.write_file(f.name)
handle_recycled = xparm.reader()
# make sure we wrote out version 2
assert handle_recycled.find_version(f.name) == 2
handle_recycled.read_file(f.name)
for handle in (handle, handle_recycled):
# Scan and goniometer stuff
assert handle.starting_frame == 1
assert handle.starting_angle == 82.0
assert handle.oscillation_range == 0.1500
assert handle.rotation_axis == (0.999997, -0.001590, -0.001580)
# Beam stuff
assert handle.wavelength == 0.976250
assert handle.beam_vector == (0.001608, 0.004392, 1.024317)
# Detector stuff
assert handle.detector_size == (2463, 2527)
assert handle.pixel_size == (0.172, 0.172)
assert handle.detector_distance == 264.928955
assert handle.detector_origin == (1224.856812, 1187.870972)
assert handle.detector_x_axis == (1.0, 0.0, 0.0)
assert handle.detector_y_axis == (0.0, 1.0, 0.0)
assert handle.detector_normal == (0.0, 0.0, 1.0)
# Crystal stuff
assert handle.space_group == 75
assert handle.unit_cell == (57.7831, 57.7831, 150.0135, 90.000, 90.000, 90.000)
assert handle.unit_cell_a_axis == (-14.918090, -22.358297, 51.151196)
assert handle.unit_cell_b_axis == (-19.858326, 51.608330, 16.766487)
assert handle.unit_cell_c_axis == (-135.447952, -34.400188, -54.539391)
# segment stuff
assert handle_recycled.num_segments == 1
assert handle_recycled.segments == [(1, 1, 2463, 1, 2527)]
assert handle_recycled.orientation == [
(0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0)]
print('OK')
def run():
if not have_dials_regression:
print "Skipping test: dials_regression not available."
return
from scitbx import matrix
from iotbx.xds import xparm, integrate_hkl
from dials.util import ioutil
from math import ceil
from dials.algorithms.spot_prediction import RotationAngles
import dxtbx
from rstbx.cftbx.coordinate_frame_converter import \
coordinate_frame_converter
# The XDS files to read from
integrate_filename = join(dials_regression, 'data/sim_mx/INTEGRATE.HKL')
gxparm_filename = join(dials_regression, 'data/sim_mx/GXPARM.XDS')
# Read the XDS files
integrate_handle = integrate_hkl.reader()
integrate_handle.read_file(integrate_filename)
gxparm_handle = xparm.reader()
gxparm_handle.read_file(gxparm_filename)
# Get the parameters we need from the GXPARM file
models = dxtbx.load(gxparm_filename)
beam = models.get_beam()
gonio = models.get_goniometer()
detector = models.get_detector()
scan = models.get_scan()
# Get the crystal parameters
space_group_type = ioutil.get_space_group_type_from_xparm(gxparm_handle)
cfc = coordinate_frame_converter(gxparm_filename)
a_vec = cfc.get('real_space_a')
b_vec = cfc.get('real_space_b')
c_vec = cfc.get('real_space_c')
unit_cell = cfc.get_unit_cell()
UB = matrix.sqr(a_vec + b_vec + c_vec).inverse()
ub_matrix = UB
# Get the minimum resolution in the integrate file
d = [unit_cell.d(h) for h in integrate_handle.hkl]
d_min = min(d)
# Get the number of frames from the max z value
xcal, ycal, zcal = zip(*integrate_handle.xyzcal)
num_frames = int(ceil(max(zcal)))
scan.set_image_range((scan.get_image_range()[0],
scan.get_image_range()[0] + num_frames - 1))
# Create the rotation angle object
ra = RotationAngles(beam.get_s0(), gonio.get_rotation_axis())
# Setup the matrices
ub = matrix.sqr(ub_matrix)
s0 = matrix.col(beam.get_s0())
m2 = matrix.col(gonio.get_rotation_axis())
# For all the miller indices
for h in integrate_handle.hkl:
h = matrix.col(h)
# Calculate the angles
angles = ra(h, ub)
# For all the angles
for phi in angles:
r = m2.axis_and_angle_as_r3_rotation_matrix(angle=phi)
pstar = r * ub * h
s1 = s0 + pstar
assert(abs(s1.length() - s0.length()) < 1e-7)
print "OK"
# Create a dict of lists of xy for each hkl
gen_phi = {}
for h in integrate_handle.hkl:
# Calculate the angles
angles = ra(h, ub)
gen_phi[h] = angles
# for phi in angles:
# try:
# a = gen_phi[h]
# a.append(phi)
# gen_phi[h] = a
# except KeyError:
# gen_phi[h] = [phi]
# For each hkl in the xds file
for hkl, xyz in zip(integrate_handle.hkl,
integrate_handle.xyzcal):
# Calculate the XDS phi value
xds_phi = scan.get_oscillation(deg=False)[0] + \
xyz[2]*scan.get_oscillation(deg=False)[1]
# Select the nearest xy to use if there are 2
my_phi = gen_phi[hkl]
if len(my_phi) == 2:
my_phi0 = my_phi[0]
my_phi1 = my_phi[1]
diff0 = abs(xds_phi - my_phi0)
diff1 = abs(xds_phi - my_phi1)
if diff0 < diff1:
my_phi = my_phi0
else:
my_phi = my_phi1
else:
my_phi = my_phi[0]
# Check the Phi values are the same
assert(abs(xds_phi - my_phi) < 0.1)
# Test Passed
print "OK"
def import_xds_xparm(xparm_file):
'''Read an XDS XPARM file, transform the parameters contained therein
into the standard coordinate frame, record this as a dictionary.'''
from iotbx.xds import xparm
handle = xparm.reader()
handle.read_file(xparm_file)
# first determine the rotation R from the XDS coordinate frame used in
# the processing to the central (i.e. imgCIF) coordinate frame. N.B.
# if the scan was e.g. a PHI scan the resulting frame could well come out
# a little odd...
axis = handle.rotation_axis
beam = handle.beam_vector
x, y = handle.detector_x_axis, handle.detector_y_axis
# XDS defines the beam vector as s0 rather than from sample -> source.
B = - matrix.col(beam).normalize()
A = matrix.col(axis).normalize()
X = matrix.col(x).normalize()
Y = matrix.col(y).normalize()
N = X.cross(Y)
_X = matrix.col([1, 0, 0])
_Y = matrix.col([0, 1, 0])
_Z = matrix.col([0, 0, 1])
R = align_reference_frame(A, _X, B, _Z)
# now transform contents of the XPARM file to the form which we want to
# return...
nx, ny = handle.detector_size
px, py = handle.pixel_size
distance = handle.detector_distance
ox, oy = handle.detector_origin
a = handle.unit_cell_a_axis
b = handle.unit_cell_b_axis
c = handle.unit_cell_c_axis
# Need to subtract 0.5 because XDS seems to do centroids in fortran coords
ox = ox - 0.5
oy = oy - 0.5
detector_origin = R * (distance * N - ox * px * X - oy * py * Y)
detector_fast = R * X
detector_slow = R * Y
rotation_axis = R * A
sample_to_source = R * B
wavelength = handle.wavelength
real_space_a = R * matrix.col(a)
real_space_b = R * matrix.col(b)
real_space_c = R * matrix.col(c)
space_group_number = handle.space_group
starting_angle = handle.starting_angle
oscillation_range = handle.oscillation_range
starting_frame = handle.starting_frame
return coordinate_frame_information(
detector_origin, detector_fast, detector_slow, (nx, ny), (px, py),
rotation_axis, sample_to_source, wavelength,
real_space_a, real_space_b, real_space_c, space_group_number,
None, None, starting_angle, oscillation_range, starting_frame,
original_rotation = R)
def run(self):
from iotbx.xds import xparm
import os
import libtbx.load_env
from libtbx.test_utils import open_tmp_file
iotbx_dir = libtbx.env.dist_path('iotbx')
filename = os.path.join(iotbx_dir, 'xds', 'tests', 'XPARM.XDS')
handle = xparm.reader()
assert handle.find_version(filename) == 1
handle.read_file(filename)
print 'OK'
filename = os.path.join(iotbx_dir, 'xds', 'tests', 'NEW_XPARM.XDS')
handle = xparm.reader()
assert handle.find_version(filename) == 2
handle.read_file(filename)
print 'OK'
f = open_tmp_file(suffix='XPARM.XDS', mode='wb')
f.close()
writer = xparm.writer(
handle.starting_frame,
handle.starting_angle,
handle.oscillation_range,
handle.rotation_axis,
handle.wavelength,
handle.beam_vector,
handle.space_group,
handle.unit_cell,
handle.unit_cell_a_axis,
handle.unit_cell_b_axis,
handle.unit_cell_c_axis,
handle.num_segments,
handle.detector_size,
handle.pixel_size,
handle.detector_origin,
handle.detector_distance,
handle.detector_x_axis,
handle.detector_y_axis,
handle.detector_normal,
handle.segments,
handle.orientation)
writer.write_file(f.name)
handle_recycled = xparm.reader()
# make sure we wrote out version 2
assert handle_recycled.find_version(f.name) == 2
handle_recycled.read_file(f.name)
for handle in (handle, handle_recycled):
# Scan and goniometer stuff
assert handle.starting_frame == 1
assert handle.starting_angle == 82.0
assert handle.oscillation_range == 0.1500
assert handle.rotation_axis == (0.999997, -0.001590, -0.001580)
# Beam stuff
assert handle.wavelength == 0.976250
assert handle.beam_vector == (0.001608, 0.004392, 1.024317)
# Detector stuff
assert handle.detector_size == (2463, 2527)
assert handle.pixel_size == (0.172, 0.172)
assert handle.detector_distance == 264.928955
assert handle.detector_origin == (1224.856812, 1187.870972)
assert handle.detector_x_axis == (1.0, 0.0, 0.0)
assert handle.detector_y_axis == (0.0, 1.0, 0.0)
assert handle.detector_normal == (0.0, 0.0, 1.0)
# Crystal stuff
assert handle.space_group == 75
assert handle.unit_cell == (57.7831, 57.7831, 150.0135, 90.000, 90.000, 90.000)
assert handle.unit_cell_a_axis == (-14.918090, -22.358297, 51.151196)
assert handle.unit_cell_b_axis == (-19.858326, 51.608330, 16.766487)
assert handle.unit_cell_c_axis == (-135.447952, -34.400188, -54.539391)
# segment stuff
assert handle_recycled.num_segments == 1
assert handle_recycled.segments == [(1, 1, 2463, 1, 2527)]
assert handle_recycled.orientation == [
(0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0)]
print 'OK'
def __init__(self):
from dials.algorithms.spot_prediction import IndexGenerator
from dials.algorithms.spot_prediction import ScanStaticRayPredictor
from dials.algorithms.spot_prediction import ray_intersection
from iotbx.xds import xparm, integrate_hkl
from dials.util import ioutil
from math import ceil
from os.path import realpath, dirname, join
import dxtbx
from rstbx.cftbx.coordinate_frame_converter import \
coordinate_frame_converter
from scitbx import matrix
# The XDS files to read from
test_path = dirname(dirname(dirname(realpath(__file__))))
integrate_filename = join(test_path, 'data/sim_mx/INTEGRATE.HKL')
gxparm_filename = join(test_path, 'data/sim_mx/GXPARM.XDS')
# Read the XDS files
self.integrate_handle = integrate_hkl.reader()
self.integrate_handle.read_file(integrate_filename)
self.gxparm_handle = xparm.reader()
self.gxparm_handle.read_file(gxparm_filename)
# Get the parameters we need from the GXPARM file
models = dxtbx.load(gxparm_filename)
self.beam = models.get_beam()
self.gonio = models.get_goniometer()
self.detector = models.get_detector()
self.scan = models.get_scan()
assert(len(self.detector) == 1)
#print self.detector
# Get crystal parameters
self.space_group_type = ioutil.get_space_group_type_from_xparm(
self.gxparm_handle)
cfc = coordinate_frame_converter(gxparm_filename)
a_vec = cfc.get('real_space_a')
b_vec = cfc.get('real_space_b')
c_vec = cfc.get('real_space_c')
self.unit_cell = cfc.get_unit_cell()
self.ub_matrix = matrix.sqr(a_vec + b_vec + c_vec).inverse()
# Get the minimum resolution in the integrate file
self.d_min = self.detector[0].get_max_resolution_at_corners(
self.beam.get_s0())
# Get the number of frames from the max z value
xcal, ycal, zcal = zip(*self.integrate_handle.xyzcal)
self.scan.set_image_range((self.scan.get_image_range()[0],
self.scan.get_image_range()[0] +
int(ceil(max(zcal)))))
# Create the index generator
generate_indices = IndexGenerator(self.unit_cell, self.space_group_type,
self.d_min)
s0 = self.beam.get_s0()
m2 = self.gonio.get_rotation_axis()
fixed_rotation = self.gonio.get_fixed_rotation()
UB = self.ub_matrix
dphi = self.scan.get_oscillation_range(deg=False)
# Create the ray predictor
self.predict_rays = ScanStaticRayPredictor(s0, m2, fixed_rotation, dphi)
# Predict the spot locations
self.reflections = self.predict_rays(
generate_indices.to_array(), UB)
# Calculate the intersection of the detector and reflection frames
success = ray_intersection(self.detector, self.reflections)
self.reflections.select(success)