def _integrate_finish(self):
'''Finish the integration - if necessary performing reindexing
based on the pointgroup and the reindexing operator.'''
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice()):
return self._mosflm_hklout
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == 0:
return self._mosflm_hklout
Debug.write('Reindexing to spacegroup %d (%s)' % \
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
hklin = self._mosflm_hklout
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
reindex.set_operator(self._intgr_reindex_operator)
if self._intgr_spacegroup_number:
reindex.set_spacegroup(self._intgr_spacegroup_number)
hklout = '%s_reindex.mtz' % hklin[:-4]
reindex.set_hklin(hklin)
reindex.set_hklout(hklout)
reindex.reindex()
return hklout
def _integrate_finish(self):
'''Finish the integration - if necessary performing reindexing
based on the pointgroup and the reindexing operator.'''
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice()):
return self._mosflm_hklout
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == 0:
return self._mosflm_hklout
Debug.write('Reindexing to spacegroup %d (%s)' % \
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
hklin = self._mosflm_hklout
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
reindex.set_operator(self._intgr_reindex_operator)
if self._intgr_spacegroup_number:
reindex.set_spacegroup(self._intgr_spacegroup_number)
hklout = '%s_reindex.mtz' % hklin[:-4]
reindex.set_hklin(hklin)
reindex.set_hklout(hklout)
reindex.reindex()
return hklout
def _integrate_finish(self):
'''Finish off the integration by running dials.export.'''
# FIXME - do we want to export every time we call this method
# (the file will not have changed) and also (more important) do
# we want a different exported MTZ file every time (I do not think
# that we do; these can be very large) - was exporter.get_xpid() ->
# now dials
exporter = self.ExportMtz()
exporter.set_reflections_filename(self._intgr_integrated_pickle)
mtz_filename = os.path.join(
self.get_working_directory(), '%s_integrated.mtz' % 'dials')
exporter.set_mtz_filename(mtz_filename)
exporter.run()
self._intgr_integrated_filename = mtz_filename
# record integrated MTZ file for e.g. BLEND.
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(
'%s %s %s %s INTEGRATE' % (pname, xname, dname, sweep), mtz_filename)
if not os.path.isfile(self._intgr_integrated_filename):
raise RuntimeError("dials.export failed: %s does not exist."
% self._intgr_integrated_filename)
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice()):
Debug.write('Not reindexing to spacegroup %d (%s)' % \
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
return mtz_filename
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == 0:
Debug.write('Not reindexing to spacegroup %d (%s)' % \
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
return mtz_filename
Debug.write('Reindexing to spacegroup %d (%s)' % \
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
hklin = mtz_filename
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
reindex.set_operator(self._intgr_reindex_operator)
if self._intgr_spacegroup_number:
reindex.set_spacegroup(self._intgr_spacegroup_number)
else:
reindex.set_spacegroup(lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice()))
hklout = '%s_reindex.mtz' % hklin[:-4]
reindex.set_hklin(hklin)
reindex.set_hklout(hklout)
reindex.reindex()
self._intgr_integrated_filename = hklout
self._intgr_cell = reindex.get_cell()
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(
'%s %s %s %s experiments' % (pname, xname, dname, sweep),
self.get_integrated_experiments())
from iotbx.reflection_file_reader import any_reflection_file
miller_arrays = any_reflection_file(hklout).as_miller_arrays()
# look for profile-fitted intensities
intensities = [ma for ma in miller_arrays
if ma.info().labels == ['IPR', 'SIGIPR']]
if len(intensities) == 0:
# look instead for summation-integrated intensities
intensities = [ma for ma in miller_arrays
if ma.info().labels == ['I', 'SIGI']]
assert len(intensities)
self._intgr_n_ref = intensities[0].size()
return hklout
def _integrate_finish(self):
'''Finish off the integration by running correct.'''
# first run the postrefinement etc with spacegroup P1
# and the current unit cell - this will be used to
# obtain a benchmark rmsd in pixels / phi and also
# cell deviations (this is working towards spotting bad
# indexing solutions) - only do this if we have no
# reindex matrix... and no postrefined cell...
p1_deviations = None
# fix for bug # 3264 -
# if we have not run integration with refined parameters, make it so...
# erm? shouldn't this therefore return if this is the principle, or
# set the flag after we have tested the lattice?
if not self._xds_data_files.has_key('GXPARM.XDS') and \
PhilIndex.params.xds.integrate.reintegrate:
Debug.write(
'Resetting integrater, to ensure refined orientation is used')
self.set_integrater_done(False)
if not self.get_integrater_reindex_matrix() and not self._intgr_cell \
and not Flags.get_no_lattice_test() and \
not self.get_integrater_sweep().get_user_lattice():
correct = self.Correct()
correct.set_data_range(self._intgr_wedge[0],
self._intgr_wedge[1])
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# FIXME should this be using the correctly transformed
# cell or are the results ok without it?!
correct.set_spacegroup_number(1)
correct.set_cell(self._intgr_refiner_cell)
correct.run()
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file('%s %s %s %s CORRECT' % \
(pname, xname, dname, sweep),
os.path.join(
self.get_working_directory(),
'CORRECT.LP'))
FileHandler.record_more_data_file(
'%s %s %s %s CORRECT' % (pname, xname, dname, sweep),
os.path.join(self.get_working_directory(), 'XDS_ASCII.HKL'))
cell = correct.get_result('cell')
cell_esd = correct.get_result('cell_esd')
Debug.write('Postrefinement in P1 results:')
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(cell))
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(cell_esd))
Debug.write('Deviations: %.2f pixels %.2f degrees' % \
(correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi')))
p1_deviations = (correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi'))
# next run the postrefinement etc with the given
# cell / lattice - this will be the assumed result...
correct = self.Correct()
correct.set_data_range(self._intgr_wedge[0],
self._intgr_wedge[1])
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# BUG # 2695 probably comes from here - need to check...
# if the pointless interface comes back with a different
# crystal setting then the unit cell stored in self._intgr_cell
# needs to be set to None...
if self.get_integrater_spacegroup_number():
correct.set_spacegroup_number(
self.get_integrater_spacegroup_number())
if not self._intgr_cell:
raise RuntimeError, 'no unit cell to recycle'
correct.set_cell(self._intgr_cell)
# BUG # 3113 - new version of XDS will try and figure the
# best spacegroup out from the intensities (and get it wrong!)
# unless we set the spacegroup and cell explicitly
if not self.get_integrater_spacegroup_number():
cell = self._intgr_refiner_cell
lattice = self._intgr_refiner.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup_number(lattice)
# this should not prevent the postrefinement from
# working correctly, else what is above would not
# work correctly (the postrefinement test)
correct.set_spacegroup_number(spacegroup_number)
correct.set_cell(cell)
Debug.write('Setting spacegroup to: %d' % spacegroup_number)
Debug.write(
'Setting cell to: %.2f %.2f %.2f %.2f %.2f %.2f' % tuple(cell))
if self.get_integrater_reindex_matrix():
# bug! if the lattice is not primitive the values in this
# reindex matrix need to be multiplied by a constant which
# depends on the Bravais lattice centering.
lattice = self._intgr_refiner.get_refiner_lattice()
import scitbx.matrix
matrix = self.get_integrater_reindex_matrix()
matrix = scitbx.matrix.sqr(matrix).transpose().elems
matrix = r_to_rt(matrix)
if lattice[1] == 'P':
mult = 1
elif lattice[1] == 'C' or lattice[1] == 'I':
mult = 2
elif lattice[1] == 'R':
mult = 3
elif lattice[1] == 'F':
mult = 4
else:
raise RuntimeError, 'unknown multiplier for lattice %s' % \
lattice
Debug.write('REIDX multiplier for lattice %s: %d' % \
(lattice, mult))
mult_matrix = [mult * m for m in matrix]
Debug.write('REIDX set to %d %d %d %d %d %d %d %d %d %d %d %d' % \
tuple(mult_matrix))
correct.set_reindex_matrix(mult_matrix)
correct.run()
# erm. just to be sure
if self.get_integrater_reindex_matrix() and \
correct.get_reindex_used():
raise RuntimeError, 'Reindex panic!'
# get the reindex operation used, which may be useful if none was
# set but XDS decided to apply one, e.g. #419.
if not self.get_integrater_reindex_matrix() and \
correct.get_reindex_used():
# convert this reindex operation to h, k, l form: n.b. this
# will involve dividing through by the lattice centring multiplier
matrix = rt_to_r(correct.get_reindex_used())
import scitbx.matrix
matrix = scitbx.matrix.sqr(matrix).transpose().elems
lattice = self._intgr_refiner.get_refiner_lattice()
if lattice[1] == 'P':
mult = 1.0
elif lattice[1] == 'C' or lattice[1] == 'I':
mult = 2.0
elif lattice[1] == 'R':
mult = 3.0
elif lattice[1] == 'F':
mult = 4.0
matrix = [m / mult for m in matrix]
reindex_op = mat_to_symop(matrix)
# assign this to self: will this reset?! make for a leaky
# abstraction and just assign this...
# self.set_integrater_reindex_operator(reindex)
self._intgr_reindex_operator = reindex_op
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file('%s %s %s %s CORRECT' % \
(pname, xname, dname, sweep),
os.path.join(self.get_working_directory(),
'CORRECT.LP'))
# should get some interesting stuff from the XDS correct file
# here, for instance the resolution range to use in integration
# (which should be fed back if not fast) and so on...
self._intgr_corrected_hklout = os.path.join(self.get_working_directory(),
'XDS_ASCII.HKL')
# also record the batch range - needed for the analysis of the
# radiation damage in chef...
self._intgr_batches_out = (self._intgr_wedge[0],
self._intgr_wedge[1])
# FIXME perhaps I should also feedback the GXPARM file here??
for file in ['GXPARM.XDS']:
self._xds_data_files[file] = correct.get_output_data_file(file)
# record the postrefined cell parameters
self._intgr_cell = correct.get_result('cell')
self._intgr_n_ref = correct.get_result('n_ref')
Debug.write('Postrefinement in "correct" spacegroup results:')
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(correct.get_result('cell')))
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(correct.get_result('cell_esd')))
Debug.write('Deviations: %.2f pixels %.2f degrees' % \
(correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi')))
Debug.write('Error correction parameters: A=%.3f B=%.3f' % \
correct.get_result('sdcorrection'))
# compute misorientation of axes
xparm_file = os.path.join(self.get_working_directory(), 'GXPARM.XDS')
from iotbx.xds import xparm
handle = xparm.reader()
handle.read_file(xparm_file)
rotn = handle.rotation_axis
beam = handle.beam_vector
dot = sum([rotn[j] * beam[j] for j in range(3)])
r = math.sqrt(sum([rotn[j] * rotn[j] for j in range(3)]))
b = math.sqrt(sum([beam[j] * beam[j] for j in range(3)]))
rtod = 180.0 / math.pi
angle = rtod * math.fabs(0.5 * math.pi - math.acos(dot / (r * b)))
Debug.write('Axis misalignment %.2f degrees' % angle)
correct_deviations = (correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi'))
if p1_deviations:
# compare and reject if both > 50% higher - though adding a little
# flexibility - 0.5 pixel / osc width slack.
pixel = p1_deviations[0]
phi = math.sqrt(0.05 * 0.05 + \
p1_deviations[1] * p1_deviations[1])
threshold = Flags.get_rejection_threshold()
Debug.write('RMSD ratio: %.2f' % (correct_deviations[0] / pixel))
Debug.write('RMSPhi ratio: %.2f' % (correct_deviations[1] / phi))
if correct_deviations[0] / pixel > threshold and \
correct_deviations[1] / phi > threshold:
Chatter.write(
'Eliminating this indexing solution as postrefinement')
Chatter.write(
'deviations rather high relative to triclinic')
raise BadLatticeError, \
'high relative deviations in postrefinement'
if not Flags.get_quick() and Flags.get_remove():
# check for alien reflections and perhaps recycle - removing them
if len(correct.get_remove()) > 0:
correct_remove = correct.get_remove()
current_remove = []
final_remove = []
# first ensure that there are no duplicate entries...
if os.path.exists(os.path.join(
self.get_working_directory(),
'REMOVE.HKL')):
for line in open(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'), 'r').readlines():
h, k, l = map(int, line.split()[:3])
z = float(line.split()[3])
if not (h, k, l, z) in current_remove:
current_remove.append((h, k, l, z))
for c in correct_remove:
if c in current_remove:
continue
final_remove.append(c)
Debug.write(
'%d alien reflections are already removed' % \
(len(correct_remove) - len(final_remove)))
else:
# we want to remove all of the new dodgy reflections
final_remove = correct_remove
remove_hkl = open(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'), 'w')
z_min = Flags.get_z_min()
rejected = 0
# write in the old reflections
for remove in current_remove:
z = remove[3]
if z >= z_min:
remove_hkl.write('%d %d %d %f\n' % remove)
else:
rejected += 1
Debug.write('Wrote %d old reflections to REMOVE.HKL' % \
(len(current_remove) - rejected))
Debug.write('Rejected %d as z < %f' % \
(rejected, z_min))
# and the new reflections
rejected = 0
used = 0
for remove in final_remove:
z = remove[3]
if z >= z_min:
used += 1
remove_hkl.write('%d %d %d %f\n' % remove)
else:
rejected += 1
Debug.write('Wrote %d new reflections to REMOVE.HKL' % \
(len(final_remove) - rejected))
Debug.write('Rejected %d as z < %f' % \
(rejected, z_min))
remove_hkl.close()
# we want to rerun the finishing step so...
# unless we have added no new reflections... or unless we
# have not confirmed the point group (see SCI-398)
if used and self.get_integrater_reindex_matrix():
self.set_integrater_finish_done(False)
else:
Debug.write(
'Going quickly so not removing %d outlier reflections...' % \
len(correct.get_remove()))
# Convert INTEGRATE.HKL to MTZ format and reapply any reindexing operations
# spacegroup changes to allow use with CCP4 / Aimless for scaling
integrate_hkl = os.path.join(
self.get_working_directory(), 'INTEGRATE.HKL')
hklout = os.path.splitext(integrate_hkl)[0] + ".mtz"
self._factory.set_working_directory(self.get_working_directory())
pointless = self._factory.Pointless()
pointless.set_xdsin(integrate_hkl)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
integrate_mtz = hklout
if self.get_integrater_reindex_operator() or \
self.get_integrater_spacegroup_number():
Debug.write('Reindexing things to MTZ')
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
if self.get_integrater_reindex_operator():
reindex.set_operator(self.get_integrater_reindex_operator())
if self.get_integrater_spacegroup_number():
reindex.set_spacegroup(self.get_integrater_spacegroup_number())
hklout = '%s_reindex.mtz' % os.path.splitext(integrate_mtz)[0]
reindex.set_hklin(integrate_mtz)
reindex.set_hklout(hklout)
reindex.reindex()
integrate_mtz = hklout
return integrate_mtz
def _integrate_finish(self):
"""Finish off the integration by running correct."""
# first run the postrefinement etc with spacegroup P1
# and the current unit cell - this will be used to
# obtain a benchmark rmsd in pixels / phi and also
# cell deviations (this is working towards spotting bad
# indexing solutions) - only do this if we have no
# reindex matrix... and no postrefined cell...
p1_deviations = None
# fix for bug # 3264 -
# if we have not run integration with refined parameters, make it so...
# erm? shouldn't this therefore return if this is the principle, or
# set the flag after we have tested the lattice?
if ("GXPARM.XDS" not in self._xds_data_files
and PhilIndex.params.xds.integrate.reintegrate):
logger.debug(
"Resetting integrater, to ensure refined orientation is used")
self.set_integrater_done(False)
if (not self.get_integrater_reindex_matrix() and not self._intgr_cell
and PhilIndex.params.xia2.settings.lattice_rejection
and not self.get_integrater_sweep().get_user_lattice()):
correct = self.Correct()
correct.set_data_range(
self._intgr_wedge[0] + self.get_frame_offset(),
self._intgr_wedge[1] + self.get_frame_offset(),
)
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# FIXME should this be using the correctly transformed
# cell or are the results ok without it?!
correct.set_spacegroup_number(1)
correct.set_cell(self._intgr_refiner_cell)
correct.run()
cell = correct.get_result("cell")
cell_esd = correct.get_result("cell_esd")
logger.debug("Postrefinement in P1 results:")
logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" % tuple(cell))
logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
tuple(cell_esd))
logger.debug("Deviations: %.2f pixels %.2f degrees" %
(correct.get_result("rmsd_pixel"),
correct.get_result("rmsd_phi")))
p1_deviations = (
correct.get_result("rmsd_pixel"),
correct.get_result("rmsd_phi"),
)
# next run the postrefinement etc with the given
# cell / lattice - this will be the assumed result...
integrate_hkl = os.path.join(self.get_working_directory(),
"INTEGRATE.HKL")
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
from dxtbx.serialize import load
from dials.algorithms.shadowing.filter import filter_shadowed_reflections
experiments_json = xparm_xds_to_experiments_json(
os.path.join(self.get_working_directory(), "XPARM.XDS"),
self.get_working_directory(),
)
experiments = load.experiment_list(experiments_json,
check_format=True)
imageset = experiments[0].imageset
masker = (imageset.get_format_class().get_instance(
imageset.paths()[0]).get_masker())
if masker is not None:
integrate_filename = integrate_hkl_to_reflection_file(
integrate_hkl, experiments_json,
self.get_working_directory())
reflections = flex.reflection_table.from_file(
integrate_filename)
t0 = time.time()
sel = filter_shadowed_reflections(experiments, reflections)
shadowed = reflections.select(sel)
t1 = time.time()
logger.debug("Filtered %i reflections in %.1f seconds" %
(sel.count(True), t1 - t0))
filter_hkl = os.path.join(self.get_working_directory(),
"FILTER.HKL")
with open(filter_hkl, "wb") as f:
detector = experiments[0].detector
for ref in shadowed:
p = detector[ref["panel"]]
ox, oy = p.get_raw_image_offset()
h, k, l = ref["miller_index"]
x, y, z = ref["xyzcal.px"]
dx, dy, dz = (2, 2, 2)
print(
"%i %i %i %.1f %.1f %.1f %.1f %.1f %.1f" %
(h, k, l, x + ox, y + oy, z, dx, dy, dz),
file=f,
)
t2 = time.time()
logger.debug("Written FILTER.HKL in %.1f seconds" % (t2 - t1))
correct = self.Correct()
correct.set_data_range(
self._intgr_wedge[0] + self.get_frame_offset(),
self._intgr_wedge[1] + self.get_frame_offset(),
)
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# BUG # 2695 probably comes from here - need to check...
# if the pointless interface comes back with a different
# crystal setting then the unit cell stored in self._intgr_cell
# needs to be set to None...
if self.get_integrater_spacegroup_number():
correct.set_spacegroup_number(
self.get_integrater_spacegroup_number())
if not self._intgr_cell:
raise RuntimeError("no unit cell to recycle")
correct.set_cell(self._intgr_cell)
# BUG # 3113 - new version of XDS will try and figure the
# best spacegroup out from the intensities (and get it wrong!)
# unless we set the spacegroup and cell explicitly
if not self.get_integrater_spacegroup_number():
cell = self._intgr_refiner_cell
lattice = self._intgr_refiner.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup_number(lattice)
# this should not prevent the postrefinement from
# working correctly, else what is above would not
# work correctly (the postrefinement test)
correct.set_spacegroup_number(spacegroup_number)
correct.set_cell(cell)
logger.debug("Setting spacegroup to: %d" % spacegroup_number)
logger.debug("Setting cell to: %.2f %.2f %.2f %.2f %.2f %.2f" %
tuple(cell))
if self.get_integrater_reindex_matrix():
# bug! if the lattice is not primitive the values in this
# reindex matrix need to be multiplied by a constant which
# depends on the Bravais lattice centering.
lattice = self._intgr_refiner.get_refiner_lattice()
matrix = self.get_integrater_reindex_matrix()
matrix = scitbx.matrix.sqr(matrix).transpose().elems
matrix = r_to_rt(matrix)
if lattice[1] == "P":
mult = 1
elif lattice[1] == "C" or lattice[1] == "I":
mult = 2
elif lattice[1] == "R":
mult = 3
elif lattice[1] == "F":
mult = 4
else:
raise RuntimeError("unknown multiplier for lattice %s" %
lattice)
logger.debug("REIDX multiplier for lattice %s: %d" %
(lattice, mult))
mult_matrix = [mult * m for m in matrix]
logger.debug("REIDX set to %d %d %d %d %d %d %d %d %d %d %d %d" %
tuple(mult_matrix))
correct.set_reindex_matrix(mult_matrix)
correct.run()
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file(
f"{pname} {xname} {dname} {sweep} CORRECT",
os.path.join(self.get_working_directory(), "CORRECT.LP"),
)
FileHandler.record_more_data_file(
f"{pname} {xname} {dname} {sweep} CORRECT",
os.path.join(self.get_working_directory(), "XDS_ASCII.HKL"),
)
# erm. just to be sure
if self.get_integrater_reindex_matrix() and correct.get_reindex_used():
raise RuntimeError("Reindex panic!")
# get the reindex operation used, which may be useful if none was
# set but XDS decided to apply one, e.g. #419.
if not self.get_integrater_reindex_matrix(
) and correct.get_reindex_used():
# convert this reindex operation to h, k, l form: n.b. this
# will involve dividing through by the lattice centring multiplier
matrix = rt_to_r(correct.get_reindex_used())
matrix = scitbx.matrix.sqr(matrix).transpose().elems
lattice = self._intgr_refiner.get_refiner_lattice()
if lattice[1] == "P":
mult = 1.0
elif lattice[1] == "C" or lattice[1] == "I":
mult = 2.0
elif lattice[1] == "R":
mult = 3.0
elif lattice[1] == "F":
mult = 4.0
matrix = [m / mult for m in matrix]
reindex_op = mat_to_symop(matrix)
# assign this to self: will this reset?! make for a leaky
# abstraction and just assign this...
# self.set_integrater_reindex_operator(reindex)
self._intgr_reindex_operator = reindex_op
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file(
f"{pname} {xname} {dname} {sweep} CORRECT",
os.path.join(self.get_working_directory(), "CORRECT.LP"),
)
# should get some interesting stuff from the XDS correct file
# here, for instance the resolution range to use in integration
# (which should be fed back if not fast) and so on...
self._intgr_corrected_hklout = os.path.join(
self.get_working_directory(), "XDS_ASCII.HKL")
# also record the batch range - needed for the analysis of the
# radiation damage in chef...
self._intgr_batches_out = (self._intgr_wedge[0], self._intgr_wedge[1])
# FIXME perhaps I should also feedback the GXPARM file here??
for file in ["GXPARM.XDS"]:
self._xds_data_files[file] = correct.get_output_data_file(file)
# record the postrefined cell parameters
self._intgr_cell = correct.get_result("cell")
self._intgr_n_ref = correct.get_result("n_ref")
logger.debug('Postrefinement in "correct" spacegroup results:')
logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
tuple(correct.get_result("cell")))
logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
tuple(correct.get_result("cell_esd")))
logger.debug(
"Deviations: %.2f pixels %.2f degrees" %
(correct.get_result("rmsd_pixel"), correct.get_result("rmsd_phi")))
logger.debug("Error correction parameters: A=%.3f B=%.3f" %
correct.get_result("sdcorrection"))
# compute misorientation of axes
xparm_file = os.path.join(self.get_working_directory(), "GXPARM.XDS")
handle = xparm.reader()
handle.read_file(xparm_file)
rotn = handle.rotation_axis
beam = handle.beam_vector
dot = sum(rotn[j] * beam[j] for j in range(3))
r = math.sqrt(sum(rotn[j] * rotn[j] for j in range(3)))
b = math.sqrt(sum(beam[j] * beam[j] for j in range(3)))
rtod = 180.0 / math.pi
angle = rtod * math.fabs(0.5 * math.pi - math.acos(dot / (r * b)))
logger.debug("Axis misalignment %.2f degrees" % angle)
correct_deviations = (
correct.get_result("rmsd_pixel"),
correct.get_result("rmsd_phi"),
)
if p1_deviations:
# compare and reject if both > 50% higher - though adding a little
# flexibility - 0.5 pixel / osc width slack.
pixel = p1_deviations[0]
phi = math.sqrt(0.05 * 0.05 + p1_deviations[1] * p1_deviations[1])
threshold = PhilIndex.params.xia2.settings.lattice_rejection_threshold
logger.debug("RMSD ratio: %.2f" % (correct_deviations[0] / pixel))
logger.debug("RMSPhi ratio: %.2f" % (correct_deviations[1] / phi))
if (correct_deviations[0] / pixel > threshold
and correct_deviations[1] / phi > threshold):
logger.info(
"Eliminating this indexing solution as postrefinement")
logger.info("deviations rather high relative to triclinic")
raise BadLatticeError(
"high relative deviations in postrefinement")
if (not PhilIndex.params.dials.fast_mode
and not PhilIndex.params.xds.keep_outliers):
# check for alien reflections and perhaps recycle - removing them
correct_remove = correct.get_remove()
if correct_remove:
current_remove = set()
final_remove = []
# first ensure that there are no duplicate entries...
if os.path.exists(
os.path.join(self.get_working_directory(),
"REMOVE.HKL")):
with open(
os.path.join(self.get_working_directory(),
"REMOVE.HKL")) as fh:
for line in fh.readlines():
h, k, l = list(map(int, line.split()[:3]))
z = float(line.split()[3])
if (h, k, l, z) not in current_remove:
current_remove.add((h, k, l, z))
for c in correct_remove:
if c in current_remove:
continue
final_remove.append(c)
logger.debug("%d alien reflections are already removed" %
(len(correct_remove) - len(final_remove)))
else:
# we want to remove all of the new dodgy reflections
final_remove = correct_remove
z_min = PhilIndex.params.xds.z_min
rejected = 0
with open(
os.path.join(self.get_working_directory(),
"REMOVE.HKL"), "w") as remove_hkl:
# write in the old reflections
for remove in current_remove:
z = remove[3]
if z >= z_min:
remove_hkl.write("%d %d %d %f\n" % remove)
else:
rejected += 1
logger.debug("Wrote %d old reflections to REMOVE.HKL" %
(len(current_remove) - rejected))
logger.debug("Rejected %d as z < %f" % (rejected, z_min))
# and the new reflections
rejected = 0
used = 0
for remove in final_remove:
z = remove[3]
if z >= z_min:
used += 1
remove_hkl.write("%d %d %d %f\n" % remove)
else:
rejected += 1
logger.debug("Wrote %d new reflections to REMOVE.HKL" %
(len(final_remove) - rejected))
logger.debug("Rejected %d as z < %f" % (rejected, z_min))
# we want to rerun the finishing step so...
# unless we have added no new reflections... or unless we
# have not confirmed the point group (see SCI-398)
if used and self.get_integrater_reindex_matrix():
self.set_integrater_finish_done(False)
else:
logger.debug(
"Going quickly so not removing %d outlier reflections..." %
len(correct.get_remove()))
# Convert INTEGRATE.HKL to MTZ format and reapply any reindexing operations
# spacegroup changes to allow use with CCP4 / Aimless for scaling
hklout = os.path.splitext(integrate_hkl)[0] + ".mtz"
self._factory.set_working_directory(self.get_working_directory())
pointless = self._factory.Pointless()
pointless.set_xdsin(integrate_hkl)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
integrate_mtz = hklout
if (self.get_integrater_reindex_operator()
or self.get_integrater_spacegroup_number()):
logger.debug("Reindexing things to MTZ")
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
if self.get_integrater_reindex_operator():
reindex.set_operator(self.get_integrater_reindex_operator())
if self.get_integrater_spacegroup_number():
reindex.set_spacegroup(self.get_integrater_spacegroup_number())
hklout = "%s_reindex.mtz" % os.path.splitext(integrate_mtz)[0]
reindex.set_hklin(integrate_mtz)
reindex.set_hklout(hklout)
reindex.reindex()
integrate_mtz = hklout
experiments_json = xparm_xds_to_experiments_json(
self._xds_data_files["GXPARM.XDS"], self.get_working_directory())
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(f"{pname} {xname} {dname} {sweep}",
experiments_json)
FileHandler.record_more_data_file(
f"{pname} {xname} {dname} {sweep} INTEGRATE", integrate_mtz)
self._intgr_experiments_filename = experiments_json
return integrate_mtz
def _integrate_finish(self):
'''Finish off the integration by running dials.export.'''
# FIXME - do we want to export every time we call this method
# (the file will not have changed) and also (more important) do
# we want a different exported MTZ file every time (I do not think
# that we do; these can be very large) - was exporter.get_xpid() ->
# now dials
exporter = self.ExportMtz()
exporter.set_reflections_filename(self._intgr_integrated_pickle)
mtz_filename = os.path.join(self.get_working_directory(),
'%s_integrated.mtz' % 'dials')
exporter.set_mtz_filename(mtz_filename)
exporter.run()
self._intgr_integrated_filename = mtz_filename
# record integrated MTZ file for e.g. BLEND.
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(
'%s %s %s %s INTEGRATE' % (pname, xname, dname, sweep),
mtz_filename)
from iotbx.reflection_file_reader import any_reflection_file
miller_arrays = any_reflection_file(
self._intgr_integrated_filename).as_miller_arrays()
# look for profile-fitted intensities
intensities = [
ma for ma in miller_arrays
if ma.info().labels == ['IPR', 'SIGIPR']
]
if len(intensities) == 0:
# look instead for summation-integrated intensities
intensities = [
ma for ma in miller_arrays
if ma.info().labels == ['I', 'SIGI']
]
assert len(intensities)
self._intgr_n_ref = intensities[0].size()
if not os.path.isfile(self._intgr_integrated_filename):
raise RuntimeError("dials.export failed: %s does not exist." %
self._intgr_integrated_filename)
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice()):
Debug.write('Not reindexing to spacegroup %d (%s)' % \
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
return mtz_filename
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == 0:
Debug.write('Not reindexing to spacegroup %d (%s)' % \
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
return mtz_filename
Debug.write('Reindexing to spacegroup %d (%s)' % \
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
hklin = mtz_filename
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
reindex.set_operator(self._intgr_reindex_operator)
if self._intgr_spacegroup_number:
reindex.set_spacegroup(self._intgr_spacegroup_number)
else:
reindex.set_spacegroup(
lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice()))
hklout = '%s_reindex.mtz' % hklin[:-4]
reindex.set_hklin(hklin)
reindex.set_hklout(hklout)
reindex.reindex()
self._intgr_integrated_filename = hklout
self._intgr_cell = reindex.get_cell()
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(
'%s %s %s %s experiments' % (pname, xname, dname, sweep),
self.get_integrated_experiments())
return hklout
def _integrate_finish(self):
"""Finish off the integration by running dials.export."""
# FIXME - do we want to export every time we call this method
# (the file will not have changed) and also (more important) do
# we want a different exported MTZ file every time (I do not think
# that we do; these can be very large) - was exporter.get_xpid() ->
# now dials
if self._output_format == "hkl":
exporter = self.ExportMtz()
exporter.set_reflections_filename(
self._intgr_integrated_reflections)
mtz_filename = os.path.join(self.get_working_directory(),
"%s_integrated.mtz" % "dials")
exporter.set_mtz_filename(mtz_filename)
exporter.run()
self._intgr_integrated_filename = mtz_filename
# record integrated MTZ file for e.g. BLEND.
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(
"%s %s %s %s INTEGRATE" % (pname, xname, dname, sweep),
mtz_filename)
from iotbx.reflection_file_reader import any_reflection_file
miller_arrays = any_reflection_file(
self._intgr_integrated_filename).as_miller_arrays()
# look for profile-fitted intensities
intensities = [
ma for ma in miller_arrays
if ma.info().labels == ["IPR", "SIGIPR"]
]
if len(intensities) == 0:
# look instead for summation-integrated intensities
intensities = [
ma for ma in miller_arrays
if ma.info().labels == ["I", "SIGI"]
]
assert len(intensities)
self._intgr_n_ref = intensities[0].size()
if not os.path.isfile(self._intgr_integrated_filename):
raise RuntimeError("dials.export failed: %s does not exist." %
self._intgr_integrated_filename)
if (self._intgr_reindex_operator is None
and self._intgr_spacegroup_number == lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice())):
Debug.write("Not reindexing to spacegroup %d (%s)" %
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
return mtz_filename
if (self._intgr_reindex_operator is None
and self._intgr_spacegroup_number == 0):
Debug.write("Not reindexing to spacegroup %d (%s)" %
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
return mtz_filename
Debug.write(
"Reindexing to spacegroup %d (%s)" %
(self._intgr_spacegroup_number, self._intgr_reindex_operator))
hklin = mtz_filename
from xia2.Wrappers.CCP4.Reindex import Reindex
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
reindex.set_operator(self._intgr_reindex_operator)
if self._intgr_spacegroup_number:
reindex.set_spacegroup(self._intgr_spacegroup_number)
else:
reindex.set_spacegroup(
lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice()))
hklout = "%s_reindex.mtz" % hklin[:-4]
reindex.set_hklin(hklin)
reindex.set_hklout(hklout)
reindex.reindex()
self._intgr_integrated_filename = hklout
self._intgr_cell = reindex.get_cell()
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(
"%s %s %s %s" % (pname, xname, dname, sweep),
self.get_integrated_experiments(),
)
FileHandler.record_more_data_file(
"%s %s %s %s" % (pname, xname, dname, sweep),
self.get_integrated_reflections(),
)
return hklout
elif self._output_format == "pickle":
if (self._intgr_reindex_operator is None
and self._intgr_spacegroup_number == lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice())):
Debug.write("Not reindexing to spacegroup %d (%s)" %
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
return self._intgr_integrated_reflections
if (self._intgr_reindex_operator is None
and self._intgr_spacegroup_number == 0):
Debug.write("Not reindexing to spacegroup %d (%s)" %
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
return self._intgr_integrated_reflections
Debug.write(
"Reindexing to spacegroup %d (%s)" %
(self._intgr_spacegroup_number, self._intgr_reindex_operator))
from xia2.Wrappers.Dials.Reindex import Reindex
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
reindex.set_cb_op(self._intgr_reindex_operator)
if self._intgr_spacegroup_number:
reindex.set_space_group(self._intgr_spacegroup_number)
else:
reindex.set_space_group(
lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice()))
reindex.set_experiments_filename(self.get_integrated_experiments())
reindex.set_indexed_filename(self.get_integrated_reflections())
reindex.run()
self._intgr_integrated_reflections = (
reindex.get_reindexed_reflections_filename())
self._intgr_integrated_filename = (
reindex.get_reindexed_reflections_filename())
self._intgr_experiments_filename = (
reindex.get_reindexed_experiments_filename())
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(
"%s %s %s %s" % (pname, xname, dname, sweep),
self.get_integrated_experiments(),
)
FileHandler.record_more_data_file(
"%s %s %s %s" % (pname, xname, dname, sweep),
self.get_integrated_reflections(),
)
return None # this will be set to intgr_hklout - better to cause failure