def run(self, method):
from xia2.Handlers.Streams import Debug
Debug.write("Running dials.index")
self.clear_command_line()
for f in self._sweep_filenames:
self.add_command_line(f)
for f in self._spot_filenames:
self.add_command_line(f)
if len(self._sweep_filenames) > 1:
self.add_command_line("auto_reduction.action=fix")
self.add_command_line("indexing.method=%s" % method)
nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
self.set_cpu_threads(nproc)
self.add_command_line("indexing.nproc=%i" % nproc)
if PhilIndex.params.xia2.settings.small_molecule:
self.add_command_line("filter_ice=false")
if self._reflections_per_degree is not None:
self.add_command_line("reflections_per_degree=%i" %
self._reflections_per_degree)
if self._fft3d_n_points is not None:
self.add_command_line(
"fft3d.reciprocal_space_grid.n_points=%i" %
self._fft3d_n_points)
if self._close_to_spindle_cutoff is not None:
self.add_command_line("close_to_spindle_cutoff=%f" %
self._close_to_spindle_cutoff)
if self._outlier_algorithm:
self.add_command_line("outlier.algorithm=%s" %
self._outlier_algorithm)
if self._max_cell:
self.add_command_line("max_cell=%g" % self._max_cell)
if self._max_cell_max_height_fraction is not None:
self.add_command_line("max_height_fraction=%g" %
self._max_cell_max_height_fraction)
if self._min_cell:
self.add_command_line("min_cell=%d" % self._min_cell)
if self._histogram_binning is not None:
self.add_command_line(
"max_cell_estimation.histogram_binning=%s" %
self._histogram_binning)
if self._nearest_neighbor_percentile is not None:
self.add_command_line(
"max_cell_estimation.nearest_neighbor_percentile=%s" %
self._nearest_neighbor_percentile)
if self._d_min_start:
self.add_command_line("d_min_start=%f" % self._d_min_start)
if self._indxr_input_lattice is not None:
from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
self._symm = lattice_to_spacegroup_number(
self._indxr_input_lattice)
self.add_command_line("known_symmetry.space_group=%s" %
self._symm)
if self._indxr_input_cell is not None:
self.add_command_line(
'known_symmetry.unit_cell="%s,%s,%s,%s,%s,%s"' %
self._indxr_input_cell)
if self._maximum_spot_error:
self.add_command_line("maximum_spot_error=%.f" %
self._maximum_spot_error)
if self._detector_fix:
self.add_command_line("detector.fix=%s" % self._detector_fix)
if self._beam_fix:
self.add_command_line("beam.fix=%s" % self._beam_fix)
if self._phil_file is not None:
self.add_command_line("%s" % self._phil_file)
self._experiment_filename = os.path.join(
self.get_working_directory(),
"%d_indexed.expt" % self.get_xpid())
self._indexed_filename = os.path.join(
self.get_working_directory(),
"%d_indexed.refl" % self.get_xpid())
self.add_command_line("output.experiments=%s" %
self._experiment_filename)
self.add_command_line("output.reflections=%s" %
self._indexed_filename)
self.start()
self.close_wait()
if not os.path.isfile(
self._experiment_filename) or not os.path.isfile(
self._indexed_filename):
# Indexing failed
with open(self.get_log_file(), "r") as fh:
if "No suitable lattice could be found" in fh.read():
raise libtbx.utils.Sorry(
"No suitable indexing solution could be found.\n\n"
"You can view the reciprocal space with:\n"
"dials.reciprocal_lattice_viewer %s" % " ".join(
os.path.normpath(
os.path.join(self.get_working_directory(),
p))
for p in self._sweep_filenames +
self._spot_filenames))
else:
raise RuntimeError(
"dials.index failed, see log file for more details: %s"
% self.get_log_file())
self.check_for_errors()
for record in self.get_all_output():
if "Too few reflections to parameterise" in record:
Debug.write(record.strip())
from dials.array_family import flex
from dxtbx.serialize import load
self._experiment_list = load.experiment_list(
self._experiment_filename)
self._reflections = flex.reflection_table.from_file(
self._indexed_filename)
crystal = self._experiment_list.crystals()[0]
self._p1_cell = crystal.get_unit_cell().parameters()
refined_sel = self._reflections.get_flags(
self._reflections.flags.used_in_refinement)
refl = self._reflections.select(refined_sel)
xc, yc, zc = refl["xyzcal.px"].parts()
xo, yo, zo = refl["xyzobs.px.value"].parts()
import math
self._nref = refl.size()
self._rmsd_x = math.sqrt(flex.mean(flex.pow2(xc - xo)))
self._rmsd_y = math.sqrt(flex.mean(flex.pow2(yc - yo)))
self._rmsd_z = math.sqrt(flex.mean(flex.pow2(zc - zo)))
def run(self, ignore_errors=False):
"""Run idxref."""
# image_header = self.get_header()
## crank through the header dictionary and replace incorrect
## information with updated values through the indexer
## interface if available...
## need to add distance, wavelength - that should be enough...
# if self.get_distance():
# image_header['distance'] = self.get_distance()
# if self.get_wavelength():
# image_header['wavelength'] = self.get_wavelength()
# if self.get_two_theta():
# image_header['two_theta'] = self.get_two_theta()
header = imageset_to_xds(
self.get_imageset(),
refined_beam_vector=self._refined_beam_vector,
refined_rotation_axis=self._refined_rotation_axis,
refined_distance=self._refined_distance,
)
xds_inp = open(os.path.join(self.get_working_directory(), "XDS.INP"), "w")
# what are we doing?
xds_inp.write("JOB=IDXREF\n")
xds_inp.write("MAXIMUM_NUMBER_OF_PROCESSORS=%d\n" % self._parallel)
# FIXME this needs to be calculated from the beam centre...
if self._refined_origin:
xds_inp.write("ORGX=%f ORGY=%f\n" % tuple(self._refined_origin))
else:
xds_inp.write("ORGX=%f ORGY=%f\n" % tuple(self._org))
# FIXME in here make sure sweep is wider than 5 degrees
# before specifying AXIS: if <= 5 degrees replace AXIS with
# nothing - base this on the maximum possible angular separation
min_frame = self._spot_range[0][0]
max_frame = self._spot_range[-1][1]
refine_params = [p for p in self._params.refine]
phi_width = self.get_phi_width()
if "AXIS" in refine_params and (max_frame - min_frame) * phi_width < 5.0:
refine_params.remove("AXIS")
xds_inp.write("REFINE(IDXREF)=%s\n" % " ".join(refine_params))
if self._starting_frame and self._starting_angle:
xds_inp.write("STARTING_FRAME=%d\n" % self._starting_frame)
xds_inp.write("STARTING_ANGLE=%f\n" % self._starting_angle)
# FIXME this looks like a potential bug - what will
# happen if the input lattice has not been set??
if self._indxr_input_cell:
self._cell = self._indxr_input_cell
if self._indxr_input_lattice:
self._symm = lattice_to_spacegroup_number(self._indxr_input_lattice)
if self._cell:
xds_inp.write("SPACE_GROUP_NUMBER=%d\n" % self._symm)
cell_format = "%6.2f %6.2f %6.2f %6.2f %6.2f %6.2f"
xds_inp.write("UNIT_CELL_CONSTANTS=%s\n" % cell_format % self._cell)
if self._a_axis:
xds_inp.write("UNIT_CELL_A-AXIS=%.2f %.2f %.2f\n" % tuple(self._a_axis))
if self._b_axis:
xds_inp.write("UNIT_CELL_B-AXIS=%.2f %.2f %.2f\n" % tuple(self._b_axis))
if self._c_axis:
xds_inp.write("UNIT_CELL_C-AXIS=%.2f %.2f %.2f\n" % tuple(self._c_axis))
for record in header:
xds_inp.write("%s\n" % record)
name_template = template_to_xds(
os.path.join(self.get_directory(), self.get_template())
)
record = "NAME_TEMPLATE_OF_DATA_FRAMES=%s\n" % name_template
xds_inp.write(record)
xds_inp.write("DATA_RANGE=%d %d\n" % self._data_range)
for spot_range in self._spot_range:
xds_inp.write("SPOT_RANGE=%d %d\n" % spot_range)
xds_inp.write("BACKGROUND_RANGE=%d %d\n" % self._background_range)
xds_inp.close()
# copy the input file...
shutil.copyfile(
os.path.join(self.get_working_directory(), "XDS.INP"),
os.path.join(
self.get_working_directory(), "%d_IDXREF.INP" % self.get_xpid()
),
)
# write the input data files...
for file_name in self._input_data_files_list:
src = self._input_data_files[file_name]
dst = os.path.join(self.get_working_directory(), file_name)
if src != dst:
shutil.copyfile(src, dst)
self.start()
self.close_wait()
xds_check_version_supported(self.get_all_output())
if not ignore_errors:
xds_check_error(self.get_all_output())
# If xds_check_error detects any errors it will raise an exception
# The caller can then continue using the run_continue_from_error()
# function. If XDS does not throw any errors we just plow on.
return self.continue_from_error()
def continue_from_error(self):
# copy the LP file
shutil.copyfile(
os.path.join(self.get_working_directory(), "IDXREF.LP"),
os.path.join(
self.get_working_directory(), "%d_IDXREF.LP" % self.get_xpid()
),
)
# parse the output
lp = open(
os.path.join(self.get_working_directory(), "IDXREF.LP"), "r"
).readlines()
self._fraction_rmsd_rmsphi = _parse_idxref_lp_quality(lp)
self._idxref_data = _parse_idxref_lp(lp)
if not self._idxref_data:
raise RuntimeError("indexing failed")
st = _parse_idxref_lp_subtree(lp)
if 2 in st:
if st[2] > st[1] / 10.0:
Debug.write("Look closely at autoindexing solution!")
self._index_tree_problem = True
for j in sorted(st):
Debug.write("%2d: %5d" % (j, st[j]))
# print out some (perhaps dire) warnings about the beam centre
# if there is really any ambiguity...
origins = _parse_idxref_index_origin(lp)
assert (0, 0, 0) in origins
quality_0 = origins[(0, 0, 0)][0]
alternatives = []
for hkl in origins:
if hkl == (0, 0, 0):
continue
if origins[hkl][0] < 4 * quality_0:
quality, delta, beam_x, beam_y = origins[hkl]
alternatives.append(
(hkl[0], hkl[1], hkl[2], quality, beam_x, beam_y)
)
if alternatives:
Debug.write("Alternative indexing possible:")
for alternative in alternatives:
Debug.write("... %3d %3d %3d %4.1f %6.1f %6.1f" % alternative)
# New algorithm in here - now use iotbx.lattice_symmetry with the
# P1 indexing solution (solution #1) to determine the list of
# allowable solutions - only consider those lattices in this
# allowed list (unless we have user input)
from xia2.Wrappers.Phenix.LatticeSymmetry import LatticeSymmetry
ls = LatticeSymmetry()
ls.set_lattice("aP")
ls.set_cell(tuple(self._idxref_data[44]["cell"]))
ls.generate()
allowed_lattices = ls.get_lattices()
for j in range(1, 45):
if j not in self._idxref_data:
continue
data = self._idxref_data[j]
lattice = data["lattice"]
fit = data["fit"]
cell = data["cell"]
mosaic = data["mosaic"]
if self._symm and self._cell and self._indxr_user_input_lattice:
if (
self._compare_cell(self._cell, cell)
and lattice_to_spacegroup_number(lattice) == self._symm
):
if lattice in self._indexing_solutions:
if self._indexing_solutions[lattice]["goodness"] < fit:
continue
self._indexing_solutions[lattice] = {
"goodness": fit,
"cell": cell,
}
else:
if lattice in allowed_lattices or (self._symm and fit < 200.0):
# bug 2417 - if we have an input lattice then we
# don't want to include anything higher symmetry
# in the results table...
if self._symm:
if lattice_to_spacegroup_number(lattice) > self._symm:
Debug.write(
"Ignoring solution with lattice %s" % lattice
)
continue
if lattice in self._indexing_solutions:
if self._indexing_solutions[lattice]["goodness"] < fit:
continue
self._indexing_solutions[lattice] = {
"goodness": fit,
"cell": cell,
}
# postprocess this list, to remove lattice solutions which are
# lower symmetry but higher penalty than the putative correct
# one, if self._symm is set...
if self._symm:
assert self._indexing_solutions, (
"No remaining indexing solutions (%s, %s)"
% (s2l(self._symm), self._symm)
)
else:
assert self._indexing_solutions, "No remaining indexing solutions"
if self._symm:
max_p = 2.0 * self._indexing_solutions[s2l(self._symm)]["goodness"]
to_remove = []
for lattice in self._indexing_solutions:
if self._indexing_solutions[lattice]["goodness"] > max_p:
to_remove.append(lattice)
for lattice in to_remove:
Debug.write("Ignoring solution with lattice %s" % lattice)
del self._indexing_solutions[lattice]
# get the highest symmetry "acceptable" solution
items = [
(k, self._indexing_solutions[k]["cell"])
for k in self._indexing_solutions
]
# if there was a preassigned cell and symmetry return now
# with everything done, else select the "top" solution and
# reindex, resetting the input cell and symmetry.
if self._cell:
# select the solution which matches the input unit cell
# actually after the changes above this should now be the
# only solution in the table..
Debug.write(
"Target unit cell: %.2f %.2f %.2f %.2f %.2f %.2f" % self._cell
)
for l in items:
if lattice_to_spacegroup_number(l[0]) == self._symm:
# this should be the correct solution...
# check the unit cell...
cell = l[1]
cell_str = "%.2f %.2f %.2f %.2f %.2f %.2f" % cell
Debug.write("Chosen unit cell: %s" % cell_str)
self._indxr_lattice = l[0]
self._indxr_cell = l[1]
self._indxr_mosaic = mosaic
else:
# select the top solution as the input cell and reset the
# "indexing done" flag
sorted_list = SortLattices(items)
# print sorted_list
self._symm = lattice_to_spacegroup_number(sorted_list[0][0])
self._cell = sorted_list[0][1]
return False
# get the refined distance &c.
beam, distance = _parse_idxref_lp_distance_etc(lp)
self._refined_beam = beam
self._refined_distance = distance
# gather the output files
for file in self._output_data_files_list:
self._output_data_files[file] = os.path.join(
self.get_working_directory(), file
)
return True
def _integrate_finish(self):
'''Finish off the integration by running correct.'''
# first run the postrefinement etc with spacegroup P1
# and the current unit cell - this will be used to
# obtain a benchmark rmsd in pixels / phi and also
# cell deviations (this is working towards spotting bad
# indexing solutions) - only do this if we have no
# reindex matrix... and no postrefined cell...
p1_deviations = None
# fix for bug # 3264 -
# if we have not run integration with refined parameters, make it so...
# erm? shouldn't this therefore return if this is the principle, or
# set the flag after we have tested the lattice?
if not self._xds_data_files.has_key('GXPARM.XDS') and \
PhilIndex.params.xds.integrate.reintegrate:
Debug.write(
'Resetting integrater, to ensure refined orientation is used')
self.set_integrater_done(False)
if not self.get_integrater_reindex_matrix() and not self._intgr_cell \
and not Flags.get_no_lattice_test() and \
not self.get_integrater_sweep().get_user_lattice():
correct = self.Correct()
correct.set_data_range(self._intgr_wedge[0],
self._intgr_wedge[1])
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# FIXME should this be using the correctly transformed
# cell or are the results ok without it?!
correct.set_spacegroup_number(1)
correct.set_cell(self._intgr_refiner_cell)
correct.run()
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file('%s %s %s %s CORRECT' % \
(pname, xname, dname, sweep),
os.path.join(
self.get_working_directory(),
'CORRECT.LP'))
FileHandler.record_more_data_file(
'%s %s %s %s CORRECT' % (pname, xname, dname, sweep),
os.path.join(self.get_working_directory(), 'XDS_ASCII.HKL'))
cell = correct.get_result('cell')
cell_esd = correct.get_result('cell_esd')
Debug.write('Postrefinement in P1 results:')
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(cell))
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(cell_esd))
Debug.write('Deviations: %.2f pixels %.2f degrees' % \
(correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi')))
p1_deviations = (correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi'))
# next run the postrefinement etc with the given
# cell / lattice - this will be the assumed result...
correct = self.Correct()
correct.set_data_range(self._intgr_wedge[0],
self._intgr_wedge[1])
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# BUG # 2695 probably comes from here - need to check...
# if the pointless interface comes back with a different
# crystal setting then the unit cell stored in self._intgr_cell
# needs to be set to None...
if self.get_integrater_spacegroup_number():
correct.set_spacegroup_number(
self.get_integrater_spacegroup_number())
if not self._intgr_cell:
raise RuntimeError, 'no unit cell to recycle'
correct.set_cell(self._intgr_cell)
# BUG # 3113 - new version of XDS will try and figure the
# best spacegroup out from the intensities (and get it wrong!)
# unless we set the spacegroup and cell explicitly
if not self.get_integrater_spacegroup_number():
cell = self._intgr_refiner_cell
lattice = self._intgr_refiner.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup_number(lattice)
# this should not prevent the postrefinement from
# working correctly, else what is above would not
# work correctly (the postrefinement test)
correct.set_spacegroup_number(spacegroup_number)
correct.set_cell(cell)
Debug.write('Setting spacegroup to: %d' % spacegroup_number)
Debug.write(
'Setting cell to: %.2f %.2f %.2f %.2f %.2f %.2f' % tuple(cell))
if self.get_integrater_reindex_matrix():
# bug! if the lattice is not primitive the values in this
# reindex matrix need to be multiplied by a constant which
# depends on the Bravais lattice centering.
lattice = self._intgr_refiner.get_refiner_lattice()
import scitbx.matrix
matrix = self.get_integrater_reindex_matrix()
matrix = scitbx.matrix.sqr(matrix).transpose().elems
matrix = r_to_rt(matrix)
if lattice[1] == 'P':
mult = 1
elif lattice[1] == 'C' or lattice[1] == 'I':
mult = 2
elif lattice[1] == 'R':
mult = 3
elif lattice[1] == 'F':
mult = 4
else:
raise RuntimeError, 'unknown multiplier for lattice %s' % \
lattice
Debug.write('REIDX multiplier for lattice %s: %d' % \
(lattice, mult))
mult_matrix = [mult * m for m in matrix]
Debug.write('REIDX set to %d %d %d %d %d %d %d %d %d %d %d %d' % \
tuple(mult_matrix))
correct.set_reindex_matrix(mult_matrix)
correct.run()
# erm. just to be sure
if self.get_integrater_reindex_matrix() and \
correct.get_reindex_used():
raise RuntimeError, 'Reindex panic!'
# get the reindex operation used, which may be useful if none was
# set but XDS decided to apply one, e.g. #419.
if not self.get_integrater_reindex_matrix() and \
correct.get_reindex_used():
# convert this reindex operation to h, k, l form: n.b. this
# will involve dividing through by the lattice centring multiplier
matrix = rt_to_r(correct.get_reindex_used())
import scitbx.matrix
matrix = scitbx.matrix.sqr(matrix).transpose().elems
lattice = self._intgr_refiner.get_refiner_lattice()
if lattice[1] == 'P':
mult = 1.0
elif lattice[1] == 'C' or lattice[1] == 'I':
mult = 2.0
elif lattice[1] == 'R':
mult = 3.0
elif lattice[1] == 'F':
mult = 4.0
matrix = [m / mult for m in matrix]
reindex_op = mat_to_symop(matrix)
# assign this to self: will this reset?! make for a leaky
# abstraction and just assign this...
# self.set_integrater_reindex_operator(reindex)
self._intgr_reindex_operator = reindex_op
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file('%s %s %s %s CORRECT' % \
(pname, xname, dname, sweep),
os.path.join(self.get_working_directory(),
'CORRECT.LP'))
# should get some interesting stuff from the XDS correct file
# here, for instance the resolution range to use in integration
# (which should be fed back if not fast) and so on...
self._intgr_corrected_hklout = os.path.join(self.get_working_directory(),
'XDS_ASCII.HKL')
# also record the batch range - needed for the analysis of the
# radiation damage in chef...
self._intgr_batches_out = (self._intgr_wedge[0],
self._intgr_wedge[1])
# FIXME perhaps I should also feedback the GXPARM file here??
for file in ['GXPARM.XDS']:
self._xds_data_files[file] = correct.get_output_data_file(file)
# record the postrefined cell parameters
self._intgr_cell = correct.get_result('cell')
self._intgr_n_ref = correct.get_result('n_ref')
Debug.write('Postrefinement in "correct" spacegroup results:')
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(correct.get_result('cell')))
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(correct.get_result('cell_esd')))
Debug.write('Deviations: %.2f pixels %.2f degrees' % \
(correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi')))
Debug.write('Error correction parameters: A=%.3f B=%.3f' % \
correct.get_result('sdcorrection'))
# compute misorientation of axes
xparm_file = os.path.join(self.get_working_directory(), 'GXPARM.XDS')
from iotbx.xds import xparm
handle = xparm.reader()
handle.read_file(xparm_file)
rotn = handle.rotation_axis
beam = handle.beam_vector
dot = sum([rotn[j] * beam[j] for j in range(3)])
r = math.sqrt(sum([rotn[j] * rotn[j] for j in range(3)]))
b = math.sqrt(sum([beam[j] * beam[j] for j in range(3)]))
rtod = 180.0 / math.pi
angle = rtod * math.fabs(0.5 * math.pi - math.acos(dot / (r * b)))
Debug.write('Axis misalignment %.2f degrees' % angle)
correct_deviations = (correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi'))
if p1_deviations:
# compare and reject if both > 50% higher - though adding a little
# flexibility - 0.5 pixel / osc width slack.
pixel = p1_deviations[0]
phi = math.sqrt(0.05 * 0.05 + \
p1_deviations[1] * p1_deviations[1])
threshold = Flags.get_rejection_threshold()
Debug.write('RMSD ratio: %.2f' % (correct_deviations[0] / pixel))
Debug.write('RMSPhi ratio: %.2f' % (correct_deviations[1] / phi))
if correct_deviations[0] / pixel > threshold and \
correct_deviations[1] / phi > threshold:
Chatter.write(
'Eliminating this indexing solution as postrefinement')
Chatter.write(
'deviations rather high relative to triclinic')
raise BadLatticeError, \
'high relative deviations in postrefinement'
if not Flags.get_quick() and Flags.get_remove():
# check for alien reflections and perhaps recycle - removing them
if len(correct.get_remove()) > 0:
correct_remove = correct.get_remove()
current_remove = []
final_remove = []
# first ensure that there are no duplicate entries...
if os.path.exists(os.path.join(
self.get_working_directory(),
'REMOVE.HKL')):
for line in open(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'), 'r').readlines():
h, k, l = map(int, line.split()[:3])
z = float(line.split()[3])
if not (h, k, l, z) in current_remove:
current_remove.append((h, k, l, z))
for c in correct_remove:
if c in current_remove:
continue
final_remove.append(c)
Debug.write(
'%d alien reflections are already removed' % \
(len(correct_remove) - len(final_remove)))
else:
# we want to remove all of the new dodgy reflections
final_remove = correct_remove
remove_hkl = open(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'), 'w')
z_min = Flags.get_z_min()
rejected = 0
# write in the old reflections
for remove in current_remove:
z = remove[3]
if z >= z_min:
remove_hkl.write('%d %d %d %f\n' % remove)
else:
rejected += 1
Debug.write('Wrote %d old reflections to REMOVE.HKL' % \
(len(current_remove) - rejected))
Debug.write('Rejected %d as z < %f' % \
(rejected, z_min))
# and the new reflections
rejected = 0
used = 0
for remove in final_remove:
z = remove[3]
if z >= z_min:
used += 1
remove_hkl.write('%d %d %d %f\n' % remove)
else:
rejected += 1
Debug.write('Wrote %d new reflections to REMOVE.HKL' % \
(len(final_remove) - rejected))
Debug.write('Rejected %d as z < %f' % \
(rejected, z_min))
remove_hkl.close()
# we want to rerun the finishing step so...
# unless we have added no new reflections... or unless we
# have not confirmed the point group (see SCI-398)
if used and self.get_integrater_reindex_matrix():
self.set_integrater_finish_done(False)
else:
Debug.write(
'Going quickly so not removing %d outlier reflections...' % \
len(correct.get_remove()))
# Convert INTEGRATE.HKL to MTZ format and reapply any reindexing operations
# spacegroup changes to allow use with CCP4 / Aimless for scaling
integrate_hkl = os.path.join(
self.get_working_directory(), 'INTEGRATE.HKL')
hklout = os.path.splitext(integrate_hkl)[0] + ".mtz"
self._factory.set_working_directory(self.get_working_directory())
pointless = self._factory.Pointless()
pointless.set_xdsin(integrate_hkl)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
integrate_mtz = hklout
if self.get_integrater_reindex_operator() or \
self.get_integrater_spacegroup_number():
Debug.write('Reindexing things to MTZ')
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
if self.get_integrater_reindex_operator():
reindex.set_operator(self.get_integrater_reindex_operator())
if self.get_integrater_spacegroup_number():
reindex.set_spacegroup(self.get_integrater_spacegroup_number())
hklout = '%s_reindex.mtz' % os.path.splitext(integrate_mtz)[0]
reindex.set_hklin(integrate_mtz)
reindex.set_hklout(hklout)
reindex.reindex()
integrate_mtz = hklout
return integrate_mtz
def _index(self):
if PhilIndex.params.dials.index.method in (libtbx.Auto, None):
if self._indxr_input_cell is not None:
indexer = self._do_indexing("real_space_grid_search")
else:
try:
indexer_fft3d = self._do_indexing(method="fft3d")
nref_3d, rmsd_3d = indexer_fft3d.get_nref_rmsds()
except Exception as e:
nref_3d = None
rmsd_3d = None
try:
indexer_fft1d = self._do_indexing(method="fft1d")
nref_1d, rmsd_1d = indexer_fft1d.get_nref_rmsds()
except Exception as e:
nref_1d = None
rmsd_1d = None
if (nref_1d is not None and nref_3d is None or
(nref_1d > nref_3d and rmsd_1d[0] < rmsd_3d[0]
and rmsd_1d[1] < rmsd_3d[1] and rmsd_1d[2] < rmsd_3d[2])):
indexer = indexer_fft1d
elif nref_3d is not None:
indexer = indexer_fft3d
else:
raise RuntimeError(e)
else:
indexer = self._do_indexing(
method=PhilIndex.params.dials.index.method)
# not strictly the P1 cell, rather the cell that was used in indexing
self._p1_cell = indexer._p1_cell
self.set_indexer_payload("indexed_filename",
indexer.get_indexed_filename())
from cctbx.sgtbx import bravais_types
from dxtbx.serialize import load
indexed_file = indexer.get_indexed_filename()
indexed_experiments = indexer.get_experiments_filename()
fast_mode = PhilIndex.params.dials.fast_mode
trust_beam_centre = PhilIndex.params.xia2.settings.trust_beam_centre
multi_sweep_indexing = PhilIndex.params.xia2.settings.multi_sweep_indexing == True
if not (trust_beam_centre or fast_mode or multi_sweep_indexing):
checksym = self.CheckIndexingSymmetry()
checksym.set_experiments_filename(indexed_experiments)
checksym.set_indexed_filename(indexed_file)
checksym.set_grid_search_scope(1)
checksym.run()
hkl_offset = checksym.get_hkl_offset()
Debug.write("hkl_offset: %s" % str(hkl_offset))
if hkl_offset is not None and hkl_offset != (0, 0, 0):
reindex = self.Reindex()
reindex.set_hkl_offset(hkl_offset)
reindex.set_indexed_filename(indexed_file)
reindex.run()
indexed_file = reindex.get_reindexed_reflections_filename()
# do some scan-static refinement - run twice, first without outlier
# rejection as the model is too far from reality to do a sensible job of
# outlier rejection
refiner = self.Refine()
refiner.set_experiments_filename(indexed_experiments)
refiner.set_indexed_filename(
reindex.get_reindexed_reflections_filename())
refiner.set_outlier_algorithm(None)
refiner.run()
indexed_experiments = refiner.get_refined_experiments_filename(
)
# now again with outlier rejection (possibly)
refiner = self.Refine()
refiner.set_experiments_filename(indexed_experiments)
refiner.set_indexed_filename(indexed_file)
refiner.run()
indexed_experiments = refiner.get_refined_experiments_filename(
)
if self._indxr_input_lattice is None:
# FIXME in here should respect the input unit cell and lattice if provided
# FIXME from this (i) populate the helper table,
# (ii) try to avoid re-running the indexing
# step if we eliminate a solution as we have all of the refined results
# already available.
rbs = self.RefineBravaisSettings()
rbs.set_experiments_filename(indexed_experiments)
rbs.set_indexed_filename(indexed_file)
if PhilIndex.params.dials.fix_geometry:
rbs.set_detector_fix('all')
rbs.set_beam_fix('all')
FileHandler.record_log_file(
'%s LATTICE' % self.get_indexer_full_name(),
rbs.get_log_file())
rbs.run()
from cctbx import crystal, sgtbx
for k in sorted(rbs.get_bravais_summary()):
summary = rbs.get_bravais_summary()[k]
# FIXME need to do this better - for the moment only accept lattices
# where R.M.S. deviation is less than twice P1 R.M.S. deviation.
if self._indxr_input_lattice is None:
if not summary['recommended']:
continue
experiments = load.experiment_list(summary['experiments_file'],
check_format=False)
cryst = experiments.crystals()[0]
cs = crystal.symmetry(unit_cell=cryst.get_unit_cell(),
space_group=cryst.get_space_group())
cb_op_best_to_ref = cs.change_of_basis_op_to_reference_setting(
)
cs_reference = cs.change_basis(cb_op_best_to_ref)
lattice = str(
bravais_types.bravais_lattice(
group=cs_reference.space_group()))
cb_op = cb_op_best_to_ref * sgtbx.change_of_basis_op(
str(summary['cb_op']))
self._solutions[k] = {
'number': k,
'mosaic': 0.0,
'metric': summary['max_angular_difference'],
'rmsd': summary['rmsd'],
'nspots': summary['nspots'],
'lattice': lattice,
'cell': cs_reference.unit_cell().parameters(),
'experiments_file': summary['experiments_file'],
'cb_op': str(cb_op)
}
self._solution = self.get_solution()
self._indxr_lattice = self._solution['lattice']
for solution in self._solutions.keys():
lattice = self._solutions[solution]['lattice']
if (self._indxr_input_lattice is not None
and self._indxr_input_lattice != lattice):
continue
if lattice in self._indxr_other_lattice_cell:
if self._indxr_other_lattice_cell[lattice]['metric'] < \
self._solutions[solution]['metric']:
continue
self._indxr_other_lattice_cell[lattice] = {
'metric': self._solutions[solution]['metric'],
'cell': self._solutions[solution]['cell']
}
self._indxr_mosaic = self._solution['mosaic']
experiment_list = load.experiment_list(
self._solution['experiments_file'])
self.set_indexer_experiment_list(experiment_list)
# reindex the output experiments list to the reference setting
# (from the best cell/conventional setting)
cb_op_to_ref = experiment_list.crystals()[0].get_space_group().info()\
.change_of_basis_op_to_reference_setting()
reindex = self.Reindex()
reindex.set_experiments_filename(
self._solution['experiments_file'])
reindex.set_cb_op(cb_op_to_ref)
reindex.set_space_group(
str(lattice_to_spacegroup_number(self._solution['lattice'])))
reindex.run()
experiments_file = reindex.get_reindexed_experiments_filename()
experiment_list = load.experiment_list(experiments_file)
self.set_indexer_experiment_list(experiment_list)
self.set_indexer_payload("experiments_filename", experiments_file)
# reindex the output reflection list to this solution
reindex = self.Reindex()
reindex.set_indexed_filename(indexed_file)
reindex.set_cb_op(self._solution['cb_op'])
reindex.set_space_group(
str(lattice_to_spacegroup_number(self._solution['lattice'])))
reindex.run()
indexed_file = reindex.get_reindexed_reflections_filename()
self.set_indexer_payload("indexed_filename", indexed_file)
else:
experiment_list = load.experiment_list(indexed_experiments)
self.set_indexer_experiment_list(experiment_list)
self.set_indexer_payload("experiments_filename",
indexed_experiments)
cryst = experiment_list.crystals()[0]
lattice = str(
bravais_types.bravais_lattice(group=cryst.get_space_group()))
self._indxr_lattice = lattice
self._solutions = {}
self._solutions[0] = {
'number': 0,
'mosaic': 0.0,
'metric': -1,
'rmsd': -1,
'nspots': -1,
'lattice': lattice,
'cell': cryst.get_unit_cell().parameters(),
'experiments_file': indexed_experiments,
'cb_op': 'a,b,c'
}
self._indxr_other_lattice_cell[lattice] = {
'metric': self._solutions[0]['metric'],
'cell': self._solutions[0]['cell']
}
return
def continue_from_error(self):
# copy the LP file
shutil.copyfile(os.path.join(self.get_working_directory(),
'IDXREF.LP'),
os.path.join(self.get_working_directory(),
'%d_IDXREF.LP' % self.get_xpid()))
# parse the output
lp = open(os.path.join(
self.get_working_directory(), 'IDXREF.LP'), 'r').readlines()
self._fraction_rmsd_rmsphi = _parse_idxref_lp_quality(lp)
self._idxref_data = _parse_idxref_lp(lp)
if not self._idxref_data:
raise RuntimeError, 'indexing failed'
st = _parse_idxref_lp_subtree(lp)
if 2 in st:
if st[2] > st[1] / 10.0:
Debug.write('Look closely at autoindexing solution!')
self._index_tree_problem = True
for j in sorted(st):
Debug.write('%2d: %5d' % (j, st[j]))
# print out some (perhaps dire) warnings about the beam centre
# if there is really any ambiguity...
origins = _parse_idxref_index_origin(lp)
assert((0, 0, 0) in origins)
quality_0 = origins[(0, 0, 0)][0]
alternatives = []
for hkl in origins:
if hkl == (0, 0, 0):
continue
if origins[hkl][0] < 4 * quality_0:
quality, delta, beam_x, beam_y = origins[hkl]
alternatives.append((hkl[0], hkl[1], hkl[2],
quality, beam_x, beam_y))
if alternatives:
Debug.write('Alternative indexing possible:')
for alternative in alternatives:
Debug.write('... %3d %3d %3d %4.1f %6.1f %6.1f' % \
alternative)
# New algorithm in here - now use iotbx.lattice_symmetry with the
# P1 indexing solution (solution #1) to determine the list of
# allowable solutions - only consider those lattices in this
# allowed list (unless we have user input)
from xia2.Wrappers.Phenix.LatticeSymmetry import LatticeSymmetry
ls = LatticeSymmetry()
ls.set_lattice('aP')
ls.set_cell(tuple(self._idxref_data[44]['cell']))
ls.generate()
allowed_lattices = ls.get_lattices()
for j in range(1, 45):
if j not in self._idxref_data:
continue
data = self._idxref_data[j]
lattice = data['lattice']
fit = data['fit']
cell = data['cell']
mosaic = data['mosaic']
reidx = data['reidx']
if self._symm and self._cell and \
self._indxr_user_input_lattice:
if self._compare_cell(self._cell, cell) and \
lattice_to_spacegroup_number(lattice) == self._symm:
if lattice in self._indexing_solutions:
if self._indexing_solutions[lattice][
'goodness'] < fit:
continue
self._indexing_solutions[lattice] = {
'goodness':fit,
'cell':cell}
else:
if lattice in allowed_lattices or \
(self._symm and fit < 200.0):
# bug 2417 - if we have an input lattice then we
# don't want to include anything higher symmetry
# in the results table...
if self._symm:
if lattice_to_spacegroup_number(lattice) \
> self._symm:
Debug.write(
'Ignoring solution with lattice %s' % \
lattice)
continue
if lattice in self._indexing_solutions:
if self._indexing_solutions[lattice][
'goodness'] < fit:
continue
self._indexing_solutions[lattice] = {
'goodness':fit,
'cell':cell}
# postprocess this list, to remove lattice solutions which are
# lower symmetry but higher penalty than the putative correct
# one, if self._symm is set...
if self._symm:
assert len(self._indexing_solutions) > 0, "No remaining indexing solutions (%s, %s)" % (s2l(self._symm), self._symm);
else:
assert len(self._indexing_solutions) > 0, "No remaining indexing solutions"
# print self._indexing_solutions
if self._symm:
max_p = 2.0 * self._indexing_solutions[
s2l(self._symm)]['goodness']
to_remove = []
for lattice in self._indexing_solutions:
if self._indexing_solutions[lattice]['goodness'] > max_p:
to_remove.append(lattice)
for lattice in to_remove:
Debug.write('Ignoring solution with lattice %s' % \
lattice)
del(self._indexing_solutions[lattice])
# get the highest symmetry "acceptable" solution
list = [(k, self._indexing_solutions[k]['cell']) for k in \
self._indexing_solutions.keys()]
# if there was a preassigned cell and symmetry return now
# with everything done, else select the "top" solution and
# reindex, resetting the input cell and symmetry.
if self._cell:
# select the solution which matches the input unit cell
# actually after the changes above this should now be the
# only solution in the table..
Debug.write(
'Target unit cell: %.2f %.2f %.2f %.2f %.2f %.2f' % \
self._cell)
for l in list:
if lattice_to_spacegroup_number(l[0]) == self._symm:
# this should be the correct solution...
# check the unit cell...
cell = l[1]
if self._compare_cell(self._cell, cell) or True:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % cell
Debug.write(
'Chosen unit cell: %s' % cell_str)
self._indxr_lattice = l[0]
self._indxr_cell = l[1]
self._indxr_mosaic = mosaic
else:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % cell
Debug.write(
'Ignoring unit cell: %s' % cell_str)
else:
# select the top solution as the input cell and reset the
# "indexing done" flag
sorted_list = SortLattices(list)
# print sorted_list
self._symm = lattice_to_spacegroup_number(sorted_list[0][0])
self._cell = sorted_list[0][1]
return False
# get the refined distance &c.
beam, distance = _parse_idxref_lp_distance_etc(lp)
self._refined_beam = beam
self._refined_distance = distance
# gather the output files
for file in self._output_data_files_list:
self._output_data_files[file] = os.path.join(
self.get_working_directory(), file)
return True
def _index(self):
'''Actually index the diffraction pattern. Note well that
this is not going to compute the matrix...'''
# acknowledge this program
Citations.cite('labelit')
Citations.cite('distl')
#self.reset()
_images = []
for i in self._indxr_images:
for j in i:
if not j in _images:
_images.append(j)
_images.sort()
images_str = '%d' % _images[0]
for i in _images[1:]:
images_str += ', %d' % i
cell_str = None
if self._indxr_input_cell:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
self._indxr_input_cell
if self._indxr_sweep_name:
# then this is a proper autoindexing run - describe this
# to the journal entry
#if len(self._fp_directory) <= 50:
#dirname = self._fp_directory
#else:
#dirname = '...%s' % self._fp_directory[-46:]
dirname = os.path.dirname(self.get_imageset().get_template())
Journal.block(
'autoindexing', self._indxr_sweep_name, 'labelit', {
'images': images_str,
'target cell': cell_str,
'target lattice': self._indxr_input_lattice,
'template': self.get_imageset().get_template(),
'directory': dirname
})
if len(_images) > 4:
raise RuntimeError('cannot use more than 4 images')
from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex
index = LabelitIndex()
index.set_working_directory(self.get_working_directory())
auto_logfiler(index)
#task = 'Autoindex from images:'
#for i in _images:
#task += ' %s' % self.get_image_name(i)
#self.set_task(task)
Debug.write('Indexing from images:')
for i in _images:
index.add_image(self.get_image_name(i))
Debug.write('%s' % self.get_image_name(i))
xsweep = self.get_indexer_sweep()
if xsweep is not None:
if xsweep.get_distance() is not None:
index.set_distance(xsweep.get_distance())
#if self.get_wavelength_prov() == 'user':
#index.set_wavelength(self.get_wavelength())
if xsweep.get_beam_centre() is not None:
index.set_beam_centre(xsweep.get_beam_centre())
if self._refine_beam is False:
index.set_refine_beam(False)
else:
index.set_refine_beam(True)
index.set_beam_search_scope(self._beam_search_scope)
if ((math.fabs(self.get_wavelength() - 1.54) < 0.01)
or (math.fabs(self.get_wavelength() - 2.29) < 0.01)):
index.set_Cu_KA_or_Cr_KA(True)
#sweep = self.get_indexer_sweep_name()
#FileHandler.record_log_file(
#'%s INDEX' % (sweep), self.get_log_file())
try:
index.run()
except RuntimeError as e:
if self._refine_beam is False:
raise e
# can we improve the situation?
if self._beam_search_scope < 4.0:
self._beam_search_scope += 4.0
# try repeating the indexing!
self.set_indexer_done(False)
return 'failed'
# otherwise this is beyond redemption
raise e
self._solutions = index.get_solutions()
# FIXME this needs to check the smilie status e.g.
# ":)" or ";(" or " ".
# FIXME need to check the value of the RMSD and raise an
# exception if the P1 solution has an RMSD > 1.0...
# Change 27/FEB/08 to support user assigned spacegroups
# (euugh!) have to "ignore" solutions with higher symmetry
# otherwise the rest of xia will override us. Bummer.
for i, solution in self._solutions.iteritems():
if self._indxr_user_input_lattice:
if (lattice_to_spacegroup(solution['lattice']) >
lattice_to_spacegroup(self._indxr_input_lattice)):
Debug.write('Ignoring solution: %s' % solution['lattice'])
del self._solutions[i]
# check the RMSD from the triclinic unit cell
if self._solutions[1]['rmsd'] > 1.0 and False:
# don't know when this is useful - but I know when it is not!
raise RuntimeError('high RMSD for triclinic solution')
# configure the "right" solution
self._solution = self.get_solution()
# now store also all of the other solutions... keyed by the
# lattice - however these should only be added if they
# have a smiley in the appropriate record, perhaps?
for solution in self._solutions.keys():
lattice = self._solutions[solution]['lattice']
if lattice in self._indxr_other_lattice_cell:
if self._indxr_other_lattice_cell[lattice]['goodness'] < \
self._solutions[solution]['metric']:
continue
self._indxr_other_lattice_cell[lattice] = {
'goodness': self._solutions[solution]['metric'],
'cell': self._solutions[solution]['cell']
}
self._indxr_lattice = self._solution['lattice']
self._indxr_cell = tuple(self._solution['cell'])
self._indxr_mosaic = self._solution['mosaic']
lms = LabelitMosflmScript()
lms.set_working_directory(self.get_working_directory())
lms.set_solution(self._solution['number'])
self._indxr_payload['mosflm_orientation_matrix'] = lms.calculate()
# get the beam centre from the mosflm script - mosflm
# may have inverted the beam centre and labelit will know
# this!
mosflm_beam_centre = lms.get_mosflm_beam()
if mosflm_beam_centre:
self._indxr_payload['mosflm_beam_centre'] = tuple(
mosflm_beam_centre)
import copy
detector = copy.deepcopy(self.get_detector())
beam = copy.deepcopy(self.get_beam())
from dxtbx.model.detector_helpers import set_mosflm_beam_centre
set_mosflm_beam_centre(detector, beam, mosflm_beam_centre)
from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
from scitbx import matrix
from cctbx import sgtbx, uctbx
from dxtbx.model import CrystalFactory
mosflm_matrix = matrix.sqr([
float(i) for line in lms.calculate()
for i in line.replace("-", " -").split()
][:9])
space_group = sgtbx.space_group_info(
lattice_to_spacegroup_number(self._solution['lattice'])).group()
crystal_model = CrystalFactory.from_mosflm_matrix(
mosflm_matrix,
unit_cell=uctbx.unit_cell(tuple(self._solution['cell'])),
space_group=space_group)
from dxtbx.model import Experiment, ExperimentList
experiment = Experiment(
beam=beam,
detector=detector,
goniometer=self.get_goniometer(),
scan=self.get_scan(),
crystal=crystal_model,
)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
# also get an estimate of the resolution limit from the
# labelit.stats_distl output... FIXME the name is wrong!
lsd = LabelitStats_distl()
lsd.set_working_directory(self.get_working_directory())
lsd.stats_distl()
resolution = 1.0e6
for i in _images:
stats = lsd.get_statistics(self.get_image_name(i))
resol = 0.5 * (stats['resol_one'] + stats['resol_two'])
if resol < resolution:
resolution = resol
self._indxr_resolution_estimate = resolution
return 'ok'
def _index(self):
"""Actually index the diffraction pattern. Note well that
this is not going to compute the matrix..."""
# acknowledge this program
if not self._indxr_images:
raise RuntimeError("No good spots found on any images")
Citations.cite("labelit")
Citations.cite("distl")
_images = []
for i in self._indxr_images:
for j in i:
if not j in _images:
_images.append(j)
_images.sort()
images_str = "%d" % _images[0]
for i in _images[1:]:
images_str += ", %d" % i
cell_str = None
if self._indxr_input_cell:
cell_str = "%.2f %.2f %.2f %.2f %.2f %.2f" % self._indxr_input_cell
if self._indxr_sweep_name:
# then this is a proper autoindexing run - describe this
# to the journal entry
if len(self._fp_directory) <= 50:
dirname = self._fp_directory
else:
dirname = "...%s" % self._fp_directory[-46:]
Journal.block(
"autoindexing",
self._indxr_sweep_name,
"labelit",
{
"images": images_str,
"target cell": cell_str,
"target lattice": self._indxr_input_lattice,
"template": self._fp_template,
"directory": dirname,
},
)
# auto_logfiler(self)
from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex
index = LabelitIndex()
index.set_working_directory(self.get_working_directory())
auto_logfiler(index)
# task = 'Autoindex from images:'
# for i in _images:
# task += ' %s' % self.get_image_name(i)
# self.set_task(task)
# self.add_command_line('--index_only')
Debug.write("Indexing from images:")
for i in _images:
index.add_image(self.get_image_name(i))
Debug.write("%s" % self.get_image_name(i))
if self._primitive_unit_cell:
index.set_primitive_unit_cell(self._primitive_unit_cell)
if self._indxr_input_cell:
index.set_max_cell(1.25 * max(self._indxr_input_cell[:3]))
xsweep = self.get_indexer_sweep()
if xsweep is not None:
if xsweep.get_distance() is not None:
index.set_distance(xsweep.get_distance())
# if self.get_wavelength_prov() == 'user':
# index.set_wavelength(self.get_wavelength())
if xsweep.get_beam_centre() is not None:
index.set_beam_centre(xsweep.get_beam_centre())
if self._refine_beam is False:
index.set_refine_beam(False)
else:
index.set_refine_beam(True)
index.set_beam_search_scope(self._beam_search_scope)
if (math.fabs(self.get_wavelength() - 1.54) <
0.01) or (math.fabs(self.get_wavelength() - 2.29) < 0.01):
index.set_Cu_KA_or_Cr_KA(True)
try:
index.run()
except RuntimeError as e:
if self._refine_beam is False:
raise e
# can we improve the situation?
if self._beam_search_scope < 4.0:
self._beam_search_scope += 4.0
# try repeating the indexing!
self.set_indexer_done(False)
return "failed"
# otherwise this is beyond redemption
raise e
self._solutions = index.get_solutions()
# FIXME this needs to check the smilie status e.g.
# ":)" or ";(" or " ".
# FIXME need to check the value of the RMSD and raise an
# exception if the P1 solution has an RMSD > 1.0...
# Change 27/FEB/08 to support user assigned spacegroups
# (euugh!) have to "ignore" solutions with higher symmetry
# otherwise the rest of xia will override us. Bummer.
for i, solution in self._solutions.iteritems():
if self._indxr_user_input_lattice:
if lattice_to_spacegroup(
solution["lattice"]) > lattice_to_spacegroup(
self._indxr_input_lattice):
Debug.write("Ignoring solution: %s" % solution["lattice"])
del self._solutions[i]
# configure the "right" solution
self._solution = self.get_solution()
# now store also all of the other solutions... keyed by the
# lattice - however these should only be added if they
# have a smiley in the appropriate record, perhaps?
for solution in self._solutions.keys():
lattice = self._solutions[solution]["lattice"]
if lattice in self._indxr_other_lattice_cell:
if (self._indxr_other_lattice_cell[lattice]["goodness"] <
self._solutions[solution]["metric"]):
continue
self._indxr_other_lattice_cell[lattice] = {
"goodness": self._solutions[solution]["metric"],
"cell": self._solutions[solution]["cell"],
}
self._indxr_lattice = self._solution["lattice"]
self._indxr_cell = tuple(self._solution["cell"])
self._indxr_mosaic = self._solution["mosaic"]
lms = LabelitMosflmMatrix()
lms.set_working_directory(self.get_working_directory())
lms.set_solution(self._solution["number"])
self._indxr_payload["mosflm_orientation_matrix"] = lms.calculate()
# get the beam centre from the mosflm script - mosflm
# may have inverted the beam centre and labelit will know
# this!
mosflm_beam_centre = lms.get_mosflm_beam()
if mosflm_beam_centre:
self._indxr_payload["mosflm_beam_centre"] = tuple(
mosflm_beam_centre)
detector = copy.deepcopy(self.get_detector())
beam = copy.deepcopy(self.get_beam())
from dxtbx.model.detector_helpers import set_mosflm_beam_centre
set_mosflm_beam_centre(detector, beam, mosflm_beam_centre)
from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
from scitbx import matrix
from cctbx import sgtbx, uctbx
from dxtbx.model import CrystalFactory
mosflm_matrix = matrix.sqr([
float(i) for line in lms.calculate()
for i in line.replace("-", " -").split()
][:9])
space_group = sgtbx.space_group_info(
lattice_to_spacegroup_number(self._solution["lattice"])).group()
crystal_model = CrystalFactory.from_mosflm_matrix(
mosflm_matrix,
unit_cell=uctbx.unit_cell(tuple(self._solution["cell"])),
space_group=space_group,
)
from dxtbx.model import Experiment, ExperimentList
experiment = Experiment(
beam=beam,
detector=detector,
goniometer=self.get_goniometer(),
scan=self.get_scan(),
crystal=crystal_model,
)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
# also get an estimate of the resolution limit from the
# labelit.stats_distl output... FIXME the name is wrong!
lsd = LabelitStats_distl()
lsd.set_working_directory(self.get_working_directory())
lsd.stats_distl()
resolution = 1.0e6
for i in _images:
stats = lsd.get_statistics(self.get_image_name(i))
resol = 0.5 * (stats["resol_one"] + stats["resol_two"])
if resol < resolution:
resolution = resol
self._indxr_resolution_estimate = resolution
return "ok"
def run(self, ignore_errors = False):
'''Run idxref.'''
#image_header = self.get_header()
## crank through the header dictionary and replace incorrect
## information with updated values through the indexer
## interface if available...
## need to add distance, wavelength - that should be enough...
#if self.get_distance():
#image_header['distance'] = self.get_distance()
#if self.get_wavelength():
#image_header['wavelength'] = self.get_wavelength()
#if self.get_two_theta():
#image_header['two_theta'] = self.get_two_theta()
header = imageset_to_xds(
self.get_imageset(),
refined_beam_vector=self._refined_beam_vector,
refined_rotation_axis=self._refined_rotation_axis,
refined_distance=self._refined_distance)
xds_inp = open(os.path.join(self.get_working_directory(),
'XDS.INP'), 'w')
# what are we doing?
xds_inp.write('JOB=IDXREF\n')
xds_inp.write('MAXIMUM_NUMBER_OF_PROCESSORS=%d\n' % \
self._parallel)
# FIXME this needs to be calculated from the beam centre...
if self._refined_origin:
xds_inp.write('ORGX=%f ORGY=%f\n' % \
tuple(self._refined_origin))
else:
xds_inp.write('ORGX=%f ORGY=%f\n' % \
tuple(self._org))
# FIXME in here make sure sweep is wider than 5 degrees
# before specifying AXIS: if <= 5 degrees replace AXIS with
# nothing - base this on the maximum possible angular separation
min_frame = self._spot_range[0][0]
max_frame = self._spot_range[-1][1]
refine_params = [p for p in self._params.refine]
phi_width = self.get_phi_width()
if ('AXIS' in refine_params
and (max_frame - min_frame) * phi_width < 5.0):
refine_params.remove('AXIS')
xds_inp.write('REFINE(IDXREF)=%s\n' %
' '.join(refine_params))
if self._starting_frame and self._starting_angle:
xds_inp.write('STARTING_FRAME=%d\n' % \
self._starting_frame)
xds_inp.write('STARTING_ANGLE=%f\n' % \
self._starting_angle)
# FIXME this looks like a potential bug - what will
# happen if the input lattice has not been set??
if self._indxr_input_cell:
self._cell = self._indxr_input_cell
if self._indxr_input_lattice:
self._symm = lattice_to_spacegroup_number(
self._indxr_input_lattice)
if self._cell:
xds_inp.write('SPACE_GROUP_NUMBER=%d\n' % self._symm)
cell_format = '%6.2f %6.2f %6.2f %6.2f %6.2f %6.2f'
xds_inp.write('UNIT_CELL_CONSTANTS=%s\n' % \
cell_format % self._cell)
if self._a_axis:
xds_inp.write('UNIT_CELL_A-AXIS=%.2f %.2f %.2f\n' %
tuple(self._a_axis))
if self._b_axis:
xds_inp.write('UNIT_CELL_B-AXIS=%.2f %.2f %.2f\n' %
tuple(self._b_axis))
if self._c_axis:
xds_inp.write('UNIT_CELL_C-AXIS=%.2f %.2f %.2f\n' %
tuple(self._c_axis))
for record in header:
xds_inp.write('%s\n' % record)
name_template = template_to_xds(
os.path.join(self.get_directory(), self.get_template()))
record = 'NAME_TEMPLATE_OF_DATA_FRAMES=%s\n' % \
name_template
xds_inp.write(record)
xds_inp.write('DATA_RANGE=%d %d\n' % self._data_range)
for spot_range in self._spot_range:
xds_inp.write('SPOT_RANGE=%d %d\n' % spot_range)
xds_inp.write('BACKGROUND_RANGE=%d %d\n' % \
self._background_range)
xds_inp.close()
# copy the input file...
shutil.copyfile(os.path.join(self.get_working_directory(),
'XDS.INP'),
os.path.join(self.get_working_directory(),
'%d_IDXREF.INP' % self.get_xpid()))
# write the input data files...
for file_name in self._input_data_files_list:
src = self._input_data_files[file_name]
dst = os.path.join(
self.get_working_directory(), file_name)
if src != dst:
shutil.copyfile(src, dst)
self.start()
self.close_wait()
xds_check_version_supported(self.get_all_output())
if not ignore_errors:
xds_check_error(self.get_all_output())
# If xds_check_error detects any errors it will raise an exception
# The caller can then continue using the run_continue_from_error()
# function. If XDS does not throw any errors we just plow on.
return self.continue_from_error()
def _index(self):
if PhilIndex.params.dials.index.method in (libtbx.Auto, None):
if self._indxr_input_cell is not None:
indexer = self._do_indexing("real_space_grid_search")
else:
try:
indexer_fft3d = self._do_indexing(method="fft3d")
nref_3d, rmsd_3d = indexer_fft3d.get_nref_rmsds()
except Exception as e:
nref_3d = None
rmsd_3d = None
indexing_failure = e
try:
indexer_fft1d = self._do_indexing(method="fft1d")
nref_1d, rmsd_1d = indexer_fft1d.get_nref_rmsds()
except Exception as e:
nref_1d = None
rmsd_1d = None
indexing_failure = e
if (nref_1d is not None and nref_3d is None or
(nref_1d > nref_3d and rmsd_1d[0] < rmsd_3d[0]
and rmsd_1d[1] < rmsd_3d[1] and rmsd_1d[2] < rmsd_3d[2])):
indexer = indexer_fft1d
elif nref_3d is not None:
indexer = indexer_fft3d
else:
raise RuntimeError(indexing_failure)
else:
indexer = self._do_indexing(
method=PhilIndex.params.dials.index.method)
# not strictly the P1 cell, rather the cell that was used in indexing
self._p1_cell = indexer._p1_cell
self.set_indexer_payload("indexed_filename",
indexer.get_indexed_filename())
indexed_file = indexer.get_indexed_filename()
indexed_experiments = indexer.get_experiments_filename()
fast_mode = PhilIndex.params.dials.fast_mode
trust_beam_centre = PhilIndex.params.xia2.settings.trust_beam_centre
multi_sweep_indexing = PhilIndex.params.xia2.settings.multi_sweep_indexing
check_indexing_symmetry = PhilIndex.params.dials.check_indexing_symmetry
if check_indexing_symmetry and not (trust_beam_centre or fast_mode
or multi_sweep_indexing):
checksym = self.CheckIndexingSymmetry()
checksym.set_experiments_filename(indexed_experiments)
checksym.set_indexed_filename(indexed_file)
checksym.set_grid_search_scope(1)
checksym.run()
hkl_offset = checksym.get_hkl_offset()
logger.debug("hkl_offset: %s", str(hkl_offset))
if hkl_offset is not None and hkl_offset != (0, 0, 0):
reindex = self.Reindex()
reindex.set_hkl_offset(hkl_offset)
reindex.set_indexed_filename(indexed_file)
reindex.run()
indexed_file = reindex.get_reindexed_reflections_filename()
# do some scan-static refinement - run twice, first without outlier
# rejection as the model is too far from reality to do a sensible job of
# outlier rejection
refiner = self.Refine()
refiner.set_experiments_filename(indexed_experiments)
refiner.set_indexed_filename(
reindex.get_reindexed_reflections_filename())
refiner.set_outlier_algorithm(None)
refiner.run()
indexed_experiments = refiner.get_refined_experiments_filename(
)
# now again with outlier rejection (possibly)
refiner = self.Refine()
refiner.set_experiments_filename(indexed_experiments)
refiner.set_indexed_filename(indexed_file)
refiner.run()
indexed_experiments = refiner.get_refined_experiments_filename(
)
if self._indxr_input_lattice is None:
# FIXME in here should respect the input unit cell and lattice if provided
# FIXME from this (i) populate the helper table,
# (ii) try to avoid re-running the indexing
# step if we eliminate a solution as we have all of the refined results
# already available.
rbs = self.RefineBravaisSettings()
rbs.set_experiments_filename(indexed_experiments)
rbs.set_indexed_filename(indexed_file)
if PhilIndex.params.dials.fix_geometry:
rbs.set_detector_fix("all")
rbs.set_beam_fix("all")
elif PhilIndex.params.dials.fix_distance:
rbs.set_detector_fix("distance")
FileHandler.record_log_file(
"%s LATTICE" % self.get_indexer_full_name(),
rbs.get_log_file())
rbs.run()
for k in sorted(rbs.get_bravais_summary()):
summary = rbs.get_bravais_summary()[k]
# FIXME need to do this better - for the moment only accept lattices
# where R.M.S. deviation is less than twice P1 R.M.S. deviation.
if self._indxr_input_lattice is None:
if not summary["recommended"]:
continue
experiments = load.experiment_list(summary["experiments_file"],
check_format=False)
cryst = experiments.crystals()[0]
cs = crystal.symmetry(unit_cell=cryst.get_unit_cell(),
space_group=cryst.get_space_group())
lattice = str(
bravais_types.bravais_lattice(group=cs.space_group()))
cb_op = sgtbx.change_of_basis_op(str(summary["cb_op"]))
self._solutions[k] = {
"number": k,
"mosaic": 0.0,
"metric": summary["max_angular_difference"],
"rmsd": summary["rmsd"],
"nspots": summary["nspots"],
"lattice": lattice,
"cell": cs.unit_cell().parameters(),
"experiments_file": summary["experiments_file"],
"cb_op": str(cb_op),
}
self._solution = self.get_solution()
self._indxr_lattice = self._solution["lattice"]
for solution in self._solutions:
lattice = self._solutions[solution]["lattice"]
if (self._indxr_input_lattice is not None
and self._indxr_input_lattice != lattice):
continue
if lattice in self._indxr_other_lattice_cell:
if (self._indxr_other_lattice_cell[lattice]["metric"] <
self._solutions[solution]["metric"]):
continue
self._indxr_other_lattice_cell[lattice] = {
"metric": self._solutions[solution]["metric"],
"cell": self._solutions[solution]["cell"],
}
self._indxr_mosaic = self._solution["mosaic"]
experiments_file = self._solution["experiments_file"]
experiment_list = load.experiment_list(experiments_file)
self.set_indexer_experiment_list(experiment_list)
self.set_indexer_payload("experiments_filename", experiments_file)
# reindex the output reflection list to this solution
reindex = self.Reindex()
reindex.set_indexed_filename(indexed_file)
reindex.set_cb_op(self._solution["cb_op"])
reindex.set_space_group(
str(lattice_to_spacegroup_number(self._solution["lattice"])))
reindex.run()
indexed_file = reindex.get_reindexed_reflections_filename()
self.set_indexer_payload("indexed_filename", indexed_file)
else:
experiment_list = load.experiment_list(indexed_experiments)
self.set_indexer_experiment_list(experiment_list)
self.set_indexer_payload("experiments_filename",
indexed_experiments)
cryst = experiment_list.crystals()[0]
lattice = str(
bravais_types.bravais_lattice(group=cryst.get_space_group()))
self._indxr_lattice = lattice
self._solutions = {}
self._solutions[0] = {
"number": 0,
"mosaic": 0.0,
"metric": -1,
"rmsd": -1,
"nspots": -1,
"lattice": lattice,
"cell": cryst.get_unit_cell().parameters(),
"experiments_file": indexed_experiments,
"cb_op": "a,b,c",
}
self._indxr_other_lattice_cell[lattice] = {
"metric": self._solutions[0]["metric"],
"cell": self._solutions[0]["cell"],
}
def _integrate_finish(self):
"""Finish off the integration by running correct."""
# first run the postrefinement etc with spacegroup P1
# and the current unit cell - this will be used to
# obtain a benchmark rmsd in pixels / phi and also
# cell deviations (this is working towards spotting bad
# indexing solutions) - only do this if we have no
# reindex matrix... and no postrefined cell...
p1_deviations = None
# fix for bug # 3264 -
# if we have not run integration with refined parameters, make it so...
# erm? shouldn't this therefore return if this is the principle, or
# set the flag after we have tested the lattice?
if ("GXPARM.XDS" not in self._xds_data_files
and PhilIndex.params.xds.integrate.reintegrate):
logger.debug(
"Resetting integrater, to ensure refined orientation is used")
self.set_integrater_done(False)
if (not self.get_integrater_reindex_matrix() and not self._intgr_cell
and PhilIndex.params.xia2.settings.lattice_rejection
and not self.get_integrater_sweep().get_user_lattice()):
correct = self.Correct()
correct.set_data_range(
self._intgr_wedge[0] + self.get_frame_offset(),
self._intgr_wedge[1] + self.get_frame_offset(),
)
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# FIXME should this be using the correctly transformed
# cell or are the results ok without it?!
correct.set_spacegroup_number(1)
correct.set_cell(self._intgr_refiner_cell)
correct.run()
cell = correct.get_result("cell")
cell_esd = correct.get_result("cell_esd")
logger.debug("Postrefinement in P1 results:")
logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" % tuple(cell))
logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
tuple(cell_esd))
logger.debug("Deviations: %.2f pixels %.2f degrees" %
(correct.get_result("rmsd_pixel"),
correct.get_result("rmsd_phi")))
p1_deviations = (
correct.get_result("rmsd_pixel"),
correct.get_result("rmsd_phi"),
)
# next run the postrefinement etc with the given
# cell / lattice - this will be the assumed result...
integrate_hkl = os.path.join(self.get_working_directory(),
"INTEGRATE.HKL")
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
from dxtbx.serialize import load
from dials.algorithms.shadowing.filter import filter_shadowed_reflections
experiments_json = xparm_xds_to_experiments_json(
os.path.join(self.get_working_directory(), "XPARM.XDS"),
self.get_working_directory(),
)
experiments = load.experiment_list(experiments_json,
check_format=True)
imageset = experiments[0].imageset
masker = (imageset.get_format_class().get_instance(
imageset.paths()[0]).get_masker())
if masker is not None:
integrate_filename = integrate_hkl_to_reflection_file(
integrate_hkl, experiments_json,
self.get_working_directory())
reflections = flex.reflection_table.from_file(
integrate_filename)
t0 = time.time()
sel = filter_shadowed_reflections(experiments, reflections)
shadowed = reflections.select(sel)
t1 = time.time()
logger.debug("Filtered %i reflections in %.1f seconds" %
(sel.count(True), t1 - t0))
filter_hkl = os.path.join(self.get_working_directory(),
"FILTER.HKL")
with open(filter_hkl, "wb") as f:
detector = experiments[0].detector
for ref in shadowed:
p = detector[ref["panel"]]
ox, oy = p.get_raw_image_offset()
h, k, l = ref["miller_index"]
x, y, z = ref["xyzcal.px"]
dx, dy, dz = (2, 2, 2)
print(
"%i %i %i %.1f %.1f %.1f %.1f %.1f %.1f" %
(h, k, l, x + ox, y + oy, z, dx, dy, dz),
file=f,
)
t2 = time.time()
logger.debug("Written FILTER.HKL in %.1f seconds" % (t2 - t1))
correct = self.Correct()
correct.set_data_range(
self._intgr_wedge[0] + self.get_frame_offset(),
self._intgr_wedge[1] + self.get_frame_offset(),
)
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# BUG # 2695 probably comes from here - need to check...
# if the pointless interface comes back with a different
# crystal setting then the unit cell stored in self._intgr_cell
# needs to be set to None...
if self.get_integrater_spacegroup_number():
correct.set_spacegroup_number(
self.get_integrater_spacegroup_number())
if not self._intgr_cell:
raise RuntimeError("no unit cell to recycle")
correct.set_cell(self._intgr_cell)
# BUG # 3113 - new version of XDS will try and figure the
# best spacegroup out from the intensities (and get it wrong!)
# unless we set the spacegroup and cell explicitly
if not self.get_integrater_spacegroup_number():
cell = self._intgr_refiner_cell
lattice = self._intgr_refiner.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup_number(lattice)
# this should not prevent the postrefinement from
# working correctly, else what is above would not
# work correctly (the postrefinement test)
correct.set_spacegroup_number(spacegroup_number)
correct.set_cell(cell)
logger.debug("Setting spacegroup to: %d" % spacegroup_number)
logger.debug("Setting cell to: %.2f %.2f %.2f %.2f %.2f %.2f" %
tuple(cell))
if self.get_integrater_reindex_matrix():
# bug! if the lattice is not primitive the values in this
# reindex matrix need to be multiplied by a constant which
# depends on the Bravais lattice centering.
lattice = self._intgr_refiner.get_refiner_lattice()
matrix = self.get_integrater_reindex_matrix()
matrix = scitbx.matrix.sqr(matrix).transpose().elems
matrix = r_to_rt(matrix)
if lattice[1] == "P":
mult = 1
elif lattice[1] == "C" or lattice[1] == "I":
mult = 2
elif lattice[1] == "R":
mult = 3
elif lattice[1] == "F":
mult = 4
else:
raise RuntimeError("unknown multiplier for lattice %s" %
lattice)
logger.debug("REIDX multiplier for lattice %s: %d" %
(lattice, mult))
mult_matrix = [mult * m for m in matrix]
logger.debug("REIDX set to %d %d %d %d %d %d %d %d %d %d %d %d" %
tuple(mult_matrix))
correct.set_reindex_matrix(mult_matrix)
correct.run()
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file(
f"{pname} {xname} {dname} {sweep} CORRECT",
os.path.join(self.get_working_directory(), "CORRECT.LP"),
)
FileHandler.record_more_data_file(
f"{pname} {xname} {dname} {sweep} CORRECT",
os.path.join(self.get_working_directory(), "XDS_ASCII.HKL"),
)
# erm. just to be sure
if self.get_integrater_reindex_matrix() and correct.get_reindex_used():
raise RuntimeError("Reindex panic!")
# get the reindex operation used, which may be useful if none was
# set but XDS decided to apply one, e.g. #419.
if not self.get_integrater_reindex_matrix(
) and correct.get_reindex_used():
# convert this reindex operation to h, k, l form: n.b. this
# will involve dividing through by the lattice centring multiplier
matrix = rt_to_r(correct.get_reindex_used())
matrix = scitbx.matrix.sqr(matrix).transpose().elems
lattice = self._intgr_refiner.get_refiner_lattice()
if lattice[1] == "P":
mult = 1.0
elif lattice[1] == "C" or lattice[1] == "I":
mult = 2.0
elif lattice[1] == "R":
mult = 3.0
elif lattice[1] == "F":
mult = 4.0
matrix = [m / mult for m in matrix]
reindex_op = mat_to_symop(matrix)
# assign this to self: will this reset?! make for a leaky
# abstraction and just assign this...
# self.set_integrater_reindex_operator(reindex)
self._intgr_reindex_operator = reindex_op
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file(
f"{pname} {xname} {dname} {sweep} CORRECT",
os.path.join(self.get_working_directory(), "CORRECT.LP"),
)
# should get some interesting stuff from the XDS correct file
# here, for instance the resolution range to use in integration
# (which should be fed back if not fast) and so on...
self._intgr_corrected_hklout = os.path.join(
self.get_working_directory(), "XDS_ASCII.HKL")
# also record the batch range - needed for the analysis of the
# radiation damage in chef...
self._intgr_batches_out = (self._intgr_wedge[0], self._intgr_wedge[1])
# FIXME perhaps I should also feedback the GXPARM file here??
for file in ["GXPARM.XDS"]:
self._xds_data_files[file] = correct.get_output_data_file(file)
# record the postrefined cell parameters
self._intgr_cell = correct.get_result("cell")
self._intgr_n_ref = correct.get_result("n_ref")
logger.debug('Postrefinement in "correct" spacegroup results:')
logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
tuple(correct.get_result("cell")))
logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
tuple(correct.get_result("cell_esd")))
logger.debug(
"Deviations: %.2f pixels %.2f degrees" %
(correct.get_result("rmsd_pixel"), correct.get_result("rmsd_phi")))
logger.debug("Error correction parameters: A=%.3f B=%.3f" %
correct.get_result("sdcorrection"))
# compute misorientation of axes
xparm_file = os.path.join(self.get_working_directory(), "GXPARM.XDS")
handle = xparm.reader()
handle.read_file(xparm_file)
rotn = handle.rotation_axis
beam = handle.beam_vector
dot = sum(rotn[j] * beam[j] for j in range(3))
r = math.sqrt(sum(rotn[j] * rotn[j] for j in range(3)))
b = math.sqrt(sum(beam[j] * beam[j] for j in range(3)))
rtod = 180.0 / math.pi
angle = rtod * math.fabs(0.5 * math.pi - math.acos(dot / (r * b)))
logger.debug("Axis misalignment %.2f degrees" % angle)
correct_deviations = (
correct.get_result("rmsd_pixel"),
correct.get_result("rmsd_phi"),
)
if p1_deviations:
# compare and reject if both > 50% higher - though adding a little
# flexibility - 0.5 pixel / osc width slack.
pixel = p1_deviations[0]
phi = math.sqrt(0.05 * 0.05 + p1_deviations[1] * p1_deviations[1])
threshold = PhilIndex.params.xia2.settings.lattice_rejection_threshold
logger.debug("RMSD ratio: %.2f" % (correct_deviations[0] / pixel))
logger.debug("RMSPhi ratio: %.2f" % (correct_deviations[1] / phi))
if (correct_deviations[0] / pixel > threshold
and correct_deviations[1] / phi > threshold):
logger.info(
"Eliminating this indexing solution as postrefinement")
logger.info("deviations rather high relative to triclinic")
raise BadLatticeError(
"high relative deviations in postrefinement")
if (not PhilIndex.params.dials.fast_mode
and not PhilIndex.params.xds.keep_outliers):
# check for alien reflections and perhaps recycle - removing them
correct_remove = correct.get_remove()
if correct_remove:
current_remove = set()
final_remove = []
# first ensure that there are no duplicate entries...
if os.path.exists(
os.path.join(self.get_working_directory(),
"REMOVE.HKL")):
with open(
os.path.join(self.get_working_directory(),
"REMOVE.HKL")) as fh:
for line in fh.readlines():
h, k, l = list(map(int, line.split()[:3]))
z = float(line.split()[3])
if (h, k, l, z) not in current_remove:
current_remove.add((h, k, l, z))
for c in correct_remove:
if c in current_remove:
continue
final_remove.append(c)
logger.debug("%d alien reflections are already removed" %
(len(correct_remove) - len(final_remove)))
else:
# we want to remove all of the new dodgy reflections
final_remove = correct_remove
z_min = PhilIndex.params.xds.z_min
rejected = 0
with open(
os.path.join(self.get_working_directory(),
"REMOVE.HKL"), "w") as remove_hkl:
# write in the old reflections
for remove in current_remove:
z = remove[3]
if z >= z_min:
remove_hkl.write("%d %d %d %f\n" % remove)
else:
rejected += 1
logger.debug("Wrote %d old reflections to REMOVE.HKL" %
(len(current_remove) - rejected))
logger.debug("Rejected %d as z < %f" % (rejected, z_min))
# and the new reflections
rejected = 0
used = 0
for remove in final_remove:
z = remove[3]
if z >= z_min:
used += 1
remove_hkl.write("%d %d %d %f\n" % remove)
else:
rejected += 1
logger.debug("Wrote %d new reflections to REMOVE.HKL" %
(len(final_remove) - rejected))
logger.debug("Rejected %d as z < %f" % (rejected, z_min))
# we want to rerun the finishing step so...
# unless we have added no new reflections... or unless we
# have not confirmed the point group (see SCI-398)
if used and self.get_integrater_reindex_matrix():
self.set_integrater_finish_done(False)
else:
logger.debug(
"Going quickly so not removing %d outlier reflections..." %
len(correct.get_remove()))
# Convert INTEGRATE.HKL to MTZ format and reapply any reindexing operations
# spacegroup changes to allow use with CCP4 / Aimless for scaling
hklout = os.path.splitext(integrate_hkl)[0] + ".mtz"
self._factory.set_working_directory(self.get_working_directory())
pointless = self._factory.Pointless()
pointless.set_xdsin(integrate_hkl)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
integrate_mtz = hklout
if (self.get_integrater_reindex_operator()
or self.get_integrater_spacegroup_number()):
logger.debug("Reindexing things to MTZ")
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
if self.get_integrater_reindex_operator():
reindex.set_operator(self.get_integrater_reindex_operator())
if self.get_integrater_spacegroup_number():
reindex.set_spacegroup(self.get_integrater_spacegroup_number())
hklout = "%s_reindex.mtz" % os.path.splitext(integrate_mtz)[0]
reindex.set_hklin(integrate_mtz)
reindex.set_hklout(hklout)
reindex.reindex()
integrate_mtz = hklout
experiments_json = xparm_xds_to_experiments_json(
self._xds_data_files["GXPARM.XDS"], self.get_working_directory())
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(f"{pname} {xname} {dname} {sweep}",
experiments_json)
FileHandler.record_more_data_file(
f"{pname} {xname} {dname} {sweep} INTEGRATE", integrate_mtz)
self._intgr_experiments_filename = experiments_json
return integrate_mtz
def run(self, method):
from xia2.Handlers.Streams import Debug
Debug.write('Running dials.index')
self.clear_command_line()
for f in self._sweep_filenames:
self.add_command_line(f)
for f in self._spot_filenames:
self.add_command_line(f)
self.add_command_line('indexing.method=%s' % method)
nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
self.set_cpu_threads(nproc)
self.add_command_line('indexing.nproc=%i' % nproc)
if PhilIndex.params.xia2.settings.small_molecule == True:
self.add_command_line('filter_ice=false')
if self._reflections_per_degree is not None:
self.add_command_line(
'reflections_per_degree=%i' %self._reflections_per_degree)
if self._fft3d_n_points is not None:
self.add_command_line(
'fft3d.reciprocal_space_grid.n_points=%i' %self._fft3d_n_points)
if self._close_to_spindle_cutoff is not None:
self.add_command_line(
'close_to_spindle_cutoff=%f' %self._close_to_spindle_cutoff)
if self._outlier_algorithm:
self.add_command_line('outlier.algorithm=%s' % self._outlier_algorithm)
if self._max_cell:
self.add_command_line('max_cell=%d' % self._max_cell)
if self._min_cell:
self.add_command_line('min_cell=%d' % self._min_cell)
if self._histogram_binning is not None:
self.add_command_line('max_cell_estimation.histogram_binning=%s' %self._histogram_binning)
if self._d_min_start:
self.add_command_line('d_min_start=%f' % self._d_min_start)
if self._indxr_input_lattice is not None:
from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
self._symm = lattice_to_spacegroup_number(
self._indxr_input_lattice)
self.add_command_line('known_symmetry.space_group=%s' % self._symm)
if self._indxr_input_cell is not None:
self.add_command_line(
'known_symmetry.unit_cell="%s,%s,%s,%s,%s,%s"' %self._indxr_input_cell)
if self._maximum_spot_error:
self.add_command_line('maximum_spot_error=%.f' %
self._maximum_spot_error)
if self._detector_fix:
self.add_command_line('detector.fix=%s' % self._detector_fix)
if self._beam_fix:
self.add_command_line('beam.fix=%s' % self._beam_fix)
if self._phil_file is not None:
self.add_command_line("%s" %self._phil_file)
self._experiment_filename = os.path.join(
self.get_working_directory(), '%d_experiments.json' %self.get_xpid())
self._indexed_filename = os.path.join(
self.get_working_directory(), '%d_indexed.pickle' %self.get_xpid())
self.add_command_line("output.experiments=%s" %self._experiment_filename)
self.add_command_line("output.reflections=%s" %self._indexed_filename)
self.start()
self.close_wait()
self.check_for_errors()
from dials.array_family import flex
from dxtbx.serialize import load
self._experiment_list = load.experiment_list(self._experiment_filename)
self._reflections = flex.reflection_table.from_pickle(
self._indexed_filename)
crystal = self._experiment_list.crystals()[0]
self._p1_cell = crystal.get_unit_cell().parameters()
refined_sel = self._reflections.get_flags(self._reflections.flags.used_in_refinement)
refl = self._reflections.select(refined_sel)
xc, yc, zc = refl['xyzcal.px'].parts()
xo, yo, zo = refl['xyzobs.px.value'].parts()
import math
self._nref = refl.size()
self._rmsd_x = math.sqrt(flex.mean(flex.pow2(xc - xo)))
self._rmsd_y = math.sqrt(flex.mean(flex.pow2(yc - yo)))
self._rmsd_z = math.sqrt(flex.mean(flex.pow2(zc - zo)))
return
def run(self, method):
from xia2.Handlers.Streams import Debug
Debug.write('Running dials.index')
self.clear_command_line()
for f in self._sweep_filenames:
self.add_command_line(f)
for f in self._spot_filenames:
self.add_command_line(f)
self.add_command_line('indexing.method=%s' % method)
nproc = Flags.get_parallel()
self.set_cpu_threads(nproc)
self.add_command_line('indexing.nproc=%i' % nproc)
if Flags.get_small_molecule():
self.add_command_line('filter_ice=false')
if self._reflections_per_degree is not None:
self.add_command_line(
'reflections_per_degree=%i' %self._reflections_per_degree)
if self._fft3d_n_points is not None:
self.add_command_line(
'fft3d.reciprocal_space_grid.n_points=%i' %self._fft3d_n_points)
if self._close_to_spindle_cutoff is not None:
self.add_command_line(
'close_to_spindle_cutoff=%f' %self._close_to_spindle_cutoff)
if self._outlier_algorithm:
self.add_command_line('outlier.algorithm=%s' % self._outlier_algorithm)
if self._max_cell:
self.add_command_line('max_cell=%d' % self._max_cell)
if self._min_cell:
self.add_command_line('min_cell=%d' % self._min_cell)
if self._d_min_start:
self.add_command_line('d_min_start=%f' % self._d_min_start)
if self._indxr_input_lattice is not None:
from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
self._symm = lattice_to_spacegroup_number(
self._indxr_input_lattice)
self.add_command_line('known_symmetry.space_group=%s' % self._symm)
if self._indxr_input_cell is not None:
self.add_command_line(
'known_symmetry.unit_cell="%s,%s,%s,%s,%s,%s"' %self._indxr_input_cell)
if self._maximum_spot_error:
self.add_command_line('maximum_spot_error=%.f' %
self._maximum_spot_error)
if self._detector_fix:
self.add_command_line('detector.fix=%s' % self._detector_fix)
if self._beam_fix:
self.add_command_line('beam.fix=%s' % self._beam_fix)
if self._phil_file is not None:
self.add_command_line("%s" %self._phil_file)
self._experiment_filename = os.path.join(
self.get_working_directory(), '%d_experiments.json' %self.get_xpid())
self._indexed_filename = os.path.join(
self.get_working_directory(), '%d_indexed.pickle' %self.get_xpid())
self.add_command_line("output.experiments=%s" %self._experiment_filename)
self.add_command_line("output.reflections=%s" %self._indexed_filename)
self.start()
self.close_wait()
self.check_for_errors()
records = self.get_all_output()
for i, record in enumerate(records):
if 'Unit cell:' in record:
self._p1_cell = map(float, record.replace('(', '').replace(
')', '').replace(',', '').split()[-6:])
if 'Final RMSDs by experiment' in record:
values = records[i+6].strip().strip('|').split('|')
if len(values):
values = [float(v) for v in values]
if values[0] == 0:
self._nref = int(values[1])
self._rmsd_x = values[2]
self._rmsd_y = values[3]
self._rmsd_z = values[4]
return
'metric':self._solutions[solution]['metric'],
'cell':self._solutions[solution]['cell']}
self._indxr_mosaic = self._solution['mosaic']
experiment_list = load.experiment_list(self._solution['experiments_file'])
self.set_indexer_experiment_list(experiment_list)
# reindex the output experiments list to the reference setting
# (from the best cell/conventional setting)
cb_op_to_ref = experiment_list.crystals()[0].get_space_group().info()\
.change_of_basis_op_to_reference_setting()
reindex = self.Reindex()
reindex.set_experiments_filename(self._solution['experiments_file'])
reindex.set_cb_op(cb_op_to_ref)
reindex.set_space_group(str(lattice_to_spacegroup_number(
self._solution['lattice'])))
reindex.run()
experiments_file = reindex.get_reindexed_experiments_filename()
experiment_list = load.experiment_list(experiments_file)
self.set_indexer_experiment_list(experiment_list)
self.set_indexer_payload("experiments_filename", experiments_file)
# reindex the output reflection list to this solution
reindex = self.Reindex()
reindex.set_indexed_filename(indexed_file)
reindex.set_cb_op(self._solution['cb_op'])
reindex.set_space_group(str(lattice_to_spacegroup_number(
self._solution['lattice'])))
reindex.run()
indexed_file = reindex.get_reindexed_reflections_filename()
self.set_indexer_payload("indexed_filename", indexed_file)
