def _do_indexing(self, method=None):
indexer = self.Index()
for spot_list in self._indxr_payload["spot_lists"]:
indexer.add_spot_filename(spot_list)
for datablock in self._indxr_payload["datablocks"]:
indexer.add_sweep_filename(datablock)
if PhilIndex.params.dials.index.phil_file is not None:
indexer.set_phil_file(PhilIndex.params.dials.index.phil_file)
if PhilIndex.params.dials.index.max_cell:
indexer.set_max_cell(PhilIndex.params.dials.index.max_cell)
if PhilIndex.params.xia2.settings.small_molecule == True:
indexer.set_min_cell(3)
if PhilIndex.params.dials.fix_geometry:
indexer.set_detector_fix('all')
indexer.set_beam_fix('all')
indexer.set_close_to_spindle_cutoff(
PhilIndex.params.dials.close_to_spindle_cutoff)
if self._indxr_input_lattice:
indexer.set_indexer_input_lattice(self._indxr_input_lattice)
Debug.write('Set lattice: %s' % self._indxr_input_lattice)
if self._indxr_input_cell:
indexer.set_indexer_input_cell(self._indxr_input_cell)
Debug.write('Set cell: %f %f %f %f %f %f' % \
self._indxr_input_cell)
original_cell = self._indxr_input_cell
if method is None:
if PhilIndex.params.dials.index.method is None:
method = 'fft3d'
Debug.write('Choosing indexing method: %s' % method)
else:
method = PhilIndex.params.dials.index.method
FileHandler.record_log_file('%s INDEX' % self.get_indexer_full_name(),
indexer.get_log_file())
indexer.run(method)
if not os.path.exists(indexer.get_experiments_filename()):
raise RuntimeError("Indexing has failed: see %s for more details."
%indexer.get_log_file())
elif not os.path.exists(indexer.get_indexed_filename()):
raise RuntimeError("Indexing has failed: %s does not exist."
%indexer.get_indexed_filename())
report = self.Report()
report.set_experiments_filename(indexer.get_experiments_filename())
report.set_reflections_filename(indexer.get_indexed_filename())
html_filename = os.path.join(
self.get_working_directory(),
'%i_dials.index.report.html' %report.get_xpid())
report.set_html_filename(html_filename)
report.run()
assert os.path.exists(html_filename)
FileHandler.record_html_file(
'%s INDEX' %self.get_indexer_full_name(), html_filename)
return indexer
def _scale_finish_chunk_3_truncate(self):
for wavelength in self._scalr_scaled_refl_files.keys():
hklin = self._scalr_scaled_refl_files[wavelength]
truncate = self._factory.Truncate()
truncate.set_hklin(hklin)
if self.get_scaler_anomalous():
truncate.set_anomalous(True)
else:
truncate.set_anomalous(False)
FileHandler.record_log_file('%s %s %s truncate' % \
(self._scalr_pname,
self._scalr_xname,
wavelength),
truncate.get_log_file())
hklout = os.path.join(self.get_working_directory(),
'%s_truncated.mtz' % wavelength)
truncate.set_hklout(hklout)
truncate.truncate()
xmlout = truncate.get_xmlout()
if xmlout is not None:
FileHandler.record_xml_file('%s %s %s truncate' % \
(self._scalr_pname,
self._scalr_xname,
wavelength),
xmlout)
Debug.write('%d absent reflections in %s removed' % \
(truncate.get_nabsent(), wavelength))
b_factor = truncate.get_b_factor()
# record the b factor somewhere (hopefully) useful...
self._scalr_statistics[(self._scalr_pname, self._scalr_xname,
wavelength)]['Wilson B factor'] = [
b_factor
]
# and record the reflection file..
self._scalr_scaled_refl_files[wavelength] = hklout
def dials_symmetry_decide_pointgroup(self, experiments, reflections):
"""Run the symmetry analyser and return it for later inspection."""
symmetry_analyser = DialsSymmetry()
symmetry_analyser.set_working_directory(self.get_working_directory())
auto_logfiler(symmetry_analyser)
FileHandler.record_log_file(
"%s %s SYMMETRY" % (self._scalr_pname, self._scalr_xname),
symmetry_analyser.get_log_file(),
)
for (exp, refl) in zip(experiments, reflections):
symmetry_analyser.add_experiments(exp)
symmetry_analyser.add_reflections(refl)
symmetry_analyser.decide_pointgroup()
return symmetry_analyser
def _index(self):
"""Actually do the autoindexing using the data prepared by the
previous method."""
images_str = "%d to %d" % tuple(self._indxr_images[0])
for i in self._indxr_images[1:]:
images_str += ", %d to %d" % tuple(i)
cell_str = None
if self._indxr_input_cell:
cell_str = "%.2f %.2f %.2f %.2f %.2f %.2f" % self._indxr_input_cell
# then this is a proper autoindexing run - describe this
# to the journal entry
dirname = self.get_directory()
Journal.block(
"autoindexing",
self._indxr_sweep_name,
"XDS",
{
"images": images_str,
"target cell": cell_str,
"target lattice": self._indxr_input_lattice,
"template": self.get_template(),
"directory": dirname,
},
)
idxref = self.Idxref()
self._index_remove_masked_regions()
for file in ["SPOT.XDS"]:
idxref.set_input_data_file(file, self._indxr_payload[file])
# edit SPOT.XDS to remove reflections in untrusted regions of the detector
idxref.set_data_range(self._indxr_images[0][0],
self._indxr_images[0][1])
idxref.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
# set the phi start etc correctly
for block in self._indxr_images[:1]:
starting_frame = block[0]
starting_angle = self.get_scan().get_angle_from_image_index(
starting_frame)
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
for block in self._indxr_images[1:]:
idxref.add_spot_range(block[0], block[1])
if self._indxr_user_input_lattice:
idxref.set_indexer_user_input_lattice(True)
if self._indxr_input_lattice and self._indxr_input_cell:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
idxref.set_indexer_input_cell(self._indxr_input_cell)
Debug.write("Set lattice: %s" % self._indxr_input_lattice)
Debug.write("Set cell: %f %f %f %f %f %f" % self._indxr_input_cell)
original_cell = self._indxr_input_cell
elif self._indxr_input_lattice:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
original_cell = None
else:
original_cell = None
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
idxref.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1])
# fixme need to check if the lattice, cell have been set already,
# and if they have, pass these in as input to the indexing job.
done = False
while not done:
try:
done = idxref.run()
# N.B. in here if the IDXREF step was being run in the first
# pass done is FALSE however there should be a refined
# P1 orientation matrix etc. available - so keep it!
except XDSException as e:
# inspect this - if we have complaints about not
# enough reflections indexed, and we have a target
# unit cell, and they are the same, well ignore it
if "solution is inaccurate" in str(e):
Debug.write("XDS complains solution inaccurate - ignoring")
done = idxref.continue_from_error()
elif ("insufficient percentage (< 70%)" in str(e)
or "insufficient percentage (< 50%)"
in str(e)) and original_cell:
done = idxref.continue_from_error()
lattice, cell, mosaic = idxref.get_indexing_solution()
# compare solutions FIXME should use xds_cell_deviation
check = PhilIndex.params.xia2.settings.xds_check_cell_deviation
for j in range(3):
# allow two percent variation in unit cell length
if (math.fabs(
(cell[j] - original_cell[j]) / original_cell[j]) >
0.02 and check):
Debug.write("XDS unhappy and solution wrong")
raise e
# and two degree difference in angle
if (math.fabs(cell[j + 3] - original_cell[j + 3]) > 2.0
and check):
Debug.write("XDS unhappy and solution wrong")
raise e
Debug.write("XDS unhappy but solution ok")
elif "insufficient percentage (< 70%)" in str(
e) or "insufficient percentage (< 50%)" in str(e):
Debug.write("XDS unhappy but solution probably ok")
done = idxref.continue_from_error()
else:
raise e
FileHandler.record_log_file(
"%s INDEX" % self.get_indexer_full_name(),
os.path.join(self.get_working_directory(), "IDXREF.LP"),
)
for file in ["SPOT.XDS", "XPARM.XDS"]:
self._indxr_payload[file] = idxref.get_output_data_file(file)
# need to get the indexing solutions out somehow...
self._indxr_other_lattice_cell = idxref.get_indexing_solutions()
self._indxr_lattice, self._indxr_cell, self._indxr_mosaic = (
idxref.get_indexing_solution())
import dxtbx
from dxtbx.serialize.xds import to_crystal
xparm_file = os.path.join(self.get_working_directory(), "XPARM.XDS")
models = dxtbx.load(xparm_file)
crystal_model = to_crystal(xparm_file)
from dxtbx.model import Experiment, ExperimentList
experiment = Experiment(
beam=models.get_beam(),
detector=models.get_detector(),
goniometer=models.get_goniometer(),
scan=models.get_scan(),
crystal=crystal_model,
# imageset=self.get_imageset(),
)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
# I will want this later on to check that the lattice was ok
self._idxref_subtree_problem = idxref.get_index_tree_problem()
return
def _scale(self):
"Perform all of the operations required to deliver the scaled data."
epochs = self._sweep_handler.get_epochs()
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_chef_unmerged(True)
sc.set_new_scales_file("%s.scales" % self._scalr_xname)
user_resolution_limits = {}
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
intgr = si.get_integrater()
if intgr.get_integrater_user_resolution():
dmin = intgr.get_integrater_high_resolution()
if (dname, sname) not in user_resolution_limits:
user_resolution_limits[(dname, sname)] = dmin
elif dmin < user_resolution_limits[(dname, sname)]:
user_resolution_limits[(dname, sname)] = dmin
start, end = si.get_batch_range()
if (dname, sname) in self._scalr_resolution_limits:
resolution, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(start, end, exclude=False, resolution=resolution, name=sname)
else:
sc.add_run(start, end, name=sname)
sc.set_hklout(
os.path.join(
self.get_working_directory(),
f"{self._scalr_pname}_{self._scalr_xname}_scaled_test.mtz",
)
)
if self.get_scaler_anomalous():
sc.set_anomalous()
# what follows, sucks
failover = PhilIndex.params.xia2.settings.failover
if failover:
try:
sc.scale()
except RuntimeError as e:
es = str(e)
if (
"bad batch" in es
or "negative scales run" in es
or "no observations" in es
):
# first ID the sweep from the batch no
batch = int(es.split()[-1])
epoch = self._identify_sweep_epoch(batch)
sweep = self._scalr_integraters[epoch].get_integrater_sweep()
# then remove it from my parent xcrystal
self.get_scaler_xcrystal().remove_sweep(sweep)
# then remove it from the scaler list of intergraters
# - this should really be a scaler interface method
del self._scalr_integraters[epoch]
# then tell the user what is happening
logger.info(
"Sweep %s gave negative scales - removing", sweep.get_name()
)
# then reset the prepare, do, finish flags
self.set_scaler_prepare_done(False)
self.set_scaler_done(False)
self.set_scaler_finish_done(False)
# and return
return
else:
raise e
else:
sc.scale()
# then gather up all of the resulting reflection files
# and convert them into the required formats (.sca, .mtz.)
loggraph = sc.parse_ccp4_loggraph()
resolution_info = {}
reflection_files = sc.get_scaled_reflection_files()
for dataset in reflection_files:
FileHandler.record_temporary_file(reflection_files[dataset])
for key in loggraph:
if "Analysis against resolution" in key:
dataset = key.split(",")[-1].strip()
resolution_info[dataset] = transpose_loggraph(loggraph[key])
# check in here that there is actually some data to scale..!
if not resolution_info:
raise RuntimeError("no resolution info")
highest_suggested_resolution = self.assess_resolution_limits(
sc.get_unmerged_reflection_file(), user_resolution_limits
)
if not self.get_scaler_done():
logger.debug("Returning as scaling not finished...")
return
batch_info = {}
for key in loggraph:
if "Analysis against Batch" in key:
dataset = key.split(",")[-1].strip()
batch_info[dataset] = transpose_loggraph(loggraph[key])
sc = self._updated_aimless()
FileHandler.record_log_file(
f"{self._scalr_pname} {self._scalr_xname} aimless", sc.get_log_file()
)
sc.set_hklin(self._prepared_reflections)
sc.set_new_scales_file("%s_final.scales" % self._scalr_xname)
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(
start, end, exclude=False, resolution=resolution_limit, name=xname
)
sc.set_hklout(
os.path.join(
self.get_working_directory(),
f"{self._scalr_pname}_{self._scalr_xname}_scaled.mtz",
)
)
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
FileHandler.record_xml_file(
f"{self._scalr_pname} {self._scalr_xname} aimless", sc.get_xmlout()
)
data = sc.get_summary()
scales_file = sc.get_new_scales_file()
loggraph = sc.parse_ccp4_loggraph()
standard_deviation_info = {}
for key in loggraph:
if "standard deviation v. Intensity" in key:
dataset = key.split(",")[-1].strip()
standard_deviation_info[dataset] = transpose_loggraph(loggraph[key])
resolution_info = {}
for key in loggraph:
if "Analysis against resolution" in key:
dataset = key.split(",")[-1].strip()
resolution_info[dataset] = transpose_loggraph(loggraph[key])
batch_info = {}
for key in loggraph:
if "Analysis against Batch" in key:
dataset = key.split(",")[-1].strip()
batch_info[dataset] = transpose_loggraph(loggraph[key])
# finally put all of the results "somewhere useful"
self._scalr_statistics = data
self._scalr_scaled_refl_files = copy.deepcopy(sc.get_scaled_reflection_files())
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_scales_file(scales_file)
self._wavelengths_in_order = []
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(
start, end, exclude=False, resolution=resolution_limit, name=sname
)
if dname not in self._wavelengths_in_order:
self._wavelengths_in_order.append(dname)
sc.set_hklout(
os.path.join(
self.get_working_directory(),
f"{self._scalr_pname}_{self._scalr_xname}_scaled.mtz",
)
)
sc.set_scalepack()
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
self._update_scaled_unit_cell()
self._scalr_scaled_reflection_files = {}
self._scalr_scaled_reflection_files["sca"] = {}
self._scalr_scaled_reflection_files["sca_unmerged"] = {}
self._scalr_scaled_reflection_files["mtz_unmerged"] = {}
for key in self._scalr_scaled_refl_files:
hklout = self._scalr_scaled_refl_files[key]
scaout = "%s.sca" % hklout[:-4]
self._scalr_scaled_reflection_files["sca"][key] = scaout
FileHandler.record_data_file(scaout)
scalepack = os.path.join(
os.path.split(hklout)[0],
os.path.split(hklout)[1]
.replace("_scaled", "_scaled_unmerged")
.replace(".mtz", ".sca"),
)
self._scalr_scaled_reflection_files["sca_unmerged"][key] = scalepack
FileHandler.record_data_file(scalepack)
mtz_unmerged = os.path.splitext(scalepack)[0] + ".mtz"
self._scalr_scaled_reflection_files["mtz_unmerged"][key] = mtz_unmerged
FileHandler.record_data_file(mtz_unmerged)
if self._scalr_cell_esd is not None:
# patch .mtz and overwrite unit cell information
import xia2.Modules.Scaler.tools as tools
override_cell = self._scalr_cell_dict.get(
f"{self._scalr_pname}_{self._scalr_xname}_{key}"
)[0]
tools.patch_mtz_unit_cell(mtz_unmerged, override_cell)
tools.patch_mtz_unit_cell(hklout, override_cell)
self._scalr_scaled_reflection_files["mtz_unmerged"][key] = mtz_unmerged
FileHandler.record_data_file(mtz_unmerged)
if PhilIndex.params.xia2.settings.merging_statistics.source == "cctbx":
for key in self._scalr_scaled_refl_files:
stats = self._compute_scaler_statistics(
self._scalr_scaled_reflection_files["mtz_unmerged"][key],
selected_band=(highest_suggested_resolution, None),
wave=key,
)
self._scalr_statistics[
(self._scalr_pname, self._scalr_xname, key)
] = stats
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_scales_file(scales_file)
self._wavelengths_in_order = []
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(
start, end, exclude=False, resolution=resolution_limit, name=sname
)
if dname not in self._wavelengths_in_order:
self._wavelengths_in_order.append(dname)
sc.set_hklout(
os.path.join(
self.get_working_directory(),
f"{self._scalr_pname}_{self._scalr_xname}_chef.mtz",
)
)
sc.set_chef_unmerged(True)
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
if not PhilIndex.params.dials.fast_mode:
try:
self._generate_absorption_map(sc)
except Exception as e:
# Map generation may fail for number of reasons, eg. matplotlib borken
logger.debug("Could not generate absorption map (%s)", e)
def _index(self):
"""Actually do the autoindexing using the data prepared by the
previous method."""
self._index_remove_masked_regions()
if self._i_or_ii is None:
self._i_or_ii = self.decide_i_or_ii()
logger.debug("Selecting I or II, chose %s", self._i_or_ii)
idxref = self.Idxref()
for file in ["SPOT.XDS"]:
idxref.set_input_data_file(file, self._indxr_payload[file])
# set the phi start etc correctly
idxref.set_data_range(self._indxr_images[0][0],
self._indxr_images[0][1])
idxref.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
if self._i_or_ii == "i":
blocks = self._index_select_images_i()
for block in blocks[:1]:
starting_frame = block[0]
starting_angle = self.get_scan().get_angle_from_image_index(
starting_frame)
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
for block in blocks[1:]:
idxref.add_spot_range(block[0], block[1])
else:
for block in self._indxr_images[:1]:
starting_frame = block[0]
starting_angle = self.get_scan().get_angle_from_image_index(
starting_frame)
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
for block in self._indxr_images[1:]:
idxref.add_spot_range(block[0], block[1])
# FIXME need to also be able to pass in the known unit
# cell and lattice if already available e.g. from
# the helper... indirectly
if self._indxr_user_input_lattice:
idxref.set_indexer_user_input_lattice(True)
if self._indxr_input_lattice and self._indxr_input_cell:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
idxref.set_indexer_input_cell(self._indxr_input_cell)
logger.debug("Set lattice: %s", self._indxr_input_lattice)
logger.debug("Set cell: %f %f %f %f %f %f" %
self._indxr_input_cell)
original_cell = self._indxr_input_cell
elif self._indxr_input_lattice:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
original_cell = None
else:
original_cell = None
# FIXED need to set the beam centre here - this needs to come
# from the input .xinfo object or header, and be converted
# to the XDS frame... done.
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
idxref.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1])
# fixme need to check if the lattice, cell have been set already,
# and if they have, pass these in as input to the indexing job.
done = False
while not done:
try:
done = idxref.run()
# N.B. in here if the IDXREF step was being run in the first
# pass done is FALSE however there should be a refined
# P1 orientation matrix etc. available - so keep it!
except XDSException as e:
# inspect this - if we have complaints about not
# enough reflections indexed, and we have a target
# unit cell, and they are the same, well ignore it
if "solution is inaccurate" in str(e):
logger.debug(
"XDS complains solution inaccurate - ignoring")
done = idxref.continue_from_error()
elif ("insufficient percentage (< 70%)" in str(e)
or "insufficient percentage (< 50%)"
in str(e)) and original_cell:
done = idxref.continue_from_error()
lattice, cell, mosaic = idxref.get_indexing_solution()
# compare solutions
check = PhilIndex.params.xia2.settings.xds_check_cell_deviation
for j in range(3):
# allow two percent variation in unit cell length
if (math.fabs(
(cell[j] - original_cell[j]) / original_cell[j]) >
0.02 and check):
logger.debug("XDS unhappy and solution wrong")
raise e
# and two degree difference in angle
if (math.fabs(cell[j + 3] - original_cell[j + 3]) > 2.0
and check):
logger.debug("XDS unhappy and solution wrong")
raise e
logger.debug("XDS unhappy but solution ok")
elif "insufficient percentage (< 70%)" in str(
e) or "insufficient percentage (< 50%)" in str(e):
logger.debug("XDS unhappy but solution probably ok")
done = idxref.continue_from_error()
else:
raise e
FileHandler.record_log_file(
"%s INDEX" % self.get_indexer_full_name(),
os.path.join(self.get_working_directory(), "IDXREF.LP"),
)
for file in ["SPOT.XDS", "XPARM.XDS"]:
self._indxr_payload[file] = idxref.get_output_data_file(file)
# need to get the indexing solutions out somehow...
self._indxr_other_lattice_cell = idxref.get_indexing_solutions()
(
self._indxr_lattice,
self._indxr_cell,
self._indxr_mosaic,
) = idxref.get_indexing_solution()
xparm_file = os.path.join(self.get_working_directory(), "XPARM.XDS")
models = dxtbx.load(xparm_file)
crystal_model = to_crystal(xparm_file)
# this information gets lost when re-creating the models from the
# XDS results - however is not refined so can simply copy from the
# input - https://github.com/xia2/xia2/issues/372
models.get_detector()[0].set_thickness(
converter.get_detector()[0].get_thickness())
experiment = Experiment(
beam=models.get_beam(),
detector=models.get_detector(),
goniometer=models.get_goniometer(),
scan=models.get_scan(),
crystal=crystal_model,
# imageset=self.get_imageset(),
)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
# I will want this later on to check that the lattice was ok
self._idxref_subtree_problem = idxref.get_index_tree_problem()
if not self.get_scaler_done():
Debug.write('Returning as scaling not finished...')
return
batch_info = { }
for key in loggraph:
if 'Analysis against Batch' in key:
dataset = key.split(',')[-1].strip()
batch_info[dataset] = transpose_loggraph(
loggraph[key])
sc = self._updated_aimless()
FileHandler.record_log_file('%s %s aimless' % (self._scalr_pname,
self._scalr_xname),
sc.get_log_file())
highest_resolution = 100.0
sc.set_hklin(self._prepared_reflections)
sc.set_intensities(PhilIndex.params.ccp4.aimless.intensities)
sc.set_new_scales_file('%s_final.scales' % self._scalr_xname)
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
def _integrate_finish(self):
'''Finish off the integration by running correct.'''
# first run the postrefinement etc with spacegroup P1
# and the current unit cell - this will be used to
# obtain a benchmark rmsd in pixels / phi and also
# cell deviations (this is working towards spotting bad
# indexing solutions) - only do this if we have no
# reindex matrix... and no postrefined cell...
p1_deviations = None
# fix for bug # 3264 -
# if we have not run integration with refined parameters, make it so...
# erm? shouldn't this therefore return if this is the principle, or
# set the flag after we have tested the lattice?
if not self._xds_data_files.has_key('GXPARM.XDS') and \
PhilIndex.params.xds.integrate.reintegrate:
Debug.write(
'Resetting integrater, to ensure refined orientation is used')
self.set_integrater_done(False)
if not self.get_integrater_reindex_matrix() and not self._intgr_cell \
and not Flags.get_no_lattice_test() and \
not self.get_integrater_sweep().get_user_lattice():
correct = self.Correct()
correct.set_data_range(self._intgr_wedge[0],
self._intgr_wedge[1])
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# FIXME should this be using the correctly transformed
# cell or are the results ok without it?!
correct.set_spacegroup_number(1)
correct.set_cell(self._intgr_refiner_cell)
correct.run()
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file('%s %s %s %s CORRECT' % \
(pname, xname, dname, sweep),
os.path.join(
self.get_working_directory(),
'CORRECT.LP'))
FileHandler.record_more_data_file(
'%s %s %s %s CORRECT' % (pname, xname, dname, sweep),
os.path.join(self.get_working_directory(), 'XDS_ASCII.HKL'))
cell = correct.get_result('cell')
cell_esd = correct.get_result('cell_esd')
Debug.write('Postrefinement in P1 results:')
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(cell))
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(cell_esd))
Debug.write('Deviations: %.2f pixels %.2f degrees' % \
(correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi')))
p1_deviations = (correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi'))
# next run the postrefinement etc with the given
# cell / lattice - this will be the assumed result...
correct = self.Correct()
correct.set_data_range(self._intgr_wedge[0],
self._intgr_wedge[1])
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# BUG # 2695 probably comes from here - need to check...
# if the pointless interface comes back with a different
# crystal setting then the unit cell stored in self._intgr_cell
# needs to be set to None...
if self.get_integrater_spacegroup_number():
correct.set_spacegroup_number(
self.get_integrater_spacegroup_number())
if not self._intgr_cell:
raise RuntimeError, 'no unit cell to recycle'
correct.set_cell(self._intgr_cell)
# BUG # 3113 - new version of XDS will try and figure the
# best spacegroup out from the intensities (and get it wrong!)
# unless we set the spacegroup and cell explicitly
if not self.get_integrater_spacegroup_number():
cell = self._intgr_refiner_cell
lattice = self._intgr_refiner.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup_number(lattice)
# this should not prevent the postrefinement from
# working correctly, else what is above would not
# work correctly (the postrefinement test)
correct.set_spacegroup_number(spacegroup_number)
correct.set_cell(cell)
Debug.write('Setting spacegroup to: %d' % spacegroup_number)
Debug.write(
'Setting cell to: %.2f %.2f %.2f %.2f %.2f %.2f' % tuple(cell))
if self.get_integrater_reindex_matrix():
# bug! if the lattice is not primitive the values in this
# reindex matrix need to be multiplied by a constant which
# depends on the Bravais lattice centering.
lattice = self._intgr_refiner.get_refiner_lattice()
import scitbx.matrix
matrix = self.get_integrater_reindex_matrix()
matrix = scitbx.matrix.sqr(matrix).transpose().elems
matrix = r_to_rt(matrix)
if lattice[1] == 'P':
mult = 1
elif lattice[1] == 'C' or lattice[1] == 'I':
mult = 2
elif lattice[1] == 'R':
mult = 3
elif lattice[1] == 'F':
mult = 4
else:
raise RuntimeError, 'unknown multiplier for lattice %s' % \
lattice
Debug.write('REIDX multiplier for lattice %s: %d' % \
(lattice, mult))
mult_matrix = [mult * m for m in matrix]
Debug.write('REIDX set to %d %d %d %d %d %d %d %d %d %d %d %d' % \
tuple(mult_matrix))
correct.set_reindex_matrix(mult_matrix)
correct.run()
# erm. just to be sure
if self.get_integrater_reindex_matrix() and \
correct.get_reindex_used():
raise RuntimeError, 'Reindex panic!'
# get the reindex operation used, which may be useful if none was
# set but XDS decided to apply one, e.g. #419.
if not self.get_integrater_reindex_matrix() and \
correct.get_reindex_used():
# convert this reindex operation to h, k, l form: n.b. this
# will involve dividing through by the lattice centring multiplier
matrix = rt_to_r(correct.get_reindex_used())
import scitbx.matrix
matrix = scitbx.matrix.sqr(matrix).transpose().elems
lattice = self._intgr_refiner.get_refiner_lattice()
if lattice[1] == 'P':
mult = 1.0
elif lattice[1] == 'C' or lattice[1] == 'I':
mult = 2.0
elif lattice[1] == 'R':
mult = 3.0
elif lattice[1] == 'F':
mult = 4.0
matrix = [m / mult for m in matrix]
reindex_op = mat_to_symop(matrix)
# assign this to self: will this reset?! make for a leaky
# abstraction and just assign this...
# self.set_integrater_reindex_operator(reindex)
self._intgr_reindex_operator = reindex_op
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file('%s %s %s %s CORRECT' % \
(pname, xname, dname, sweep),
os.path.join(self.get_working_directory(),
'CORRECT.LP'))
# should get some interesting stuff from the XDS correct file
# here, for instance the resolution range to use in integration
# (which should be fed back if not fast) and so on...
self._intgr_corrected_hklout = os.path.join(self.get_working_directory(),
'XDS_ASCII.HKL')
# also record the batch range - needed for the analysis of the
# radiation damage in chef...
self._intgr_batches_out = (self._intgr_wedge[0],
self._intgr_wedge[1])
# FIXME perhaps I should also feedback the GXPARM file here??
for file in ['GXPARM.XDS']:
self._xds_data_files[file] = correct.get_output_data_file(file)
# record the postrefined cell parameters
self._intgr_cell = correct.get_result('cell')
self._intgr_n_ref = correct.get_result('n_ref')
Debug.write('Postrefinement in "correct" spacegroup results:')
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(correct.get_result('cell')))
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(correct.get_result('cell_esd')))
Debug.write('Deviations: %.2f pixels %.2f degrees' % \
(correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi')))
Debug.write('Error correction parameters: A=%.3f B=%.3f' % \
correct.get_result('sdcorrection'))
# compute misorientation of axes
xparm_file = os.path.join(self.get_working_directory(), 'GXPARM.XDS')
from iotbx.xds import xparm
handle = xparm.reader()
handle.read_file(xparm_file)
rotn = handle.rotation_axis
beam = handle.beam_vector
dot = sum([rotn[j] * beam[j] for j in range(3)])
r = math.sqrt(sum([rotn[j] * rotn[j] for j in range(3)]))
b = math.sqrt(sum([beam[j] * beam[j] for j in range(3)]))
rtod = 180.0 / math.pi
angle = rtod * math.fabs(0.5 * math.pi - math.acos(dot / (r * b)))
Debug.write('Axis misalignment %.2f degrees' % angle)
correct_deviations = (correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi'))
if p1_deviations:
# compare and reject if both > 50% higher - though adding a little
# flexibility - 0.5 pixel / osc width slack.
pixel = p1_deviations[0]
phi = math.sqrt(0.05 * 0.05 + \
p1_deviations[1] * p1_deviations[1])
threshold = Flags.get_rejection_threshold()
Debug.write('RMSD ratio: %.2f' % (correct_deviations[0] / pixel))
Debug.write('RMSPhi ratio: %.2f' % (correct_deviations[1] / phi))
if correct_deviations[0] / pixel > threshold and \
correct_deviations[1] / phi > threshold:
Chatter.write(
'Eliminating this indexing solution as postrefinement')
Chatter.write(
'deviations rather high relative to triclinic')
raise BadLatticeError, \
'high relative deviations in postrefinement'
if not Flags.get_quick() and Flags.get_remove():
# check for alien reflections and perhaps recycle - removing them
if len(correct.get_remove()) > 0:
correct_remove = correct.get_remove()
current_remove = []
final_remove = []
# first ensure that there are no duplicate entries...
if os.path.exists(os.path.join(
self.get_working_directory(),
'REMOVE.HKL')):
for line in open(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'), 'r').readlines():
h, k, l = map(int, line.split()[:3])
z = float(line.split()[3])
if not (h, k, l, z) in current_remove:
current_remove.append((h, k, l, z))
for c in correct_remove:
if c in current_remove:
continue
final_remove.append(c)
Debug.write(
'%d alien reflections are already removed' % \
(len(correct_remove) - len(final_remove)))
else:
# we want to remove all of the new dodgy reflections
final_remove = correct_remove
remove_hkl = open(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'), 'w')
z_min = Flags.get_z_min()
rejected = 0
# write in the old reflections
for remove in current_remove:
z = remove[3]
if z >= z_min:
remove_hkl.write('%d %d %d %f\n' % remove)
else:
rejected += 1
Debug.write('Wrote %d old reflections to REMOVE.HKL' % \
(len(current_remove) - rejected))
Debug.write('Rejected %d as z < %f' % \
(rejected, z_min))
# and the new reflections
rejected = 0
used = 0
for remove in final_remove:
z = remove[3]
if z >= z_min:
used += 1
remove_hkl.write('%d %d %d %f\n' % remove)
else:
rejected += 1
Debug.write('Wrote %d new reflections to REMOVE.HKL' % \
(len(final_remove) - rejected))
Debug.write('Rejected %d as z < %f' % \
(rejected, z_min))
remove_hkl.close()
# we want to rerun the finishing step so...
# unless we have added no new reflections... or unless we
# have not confirmed the point group (see SCI-398)
if used and self.get_integrater_reindex_matrix():
self.set_integrater_finish_done(False)
else:
Debug.write(
'Going quickly so not removing %d outlier reflections...' % \
len(correct.get_remove()))
# Convert INTEGRATE.HKL to MTZ format and reapply any reindexing operations
# spacegroup changes to allow use with CCP4 / Aimless for scaling
integrate_hkl = os.path.join(
self.get_working_directory(), 'INTEGRATE.HKL')
hklout = os.path.splitext(integrate_hkl)[0] + ".mtz"
self._factory.set_working_directory(self.get_working_directory())
pointless = self._factory.Pointless()
pointless.set_xdsin(integrate_hkl)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
integrate_mtz = hklout
if self.get_integrater_reindex_operator() or \
self.get_integrater_spacegroup_number():
Debug.write('Reindexing things to MTZ')
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
if self.get_integrater_reindex_operator():
reindex.set_operator(self.get_integrater_reindex_operator())
if self.get_integrater_spacegroup_number():
reindex.set_spacegroup(self.get_integrater_spacegroup_number())
hklout = '%s_reindex.mtz' % os.path.splitext(integrate_mtz)[0]
reindex.set_hklin(integrate_mtz)
reindex.set_hklout(hklout)
reindex.reindex()
integrate_mtz = hklout
return integrate_mtz
def _mosflm_parallel_integrate(self):
'''Perform the integration as before, but this time as a
number of parallel Mosflm jobs (hence, in separate directories)
and including a step of pre-refinement of the mosaic spread and
missets. This will all be kind of explicit and hence probably
messy!'''
refinr = self.get_integrater_refiner()
lattice = refinr.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup(lattice)
mosaic = refinr.get_refiner_payload('mosaic')
beam = refinr.get_refiner_payload('beam')
distance = refinr.get_refiner_payload('distance')
matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')
integration_params = refinr.get_refiner_payload(
'mosflm_integration_parameters')
if integration_params:
if 'separation' in integration_params:
self.set_integrater_parameter(
'mosflm', 'separation',
'%s %s' % tuple(integration_params['separation']))
if 'raster' in integration_params:
self.set_integrater_parameter(
'mosflm', 'raster',
'%d %d %d %d %d' % tuple(integration_params['raster']))
refinr.set_refiner_payload('mosflm_integration_parameters', None)
pname, xname, dname = self.get_integrater_project_info()
# what follows below should (i) be run in separate directories
# and (ii) be repeated N=parallel times.
nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
parallel = nproc
# FIXME this is something of a kludge - if too few frames refinement
# and integration does not work well... ideally want at least 15
# frames / chunk (say)
nframes = self._intgr_wedge[1] - self._intgr_wedge[0] + 1
if parallel > nframes / 15:
parallel = nframes // 15
if not parallel:
raise RuntimeError('parallel not set')
if parallel < 2:
raise RuntimeError('parallel not parallel: %s' % parallel)
jobs = []
hklouts = []
nref = 0
# calculate the chunks to use
offset = self.get_frame_offset()
start = self._intgr_wedge[0] - offset
end = self._intgr_wedge[1] - offset
left_images = 1 + end - start
left_chunks = parallel
chunks = []
while left_images > 0:
size = left_images // left_chunks
chunks.append((start, start + size - 1))
start += size
left_images -= size
left_chunks -= 1
summary_files = []
for j in range(parallel):
# make some working directories, as necessary - chunk-(0:N-1)
wd = os.path.join(self.get_working_directory(), 'chunk-%d' % j)
if not os.path.exists(wd):
os.makedirs(wd)
job = MosflmIntegrate()
job.set_working_directory(wd)
auto_logfiler(job)
l = refinr.get_refiner_lattice()
# create the starting point
f = open(os.path.join(wd, 'xiaintegrate-%s.mat' % l), 'w')
for m in matrix:
f.write(m)
f.close()
spacegroup_number = lattice_to_spacegroup(lattice)
job.set_refine_profiles(self._mosflm_refine_profiles)
# N.B. for harvesting need to append N to dname.
if pname is not None and xname is not None and dname is not None:
Debug.write('Harvesting: %s/%s/%s' % (pname, xname, dname))
harvest_dir = self.get_working_directory()
temp_dname = '%s_%s' % \
(dname, self.get_integrater_sweep_name())
job.set_pname_xname_dname(pname, xname, temp_dname)
job.set_template(os.path.basename(self.get_template()))
job.set_directory(self.get_directory())
# check for ice - and if so, exclude (ranges taken from
# XDS documentation)
if self.get_integrater_ice() != 0:
Debug.write('Excluding ice rings')
job.set_exclude_ice(True)
# exclude specified resolution ranges
if len(self.get_integrater_excluded_regions()) != 0:
regions = self.get_integrater_excluded_regions()
Debug.write('Excluding regions: %s' % repr(regions))
job.set_exclude_regions(regions)
mask = standard_mask(self.get_detector())
for m in mask:
job.add_instruction(m)
job.set_input_mat_file('xiaintegrate-%s.mat' % l)
job.set_beam_centre(beam)
job.set_distance(distance)
job.set_space_group_number(spacegroup_number)
job.set_mosaic(mosaic)
if self.get_wavelength_prov() == 'user':
job.set_wavelength(self.get_wavelength())
parameters = self.get_integrater_parameters('mosflm')
job.update_parameters(parameters)
if self._mosflm_gain:
job.set_gain(self._mosflm_gain)
# check for resolution limits
if self._intgr_reso_high > 0.0:
job.set_d_min(self._intgr_reso_high)
if self._intgr_reso_low:
job.set_d_max(self._intgr_reso_low)
if PhilIndex.params.general.backstop_mask:
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask = mask.calculate_mask_mosflm(self.get_header())
job.set_mask(mask)
detector = self.get_detector()
detector_width, detector_height = detector[0].get_image_size_mm()
lim_x = 0.5 * detector_width
lim_y = 0.5 * detector_height
Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
job.set_limits(lim_x, lim_y)
job.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
job.set_pre_refinement(True)
job.set_image_range(chunks[j])
# these are now running so ...
jobs.append(job)
continue
# ok, at this stage I need to ...
#
# (i) accumulate the statistics as a function of batch
# (ii) mong them into a single block
#
# This is likely to be a pain in the arse!
first_integrated_batch = 1.0e6
last_integrated_batch = -1.0e6
all_residuals = []
threads = []
for j in range(parallel):
job = jobs[j]
# now wait for them to finish - first wait will really be the
# first one, then all should be finished...
thread = Background(job, 'run')
thread.start()
threads.append(thread)
mosaics = []
postref_result = {}
integrated_images_first = 1.0e6
integrated_images_last = -1.0e6
self._intgr_per_image_statistics = {}
for j in range(parallel):
thread = threads[j]
thread.stop()
job = jobs[j]
# get the log file
output = job.get_all_output()
# record a copy of it, perhaps - though not if parallel
if self.get_integrater_sweep_name() and False:
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file(
'%s %s %s %s mosflm integrate' % \
(self.get_integrater_sweep_name(),
pname, xname, '%s_%d' % (dname, j)),
job.get_log_file())
# look for things that we want to know...
# that is, the output reflection file name, the updated
# value for the gain (if present,) any warnings, errors,
# or just interesting facts.
batches = job.get_batches_out()
integrated_images_first = min(batches[0], integrated_images_first)
integrated_images_last = max(batches[1], integrated_images_last)
mosaics.extend(job.get_mosaic_spreads())
if min(mosaics) < 0:
raise IntegrationError('negative mosaic spread: %s' %
min(mosaic))
if (job.get_detector_gain_error()
and not (self.get_imageset().get_detector()[0].get_type()
== 'SENSOR_PAD')):
gain = job.get_suggested_gain()
if gain is not None:
self.set_integrater_parameter('mosflm', 'gain', gain)
self.set_integrater_export_parameter(
'mosflm', 'gain', gain)
if self._mosflm_gain:
Debug.write('GAIN updated to %f' % gain)
else:
Debug.write('GAIN found to be %f' % gain)
self._mosflm_gain = gain
self._mosflm_rerun_integration = True
hklout = job.get_hklout()
Debug.write('Integration output: %s' % hklout)
hklouts.append(hklout)
nref += job.get_nref()
# if a BGSIG error happened try not refining the
# profile and running again...
if job.get_bgsig_too_large():
if not self._mosflm_refine_profiles:
raise RuntimeError('BGSIG error with profiles fixed')
Debug.write('BGSIG error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
if job.get_getprof_error():
Debug.write('GETPROF error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
# here
# write the report for each image as .*-#$ to Chatter -
# detailed report will be written automagically to science...
self._intgr_per_image_statistics.update(
job.get_per_image_statistics())
postref_result.update(job.get_postref_result())
# inspect the output for e.g. very high weighted residuals
all_residuals.extend(job.get_residuals())
self._intgr_batches_out = (integrated_images_first,
integrated_images_last)
if mosaics and len(mosaics) > 0:
self.set_integrater_mosaic_min_mean_max(
min(mosaics),
sum(mosaics) / len(mosaics), max(mosaics))
else:
m = indxr.get_indexer_mosaic()
self.set_integrater_mosaic_min_mean_max(m, m, m)
Chatter.write(self.show_per_image_statistics())
Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
self.get_integrater_mosaic_min_mean_max())
# gather the statistics from the postrefinement for all sweeps
# now write this to a postrefinement log
postref_log = os.path.join(self.get_working_directory(),
'postrefinement.log')
fout = open(postref_log, 'w')
fout.write('$TABLE: Postrefinement for %s:\n' % \
self._intgr_sweep_name)
fout.write('$GRAPHS: Missetting angles:A:1, 2, 3, 4: $$\n')
fout.write('Batch PhiX PhiY PhiZ $$ Batch PhiX PhiY PhiZ $$\n')
for image in sorted(postref_result):
phix = postref_result[image].get('phix', 0.0)
phiy = postref_result[image].get('phiy', 0.0)
phiz = postref_result[image].get('phiz', 0.0)
fout.write('%d %5.2f %5.2f %5.2f\n' % \
(image, phix, phiy, phiz))
fout.write('$$\n')
fout.close()
if self.get_integrater_sweep_name():
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file('%s %s %s %s postrefinement' % \
(self.get_integrater_sweep_name(),
pname, xname, dname),
postref_log)
hklouts.sort()
hklout = os.path.join(self.get_working_directory(),
os.path.split(hklouts[0])[-1])
Debug.write('Sorting data to %s' % hklout)
for hklin in hklouts:
Debug.write('<= %s' % hklin)
sortmtz = Sortmtz()
sortmtz.set_hklout(hklout)
for hklin in hklouts:
sortmtz.add_hklin(hklin)
sortmtz.sort()
self._mosflm_hklout = hklout
return self._mosflm_hklout
def _do_indexing(self, method=None):
indexer = self.Index()
for spot_list in self._indxr_payload["spot_lists"]:
indexer.add_spot_filename(spot_list)
for filename in self._indxr_payload["experiments"]:
indexer.add_sweep_filename(filename)
if PhilIndex.params.dials.index.phil_file is not None:
indexer.set_phil_file(PhilIndex.params.dials.index.phil_file)
indexer.set_max_cell(
max_cell=PhilIndex.params.dials.index.max_cell,
max_height_fraction=PhilIndex.params.dials.index.
max_cell_estimation.max_height_fraction,
)
if PhilIndex.params.xia2.settings.small_molecule:
indexer.set_min_cell(3)
if PhilIndex.params.dials.fix_geometry:
indexer.set_detector_fix("all")
indexer.set_beam_fix("all")
elif PhilIndex.params.dials.fix_distance:
indexer.set_detector_fix("distance")
indexer.set_close_to_spindle_cutoff(
PhilIndex.params.dials.close_to_spindle_cutoff)
if self._indxr_input_lattice:
indexer.set_indexer_input_lattice(self._indxr_input_lattice)
logger.debug("Set lattice: %s", self._indxr_input_lattice)
if self._indxr_input_cell:
indexer.set_indexer_input_cell(self._indxr_input_cell)
logger.debug("Set cell: %f %f %f %f %f %f" %
self._indxr_input_cell)
if method is None:
if PhilIndex.params.dials.index.method is None:
method = "fft3d"
logger.debug("Choosing indexing method: %s", method)
else:
method = PhilIndex.params.dials.index.method
FileHandler.record_log_file("%s INDEX" % self.get_indexer_full_name(),
indexer.get_log_file())
indexer.run(method)
if not os.path.exists(indexer.get_experiments_filename()):
raise RuntimeError(
"Indexing has failed: see %s for more details." %
indexer.get_log_file())
elif not os.path.exists(indexer.get_indexed_filename()):
raise RuntimeError("Indexing has failed: %s does not exist." %
indexer.get_indexed_filename())
report = self.Report()
report.set_experiments_filename(indexer.get_experiments_filename())
report.set_reflections_filename(indexer.get_indexed_filename())
html_filename = os.path.join(
self.get_working_directory(),
"%i_dials.index.report.html" % report.get_xpid(),
)
report.set_html_filename(html_filename)
report.run()
FileHandler.record_html_file("%s INDEX" % self.get_indexer_full_name(),
html_filename)
return indexer
def _sort_together_data_ccp4(self):
'''Sort together in the right order (rebatching as we go) the sweeps
we want to scale together.'''
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
hklin = si.get_reflections()
# limit the reflections - e.g. if we are re-running the scaling step
# on just a subset of the integrated data
hklin = si.get_reflections()
limit_batch_range = None
for sweep in PhilIndex.params.xia2.settings.sweep:
if sweep.id == sname and sweep.range is not None:
limit_batch_range = sweep.range
break
if limit_batch_range is not None:
Debug.write('Limiting batch range for %s: %s' %
(sname, limit_batch_range))
start, end = limit_batch_range
hklout = os.path.splitext(hklin)[0] + '_tmp.mtz'
FileHandler.record_temporary_file(hklout)
rb = self._factory.Pointless()
rb.set_hklin(hklin)
rb.set_hklout(hklout)
rb.limit_batches(start, end)
si.set_reflections(hklout)
si.set_batches(limit_batch_range)
# keep a count of the maximum number of batches in a block -
# this will be used to make rebatch work below.
hklin = si.get_reflections()
batches = MtzUtils.batches_from_mtz(hklin)
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
# then rebatch the files, to make sure that the batch numbers are
# in the same order as the epochs of data collection.
counter = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_integrated.mtz' % \
(pname, xname, dname, sname))
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
from xia2.Modules.Scaler.rebatch import rebatch
new_batches = rebatch(hklin,
hklout,
first_batch=counter * max_batches + 1,
pname=pname,
xname=xname,
dname=dname)
# update the "input information"
si.set_reflections(hklout)
si.set_batches(new_batches)
# update the counter & recycle
counter += 1
s = self._factory.Sortmtz()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(hklout)
for epoch in self._sweep_handler.get_epochs():
s.add_hklin(
self._sweep_handler.get_sweep_information(
epoch).get_reflections())
s.sort()
# verify that the measurements are in the correct setting
# choice for the spacegroup
hklin = hklout
hklout = hklin.replace('sorted.mtz', 'temp.mtz')
if not self.get_scaler_reference_reflection_file():
if PhilIndex.params.xia2.settings.symmetry.program == 'dials':
p = self._factory.dials_symmetry()
else:
p = self._factory.Pointless()
FileHandler.record_log_file('%s %s pointless' % \
(self._scalr_pname,
self._scalr_xname),
p.get_log_file())
if len(self._sweep_handler.get_epochs()) > 1:
p.set_hklin(hklin)
else:
# permit the use of pointless preparation...
epoch = self._sweep_handler.get_epochs()[0]
p.set_hklin(
self._prepare_pointless_hklin(
hklin,
self._sweep_handler.get_sweep_information(
epoch).get_integrater().get_phi_width()))
if self._scalr_input_spacegroup:
Debug.write('Assigning user input spacegroup: %s' % \
self._scalr_input_spacegroup)
p.decide_spacegroup()
spacegroup = p.get_spacegroup()
reindex_operator = p.get_spacegroup_reindex_operator()
Debug.write('Pointless thought %s (reindex as %s)' % \
(spacegroup, reindex_operator))
spacegroup = self._scalr_input_spacegroup
reindex_operator = 'h,k,l'
self._spacegroup_reindex_operator = reindex_operator
else:
p.decide_spacegroup()
spacegroup = p.get_spacegroup()
reindex_operator = p.get_spacegroup_reindex_operator()
self._spacegroup_reindex_operator = clean_reindex_operator(
reindex_operator)
Debug.write('Pointless thought %s (reindex as %s)' % \
(spacegroup, reindex_operator))
if self._scalr_input_spacegroup:
self._scalr_likely_spacegroups = [self._scalr_input_spacegroup]
else:
self._scalr_likely_spacegroups = p.get_likely_spacegroups()
Chatter.write('Likely spacegroups:')
for spag in self._scalr_likely_spacegroups:
Chatter.write('%s' % spag)
Chatter.write(
'Reindexing to first spacegroup setting: %s (%s)' % \
(spacegroup, clean_reindex_operator(reindex_operator)))
else:
spacegroup = MtzUtils.space_group_name_from_mtz(
self.get_scaler_reference_reflection_file())
reindex_operator = 'h,k,l'
self._scalr_likely_spacegroups = [spacegroup]
Debug.write('Assigning spacegroup %s from reference' % \
spacegroup)
# then run reindex to set the correct spacegroup
ri = self._factory.Reindex()
ri.set_hklin(hklin)
ri.set_hklout(hklout)
ri.set_spacegroup(spacegroup)
ri.set_operator(reindex_operator)
ri.reindex()
FileHandler.record_temporary_file(hklout)
# then resort the reflections (one last time!)
s = self._factory.Sortmtz()
temp = hklin
hklin = hklout
hklout = temp
s.add_hklin(hklin)
s.set_hklout(hklout)
s.sort()
# done preparing!
self._prepared_reflections = s.get_hklout()
def _refine(self):
for epoch, idxr in self._refinr_indexers.iteritems():
# decide what images we are going to process, if not already
# specified
#if not self._intgr_wedge:
#images = self.get_matching_images()
#self.set_integrater_wedge(min(images),
#max(images))
#Debug.write('DIALS INTEGRATE PREPARE:')
#Debug.write('Wavelength: %.6f' % self.get_wavelength())
#Debug.write('Distance: %.2f' % self.get_distance())
#if not self._intgr_indexer:
#self.set_integrater_indexer(DialsIndexer())
#self.get_integrater_indexer().set_indexer_sweep(
#self.get_integrater_sweep())
#self._intgr_indexer.set_working_directory(
#self.get_working_directory())
#self._intgr_indexer.setup_from_imageset(self.get_imageset())
#if self.get_frame_wedge():
#wedge = self.get_frame_wedge()
#Debug.write('Propogating wedge limit: %d %d' % wedge)
#self._intgr_indexer.set_frame_wedge(wedge[0], wedge[1],
#apply_offset = False)
## this needs to be set up from the contents of the
## Integrater frame processer - wavelength &c.
#if self.get_beam_centre():
#self._intgr_indexer.set_beam_centre(self.get_beam_centre())
#if self.get_distance():
#self._intgr_indexer.set_distance(self.get_distance())
#if self.get_wavelength():
#self._intgr_indexer.set_wavelength(
#self.get_wavelength())
# get the unit cell from this indexer to initiate processing
# if it is new... and also copy out all of the information for
# the Dials indexer if not...
experiments = idxr.get_indexer_experiment_list()
indexed_experiments = idxr.get_indexer_payload("experiments_filename")
indexed_reflections = idxr.get_indexer_payload("indexed_filename")
if len(experiments) > 1:
xsweeps = idxr._indxr_sweeps
assert len(xsweeps) == len(experiments)
assert len(self._refinr_sweeps) == 1 # don't currently support joint refinement
xsweep = self._refinr_sweeps[0]
i = xsweeps.index(xsweep)
experiments = experiments[i:i+1]
# Extract and output experiment and reflections for current sweep
indexed_experiments = os.path.join(
self.get_working_directory(),
"%s_indexed_experiments.json" %xsweep.get_name())
indexed_reflections = os.path.join(
self.get_working_directory(),
"%s_indexed_reflections.pickle" %xsweep.get_name())
from dxtbx.serialize import dump
dump.experiment_list(experiments, indexed_experiments)
from libtbx import easy_pickle
from scitbx.array_family import flex
reflections = easy_pickle.load(
idxr.get_indexer_payload("indexed_filename"))
sel = reflections['id'] == i
assert sel.count(True) > 0
imageset_id = reflections['imageset_id'].select(sel)
assert imageset_id.all_eq(imageset_id[0])
sel = reflections['imageset_id'] == imageset_id[0]
reflections = reflections.select(sel)
# set indexed reflections to id == 0 and imageset_id == 0
reflections['id'].set_selected(reflections['id'] == i, 0)
reflections['imageset_id'] = flex.int(len(reflections), 0)
easy_pickle.dump(indexed_reflections, reflections)
assert len(experiments.crystals()) == 1 # currently only handle one lattice/sweep
crystal_model = experiments.crystals()[0]
lattice = idxr.get_indexer_lattice()
# check if the lattice was user assigned...
user_assigned = idxr.get_indexer_user_input_lattice()
# XXX check that the indexer is an Dials indexer - if not then
# create one...
# set a low resolution limit (which isn't really used...)
# this should perhaps be done more intelligently from an
# analysis of the spot list or something...?
#if not self.get_integrater_low_resolution():
#dmax = idxr.get_indexer_low_resolution()
#self.set_integrater_low_resolution(dmax)
#Debug.write('Low resolution set to: %s' % \
#self.get_integrater_low_resolution())
## copy the data across
from dxtbx.serialize import load, dump
refiner = self.Refine()
refiner.set_experiments_filename(indexed_experiments)
refiner.set_indexed_filename(indexed_reflections)
# XXX Temporary workaround for dials.refine error for scan_varying
# refinement with smaller wedges
total_phi_range = idxr._indxr_imagesets[0].get_scan().get_oscillation_range()[1]
if total_phi_range < 5: # arbitrary value
refiner.set_scan_varying(False)
elif total_phi_range < 36:
refiner.set_interval_width_degrees(total_phi_range/2)
FileHandler.record_log_file('%s REFINE' % idxr.get_indexer_full_name(),
refiner.get_log_file())
refiner.run()
self._refinr_experiments_filename \
= refiner.get_refined_experiments_filename()
experiments = load.experiment_list(self._refinr_experiments_filename)
self._refinr_indexed_filename = refiner.get_refined_filename()
self.set_refiner_payload("experiments.json", self._refinr_experiments_filename)
self.set_refiner_payload("reflections.pickle", self._refinr_indexed_filename)
# this is the result of the cell refinement
self._refinr_cell = experiments.crystals()[0].get_unit_cell().parameters()
def _scale(self):
"""Perform all of the operations required to deliver the scaled
data."""
sweep_infos = [
self._sweep_handler.get_sweep_information(e)
for e in self._sweep_handler.get_epochs()
]
if self._scalr_corrections:
Journal.block(
"scaling",
self.get_scaler_xcrystal().get_name(),
"Dials",
{
"scaling model": "automatic",
"absorption": self._scalr_correct_absorption,
"decay": self._scalr_correct_decay,
},
)
else:
Journal.block(
"scaling",
self.get_scaler_xcrystal().get_name(),
"Dials",
{"scaling model": "default"},
)
### Set the parameters and datafiles for dials.scale
self._scaler = DialsScale()
self._scaler = self._updated_dials_scaler()
if self._scaled_experiments and self._scaled_reflections:
# going to continue-where-left-off
self._scaler.add_experiments_json(self._scaled_experiments)
self._scaler.add_reflections_file(self._scaled_reflections)
else:
for si in sweep_infos:
self._scaler.add_experiments_json(si.get_experiments())
self._scaler.add_reflections_file(si.get_reflections())
### Set the unmerged mtz filepath
self._scalr_scaled_reflection_files = {}
self._scalr_scaled_reflection_files["mtz_unmerged"] = {}
# First set the unmerged mtz output filename. Note that this is the
# same for MAD datasets too, as need a single unmerged for merging
# stats calc. For the merged mtz this is different.
scaled_unmerged_mtz_path = os.path.join(
self.get_working_directory(),
"%s_%s_scaled_unmerged.mtz" % (self._scalr_pname, self._scalr_xname),
)
self._scaler.set_scaled_unmerged_mtz([scaled_unmerged_mtz_path])
self._scaler.set_crystal_name(self._scalr_xname) # Name goes in mtz
### Set the merged mtz filepath(s), making into account MAD case.
# Find number of dnames (i.e. number of wavelengths)
dnames_set = OrderedSet()
for si in sweep_infos:
dnames_set.add(si.get_project_info()[2])
scaled_mtz_path = os.path.join(
self.get_working_directory(),
"%s_%s_scaled.mtz" % (self._scalr_pname, self._scalr_xname),
)
if len(dnames_set) == 1:
self._scaler.set_scaled_mtz([scaled_mtz_path])
self._scalr_scaled_reflection_files["mtz"] = {
dnames_set[0]: scaled_mtz_path
}
self._scalr_scaled_reflection_files["mtz_unmerged"] = {
dnames_set[0]: scaled_unmerged_mtz_path
}
else:
merged_mtz_files = []
self._scalr_scaled_reflection_files["mtz"] = {}
for dname in dnames_set:
this_mtz_path = scaled_mtz_path.rstrip(".mtz") + ("_%s.mtz" % dname)
merged_mtz_files.append(this_mtz_path)
self._scalr_scaled_reflection_files["mtz"][dname] = scaled_mtz_path
# Note - we aren't logging individual unmerged here as not
# generating until later.
self._scaler.set_scaled_mtz(merged_mtz_files)
### Set the resolution limit if applicable
user_resolution_limits = {}
highest_resolution = 100.0
for si in sweep_infos:
dname = si.get_project_info()[2]
sname = si.get_sweep_name()
intgr = si.get_integrater()
if intgr.get_integrater_user_resolution():
# record user resolution here but don't use it until later - why?
dmin = intgr.get_integrater_high_resolution()
if (dname, sname) not in user_resolution_limits:
user_resolution_limits[(dname, sname)] = dmin
elif dmin < user_resolution_limits[(dname, sname)]:
user_resolution_limits[(dname, sname)] = dmin
if (dname, sname) in self._scalr_resolution_limits:
d_min, _ = self._scalr_resolution_limits[(dname, sname)]
if d_min < highest_resolution:
highest_resolution = d_min
if highest_resolution < 99.9:
self._scaler.set_resolution(d_min=highest_resolution)
### Setup final job details and run scale
self._scaler.set_working_directory(self.get_working_directory())
auto_logfiler(self._scaler)
FileHandler.record_log_file(
"%s %s SCALE" % (self._scalr_pname, self._scalr_xname),
self._scaler.get_log_file(),
)
self._scaler.scale()
self._scaled_experiments = self._scaler.get_scaled_experiments()
self._scaled_reflections = self._scaler.get_scaled_reflections()
FileHandler.record_data_file(scaled_unmerged_mtz_path)
# make it so that only scaled.expt and scaled.refl are
# the files that dials.scale knows about, so that if scale is called again,
# scaling resumes from where it left off.
self._scaler.clear_datafiles()
# log datafiles here, picked up from here in commonscaler methods.
if len(dnames_set) == 1:
hklout = copy.deepcopy(self._scaler.get_scaled_mtz()[0])
self._scalr_scaled_refl_files = {dnames_set[0]: hklout}
FileHandler.record_data_file(hklout)
else:
self._scalr_scaled_refl_files = {}
for i, dname in enumerate(dnames_set):
hklout = copy.deepcopy(self._scaler.get_scaled_mtz()[i])
self._scalr_scaled_refl_files[dname] = hklout
FileHandler.record_data_file(hklout)
### Calculate the resolution limit and set done False if applicable
highest_suggested_resolution = self.assess_resolution_limits(
self._scaler.get_unmerged_reflection_file(),
user_resolution_limits,
use_misigma=False,
)
if not self.get_scaler_done():
# reset for when resolution limit applied
Debug.write("Returning as scaling not finished...")
return
### For MAD case, generate individual unmerged mtz for stats.
if len(dnames_set) > 1:
unmerged_mtz_files = []
scaler = DialsScale()
scaler.set_working_directory(self.get_working_directory())
scaler.set_export_mtz_only()
scaler.add_experiments_json(self._scaled_experiments)
scaler.add_reflections_file(self._scaled_reflections)
for dname in dnames_set:
this_mtz_path = scaled_unmerged_mtz_path.rstrip(".mtz") + (
"_%s.mtz" % dname
)
unmerged_mtz_files.append(this_mtz_path)
self._scalr_scaled_reflection_files["mtz_unmerged"][
dname
] = this_mtz_path
scaler.set_scaled_unmerged_mtz(unmerged_mtz_files)
scaler.scale()
for f in scaler.get_scaled_unmerged_mtz(): # a list
FileHandler.record_data_file(f)
# set refls, exps & unmerged mtz names"
if PhilIndex.params.xia2.settings.merging_statistics.source == "cctbx":
for key in self._scalr_scaled_refl_files:
stats = self._compute_scaler_statistics(
self._scalr_scaled_reflection_files["mtz_unmerged"][key],
selected_band=(highest_suggested_resolution, None),
wave=key,
)
self._scalr_statistics[
(self._scalr_pname, self._scalr_xname, key)
] = stats
# Run twotheta refine
self._update_scaled_unit_cell()
def _integrate_finish(self):
"""Finish off the integration by running correct."""
# first run the postrefinement etc with spacegroup P1
# and the current unit cell - this will be used to
# obtain a benchmark rmsd in pixels / phi and also
# cell deviations (this is working towards spotting bad
# indexing solutions) - only do this if we have no
# reindex matrix... and no postrefined cell...
p1_deviations = None
# fix for bug # 3264 -
# if we have not run integration with refined parameters, make it so...
# erm? shouldn't this therefore return if this is the principle, or
# set the flag after we have tested the lattice?
if ("GXPARM.XDS" not in self._xds_data_files
and PhilIndex.params.xds.integrate.reintegrate):
logger.debug(
"Resetting integrater, to ensure refined orientation is used")
self.set_integrater_done(False)
if (not self.get_integrater_reindex_matrix() and not self._intgr_cell
and PhilIndex.params.xia2.settings.lattice_rejection
and not self.get_integrater_sweep().get_user_lattice()):
correct = self.Correct()
correct.set_data_range(
self._intgr_wedge[0] + self.get_frame_offset(),
self._intgr_wedge[1] + self.get_frame_offset(),
)
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# FIXME should this be using the correctly transformed
# cell or are the results ok without it?!
correct.set_spacegroup_number(1)
correct.set_cell(self._intgr_refiner_cell)
correct.run()
cell = correct.get_result("cell")
cell_esd = correct.get_result("cell_esd")
logger.debug("Postrefinement in P1 results:")
logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" % tuple(cell))
logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
tuple(cell_esd))
logger.debug("Deviations: %.2f pixels %.2f degrees" %
(correct.get_result("rmsd_pixel"),
correct.get_result("rmsd_phi")))
p1_deviations = (
correct.get_result("rmsd_pixel"),
correct.get_result("rmsd_phi"),
)
# next run the postrefinement etc with the given
# cell / lattice - this will be the assumed result...
integrate_hkl = os.path.join(self.get_working_directory(),
"INTEGRATE.HKL")
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
from dxtbx.serialize import load
from dials.algorithms.shadowing.filter import filter_shadowed_reflections
experiments_json = xparm_xds_to_experiments_json(
os.path.join(self.get_working_directory(), "XPARM.XDS"),
self.get_working_directory(),
)
experiments = load.experiment_list(experiments_json,
check_format=True)
imageset = experiments[0].imageset
masker = (imageset.get_format_class().get_instance(
imageset.paths()[0]).get_masker())
if masker is not None:
integrate_filename = integrate_hkl_to_reflection_file(
integrate_hkl, experiments_json,
self.get_working_directory())
reflections = flex.reflection_table.from_file(
integrate_filename)
t0 = time.time()
sel = filter_shadowed_reflections(experiments, reflections)
shadowed = reflections.select(sel)
t1 = time.time()
logger.debug("Filtered %i reflections in %.1f seconds" %
(sel.count(True), t1 - t0))
filter_hkl = os.path.join(self.get_working_directory(),
"FILTER.HKL")
with open(filter_hkl, "wb") as f:
detector = experiments[0].detector
for ref in shadowed:
p = detector[ref["panel"]]
ox, oy = p.get_raw_image_offset()
h, k, l = ref["miller_index"]
x, y, z = ref["xyzcal.px"]
dx, dy, dz = (2, 2, 2)
print(
"%i %i %i %.1f %.1f %.1f %.1f %.1f %.1f" %
(h, k, l, x + ox, y + oy, z, dx, dy, dz),
file=f,
)
t2 = time.time()
logger.debug("Written FILTER.HKL in %.1f seconds" % (t2 - t1))
correct = self.Correct()
correct.set_data_range(
self._intgr_wedge[0] + self.get_frame_offset(),
self._intgr_wedge[1] + self.get_frame_offset(),
)
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# BUG # 2695 probably comes from here - need to check...
# if the pointless interface comes back with a different
# crystal setting then the unit cell stored in self._intgr_cell
# needs to be set to None...
if self.get_integrater_spacegroup_number():
correct.set_spacegroup_number(
self.get_integrater_spacegroup_number())
if not self._intgr_cell:
raise RuntimeError("no unit cell to recycle")
correct.set_cell(self._intgr_cell)
# BUG # 3113 - new version of XDS will try and figure the
# best spacegroup out from the intensities (and get it wrong!)
# unless we set the spacegroup and cell explicitly
if not self.get_integrater_spacegroup_number():
cell = self._intgr_refiner_cell
lattice = self._intgr_refiner.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup_number(lattice)
# this should not prevent the postrefinement from
# working correctly, else what is above would not
# work correctly (the postrefinement test)
correct.set_spacegroup_number(spacegroup_number)
correct.set_cell(cell)
logger.debug("Setting spacegroup to: %d" % spacegroup_number)
logger.debug("Setting cell to: %.2f %.2f %.2f %.2f %.2f %.2f" %
tuple(cell))
if self.get_integrater_reindex_matrix():
# bug! if the lattice is not primitive the values in this
# reindex matrix need to be multiplied by a constant which
# depends on the Bravais lattice centering.
lattice = self._intgr_refiner.get_refiner_lattice()
matrix = self.get_integrater_reindex_matrix()
matrix = scitbx.matrix.sqr(matrix).transpose().elems
matrix = r_to_rt(matrix)
if lattice[1] == "P":
mult = 1
elif lattice[1] == "C" or lattice[1] == "I":
mult = 2
elif lattice[1] == "R":
mult = 3
elif lattice[1] == "F":
mult = 4
else:
raise RuntimeError("unknown multiplier for lattice %s" %
lattice)
logger.debug("REIDX multiplier for lattice %s: %d" %
(lattice, mult))
mult_matrix = [mult * m for m in matrix]
logger.debug("REIDX set to %d %d %d %d %d %d %d %d %d %d %d %d" %
tuple(mult_matrix))
correct.set_reindex_matrix(mult_matrix)
correct.run()
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file(
f"{pname} {xname} {dname} {sweep} CORRECT",
os.path.join(self.get_working_directory(), "CORRECT.LP"),
)
FileHandler.record_more_data_file(
f"{pname} {xname} {dname} {sweep} CORRECT",
os.path.join(self.get_working_directory(), "XDS_ASCII.HKL"),
)
# erm. just to be sure
if self.get_integrater_reindex_matrix() and correct.get_reindex_used():
raise RuntimeError("Reindex panic!")
# get the reindex operation used, which may be useful if none was
# set but XDS decided to apply one, e.g. #419.
if not self.get_integrater_reindex_matrix(
) and correct.get_reindex_used():
# convert this reindex operation to h, k, l form: n.b. this
# will involve dividing through by the lattice centring multiplier
matrix = rt_to_r(correct.get_reindex_used())
matrix = scitbx.matrix.sqr(matrix).transpose().elems
lattice = self._intgr_refiner.get_refiner_lattice()
if lattice[1] == "P":
mult = 1.0
elif lattice[1] == "C" or lattice[1] == "I":
mult = 2.0
elif lattice[1] == "R":
mult = 3.0
elif lattice[1] == "F":
mult = 4.0
matrix = [m / mult for m in matrix]
reindex_op = mat_to_symop(matrix)
# assign this to self: will this reset?! make for a leaky
# abstraction and just assign this...
# self.set_integrater_reindex_operator(reindex)
self._intgr_reindex_operator = reindex_op
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file(
f"{pname} {xname} {dname} {sweep} CORRECT",
os.path.join(self.get_working_directory(), "CORRECT.LP"),
)
# should get some interesting stuff from the XDS correct file
# here, for instance the resolution range to use in integration
# (which should be fed back if not fast) and so on...
self._intgr_corrected_hklout = os.path.join(
self.get_working_directory(), "XDS_ASCII.HKL")
# also record the batch range - needed for the analysis of the
# radiation damage in chef...
self._intgr_batches_out = (self._intgr_wedge[0], self._intgr_wedge[1])
# FIXME perhaps I should also feedback the GXPARM file here??
for file in ["GXPARM.XDS"]:
self._xds_data_files[file] = correct.get_output_data_file(file)
# record the postrefined cell parameters
self._intgr_cell = correct.get_result("cell")
self._intgr_n_ref = correct.get_result("n_ref")
logger.debug('Postrefinement in "correct" spacegroup results:')
logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
tuple(correct.get_result("cell")))
logger.debug("%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f" %
tuple(correct.get_result("cell_esd")))
logger.debug(
"Deviations: %.2f pixels %.2f degrees" %
(correct.get_result("rmsd_pixel"), correct.get_result("rmsd_phi")))
logger.debug("Error correction parameters: A=%.3f B=%.3f" %
correct.get_result("sdcorrection"))
# compute misorientation of axes
xparm_file = os.path.join(self.get_working_directory(), "GXPARM.XDS")
handle = xparm.reader()
handle.read_file(xparm_file)
rotn = handle.rotation_axis
beam = handle.beam_vector
dot = sum(rotn[j] * beam[j] for j in range(3))
r = math.sqrt(sum(rotn[j] * rotn[j] for j in range(3)))
b = math.sqrt(sum(beam[j] * beam[j] for j in range(3)))
rtod = 180.0 / math.pi
angle = rtod * math.fabs(0.5 * math.pi - math.acos(dot / (r * b)))
logger.debug("Axis misalignment %.2f degrees" % angle)
correct_deviations = (
correct.get_result("rmsd_pixel"),
correct.get_result("rmsd_phi"),
)
if p1_deviations:
# compare and reject if both > 50% higher - though adding a little
# flexibility - 0.5 pixel / osc width slack.
pixel = p1_deviations[0]
phi = math.sqrt(0.05 * 0.05 + p1_deviations[1] * p1_deviations[1])
threshold = PhilIndex.params.xia2.settings.lattice_rejection_threshold
logger.debug("RMSD ratio: %.2f" % (correct_deviations[0] / pixel))
logger.debug("RMSPhi ratio: %.2f" % (correct_deviations[1] / phi))
if (correct_deviations[0] / pixel > threshold
and correct_deviations[1] / phi > threshold):
logger.info(
"Eliminating this indexing solution as postrefinement")
logger.info("deviations rather high relative to triclinic")
raise BadLatticeError(
"high relative deviations in postrefinement")
if (not PhilIndex.params.dials.fast_mode
and not PhilIndex.params.xds.keep_outliers):
# check for alien reflections and perhaps recycle - removing them
correct_remove = correct.get_remove()
if correct_remove:
current_remove = set()
final_remove = []
# first ensure that there are no duplicate entries...
if os.path.exists(
os.path.join(self.get_working_directory(),
"REMOVE.HKL")):
with open(
os.path.join(self.get_working_directory(),
"REMOVE.HKL")) as fh:
for line in fh.readlines():
h, k, l = list(map(int, line.split()[:3]))
z = float(line.split()[3])
if (h, k, l, z) not in current_remove:
current_remove.add((h, k, l, z))
for c in correct_remove:
if c in current_remove:
continue
final_remove.append(c)
logger.debug("%d alien reflections are already removed" %
(len(correct_remove) - len(final_remove)))
else:
# we want to remove all of the new dodgy reflections
final_remove = correct_remove
z_min = PhilIndex.params.xds.z_min
rejected = 0
with open(
os.path.join(self.get_working_directory(),
"REMOVE.HKL"), "w") as remove_hkl:
# write in the old reflections
for remove in current_remove:
z = remove[3]
if z >= z_min:
remove_hkl.write("%d %d %d %f\n" % remove)
else:
rejected += 1
logger.debug("Wrote %d old reflections to REMOVE.HKL" %
(len(current_remove) - rejected))
logger.debug("Rejected %d as z < %f" % (rejected, z_min))
# and the new reflections
rejected = 0
used = 0
for remove in final_remove:
z = remove[3]
if z >= z_min:
used += 1
remove_hkl.write("%d %d %d %f\n" % remove)
else:
rejected += 1
logger.debug("Wrote %d new reflections to REMOVE.HKL" %
(len(final_remove) - rejected))
logger.debug("Rejected %d as z < %f" % (rejected, z_min))
# we want to rerun the finishing step so...
# unless we have added no new reflections... or unless we
# have not confirmed the point group (see SCI-398)
if used and self.get_integrater_reindex_matrix():
self.set_integrater_finish_done(False)
else:
logger.debug(
"Going quickly so not removing %d outlier reflections..." %
len(correct.get_remove()))
# Convert INTEGRATE.HKL to MTZ format and reapply any reindexing operations
# spacegroup changes to allow use with CCP4 / Aimless for scaling
hklout = os.path.splitext(integrate_hkl)[0] + ".mtz"
self._factory.set_working_directory(self.get_working_directory())
pointless = self._factory.Pointless()
pointless.set_xdsin(integrate_hkl)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
integrate_mtz = hklout
if (self.get_integrater_reindex_operator()
or self.get_integrater_spacegroup_number()):
logger.debug("Reindexing things to MTZ")
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
if self.get_integrater_reindex_operator():
reindex.set_operator(self.get_integrater_reindex_operator())
if self.get_integrater_spacegroup_number():
reindex.set_spacegroup(self.get_integrater_spacegroup_number())
hklout = "%s_reindex.mtz" % os.path.splitext(integrate_mtz)[0]
reindex.set_hklin(integrate_mtz)
reindex.set_hklout(hklout)
reindex.reindex()
integrate_mtz = hklout
experiments_json = xparm_xds_to_experiments_json(
self._xds_data_files["GXPARM.XDS"], self.get_working_directory())
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(f"{pname} {xname} {dname} {sweep}",
experiments_json)
FileHandler.record_more_data_file(
f"{pname} {xname} {dname} {sweep} INTEGRATE", integrate_mtz)
self._intgr_experiments_filename = experiments_json
return integrate_mtz
def _integrate(self):
'''Actually do the integration - in XDS terms this will mean running
DEFPIX and INTEGRATE to measure all the reflections.'''
experiment = self._intgr_refiner.get_refined_experiment_list(
self.get_integrater_epoch())[0]
crystal_model = experiment.crystal
self._intgr_refiner_cell = crystal_model.get_unit_cell().parameters()
images_str = '%d to %d' % tuple(self._intgr_wedge)
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % tuple(
self._intgr_refiner_cell)
if len(self._fp_directory) <= 50:
dirname = self._fp_directory
else:
dirname = '...%s' % self._fp_directory[-46:]
Journal.block(
'integrating', self._intgr_sweep_name, 'XDS', {
'images': images_str,
'cell': cell_str,
'lattice': self._intgr_refiner.get_refiner_lattice(),
'template': self._fp_template,
'directory': dirname,
'resolution': '%.2f' % self._intgr_reso_high
})
defpix = self.Defpix()
# pass in the correct data
for file in [
'X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf', 'BKGINIT.cbf',
'XPARM.XDS'
]:
defpix.set_input_data_file(file, self._xds_data_files[file])
defpix.set_data_range(self._intgr_wedge[0] + self.get_frame_offset(),
self._intgr_wedge[1] + self.get_frame_offset())
if self.get_integrater_high_resolution() > 0.0 and \
self.get_integrater_user_resolution():
Debug.write('Setting resolution limit in DEFPIX to %.2f' % \
self.get_integrater_high_resolution())
defpix.set_resolution_high(self.get_integrater_high_resolution())
defpix.set_resolution_low(self.get_integrater_low_resolution())
elif self.get_integrater_low_resolution():
Debug.write('Setting low resolution limit in DEFPIX to %.2f' % \
self.get_integrater_low_resolution())
defpix.set_resolution_high(0.0)
defpix.set_resolution_low(self.get_integrater_low_resolution())
defpix.run()
# and gather the result files
for file in ['BKGPIX.cbf', 'ABS.cbf']:
self._xds_data_files[file] = defpix.get_output_data_file(file)
integrate = self.Integrate()
if self._xds_integrate_parameters:
integrate.set_updates(self._xds_integrate_parameters)
# decide what images we are going to process, if not already
# specified
if not self._intgr_wedge:
images = self.get_matching_images()
self.set_integrater_wedge(min(images), max(images))
integrate.set_data_range(
self._intgr_wedge[0] + self.get_frame_offset(),
self._intgr_wedge[1] + self.get_frame_offset())
for file in [
'X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf', 'BLANK.cbf',
'BKGPIX.cbf', 'GAIN.cbf'
]:
integrate.set_input_data_file(file, self._xds_data_files[file])
if 'GXPARM.XDS' in self._xds_data_files:
Debug.write('Using globally refined parameters')
integrate.set_input_data_file('XPARM.XDS',
self._xds_data_files['GXPARM.XDS'])
integrate.set_refined_xparm()
else:
integrate.set_input_data_file('XPARM.XDS',
self._xds_data_files['XPARM.XDS'])
integrate.run()
self._intgr_per_image_statistics = integrate.get_per_image_statistics()
Chatter.write(self.show_per_image_statistics())
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file('%s %s %s %s INTEGRATE' % \
(pname, xname, dname, sweep),
os.path.join(self.get_working_directory(),
'INTEGRATE.LP'))
# and copy the first pass INTEGRATE.HKL...
lattice = self._intgr_refiner.get_refiner_lattice()
if not os.path.exists(
os.path.join(self.get_working_directory(),
'INTEGRATE-%s.HKL' % lattice)):
here = self.get_working_directory()
shutil.copyfile(os.path.join(here, 'INTEGRATE.HKL'),
os.path.join(here, 'INTEGRATE-%s.HKL' % lattice))
# record INTEGRATE.HKL for e.g. BLEND.
FileHandler.record_more_data_file(
'%s %s %s %s INTEGRATE' % (pname, xname, dname, sweep),
os.path.join(self.get_working_directory(), 'INTEGRATE.HKL'))
# should the existence of these require that I rerun the
# integration or can we assume that the application of a
# sensible resolution limit will achieve this??
self._xds_integrate_parameters = integrate.get_updates()
# record the mosaic spread &c.
m_min, m_mean, m_max = integrate.get_mosaic()
self.set_integrater_mosaic_min_mean_max(m_min, m_mean, m_max)
Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
self.get_integrater_mosaic_min_mean_max())
return os.path.join(self.get_working_directory(), 'INTEGRATE.HKL')
def _sort_together_data_xds_one_sweep(self):
assert(len(self._sweep_information) == 1)
epoch = self._sweep_information.keys()[0]
hklin = self._sweep_information[epoch]['scaled_reflections']
if Flags.get_chef():
self._sweep_information_to_chef()
if self.get_scaler_reference_reflection_file():
md = self._factory.Mtzdump()
md.set_hklin(self.get_scaler_reference_reflection_file())
md.dump()
spacegroups = [md.get_spacegroup()]
reindex_operator = 'h,k,l'
elif self._scalr_input_spacegroup:
Debug.write('Assigning user input spacegroup: %s' % \
self._scalr_input_spacegroup)
spacegroups = [self._scalr_input_spacegroup]
reindex_operator = 'h,k,l'
else:
pointless = self._factory.Pointless()
pointless.set_hklin(hklin)
pointless.decide_spacegroup()
FileHandler.record_log_file('%s %s pointless' % \
(self._scalr_pname,
self._scalr_xname),
pointless.get_log_file())
spacegroups = pointless.get_likely_spacegroups()
reindex_operator = pointless.get_spacegroup_reindex_operator()
self._scalr_likely_spacegroups = spacegroups
spacegroup = self._scalr_likely_spacegroups[0]
self._scalr_reindex_operator = reindex_operator
Chatter.write('Likely spacegroups:')
for spag in self._scalr_likely_spacegroups:
Chatter.write('%s' % spag)
Chatter.write(
'Reindexing to first spacegroup setting: %s (%s)' % \
(spacegroup, clean_reindex_operator(reindex_operator)))
hklout = os.path.join(self.get_working_directory(),
'%s_%s_reindex.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(hklout)
if reindex_operator == '[h,k,l]':
# just assign spacegroup
from iotbx import mtz
from cctbx import sgtbx
s = sgtbx.space_group(sgtbx.space_group_symbols(
str(spacegroup)).hall())
m = mtz.object(hklin)
m.set_space_group(s).write(hklout)
self._scalr_cell = m.crystals()[-1].unit_cell().parameters()
Debug.write(
'Updating unit cell to %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(self._scalr_cell))
del(m)
del(s)
else:
ri = self._factory.Reindex()
ri.set_hklin(hklin)
ri.set_hklout(hklout)
ri.set_spacegroup(spacegroup)
ri.set_operator(reindex_operator)
ri.reindex()
Debug.write(
'Updating unit cell to %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(ri.get_cell()))
self._scalr_cell = tuple(ri.get_cell())
hklin = hklout
hklout = os.path.join(self.get_working_directory(),
'%s_%s_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s = self._factory.Sortmtz()
s.set_hklin(hklin)
s.set_hklout(hklout)
s.sort(vrset = -99999999.0)
self._prepared_reflections = hklout
return
def _sort_together_data_ccp4(self):
'''Sort together in the right order (rebatching as we go) the sweeps
we want to scale together.'''
max_batches = 0
for e in self._sweep_handler.get_epochs():
if Flags.get_small_molecule():
continue
si = self._sweep_handler.get_sweep_information(e)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
# limit the reflections - e.g. if we are re-running the scaling step
# on just a subset of the integrated data
hklin = si.get_reflections()
limit_batch_range = None
for sweep in PhilIndex.params.xia2.settings.sweep:
if sweep.id == sname and sweep.range is not None:
limit_batch_range = sweep.range
break
if limit_batch_range is not None:
Debug.write('Limiting batch range for %s: %s' %(sname, limit_batch_range))
start, end = limit_batch_range
hklout = os.path.splitext(hklin)[0] + '_tmp.mtz'
FileHandler.record_temporary_file(hklout)
rb = self._factory.Pointless()
rb.set_hklin(hklin)
rb.set_hklout(hklout)
rb.limit_batches(start, end)
si.set_reflections(hklout)
si.set_batches(limit_batch_range)
# keep a count of the maximum number of batches in a block -
# this will be used to make rebatch work below.
hklin = si.get_reflections()
md = self._factory.Mtzdump()
md.set_hklin(hklin)
md.dump()
batches = md.get_batches()
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
datasets = md.get_datasets()
Debug.write('In reflection file %s found:' % hklin)
for d in datasets:
Debug.write('... %s' % d)
dataset_info = md.get_dataset_info(datasets[0])
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
# then rebatch the files, to make sure that the batch numbers are
# in the same order as the epochs of data collection.
counter = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
rb = self._factory.Rebatch()
hklin = si.get_reflections()
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_integrated.mtz' % \
(pname, xname, dname, sname))
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
rb.set_hklin(hklin)
rb.set_first_batch(counter * max_batches + 1)
rb.set_project_info(pname, xname, dname)
rb.set_hklout(hklout)
new_batches = rb.rebatch()
# update the "input information"
si.set_reflections(hklout)
si.set_batches(new_batches)
# update the counter & recycle
counter += 1
s = self._factory.Sortmtz()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(hklout)
for epoch in self._sweep_handler.get_epochs():
s.add_hklin(self._sweep_handler.get_sweep_information(
epoch).get_reflections())
s.sort()
# verify that the measurements are in the correct setting
# choice for the spacegroup
hklin = hklout
hklout = hklin.replace('sorted.mtz', 'temp.mtz')
if not self.get_scaler_reference_reflection_file():
p = self._factory.Pointless()
FileHandler.record_log_file('%s %s pointless' % \
(self._scalr_pname,
self._scalr_xname),
p.get_log_file())
if len(self._sweep_handler.get_epochs()) > 1:
p.set_hklin(hklin)
else:
# permit the use of pointless preparation...
epoch = self._sweep_handler.get_epochs()[0]
p.set_hklin(self._prepare_pointless_hklin(
hklin, self._sweep_handler.get_sweep_information(
epoch).get_integrater().get_phi_width()))
if self._scalr_input_spacegroup:
Debug.write('Assigning user input spacegroup: %s' % \
self._scalr_input_spacegroup)
p.decide_spacegroup()
spacegroup = p.get_spacegroup()
reindex_operator = p.get_spacegroup_reindex_operator()
Debug.write('Pointless thought %s (reindex as %s)' % \
(spacegroup, reindex_operator))
spacegroup = self._scalr_input_spacegroup
reindex_operator = 'h,k,l'
elif Flags.get_small_molecule() and False:
p.decide_pointgroup()
spacegroup = p.get_pointgroup()
reindex_operator = p.get_reindex_operator()
Debug.write('Pointless thought %s (reindex as %s)' % \
(spacegroup, reindex_operator))
self._scalr_likely_spacegroups = [spacegroup]
else:
p.decide_spacegroup()
spacegroup = p.get_spacegroup()
reindex_operator = p.get_spacegroup_reindex_operator()
Debug.write('Pointless thought %s (reindex as %s)' % \
(spacegroup, reindex_operator))
if self._scalr_input_spacegroup:
self._scalr_likely_spacegroups = [self._scalr_input_spacegroup]
else:
self._scalr_likely_spacegroups = p.get_likely_spacegroups()
Chatter.write('Likely spacegroups:')
for spag in self._scalr_likely_spacegroups:
Chatter.write('%s' % spag)
Chatter.write(
'Reindexing to first spacegroup setting: %s (%s)' % \
(spacegroup, clean_reindex_operator(reindex_operator)))
else:
md = self._factory.Mtzdump()
md.set_hklin(self.get_scaler_reference_reflection_file())
md.dump()
spacegroup = md.get_spacegroup()
reindex_operator = 'h,k,l'
self._scalr_likely_spacegroups = [spacegroup]
Debug.write('Assigning spacegroup %s from reference' % \
spacegroup)
# then run reindex to set the correct spacegroup
ri = self._factory.Reindex()
ri.set_hklin(hklin)
ri.set_hklout(hklout)
ri.set_spacegroup(spacegroup)
ri.set_operator(reindex_operator)
ri.reindex()
FileHandler.record_temporary_file(hklout)
# then resort the reflections (one last time!)
s = self._factory.Sortmtz()
temp = hklin
hklin = hklout
hklout = temp
s.add_hklin(hklin)
s.set_hklout(hklout)
s.sort()
# done preparing!
self._prepared_reflections = s.get_hklout()
return
def _sort_together_data_xds(self):
if len(self._sweep_information) == 1:
return self._sort_together_data_xds_one_sweep()
max_batches = 0
for epoch in self._sweep_information.keys():
hklin = self._sweep_information[epoch]['scaled_reflections']
md = self._factory.Mtzdump()
md.set_hklin(hklin)
md.dump()
if self._sweep_information[epoch]['batches'] == [0, 0]:
Chatter.write('Getting batches from %s' % hklin)
batches = md.get_batches()
self._sweep_information[epoch]['batches'] = [min(batches),
max(batches)]
Chatter.write('=> %d to %d' % (min(batches),
max(batches)))
batches = self._sweep_information[epoch]['batches']
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
datasets = md.get_datasets()
Debug.write('In reflection file %s found:' % hklin)
for d in datasets:
Debug.write('... %s' % d)
dataset_info = md.get_dataset_info(datasets[0])
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
epochs = self._sweep_information.keys()
epochs.sort()
counter = 0
for epoch in epochs:
rb = self._factory.Rebatch()
hklin = self._sweep_information[epoch]['scaled_reflections']
pname = self._sweep_information[epoch]['pname']
xname = self._sweep_information[epoch]['xname']
dname = self._sweep_information[epoch]['dname']
sname = self._sweep_information[epoch]['sname']
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%d.mtz' % \
(pname, xname, dname, counter))
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
# record this for future reference - will be needed in the
# radiation damage analysis...
# hack - reset this as it gets in a muddle...
intgr = self._sweep_information[epoch]['integrater']
self._sweep_information[epoch][
'batches'] = intgr.get_integrater_batches()
first_batch = min(self._sweep_information[epoch]['batches'])
self._sweep_information[epoch][
'batch_offset'] = counter * max_batches - first_batch + 1
rb.set_hklin(hklin)
rb.set_first_batch(counter * max_batches + 1)
rb.set_hklout(hklout)
new_batches = rb.rebatch()
# update the "input information"
self._sweep_information[epoch]['hklin'] = hklout
self._sweep_information[epoch]['batches'] = new_batches
# update the counter & recycle
counter += 1
if Flags.get_chef():
self._sweep_information_to_chef()
s = self._factory.Sortmtz()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(hklout)
for epoch in epochs:
s.add_hklin(self._sweep_information[epoch]['hklin'])
s.sort(vrset = -99999999.0)
self._prepared_reflections = hklout
if self.get_scaler_reference_reflection_file():
md = self._factory.Mtzdump()
md.set_hklin(self.get_scaler_reference_reflection_file())
md.dump()
spacegroups = [md.get_spacegroup()]
reindex_operator = 'h,k,l'
else:
pointless = self._factory.Pointless()
pointless.set_hklin(hklout)
pointless.decide_spacegroup()
FileHandler.record_log_file('%s %s pointless' % \
(self._scalr_pname,
self._scalr_xname),
pointless.get_log_file())
spacegroups = pointless.get_likely_spacegroups()
reindex_operator = pointless.get_spacegroup_reindex_operator()
if self._scalr_input_spacegroup:
Debug.write('Assigning user input spacegroup: %s' % \
self._scalr_input_spacegroup)
spacegroups = [self._scalr_input_spacegroup]
reindex_operator = 'h,k,l'
self._scalr_likely_spacegroups = spacegroups
spacegroup = self._scalr_likely_spacegroups[0]
self._scalr_reindex_operator = reindex_operator
Chatter.write('Likely spacegroups:')
for spag in self._scalr_likely_spacegroups:
Chatter.write('%s' % spag)
Chatter.write(
'Reindexing to first spacegroup setting: %s (%s)' % \
(spacegroup, clean_reindex_operator(reindex_operator)))
hklin = self._prepared_reflections
hklout = os.path.join(self.get_working_directory(),
'%s_%s_reindex.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(hklout)
ri = self._factory.Reindex()
ri.set_hklin(hklin)
ri.set_hklout(hklout)
ri.set_spacegroup(spacegroup)
ri.set_operator(reindex_operator)
ri.reindex()
hklin = hklout
hklout = os.path.join(self.get_working_directory(),
'%s_%s_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s = self._factory.Sortmtz()
s.set_hklin(hklin)
s.set_hklout(hklout)
s.sort(vrset = -99999999.0)
self._prepared_reflections = hklout
Debug.write(
'Updating unit cell to %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(ri.get_cell()))
self._scalr_cell = tuple(ri.get_cell())
return
def _integrate(self):
"""Actually do the integration - in XDS terms this will mean running
DEFPIX and INTEGRATE to measure all the reflections."""
images_str = "%d to %d" % tuple(self._intgr_wedge)
cell_str = "%.2f %.2f %.2f %.2f %.2f %.2f" % tuple(self._intgr_cell)
if len(self._fp_directory) <= 50:
dirname = self._fp_directory
else:
dirname = "...%s" % self._fp_directory[-46:]
Journal.block(
"integrating",
self._intgr_sweep_name,
"DIALS",
{
"images": images_str,
"cell": cell_str,
"lattice": self.get_integrater_refiner().get_refiner_lattice(),
"template": self._fp_template,
"directory": dirname,
"resolution": "%.2f" % self._intgr_reso_high,
},
)
integrate = self.Integrate()
# decide what images we are going to process, if not already
# specified
if not self._intgr_wedge:
images = self.get_matching_images()
self.set_integrater_wedge(min(images), max(images))
imageset = self.get_imageset()
beam = imageset.get_beam()
detector = imageset.get_detector()
d_min_limit = detector.get_max_resolution(beam.get_s0())
if (d_min_limit > self._intgr_reso_high or
PhilIndex.params.xia2.settings.resolution.keep_all_reflections
):
Debug.write("Overriding high resolution limit: %f => %f" %
(self._intgr_reso_high, d_min_limit))
self._intgr_reso_high = d_min_limit
integrate.set_experiments_filename(self._intgr_experiments_filename)
integrate.set_reflections_filename(self._intgr_indexed_filename)
if PhilIndex.params.dials.integrate.d_max:
integrate.set_d_max(PhilIndex.params.dials.integrate.d_max)
else:
integrate.set_d_max(self._intgr_reso_low)
if PhilIndex.params.dials.integrate.d_min:
integrate.set_d_min(PhilIndex.params.dials.integrate.d_min)
else:
integrate.set_d_min(self._intgr_reso_high)
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file(
"%s %s %s %s INTEGRATE" % (pname, xname, dname, sweep),
integrate.get_log_file(),
)
try:
integrate.run()
except xia2.Wrappers.Dials.Integrate.DIALSIntegrateError as e:
s = str(e)
if ("dials.integrate requires more memory than is available." in s
and not self._intgr_reso_high):
# Try to estimate a more sensible resolution limit for integration
# in case we were just integrating noise to the edge of the detector
images = self._integrate_select_images_wedges()
Debug.write(
"Integrating subset of images to estimate resolution limit.\n"
"Integrating images %s" % images)
integrate = self.Integrate()
integrate.set_experiments_filename(
self._intgr_experiments_filename)
integrate.set_reflections_filename(
self._intgr_indexed_filename)
integrate.set_d_max(self._intgr_reso_low)
integrate.set_d_min(self._intgr_reso_high)
for (start, stop) in images:
integrate.add_scan_range(
start - self.get_matching_images()[0],
stop - self.get_matching_images()[0],
)
integrate.set_reflections_per_degree(1000)
integrate.run()
integrated_reflections = integrate.get_integrated_filename()
from xia2.Wrappers.Dials.EstimateResolutionLimit import (
EstimateResolutionLimit, )
d_min_estimater = EstimateResolutionLimit()
d_min_estimater.set_working_directory(
self.get_working_directory())
auto_logfiler(d_min_estimater)
d_min_estimater.set_experiments_filename(
self._intgr_experiments_filename)
d_min_estimater.set_reflections_filename(
integrated_reflections)
d_min = d_min_estimater.run()
Debug.write("Estimate for d_min: %.2f" % d_min)
Debug.write("Re-running integration to this resolution limit")
self._intgr_reso_high = d_min
self.set_integrater_done(False)
return
raise Sorry(e)
self._intgr_experiments_filename = integrate.get_integrated_experiments(
)
# also record the batch range - needed for the analysis of the
# radiation damage in chef...
self._intgr_batches_out = (self._intgr_wedge[0], self._intgr_wedge[1])
# FIXME (i) record the log file, (ii) get more information out from the
# integration log on the quality of the data and (iii) the mosaic spread
# range observed and R.M.S. deviations.
self._intgr_integrated_reflections = integrate.get_integrated_reflections(
)
if not os.path.isfile(self._intgr_integrated_reflections):
raise RuntimeError("Integration failed: %s does not exist." %
self._intgr_integrated_reflections)
self._intgr_per_image_statistics = integrate.get_per_image_statistics()
Chatter.write(self.show_per_image_statistics())
report = self.Report()
html_filename = os.path.join(
self.get_working_directory(),
"%i_dials.integrate.report.html" % report.get_xpid(),
)
report.set_html_filename(html_filename)
report.run(wait_for_completion=True)
FileHandler.record_html_file(
"%s %s %s %s INTEGRATE" % (pname, xname, dname, sweep),
html_filename)
from dxtbx.serialize import load
experiments = load.experiment_list(self._intgr_experiments_filename)
profile = experiments.profiles()[0]
mosaic = profile.sigma_m()
try:
m_min, m_max, m_mean = mosaic.min_max_mean().as_tuple()
self.set_integrater_mosaic_min_mean_max(m_min, m_mean, m_max)
except AttributeError as e:
self.set_integrater_mosaic_min_mean_max(mosaic, mosaic, mosaic)
Chatter.write("Mosaic spread: %.3f < %.3f < %.3f" %
self.get_integrater_mosaic_min_mean_max())
return self._intgr_integrated_reflections
def _sort_together_data_xds(self):
if len(self._sweep_information) == 1:
return self._sort_together_data_xds_one_sweep()
max_batches = 0
for epoch in self._sweep_information.keys():
hklin = self._sweep_information[epoch]['scaled_reflections']
if self._sweep_information[epoch]['batches'] == [0, 0]:
Chatter.write('Getting batches from %s' % hklin)
batches = MtzUtils.batches_from_mtz(hklin)
self._sweep_information[epoch]['batches'] = [
min(batches), max(batches)
]
Chatter.write('=> %d to %d' % (min(batches), max(batches)))
batches = self._sweep_information[epoch]['batches']
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
epochs = sorted(self._sweep_information.keys())
counter = 0
for epoch in epochs:
hklin = self._sweep_information[epoch]['scaled_reflections']
pname = self._sweep_information[epoch]['pname']
xname = self._sweep_information[epoch]['xname']
dname = self._sweep_information[epoch]['dname']
sname = self._sweep_information[epoch]['sname']
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%d.mtz' % \
(pname, xname, dname, counter))
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
# record this for future reference - will be needed in the
# radiation damage analysis...
# hack - reset this as it gets in a muddle...
intgr = self._sweep_information[epoch]['integrater']
self._sweep_information[epoch][
'batches'] = intgr.get_integrater_batches()
first_batch = min(self._sweep_information[epoch]['batches'])
offset = counter * max_batches - first_batch + 1
self._sweep_information[epoch]['batch_offset'] = offset
from xia2.Modules.Scaler.rebatch import rebatch
new_batches = rebatch(hklin,
hklout,
add_batch=offset,
pname=pname,
xname=xname,
dname=dname)
# update the "input information"
self._sweep_information[epoch]['hklin'] = hklout
self._sweep_information[epoch]['batches'] = new_batches
# update the counter & recycle
counter += 1
s = self._factory.Sortmtz()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(hklout)
for epoch in epochs:
s.add_hklin(self._sweep_information[epoch]['hklin'])
s.sort(vrset=-99999999.0)
self._prepared_reflections = hklout
if self.get_scaler_reference_reflection_file():
spacegroups = [
MtzUtils.space_group_name_from_mtz(
self.get_scaler_reference_reflection_file())
]
reindex_operator = 'h,k,l'
else:
pointless = self._factory.Pointless()
pointless.set_hklin(hklout)
pointless.decide_spacegroup()
FileHandler.record_log_file('%s %s pointless' % \
(self._scalr_pname,
self._scalr_xname),
pointless.get_log_file())
spacegroups = pointless.get_likely_spacegroups()
reindex_operator = pointless.get_spacegroup_reindex_operator()
if self._scalr_input_spacegroup:
Debug.write('Assigning user input spacegroup: %s' % \
self._scalr_input_spacegroup)
spacegroups = [self._scalr_input_spacegroup]
reindex_operator = 'h,k,l'
self._scalr_likely_spacegroups = spacegroups
spacegroup = self._scalr_likely_spacegroups[0]
self._scalr_reindex_operator = reindex_operator
Chatter.write('Likely spacegroups:')
for spag in self._scalr_likely_spacegroups:
Chatter.write('%s' % spag)
Chatter.write(
'Reindexing to first spacegroup setting: %s (%s)' % \
(spacegroup, clean_reindex_operator(reindex_operator)))
hklin = self._prepared_reflections
hklout = os.path.join(self.get_working_directory(),
'%s_%s_reindex.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(hklout)
ri = self._factory.Reindex()
ri.set_hklin(hklin)
ri.set_hklout(hklout)
ri.set_spacegroup(spacegroup)
ri.set_operator(reindex_operator)
ri.reindex()
hklin = hklout
hklout = os.path.join(self.get_working_directory(),
'%s_%s_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s = self._factory.Sortmtz()
s.set_hklin(hklin)
s.set_hklout(hklout)
s.sort(vrset=-99999999.0)
self._prepared_reflections = hklout
Debug.write(
'Updating unit cell to %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(ri.get_cell()))
self._scalr_cell = tuple(ri.get_cell())
return
def _refine(self):
for idxr in set(self._refinr_indexers.values()):
experiments = idxr.get_indexer_experiment_list()
indexed_experiments = idxr.get_indexer_payload(
"experiments_filename")
indexed_reflections = idxr.get_indexer_payload("indexed_filename")
# If multiple sweeps but not doing joint refinement, get only the
# relevant reflections.
multi_sweep = PhilIndex.params.xia2.settings.multi_sweep_refinement
if len(experiments) > 1 and not multi_sweep:
xsweeps = idxr._indxr_sweeps
assert len(xsweeps) == len(experiments)
# Don't do joint refinement
assert len(self._refinr_sweeps) == 1
xsweep = self._refinr_sweeps[0]
i = xsweeps.index(xsweep)
experiments = experiments[i:i + 1]
# Extract and output experiment and reflections for current sweep
indexed_experiments = os.path.join(
self.get_working_directory(),
"%s_indexed.expt" % xsweep.get_name())
indexed_reflections = os.path.join(
self.get_working_directory(),
"%s_indexed.refl" % xsweep.get_name())
experiments.as_file(indexed_experiments)
reflections = flex.reflection_table.from_file(
idxr.get_indexer_payload("indexed_filename"))
sel = reflections["id"] == i
assert sel.count(True) > 0
imageset_id = reflections["imageset_id"].select(sel)
assert imageset_id.all_eq(imageset_id[0])
sel = reflections["imageset_id"] == imageset_id[0]
reflections = reflections.select(sel)
# set indexed reflections to id == 0 and imageset_id == 0
reflections["id"].set_selected(reflections["id"] == i, 0)
reflections["imageset_id"] = flex.int(len(reflections), 0)
reflections.as_file(indexed_reflections)
# currently only handle one lattice/refiner
assert len(experiments.crystals()) == 1
scan_static = PhilIndex.params.dials.refine.scan_static
# Avoid doing scan-varying refinement on narrow wedges.
scan_oscillation_ranges = []
for experiment in experiments:
start, end = experiment.scan.get_oscillation_range()
scan_oscillation_ranges.append(end - start)
min_oscillation_range = min(scan_oscillation_ranges)
if (PhilIndex.params.dials.refine.scan_varying
and min_oscillation_range > 5
and not PhilIndex.params.dials.fast_mode):
scan_varying = PhilIndex.params.dials.refine.scan_varying
else:
scan_varying = False
if scan_static:
refiner = self.Refine()
refiner.set_experiments_filename(indexed_experiments)
refiner.set_indexed_filename(indexed_reflections)
refiner.set_scan_varying(False)
refiner.run()
self._refinr_experiments_filename = (
refiner.get_refined_experiments_filename())
self._refinr_indexed_filename = refiner.get_refined_filename()
else:
self._refinr_experiments_filename = indexed_experiments
self._refinr_indexed_filename = indexed_reflections
if scan_varying:
refiner = self.Refine()
refiner.set_experiments_filename(
self._refinr_experiments_filename)
refiner.set_indexed_filename(self._refinr_indexed_filename)
if min_oscillation_range < 36:
refiner.set_interval_width_degrees(min_oscillation_range /
2)
refiner.run()
self._refinr_experiments_filename = (
refiner.get_refined_experiments_filename())
self._refinr_indexed_filename = refiner.get_refined_filename()
if scan_static or scan_varying:
FileHandler.record_log_file(
"%s REFINE" % idxr.get_indexer_full_name(),
refiner.get_log_file())
report = self.Report()
report.set_experiments_filename(
self._refinr_experiments_filename)
report.set_reflections_filename(self._refinr_indexed_filename)
html_filename = os.path.join(
self.get_working_directory(),
"%i_dials.refine.report.html" % report.get_xpid(),
)
report.set_html_filename(html_filename)
report.run()
FileHandler.record_html_file(
"%s REFINE" % idxr.get_indexer_full_name(), html_filename)
experiments = ExperimentList.from_file(
self._refinr_experiments_filename)
self.set_refiner_payload("models.expt",
self._refinr_experiments_filename)
self.set_refiner_payload("observations.refl",
self._refinr_indexed_filename)
# this is the result of the cell refinement
self._refinr_cell = experiments.crystals()[0].get_unit_cell(
).parameters()
def _sort_together_data_xds_one_sweep(self):
assert len(self._sweep_information) == 1
epoch = self._sweep_information.keys()[0]
hklin = self._sweep_information[epoch]['scaled_reflections']
if self.get_scaler_reference_reflection_file():
spacegroups = [
MtzUtils.space_group_name_from_mtz(
self.get_scaler_reference_reflection_file())
]
reindex_operator = 'h,k,l'
elif self._scalr_input_spacegroup:
Debug.write('Assigning user input spacegroup: %s' % \
self._scalr_input_spacegroup)
spacegroups = [self._scalr_input_spacegroup]
reindex_operator = 'h,k,l'
else:
pointless = self._factory.Pointless()
pointless.set_hklin(hklin)
pointless.decide_spacegroup()
FileHandler.record_log_file('%s %s pointless' % \
(self._scalr_pname,
self._scalr_xname),
pointless.get_log_file())
spacegroups = pointless.get_likely_spacegroups()
reindex_operator = pointless.get_spacegroup_reindex_operator()
self._scalr_likely_spacegroups = spacegroups
spacegroup = self._scalr_likely_spacegroups[0]
self._scalr_reindex_operator = clean_reindex_operator(reindex_operator)
Chatter.write('Likely spacegroups:')
for spag in self._scalr_likely_spacegroups:
Chatter.write('%s' % spag)
Chatter.write(
'Reindexing to first spacegroup setting: %s (%s)' % \
(spacegroup, clean_reindex_operator(reindex_operator)))
hklout = os.path.join(self.get_working_directory(),
'%s_%s_reindex.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(hklout)
if reindex_operator == '[h,k,l]':
# just assign spacegroup
from cctbx import sgtbx
s = sgtbx.space_group(
sgtbx.space_group_symbols(str(spacegroup)).hall())
m = mtz.object(hklin)
m.set_space_group(s).write(hklout)
self._scalr_cell = m.crystals()[-1].unit_cell().parameters()
Debug.write(
'Updating unit cell to %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(self._scalr_cell))
del m
del s
else:
ri = self._factory.Reindex()
ri.set_hklin(hklin)
ri.set_hklout(hklout)
ri.set_spacegroup(spacegroup)
ri.set_operator(reindex_operator)
ri.reindex()
Debug.write(
'Updating unit cell to %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(ri.get_cell()))
self._scalr_cell = tuple(ri.get_cell())
hklin = hklout
hklout = os.path.join(self.get_working_directory(),
'%s_%s_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s = self._factory.Sortmtz()
s.set_hklin(hklin)
s.set_hklout(hklout)
s.sort(vrset=-99999999.0)
self._prepared_reflections = hklout
def _integrate(self):
"""Actually do the integration - in XDS terms this will mean running
DEFPIX and INTEGRATE to measure all the reflections."""
integrate = self.Integrate()
# decide what images we are going to process, if not already
# specified
if not self._intgr_wedge:
images = self.get_matching_images()
self.set_integrater_wedge(min(images), max(images))
imageset = self.get_imageset()
beam = imageset.get_beam()
detector = imageset.get_detector()
d_min_limit = detector.get_max_resolution(beam.get_s0())
if (d_min_limit > self._intgr_reso_high or
PhilIndex.params.xia2.settings.resolution.keep_all_reflections
):
logger.debug("Overriding high resolution limit: %f => %f" %
(self._intgr_reso_high, d_min_limit))
self._intgr_reso_high = d_min_limit
integrate.set_experiments_filename(self._intgr_experiments_filename)
integrate.set_reflections_filename(self._intgr_indexed_filename)
if PhilIndex.params.dials.integrate.d_max:
integrate.set_d_max(PhilIndex.params.dials.integrate.d_max)
else:
integrate.set_d_max(self._intgr_reso_low)
if PhilIndex.params.dials.integrate.d_min:
integrate.set_d_min(PhilIndex.params.dials.integrate.d_min)
else:
integrate.set_d_min(self._intgr_reso_high)
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file(
f"{pname} {xname} {dname} {sweep} INTEGRATE",
integrate.get_log_file(),
)
integrate.run()
self._intgr_experiments_filename = integrate.get_integrated_experiments(
)
# also record the batch range - needed for the analysis of the
# radiation damage in chef...
self._intgr_batches_out = (self._intgr_wedge[0], self._intgr_wedge[1])
# FIXME (i) record the log file, (ii) get more information out from the
# integration log on the quality of the data and (iii) the mosaic spread
# range observed and R.M.S. deviations.
self._intgr_integrated_reflections = integrate.get_integrated_reflections(
)
if not os.path.isfile(self._intgr_integrated_reflections):
raise RuntimeError("Integration failed: %s does not exist." %
self._intgr_integrated_reflections)
self._intgr_per_image_statistics = integrate.get_per_image_statistics()
logger.info(self.show_per_image_statistics())
report = self.Report()
html_filename = os.path.join(
self.get_working_directory(),
"%i_dials.integrate.report.html" % report.get_xpid(),
)
report.set_html_filename(html_filename)
report.run(wait_for_completion=True)
FileHandler.record_html_file(
f"{pname} {xname} {dname} {sweep} INTEGRATE", html_filename)
experiments = load.experiment_list(self._intgr_experiments_filename)
profile = experiments.profiles()[0]
mosaic = profile.sigma_m()
try:
m_min, m_max, m_mean = mosaic.min_max_mean().as_tuple()
self.set_integrater_mosaic_min_mean_max(m_min, m_mean, m_max)
except AttributeError:
self.set_integrater_mosaic_min_mean_max(mosaic, mosaic, mosaic)
logger.info("Mosaic spread: %.3f < %.3f < %.3f" %
self.get_integrater_mosaic_min_mean_max())
# If running in high-pressure mode, run dials.anvil_correction to
# correct for the attenuation of the incident and diffracted beams by the
# diamond anvils.
if self.high_pressure:
self._anvil_correction()
return self._intgr_integrated_reflections
def _mosflm_integrate(self):
'''Perform the actual integration, based on the results of the
cell refinement or indexing (they have the equivalent form.)'''
refinr = self.get_integrater_refiner()
if not refinr.get_refiner_payload('mosflm_orientation_matrix'):
raise RuntimeError('unexpected situation in indexing')
lattice = refinr.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup(lattice)
mosaic = refinr.get_refiner_payload('mosaic')
beam = refinr.get_refiner_payload('beam')
distance = refinr.get_refiner_payload('distance')
matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')
integration_params = refinr.get_refiner_payload(
'mosflm_integration_parameters')
if integration_params:
if 'separation' in integration_params:
self.set_integrater_parameter(
'mosflm', 'separation',
'%s %s' % tuple(integration_params['separation']))
if 'raster' in integration_params:
self.set_integrater_parameter(
'mosflm', 'raster',
'%d %d %d %d %d' % tuple(integration_params['raster']))
refinr.set_refiner_payload('mosflm_integration_parameters', None)
f = open(
os.path.join(self.get_working_directory(), 'xiaintegrate.mat'),
'w')
for m in matrix:
f.write(m)
f.close()
# then start the integration
integrater = MosflmIntegrate()
integrater.set_working_directory(self.get_working_directory())
auto_logfiler(integrater)
integrater.set_refine_profiles(self._mosflm_refine_profiles)
pname, xname, dname = self.get_integrater_project_info()
if pname is not None and xname is not None and dname is not None:
Debug.write('Harvesting: %s/%s/%s' % (pname, xname, dname))
harvest_dir = self.get_working_directory()
# harvest file name will be %s.mosflm_run_start_end % dname
temp_dname = '%s_%s' % \
(dname, self.get_integrater_sweep_name())
integrater.set_pname_xname_dname(pname, xname, temp_dname)
integrater.set_template(os.path.basename(self.get_template()))
integrater.set_directory(self.get_directory())
# check for ice - and if so, exclude (ranges taken from
# XDS documentation)
if self.get_integrater_ice() != 0:
Debug.write('Excluding ice rings')
integrater.set_exclude_ice(True)
# exclude specified resolution ranges
if len(self.get_integrater_excluded_regions()) != 0:
regions = self.get_integrater_excluded_regions()
Debug.write('Excluding regions: %s' % repr(regions))
integrater.set_exclude_regions(regions)
mask = standard_mask(self.get_detector())
for m in mask:
integrater.add_instruction(m)
integrater.set_input_mat_file('xiaintegrate.mat')
integrater.set_beam_centre(beam)
integrater.set_distance(distance)
integrater.set_space_group_number(spacegroup_number)
integrater.set_mosaic(mosaic)
if self.get_wavelength_prov() == 'user':
integrater.set_wavelength(self.get_wavelength())
parameters = self.get_integrater_parameters('mosflm')
integrater.update_parameters(parameters)
if self._mosflm_gain:
integrater.set_gain(self._mosflm_gain)
# check for resolution limits
if self._intgr_reso_high > 0.0:
integrater.set_d_min(self._intgr_reso_high)
if self._intgr_reso_low:
integrater.set_d_max(self._intgr_reso_low)
if PhilIndex.params.general.backstop_mask:
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask = mask.calculate_mask_mosflm(self.get_header())
integrater.set_mask(mask)
detector = self.get_detector()
detector_width, detector_height = detector[0].get_image_size_mm()
lim_x = 0.5 * detector_width
lim_y = 0.5 * detector_height
Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
integrater.set_limits(lim_x, lim_y)
integrater.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
offset = self.get_frame_offset()
integrater.set_image_range(
(self._intgr_wedge[0] - offset, self._intgr_wedge[1] - offset))
try:
integrater.run()
except RuntimeError as e:
if 'integration failed: reason unknown' in str(e):
Chatter.write('Mosflm has failed in integration')
message = 'The input was:\n\n'
for input in integrater.get_all_input():
message += ' %s' % input
Chatter.write(message)
raise
FileHandler.record_log_file(
'%s %s %s %s mosflm integrate' % \
(self.get_integrater_sweep_name(),
pname, xname, dname),
integrater.get_log_file())
self._intgr_per_image_statistics = integrater.get_per_image_statistics(
)
self._mosflm_hklout = integrater.get_hklout()
Debug.write('Integration output: %s' % self._mosflm_hklout)
self._intgr_n_ref = integrater.get_nref()
# if a BGSIG error happened try not refining the
# profile and running again...
if integrater.get_bgsig_too_large():
if not self._mosflm_refine_profiles:
raise RuntimeError('BGSIG error with profiles fixed')
Debug.write('BGSIG error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
if integrater.get_getprof_error():
Debug.write('GETPROF error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
if (integrater.get_detector_gain_error()
and not (self.get_imageset().get_detector()[0].get_type()
== 'SENSOR_PAD')):
gain = integrater.get_suggested_gain()
if gain is not None:
self.set_integrater_parameter('mosflm', 'gain', gain)
self.set_integrater_export_parameter('mosflm', 'gain', gain)
if self._mosflm_gain:
Debug.write('GAIN updated to %f' % gain)
else:
Debug.write('GAIN found to be %f' % gain)
self._mosflm_gain = gain
self._mosflm_rerun_integration = True
if not self._mosflm_hklout:
raise RuntimeError('processing abandoned')
self._intgr_batches_out = integrater.get_batches_out()
mosaics = integrater.get_mosaic_spreads()
if mosaics and len(mosaics) > 0:
self.set_integrater_mosaic_min_mean_max(
min(mosaics),
sum(mosaics) / len(mosaics), max(mosaics))
else:
m = indxr.get_indexer_mosaic()
self.set_integrater_mosaic_min_mean_max(m, m, m)
# write the report for each image as .*-#$ to Chatter -
# detailed report will be written automagically to science...
Chatter.write(self.show_per_image_statistics())
Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
self.get_integrater_mosaic_min_mean_max())
# gather the statistics from the postrefinement
postref_result = integrater.get_postref_result()
# now write this to a postrefinement log
postref_log = os.path.join(self.get_working_directory(),
'postrefinement.log')
fout = open(postref_log, 'w')
fout.write('$TABLE: Postrefinement for %s:\n' % \
self._intgr_sweep_name)
fout.write('$GRAPHS: Missetting angles:A:1, 2, 3, 4: $$\n')
fout.write('Batch PhiX PhiY PhiZ $$ Batch PhiX PhiY PhiZ $$\n')
for image in sorted(postref_result):
phix = postref_result[image].get('phix', 0.0)
phiy = postref_result[image].get('phiy', 0.0)
phiz = postref_result[image].get('phiz', 0.0)
fout.write('%d %5.2f %5.2f %5.2f\n' % \
(image, phix, phiy, phiz))
fout.write('$$\n')
fout.close()
if self.get_integrater_sweep_name():
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file('%s %s %s %s postrefinement' % \
(self.get_integrater_sweep_name(),
pname, xname, dname),
postref_log)
return self._mosflm_hklout
def _scale(self):
'''Perform all of the operations required to deliver the scaled
data.'''
epochs = self._sweep_handler.get_epochs()
if self._scalr_corrections:
Journal.block(
'scaling', self.get_scaler_xcrystal().get_name(), 'CCP4',
{'scaling model':'automatic',
'absorption':self._scalr_correct_absorption,
'decay':self._scalr_correct_decay
})
else:
Journal.block(
'scaling', self.get_scaler_xcrystal().get_name(), 'CCP4',
{'scaling model':'default'})
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_chef_unmerged(True)
sc.set_new_scales_file('%s.scales' % self._scalr_xname)
user_resolution_limits = {}
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
intgr = si.get_integrater()
if intgr.get_integrater_user_resolution():
dmin = intgr.get_integrater_high_resolution()
if (dname, sname) not in user_resolution_limits:
user_resolution_limits[(dname, sname)] = dmin
elif dmin < user_resolution_limits[(dname, sname)]:
user_resolution_limits[(dname, sname)] = dmin
start, end = si.get_batch_range()
if (dname, sname) in self._scalr_resolution_limits:
resolution, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(start, end, exclude=False, resolution=resolution, name=sname)
else:
sc.add_run(start, end, name=sname)
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_scaled_test.mtz' % \
(self._scalr_pname, self._scalr_xname)))
if self.get_scaler_anomalous():
sc.set_anomalous()
# what follows, sucks
failover = PhilIndex.params.xia2.settings.failover
if failover:
try:
sc.scale()
except RuntimeError as e:
es = str(e)
if 'bad batch' in es or \
'negative scales run' in es or \
'no observations' in es:
# first ID the sweep from the batch no
batch = int(es.split()[-1])
epoch = self._identify_sweep_epoch(batch)
sweep = self._scalr_integraters[epoch].get_integrater_sweep()
# then remove it from my parent xcrystal
self.get_scaler_xcrystal().remove_sweep(sweep)
# then remove it from the scaler list of intergraters
# - this should really be a scaler interface method
del self._scalr_integraters[epoch]
# then tell the user what is happening
Chatter.write(
'Sweep %s gave negative scales - removing' % \
sweep.get_name())
# then reset the prepare, do, finish flags
self.set_scaler_prepare_done(False)
self.set_scaler_done(False)
self.set_scaler_finish_done(False)
# and return
return
else:
raise e
else:
sc.scale()
# then gather up all of the resulting reflection files
# and convert them into the required formats (.sca, .mtz.)
data = sc.get_summary()
loggraph = sc.parse_ccp4_loggraph()
resolution_info = {}
reflection_files = sc.get_scaled_reflection_files()
for dataset in reflection_files:
FileHandler.record_temporary_file(reflection_files[dataset])
for key in loggraph:
if 'Analysis against resolution' in key:
dataset = key.split(',')[-1].strip()
resolution_info[dataset] = transpose_loggraph(loggraph[key])
highest_resolution = 100.0
highest_suggested_resolution = None
# check in here that there is actually some data to scale..!
if len(resolution_info) == 0:
raise RuntimeError('no resolution info')
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
intgr = si.get_integrater()
start, end = si.get_batch_range()
if (dname, sname) in self._scalr_resolution_limits:
continue
elif (dname, sname) in user_resolution_limits:
limit = user_resolution_limits[(dname, sname)]
self._scalr_resolution_limits[(dname, sname)] = (limit, None)
if limit < highest_resolution:
highest_resolution = limit
Chatter.write('Resolution limit for %s: %5.2f (user provided)' % \
(dname, limit))
continue
hklin = sc.get_unmerged_reflection_file()
limit, reasoning = self._estimate_resolution_limit(
hklin, batch_range=(start, end))
if PhilIndex.params.xia2.settings.resolution.keep_all_reflections == True:
suggested = limit
if highest_suggested_resolution is None or limit < highest_suggested_resolution:
highest_suggested_resolution = limit
limit = intgr.get_detector().get_max_resolution(intgr.get_beam_obj().get_s0())
self._scalr_resolution_limits[(dname, sname)] = (limit, suggested)
Debug.write('keep_all_reflections set, using detector limits')
Debug.write('Resolution for sweep %s: %.2f' % \
(sname, limit))
if not (dname, sname) in self._scalr_resolution_limits:
self._scalr_resolution_limits[(dname, sname)] = (limit, None)
self.set_scaler_done(False)
if limit < highest_resolution:
highest_resolution = limit
limit, suggested = self._scalr_resolution_limits[(dname, sname)]
if suggested is None or limit == suggested:
reasoning_str = ''
if reasoning:
reasoning_str = ' (%s)' % reasoning
Chatter.write('Resolution for sweep %s/%s: %.2f%s' % \
(dname, sname, limit, reasoning_str))
else:
Chatter.write('Resolution limit for %s/%s: %5.2f (%5.2f suggested)' % \
(dname, sname, limit, suggested))
if highest_suggested_resolution is not None and \
highest_resolution >= (highest_suggested_resolution - 0.004):
Debug.write('Dropping resolution cut-off suggestion since it is'
' essentially identical to the actual resolution limit.')
highest_suggested_resolution = None
self._scalr_highest_resolution = highest_resolution
self._scalr_highest_suggested_resolution = highest_suggested_resolution
if highest_suggested_resolution is not None:
Debug.write('Suggested highest resolution is %5.2f (%5.2f suggested)' % \
(highest_resolution, highest_suggested_resolution))
else:
Debug.write('Scaler highest resolution set to %5.2f' % \
highest_resolution)
if not self.get_scaler_done():
Debug.write('Returning as scaling not finished...')
return
batch_info = {}
for key in loggraph:
if 'Analysis against Batch' in key:
dataset = key.split(',')[-1].strip()
batch_info[dataset] = transpose_loggraph(loggraph[key])
sc = self._updated_aimless()
FileHandler.record_log_file('%s %s aimless' % (self._scalr_pname,
self._scalr_xname),
sc.get_log_file())
sc.set_hklin(self._prepared_reflections)
sc.set_new_scales_file('%s_final.scales' % self._scalr_xname)
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(start, end, exclude=False, resolution=resolution_limit,
name=xname)
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_scaled.mtz' % \
(self._scalr_pname, self._scalr_xname)))
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
FileHandler.record_xml_file('%s %s aimless xml' % (self._scalr_pname,
self._scalr_xname),
sc.get_xmlout())
data = sc.get_summary()
scales_file = sc.get_new_scales_file()
loggraph = sc.parse_ccp4_loggraph()
standard_deviation_info = {}
for key in loggraph:
if 'standard deviation v. Intensity' in key:
dataset = key.split(',')[-1].strip()
standard_deviation_info[dataset] = transpose_loggraph(loggraph[key])
resolution_info = {}
for key in loggraph:
if 'Analysis against resolution' in key:
dataset = key.split(',')[-1].strip()
resolution_info[dataset] = transpose_loggraph(loggraph[key])
batch_info = {}
for key in loggraph:
if 'Analysis against Batch' in key:
dataset = key.split(',')[-1].strip()
batch_info[dataset] = transpose_loggraph(loggraph[key])
# finally put all of the results "somewhere useful"
self._scalr_statistics = data
self._scalr_scaled_refl_files = copy.deepcopy(
sc.get_scaled_reflection_files())
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_scales_file(scales_file)
self._wavelengths_in_order = []
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(start, end, exclude=False, resolution=resolution_limit,
name=sname)
if not dname in self._wavelengths_in_order:
self._wavelengths_in_order.append(dname)
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_scaled.mtz' % \
(self._scalr_pname,
self._scalr_xname)))
sc.set_scalepack()
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
self._update_scaled_unit_cell()
self._scalr_scaled_reflection_files = {}
self._scalr_scaled_reflection_files['sca'] = {}
self._scalr_scaled_reflection_files['sca_unmerged'] = {}
self._scalr_scaled_reflection_files['mtz_unmerged'] = {}
for key in self._scalr_scaled_refl_files:
hklout = self._scalr_scaled_refl_files[key]
scaout = '%s.sca' % hklout[:-4]
self._scalr_scaled_reflection_files['sca'][key] = scaout
FileHandler.record_data_file(scaout)
scalepack = os.path.join(os.path.split(hklout)[0],
os.path.split(hklout)[1].replace(
'_scaled', '_scaled_unmerged').replace('.mtz', '.sca'))
self._scalr_scaled_reflection_files['sca_unmerged'][key] = scalepack
FileHandler.record_data_file(scalepack)
mtz_unmerged = os.path.splitext(scalepack)[0] + '.mtz'
self._scalr_scaled_reflection_files['mtz_unmerged'][key] = mtz_unmerged
FileHandler.record_data_file(mtz_unmerged)
if self._scalr_cell_esd is not None:
# patch .mtz and overwrite unit cell information
import xia2.Modules.Scaler.tools as tools
override_cell = self._scalr_cell_dict.get('%s_%s_%s' %
(self._scalr_pname, self._scalr_xname, key))[0]
tools.patch_mtz_unit_cell(mtz_unmerged, override_cell)
tools.patch_mtz_unit_cell(hklout, override_cell)
self._scalr_scaled_reflection_files['mtz_unmerged'][key] = mtz_unmerged
FileHandler.record_data_file(mtz_unmerged)
if PhilIndex.params.xia2.settings.merging_statistics.source == 'cctbx':
for key in self._scalr_scaled_refl_files:
stats = self._compute_scaler_statistics(
self._scalr_scaled_reflection_files['mtz_unmerged'][key],
selected_band=(highest_suggested_resolution, None), wave=key)
self._scalr_statistics[
(self._scalr_pname, self._scalr_xname, key)] = stats
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_scales_file(scales_file)
self._wavelengths_in_order = []
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(start, end, exclude=False, resolution=resolution_limit,
name=sname)
if not dname in self._wavelengths_in_order:
self._wavelengths_in_order.append(dname)
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_chef.mtz' % \
(self._scalr_pname,
self._scalr_xname)))
sc.set_chef_unmerged(True)
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
if not PhilIndex.params.dials.fast_mode:
try:
self._generate_absorption_map(sc)
except Exception as e:
# Map generation may fail for number of reasons, eg. matplotlib borken
Debug.write("Could not generate absorption map (%s)" % e)
def _index(self):
if PhilIndex.params.dials.index.method in (libtbx.Auto, None):
if self._indxr_input_cell is not None:
indexer = self._do_indexing("real_space_grid_search")
else:
try:
indexer_fft3d = self._do_indexing(method="fft3d")
nref_3d, rmsd_3d = indexer_fft3d.get_nref_rmsds()
except Exception as e:
nref_3d = None
rmsd_3d = None
indexing_failure = e
try:
indexer_fft1d = self._do_indexing(method="fft1d")
nref_1d, rmsd_1d = indexer_fft1d.get_nref_rmsds()
except Exception as e:
nref_1d = None
rmsd_1d = None
indexing_failure = e
if (nref_1d is not None and nref_3d is None or
(nref_1d > nref_3d and rmsd_1d[0] < rmsd_3d[0]
and rmsd_1d[1] < rmsd_3d[1] and rmsd_1d[2] < rmsd_3d[2])):
indexer = indexer_fft1d
elif nref_3d is not None:
indexer = indexer_fft3d
else:
raise RuntimeError(indexing_failure)
else:
indexer = self._do_indexing(
method=PhilIndex.params.dials.index.method)
# not strictly the P1 cell, rather the cell that was used in indexing
self._p1_cell = indexer._p1_cell
self.set_indexer_payload("indexed_filename",
indexer.get_indexed_filename())
indexed_file = indexer.get_indexed_filename()
indexed_experiments = indexer.get_experiments_filename()
fast_mode = PhilIndex.params.dials.fast_mode
trust_beam_centre = PhilIndex.params.xia2.settings.trust_beam_centre
multi_sweep_indexing = PhilIndex.params.xia2.settings.multi_sweep_indexing
check_indexing_symmetry = PhilIndex.params.dials.check_indexing_symmetry
if check_indexing_symmetry and not (trust_beam_centre or fast_mode
or multi_sweep_indexing):
checksym = self.CheckIndexingSymmetry()
checksym.set_experiments_filename(indexed_experiments)
checksym.set_indexed_filename(indexed_file)
checksym.set_grid_search_scope(1)
checksym.run()
hkl_offset = checksym.get_hkl_offset()
logger.debug("hkl_offset: %s", str(hkl_offset))
if hkl_offset is not None and hkl_offset != (0, 0, 0):
reindex = self.Reindex()
reindex.set_hkl_offset(hkl_offset)
reindex.set_indexed_filename(indexed_file)
reindex.run()
indexed_file = reindex.get_reindexed_reflections_filename()
# do some scan-static refinement - run twice, first without outlier
# rejection as the model is too far from reality to do a sensible job of
# outlier rejection
refiner = self.Refine()
refiner.set_experiments_filename(indexed_experiments)
refiner.set_indexed_filename(
reindex.get_reindexed_reflections_filename())
refiner.set_outlier_algorithm(None)
refiner.run()
indexed_experiments = refiner.get_refined_experiments_filename(
)
# now again with outlier rejection (possibly)
refiner = self.Refine()
refiner.set_experiments_filename(indexed_experiments)
refiner.set_indexed_filename(indexed_file)
refiner.run()
indexed_experiments = refiner.get_refined_experiments_filename(
)
if self._indxr_input_lattice is None:
# FIXME in here should respect the input unit cell and lattice if provided
# FIXME from this (i) populate the helper table,
# (ii) try to avoid re-running the indexing
# step if we eliminate a solution as we have all of the refined results
# already available.
rbs = self.RefineBravaisSettings()
rbs.set_experiments_filename(indexed_experiments)
rbs.set_indexed_filename(indexed_file)
if PhilIndex.params.dials.fix_geometry:
rbs.set_detector_fix("all")
rbs.set_beam_fix("all")
elif PhilIndex.params.dials.fix_distance:
rbs.set_detector_fix("distance")
FileHandler.record_log_file(
"%s LATTICE" % self.get_indexer_full_name(),
rbs.get_log_file())
rbs.run()
for k in sorted(rbs.get_bravais_summary()):
summary = rbs.get_bravais_summary()[k]
# FIXME need to do this better - for the moment only accept lattices
# where R.M.S. deviation is less than twice P1 R.M.S. deviation.
if self._indxr_input_lattice is None:
if not summary["recommended"]:
continue
experiments = load.experiment_list(summary["experiments_file"],
check_format=False)
cryst = experiments.crystals()[0]
cs = crystal.symmetry(unit_cell=cryst.get_unit_cell(),
space_group=cryst.get_space_group())
lattice = str(
bravais_types.bravais_lattice(group=cs.space_group()))
cb_op = sgtbx.change_of_basis_op(str(summary["cb_op"]))
self._solutions[k] = {
"number": k,
"mosaic": 0.0,
"metric": summary["max_angular_difference"],
"rmsd": summary["rmsd"],
"nspots": summary["nspots"],
"lattice": lattice,
"cell": cs.unit_cell().parameters(),
"experiments_file": summary["experiments_file"],
"cb_op": str(cb_op),
}
self._solution = self.get_solution()
self._indxr_lattice = self._solution["lattice"]
for solution in self._solutions:
lattice = self._solutions[solution]["lattice"]
if (self._indxr_input_lattice is not None
and self._indxr_input_lattice != lattice):
continue
if lattice in self._indxr_other_lattice_cell:
if (self._indxr_other_lattice_cell[lattice]["metric"] <
self._solutions[solution]["metric"]):
continue
self._indxr_other_lattice_cell[lattice] = {
"metric": self._solutions[solution]["metric"],
"cell": self._solutions[solution]["cell"],
}
self._indxr_mosaic = self._solution["mosaic"]
experiments_file = self._solution["experiments_file"]
experiment_list = load.experiment_list(experiments_file)
self.set_indexer_experiment_list(experiment_list)
self.set_indexer_payload("experiments_filename", experiments_file)
# reindex the output reflection list to this solution
reindex = self.Reindex()
reindex.set_indexed_filename(indexed_file)
reindex.set_cb_op(self._solution["cb_op"])
reindex.set_space_group(
str(lattice_to_spacegroup_number(self._solution["lattice"])))
reindex.run()
indexed_file = reindex.get_reindexed_reflections_filename()
self.set_indexer_payload("indexed_filename", indexed_file)
else:
experiment_list = load.experiment_list(indexed_experiments)
self.set_indexer_experiment_list(experiment_list)
self.set_indexer_payload("experiments_filename",
indexed_experiments)
cryst = experiment_list.crystals()[0]
lattice = str(
bravais_types.bravais_lattice(group=cryst.get_space_group()))
self._indxr_lattice = lattice
self._solutions = {}
self._solutions[0] = {
"number": 0,
"mosaic": 0.0,
"metric": -1,
"rmsd": -1,
"nspots": -1,
"lattice": lattice,
"cell": cryst.get_unit_cell().parameters(),
"experiments_file": indexed_experiments,
"cb_op": "a,b,c",
}
self._indxr_other_lattice_cell[lattice] = {
"metric": self._solutions[0]["metric"],
"cell": self._solutions[0]["cell"],
}
def _integrate(self):
'''Actually do the integration - in XDS terms this will mean running
DEFPIX and INTEGRATE to measure all the reflections.'''
images_str = '%d to %d' % tuple(self._intgr_wedge)
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % tuple(self._intgr_cell)
if len(self._fp_directory) <= 50:
dirname = self._fp_directory
else:
dirname = '...%s' % self._fp_directory[-46:]
Journal.block(
'integrating', self._intgr_sweep_name, 'DIALS', {
'images': images_str,
'cell': cell_str,
'lattice': self.get_integrater_refiner().get_refiner_lattice(),
'template': self._fp_template,
'directory': dirname,
'resolution': '%.2f' % self._intgr_reso_high
})
integrate = self.Integrate()
# decide what images we are going to process, if not already
# specified
if not self._intgr_wedge:
images = self.get_matching_images()
self.set_integrater_wedge(min(images), max(images))
imageset = self.get_imageset()
beam = imageset.get_beam()
detector = imageset.get_detector()
d_min_limit = detector.get_max_resolution(beam.get_s0())
if d_min_limit > self._intgr_reso_high \
or PhilIndex.params.xia2.settings.resolution.keep_all_reflections:
Debug.write('Overriding high resolution limit: %f => %f' % \
(self._intgr_reso_high, d_min_limit))
self._intgr_reso_high = d_min_limit
integrate.set_experiments_filename(self._intgr_experiments_filename)
integrate.set_reflections_filename(self._intgr_indexed_filename)
integrate.set_d_max(self._intgr_reso_low)
integrate.set_d_min(self._intgr_reso_high)
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file('%s %s %s %s INTEGRATE' % \
(pname, xname, dname, sweep),
integrate.get_log_file())
try:
integrate.run()
except xia2.Wrappers.Dials.Integrate.DIALSIntegrateError as e:
s = str(e)
if ('dials.integrate requires more memory than is available.' in s
and not self._intgr_reso_high):
# Try to estimate a more sensible resolution limit for integration
# in case we were just integrating noise to the edge of the detector
images = self._integrate_select_images_wedges()
Debug.write(
'Integrating subset of images to estimate resolution limit.\n'
'Integrating images %s' % images)
integrate = self.Integrate()
integrate.set_experiments_filename(
self._intgr_experiments_filename)
integrate.set_reflections_filename(
self._intgr_indexed_filename)
integrate.set_d_max(self._intgr_reso_low)
integrate.set_d_min(self._intgr_reso_high)
for (start, stop) in images:
integrate.add_scan_range(
start - self.get_matching_images()[0],
stop - self.get_matching_images()[0])
integrate.set_reflections_per_degree(1000)
integrate.run()
integrated_pickle = integrate.get_integrated_filename()
from xia2.Wrappers.Dials.EstimateResolutionLimit import EstimateResolutionLimit
d_min_estimater = EstimateResolutionLimit()
d_min_estimater.set_working_directory(
self.get_working_directory())
auto_logfiler(d_min_estimater)
d_min_estimater.set_experiments_filename(
self._intgr_experiments_filename)
d_min_estimater.set_reflections_filename(integrated_pickle)
d_min = d_min_estimater.run()
Debug.write('Estimate for d_min: %.2f' % d_min)
Debug.write('Re-running integration to this resolution limit')
self._intgr_reso_high = d_min
self.set_integrater_done(False)
return
raise Sorry(e)
self._intgr_experiments_filename = integrate.get_integrated_experiments(
)
# also record the batch range - needed for the analysis of the
# radiation damage in chef...
self._intgr_batches_out = (self._intgr_wedge[0], self._intgr_wedge[1])
# FIXME (i) record the log file, (ii) get more information out from the
# integration log on the quality of the data and (iii) the mosaic spread
# range observed and R.M.S. deviations.
self._intgr_integrated_pickle = integrate.get_integrated_reflections()
if not os.path.isfile(self._intgr_integrated_pickle):
raise RuntimeError("Integration failed: %s does not exist." %
self._intgr_integrated_pickle)
self._intgr_per_image_statistics = integrate.get_per_image_statistics()
Chatter.write(self.show_per_image_statistics())
report = self.Report()
html_filename = os.path.join(
self.get_working_directory(),
'%i_dials.integrate.report.html' % report.get_xpid())
report.set_html_filename(html_filename)
report.run()
FileHandler.record_html_file('%s %s %s %s INTEGRATE' % \
(pname, xname, dname, sweep),
html_filename)
import dials
from dxtbx.serialize import load
experiments = load.experiment_list(self._intgr_experiments_filename)
profile = experiments.profiles()[0]
mosaic = profile.sigma_m()
self.set_integrater_mosaic_min_mean_max(mosaic, mosaic, mosaic)
Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
self.get_integrater_mosaic_min_mean_max())
return self._intgr_integrated_pickle
def _index(self):
'''Actually do the autoindexing using the data prepared by the
previous method.'''
images_str = '%d to %d' % tuple(self._indxr_images[0])
for i in self._indxr_images[1:]:
images_str += ', %d to %d' % tuple(i)
cell_str = None
if self._indxr_input_cell:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
self._indxr_input_cell
# then this is a proper autoindexing run - describe this
# to the journal entry
#if len(self._fp_directory) <= 50:
#dirname = self._fp_directory
#else:
#dirname = '...%s' % self._fp_directory[-46:]
dirname = self.get_directory()
Journal.block(
'autoindexing', self._indxr_sweep_name, 'XDS', {
'images': images_str,
'target cell': cell_str,
'target lattice': self._indxr_input_lattice,
'template': self.get_template(),
'directory': dirname
})
self._index_remove_masked_regions()
if self._i_or_ii is None:
self._i_or_ii = self.decide_i_or_ii()
Debug.write('Selecting I or II, chose %s' % self._i_or_ii)
idxref = self.Idxref()
for file in ['SPOT.XDS']:
idxref.set_input_data_file(file, self._indxr_payload[file])
# set the phi start etc correctly
idxref.set_data_range(self._indxr_images[0][0],
self._indxr_images[0][1])
idxref.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
if self._i_or_ii == 'i':
blocks = self._index_select_images_i()
for block in blocks[:1]:
starting_frame = block[0]
starting_angle = self.get_scan().get_angle_from_image_index(
starting_frame)
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
for block in blocks[1:]:
idxref.add_spot_range(block[0], block[1])
else:
for block in self._indxr_images[:1]:
starting_frame = block[0]
starting_angle = self.get_scan().get_angle_from_image_index(
starting_frame)
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
for block in self._indxr_images[1:]:
idxref.add_spot_range(block[0], block[1])
# FIXME need to also be able to pass in the known unit
# cell and lattice if already available e.g. from
# the helper... indirectly
if self._indxr_user_input_lattice:
idxref.set_indexer_user_input_lattice(True)
if self._indxr_input_lattice and self._indxr_input_cell:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
idxref.set_indexer_input_cell(self._indxr_input_cell)
Debug.write('Set lattice: %s' % self._indxr_input_lattice)
Debug.write('Set cell: %f %f %f %f %f %f' % \
self._indxr_input_cell)
original_cell = self._indxr_input_cell
elif self._indxr_input_lattice:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
original_cell = None
else:
original_cell = None
# FIXED need to set the beam centre here - this needs to come
# from the input .xinfo object or header, and be converted
# to the XDS frame... done.
#mosflm_beam_centre = self.get_beam_centre()
#xds_beam_centre = beam_centre_mosflm_to_xds(
#mosflm_beam_centre[0], mosflm_beam_centre[1], self.get_header())
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
idxref.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1])
# fixme need to check if the lattice, cell have been set already,
# and if they have, pass these in as input to the indexing job.
done = False
while not done:
try:
done = idxref.run()
# N.B. in here if the IDXREF step was being run in the first
# pass done is FALSE however there should be a refined
# P1 orientation matrix etc. available - so keep it!
except XDSException as e:
# inspect this - if we have complaints about not
# enough reflections indexed, and we have a target
# unit cell, and they are the same, well ignore it
if 'solution is inaccurate' in str(e):
Debug.write('XDS complains solution inaccurate - ignoring')
done = idxref.continue_from_error()
elif ('insufficient percentage (< 70%)' in str(e) or
'insufficient percentage (< 50%)' in str(e)) and \
original_cell:
done = idxref.continue_from_error()
lattice, cell, mosaic = \
idxref.get_indexing_solution()
# compare solutions
check = PhilIndex.params.xia2.settings.xds_check_cell_deviation
for j in range(3):
# allow two percent variation in unit cell length
if math.fabs((cell[j] - original_cell[j]) / \
original_cell[j]) > 0.02 and check:
Debug.write('XDS unhappy and solution wrong')
raise e
# and two degree difference in angle
if math.fabs(cell[j + 3] - original_cell[j + 3]) \
> 2.0 and check:
Debug.write('XDS unhappy and solution wrong')
raise e
Debug.write('XDS unhappy but solution ok')
elif 'insufficient percentage (< 70%)' in str(e) or \
'insufficient percentage (< 50%)' in str(e):
Debug.write('XDS unhappy but solution probably ok')
done = idxref.continue_from_error()
else:
raise e
FileHandler.record_log_file(
'%s INDEX' % self.get_indexer_full_name(),
os.path.join(self.get_working_directory(), 'IDXREF.LP'))
for file in ['SPOT.XDS', 'XPARM.XDS']:
self._indxr_payload[file] = idxref.get_output_data_file(file)
# need to get the indexing solutions out somehow...
self._indxr_other_lattice_cell = idxref.get_indexing_solutions()
self._indxr_lattice, self._indxr_cell, self._indxr_mosaic = \
idxref.get_indexing_solution()
import dxtbx
from dxtbx.serialize.xds import to_crystal
xparm_file = os.path.join(self.get_working_directory(), 'XPARM.XDS')
models = dxtbx.load(xparm_file)
crystal_model = to_crystal(xparm_file)
from dxtbx.model import Experiment, ExperimentList
experiment = Experiment(
beam=models.get_beam(),
detector=models.get_detector(),
goniometer=models.get_goniometer(),
scan=models.get_scan(),
crystal=crystal_model,
#imageset=self.get_imageset(),
)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
# I will want this later on to check that the lattice was ok
self._idxref_subtree_problem = idxref.get_index_tree_problem()
return
(self._intgr_wedge[0] - offset, self._intgr_wedge[1] - offset))
try:
integrater.run()
except RuntimeError, e:
if 'integration failed: reason unknown' in str(e):
Chatter.write('Mosflm has failed in integration')
message = 'The input was:\n\n'
for input in integrater.get_all_input():
message += ' %s' % input
Chatter.write(message)
raise
FileHandler.record_log_file(
'%s %s %s %s mosflm integrate' % \
(self.get_integrater_sweep_name(),
pname, xname, dname),
integrater.get_log_file())
self._intgr_per_image_statistics = integrater.get_per_image_statistics()
self._mosflm_hklout = integrater.get_hklout()
Debug.write('Integration output: %s' %self._mosflm_hklout)
self._intgr_n_ref = integrater.get_nref()
# if a BGSIG error happened try not refining the
# profile and running again...
if integrater.get_bgsig_too_large():
if not self._mosflm_refine_profiles:
def _integrate(self):
'''Actually do the integration - in XDS terms this will mean running
DEFPIX and INTEGRATE to measure all the reflections.'''
experiment = self._intgr_refiner.get_refined_experiment_list(
self.get_integrater_epoch())[0]
crystal_model = experiment.crystal
self._intgr_refiner_cell = crystal_model.get_unit_cell().parameters()
images_str = '%d to %d' % tuple(self._intgr_wedge)
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' %tuple(self._intgr_refiner_cell)
if len(self._fp_directory) <= 50:
dirname = self._fp_directory
else:
dirname = '...%s' % self._fp_directory[-46:]
Journal.block(
'integrating', self._intgr_sweep_name, 'XDS',
{'images':images_str,
'cell':cell_str,
'lattice':self._intgr_refiner.get_refiner_lattice(),
'template':self._fp_template,
'directory':dirname,
'resolution':'%.2f' % self._intgr_reso_high})
first_image_in_wedge = self.get_image_name(self._intgr_wedge[0])
defpix = self.Defpix()
# pass in the correct data
for file in ['X-CORRECTIONS.cbf',
'Y-CORRECTIONS.cbf',
'BKGINIT.cbf',
'XPARM.XDS']:
defpix.set_input_data_file(file, self._xds_data_files[file])
defpix.set_data_range(self._intgr_wedge[0],
self._intgr_wedge[1])
if self.get_integrater_high_resolution() > 0.0 and \
self.get_integrater_user_resolution():
Debug.write('Setting resolution limit in DEFPIX to %.2f' % \
self.get_integrater_high_resolution())
defpix.set_resolution_high(self.get_integrater_high_resolution())
defpix.set_resolution_low(self.get_integrater_low_resolution())
elif self.get_integrater_low_resolution():
Debug.write('Setting low resolution limit in DEFPIX to %.2f' % \
self.get_integrater_low_resolution())
defpix.set_resolution_high(0.0)
defpix.set_resolution_low(self.get_integrater_low_resolution())
defpix.run()
# and gather the result files
for file in ['BKGPIX.cbf',
'ABS.cbf']:
self._xds_data_files[file] = defpix.get_output_data_file(file)
integrate = self.Integrate()
if self._xds_integrate_parameters:
integrate.set_updates(self._xds_integrate_parameters)
# decide what images we are going to process, if not already
# specified
if not self._intgr_wedge:
images = self.get_matching_images()
self.set_integrater_wedge(min(images),
max(images))
first_image_in_wedge = self.get_image_name(self._intgr_wedge[0])
integrate.set_data_range(self._intgr_wedge[0],
self._intgr_wedge[1])
for file in ['X-CORRECTIONS.cbf',
'Y-CORRECTIONS.cbf',
'BLANK.cbf',
'BKGPIX.cbf',
'GAIN.cbf']:
integrate.set_input_data_file(file, self._xds_data_files[file])
if self._xds_data_files.has_key('GXPARM.XDS'):
Debug.write('Using globally refined parameters')
integrate.set_input_data_file(
'XPARM.XDS', self._xds_data_files['GXPARM.XDS'])
integrate.set_refined_xparm()
else:
integrate.set_input_data_file(
'XPARM.XDS', self._xds_data_files['XPARM.XDS'])
integrate.run()
self._intgr_per_image_statistics = integrate.get_per_image_statistics()
Chatter.write(self.show_per_image_statistics())
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file('%s %s %s %s INTEGRATE' % \
(pname, xname, dname, sweep),
os.path.join(self.get_working_directory(),
'INTEGRATE.LP'))
# and copy the first pass INTEGRATE.HKL...
lattice = self._intgr_refiner.get_refiner_lattice()
if not os.path.exists(os.path.join(
self.get_working_directory(),
'INTEGRATE-%s.HKL' % lattice)):
here = self.get_working_directory()
shutil.copyfile(os.path.join(here, 'INTEGRATE.HKL'),
os.path.join(here, 'INTEGRATE-%s.HKL' % lattice))
# record INTEGRATE.HKL for e.g. BLEND.
FileHandler.record_more_data_file(
'%s %s %s %s INTEGRATE' % (pname, xname, dname, sweep),
os.path.join(self.get_working_directory(), 'INTEGRATE.HKL'))
# should the existence of these require that I rerun the
# integration or can we assume that the application of a
# sensible resolution limit will achieve this??
self._xds_integrate_parameters = integrate.get_updates()
# record the mosaic spread &c.
m_min, m_mean, m_max = integrate.get_mosaic()
self.set_integrater_mosaic_min_mean_max(m_min, m_mean, m_max)
Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
self.get_integrater_mosaic_min_mean_max())
return os.path.join(self.get_working_directory(), 'INTEGRATE.HKL')
def _mosflm_parallel_integrate(self):
'''Perform the integration as before, but this time as a
number of parallel Mosflm jobs (hence, in separate directories)
and including a step of pre-refinement of the mosaic spread and
missets. This will all be kind of explicit and hence probably
messy!'''
refinr = self.get_integrater_refiner()
lattice = refinr.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup(lattice)
mosaic = refinr.get_refiner_payload('mosaic')
beam = refinr.get_refiner_payload('beam')
distance = refinr.get_refiner_payload('distance')
matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')
integration_params = refinr.get_refiner_payload(
'mosflm_integration_parameters')
if integration_params:
if 'separation' in integration_params:
self.set_integrater_parameter(
'mosflm', 'separation',
'%s %s' % tuple(integration_params['separation']))
if 'raster' in integration_params:
self.set_integrater_parameter(
'mosflm', 'raster',
'%d %d %d %d %d' % tuple(integration_params['raster']))
refinr.set_refiner_payload('mosflm_integration_parameters', None)
pname, xname, dname = self.get_integrater_project_info()
# what follows below should (i) be run in separate directories
# and (ii) be repeated N=parallel times.
nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
parallel = nproc
# FIXME this is something of a kludge - if too few frames refinement
# and integration does not work well... ideally want at least 15
# frames / chunk (say)
nframes = self._intgr_wedge[1] - self._intgr_wedge[0] + 1
if parallel > nframes / 15:
parallel = nframes // 15
if not parallel:
raise RuntimeError, 'parallel not set'
if parallel < 2:
raise RuntimeError, 'parallel not parallel: %s' % parallel
jobs = []
hklouts = []
nref = 0
# calculate the chunks to use
offset = self.get_frame_offset()
start = self._intgr_wedge[0] - offset
end = self._intgr_wedge[1] - offset
left_images = 1 + end - start
left_chunks = parallel
chunks = []
while left_images > 0:
size = left_images // left_chunks
chunks.append((start, start + size - 1))
start += size
left_images -= size
left_chunks -= 1
summary_files = []
for j in range(parallel):
# make some working directories, as necessary - chunk-(0:N-1)
wd = os.path.join(self.get_working_directory(),
'chunk-%d' % j)
if not os.path.exists(wd):
os.makedirs(wd)
job = MosflmIntegrate()
job.set_working_directory(wd)
auto_logfiler(job)
l = refinr.get_refiner_lattice()
# create the starting point
f = open(os.path.join(wd, 'xiaintegrate-%s.mat' % l), 'w')
for m in matrix:
f.write(m)
f.close()
spacegroup_number = lattice_to_spacegroup(lattice)
job.set_refine_profiles(self._mosflm_refine_profiles)
# N.B. for harvesting need to append N to dname.
if pname is not None and xname is not None and dname is not None:
Debug.write('Harvesting: %s/%s/%s' %
(pname, xname, dname))
harvest_dir = self.get_working_directory()
temp_dname = '%s_%s' % \
(dname, self.get_integrater_sweep_name())
job.set_pname_xname_dname(pname, xname, temp_dname)
job.set_template(os.path.basename(self.get_template()))
job.set_directory(self.get_directory())
# check for ice - and if so, exclude (ranges taken from
# XDS documentation)
if self.get_integrater_ice() != 0:
Debug.write('Excluding ice rings')
job.set_exclude_ice(True)
# exclude specified resolution ranges
if len(self.get_integrater_excluded_regions()) != 0:
regions = self.get_integrater_excluded_regions()
Debug.write('Excluding regions: %s' % `regions`)
job.set_exclude_regions(regions)
mask = standard_mask(self.get_detector())
for m in mask:
job.add_instruction(m)
job.set_input_mat_file('xiaintegrate-%s.mat' % l)
job.set_beam_centre(beam)
job.set_distance(distance)
job.set_space_group_number(spacegroup_number)
job.set_mosaic(mosaic)
if self.get_wavelength_prov() == 'user':
job.set_wavelength(self.get_wavelength())
parameters = self.get_integrater_parameters('mosflm')
job.update_parameters(parameters)
if self._mosflm_gain:
job.set_gain(self._mosflm_gain)
# check for resolution limits
if self._intgr_reso_high > 0.0:
job.set_d_min(self._intgr_reso_high)
if self._intgr_reso_low:
job.set_d_max(self._intgr_reso_low)
if PhilIndex.params.general.backstop_mask:
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask = mask.calculate_mask_mosflm(self.get_header())
job.set_mask(mask)
detector = self.get_detector()
detector_width, detector_height = detector[0].get_image_size_mm()
lim_x = 0.5 * detector_width
lim_y = 0.5 * detector_height
Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
job.set_limits(lim_x, lim_y)
job.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
job.set_pre_refinement(True)
job.set_image_range(chunks[j])
# these are now running so ...
jobs.append(job)
continue
# ok, at this stage I need to ...
#
# (i) accumulate the statistics as a function of batch
# (ii) mong them into a single block
#
# This is likely to be a pain in the arse!
first_integrated_batch = 1.0e6
last_integrated_batch = -1.0e6
all_residuals = []
threads = []
for j in range(parallel):
job = jobs[j]
# now wait for them to finish - first wait will really be the
# first one, then all should be finished...
thread = Background(job, 'run')
thread.start()
threads.append(thread)
mosaics = []
postref_result = { }
integrated_images_first = 1.0e6
integrated_images_last = -1.0e6
self._intgr_per_image_statistics = {}
for j in range(parallel):
thread = threads[j]
thread.stop()
job = jobs[j]
# get the log file
output = job.get_all_output()
# record a copy of it, perhaps - though not if parallel
if self.get_integrater_sweep_name() and False:
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file(
'%s %s %s %s mosflm integrate' % \
(self.get_integrater_sweep_name(),
pname, xname, '%s_%d' % (dname, j)),
job.get_log_file())
# look for things that we want to know...
# that is, the output reflection file name, the updated
# value for the gain (if present,) any warnings, errors,
# or just interesting facts.
batches = job.get_batches_out()
integrated_images_first = min(batches[0], integrated_images_first)
integrated_images_last = max(batches[1], integrated_images_last)
mosaics.extend(job.get_mosaic_spreads())
if min(mosaics) < 0:
raise IntegrationError, 'negative mosaic spread: %s' % min(mosaic)
if (job.get_detector_gain_error() and not
(self.get_imageset().get_detector()[0].get_type() == 'SENSOR_PAD')):
gain = job.get_suggested_gain()
if gain is not None:
self.set_integrater_parameter('mosflm', 'gain', gain)
self.set_integrater_export_parameter('mosflm', 'gain', gain)
if self._mosflm_gain:
Debug.write('GAIN updated to %f' % gain)
else:
Debug.write('GAIN found to be %f' % gain)
self._mosflm_gain = gain
self._mosflm_rerun_integration = True
hklout = job.get_hklout()
Debug.write('Integration output: %s' % hklout)
hklouts.append(hklout)
nref += job.get_nref()
# if a BGSIG error happened try not refining the
# profile and running again...
if job.get_bgsig_too_large():
if not self._mosflm_refine_profiles:
raise RuntimeError, 'BGSIG error with profiles fixed'
Debug.write(
'BGSIG error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
if job.get_getprof_error():
Debug.write(
'GETPROF error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
# here
# write the report for each image as .*-#$ to Chatter -
# detailed report will be written automagically to science...
self._intgr_per_image_statistics.update(job.get_per_image_statistics())
postref_result.update(job.get_postref_result())
# inspect the output for e.g. very high weighted residuals
all_residuals.extend(job.get_residuals())
self._intgr_batches_out = (integrated_images_first,
integrated_images_last)
if mosaics and len(mosaics) > 0:
self.set_integrater_mosaic_min_mean_max(
min(mosaics), sum(mosaics) / len(mosaics), max(mosaics))
else:
m = indxr.get_indexer_mosaic()
self.set_integrater_mosaic_min_mean_max(m, m, m)
Chatter.write(self.show_per_image_statistics())
Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
self.get_integrater_mosaic_min_mean_max())
# gather the statistics from the postrefinement for all sweeps
# now write this to a postrefinement log
postref_log = os.path.join(self.get_working_directory(),
'postrefinement.log')
fout = open(postref_log, 'w')
fout.write('$TABLE: Postrefinement for %s:\n' % \
self._intgr_sweep_name)
fout.write('$GRAPHS: Missetting angles:A:1, 2, 3, 4: $$\n')
fout.write('Batch PhiX PhiY PhiZ $$ Batch PhiX PhiY PhiZ $$\n')
for image in sorted(postref_result):
phix = postref_result[image].get('phix', 0.0)
phiy = postref_result[image].get('phiy', 0.0)
phiz = postref_result[image].get('phiz', 0.0)
fout.write('%d %5.2f %5.2f %5.2f\n' % \
(image, phix, phiy, phiz))
fout.write('$$\n')
fout.close()
if self.get_integrater_sweep_name():
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file('%s %s %s %s postrefinement' % \
(self.get_integrater_sweep_name(),
pname, xname, dname),
postref_log)
hklouts.sort()
hklout = os.path.join(self.get_working_directory(),
os.path.split(hklouts[0])[-1])
Debug.write('Sorting data to %s' % hklout)
for hklin in hklouts:
Debug.write('<= %s' % hklin)
sortmtz = Sortmtz()
sortmtz.set_hklout(hklout)
for hklin in hklouts:
sortmtz.add_hklin(hklin)
sortmtz.sort()
self._mosflm_hklout = hklout
return self._mosflm_hklout
def _index(self):
if PhilIndex.params.dials.index.method in (libtbx.Auto, None):
if self._indxr_input_cell is not None:
indexer = self._do_indexing("real_space_grid_search")
else:
try:
indexer_fft3d = self._do_indexing(method="fft3d")
nref_3d, rmsd_3d = indexer_fft3d.get_nref_rmsds()
except Exception as e:
nref_3d = None
rmsd_3d = None
try:
indexer_fft1d = self._do_indexing(method="fft1d")
nref_1d, rmsd_1d = indexer_fft1d.get_nref_rmsds()
except Exception as e:
nref_1d = None
rmsd_1d = None
if (nref_1d is not None and nref_3d is None or
(nref_1d > nref_3d and rmsd_1d[0] < rmsd_3d[0]
and rmsd_1d[1] < rmsd_3d[1] and rmsd_1d[2] < rmsd_3d[2])):
indexer = indexer_fft1d
elif nref_3d is not None:
indexer = indexer_fft3d
else:
raise RuntimeError(e)
else:
indexer = self._do_indexing(
method=PhilIndex.params.dials.index.method)
# not strictly the P1 cell, rather the cell that was used in indexing
self._p1_cell = indexer._p1_cell
self.set_indexer_payload("indexed_filename",
indexer.get_indexed_filename())
from cctbx.sgtbx import bravais_types
from dxtbx.serialize import load
indexed_file = indexer.get_indexed_filename()
indexed_experiments = indexer.get_experiments_filename()
fast_mode = PhilIndex.params.dials.fast_mode
trust_beam_centre = PhilIndex.params.xia2.settings.trust_beam_centre
multi_sweep_indexing = PhilIndex.params.xia2.settings.multi_sweep_indexing == True
if not (trust_beam_centre or fast_mode or multi_sweep_indexing):
checksym = self.CheckIndexingSymmetry()
checksym.set_experiments_filename(indexed_experiments)
checksym.set_indexed_filename(indexed_file)
checksym.set_grid_search_scope(1)
checksym.run()
hkl_offset = checksym.get_hkl_offset()
Debug.write("hkl_offset: %s" % str(hkl_offset))
if hkl_offset is not None and hkl_offset != (0, 0, 0):
reindex = self.Reindex()
reindex.set_hkl_offset(hkl_offset)
reindex.set_indexed_filename(indexed_file)
reindex.run()
indexed_file = reindex.get_reindexed_reflections_filename()
# do some scan-static refinement - run twice, first without outlier
# rejection as the model is too far from reality to do a sensible job of
# outlier rejection
refiner = self.Refine()
refiner.set_experiments_filename(indexed_experiments)
refiner.set_indexed_filename(
reindex.get_reindexed_reflections_filename())
refiner.set_outlier_algorithm(None)
refiner.run()
indexed_experiments = refiner.get_refined_experiments_filename(
)
# now again with outlier rejection (possibly)
refiner = self.Refine()
refiner.set_experiments_filename(indexed_experiments)
refiner.set_indexed_filename(indexed_file)
refiner.run()
indexed_experiments = refiner.get_refined_experiments_filename(
)
if self._indxr_input_lattice is None:
# FIXME in here should respect the input unit cell and lattice if provided
# FIXME from this (i) populate the helper table,
# (ii) try to avoid re-running the indexing
# step if we eliminate a solution as we have all of the refined results
# already available.
rbs = self.RefineBravaisSettings()
rbs.set_experiments_filename(indexed_experiments)
rbs.set_indexed_filename(indexed_file)
if PhilIndex.params.dials.fix_geometry:
rbs.set_detector_fix('all')
rbs.set_beam_fix('all')
FileHandler.record_log_file(
'%s LATTICE' % self.get_indexer_full_name(),
rbs.get_log_file())
rbs.run()
from cctbx import crystal, sgtbx
for k in sorted(rbs.get_bravais_summary()):
summary = rbs.get_bravais_summary()[k]
# FIXME need to do this better - for the moment only accept lattices
# where R.M.S. deviation is less than twice P1 R.M.S. deviation.
if self._indxr_input_lattice is None:
if not summary['recommended']:
continue
experiments = load.experiment_list(summary['experiments_file'],
check_format=False)
cryst = experiments.crystals()[0]
cs = crystal.symmetry(unit_cell=cryst.get_unit_cell(),
space_group=cryst.get_space_group())
cb_op_best_to_ref = cs.change_of_basis_op_to_reference_setting(
)
cs_reference = cs.change_basis(cb_op_best_to_ref)
lattice = str(
bravais_types.bravais_lattice(
group=cs_reference.space_group()))
cb_op = cb_op_best_to_ref * sgtbx.change_of_basis_op(
str(summary['cb_op']))
self._solutions[k] = {
'number': k,
'mosaic': 0.0,
'metric': summary['max_angular_difference'],
'rmsd': summary['rmsd'],
'nspots': summary['nspots'],
'lattice': lattice,
'cell': cs_reference.unit_cell().parameters(),
'experiments_file': summary['experiments_file'],
'cb_op': str(cb_op)
}
self._solution = self.get_solution()
self._indxr_lattice = self._solution['lattice']
for solution in self._solutions.keys():
lattice = self._solutions[solution]['lattice']
if (self._indxr_input_lattice is not None
and self._indxr_input_lattice != lattice):
continue
if lattice in self._indxr_other_lattice_cell:
if self._indxr_other_lattice_cell[lattice]['metric'] < \
self._solutions[solution]['metric']:
continue
self._indxr_other_lattice_cell[lattice] = {
'metric': self._solutions[solution]['metric'],
'cell': self._solutions[solution]['cell']
}
self._indxr_mosaic = self._solution['mosaic']
experiment_list = load.experiment_list(
self._solution['experiments_file'])
self.set_indexer_experiment_list(experiment_list)
# reindex the output experiments list to the reference setting
# (from the best cell/conventional setting)
cb_op_to_ref = experiment_list.crystals()[0].get_space_group().info()\
.change_of_basis_op_to_reference_setting()
reindex = self.Reindex()
reindex.set_experiments_filename(
self._solution['experiments_file'])
reindex.set_cb_op(cb_op_to_ref)
reindex.set_space_group(
str(lattice_to_spacegroup_number(self._solution['lattice'])))
reindex.run()
experiments_file = reindex.get_reindexed_experiments_filename()
experiment_list = load.experiment_list(experiments_file)
self.set_indexer_experiment_list(experiment_list)
self.set_indexer_payload("experiments_filename", experiments_file)
# reindex the output reflection list to this solution
reindex = self.Reindex()
reindex.set_indexed_filename(indexed_file)
reindex.set_cb_op(self._solution['cb_op'])
reindex.set_space_group(
str(lattice_to_spacegroup_number(self._solution['lattice'])))
reindex.run()
indexed_file = reindex.get_reindexed_reflections_filename()
self.set_indexer_payload("indexed_filename", indexed_file)
else:
experiment_list = load.experiment_list(indexed_experiments)
self.set_indexer_experiment_list(experiment_list)
self.set_indexer_payload("experiments_filename",
indexed_experiments)
cryst = experiment_list.crystals()[0]
lattice = str(
bravais_types.bravais_lattice(group=cryst.get_space_group()))
self._indxr_lattice = lattice
self._solutions = {}
self._solutions[0] = {
'number': 0,
'mosaic': 0.0,
'metric': -1,
'rmsd': -1,
'nspots': -1,
'lattice': lattice,
'cell': cryst.get_unit_cell().parameters(),
'experiments_file': indexed_experiments,
'cb_op': 'a,b,c'
}
self._indxr_other_lattice_cell[lattice] = {
'metric': self._solutions[0]['metric'],
'cell': self._solutions[0]['cell']
}
return
def _scale(self):
'''Actually scale all of the data together.'''
from xia2.Handlers.Environment import debug_memory_usage
debug_memory_usage()
Journal.block(
'scaling', self.get_scaler_xcrystal().get_name(), 'XSCALE',
{'scaling model':'default (all)'})
epochs = self._sweep_information.keys()
epochs.sort()
xscale = self.XScale()
xscale.set_spacegroup_number(self._xds_spacegroup)
xscale.set_cell(self._scalr_cell)
Debug.write('Set CELL: %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(self._scalr_cell))
Debug.write('Set SPACEGROUP_NUMBER: %d' % \
self._xds_spacegroup)
Debug.write('Gathering measurements for scaling')
for epoch in epochs:
# get the prepared reflections
reflections = self._sweep_information[epoch][
'prepared_reflections']
# and the get wavelength that this belongs to
dname = self._sweep_information[epoch]['dname']
sname = self._sweep_information[epoch]['sname']
# and the resolution range for the reflections
intgr = self._sweep_information[epoch]['integrater']
Debug.write('Epoch: %d' % epoch)
Debug.write('HKL: %s (%s/%s)' % (reflections, dname, sname))
resolution_low = intgr.get_integrater_low_resolution()
resolution_high, _ = self._scalr_resolution_limits.get((dname, sname), (0.0, None))
resolution = (resolution_high, resolution_low)
xscale.add_reflection_file(reflections, dname, resolution)
# set the global properties of the sample
xscale.set_crystal(self._scalr_xname)
xscale.set_anomalous(self._scalr_anomalous)
debug_memory_usage()
xscale.run()
scale_factor = xscale.get_scale_factor()
Debug.write('XSCALE scale factor found to be: %e' % scale_factor)
# record the log file
pname = self._scalr_pname
xname = self._scalr_xname
FileHandler.record_log_file('%s %s XSCALE' % \
(pname, xname),
os.path.join(self.get_working_directory(),
'XSCALE.LP'))
# check for outlier reflections and if a number are found
# then iterate (that is, rerun XSCALE, rejecting these outliers)
if not PhilIndex.params.dials.fast_mode and not PhilIndex.params.xds.keep_outliers:
xscale_remove = xscale.get_remove()
if xscale_remove:
current_remove = []
final_remove = []
# first ensure that there are no duplicate entries...
if os.path.exists(os.path.join(
self.get_working_directory(),
'REMOVE.HKL')):
for line in open(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'), 'r').readlines():
h, k, l = map(int, line.split()[:3])
z = float(line.split()[3])
if not (h, k, l, z) in current_remove:
current_remove.append((h, k, l, z))
for c in xscale_remove:
if c in current_remove:
continue
final_remove.append(c)
Debug.write(
'%d alien reflections are already removed' % \
(len(xscale_remove) - len(final_remove)))
else:
# we want to remove all of the new dodgy reflections
final_remove = xscale_remove
remove_hkl = open(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'), 'w')
z_min = PhilIndex.params.xds.z_min
rejected = 0
# write in the old reflections
for remove in current_remove:
z = remove[3]
if z >= z_min:
remove_hkl.write('%d %d %d %f\n' % remove)
else:
rejected += 1
Debug.write('Wrote %d old reflections to REMOVE.HKL' % \
(len(current_remove) - rejected))
Debug.write('Rejected %d as z < %f' % \
(rejected, z_min))
# and the new reflections
rejected = 0
used = 0
for remove in final_remove:
z = remove[3]
if z >= z_min:
used += 1
remove_hkl.write('%d %d %d %f\n' % remove)
else:
rejected += 1
Debug.write('Wrote %d new reflections to REMOVE.HKL' % \
(len(final_remove) - rejected))
Debug.write('Rejected %d as z < %f' % \
(rejected, z_min))
remove_hkl.close()
# we want to rerun the finishing step so...
# unless we have added no new reflections
if used:
self.set_scaler_done(False)
if not self.get_scaler_done():
Chatter.write('Excluding outlier reflections Z > %.2f' %
PhilIndex.params.xds.z_min)
return
debug_memory_usage()
# now get the reflection files out and merge them with aimless
output_files = xscale.get_output_reflection_files()
wavelength_names = output_files.keys()
# these are per wavelength - also allow for user defined resolution
# limits a la bug # 3183. No longer...
for epoch in self._sweep_information.keys():
input = self._sweep_information[epoch]
intgr = input['integrater']
rkey = input['dname'], input['sname']
if intgr.get_integrater_user_resolution():
dmin = intgr.get_integrater_high_resolution()
if rkey not in self._user_resolution_limits:
self._scalr_resolution_limits[rkey] = (dmin, None)
self._user_resolution_limits[rkey] = dmin
elif dmin < self._user_resolution_limits[rkey]:
self._scalr_resolution_limits[rkey] = (dmin, None)
self._user_resolution_limits[rkey] = dmin
self._scalr_scaled_refl_files = { }
self._scalr_statistics = { }
max_batches = 0
mtz_dict = { }
project_info = { }
for epoch in self._sweep_information.keys():
pname = self._scalr_pname
xname = self._scalr_xname
dname = self._sweep_information[epoch]['dname']
reflections = os.path.split(
self._sweep_information[epoch]['prepared_reflections'])[-1]
project_info[reflections] = (pname, xname, dname)
for epoch in self._sweep_information.keys():
self._sweep_information[epoch]['scaled_reflections'] = None
debug_memory_usage()
for wavelength in wavelength_names:
hklin = output_files[wavelength]
xsh = XDSScalerHelper()
xsh.set_working_directory(self.get_working_directory())
ref = xsh.split_and_convert_xscale_output(
hklin, 'SCALED_', project_info, 1.0 / scale_factor)
for hklout in ref.keys():
for epoch in self._sweep_information.keys():
if os.path.split(self._sweep_information[epoch][
'prepared_reflections'])[-1] == \
os.path.split(hklout)[-1]:
if self._sweep_information[epoch][
'scaled_reflections'] is not None:
raise RuntimeError, 'duplicate entries'
self._sweep_information[epoch][
'scaled_reflections'] = ref[hklout]
del(xsh)
debug_memory_usage()
for epoch in self._sweep_information.keys():
hklin = self._sweep_information[epoch]['scaled_reflections']
dname = self._sweep_information[epoch]['dname']
sname = self._sweep_information[epoch]['sname']
hkl_copy = os.path.join(self.get_working_directory(),
'R_%s' % os.path.split(hklin)[-1])
if not os.path.exists(hkl_copy):
shutil.copyfile(hklin, hkl_copy)
# let's properly listen to the user's resolution limit needs...
if self._user_resolution_limits.get((dname, sname), False):
resolution = self._user_resolution_limits[(dname, sname)]
else:
if PhilIndex.params.xia2.settings.resolution.keep_all_reflections == True:
try:
resolution = intgr.get_detector().get_max_resolution(intgr.get_beam_obj().get_s0())
Debug.write('keep_all_reflections set, using detector limits')
except Exception:
resolution = self._estimate_resolution_limit(hklin)
else:
resolution = self._estimate_resolution_limit(hklin)
Chatter.write('Resolution for sweep %s/%s: %.2f' % \
(dname, sname, resolution))
if (dname, sname) not in self._scalr_resolution_limits:
self._scalr_resolution_limits[(dname, sname)] = (resolution, None)
self.set_scaler_done(False)
else:
if resolution < self._scalr_resolution_limits[(dname, sname)][0]:
self._scalr_resolution_limits[(dname, sname)] = (resolution, None)
self.set_scaler_done(False)
debug_memory_usage()
if not self.get_scaler_done():
Debug.write('Returning as scaling not finished...')
return
self._sort_together_data_xds()
highest_resolution = min(limit for limit, _ in self._scalr_resolution_limits.values())
self._scalr_highest_resolution = highest_resolution
Debug.write('Scaler highest resolution set to %5.2f' % \
highest_resolution)
if not self.get_scaler_done():
Debug.write('Returning as scaling not finished...')
return
sdadd_full = 0.0
sdb_full = 0.0
# ---------- FINAL MERGING ----------
sc = self._factory.Aimless()
FileHandler.record_log_file('%s %s aimless' % (self._scalr_pname,
self._scalr_xname),
sc.get_log_file())
sc.set_resolution(highest_resolution)
sc.set_hklin(self._prepared_reflections)
sc.set_new_scales_file('%s_final.scales' % self._scalr_xname)
if sdadd_full == 0.0 and sdb_full == 0.0:
pass
else:
sc.add_sd_correction('both', 1.0, sdadd_full, sdb_full)
for epoch in epochs:
input = self._sweep_information[epoch]
start, end = (min(input['batches']), max(input['batches']))
rkey = input['dname'], input['sname']
run_resolution_limit, _ = self._scalr_resolution_limits[rkey]
sc.add_run(start, end, exclude = False,
resolution = run_resolution_limit,
name = input['sname'])
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_scaled.mtz' % \
(self._scalr_pname, self._scalr_xname)))
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.multi_merge()
FileHandler.record_xml_file('%s %s aimless xml' % (self._scalr_pname,
self._scalr_xname),
sc.get_xmlout())
data = sc.get_summary()
loggraph = sc.parse_ccp4_loggraph()
standard_deviation_info = { }
for key in loggraph.keys():
if 'standard deviation v. Intensity' in key:
dataset = key.split(',')[-1].strip()
standard_deviation_info[dataset] = transpose_loggraph(
loggraph[key])
resolution_info = { }
for key in loggraph.keys():
if 'Analysis against resolution' in key:
dataset = key.split(',')[-1].strip()
resolution_info[dataset] = transpose_loggraph(
loggraph[key])
# and also radiation damage stuff...
batch_info = { }
for key in loggraph.keys():
if 'Analysis against Batch' in key:
dataset = key.split(',')[-1].strip()
batch_info[dataset] = transpose_loggraph(
loggraph[key])
# finally put all of the results "somewhere useful"
self._scalr_statistics = data
self._scalr_scaled_refl_files = copy.deepcopy(
sc.get_scaled_reflection_files())
self._scalr_scaled_reflection_files = { }
# also output the unmerged scalepack format files...
sc = self._factory.Aimless()
sc.set_resolution(highest_resolution)
sc.set_hklin(self._prepared_reflections)
sc.set_scalepack()
for epoch in epochs:
input = self._sweep_information[epoch]
start, end = (min(input['batches']), max(input['batches']))
rkey = input['dname'], input['sname']
run_resolution_limit, _ = self._scalr_resolution_limits[rkey]
sc.add_run(start, end, exclude = False,
resolution = run_resolution_limit,
name = input['sname'])
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_scaled.mtz' % \
(self._scalr_pname,
self._scalr_xname)))
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.multi_merge()
self._scalr_scaled_reflection_files['sca_unmerged'] = { }
self._scalr_scaled_reflection_files['mtz_unmerged'] = { }
for dataset in sc.get_scaled_reflection_files().keys():
hklout = sc.get_scaled_reflection_files()[dataset]
# then mark the scalepack files for copying...
scalepack = os.path.join(os.path.split(hklout)[0],
os.path.split(hklout)[1].replace(
'_scaled', '_scaled_unmerged').replace('.mtz', '.sca'))
self._scalr_scaled_reflection_files['sca_unmerged'][
dataset] = scalepack
FileHandler.record_data_file(scalepack)
mtz_unmerged = os.path.splitext(scalepack)[0] + '.mtz'
self._scalr_scaled_reflection_files['mtz_unmerged'][dataset] = mtz_unmerged
FileHandler.record_data_file(mtz_unmerged)
if PhilIndex.params.xia2.settings.merging_statistics.source == 'cctbx':
for key in self._scalr_scaled_refl_files:
stats = self._compute_scaler_statistics(
self._scalr_scaled_reflection_files['mtz_unmerged'][key], wave=key)
self._scalr_statistics[
(self._scalr_pname, self._scalr_xname, key)] = stats
# convert reflection files to .sca format - use mtz2various for this
self._scalr_scaled_reflection_files['sca'] = { }
self._scalr_scaled_reflection_files['hkl'] = { }
for key in self._scalr_scaled_refl_files:
f = self._scalr_scaled_refl_files[key]
scaout = '%s.sca' % f[:-4]
m2v = self._factory.Mtz2various()
m2v.set_hklin(f)
m2v.set_hklout(scaout)
m2v.convert()
self._scalr_scaled_reflection_files['sca'][key] = scaout
FileHandler.record_data_file(scaout)
if PhilIndex.params.xia2.settings.small_molecule == True:
hklout = '%s.hkl' % f[:-4]
m2v = self._factory.Mtz2various()
m2v.set_hklin(f)
m2v.set_hklout(hklout)
m2v.convert_shelx()
self._scalr_scaled_reflection_files['hkl'][key] = hklout
FileHandler.record_data_file(hklout)
def _refine(self):
for epoch, idxr in self._refinr_indexers.iteritems():
experiments = idxr.get_indexer_experiment_list()
indexed_experiments = idxr.get_indexer_payload(
"experiments_filename")
indexed_reflections = idxr.get_indexer_payload("indexed_filename")
if len(experiments) > 1:
xsweeps = idxr._indxr_sweeps
assert len(xsweeps) == len(experiments)
assert len(self._refinr_sweeps
) == 1 # don't currently support joint refinement
xsweep = self._refinr_sweeps[0]
i = xsweeps.index(xsweep)
experiments = experiments[i:i + 1]
# Extract and output experiment and reflections for current sweep
indexed_experiments = os.path.join(
self.get_working_directory(),
"%s_indexed_experiments.json" % xsweep.get_name())
indexed_reflections = os.path.join(
self.get_working_directory(),
"%s_indexed_reflections.pickle" % xsweep.get_name())
from dxtbx.serialize import dump
dump.experiment_list(experiments, indexed_experiments)
from libtbx import easy_pickle
from scitbx.array_family import flex
reflections = easy_pickle.load(
idxr.get_indexer_payload("indexed_filename"))
sel = reflections['id'] == i
assert sel.count(True) > 0
imageset_id = reflections['imageset_id'].select(sel)
assert imageset_id.all_eq(imageset_id[0])
sel = reflections['imageset_id'] == imageset_id[0]
reflections = reflections.select(sel)
# set indexed reflections to id == 0 and imageset_id == 0
reflections['id'].set_selected(reflections['id'] == i, 0)
reflections['imageset_id'] = flex.int(len(reflections), 0)
easy_pickle.dump(indexed_reflections, reflections)
assert len(experiments.crystals()
) == 1 # currently only handle one lattice/sweep
crystal_model = experiments.crystals()[0]
lattice = idxr.get_indexer_lattice()
from dxtbx.serialize import load
scan_static = PhilIndex.params.dials.refine.scan_static
# XXX Temporary workaround for dials.refine error for scan_varying
# refinement with smaller wedges
start, end = experiments[0].scan.get_oscillation_range()
total_phi_range = end - start
if (PhilIndex.params.dials.refine.scan_varying
and total_phi_range > 5
and not PhilIndex.params.dials.fast_mode):
scan_varying = PhilIndex.params.dials.refine.scan_varying
else:
scan_varying = False
if scan_static:
refiner = self.Refine()
refiner.set_experiments_filename(indexed_experiments)
refiner.set_indexed_filename(indexed_reflections)
refiner.set_scan_varying(False)
refiner.run()
self._refinr_experiments_filename \
= refiner.get_refined_experiments_filename()
self._refinr_indexed_filename = refiner.get_refined_filename()
else:
self._refinr_experiments_filename = indexed_experiments
self._refinr_indexed_filename = indexed_reflections
if scan_varying:
refiner = self.Refine()
refiner.set_experiments_filename(
self._refinr_experiments_filename)
refiner.set_indexed_filename(self._refinr_indexed_filename)
if total_phi_range < 36:
refiner.set_interval_width_degrees(total_phi_range / 2)
refiner.run()
self._refinr_experiments_filename \
= refiner.get_refined_experiments_filename()
self._refinr_indexed_filename = refiner.get_refined_filename()
if scan_static or scan_varying:
FileHandler.record_log_file(
'%s REFINE' % idxr.get_indexer_full_name(),
refiner.get_log_file())
report = self.Report()
report.set_experiments_filename(
self._refinr_experiments_filename)
report.set_reflections_filename(self._refinr_indexed_filename)
html_filename = os.path.join(
self.get_working_directory(),
'%i_dials.refine.report.html' % report.get_xpid())
report.set_html_filename(html_filename)
report.run()
FileHandler.record_html_file(
'%s REFINE' % idxr.get_indexer_full_name(), html_filename)
experiments = load.experiment_list(
self._refinr_experiments_filename)
self.set_refiner_payload("experiments.json",
self._refinr_experiments_filename)
self.set_refiner_payload("reflections.pickle",
self._refinr_indexed_filename)
# this is the result of the cell refinement
self._refinr_cell = experiments.crystals()[0].get_unit_cell(
).parameters()
def _integrate(self):
"""Actually do the integration - in XDS terms this will mean running
DEFPIX and INTEGRATE to measure all the reflections."""
experiment = self._intgr_refiner.get_refined_experiment_list(
self.get_integrater_epoch())[0]
crystal_model = experiment.crystal
self._intgr_refiner_cell = crystal_model.get_unit_cell().parameters()
defpix = self.Defpix()
# pass in the correct data
for file in (
"X-CORRECTIONS.cbf",
"Y-CORRECTIONS.cbf",
"BKGINIT.cbf",
"XPARM.XDS",
):
defpix.set_input_data_file(file, self._xds_data_files[file])
defpix.set_data_range(
self._intgr_wedge[0] + self.get_frame_offset(),
self._intgr_wedge[1] + self.get_frame_offset(),
)
if (self.get_integrater_high_resolution() > 0.0
and self.get_integrater_user_resolution()):
logger.debug("Setting resolution limit in DEFPIX to %.2f" %
self.get_integrater_high_resolution())
defpix.set_resolution_high(self.get_integrater_high_resolution())
defpix.set_resolution_low(self.get_integrater_low_resolution())
elif self.get_integrater_low_resolution():
logger.debug("Setting low resolution limit in DEFPIX to %.2f" %
self.get_integrater_low_resolution())
defpix.set_resolution_high(0.0)
defpix.set_resolution_low(self.get_integrater_low_resolution())
defpix.run()
# and gather the result files
for file in ("BKGPIX.cbf", "ABS.cbf"):
self._xds_data_files[file] = defpix.get_output_data_file(file)
integrate = self.Integrate()
if self._xds_integrate_parameters:
integrate.set_updates(self._xds_integrate_parameters)
# decide what images we are going to process, if not already
# specified
if not self._intgr_wedge:
images = self.get_matching_images()
self.set_integrater_wedge(min(images), max(images))
integrate.set_data_range(
self._intgr_wedge[0] + self.get_frame_offset(),
self._intgr_wedge[1] + self.get_frame_offset(),
)
for file in (
"X-CORRECTIONS.cbf",
"Y-CORRECTIONS.cbf",
"BLANK.cbf",
"BKGPIX.cbf",
"GAIN.cbf",
):
integrate.set_input_data_file(file, self._xds_data_files[file])
if "GXPARM.XDS" in self._xds_data_files:
logger.debug("Using globally refined parameters")
integrate.set_input_data_file("XPARM.XDS",
self._xds_data_files["GXPARM.XDS"])
integrate.set_refined_xparm()
else:
integrate.set_input_data_file("XPARM.XDS",
self._xds_data_files["XPARM.XDS"])
integrate.run()
self._intgr_per_image_statistics = integrate.get_per_image_statistics()
logger.info(self.show_per_image_statistics())
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file(
f"{pname} {xname} {dname} {sweep} INTEGRATE",
os.path.join(self.get_working_directory(), "INTEGRATE.LP"),
)
# and copy the first pass INTEGRATE.HKL...
lattice = self._intgr_refiner.get_refiner_lattice()
if not os.path.exists(
os.path.join(self.get_working_directory(),
"INTEGRATE-%s.HKL" % lattice)):
here = self.get_working_directory()
shutil.copyfile(
os.path.join(here, "INTEGRATE.HKL"),
os.path.join(here, "INTEGRATE-%s.HKL" % lattice),
)
# record INTEGRATE.HKL
FileHandler.record_more_data_file(
f"{pname} {xname} {dname} {sweep} INTEGRATE",
os.path.join(self.get_working_directory(), "INTEGRATE.HKL"),
)
# should the existence of these require that I rerun the
# integration or can we assume that the application of a
# sensible resolution limit will achieve this??
self._xds_integrate_parameters = integrate.get_updates()
# record the mosaic spread &c.
m_min, m_mean, m_max = integrate.get_mosaic()
self.set_integrater_mosaic_min_mean_max(m_min, m_mean, m_max)
logger.info("Mosaic spread: %.3f < %.3f < %.3f" %
self.get_integrater_mosaic_min_mean_max())
return os.path.join(self.get_working_directory(), "INTEGRATE.HKL")
def _integrate(self):
'''Actually do the integration - in XDS terms this will mean running
DEFPIX and INTEGRATE to measure all the reflections.'''
images_str = '%d to %d' % tuple(self._intgr_wedge)
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % tuple(self._intgr_cell)
if len(self._fp_directory) <= 50:
dirname = self._fp_directory
else:
dirname = '...%s' % self._fp_directory[-46:]
Journal.block(
'integrating', self._intgr_sweep_name, 'DIALS',
{'images':images_str,
'cell':cell_str,
'lattice':self.get_integrater_refiner().get_refiner_lattice(),
'template':self._fp_template,
'directory':dirname,
'resolution':'%.2f' % self._intgr_reso_high})
integrate = self.Integrate()
# decide what images we are going to process, if not already
# specified
if not self._intgr_wedge:
images = self.get_matching_images()
self.set_integrater_wedge(min(images),
max(images))
imageset = self.get_imageset()
beam = imageset.get_beam()
detector = imageset.get_detector()
d_min_limit = detector.get_max_resolution(beam.get_s0())
if d_min_limit > self._intgr_reso_high \
or PhilIndex.params.xia2.settings.resolution.keep_all_reflections:
Debug.write('Overriding high resolution limit: %f => %f' % \
(self._intgr_reso_high, d_min_limit))
self._intgr_reso_high = d_min_limit
integrate.set_experiments_filename(self._intgr_experiments_filename)
integrate.set_reflections_filename(self._intgr_indexed_filename)
integrate.set_d_max(self._intgr_reso_low)
integrate.set_d_min(self._intgr_reso_high)
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file('%s %s %s %s INTEGRATE' % \
(pname, xname, dname, sweep),
integrate.get_log_file())
try:
integrate.run()
except RuntimeError, e:
s = str(e)
if ('dials.integrate requires more memory than is available.' in s
and not self._intgr_reso_high):
# Try to estimate a more sensible resolution limit for integration
# in case we were just integrating noise to the edge of the detector
images = self._integrate_select_images_wedges()
Debug.write(
'Integrating subset of images to estimate resolution limit.\n'
'Integrating images %s' %images)
integrate = self.Integrate()
integrate.set_experiments_filename(self._intgr_experiments_filename)
integrate.set_reflections_filename(self._intgr_indexed_filename)
integrate.set_d_max(self._intgr_reso_low)
integrate.set_d_min(self._intgr_reso_high)
for (start, stop) in images:
integrate.add_scan_range(start-self.get_matching_images()[0], stop-self.get_matching_images()[0])
integrate.set_reflections_per_degree(1000)
integrate.run()
integrated_pickle = integrate.get_integrated_filename()
from xia2.Wrappers.Dials.EstimateResolutionLimit import EstimateResolutionLimit
d_min_estimater = EstimateResolutionLimit()
d_min_estimater.set_working_directory(self.get_working_directory())
auto_logfiler(d_min_estimater)
d_min_estimater.set_experiments_filename(self._intgr_experiments_filename)
d_min_estimater.set_reflections_filename(integrated_pickle)
d_min = d_min_estimater.run()
Debug.write('Estimate for d_min: %.2f' %d_min)
Debug.write('Re-running integration to this resolution limit')
self._intgr_reso_high = d_min
self.set_integrater_done(False)
return
raise
# FIXME in here should respect the input unit cell and lattice if provided
# FIXME from this (i) populate the helper table,
# (ii) try to avoid re-running the indexing
# step if we eliminate a solution as we have all of the refined results
# already available.
rbs = self.RefineBravaisSettings()
rbs.set_experiments_filename(indexed_experiments)
rbs.set_indexed_filename(indexed_file)
if PhilIndex.params.dials.fix_geometry:
rbs.set_detector_fix('all')
rbs.set_beam_fix('all')
FileHandler.record_log_file('%s LATTICE' % self.get_indexer_full_name(),
rbs.get_log_file())
rbs.run()
from cctbx import crystal, sgtbx
for k in sorted(rbs.get_bravais_summary()):
summary = rbs.get_bravais_summary()[k]
# FIXME need to do this better - for the moment only accept lattices
# where R.M.S. deviation is less than twice P1 R.M.S. deviation.
if self._indxr_input_lattice is None:
if not summary['recommended']:
continue
experiments = load.experiment_list(
