def _do_indexing(self, method=None):
indexer = self.Index()
for spot_list in self._indxr_payload["spot_lists"]:
indexer.add_spot_filename(spot_list)
for datablock in self._indxr_payload["datablocks"]:
indexer.add_sweep_filename(datablock)
if PhilIndex.params.dials.index.phil_file is not None:
indexer.set_phil_file(PhilIndex.params.dials.index.phil_file)
indexer.set_max_cell(max_cell=PhilIndex.params.dials.index.max_cell,
max_height_fraction=PhilIndex.params.dials.index.
max_cell_estimation.max_height_fraction)
if PhilIndex.params.xia2.settings.small_molecule == True:
indexer.set_min_cell(3)
if PhilIndex.params.dials.fix_geometry:
indexer.set_detector_fix('all')
indexer.set_beam_fix('all')
indexer.set_close_to_spindle_cutoff(
PhilIndex.params.dials.close_to_spindle_cutoff)
if self._indxr_input_lattice:
indexer.set_indexer_input_lattice(self._indxr_input_lattice)
Debug.write('Set lattice: %s' % self._indxr_input_lattice)
if self._indxr_input_cell:
indexer.set_indexer_input_cell(self._indxr_input_cell)
Debug.write('Set cell: %f %f %f %f %f %f' % \
self._indxr_input_cell)
original_cell = self._indxr_input_cell
if method is None:
if PhilIndex.params.dials.index.method is None:
method = 'fft3d'
Debug.write('Choosing indexing method: %s' % method)
else:
method = PhilIndex.params.dials.index.method
FileHandler.record_log_file('%s INDEX' % self.get_indexer_full_name(),
indexer.get_log_file())
indexer.run(method)
if not os.path.exists(indexer.get_experiments_filename()):
raise RuntimeError(
"Indexing has failed: see %s for more details." %
indexer.get_log_file())
elif not os.path.exists(indexer.get_indexed_filename()):
raise RuntimeError("Indexing has failed: %s does not exist." %
indexer.get_indexed_filename())
report = self.Report()
report.set_experiments_filename(indexer.get_experiments_filename())
report.set_reflections_filename(indexer.get_indexed_filename())
html_filename = os.path.join(
self.get_working_directory(),
'%i_dials.index.report.html' % report.get_xpid())
report.set_html_filename(html_filename)
report.run()
FileHandler.record_html_file('%s INDEX' % self.get_indexer_full_name(),
html_filename)
return indexer
def _do_indexing(self, method=None):
indexer = self.Index()
for spot_list in self._indxr_payload["spot_lists"]:
indexer.add_spot_filename(spot_list)
for datablock in self._indxr_payload["datablocks"]:
indexer.add_sweep_filename(datablock)
if PhilIndex.params.dials.index.phil_file is not None:
indexer.set_phil_file(PhilIndex.params.dials.index.phil_file)
if PhilIndex.params.dials.index.max_cell:
indexer.set_max_cell(PhilIndex.params.dials.index.max_cell)
if PhilIndex.params.xia2.settings.small_molecule == True:
indexer.set_min_cell(3)
if PhilIndex.params.dials.fix_geometry:
indexer.set_detector_fix('all')
indexer.set_beam_fix('all')
indexer.set_close_to_spindle_cutoff(
PhilIndex.params.dials.close_to_spindle_cutoff)
if self._indxr_input_lattice:
indexer.set_indexer_input_lattice(self._indxr_input_lattice)
Debug.write('Set lattice: %s' % self._indxr_input_lattice)
if self._indxr_input_cell:
indexer.set_indexer_input_cell(self._indxr_input_cell)
Debug.write('Set cell: %f %f %f %f %f %f' % \
self._indxr_input_cell)
original_cell = self._indxr_input_cell
if method is None:
if PhilIndex.params.dials.index.method is None:
method = 'fft3d'
Debug.write('Choosing indexing method: %s' % method)
else:
method = PhilIndex.params.dials.index.method
FileHandler.record_log_file('%s INDEX' % self.get_indexer_full_name(),
indexer.get_log_file())
indexer.run(method)
if not os.path.exists(indexer.get_experiments_filename()):
raise RuntimeError("Indexing has failed: see %s for more details."
%indexer.get_log_file())
elif not os.path.exists(indexer.get_indexed_filename()):
raise RuntimeError("Indexing has failed: %s does not exist."
%indexer.get_indexed_filename())
report = self.Report()
report.set_experiments_filename(indexer.get_experiments_filename())
report.set_reflections_filename(indexer.get_indexed_filename())
html_filename = os.path.join(
self.get_working_directory(),
'%i_dials.index.report.html' %report.get_xpid())
report.set_html_filename(html_filename)
report.run()
assert os.path.exists(html_filename)
FileHandler.record_html_file(
'%s INDEX' %self.get_indexer_full_name(), html_filename)
return indexer
def _scale_finish_chunk_2_report(self):
from cctbx.array_family import flex
from iotbx.reflection_file_reader import any_reflection_file
from xia2.lib.bits import auto_logfiler
from xia2.Wrappers.XIA.Report import Report
for wavelength in self._scalr_scaled_refl_files.keys():
mtz_unmerged = self._scalr_scaled_reflection_files['mtz_unmerged'][
wavelength]
reader = any_reflection_file(mtz_unmerged)
mtz_object = reader.file_content()
batches = mtz_object.as_miller_arrays_dict()['HKL_base',
'HKL_base', 'BATCH']
dose = flex.double(batches.size(), -1)
batch_to_dose = self.get_batch_to_dose()
for i, b in enumerate(batches.data()):
dose[i] = batch_to_dose[b]
c = mtz_object.crystals()[0]
d = c.datasets()[0]
d.add_column('DOSE', 'R').set_values(dose.as_float())
tmp_mtz = os.path.join(self.get_working_directory(),
'dose_tmp.mtz')
mtz_object.write(tmp_mtz)
hklin = tmp_mtz
FileHandler.record_temporary_file(hklin)
report = Report()
report.set_working_directory(self.get_working_directory())
report.set_mtz_filename(hklin)
htmlout = os.path.join(
self.get_working_directory(), '%s_%s_%s_report.html' %
(self._scalr_pname, self._scalr_xname, wavelength))
report.set_html_filename(htmlout)
report.set_chef_min_completeness(0.95) # sensible?
auto_logfiler(report)
try:
report.run()
FileHandler.record_html_file(
'%s %s %s report' %
(self._scalr_pname, self._scalr_xname, wavelength),
htmlout)
except Exception as e:
Debug.write('xia2.report failed:')
Debug.write(str(e))
def _refine(self):
for epoch, idxr in self._refinr_indexers.iteritems():
experiments = idxr.get_indexer_experiment_list()
indexed_experiments = idxr.get_indexer_payload(
"experiments_filename")
indexed_reflections = idxr.get_indexer_payload("indexed_filename")
if len(experiments) > 1:
xsweeps = idxr._indxr_sweeps
assert len(xsweeps) == len(experiments)
assert len(self._refinr_sweeps
) == 1 # don't currently support joint refinement
xsweep = self._refinr_sweeps[0]
i = xsweeps.index(xsweep)
experiments = experiments[i:i + 1]
# Extract and output experiment and reflections for current sweep
indexed_experiments = os.path.join(
self.get_working_directory(),
"%s_indexed_experiments.json" % xsweep.get_name())
indexed_reflections = os.path.join(
self.get_working_directory(),
"%s_indexed_reflections.pickle" % xsweep.get_name())
from dxtbx.serialize import dump
dump.experiment_list(experiments, indexed_experiments)
from libtbx import easy_pickle
from scitbx.array_family import flex
reflections = easy_pickle.load(
idxr.get_indexer_payload("indexed_filename"))
sel = reflections['id'] == i
assert sel.count(True) > 0
imageset_id = reflections['imageset_id'].select(sel)
assert imageset_id.all_eq(imageset_id[0])
sel = reflections['imageset_id'] == imageset_id[0]
reflections = reflections.select(sel)
# set indexed reflections to id == 0 and imageset_id == 0
reflections['id'].set_selected(reflections['id'] == i, 0)
reflections['imageset_id'] = flex.int(len(reflections), 0)
easy_pickle.dump(indexed_reflections, reflections)
assert len(experiments.crystals()
) == 1 # currently only handle one lattice/sweep
crystal_model = experiments.crystals()[0]
lattice = idxr.get_indexer_lattice()
from dxtbx.serialize import load
scan_static = PhilIndex.params.dials.refine.scan_static
# XXX Temporary workaround for dials.refine error for scan_varying
# refinement with smaller wedges
start, end = experiments[0].scan.get_oscillation_range()
total_phi_range = end - start
if (PhilIndex.params.dials.refine.scan_varying
and total_phi_range > 5
and not PhilIndex.params.dials.fast_mode):
scan_varying = PhilIndex.params.dials.refine.scan_varying
else:
scan_varying = False
if scan_static:
refiner = self.Refine()
refiner.set_experiments_filename(indexed_experiments)
refiner.set_indexed_filename(indexed_reflections)
refiner.set_scan_varying(False)
refiner.run()
self._refinr_experiments_filename \
= refiner.get_refined_experiments_filename()
self._refinr_indexed_filename = refiner.get_refined_filename()
else:
self._refinr_experiments_filename = indexed_experiments
self._refinr_indexed_filename = indexed_reflections
if scan_varying:
refiner = self.Refine()
refiner.set_experiments_filename(
self._refinr_experiments_filename)
refiner.set_indexed_filename(self._refinr_indexed_filename)
if total_phi_range < 36:
refiner.set_interval_width_degrees(total_phi_range / 2)
refiner.run()
self._refinr_experiments_filename \
= refiner.get_refined_experiments_filename()
self._refinr_indexed_filename = refiner.get_refined_filename()
if scan_static or scan_varying:
FileHandler.record_log_file(
'%s REFINE' % idxr.get_indexer_full_name(),
refiner.get_log_file())
report = self.Report()
report.set_experiments_filename(
self._refinr_experiments_filename)
report.set_reflections_filename(self._refinr_indexed_filename)
html_filename = os.path.join(
self.get_working_directory(),
'%i_dials.refine.report.html' % report.get_xpid())
report.set_html_filename(html_filename)
report.run()
FileHandler.record_html_file(
'%s REFINE' % idxr.get_indexer_full_name(), html_filename)
experiments = load.experiment_list(
self._refinr_experiments_filename)
self.set_refiner_payload("experiments.json",
self._refinr_experiments_filename)
self.set_refiner_payload("reflections.pickle",
self._refinr_indexed_filename)
# this is the result of the cell refinement
self._refinr_cell = experiments.crystals()[0].get_unit_cell(
).parameters()
def _do_indexing(self, method=None):
indexer = self.Index()
for spot_list in self._indxr_payload["spot_lists"]:
indexer.add_spot_filename(spot_list)
for filename in self._indxr_payload["experiments"]:
indexer.add_sweep_filename(filename)
if PhilIndex.params.dials.index.phil_file is not None:
indexer.set_phil_file(PhilIndex.params.dials.index.phil_file)
indexer.set_max_cell(
max_cell=PhilIndex.params.dials.index.max_cell,
max_height_fraction=PhilIndex.params.dials.index.
max_cell_estimation.max_height_fraction,
)
if PhilIndex.params.xia2.settings.small_molecule:
indexer.set_min_cell(3)
if PhilIndex.params.dials.fix_geometry:
indexer.set_detector_fix("all")
indexer.set_beam_fix("all")
elif PhilIndex.params.dials.fix_distance:
indexer.set_detector_fix("distance")
indexer.set_close_to_spindle_cutoff(
PhilIndex.params.dials.close_to_spindle_cutoff)
if self._indxr_input_lattice:
indexer.set_indexer_input_lattice(self._indxr_input_lattice)
logger.debug("Set lattice: %s", self._indxr_input_lattice)
if self._indxr_input_cell:
indexer.set_indexer_input_cell(self._indxr_input_cell)
logger.debug("Set cell: %f %f %f %f %f %f" %
self._indxr_input_cell)
if method is None:
if PhilIndex.params.dials.index.method is None:
method = "fft3d"
logger.debug("Choosing indexing method: %s", method)
else:
method = PhilIndex.params.dials.index.method
FileHandler.record_log_file("%s INDEX" % self.get_indexer_full_name(),
indexer.get_log_file())
indexer.run(method)
if not os.path.exists(indexer.get_experiments_filename()):
raise RuntimeError(
"Indexing has failed: see %s for more details." %
indexer.get_log_file())
elif not os.path.exists(indexer.get_indexed_filename()):
raise RuntimeError("Indexing has failed: %s does not exist." %
indexer.get_indexed_filename())
report = self.Report()
report.set_experiments_filename(indexer.get_experiments_filename())
report.set_reflections_filename(indexer.get_indexed_filename())
html_filename = os.path.join(
self.get_working_directory(),
"%i_dials.index.report.html" % report.get_xpid(),
)
report.set_html_filename(html_filename)
report.run()
FileHandler.record_html_file("%s INDEX" % self.get_indexer_full_name(),
html_filename)
return indexer
def _integrate(self):
'''Actually do the integration - in XDS terms this will mean running
DEFPIX and INTEGRATE to measure all the reflections.'''
images_str = '%d to %d' % tuple(self._intgr_wedge)
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % tuple(self._intgr_cell)
if len(self._fp_directory) <= 50:
dirname = self._fp_directory
else:
dirname = '...%s' % self._fp_directory[-46:]
Journal.block(
'integrating', self._intgr_sweep_name, 'DIALS', {
'images': images_str,
'cell': cell_str,
'lattice': self.get_integrater_refiner().get_refiner_lattice(),
'template': self._fp_template,
'directory': dirname,
'resolution': '%.2f' % self._intgr_reso_high
})
integrate = self.Integrate()
# decide what images we are going to process, if not already
# specified
if not self._intgr_wedge:
images = self.get_matching_images()
self.set_integrater_wedge(min(images), max(images))
imageset = self.get_imageset()
beam = imageset.get_beam()
detector = imageset.get_detector()
d_min_limit = detector.get_max_resolution(beam.get_s0())
if d_min_limit > self._intgr_reso_high \
or PhilIndex.params.xia2.settings.resolution.keep_all_reflections:
Debug.write('Overriding high resolution limit: %f => %f' % \
(self._intgr_reso_high, d_min_limit))
self._intgr_reso_high = d_min_limit
integrate.set_experiments_filename(self._intgr_experiments_filename)
integrate.set_reflections_filename(self._intgr_indexed_filename)
integrate.set_d_max(self._intgr_reso_low)
integrate.set_d_min(self._intgr_reso_high)
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file('%s %s %s %s INTEGRATE' % \
(pname, xname, dname, sweep),
integrate.get_log_file())
try:
integrate.run()
except xia2.Wrappers.Dials.Integrate.DIALSIntegrateError as e:
s = str(e)
if ('dials.integrate requires more memory than is available.' in s
and not self._intgr_reso_high):
# Try to estimate a more sensible resolution limit for integration
# in case we were just integrating noise to the edge of the detector
images = self._integrate_select_images_wedges()
Debug.write(
'Integrating subset of images to estimate resolution limit.\n'
'Integrating images %s' % images)
integrate = self.Integrate()
integrate.set_experiments_filename(
self._intgr_experiments_filename)
integrate.set_reflections_filename(
self._intgr_indexed_filename)
integrate.set_d_max(self._intgr_reso_low)
integrate.set_d_min(self._intgr_reso_high)
for (start, stop) in images:
integrate.add_scan_range(
start - self.get_matching_images()[0],
stop - self.get_matching_images()[0])
integrate.set_reflections_per_degree(1000)
integrate.run()
integrated_pickle = integrate.get_integrated_filename()
from xia2.Wrappers.Dials.EstimateResolutionLimit import EstimateResolutionLimit
d_min_estimater = EstimateResolutionLimit()
d_min_estimater.set_working_directory(
self.get_working_directory())
auto_logfiler(d_min_estimater)
d_min_estimater.set_experiments_filename(
self._intgr_experiments_filename)
d_min_estimater.set_reflections_filename(integrated_pickle)
d_min = d_min_estimater.run()
Debug.write('Estimate for d_min: %.2f' % d_min)
Debug.write('Re-running integration to this resolution limit')
self._intgr_reso_high = d_min
self.set_integrater_done(False)
return
raise Sorry(e)
self._intgr_experiments_filename = integrate.get_integrated_experiments(
)
# also record the batch range - needed for the analysis of the
# radiation damage in chef...
self._intgr_batches_out = (self._intgr_wedge[0], self._intgr_wedge[1])
# FIXME (i) record the log file, (ii) get more information out from the
# integration log on the quality of the data and (iii) the mosaic spread
# range observed and R.M.S. deviations.
self._intgr_integrated_pickle = integrate.get_integrated_reflections()
if not os.path.isfile(self._intgr_integrated_pickle):
raise RuntimeError("Integration failed: %s does not exist." %
self._intgr_integrated_pickle)
self._intgr_per_image_statistics = integrate.get_per_image_statistics()
Chatter.write(self.show_per_image_statistics())
report = self.Report()
html_filename = os.path.join(
self.get_working_directory(),
'%i_dials.integrate.report.html' % report.get_xpid())
report.set_html_filename(html_filename)
report.run()
FileHandler.record_html_file('%s %s %s %s INTEGRATE' % \
(pname, xname, dname, sweep),
html_filename)
import dials
from dxtbx.serialize import load
experiments = load.experiment_list(self._intgr_experiments_filename)
profile = experiments.profiles()[0]
mosaic = profile.sigma_m()
self.set_integrater_mosaic_min_mean_max(mosaic, mosaic, mosaic)
Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
self.get_integrater_mosaic_min_mean_max())
return self._intgr_integrated_pickle
def _integrate(self):
"""Actually do the integration - in XDS terms this will mean running
DEFPIX and INTEGRATE to measure all the reflections."""
integrate = self.Integrate()
# decide what images we are going to process, if not already
# specified
if not self._intgr_wedge:
images = self.get_matching_images()
self.set_integrater_wedge(min(images), max(images))
imageset = self.get_imageset()
beam = imageset.get_beam()
detector = imageset.get_detector()
d_min_limit = detector.get_max_resolution(beam.get_s0())
if (d_min_limit > self._intgr_reso_high or
PhilIndex.params.xia2.settings.resolution.keep_all_reflections
):
logger.debug("Overriding high resolution limit: %f => %f" %
(self._intgr_reso_high, d_min_limit))
self._intgr_reso_high = d_min_limit
integrate.set_experiments_filename(self._intgr_experiments_filename)
integrate.set_reflections_filename(self._intgr_indexed_filename)
if PhilIndex.params.dials.integrate.d_max:
integrate.set_d_max(PhilIndex.params.dials.integrate.d_max)
else:
integrate.set_d_max(self._intgr_reso_low)
if PhilIndex.params.dials.integrate.d_min:
integrate.set_d_min(PhilIndex.params.dials.integrate.d_min)
else:
integrate.set_d_min(self._intgr_reso_high)
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file(
f"{pname} {xname} {dname} {sweep} INTEGRATE",
integrate.get_log_file(),
)
integrate.run()
self._intgr_experiments_filename = integrate.get_integrated_experiments(
)
# also record the batch range - needed for the analysis of the
# radiation damage in chef...
self._intgr_batches_out = (self._intgr_wedge[0], self._intgr_wedge[1])
# FIXME (i) record the log file, (ii) get more information out from the
# integration log on the quality of the data and (iii) the mosaic spread
# range observed and R.M.S. deviations.
self._intgr_integrated_reflections = integrate.get_integrated_reflections(
)
if not os.path.isfile(self._intgr_integrated_reflections):
raise RuntimeError("Integration failed: %s does not exist." %
self._intgr_integrated_reflections)
self._intgr_per_image_statistics = integrate.get_per_image_statistics()
logger.info(self.show_per_image_statistics())
report = self.Report()
html_filename = os.path.join(
self.get_working_directory(),
"%i_dials.integrate.report.html" % report.get_xpid(),
)
report.set_html_filename(html_filename)
report.run(wait_for_completion=True)
FileHandler.record_html_file(
f"{pname} {xname} {dname} {sweep} INTEGRATE", html_filename)
experiments = load.experiment_list(self._intgr_experiments_filename)
profile = experiments.profiles()[0]
mosaic = profile.sigma_m()
try:
m_min, m_max, m_mean = mosaic.min_max_mean().as_tuple()
self.set_integrater_mosaic_min_mean_max(m_min, m_mean, m_max)
except AttributeError:
self.set_integrater_mosaic_min_mean_max(mosaic, mosaic, mosaic)
logger.info("Mosaic spread: %.3f < %.3f < %.3f" %
self.get_integrater_mosaic_min_mean_max())
# If running in high-pressure mode, run dials.anvil_correction to
# correct for the attenuation of the incident and diffracted beams by the
# diamond anvils.
if self.high_pressure:
self._anvil_correction()
return self._intgr_integrated_reflections
def _refine(self):
for idxr in set(self._refinr_indexers.values()):
experiments = idxr.get_indexer_experiment_list()
indexed_experiments = idxr.get_indexer_payload(
"experiments_filename")
indexed_reflections = idxr.get_indexer_payload("indexed_filename")
# If multiple sweeps but not doing joint refinement, get only the
# relevant reflections.
multi_sweep = PhilIndex.params.xia2.settings.multi_sweep_refinement
if len(experiments) > 1 and not multi_sweep:
xsweeps = idxr._indxr_sweeps
assert len(xsweeps) == len(experiments)
# Don't do joint refinement
assert len(self._refinr_sweeps) == 1
xsweep = self._refinr_sweeps[0]
i = xsweeps.index(xsweep)
experiments = experiments[i:i + 1]
# Extract and output experiment and reflections for current sweep
indexed_experiments = os.path.join(
self.get_working_directory(),
"%s_indexed.expt" % xsweep.get_name())
indexed_reflections = os.path.join(
self.get_working_directory(),
"%s_indexed.refl" % xsweep.get_name())
experiments.as_file(indexed_experiments)
reflections = flex.reflection_table.from_file(
idxr.get_indexer_payload("indexed_filename"))
sel = reflections["id"] == i
assert sel.count(True) > 0
imageset_id = reflections["imageset_id"].select(sel)
assert imageset_id.all_eq(imageset_id[0])
sel = reflections["imageset_id"] == imageset_id[0]
reflections = reflections.select(sel)
# set indexed reflections to id == 0 and imageset_id == 0
reflections["id"].set_selected(reflections["id"] == i, 0)
reflections["imageset_id"] = flex.int(len(reflections), 0)
reflections.as_file(indexed_reflections)
# currently only handle one lattice/refiner
assert len(experiments.crystals()) == 1
scan_static = PhilIndex.params.dials.refine.scan_static
# Avoid doing scan-varying refinement on narrow wedges.
scan_oscillation_ranges = []
for experiment in experiments:
start, end = experiment.scan.get_oscillation_range()
scan_oscillation_ranges.append(end - start)
min_oscillation_range = min(scan_oscillation_ranges)
if (PhilIndex.params.dials.refine.scan_varying
and min_oscillation_range > 5
and not PhilIndex.params.dials.fast_mode):
scan_varying = PhilIndex.params.dials.refine.scan_varying
else:
scan_varying = False
if scan_static:
refiner = self.Refine()
refiner.set_experiments_filename(indexed_experiments)
refiner.set_indexed_filename(indexed_reflections)
refiner.set_scan_varying(False)
refiner.run()
self._refinr_experiments_filename = (
refiner.get_refined_experiments_filename())
self._refinr_indexed_filename = refiner.get_refined_filename()
else:
self._refinr_experiments_filename = indexed_experiments
self._refinr_indexed_filename = indexed_reflections
if scan_varying:
refiner = self.Refine()
refiner.set_experiments_filename(
self._refinr_experiments_filename)
refiner.set_indexed_filename(self._refinr_indexed_filename)
if min_oscillation_range < 36:
refiner.set_interval_width_degrees(min_oscillation_range /
2)
refiner.run()
self._refinr_experiments_filename = (
refiner.get_refined_experiments_filename())
self._refinr_indexed_filename = refiner.get_refined_filename()
if scan_static or scan_varying:
FileHandler.record_log_file(
"%s REFINE" % idxr.get_indexer_full_name(),
refiner.get_log_file())
report = self.Report()
report.set_experiments_filename(
self._refinr_experiments_filename)
report.set_reflections_filename(self._refinr_indexed_filename)
html_filename = os.path.join(
self.get_working_directory(),
"%i_dials.refine.report.html" % report.get_xpid(),
)
report.set_html_filename(html_filename)
report.run()
FileHandler.record_html_file(
"%s REFINE" % idxr.get_indexer_full_name(), html_filename)
experiments = ExperimentList.from_file(
self._refinr_experiments_filename)
self.set_refiner_payload("models.expt",
self._refinr_experiments_filename)
self.set_refiner_payload("observations.refl",
self._refinr_indexed_filename)
# this is the result of the cell refinement
self._refinr_cell = experiments.crystals()[0].get_unit_cell(
).parameters()
def _scale_finish(self):
# compute anomalous signals if anomalous
if self.get_scaler_anomalous():
for key in self._scalr_scaled_refl_files:
f = self._scalr_scaled_refl_files[key]
from iotbx import mtz
m = mtz.object(f)
if m.space_group().is_centric():
Debug.write('Spacegroup is centric: %s' % f)
continue
Debug.write('Running anomalous signal analysis on %s' % f)
a_s = anomalous_signals(f)
self._scalr_statistics[
(self._scalr_pname, self._scalr_xname, key)
]['dF/F'] = [a_s[0]]
self._scalr_statistics[
(self._scalr_pname, self._scalr_xname, key)
]['dI/s(dI)'] = [a_s[1]]
# next transform to F's from I's etc.
if len(self._scalr_scaled_refl_files.keys()) == 0:
raise RuntimeError, 'no reflection files stored'
# run xia2.report on each unmerged mtz file
from iotbx.reflection_file_reader import any_reflection_file
from iotbx import mtz
from cctbx.array_family import flex
for wavelength in self._scalr_scaled_refl_files.keys():
mtz_unmerged = self._scalr_scaled_reflection_files['mtz_unmerged'][wavelength]
reader = any_reflection_file(mtz_unmerged)
mtz_object = reader.file_content()
batches = mtz_object.as_miller_arrays_dict()['HKL_base', 'HKL_base', 'BATCH']
dose = flex.double(batches.size(), -1)
batch_to_dose = self.get_batch_to_dose()
for i, b in enumerate(batches.data()):
dose[i] = batch_to_dose[b]
c = mtz_object.crystals()[0]
d = c.datasets()[0]
d.add_column('DOSE', 'R').set_values(dose.as_float())
tmp_mtz = os.path.join(self.get_working_directory(), 'dose_tmp.mtz')
mtz_object.write(tmp_mtz)
hklin = tmp_mtz
FileHandler.record_temporary_file(hklin)
from xia2.Wrappers.XIA.Report import Report
report = Report()
report.set_working_directory(self.get_working_directory())
report.set_mtz_filename(hklin)
htmlout = os.path.join(
self.get_working_directory(), '%s_%s_%s_report.html' %(
self._scalr_pname, self._scalr_xname, wavelength))
report.set_html_filename(htmlout)
report.set_chef_min_completeness(0.95) # sensible?
from xia2.lib.bits import auto_logfiler
auto_logfiler(report)
try:
report.run()
except Exception, e:
Debug.write('xia2.report failed:')
Debug.write(str(e))
continue
FileHandler.record_html_file(
'%s %s %s report' %(
self._scalr_pname, self._scalr_xname, wavelength), htmlout)
def _integrate(self):
"""Actually do the integration - in XDS terms this will mean running
DEFPIX and INTEGRATE to measure all the reflections."""
images_str = "%d to %d" % tuple(self._intgr_wedge)
cell_str = "%.2f %.2f %.2f %.2f %.2f %.2f" % tuple(self._intgr_cell)
if len(self._fp_directory) <= 50:
dirname = self._fp_directory
else:
dirname = "...%s" % self._fp_directory[-46:]
Journal.block(
"integrating",
self._intgr_sweep_name,
"DIALS",
{
"images": images_str,
"cell": cell_str,
"lattice": self.get_integrater_refiner().get_refiner_lattice(),
"template": self._fp_template,
"directory": dirname,
"resolution": "%.2f" % self._intgr_reso_high,
},
)
integrate = self.Integrate()
# decide what images we are going to process, if not already
# specified
if not self._intgr_wedge:
images = self.get_matching_images()
self.set_integrater_wedge(min(images), max(images))
imageset = self.get_imageset()
beam = imageset.get_beam()
detector = imageset.get_detector()
d_min_limit = detector.get_max_resolution(beam.get_s0())
if (d_min_limit > self._intgr_reso_high or
PhilIndex.params.xia2.settings.resolution.keep_all_reflections
):
Debug.write("Overriding high resolution limit: %f => %f" %
(self._intgr_reso_high, d_min_limit))
self._intgr_reso_high = d_min_limit
integrate.set_experiments_filename(self._intgr_experiments_filename)
integrate.set_reflections_filename(self._intgr_indexed_filename)
if PhilIndex.params.dials.integrate.d_max:
integrate.set_d_max(PhilIndex.params.dials.integrate.d_max)
else:
integrate.set_d_max(self._intgr_reso_low)
if PhilIndex.params.dials.integrate.d_min:
integrate.set_d_min(PhilIndex.params.dials.integrate.d_min)
else:
integrate.set_d_min(self._intgr_reso_high)
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file(
"%s %s %s %s INTEGRATE" % (pname, xname, dname, sweep),
integrate.get_log_file(),
)
try:
integrate.run()
except xia2.Wrappers.Dials.Integrate.DIALSIntegrateError as e:
s = str(e)
if ("dials.integrate requires more memory than is available." in s
and not self._intgr_reso_high):
# Try to estimate a more sensible resolution limit for integration
# in case we were just integrating noise to the edge of the detector
images = self._integrate_select_images_wedges()
Debug.write(
"Integrating subset of images to estimate resolution limit.\n"
"Integrating images %s" % images)
integrate = self.Integrate()
integrate.set_experiments_filename(
self._intgr_experiments_filename)
integrate.set_reflections_filename(
self._intgr_indexed_filename)
integrate.set_d_max(self._intgr_reso_low)
integrate.set_d_min(self._intgr_reso_high)
for (start, stop) in images:
integrate.add_scan_range(
start - self.get_matching_images()[0],
stop - self.get_matching_images()[0],
)
integrate.set_reflections_per_degree(1000)
integrate.run()
integrated_reflections = integrate.get_integrated_filename()
from xia2.Wrappers.Dials.EstimateResolutionLimit import (
EstimateResolutionLimit, )
d_min_estimater = EstimateResolutionLimit()
d_min_estimater.set_working_directory(
self.get_working_directory())
auto_logfiler(d_min_estimater)
d_min_estimater.set_experiments_filename(
self._intgr_experiments_filename)
d_min_estimater.set_reflections_filename(
integrated_reflections)
d_min = d_min_estimater.run()
Debug.write("Estimate for d_min: %.2f" % d_min)
Debug.write("Re-running integration to this resolution limit")
self._intgr_reso_high = d_min
self.set_integrater_done(False)
return
raise Sorry(e)
self._intgr_experiments_filename = integrate.get_integrated_experiments(
)
# also record the batch range - needed for the analysis of the
# radiation damage in chef...
self._intgr_batches_out = (self._intgr_wedge[0], self._intgr_wedge[1])
# FIXME (i) record the log file, (ii) get more information out from the
# integration log on the quality of the data and (iii) the mosaic spread
# range observed and R.M.S. deviations.
self._intgr_integrated_reflections = integrate.get_integrated_reflections(
)
if not os.path.isfile(self._intgr_integrated_reflections):
raise RuntimeError("Integration failed: %s does not exist." %
self._intgr_integrated_reflections)
self._intgr_per_image_statistics = integrate.get_per_image_statistics()
Chatter.write(self.show_per_image_statistics())
report = self.Report()
html_filename = os.path.join(
self.get_working_directory(),
"%i_dials.integrate.report.html" % report.get_xpid(),
)
report.set_html_filename(html_filename)
report.run(wait_for_completion=True)
FileHandler.record_html_file(
"%s %s %s %s INTEGRATE" % (pname, xname, dname, sweep),
html_filename)
from dxtbx.serialize import load
experiments = load.experiment_list(self._intgr_experiments_filename)
profile = experiments.profiles()[0]
mosaic = profile.sigma_m()
try:
m_min, m_max, m_mean = mosaic.min_max_mean().as_tuple()
self.set_integrater_mosaic_min_mean_max(m_min, m_mean, m_max)
except AttributeError as e:
self.set_integrater_mosaic_min_mean_max(mosaic, mosaic, mosaic)
Chatter.write("Mosaic spread: %.3f < %.3f < %.3f" %
self.get_integrater_mosaic_min_mean_max())
return self._intgr_integrated_reflections
