def exercise_dials_wrappers(nproc=None):
if not have_dials_regression:
print "Skipping exercise_dials_wrappers(): dials_regression not configured"
return
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
from xia2.Wrappers.Dials.Import import Import
from xia2.Wrappers.Dials.Spotfinder import Spotfinder
from xia2.Wrappers.Dials.Index import Index
from xia2.Wrappers.Dials.Integrate import Integrate
from xia2.Wrappers.Dials.RefineBravaisSettings import RefineBravaisSettings
from xia2.Wrappers.Dials.Refine import Refine
from xia2.Wrappers.Dials.Reindex import Reindex
from xia2.Wrappers.Dials.ExportMtz import ExportMtz
from xia2.Wrappers.Dials.ExportXDSASCII import ExportXDSASCII
xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
template = os.path.join(xia2_demo_data, "insulin_1_%03i.img")
scan_ranges = [(1, 45)]
image_range = (1, 45)
cwd = os.path.abspath(os.curdir)
tmp_dir = os.path.abspath(open_tmp_directory())
os.chdir(tmp_dir)
print "Begin importing"
importer = Import()
importer.setup_from_image(template %1)
importer.set_image_range(image_range)
importer.run()
print ''.join(importer.get_all_output())
print "Done importing"
print "Begin spotfinding"
spotfinder = Spotfinder()
spotfinder.set_input_sweep_filename(importer.get_sweep_filename())
spotfinder.set_scan_ranges(scan_ranges)
spotfinder.run()
print ''.join(spotfinder.get_all_output())
print "Done spotfinding"
print "Begin indexing"
indexer = Index()
indexer.add_spot_filename(spotfinder.get_spot_filename())
indexer.add_sweep_filename(importer.get_sweep_filename())
indexer.run('fft3d')
print ''.join(indexer.get_all_output())
print "Done indexing"
print "Begin refining"
rbs = RefineBravaisSettings()
rbs.set_experiments_filename(indexer.get_experiments_filename())
rbs.set_indexed_filename(indexer.get_indexed_filename())
rbs.run()
print ''.join(rbs.get_all_output())
print "Done refining"
bravais_setting_22 = rbs.get_bravais_summary()[22]
assert bravais_setting_22['bravais'] == 'cI'
assert bravais_setting_22['cb_op'] == 'b+c,a+c,a+b'
assert approx_equal(bravais_setting_22['unit_cell'],
(78.14,78.14,78.14,90,90,90), eps=1e-1)
bravais_setting_22_json = bravais_setting_22['experiments_file']
assert os.path.exists(bravais_setting_22_json)
print "Begin reindexing"
reindexer = Reindex()
reindexer.set_experiments_filename(indexer.get_experiments_filename())
reindexer.set_indexed_filename(indexer.get_indexed_filename())
reindexer.set_cb_op(bravais_setting_22['cb_op'])
reindexer.run()
assert os.path.exists(reindexer.get_reindexed_experiments_filename())
assert os.path.exists(reindexer.get_reindexed_reflections_filename())
print ''.join(reindexer.get_all_output())
print "Done reindexing"
print "Begin refining"
refiner = Refine()
refiner.set_experiments_filename(bravais_setting_22_json)
refiner.set_indexed_filename(reindexer.get_reindexed_reflections_filename())
refiner.set_scan_varying(True)
refiner.run()
assert os.path.exists(refiner.get_refined_experiments_filename())
print ''.join(refiner.get_all_output())
print "Done refining"
print "Begin integrating"
integrater = Integrate()
integrater.set_experiments_filename(refiner.get_refined_experiments_filename())
integrater.set_reflections_filename(reindexer.get_reindexed_reflections_filename())
integrater.run()
print ''.join(integrater.get_all_output())
print "Done integrating"
print "Begin exporting"
exporter = ExportMtz()
exporter.set_experiments_filename(integrater.get_integrated_experiments())
exporter.set_reflections_filename(integrater.get_integrated_reflections())
exporter.run()
print ''.join(exporter.get_all_output())
print "Done exporting"
assert os.path.exists(exporter.get_mtz_filename())
print "Begin exporting xds_ascii"
exporter = ExportXDSASCII()
exporter.set_experiments_filename(integrater.get_integrated_experiments())
exporter.set_reflections_filename(integrater.get_integrated_reflections())
exporter.run()
print ''.join(exporter.get_all_output())
print "Done exporting"
assert os.path.exists(exporter.get_hkl_filename())
import shutil
shutil.copy(indexer.get_experiments_filename(), "copy.json")
shutil.copy(indexer.get_indexed_filename(), "copy.pickle")
print "Begin combining"
from xia2.Wrappers.Dials.CombineExperiments import CombineExperiments
exporter = CombineExperiments()
exporter.add_experiments(indexer.get_experiments_filename())
exporter.add_experiments("copy.json")
exporter.add_reflections(indexer.get_indexed_filename())
exporter.add_reflections("copy.pickle")
exporter.run()
print ''.join(exporter.get_all_output())
print "Done combining"
assert os.path.exists(exporter.get_combined_experiments_filename())
assert os.path.exists(exporter.get_combined_reflections_filename())
def exercise_dials_wrappers(template, nproc=None):
if nproc is not None:
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
scan_ranges = [(1, 45)]
image_range = (1, 45)
print("Begin importing")
importer = Import()
importer.setup_from_image(template % 1)
importer.set_image_range(image_range)
importer.run()
print("".join(importer.get_all_output()))
print("Done importing")
print("Begin spotfinding")
spotfinder = Spotfinder()
spotfinder.set_input_sweep_filename(importer.get_sweep_filename())
spotfinder.set_scan_ranges(scan_ranges)
spotfinder.run()
print("".join(spotfinder.get_all_output()))
print("Done spotfinding")
print("Begin indexing")
indexer = Index()
indexer.add_spot_filename(spotfinder.get_spot_filename())
indexer.add_sweep_filename(importer.get_sweep_filename())
indexer.run("fft3d")
print("".join(indexer.get_all_output()))
print("Done indexing")
print("Begin refining")
rbs = RefineBravaisSettings()
rbs.set_experiments_filename(indexer.get_experiments_filename())
rbs.set_indexed_filename(indexer.get_indexed_filename())
rbs.run()
print("".join(rbs.get_all_output()))
print("Done refining")
bravais_setting_22 = rbs.get_bravais_summary()[22]
assert bravais_setting_22["bravais"] == "cI"
assert bravais_setting_22["cb_op"] == "b+c,a+c,a+b"
assert bravais_setting_22["unit_cell"] == pytest.approx(
(78.14, 78.14, 78.14, 90, 90, 90), abs=1e-1
)
bravais_setting_22_json = bravais_setting_22["experiments_file"]
assert os.path.exists(bravais_setting_22_json)
print("Begin reindexing")
reindexer = Reindex()
reindexer.set_experiments_filename(indexer.get_experiments_filename())
reindexer.set_indexed_filename(indexer.get_indexed_filename())
reindexer.set_cb_op(bravais_setting_22["cb_op"])
reindexer.run()
assert os.path.exists(reindexer.get_reindexed_experiments_filename())
assert os.path.exists(reindexer.get_reindexed_reflections_filename())
print("".join(reindexer.get_all_output()))
print("Done reindexing")
print("Begin refining")
refiner = Refine()
refiner.set_experiments_filename(bravais_setting_22_json)
refiner.set_indexed_filename(reindexer.get_reindexed_reflections_filename())
refiner.set_scan_varying(True)
refiner.run()
assert os.path.exists(refiner.get_refined_experiments_filename())
print("".join(refiner.get_all_output()))
print("Done refining")
print("Begin integrating")
integrater = Integrate()
integrater.set_experiments_filename(refiner.get_refined_experiments_filename())
integrater.set_reflections_filename(reindexer.get_reindexed_reflections_filename())
integrater.run()
print("".join(integrater.get_all_output()))
print("Done integrating")
print("Begin exporting")
exporter = ExportMtz()
exporter.set_experiments_filename(integrater.get_integrated_experiments())
exporter.set_reflections_filename(integrater.get_integrated_reflections())
exporter.run()
print("".join(exporter.get_all_output()))
print("Done exporting")
assert os.path.exists(exporter.get_mtz_filename())
print("Begin exporting xds_ascii")
exporter = ExportXDSASCII()
exporter.set_experiments_filename(integrater.get_integrated_experiments())
exporter.set_reflections_filename(integrater.get_integrated_reflections())
exporter.run()
print("".join(exporter.get_all_output()))
print("Done exporting")
assert os.path.exists(exporter.get_hkl_filename())
# Test combine experiments wrapper. Duplicate the file and adjust the
# identifier to allow combination.
shutil.copy(indexer.get_experiments_filename(), "copy.expt")
shutil.copy(indexer.get_indexed_filename(), "copy.refl")
expts = load.experiment_list("copy.expt")
expts[0].identifier = "0"
expts.as_file("copy.expt")
refls = flex.reflection_table.from_file("copy.refl")
refls.experiment_identifiers()[0] = "0"
refls.as_file("copy.refl")
exporter = CombineExperiments()
exporter.add_experiments(indexer.get_experiments_filename())
exporter.add_experiments("copy.expt")
exporter.add_reflections(indexer.get_indexed_filename())
exporter.add_reflections("copy.refl")
exporter.run()
print("".join(exporter.get_all_output()))
print("Done combining")
assert os.path.exists(exporter.get_combined_experiments_filename())
assert os.path.exists(exporter.get_combined_reflections_filename())
def exercise_dials_wrappers(dials_regression, nproc=None):
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
from xia2.Wrappers.Dials.Import import Import
from xia2.Wrappers.Dials.Spotfinder import Spotfinder
from xia2.Wrappers.Dials.Index import Index
from xia2.Wrappers.Dials.Integrate import Integrate
from xia2.Wrappers.Dials.RefineBravaisSettings import RefineBravaisSettings
from xia2.Wrappers.Dials.Refine import Refine
from xia2.Wrappers.Dials.Reindex import Reindex
from xia2.Wrappers.Dials.ExportMtz import ExportMtz
from xia2.Wrappers.Dials.ExportXDSASCII import ExportXDSASCII
xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
template = os.path.join(xia2_demo_data, "insulin_1_%03i.img")
scan_ranges = [(1, 45)]
image_range = (1, 45)
print("Begin importing")
importer = Import()
importer.setup_from_image(template % 1)
importer.set_image_range(image_range)
importer.run()
print(''.join(importer.get_all_output()))
print("Done importing")
print("Begin spotfinding")
spotfinder = Spotfinder()
spotfinder.set_input_sweep_filename(importer.get_sweep_filename())
spotfinder.set_scan_ranges(scan_ranges)
spotfinder.run()
print(''.join(spotfinder.get_all_output()))
print("Done spotfinding")
print("Begin indexing")
indexer = Index()
indexer.add_spot_filename(spotfinder.get_spot_filename())
indexer.add_sweep_filename(importer.get_sweep_filename())
indexer.run('fft3d')
print(''.join(indexer.get_all_output()))
print("Done indexing")
print("Begin refining")
rbs = RefineBravaisSettings()
rbs.set_experiments_filename(indexer.get_experiments_filename())
rbs.set_indexed_filename(indexer.get_indexed_filename())
rbs.run()
print(''.join(rbs.get_all_output()))
print("Done refining")
bravais_setting_22 = rbs.get_bravais_summary()[22]
assert bravais_setting_22['bravais'] == 'cI'
assert bravais_setting_22['cb_op'] == 'b+c,a+c,a+b'
assert approx_equal(bravais_setting_22['unit_cell'],
(78.14, 78.14, 78.14, 90, 90, 90),
eps=1e-1)
bravais_setting_22_json = bravais_setting_22['experiments_file']
assert os.path.exists(bravais_setting_22_json)
print("Begin reindexing")
reindexer = Reindex()
reindexer.set_experiments_filename(indexer.get_experiments_filename())
reindexer.set_indexed_filename(indexer.get_indexed_filename())
reindexer.set_cb_op(bravais_setting_22['cb_op'])
reindexer.run()
assert os.path.exists(reindexer.get_reindexed_experiments_filename())
assert os.path.exists(reindexer.get_reindexed_reflections_filename())
print(''.join(reindexer.get_all_output()))
print("Done reindexing")
print("Begin refining")
refiner = Refine()
refiner.set_experiments_filename(bravais_setting_22_json)
refiner.set_indexed_filename(
reindexer.get_reindexed_reflections_filename())
refiner.set_scan_varying(True)
refiner.run()
assert os.path.exists(refiner.get_refined_experiments_filename())
print(''.join(refiner.get_all_output()))
print("Done refining")
print("Begin integrating")
integrater = Integrate()
integrater.set_experiments_filename(
refiner.get_refined_experiments_filename())
integrater.set_reflections_filename(
reindexer.get_reindexed_reflections_filename())
integrater.run()
print(''.join(integrater.get_all_output()))
print("Done integrating")
print("Begin exporting")
exporter = ExportMtz()
exporter.set_experiments_filename(integrater.get_integrated_experiments())
exporter.set_reflections_filename(integrater.get_integrated_reflections())
exporter.run()
print(''.join(exporter.get_all_output()))
print("Done exporting")
assert os.path.exists(exporter.get_mtz_filename())
print("Begin exporting xds_ascii")
exporter = ExportXDSASCII()
exporter.set_experiments_filename(integrater.get_integrated_experiments())
exporter.set_reflections_filename(integrater.get_integrated_reflections())
exporter.run()
print(''.join(exporter.get_all_output()))
print("Done exporting")
assert os.path.exists(exporter.get_hkl_filename())
import shutil
shutil.copy(indexer.get_experiments_filename(), "copy.json")
shutil.copy(indexer.get_indexed_filename(), "copy.pickle")
print("Begin combining")
from xia2.Wrappers.Dials.CombineExperiments import CombineExperiments
exporter = CombineExperiments()
exporter.add_experiments(indexer.get_experiments_filename())
exporter.add_experiments("copy.json")
exporter.add_reflections(indexer.get_indexed_filename())
exporter.add_reflections("copy.pickle")
exporter.run()
print(''.join(exporter.get_all_output()))
print("Done combining")
assert os.path.exists(exporter.get_combined_experiments_filename())
assert os.path.exists(exporter.get_combined_reflections_filename())
def run(self):
from xia2.Handlers.Streams import Chatter, Debug
if self._reindexing_operators:
Debug.write("Reindexing sweeps for dials.two_theta_refine")
from xia2.lib.bits import auto_logfiler
from xia2.Wrappers.Dials.Reindex import Reindex
self._reindexed_experiments, self._reindexed_reflections = [], []
for e, p, op in zip(
self._experiments,
self._reflection_files,
self._reindexing_operators,
):
reindexer = Reindex()
reindexer.set_cb_op(op)
reindexer.set_experiments_filename(e)
reindexer.set_indexed_filename(p)
reindexer.set_working_directory(
self.get_working_directory())
auto_logfiler(reindexer)
reindexer.run()
self._reindexed_experiments.append(
reindexer.get_reindexed_experiments_filename())
self._reindexed_reflections.append(
reindexer.get_reindexed_reflections_filename())
Debug.write("Running dials.two_theta_refine")
self._output_cif = os.path.join(
self.get_working_directory(),
"%s_dials.two_theta_refine.cif" % self.get_xpid(),
)
self._output_mmcif = os.path.join(
self.get_working_directory(),
"%s_dials.two_theta_refine.mmcif" % self.get_xpid(),
)
if not self._output_p4p:
self._output_p4p = os.path.join(
self.get_working_directory(),
"%s_dials.two_theta_refine.p4p" % self.get_xpid(),
)
self._output_correlation_plot = os.path.join(
self.get_working_directory(),
"%s_dials.two_theta_refine.png" % self.get_xpid(),
)
self._output_experiments = os.path.join(
self.get_working_directory(),
"%s_refined_cell.expt" % self.get_xpid())
self.clear_command_line()
if self._reindexing_operators:
for experiment in self._reindexed_experiments:
self.add_command_line(experiment)
for reflection_file in self._reindexed_reflections:
self.add_command_line(reflection_file)
else:
for experiment in self._experiments:
self.add_command_line(experiment)
for reflection_file in self._reflection_files:
self.add_command_line(reflection_file)
self.add_command_line("combine_crystal_models=%s" %
self._combine_crystal_models)
self.add_command_line("output.cif=%s" % self._output_cif)
self.add_command_line("output.mmcif=%s" % self._output_mmcif)
self.add_command_line("output.p4p=%s" % self._output_p4p)
if self._output_correlation_plot is not None:
self.add_command_line("output.correlation_plot.filename=%s" %
self._output_correlation_plot)
if self._output_experiments is not None:
self.add_command_line("output.experiments=%s" %
self._output_experiments)
if self._phil_file is not None:
self.add_command_line("%s" % self._phil_file)
self.start()
self.close_wait()
if not os.path.isfile(self._output_cif):
Chatter.write(
"TwoTheta refinement failed, see log file for more details:\n %s"
% self.get_log_file())
raise RuntimeError("unit cell not refined")
self.check_for_errors()
from dxtbx.model.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(
self.get_output_experiments(), check_format=False)
self._crystal = experiments.crystals()[0]
def _determine_scaled_pointgroup(self):
"""Rerun symmetry after scaling to check for consistent space group. If not,
then new space group should be used and data rescaled."""
from cctbx import crystal
exp_crystal = load.experiment_list(self._scaler.get_scaled_experiments())[
0
].crystal
cs = crystal.symmetry(
space_group=exp_crystal.get_space_group(),
unit_cell=exp_crystal.get_unit_cell(),
)
cs_ref = cs.as_reference_setting()
current_pointgroup = cs_ref.space_group()
current_patt_group = (
current_pointgroup.build_derived_patterson_group().type().lookup_symbol()
)
Debug.write(
"Space group used in scaling: %s"
% current_pointgroup.type().lookup_symbol()
)
first = self._sweep_handler.get_epochs()[0]
si = self._sweep_handler.get_sweep_information(first)
refiner = si.get_integrater().get_integrater_refiner()
point_group, reindex_op, _, _, reind_refl, reind_exp, reindex_initial = self._dials_symmetry_indexer_jiffy(
[self._scaler.get_scaled_experiments()],
[self._scaler.get_scaled_reflections()],
[refiner],
)
Debug.write(
"Point group determined by dials.symmetry on scaled dataset: %s"
% point_group
)
sginfo = space_group_info(symbol=point_group)
patt_group = (
sginfo.group().build_derived_patterson_group().type().lookup_symbol()
)
self._scaler_symmetry_check_count += 1
if patt_group != current_patt_group:
if reindex_initial:
reindexer = DialsReindex()
reindexer.set_working_directory(self.get_working_directory())
auto_logfiler(reindexer)
reindexer.set_experiments_filename(
self._scaler.get_scaled_experiments()
)
reindexer.set_indexed_filename(self._scaler.get_scaled_reflections())
reindexer.set_cb_op(reindex_op)
reindexer.run()
self._scaler.set_scaled_experiments(
reindexer.get_reindexed_experiments_filename()
)
self._scaler.set_scaled_reflections(
reindexer.get_reindexed_reflections_filename()
)
else:
self._scaler.set_scaled_experiments(reind_exp)
self._scaler.set_scaled_reflections(reind_refl)
self.set_scaler_done(False)
Chatter.write(
"""Inconsistent space groups determined before and after scaling: %s, %s \n
Data will be rescaled in new point group"""
% (current_patt_group, patt_group)
)
return
else:
Chatter.write("Consistent space group determined before and after scaling")
def apply_reindex_operator_to_sweep_info(self, si, reindex_operator, reason):
"""Use a reindex operator to reindex the data.
Take the data from the sweep info, reindex using
dials.reindex, and set the new data into the si.
"""
reindexer = DialsReindex()
reindexer.set_working_directory(self.get_working_directory())
auto_logfiler(reindexer)
reindexer.set_indexed_filename(si.get_reflections())
reindexer.set_experiments_filename(si.get_experiments())
reindexer.set_cb_op(reindex_operator)
reindexer.run()
si.set_reflections(reindexer.get_reindexed_reflections_filename())
si.set_experiments(reindexer.get_reindexed_experiments_filename())
Debug.write(
"Reindexed with operator %s, reason is %s" % (reindex_operator, reason)
)
def _scale_prepare(self):
"""Perform all of the preparation required to deliver the scaled
data. This should sort together the reflection files, ensure that
they are correctly indexed (via dials.symmetry) and generally tidy
things up."""
# AIM discover symmetry and reindex with dials.symmetry, and set the correct
# reflections in si.reflections, si.experiments
self._helper.set_working_directory(self.get_working_directory())
self._factory.set_working_directory(self.get_working_directory())
need_to_return = False
self._sweep_handler = SweepInformationHandler(self._scalr_integraters)
p, x = self._sweep_handler.get_project_info()
self._scalr_pname = p
self._scalr_xname = x
self._helper.set_pname_xname(p, x)
Journal.block(
"gathering",
self.get_scaler_xcrystal().get_name(),
"Dials",
{"working directory": self.get_working_directory()},
)
# First do stuff to work out if excluding any data
# Note - does this actually work? I couldn't seem to get it to work
# in either this pipeline or the standard dials pipeline
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
_, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
exclude_sweep = False
for sweep in PhilIndex.params.xia2.settings.sweep:
if sweep.id == sname and sweep.exclude:
exclude_sweep = True
break
if exclude_sweep:
self._sweep_handler.remove_epoch(epoch)
Debug.write("Excluding sweep %s" % sname)
else:
Journal.entry({"adding data from": "%s/%s/%s" % (xname, dname, sname)})
# If multiple files, want to run symmetry to check for consistent indexing
# also
# try to reproduce what CCP4ScalerA is doing
# first assign identifiers to avoid dataset-id collisions
# Idea is that this should be called anytime you get data anew from the
# integrater, to intercept and assign unique ids, then set in the
# sweep_information (si) and always use si.set_reflections/
# si.get_reflections as we process.
# self._sweep_handler = self._helper.assign_and_return_datasets(
# self._sweep_handler
# ) symmetry now sorts out identifiers.
need_to_return = False
if self._scalr_input_pointgroup:
self._input_pointgroup_scale_prepare()
elif (
len(self._sweep_handler.get_epochs()) > 1
and PhilIndex.params.xia2.settings.multi_sweep_indexing
):
need_to_return = self._multi_sweep_scale_prepare()
else:
need_to_return = self._standard_scale_prepare()
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
### After this point, point group is good and only need to
### reindex to consistent setting. Don't need to call back to the
### integator, just use the data in the sweep info.
# First work out if we're going to reindex against external reference
param = PhilIndex.params.xia2.settings.scale
using_external_references = False
reference_refl = None
reference_expt = None
if param.reference_reflection_file:
if not param.reference_experiment_file:
Chatter.write(
"""
No DIALS reference experiments file provided, reference reflection file will
not be used. Reference mtz files for reindexing not currently supported for
pipeline=dials (supported for pipeline=dials-aimless).
"""
)
else:
reference_refl = param.reference_reflection_file
reference_expt = param.reference_experiment_file
using_external_references = True
Debug.write("Using reference reflections %s" % reference_refl)
Debug.write("Using reference experiments %s" % reference_expt)
if len(self._sweep_handler.get_epochs()) > 1:
if PhilIndex.params.xia2.settings.unify_setting:
self.unify_setting()
if PhilIndex.params.xia2.settings.use_brehm_diederichs:
self.brehm_diederichs_reindexing()
# If not using Brehm-deidrichs reindexing, set reference as first
# sweep, unless using external reference.
elif not using_external_references:
Debug.write("First sweep will be used as reference for reindexing")
first = self._sweep_handler.get_epochs()[0]
si = self._sweep_handler.get_sweep_information(first)
reference_expt = si.get_experiments()
reference_refl = si.get_reflections()
# Now reindex to be consistent with first dataset - run reindex on each
# dataset with reference (unless did brehm diederichs and didn't supply
# a reference file)
if reference_refl and reference_expt:
exp = load.experiment_list(reference_expt)
reference_cell = exp[0].crystal.get_unit_cell().parameters()
# ---------- REINDEX TO CORRECT (REFERENCE) SETTING ----------
Chatter.write("Reindexing all datasets to common reference")
if using_external_references:
epochs = self._sweep_handler.get_epochs()
else:
epochs = self._sweep_handler.get_epochs()[1:]
for epoch in epochs:
# if we are working with unified UB matrix then this should not
# be a problem here (note, *if*; *should*)
# what about e.g. alternative P1 settings?
# see JIRA MXSW-904
if PhilIndex.params.xia2.settings.unify_setting:
continue
reindexer = DialsReindex()
reindexer.set_working_directory(self.get_working_directory())
auto_logfiler(reindexer)
si = self._sweep_handler.get_sweep_information(epoch)
reindexer.set_reference_filename(reference_expt)
reindexer.set_reference_reflections(reference_refl)
reindexer.set_indexed_filename(si.get_reflections())
reindexer.set_experiments_filename(si.get_experiments())
reindexer.run()
# At this point, CCP4ScalerA would reset in integrator so that
# the integrater calls reindex, no need to do that here as
# have access to the files and will never need to reintegrate.
si.set_reflections(reindexer.get_reindexed_reflections_filename())
si.set_experiments(reindexer.get_reindexed_experiments_filename())
# FIXME how to get some indication of the reindexing used?
exp = load.experiment_list(
reindexer.get_reindexed_experiments_filename()
)
cell = exp[0].crystal.get_unit_cell().parameters()
# Note - no lattice check as this will already be caught by reindex
Debug.write("Cell: %.2f %.2f %.2f %.2f %.2f %.2f" % cell)
Debug.write("Ref: %.2f %.2f %.2f %.2f %.2f %.2f" % reference_cell)
for j in range(6):
if (
math.fabs((cell[j] - reference_cell[j]) / reference_cell[j])
> 0.1
):
raise RuntimeError(
"unit cell parameters differ in %s and %s"
% (reference_expt, si.get_reflections())
)
# Now make sure all batches ok before finish preparing
# This should be made safer, currently after dials.scale there is no
# concept of 'batch', dials.export uses the calculate_batch_offsets
# to assign batches, giving the same result as below.
experiments_to_rebatch = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
experiment = si.get_experiments()
experiments_to_rebatch.append(load.experiment_list(experiment)[0])
offsets = calculate_batch_offsets(experiments_to_rebatch)
for i, epoch in enumerate(self._sweep_handler.get_epochs()):
si = self._sweep_handler.get_sweep_information(epoch)
r = si.get_batch_range()
si.set_batch_offset(offsets[i])
si.set_batches([r[0] + offsets[i], r[1] + offsets[i]])
def load_sweeps_with_common_indexing():
assert os.path.exists('xia2.json')
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
import dials # required for gaussian_rs warning
from xia2.Wrappers.Dials.Reindex import Reindex
Citations.cite('dials')
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
import cPickle as pickle
crystals = xinfo.get_crystals()
assert len(crystals) == 1
crystal = next(crystals.itervalues())
working_directory = Environment.generate_directory([crystal.get_name(), 'analysis'])
os.chdir(working_directory)
scaler = crystal._get_scaler()
epoch_to_batches = {}
epoch_to_integrated_intensities = {}
epoch_to_sweep_name = {}
# Aimless only
epochs = scaler._sweep_handler.get_epochs()
reference_cell = None
reference_lattice = None
reference_vectors = None
reference_wavelength = None
# Reindex each sweep to same setting
all_miller_indices = flex.miller_index()
all_two_thetas = flex.double()
for epoch in epochs:
si = scaler._sweep_handler.get_sweep_information(epoch)
Chatter.smallbanner(si.get_sweep_name(), True)
Debug.smallbanner(si.get_sweep_name(), True)
intgr = si.get_integrater()
experiments_filename = intgr.get_integrated_experiments()
reflections_filename = intgr.get_integrated_reflections()
refiner = intgr.get_integrater_refiner()
Debug.write('experiment: %s' % experiments_filename)
Debug.write('reflection: %s' % reflections_filename)
# Use setting of first sweep as reference
if reference_vectors is None:
reference_vectors = experiments_filename
# Assume that all sweeps have the same lattice system
if reference_lattice is None:
reference_lattice = refiner.get_refiner_lattice()
else:
assert reference_lattice == refiner.get_refiner_lattice()
Debug.write("lattice: %s" % refiner.get_refiner_lattice())
# Read .json file for sweep
db = ExperimentListFactory.from_json_file(experiments_filename)
# Assume that each file only contains a single experiment
assert (len(db) == 1)
db = db[0]
# Get beam vector
s0 = db.beam.get_unit_s0()
# Use the unit cell of the first sweep as reference
if reference_cell is None:
reference_cell = db.crystal.get_unit_cell()
Debug.write("Reference cell: %s" % str(reference_cell))
dials_reindex = Reindex()
dials_reindex.set_working_directory(working_directory)
dials_reindex.set_cb_op("auto")
dials_reindex.set_reference_filename(reference_vectors)
dials_reindex.set_experiments_filename(experiments_filename)
dials_reindex.set_indexed_filename(reflections_filename)
auto_logfiler(dials_reindex)
dials_reindex.run()
# Assume that all data are collected at same wavelength
if reference_wavelength is None:
reference_wavelength = intgr.get_wavelength()
else:
assert abs(reference_wavelength - intgr.get_wavelength()) < 0.01
Debug.write("wavelength: %f A" % intgr.get_wavelength())
Debug.write("distance: %f mm" % intgr.get_distance())
# Get integrated reflection data
import dials
with open(dials_reindex.get_reindexed_reflections_filename(), 'rb') as fh:
reflections = pickle.load(fh)
selection = reflections.get_flags(reflections.flags.used_in_refinement)
Chatter.write("Found %d reflections used in refinement (out of %d entries)" % (selection.count(True), len(reflections['miller_index'])))
reflections = reflections.select(selection)
# Filter bad reflections
selection = reflections['intensity.sum.variance'] <= 0
if selection.count(True) > 0:
reflections.del_selected(selection)
print 'Removing %d reflections with negative variance' % \
selection.count(True)
if 'intensity.prf.variance' in reflections:
selection = reflections['intensity.prf.variance'] <= 0
if selection.count(True) > 0:
reflections.del_selected(selection)
print 'Removing %d profile reflections with negative variance' % \
selection.count(True)
# Find the observed 2theta angles
miller_indices = flex.miller_index()
two_thetas_obs = flex.double()
for pixel, panel, hkl in zip(reflections['xyzobs.px.value'], reflections['panel'], reflections['miller_index']):
assert hkl != (0, 0, 0)
two_thetas_obs.append(db.detector[panel].get_two_theta_at_pixel(s0, pixel[0:2]))
miller_indices.append(hkl)
# Convert observed 2theta angles to degrees
two_thetas_obs = two_thetas_obs * 180 / 3.14159265359
Chatter.write("Remaining %d reflections are in 2theta range %.3f - %.3f deg" % (len(miller_indices), min(two_thetas_obs), max(two_thetas_obs)))
all_miller_indices.extend(miller_indices)
all_two_thetas.extend(two_thetas_obs)
return all_miller_indices, all_two_thetas, reference_cell, reference_lattice, reference_wavelength
def run(self):
from xia2.Handlers.Streams import Chatter, Debug
if self._reindexing_operator:
Debug.write('Reindexing sweeps for dials.two_theta_refine')
from xia2.lib.bits import auto_logfiler
from xia2.Wrappers.Dials.Reindex import Reindex
self._reindexed_experiments, self._reindexed_reflections = [], []
for e, p in zip(self._experiments, self._pickles):
reindexer = Reindex()
reindexer.set_cb_op(self._reindexing_operator)
reindexer.set_experiments_filename(e)
reindexer.set_indexed_filename(p)
reindexer.set_working_directory(self.get_working_directory())
auto_logfiler(reindexer)
reindexer.run()
self._reindexed_experiments.append(reindexer.get_reindexed_experiments_filename())
self._reindexed_reflections.append(reindexer.get_reindexed_reflections_filename())
Debug.write('Running dials.two_theta_refine')
self._output_cif = os.path.join(
self.get_working_directory(),
'%s_dials.two_theta_refine.cif' % self.get_xpid())
self._output_mmcif = os.path.join(
self.get_working_directory(),
'%s_dials.two_theta_refine.mmcif' % self.get_xpid())
self._output_correlation_plot = os.path.join(
self.get_working_directory(),
'%s_dials.two_theta_refine.png' % self.get_xpid())
self._output_experiments = os.path.join(
self.get_working_directory(),
'%s_refined_cell.json' % self.get_xpid())
self.clear_command_line()
if self._reindexing_operator:
for experiment in self._reindexed_experiments:
self.add_command_line(experiment)
for pickle in self._reindexed_reflections:
self.add_command_line(pickle)
else:
for experiment in self._experiments:
self.add_command_line(experiment)
for pickle in self._pickles:
self.add_command_line(pickle)
self.add_command_line('output.cif=%s' % self._output_cif)
self.add_command_line('output.mmcif=%s' % self._output_mmcif)
if self._output_correlation_plot is not None:
self.add_command_line(
'output.correlation_plot.filename=%s' % self._output_correlation_plot)
if self._output_experiments is not None:
self.add_command_line(
'output.experiments=%s' % self._output_experiments)
if self._phil_file is not None:
self.add_command_line('%s' %self._phil_file)
self.start()
self.close_wait()
if not os.path.isfile(self._output_cif):
Chatter.write(
"TwoTheta refinement failed, see log file for more details:\n %s" % self.get_log_file())
raise RuntimeError, 'unit cell not refined'
self.check_for_errors()
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(self.get_output_experiments())
self._crystal = experiments.crystals()[0]
