def test_xds_scalerA(ccp4, xia2_regression_build, tmpdir, nproc):
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
template = os.path.join(xia2_regression_build, "test_data", "insulin",
"insulin_1_###.img")
tmpdir.chdir()
tmpdir = tmpdir.strpath
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater
from xia2.Modules.Scaler.XDSScalerA import XDSScalerA
indexer = XDSIndexer()
indexer.set_working_directory(tmpdir)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
with mock.patch.object(sys, 'argv', []):
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.XDSRefiner import XDSRefiner
refiner = XDSRefiner()
refiner.set_working_directory(tmpdir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
integrater = XDSIntegrater()
integrater.set_working_directory(tmpdir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
#integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.set_integrater_sweep_name('SWEEP1')
integrater.set_integrater_project_info('CRYST1', 'WAVE1', 'SWEEP1')
scaler = XDSScalerA()
scaler.add_scaler_integrater(integrater)
scaler.set_scaler_xcrystal(cryst)
scaler.set_scaler_project_info('CRYST1', 'WAVE1')
check_scaler_files_exist(scaler)
# test serialization of scaler
json_str = scaler.as_json()
#print json_str
scaler2 = XDSScalerA.from_json(string=json_str)
scaler2.set_scaler_xcrystal(cryst)
check_scaler_files_exist(scaler2)
scaler2.set_scaler_finish_done(False)
check_scaler_files_exist(scaler2)
scaler2.set_scaler_done(False)
check_scaler_files_exist(scaler2)
scaler2._scalr_integraters = {} # XXX
scaler2.add_scaler_integrater(integrater)
scaler2.set_scaler_prepare_done(False)
check_scaler_files_exist(scaler2)
def exercise_xds_integrater(nproc=None):
if not have_dials_regression:
print "Skipping exercise_xds_integrater(): dials_regression not configured"
return
if nproc is not None:
from xia2.Handlers.Flags import Flags
Flags.set_parallel(nproc)
xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
cwd = os.path.abspath(os.curdir)
tmp_dir = os.path.abspath(open_tmp_directory())
os.chdir(tmp_dir)
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater
indexer = XDSIndexer()
indexer.set_working_directory(tmp_dir)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.XDSRefiner import XDSRefiner
refiner = XDSRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
#refiner.refine()
integrater = XDSIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
#integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.integrate()
integrater_intensities = integrater.get_integrater_intensities()
assert os.path.exists(integrater_intensities)
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(integrater_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=400)
assert mtz_object.column_labels() == [
'H', 'K', 'L', 'M_ISYM', 'BATCH', 'I', 'SIGI', 'FRACTIONCALC',
'XDET', 'YDET', 'ROT', 'LP', 'FLAG']
corrected_intensities = integrater.get_integrater_corrected_intensities()
assert os.path.exists(corrected_intensities)
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(corrected_intensities)
assert reader.file_type() == "xds_ascii"
ma = reader.as_miller_arrays(merge_equivalents=False)[0]
assert approx_equal(ma.size(), 50000, eps=400)
#assert integrater.get_integrater_reindex_operator() == 'x,z,-y'
assert integrater.get_integrater_wedge() == (1, 45)
assert approx_equal(integrater.get_integrater_cell(),
[78.066, 78.066, 78.066, 90, 90, 90], eps=3e-2)
assert approx_equal(integrater.get_integrater_mosaic_min_mean_max(),
(0.181, 0.181, 0.181), eps=1e-3)
# test serialization of integrater
json_str = integrater.as_json()
#print json_str
integrater2 = XDSIntegrater.from_json(string=json_str)
integrater2.set_integrater_sweep(sweep, reset=False)
integrater2_intensities = integrater.get_integrater_intensities()
assert integrater2_intensities == integrater_intensities
integrater2.set_integrater_finish_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=400)
integrater2.set_integrater_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=350)
integrater2.set_integrater_prepare_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=300)
def exercise_xds_indexer(xia2_regression_build, tmp_dir, nproc=None):
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
xia2_demo_data = os.path.join(xia2_regression_build, "test_data",
"insulin")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
indexer = XDSIndexer()
indexer.set_working_directory(tmp_dir)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
indexer.index()
assert approx_equal(indexer.get_indexer_cell(),
(78.076, 78.076, 78.076, 90, 90, 90),
eps=1), indexer.get_indexer_cell()
experiment = indexer.get_indexer_experiment_list()[0]
sgi = experiment.crystal.get_space_group().info()
assert sgi.type().number() == 197
beam_centre = indexer.get_indexer_beam_centre()
assert approx_equal(beam_centre, (94.4221, 94.5096), eps=1e-1)
assert indexer.get_indexer_images() == [(1, 5), (20, 24), (41, 45)]
print(indexer.get_indexer_experiment_list()[0].crystal)
print(indexer.get_indexer_experiment_list()[0].detector)
# test serialization of indexer
json_str = indexer.as_json()
print(json_str)
indexer2 = XDSIndexer.from_json(string=json_str)
indexer2.index()
assert approx_equal(indexer.get_indexer_cell(),
indexer2.get_indexer_cell())
assert approx_equal(indexer.get_indexer_beam_centre(),
indexer2.get_indexer_beam_centre())
assert approx_equal(indexer.get_indexer_images(),
indexer2.get_indexer_images())
indexer.eliminate()
indexer2.eliminate()
assert approx_equal(indexer.get_indexer_cell(),
indexer2.get_indexer_cell())
assert indexer.get_indexer_lattice() == 'hR'
assert indexer2.get_indexer_lattice() == 'hR'
def exercise_xds_integrater(dials_data, tmp_dir, nproc=None):
if nproc:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
template = dials_data("insulin").join("insulin_1_###.img").strpath
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater
indexer = XDSIndexer()
indexer.set_working_directory(tmp_dir)
from dxtbx.model.experiment_list import ExperimentListTemplateImporter
importer = ExperimentListTemplateImporter([template])
experiments = importer.experiments
imageset = experiments.imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.XDSRefiner import XDSRefiner
refiner = XDSRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
# refiner.refine()
integrater = XDSIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
integrater.set_integrater_sweep(sweep)
integrater.integrate()
from iotbx.reflection_file_reader import any_reflection_file
integrater_intensities = integrater.get_integrater_intensities()
assert os.path.exists(integrater_intensities)
reader = any_reflection_file(integrater_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=400)
assert mtz_object.column_labels() == [
"H",
"K",
"L",
"M_ISYM",
"BATCH",
"I",
"SIGI",
"FRACTIONCALC",
"XDET",
"YDET",
"ROT",
"LP",
"FLAG",
]
corrected_intensities = integrater.get_integrater_corrected_intensities()
assert os.path.exists(corrected_intensities)
reader = any_reflection_file(corrected_intensities)
assert reader.file_type() == "xds_ascii"
ma = reader.as_miller_arrays(merge_equivalents=False)[0]
assert approx_equal(ma.size(), 50000, eps=400)
assert integrater.get_integrater_wedge() == (1, 45)
assert approx_equal(
integrater.get_integrater_cell(), [78.066, 78.066, 78.066, 90, 90, 90], eps=1
)
assert approx_equal(
integrater.get_integrater_mosaic_min_mean_max(), (0.180, 0.180, 0.180), eps=1e-1
)
# test serialization of integrater
json_str = integrater.as_json()
# print(json_str)
integrater2 = XDSIntegrater.from_json(string=json_str)
integrater2.set_integrater_sweep(sweep, reset=False)
integrater2_intensities = integrater.get_integrater_intensities()
assert integrater2_intensities == integrater_intensities
integrater2.set_integrater_finish_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=400)
integrater2.set_integrater_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=450)
integrater2.set_integrater_prepare_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50100, eps=400)
def _integrate_prepare(self):
'''Prepare for integration - in XDS terms this may mean rerunning
IDXREF to get the XPARM etc. DEFPIX is considered part of the full
integration as it is resolution dependent.'''
from xia2.Handlers.Citations import Citations
Citations.cite('xds')
# decide what images we are going to process, if not already
# specified
if not self._intgr_wedge:
images = self.get_matching_images()
self.set_integrater_wedge(min(images),
max(images))
Debug.write('XDS INTEGRATE PREPARE:')
Debug.write('Wavelength: %.6f' % self.get_wavelength())
Debug.write('Distance: %.2f' % self.get_distance())
idxr = self._intgr_refiner.get_refiner_indexer(self.get_integrater_epoch())
if idxr is None:
idxr = XDSIndexer()
self._intgr_refiner.add_refiner_indexer(
self.get_integrater_epoch(), idxr)
self.set_integrater_prepare_done(False)
#self.set_integrater_indexer()
idxr.set_indexer_sweep(self.get_integrater_sweep())
idxr.set_working_directory(self.get_working_directory())
idxr.setup_from_imageset(self.get_imageset())
if self.get_frame_wedge():
wedge = self.get_frame_wedge()
Debug.write('Propogating wedge limit: %d %d' % wedge)
idxr.set_frame_wedge(wedge[0], wedge[1], apply_offset = False)
# this needs to be set up from the contents of the
# Integrater frame processer - wavelength &c.
if self.get_beam_centre():
idxr.set_beam_centre(self.get_beam_centre())
if self.get_distance():
idxr.set_distance(self.get_distance())
if self.get_wavelength():
idxr.set_wavelength(self.get_wavelength())
# get the unit cell from this indexer to initiate processing
# if it is new... and also copy out all of the information for
# the XDS indexer if not...
# copy the data across
self._xds_data_files = copy.deepcopy(
self._intgr_refiner.get_refiner_payload(self.get_integrater_epoch()))
Debug.write('Files available at the end of XDS integrate prepare:')
for f in self._xds_data_files.keys():
Debug.write('%s' % f)
experiment = self._intgr_refiner.get_refined_experiment_list(
self.get_integrater_epoch())[0]
# copy across the trusted_range - it got lost along the way
old_detector = self.get_detector()
self.set_detector(experiment.detector)
for p1, p2 in zip(old_detector, self.get_detector()):
p2.set_trusted_range(p1.get_trusted_range())
self.set_beam_obj(experiment.beam)
self.set_goniometer(experiment.goniometer)
# set a low resolution limit (which isn't really used...)
# this should perhaps be done more intelligently from an
# analysis of the spot list or something...?
if not self.get_integrater_low_resolution():
dmax = self._intgr_refiner.get_indexer_low_resolution(
self.get_integrater_epoch())
self.set_integrater_low_resolution(dmax)
Debug.write('Low resolution set to: %s' % \
self.get_integrater_low_resolution())
# delete things we should not know e.g. the postrefined cell from
# CORRECT - c/f bug # 2695
self._intgr_cell = None
self._intgr_spacegroup_number = None
return
def _integrate_prepare(self):
"""Prepare for integration - in XDS terms this may mean rerunning
IDXREF to get the XPARM etc. DEFPIX is considered part of the full
integration as it is resolution dependent."""
Citations.cite("xds")
# decide what images we are going to process, if not already
# specified
if not self._intgr_wedge:
images = self.get_matching_images()
self.set_integrater_wedge(
min(images) + self.get_frame_offset(),
max(images) + self.get_frame_offset(),
)
logger.debug("XDS INTEGRATE PREPARE:")
logger.debug("Wavelength: %.6f" % self.get_wavelength())
logger.debug("Distance: %.2f" % self.get_distance())
idxr = self._intgr_refiner.get_refiner_indexer(
self.get_integrater_epoch())
if idxr is None:
idxr = XDSIndexer()
self._intgr_refiner.add_refiner_indexer(
self.get_integrater_epoch(), idxr)
self.set_integrater_prepare_done(False)
# self.set_integrater_indexer()
idxr.set_indexer_sweep(self.get_integrater_sweep())
idxr.set_working_directory(self.get_working_directory())
idxr.setup_from_imageset(self.get_imageset())
if self.get_frame_wedge():
wedge = self.get_frame_wedge()
logger.debug("Propogating wedge limit: %d %d" % wedge)
idxr.set_frame_wedge(wedge[0], wedge[1], apply_offset=False)
# this needs to be set up from the contents of the
# Integrater frame processer - wavelength &c.
if self.get_beam_centre():
idxr.set_beam_centre(self.get_beam_centre())
if self.get_distance():
idxr.set_distance(self.get_distance())
if self.get_wavelength():
idxr.set_wavelength(self.get_wavelength())
# get the unit cell from this indexer to initiate processing
# if it is new... and also copy out all of the information for
# the XDS indexer if not...
# copy the data across
self._xds_data_files = copy.deepcopy(
self._intgr_refiner.get_refiner_payload(
self.get_integrater_epoch()))
if self._xds_data_files is None:
self._xds_data_files = {}
logger.debug("Files available at the end of XDS integrate prepare:")
for f in self._xds_data_files:
logger.debug("%s" % f)
experiment = self._intgr_refiner.get_refined_experiment_list(
self.get_integrater_epoch())[0]
# copy across the trusted_range - it got lost along the way
old_detector = self.get_detector()
self.set_detector(experiment.detector)
for p1, p2 in zip(old_detector, self.get_detector()):
p2.set_trusted_range(p1.get_trusted_range())
self.set_beam_obj(experiment.beam)
self.set_goniometer(experiment.goniometer)
# set a low resolution limit (which isn't really used...)
# this should perhaps be done more intelligently from an
# analysis of the spot list or something...?
if not self.get_integrater_low_resolution():
dmax = self._intgr_refiner.get_indexer_low_resolution(
self.get_integrater_epoch())
self.set_integrater_low_resolution(dmax)
logger.debug("Low resolution set to: %s" %
self.get_integrater_low_resolution())
# delete things we should not know e.g. the postrefined cell from
# CORRECT - c/f bug # 2695
self._intgr_cell = None
self._intgr_spacegroup_number = None
def _refine_prepare(self):
for epoch, idxr in self._refinr_indexers.iteritems():
experiments = idxr.get_indexer_experiment_list()
assert len(
experiments) == 1 # currently only handle one lattice/sweep
experiment = experiments[0]
crystal_model = experiment.crystal
lattice = idxr.get_indexer_lattice()
# check if the lattice was user assigned...
user_assigned = idxr.get_indexer_user_input_lattice()
# hack to figure out if we did indexing with Dials - if so then need to
# run XYCORR, INIT, and then dials.export_xds before we are ready to
# integrate with XDS
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
if isinstance(idxr, DialsIndexer):
sweep = idxr.get_indexer_sweep()
imageset = idxr._indxr_imagesets[0]
scan = imageset.get_scan()
first, last = scan.get_image_range()
phi_width = scan.get_oscillation()[1]
last_background = int(round(5.0 / phi_width)) - 1 + first
last_background = min(last, last_background)
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
xds_idxr = XDSIndexer()
xds_idxr.set_working_directory(self.get_working_directory())
xds_idxr.set_indexer_sweep(sweep)
xds_idxr.add_indexer_imageset(imageset)
# next start to process these - first xycorr
# FIXME run these *afterwards* as then we have a refined detector geometry
# so the parallax correction etc. should be slightly better.
#self._indxr_images = [(first, last)]
xycorr = xds_idxr.Xycorr()
xycorr.set_data_range(first, last)
xycorr.set_background_range(first, last_background)
xycorr.set_working_directory(self.get_working_directory())
xycorr.run()
xds_data_files = {}
for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']:
xds_data_files[file] = xycorr.get_output_data_file(file)
# next start to process these - then init
init = xds_idxr.Init()
for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']:
init.set_input_data_file(file, xds_data_files[file])
init.set_data_range(first, last)
init.set_background_range(first, last_background)
init.set_working_directory(self.get_working_directory())
init.run()
for file in ['BLANK.cbf', 'BKGINIT.cbf', 'GAIN.cbf']:
xds_data_files[file] = init.get_output_data_file(file)
exporter = self.ExportXDS()
exporter.set_experiments_filename(
idxr.get_solution()['experiments_file'])
exporter.run()
for file in ['XPARM.XDS']:
xds_data_files[file] = os.path.join(
self.get_working_directory(), 'xds', file)
for k, v in xds_data_files.items():
idxr.set_indexer_payload(k, v)
# check that the indexer is an XDS indexer - if not then
# create one...
elif not idxr.get_indexer_payload('XPARM.XDS'):
Debug.write('Generating an XDS indexer')
idxr_old = idxr
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
idxr = XDSIndexer()
idxr.set_indexer_sweep(idxr_old.get_indexer_sweep())
self._refinr_indexers[epoch] = idxr
self.set_refiner_prepare_done(False)
# note to self for the future - this set will reset the
# integrater prepare done flag - this means that we will
# go through this routine all over again. However this
# is not a problem as all that will happen is that the
# results will be re-got, no additional processing will
# be performed...
# set the indexer up as per the frameprocessor interface...
# this would usually happen within the IndexerFactory.
idxr.set_indexer_sweep_name(idxr_old.get_indexer_sweep_name())
idxr.add_indexer_imageset(idxr_old.get_imageset())
idxr.set_working_directory(idxr_old.get_working_directory())
# now copy information from the old indexer to the new
# one - lattice, cell, distance etc.
# bug # 2434 - providing the correct target cell
# may be screwing things up - perhaps it would
# be best to allow XDS just to index with a free
# cell but target lattice??
cell = crystal_model.get_unit_cell().parameters()
check = PhilIndex.params.xia2.settings.xds_check_cell_deviation
# FIXME this was changed in #42 but not sure logic is right
if not check:
Debug.write(
'Inputting target cell: %.2f %.2f %.2f %.2f %.2f %.2f' % \
cell)
idxr.set_indexer_input_cell(cell)
input_cell = cell
from cctbx.sgtbx import bravais_types
lattice = str(
bravais_types.bravais_lattice(
group=crystal_model.get_space_group()))
idxr.set_indexer_input_lattice(lattice)
if user_assigned:
Debug.write('Assigning the user given lattice: %s' % \
lattice)
idxr.set_indexer_user_input_lattice(True)
idxr.set_detector(experiment.detector)
idxr.set_beam(experiment.beam)
idxr.set_goniometer(experiment.goniometer)
# re-get the unit cell &c. and check that the indexing
# worked correctly
Debug.write('Rerunning indexing with XDS')
experiments = idxr.get_indexer_experiment_list()
assert len(experiments
) == 1 # currently only handle one lattice/sweep
experiment = experiments[0]
crystal_model = experiment.crystal
def test_xds_scalerA(regression_test, ccp4, xds, dials_data, run_in_tmpdir,
nproc):
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
template = dials_data("insulin").join("insulin_1_###.img").strpath
tmpdir = run_in_tmpdir.strpath
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater
from xia2.Modules.Scaler.XDSScalerA import XDSScalerA
indexer = XDSIndexer()
indexer.set_working_directory(tmpdir)
from dxtbx.model.experiment_list import ExperimentListTemplateImporter
importer = ExperimentListTemplateImporter([template])
experiments = importer.experiments
imageset = experiments.imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
with mock.patch.object(sys, "argv", []):
sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.XDSRefiner import XDSRefiner
refiner = XDSRefiner()
refiner.set_working_directory(tmpdir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
integrater = XDSIntegrater()
integrater.set_working_directory(tmpdir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
# integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.set_integrater_sweep_name("SWEEP1")
integrater.set_integrater_project_info("CRYST1", "WAVE1", "SWEEP1")
scaler = XDSScalerA(base_path=pathlib.Path(tmpdir))
scaler.add_scaler_integrater(integrater)
scaler.set_scaler_xcrystal(cryst)
scaler.set_scaler_project_info("CRYST1", "WAVE1")
check_scaler_files_exist(scaler)
# test serialization of scaler
json_str = scaler.as_json()
# print json_str
scaler2 = XDSScalerA.from_json(string=json_str)
scaler2.set_scaler_xcrystal(cryst)
check_scaler_files_exist(scaler2)
scaler2.set_scaler_done(False)
check_scaler_files_exist(scaler2)
scaler2._scalr_integraters = {} # XXX
scaler2.add_scaler_integrater(integrater)
scaler2.set_scaler_prepare_done(False)
check_scaler_files_exist(scaler2)
def exercise_xds_integrater(xia2_regression_build, tmp_dir, nproc=None):
if nproc:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
xia2_demo_data = os.path.join(xia2_regression_build, "test_data",
"insulin")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater
indexer = XDSIndexer()
indexer.set_working_directory(tmp_dir)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.XDSRefiner import XDSRefiner
refiner = XDSRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
#refiner.refine()
integrater = XDSIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
integrater.set_integrater_sweep(sweep)
integrater.integrate()
from iotbx.reflection_file_reader import any_reflection_file
integrater_intensities = integrater.get_integrater_intensities()
assert os.path.exists(integrater_intensities)
reader = any_reflection_file(integrater_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=400)
assert mtz_object.column_labels() == [
'H', 'K', 'L', 'M_ISYM', 'BATCH', 'I', 'SIGI', 'FRACTIONCALC', 'XDET',
'YDET', 'ROT', 'LP', 'FLAG'
]
corrected_intensities = integrater.get_integrater_corrected_intensities()
assert os.path.exists(corrected_intensities)
reader = any_reflection_file(corrected_intensities)
assert reader.file_type() == "xds_ascii"
ma = reader.as_miller_arrays(merge_equivalents=False)[0]
assert approx_equal(ma.size(), 50000, eps=400)
assert integrater.get_integrater_wedge() == (1, 45)
assert approx_equal(integrater.get_integrater_cell(),
[78.066, 78.066, 78.066, 90, 90, 90],
eps=1)
assert approx_equal(integrater.get_integrater_mosaic_min_mean_max(),
(0.180, 0.180, 0.180),
eps=1e-1)
# test serialization of integrater
json_str = integrater.as_json()
#print(json_str)
integrater2 = XDSIntegrater.from_json(string=json_str)
integrater2.set_integrater_sweep(sweep, reset=False)
integrater2_intensities = integrater.get_integrater_intensities()
assert integrater2_intensities == integrater_intensities
integrater2.set_integrater_finish_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=400)
integrater2.set_integrater_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=450)
integrater2.set_integrater_prepare_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50100, eps=400)
def exercise_xds_scaler(nproc=None):
if not have_dials_regression:
print "Skipping exercise_xds_scaler(): dials_regression not configured"
return
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
cwd = os.path.abspath(os.curdir)
tmp_dir = os.path.abspath(open_tmp_directory())
os.chdir(tmp_dir)
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater
from xia2.Modules.Scaler.XDSScalerA import XDSScalerA
indexer = XDSIndexer()
indexer.set_working_directory(tmp_dir)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.XDSRefiner import XDSRefiner
refiner = XDSRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
integrater = XDSIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
#integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.set_integrater_sweep_name('SWEEP1')
integrater.set_integrater_project_info('CRYST1', 'WAVE1', 'SWEEP1')
scaler = XDSScalerA()
scaler.add_scaler_integrater(integrater)
scaler.set_scaler_xcrystal(cryst)
scaler.set_scaler_project_info('CRYST1', 'WAVE1')
check_scaler_files_exist(scaler)
# test serialization of scaler
json_str = scaler.as_json()
#print json_str
scaler2 = XDSScalerA.from_json(string=json_str)
scaler2.set_scaler_xcrystal(cryst)
check_scaler_files_exist(scaler2)
scaler2.set_scaler_finish_done(False)
check_scaler_files_exist(scaler2)
scaler2.set_scaler_done(False)
check_scaler_files_exist(scaler2)
scaler2._scalr_integraters = {} # XXX
scaler2.add_scaler_integrater(integrater)
scaler2.set_scaler_prepare_done(False)
check_scaler_files_exist(scaler2)
def _index_finish(self):
self._modify_xparm_xds()
XDSIndexer._index_finish(self)
def exercise_xds_indexer(dials_data, tmp_dir, nproc=None):
if nproc is not None:
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
template = dials_data("insulin").join("insulin_1_###.img").strpath
indexer = XDSIndexer()
indexer.set_working_directory(tmp_dir)
importer = ExperimentListTemplateImporter([template])
experiments = importer.experiments
imageset = experiments.imagesets()[0]
indexer.add_indexer_imageset(imageset)
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
indexer.index()
assert indexer.get_indexer_cell() == pytest.approx(
(78.076, 78.076, 78.076, 90, 90, 90),
abs=1), indexer.get_indexer_cell()
experiment = indexer.get_indexer_experiment_list()[0]
sgi = experiment.crystal.get_space_group().info()
assert sgi.type().number() == 197
beam_centre = indexer.get_indexer_beam_centre()
assert beam_centre == pytest.approx((94.4221, 94.5096), abs=1e-1)
assert indexer.get_indexer_images() == [(1, 5), (20, 24), (41, 45)]
print(indexer.get_indexer_experiment_list()[0].crystal)
print(indexer.get_indexer_experiment_list()[0].detector)
# test serialization of indexer
json_str = indexer.as_json()
print(json_str)
indexer2 = XDSIndexer.from_json(string=json_str)
indexer2.index()
assert indexer.get_indexer_cell() == pytest.approx(
indexer2.get_indexer_cell())
assert indexer.get_indexer_beam_centre() == pytest.approx(
indexer2.get_indexer_beam_centre())
assert indexer.get_indexer_images() == [
tuple(i) for i in indexer2.get_indexer_images()
]
indexer.eliminate()
indexer2.eliminate()
assert indexer.get_indexer_cell() == pytest.approx(
indexer2.get_indexer_cell())
assert indexer.get_indexer_lattice() == "hR"
assert indexer2.get_indexer_lattice() == "hR"
def exercise_xds_indexer(nproc=None):
if not have_dials_regression:
print "Skipping exercise_xds_indexer(): dials_regression not configured"
return
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
cwd = os.path.abspath(os.curdir)
tmp_dir = os.path.abspath(open_tmp_directory())
os.chdir(tmp_dir)
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
indexer = XDSIndexer()
indexer.set_working_directory(tmp_dir)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
indexer.index()
assert approx_equal(
indexer.get_indexer_cell(), (78.07336, 78.07337, 78.07336, 90, 90, 90), eps=1e-3
), indexer.get_indexer_cell()
experiment = indexer.get_indexer_experiment_list()[0]
sgi = experiment.crystal.get_space_group().info()
assert sgi.type().number() == 197
beam_centre = indexer.get_indexer_beam_centre()
assert approx_equal(beam_centre, (94.4221, 94.5096), eps=1e-3)
assert indexer.get_indexer_images() == [(1, 5), (20, 24), (41, 45)]
print indexer.get_indexer_experiment_list()[0].crystal
print indexer.get_indexer_experiment_list()[0].detector
# test serialization of indexer
json_str = indexer.as_json()
print json_str
indexer2 = XDSIndexer.from_json(string=json_str)
indexer2.index()
assert approx_equal(indexer.get_indexer_cell(), indexer2.get_indexer_cell())
assert approx_equal(indexer.get_indexer_beam_centre(), indexer2.get_indexer_beam_centre())
assert approx_equal(indexer.get_indexer_images(), indexer2.get_indexer_images())
indexer.eliminate()
indexer2.eliminate()
assert approx_equal(indexer.get_indexer_cell(), indexer2.get_indexer_cell())
assert indexer.get_indexer_lattice() == "hR"
assert indexer2.get_indexer_lattice() == "hR"
def _refine_prepare(self):
for epoch, idxr in self._refinr_indexers.iteritems():
experiments = idxr.get_indexer_experiment_list()
assert len(experiments) == 1 # currently only handle one lattice/sweep
experiment = experiments[0]
crystal_model = experiment.crystal
lattice = idxr.get_indexer_lattice()
# check if the lattice was user assigned...
user_assigned = idxr.get_indexer_user_input_lattice()
# hack to figure out if we did indexing with Dials - if so then need to
# run XYCORR, INIT, and then dials.export_xds before we are ready to
# integrate with XDS
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
if isinstance(idxr, DialsIndexer):
sweep = idxr.get_indexer_sweep()
imageset = idxr._indxr_imagesets[0]
scan = imageset.get_scan()
first, last = scan.get_image_range()
phi_width = scan.get_oscillation()[1]
last_background = int(round(5.0 / phi_width)) - 1 + first
last_background = min(last, last_background)
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
xds_idxr = XDSIndexer()
xds_idxr.set_working_directory(self.get_working_directory())
xds_idxr.set_indexer_sweep(sweep)
xds_idxr.add_indexer_imageset(imageset)
# next start to process these - first xycorr
# FIXME run these *afterwards* as then we have a refined detector geometry
# so the parallax correction etc. should be slightly better.
#self._indxr_images = [(first, last)]
xycorr = xds_idxr.Xycorr()
xycorr.set_data_range(first, last)
xycorr.set_background_range(first, last_background)
xycorr.set_working_directory(self.get_working_directory())
xycorr.run()
xds_data_files = {}
for file in ['X-CORRECTIONS.cbf',
'Y-CORRECTIONS.cbf']:
xds_data_files[file] = xycorr.get_output_data_file(file)
# next start to process these - then init
init = xds_idxr.Init()
for file in ['X-CORRECTIONS.cbf',
'Y-CORRECTIONS.cbf']:
init.set_input_data_file(file, xds_data_files[file])
init.set_data_range(first, last)
init.set_background_range(first, last_background)
init.set_working_directory(self.get_working_directory())
init.run()
for file in ['BLANK.cbf',
'BKGINIT.cbf',
'GAIN.cbf']:
xds_data_files[file] = init.get_output_data_file(file)
exporter = self.ExportXDS()
exporter.set_experiments_filename(
idxr.get_solution()['experiments_file'])
exporter.run()
for file in ['XPARM.XDS']:
xds_data_files[file] = os.path.join(
self.get_working_directory(), 'xds', file)
for k, v in xds_data_files.items():
idxr.set_indexer_payload(k, v)
# check that the indexer is an XDS indexer - if not then
# create one...
elif not idxr.get_indexer_payload('XPARM.XDS'):
Debug.write('Generating an XDS indexer')
idxr_old = idxr
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
idxr = XDSIndexer()
idxr.set_indexer_sweep(idxr_old.get_indexer_sweep())
self._refinr_indexers[epoch] = idxr
self.set_refiner_prepare_done(False)
# note to self for the future - this set will reset the
# integrater prepare done flag - this means that we will
# go through this routine all over again. However this
# is not a problem as all that will happen is that the
# results will be re-got, no additional processing will
# be performed...
# set the indexer up as per the frameprocessor interface...
# this would usually happen within the IndexerFactory.
idxr.set_indexer_sweep_name(idxr_old.get_indexer_sweep_name())
idxr.add_indexer_imageset(idxr_old.get_imageset())
idxr.set_working_directory(idxr_old.get_working_directory())
# now copy information from the old indexer to the new
# one - lattice, cell, distance etc.
# bug # 2434 - providing the correct target cell
# may be screwing things up - perhaps it would
# be best to allow XDS just to index with a free
# cell but target lattice??
cell = crystal_model.get_unit_cell().parameters()
check = PhilIndex.params.xia2.settings.xds_check_cell_deviation
# FIXME this was changed in #42 but not sure logic is right
if not check:
Debug.write(
'Inputting target cell: %.2f %.2f %.2f %.2f %.2f %.2f' % \
cell)
idxr.set_indexer_input_cell(cell)
input_cell = cell
from cctbx.sgtbx import bravais_types
lattice = str(
bravais_types.bravais_lattice(group=crystal_model.get_space_group()))
idxr.set_indexer_input_lattice(lattice)
if user_assigned:
Debug.write('Assigning the user given lattice: %s' % \
lattice)
idxr.set_indexer_user_input_lattice(True)
idxr.set_detector(experiment.detector)
idxr.set_beam(experiment.beam)
idxr.set_goniometer(experiment.goniometer)
# re-get the unit cell &c. and check that the indexing
# worked correctly
Debug.write('Rerunning indexing with XDS')
experiments = idxr.get_indexer_experiment_list()
assert len(experiments) == 1 # currently only handle one lattice/sweep
experiment = experiments[0]
crystal_model = experiment.crystal