def add_sweep(self, name, sample, directory=None, image=None,
beam=None, reversephi=False, distance=None,
gain=0.0, dmin=0.0, dmax=0.0, polarization=0.0,
frames_to_process=None, user_lattice=None,
user_cell=None, epoch=0, ice=False, excluded_regions=None):
'''Add a sweep to this wavelength.'''
if excluded_regions is None:
excluded_regions = []
xsweep = XSweep(name, self,
sample=sample,
directory=directory,
image=image,
beam=beam,
reversephi=reversephi,
distance=distance,
gain=gain,
dmin=dmin,
dmax=dmax,
polarization=polarization,
frames_to_process=frames_to_process,
user_lattice=user_lattice,
user_cell=user_cell,
epoch=epoch,
ice=ice,
excluded_regions=excluded_regions)
self._sweeps.append(xsweep)
return xsweep
def exercise_xds_indexer(dials_data, tmp_dir, nproc=None):
if nproc is not None:
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
template = dials_data("insulin").join("insulin_1_###.img").strpath
indexer = XDSIndexer()
indexer.set_working_directory(tmp_dir)
importer = ExperimentListTemplateImporter([template])
experiments = importer.experiments
imageset = experiments.imagesets()[0]
indexer.add_indexer_imageset(imageset)
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
indexer.index()
assert indexer.get_indexer_cell() == pytest.approx(
(78.076, 78.076, 78.076, 90, 90, 90),
abs=1), indexer.get_indexer_cell()
experiment = indexer.get_indexer_experiment_list()[0]
sgi = experiment.crystal.get_space_group().info()
assert sgi.type().number() == 197
beam_centre = indexer.get_indexer_beam_centre()
assert beam_centre == pytest.approx((94.4221, 94.5096), abs=1e-1)
assert indexer.get_indexer_images() == [(1, 5), (20, 24), (41, 45)]
print(indexer.get_indexer_experiment_list()[0].crystal)
print(indexer.get_indexer_experiment_list()[0].detector)
# test serialization of indexer
json_str = indexer.as_json()
print(json_str)
indexer2 = XDSIndexer.from_json(string=json_str)
indexer2.index()
assert indexer.get_indexer_cell() == pytest.approx(
indexer2.get_indexer_cell())
assert indexer.get_indexer_beam_centre() == pytest.approx(
indexer2.get_indexer_beam_centre())
assert indexer.get_indexer_images() == [
tuple(i) for i in indexer2.get_indexer_images()
]
indexer.eliminate()
indexer2.eliminate()
assert indexer.get_indexer_cell() == pytest.approx(
indexer2.get_indexer_cell())
assert indexer.get_indexer_lattice() == "hR"
assert indexer2.get_indexer_lattice() == "hR"
def from_dict(cls, obj):
assert obj['__id__'] == 'XWavelength'
return_obj = cls(name=None, crystal=None, wavelength=None)
for k, v in obj.iteritems():
if k == '_sweeps':
v = [XSweep.from_dict(s_dict) for s_dict in v]
for sweep in v:
sweep._wavelength = return_obj
setattr(return_obj, k, v)
return return_obj
def exercise_dials_indexer(dials_data, tmp_dir, nproc=None):
if nproc is not None:
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
template = dials_data("insulin").join("insulin_1_###.img").strpath
indexer = DialsIndexer()
indexer.set_working_directory(tmp_dir)
importer = ExperimentListTemplateImporter([template])
experiments = importer.experiments
imageset = experiments.imagesets()[0]
indexer.add_indexer_imageset(imageset)
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
indexer.index()
assert indexer.get_indexer_cell() == pytest.approx(
(78.14, 78.14, 78.14, 90, 90, 90), rel=1e-3)
solution = indexer.get_solution()
assert solution["rmsd"] == pytest.approx(0.03545, abs=1e-3)
assert solution["metric"] == pytest.approx(0.02517, abs=5e-3)
assert solution["number"] == 22
assert solution["lattice"] == "cI"
beam_centre = indexer.get_indexer_beam_centre()
assert beam_centre == pytest.approx((94.41567208118963, 94.51337522659865),
abs=1e-3)
print(indexer.get_indexer_experiment_list()[0].crystal)
print(indexer.get_indexer_experiment_list()[0].detector)
# test serialization of indexer
json_str = indexer.as_json()
indexer2 = DialsIndexer.from_json(string=json_str)
indexer2.index()
assert indexer.get_indexer_cell() == pytest.approx(
indexer2.get_indexer_cell())
assert indexer.get_indexer_beam_centre() == pytest.approx(
indexer2.get_indexer_beam_centre())
indexer.eliminate()
indexer2.eliminate()
assert indexer.get_indexer_cell() == pytest.approx(
indexer2.get_indexer_cell())
assert indexer.get_indexer_lattice() == "hR"
assert indexer2.get_indexer_lattice() == "hR"
def exercise_dials_multi_indexer(nproc=None):
template = "/Volumes/touro/data/i19/4sweep/56_Reza-H2-normalhem_100K_3.75mmAl/56-RezaH2-hem_%02i_#####.cbf"
cwd = os.path.abspath(os.curdir)
tmp_dir = os.path.abspath(open_tmp_directory())
os.chdir(tmp_dir)
cryst = None
wav = None
samp = None
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.trust_beam_centre = True
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
indexer = DialsIndexer()
indexer.set_working_directory(tmp_dir)
for i in range(4):
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template % (i + 1)])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
if cryst is None or wav is None:
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst,
imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.add_indexer_sweep(sweep)
indexer.index()
assert approx_equal(indexer.get_indexer_cell(),
(9.088, 12.415, 17.420, 90.000, 90.000, 90.000),
eps=1e-2)
def exercise_xds_integrater(xia2_regression_build, tmp_dir, nproc=None):
if nproc:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
xia2_demo_data = os.path.join(xia2_regression_build, "test_data",
"insulin")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater
indexer = XDSIndexer()
indexer.set_working_directory(tmp_dir)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.XDSRefiner import XDSRefiner
refiner = XDSRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
#refiner.refine()
integrater = XDSIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
integrater.set_integrater_sweep(sweep)
integrater.integrate()
from iotbx.reflection_file_reader import any_reflection_file
integrater_intensities = integrater.get_integrater_intensities()
assert os.path.exists(integrater_intensities)
reader = any_reflection_file(integrater_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=400)
assert mtz_object.column_labels() == [
'H', 'K', 'L', 'M_ISYM', 'BATCH', 'I', 'SIGI', 'FRACTIONCALC', 'XDET',
'YDET', 'ROT', 'LP', 'FLAG'
]
corrected_intensities = integrater.get_integrater_corrected_intensities()
assert os.path.exists(corrected_intensities)
reader = any_reflection_file(corrected_intensities)
assert reader.file_type() == "xds_ascii"
ma = reader.as_miller_arrays(merge_equivalents=False)[0]
assert approx_equal(ma.size(), 50000, eps=400)
assert integrater.get_integrater_wedge() == (1, 45)
assert approx_equal(integrater.get_integrater_cell(),
[78.066, 78.066, 78.066, 90, 90, 90],
eps=1)
assert approx_equal(integrater.get_integrater_mosaic_min_mean_max(),
(0.180, 0.180, 0.180),
eps=1e-1)
# test serialization of integrater
json_str = integrater.as_json()
#print(json_str)
integrater2 = XDSIntegrater.from_json(string=json_str)
integrater2.set_integrater_sweep(sweep, reset=False)
integrater2_intensities = integrater.get_integrater_intensities()
assert integrater2_intensities == integrater_intensities
integrater2.set_integrater_finish_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=400)
integrater2.set_integrater_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=450)
integrater2.set_integrater_prepare_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50100, eps=400)
def exercise_xds_integrater(nproc=None):
if not have_dials_regression:
print "Skipping exercise_xds_integrater(): dials_regression not configured"
return
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
cwd = os.path.abspath(os.curdir)
tmp_dir = os.path.abspath(open_tmp_directory())
os.chdir(tmp_dir)
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater
indexer = XDSIndexer()
indexer.set_working_directory(tmp_dir)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.XDSRefiner import XDSRefiner
refiner = XDSRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
#refiner.refine()
integrater = XDSIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
#integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.integrate()
integrater_intensities = integrater.get_integrater_intensities()
assert os.path.exists(integrater_intensities)
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(integrater_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=400)
assert mtz_object.column_labels() == [
'H', 'K', 'L', 'M_ISYM', 'BATCH', 'I', 'SIGI', 'FRACTIONCALC',
'XDET', 'YDET', 'ROT', 'LP', 'FLAG']
corrected_intensities = integrater.get_integrater_corrected_intensities()
assert os.path.exists(corrected_intensities)
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(corrected_intensities)
assert reader.file_type() == "xds_ascii"
ma = reader.as_miller_arrays(merge_equivalents=False)[0]
assert approx_equal(ma.size(), 50000, eps=400)
#assert integrater.get_integrater_reindex_operator() == 'x,z,-y'
assert integrater.get_integrater_wedge() == (1, 45)
assert approx_equal(integrater.get_integrater_cell(),
[78.066, 78.066, 78.066, 90, 90, 90], eps=3e-2)
assert approx_equal(integrater.get_integrater_mosaic_min_mean_max(),
(0.180, 0.180, 0.180), eps=1e-3)
# test serialization of integrater
json_str = integrater.as_json()
#print json_str
integrater2 = XDSIntegrater.from_json(string=json_str)
integrater2.set_integrater_sweep(sweep, reset=False)
integrater2_intensities = integrater.get_integrater_intensities()
assert integrater2_intensities == integrater_intensities
integrater2.set_integrater_finish_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=400)
integrater2.set_integrater_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=350)
integrater2.set_integrater_prepare_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=300)
def exercise_mosflm_integrater(dials_data, tmp_dir, nproc):
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
template = dials_data("insulin").join("insulin_1_###.img").strpath
# otherwise if this test is running multiple times simultaneously two mosflm
# processes try to write to the same genfile
os.environ["CCP4_SCR"] = tmp_dir
from xia2.Modules.Indexer.MosflmIndexer import MosflmIndexer
from xia2.Modules.Integrater.MosflmIntegrater import MosflmIntegrater
from dxtbx.model.experiment_list import ExperimentListTemplateImporter
indexer = MosflmIndexer()
indexer.set_working_directory(tmp_dir)
importer = ExperimentListTemplateImporter([template])
experiments = importer.experiments
imageset = experiments.imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.MosflmRefiner import MosflmRefiner
refiner = MosflmRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
nref_error = 500
integrater = MosflmIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
# integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.integrate()
integrater_intensities = integrater.get_integrater_intensities()
assert os.path.exists(integrater_intensities)
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(integrater_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert (abs(mtz_object.n_reflections() - 81116) <
nref_error), mtz_object.n_reflections()
assert mtz_object.column_labels() == [
"H",
"K",
"L",
"M_ISYM",
"BATCH",
"I",
"SIGI",
"IPR",
"SIGIPR",
"FRACTIONCALC",
"XDET",
"YDET",
"ROT",
"WIDTH",
"LP",
"MPART",
"FLAG",
"BGPKRATIOS",
]
assert integrater.get_integrater_wedge() == (1, 45)
assert approx_equal(
integrater.get_integrater_cell(),
(78.014, 78.014, 78.014, 90.0, 90.0, 90.0),
eps=1e-2,
)
# test serialization of integrater
json_str = integrater.as_json()
# print json_str
integrater2 = MosflmIntegrater.from_json(string=json_str)
integrater2.set_integrater_sweep(sweep, reset=False)
integrater2_intensities = integrater.get_integrater_intensities()
assert integrater2_intensities == integrater_intensities
integrater2.set_integrater_finish_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert (abs(mtz_object.n_reflections() - 81116) <
nref_error), mtz_object.n_reflections()
integrater2.set_integrater_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert (abs(mtz_object.n_reflections() - 81116) <
nref_error), mtz_object.n_reflections()
integrater2.set_integrater_prepare_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert (abs(mtz_object.n_reflections() - 81116) <
nref_error), mtz_object.n_reflections()
def test_xds_scalerA(ccp4, xia2_regression_build, tmpdir, nproc):
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
template = os.path.join(xia2_regression_build, "test_data", "insulin",
"insulin_1_###.img")
tmpdir.chdir()
tmpdir = tmpdir.strpath
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater
from xia2.Modules.Scaler.XDSScalerA import XDSScalerA
indexer = XDSIndexer()
indexer.set_working_directory(tmpdir)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
with mock.patch.object(sys, 'argv', []):
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.XDSRefiner import XDSRefiner
refiner = XDSRefiner()
refiner.set_working_directory(tmpdir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
integrater = XDSIntegrater()
integrater.set_working_directory(tmpdir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
#integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.set_integrater_sweep_name('SWEEP1')
integrater.set_integrater_project_info('CRYST1', 'WAVE1', 'SWEEP1')
scaler = XDSScalerA()
scaler.add_scaler_integrater(integrater)
scaler.set_scaler_xcrystal(cryst)
scaler.set_scaler_project_info('CRYST1', 'WAVE1')
check_scaler_files_exist(scaler)
# test serialization of scaler
json_str = scaler.as_json()
#print json_str
scaler2 = XDSScalerA.from_json(string=json_str)
scaler2.set_scaler_xcrystal(cryst)
check_scaler_files_exist(scaler2)
scaler2.set_scaler_finish_done(False)
check_scaler_files_exist(scaler2)
scaler2.set_scaler_done(False)
check_scaler_files_exist(scaler2)
scaler2._scalr_integraters = {} # XXX
scaler2.add_scaler_integrater(integrater)
scaler2.set_scaler_prepare_done(False)
check_scaler_files_exist(scaler2)
def exercise_dials_integrater(nproc=None):
if not have_dials_regression:
print "Skipping exercise_dials_integrater(): dials_regression not configured"
return
if nproc is not None:
from xia2.Handlers.Flags import Flags
Flags.set_parallel(nproc)
xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
cwd = os.path.abspath(os.curdir)
tmp_dir = os.path.abspath(open_tmp_directory())
os.chdir(tmp_dir)
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater
from dxtbx.datablock import DataBlockTemplateImporter
indexer = DialsIndexer()
indexer.set_working_directory(tmp_dir)
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.DialsRefiner import DialsRefiner
refiner = DialsRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
#refiner.refine()
integrater = DialsIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
#integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.integrate()
integrater_intensities = integrater.get_integrater_intensities()
assert os.path.exists(integrater_intensities)
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(integrater_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 48117) < 100, mtz_object.n_reflections()
assert mtz_object.column_labels() == [
'H', 'K', 'L', 'M_ISYM', 'BATCH', 'IPR', 'SIGIPR', 'I', 'SIGI',
'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'LP', 'DQE']
assert integrater.get_integrater_wedge() == (1, 45)
assert approx_equal(integrater.get_integrater_cell(),
(78.14, 78.14, 78.14, 90, 90, 90), eps=1e-1)
# test serialization of integrater
json_str = integrater.as_json()
#print json_str
integrater2 = DialsIntegrater.from_json(string=json_str)
integrater2.set_integrater_sweep(sweep, reset=False)
integrater2_intensities = integrater.get_integrater_intensities()
assert integrater2_intensities == integrater_intensities
integrater2.set_integrater_finish_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 48117) < 100, mtz_object.n_reflections()
integrater2.set_integrater_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 48117) < 100, mtz_object.n_reflections()
integrater2.set_integrater_prepare_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 48117) < 100, mtz_object.n_reflections()
def exercise_dials_indexer(xia2_regression_build, tmp_dir, nproc=None):
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
xia2_demo_data = os.path.join(xia2_regression_build, "test_data",
"insulin")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
indexer = DialsIndexer()
indexer.set_working_directory(tmp_dir)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
indexer.index()
assert approx_equal(indexer.get_indexer_cell(),
(78.14, 78.14, 78.14, 90, 90, 90),
eps=1e-1)
solution = indexer.get_solution()
assert approx_equal(solution['rmsd'], 0.041, eps=1e-2)
assert approx_equal(solution['metric'], 0.027, eps=1e-2)
assert solution['number'] == 22
assert solution['lattice'] == 'cI'
beam_centre = indexer.get_indexer_beam_centre()
assert approx_equal(beam_centre, (94.4223, 94.5097), eps=1e-2)
print(indexer.get_indexer_experiment_list()[0].crystal)
print(indexer.get_indexer_experiment_list()[0].detector)
# test serialization of indexer
json_str = indexer.as_json()
#print(json_str)
indexer2 = DialsIndexer.from_json(string=json_str)
indexer2.index()
assert approx_equal(indexer.get_indexer_cell(),
indexer2.get_indexer_cell())
assert approx_equal(indexer.get_indexer_beam_centre(),
indexer2.get_indexer_beam_centre())
indexer.eliminate()
indexer2.eliminate()
assert approx_equal(indexer.get_indexer_cell(),
indexer2.get_indexer_cell())
assert indexer.get_indexer_lattice() == 'hR'
assert indexer2.get_indexer_lattice() == 'hR'
def exercise_dials_integrater(dials_data, tmp_dir, nproc=None):
if nproc:
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
template = dials_data("insulin").join("insulin_1_###.img").strpath
indexer = DialsIndexer()
indexer.set_working_directory(tmp_dir)
importer = ExperimentListTemplateImporter([template])
experiments = importer.experiments
imageset = experiments.imagesets()[0]
indexer.add_indexer_imageset(imageset)
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
refiner = DialsRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
# refiner.refine()
integrater = DialsIntegrater()
integrater.set_output_format("hkl")
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
# integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.integrate()
integrater_intensities = integrater.get_integrater_intensities()
assert os.path.exists(integrater_intensities)
reader = any_reflection_file(integrater_intensities)
assert reader.file_type() == "ccp4_mtz", repr(integrater_intensities)
mtz_object = reader.file_content()
expected_reflections = 47623
assert (abs(mtz_object.n_reflections() - expected_reflections) <
300), mtz_object.n_reflections()
assert mtz_object.column_labels() == [
"H",
"K",
"L",
"M_ISYM",
"BATCH",
"IPR",
"SIGIPR",
"I",
"SIGI",
"BG",
"SIGBG",
"FRACTIONCALC",
"XDET",
"YDET",
"ROT",
"LP",
"QE",
]
assert integrater.get_integrater_wedge() == (1, 45)
assert integrater.get_integrater_cell() == pytest.approx(
(78.14, 78.14, 78.14, 90, 90, 90), abs=1e-1)
# test serialization of integrater
json_str = integrater.as_json()
# print(json_str)
integrater2 = DialsIntegrater.from_json(string=json_str)
integrater2.set_integrater_sweep(sweep, reset=False)
integrater2_intensities = integrater.get_integrater_intensities()
assert integrater2_intensities == integrater_intensities
integrater2.set_integrater_finish_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert (abs(mtz_object.n_reflections() - expected_reflections) <
300), mtz_object.n_reflections()
integrater2.set_integrater_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert (abs(mtz_object.n_reflections() - expected_reflections) <
300), mtz_object.n_reflections()
integrater2.set_integrater_prepare_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert (abs(mtz_object.n_reflections() - expected_reflections) <
300), mtz_object.n_reflections()
# Test that diamond anvil cell attenuation correction does something.
# That it does the right thing is left as a matter for the DIALS tests.
integrater3 = DialsIntegrater.from_json(string=json_str)
integrater3.set_integrater_sweep(sweep, reset=False)
integrater3.set_integrater_done(False)
integrater3.high_pressure = True
# Don't get .hkl output because we're applying the attenuation correction to data
# that weren't actually collected with a diamond anvil cell and some integrated
# intensities will be rather nonsensical, which causes an error
# 'cctbx Error: Inconsistent observation/sigma pair in columns: IPR, SIGIPR',
# when some internal .hkl consistency checks are run, which is not meaningful here.
integrater3.set_output_format("pickle")
# Compare the first ten profile-fitted integrated intensities without correction.
control_reflections = flex.reflection_table.from_file(
integrater2.get_integrated_reflections())
valid = control_reflections.get_flags(
control_reflections.flags.integrated_prf)
valid = valid.iselection()[:10]
control_reflections = control_reflections.select(valid)
# Get the first ten profile-fitted integrated intensities with DAC correction.
corrected_reflections = flex.reflection_table.from_file(
integrater3.get_integrated_reflections())
valid = corrected_reflections.get_flags(
corrected_reflections.flags.integrated_prf)
valid = valid.iselection()[:10]
corrected_reflections = corrected_reflections.select(valid)
# Check that we're comparing equivalent reflections.
assert control_reflections["miller_index"] == corrected_reflections[
"miller_index"]
control_intensities = control_reflections["intensity.prf.value"]
corrected_intensities = corrected_reflections["intensity.prf.value"]
# Check that the reflection intensities are not the same.
assert pytest.approx(control_intensities) != corrected_intensities
def exercise_mosflm_integrater(dials_regression, tmp_dir, nproc):
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
# otherwise if this test is running multiple times simultaneously two mosflm
# processes try to write to the same genfile
os.environ['CCP4_SCR'] = tmp_dir
from xia2.Modules.Indexer.MosflmIndexer import MosflmIndexer
from xia2.Modules.Integrater.MosflmIntegrater import MosflmIntegrater
from dxtbx.datablock import DataBlockTemplateImporter
indexer = MosflmIndexer()
indexer.set_working_directory(tmp_dir)
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.MosflmRefiner import MosflmRefiner
refiner = MosflmRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
nref_error = 500
integrater = MosflmIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
#integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.integrate()
integrater_intensities = integrater.get_integrater_intensities()
assert os.path.exists(integrater_intensities)
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(integrater_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 81116) < nref_error, \
mtz_object.n_reflections()
assert mtz_object.column_labels() == [
'H', 'K', 'L', 'M_ISYM', 'BATCH', 'I', 'SIGI', 'IPR', 'SIGIPR',
'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'WIDTH', 'LP', 'MPART', 'FLAG',
'BGPKRATIOS']
assert integrater.get_integrater_wedge() == (1, 45)
assert approx_equal(integrater.get_integrater_cell(),
(78.014, 78.014, 78.014, 90.0, 90.0, 90.0), eps=1e-2)
# test serialization of integrater
json_str = integrater.as_json()
#print json_str
integrater2 = MosflmIntegrater.from_json(string=json_str)
integrater2.set_integrater_sweep(sweep, reset=False)
integrater2_intensities = integrater.get_integrater_intensities()
assert integrater2_intensities == integrater_intensities
integrater2.set_integrater_finish_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 81116) < nref_error, \
mtz_object.n_reflections()
integrater2.set_integrater_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 81116) < nref_error, \
mtz_object.n_reflections()
integrater2.set_integrater_prepare_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 81116) < nref_error, \
mtz_object.n_reflections()
def exercise_dials_refiner(nproc=None):
if not have_xia2_regression:
print "Skipping exercise_dials_refiner(): xia2_regression not configured"
return
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
xia2_demo_data = os.path.join(xia2_regression, "test_data", "mad_example")
template = os.path.join(xia2_demo_data, "12287_1_E%i_%03i.img")
cwd = os.path.abspath(os.curdir)
tmp_dir = os.path.abspath(open_tmp_directory())
os.chdir(tmp_dir)
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
from xia2.Modules.Refiner.DialsRefiner import DialsRefiner
from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater
indexer1 = DialsIndexer()
indexer1.set_working_directory(tmp_dir)
indexer1.setup_from_image(template % (1, 1))
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav1 = XWavelength("WAVE1", cryst, indexer1.get_wavelength())
samp1 = XSample("X1", cryst)
directory, image = os.path.split(template % (1, 1))
sweep1 = XSweep('SWEEP1', wav1, samp1, directory=directory, image=image)
indexer1.set_indexer_sweep(sweep1)
indexer2 = DialsIndexer()
indexer2.set_working_directory(tmp_dir)
indexer2.setup_from_image(template % (2, 1))
wav2 = XWavelength("WAVE2", cryst, indexer2.get_wavelength())
samp2 = XSample("X2", cryst)
directory, image = os.path.split(template % (2, 1))
sweep2 = XSweep('SWEEP2', wav2, samp2, directory=directory, image=image)
indexer2.set_indexer_sweep(sweep2)
refiner = DialsRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(indexer1)
refiner.add_refiner_indexer(indexer2)
refined_experiment_list = refiner.get_refined_experiment_list()
assert refined_experiment_list is not None
assert len(refined_experiment_list.detectors()) == 1
refined_detector = refined_experiment_list[0].detector
# test serialization of refiner
json_str = refiner.as_json()
#print json_str
refiner2 = DialsRefiner.from_json(string=json_str)
refined_expts_2 = refiner2.get_refined_experiment_list()
assert refined_expts_2[0].detector == refined_detector
refiner2.set_refiner_finish_done(False)
refined_expts_2 = refiner2.get_refined_experiment_list()
assert refined_expts_2[0].detector == refined_detector
refiner2.set_refiner_done(False)
refined_expts_2 = refiner2.get_refined_experiment_list()
assert refined_expts_2[0].detector == refined_detector
refiner2.set_refiner_prepare_done(False)
refined_expts_2 = refiner2.get_refined_experiment_list()
assert refined_expts_2[0].detector == refined_detector
assert (indexer1.get_indexer_experiment_list()[0].detector !=
indexer2.get_indexer_experiment_list()[0].detector)
assert (indexer1.get_indexer_experiment_list()[0].beam !=
indexer2.get_indexer_experiment_list()[0].beam)
indexer1.set_indexer_experiment_list(refined_experiment_list[0:1])
indexer2.set_indexer_experiment_list(refined_experiment_list[1:2])
assert (indexer1.get_indexer_experiment_list()[0].detector ==
indexer2.get_indexer_experiment_list()[0].detector)
assert (indexer1.get_indexer_experiment_list()[0].goniometer ==
indexer2.get_indexer_experiment_list()[0].goniometer)
def exercise_ccp4_scaler(nproc=None):
if not have_dials_regression:
print "Skipping exercise_ccp4_scaler(): dials_regression not configured"
return
if nproc is not None:
from xia2.Handlers.Flags import Flags
Flags.set_parallel(nproc)
xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
cwd = os.path.abspath(os.curdir)
tmp_dir = os.path.abspath(open_tmp_directory())
os.chdir(tmp_dir)
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
from xia2.Modules.Refiner.DialsRefiner import DialsRefiner
from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater
from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA
indexer = DialsIndexer()
indexer.set_working_directory(tmp_dir)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
refiner = DialsRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
integrater = DialsIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
#integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.set_integrater_sweep_name('SWEEP1')
integrater.set_integrater_project_info('CRYST1', 'WAVE1', 'SWEEP1')
scaler = CCP4ScalerA()
scaler.add_scaler_integrater(integrater)
scaler.set_scaler_xcrystal(cryst)
scaler.set_scaler_project_info('CRYST1', 'WAVE1')
check_scaler_files_exist(scaler)
# test serialization of scaler
json_str = scaler.as_json()
#print json_str
scaler2 = CCP4ScalerA.from_json(string=json_str)
scaler2.set_scaler_xcrystal(cryst)
check_scaler_files_exist(scaler2)
scaler2.set_scaler_finish_done(False)
check_scaler_files_exist(scaler2)
scaler2.set_scaler_done(False)
check_scaler_files_exist(scaler2)
scaler2._scalr_integraters = {} # XXX
scaler2.add_scaler_integrater(integrater)
scaler2.set_scaler_prepare_done(False)
check_scaler_files_exist(scaler2)
self.integrate()
from xia2.Wrappers.Dials.ExportXDSASCII import ExportXDSASCII
exporter = ExportXDSASCII()
exporter.set_experiments_filename(self.get_integrated_experiments())
exporter.set_reflections_filename(self.get_integrated_reflections())
exporter.set_working_directory(self.get_working_directory())
auto_logfiler(exporter)
self._intgr_corrected_hklout = os.path.join(
self.get_working_directory(), '%i_DIALS.HKL' % exporter.get_xpid())
exporter.set_hkl_filename(self._intgr_corrected_hklout)
exporter.run()
assert os.path.exists(self._intgr_corrected_hklout)
return self._intgr_corrected_hklout
if __name__ == '__main__':
# run a demo test
di = DialsIntegrater()
di.setup_from_image(sys.argv[1])
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, di.get_wavelength())
directory, image = os.path.split(sys.argv[1])
sweep = XSweep('SWEEP1', wav, directory=directory, image=image)
di.set_integrater_sweep(sweep)
di.integrate()
def xia2_main(stop_after=None):
'''Actually process something...'''
Citations.cite('xia2')
# print versions of related software
from dials.util.version import dials_version
Chatter.write(dials_version())
start_time = time.time()
CommandLine = get_command_line()
start_dir = Flags.get_starting_directory()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
no_images = True
for name in xtals.keys():
xtal = xtals[name]
if not xtal.get_all_image_names():
Chatter.write('-----------------------------------' + \
'-' * len(name))
Chatter.write('| No images assigned for crystal %s |' % name)
Chatter.write('-----------------------------------' + '-' \
* len(name))
else:
no_images = False
args = []
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
from libtbx import group_args
xinfo = CommandLine.get_xinfo()
if os.path.exists('xia2.json'):
from xia2.Schema.XProject import XProject
xinfo_new = xinfo
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
crystals_new = xinfo_new.get_crystals()
for crystal_id in crystals_new.keys():
if crystal_id not in crystals:
crystals[crystal_id] = crystals_new[crystal_id]
continue
crystals[crystal_id]._scaler = None # reset scaler
for wavelength_id in crystals_new[crystal_id].get_wavelength_names():
wavelength_new = crystals_new[crystal_id].get_xwavelength(wavelength_id)
if wavelength_id not in crystals[crystal_id].get_wavelength_names():
crystals[crystal_id].add_wavelength(
crystals_new[crystal_id].get_xwavelength(wavelength_new))
continue
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
sweeps_new = wavelength_new.get_sweeps()
sweeps = wavelength.get_sweeps()
sweep_names = [s.get_name() for s in sweeps]
sweep_keys = [
(s.get_directory(), s.get_template(), s.get_image_range())
for s in sweeps]
for sweep in sweeps_new:
if ((sweep.get_directory(), sweep.get_template(),
sweep.get_image_range()) not in sweep_keys):
if sweep.get_name() in sweep_names:
i = 1
while 'SWEEEP%i' %i in sweep_names:
i += 1
sweep._name = 'SWEEP%i' %i
break
wavelength.add_sweep(
name=sweep.get_name(),
directory=sweep.get_directory(),
image=sweep.get_image(),
beam=sweep.get_beam_centre(),
reversephi=sweep.get_reversephi(),
distance=sweep.get_distance(),
gain=sweep.get_gain(),
dmin=sweep.get_resolution_high(),
dmax=sweep.get_resolution_low(),
polarization=sweep.get_polarization(),
frames_to_process=sweep.get_frames_to_process(),
user_lattice=sweep.get_user_lattice(),
user_cell=sweep.get_user_cell(),
epoch=sweep._epoch,
ice=sweep._ice,
excluded_regions=sweep._excluded_regions,
)
sweep_names.append(sweep.get_name())
crystals = xinfo.get_crystals()
failover = params.xia2.settings.failover
if njob > 1:
driver_type = mp_params.type
command_line_args = CommandLine.get_argv()[1:]
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
sweep._get_indexer()
sweep._get_refiner()
sweep._get_integrater()
args.append((
group_args(
driver_type=driver_type,
stop_after=stop_after,
failover=failover,
command_line_args=command_line_args,
nproc=mp_params.nproc,
crystal_id=crystal_id,
wavelength_id=wavelength_id,
sweep_id=sweep.get_name(),
),))
from xia2.Driver.DriverFactory import DriverFactory
default_driver_type = DriverFactory.get_driver_type()
# run every nth job on the current computer (no need to submit to qsub)
for i_job, arg in enumerate(args):
if (i_job % njob) == 0:
arg[0].driver_type = default_driver_type
if mp_params.type == "qsub":
method = "sge"
else:
method = "multiprocessing"
nproc = mp_params.nproc
qsub_command = mp_params.qsub_command
if not qsub_command:
qsub_command = 'qsub'
qsub_command = '%s -V -cwd -pe smp %d' %(qsub_command, nproc)
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_sweep, args, processes=njob,
#method=method,
method="multiprocessing",
qsub_command=qsub_command,
preserve_order=True,
preserve_exception_message=True)
# Hack to update sweep with the serialized indexers/refiners/integraters
i_sweep = 0
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
success, output, xsweep_dict = results[i_sweep]
assert xsweep_dict is not None
if output is not None:
Chatter.write(output)
if not success:
Chatter.write('Sweep failed: removing %s' %sweep.get_name())
remove_sweeps.append(sweep)
else:
Chatter.write('Loading sweep: %s' % sweep.get_name())
from xia2.Schema.XSweep import XSweep
new_sweep = XSweep.from_dict(xsweep_dict)
sweep._indexer = new_sweep._indexer
sweep._refiner = new_sweep._refiner
sweep._integrater = new_sweep._integrater
i_sweep += 1
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
else:
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
try:
if stop_after == 'index':
sweep.get_indexer_cell()
else:
sweep.get_integrater_intensities()
sweep.serialize()
except Exception, e:
if failover:
Chatter.write('Processing sweep %s failed: %s' % \
(sweep.get_name(), str(e)))
remove_sweeps.append(sweep)
else:
raise
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
exporter = ExportXDSASCII()
exporter.set_experiments_filename(self.get_integrated_experiments())
exporter.set_reflections_filename(self.get_integrated_reflections())
exporter.set_working_directory(self.get_working_directory())
auto_logfiler(exporter)
self._intgr_corrected_hklout = os.path.join(
self.get_working_directory(), "%i_DIALS.HKL" % exporter.get_xpid())
exporter.set_hkl_filename(self._intgr_corrected_hklout)
exporter.run()
assert os.path.exists(self._intgr_corrected_hklout)
return self._intgr_corrected_hklout
if __name__ == "__main__":
# run a demo test
di = DialsIntegrater()
di.setup_from_image(sys.argv[1])
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, di.get_wavelength())
directory, image = os.path.split(sys.argv[1])
sweep = XSweep("SWEEP1", wav, directory=directory, image=image)
di.set_integrater_sweep(sweep)
di.integrate()
def xia2_main(stop_after=None):
"""Actually process something..."""
Citations.cite("xia2")
# print versions of related software
logger.info(dials_version())
ccp4_version = get_ccp4_version()
if ccp4_version:
logger.info("CCP4 %s", ccp4_version)
start_time = time.time()
CommandLine = get_command_line()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
for name, xtal in xtals.items():
if not xtal.get_all_image_names():
logger.info("-----------------------------------" +
"-" * len(name))
logger.info("| No images assigned for crystal %s |", name)
logger.info("-----------------------------------" +
"-" * len(name))
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
xinfo = CommandLine.get_xinfo()
logger.info("Project directory: %s", xinfo.path)
if (params.xia2.settings.developmental.continue_from_previous_job
and os.path.exists("xia2.json")):
logger.debug("==== Starting from existing xia2.json ====")
xinfo_new = xinfo
xinfo = XProject.from_json(filename="xia2.json")
crystals = xinfo.get_crystals()
crystals_new = xinfo_new.get_crystals()
for crystal_id in crystals_new:
if crystal_id not in crystals:
crystals[crystal_id] = crystals_new[crystal_id]
continue
crystals[crystal_id]._scaler = None # reset scaler
for wavelength_id in crystals_new[crystal_id].get_wavelength_names(
):
wavelength_new = crystals_new[crystal_id].get_xwavelength(
wavelength_id)
if wavelength_id not in crystals[
crystal_id].get_wavelength_names():
crystals[crystal_id].add_wavelength(
crystals_new[crystal_id].get_xwavelength(
wavelength_new))
continue
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps_new = wavelength_new.get_sweeps()
sweeps = wavelength.get_sweeps()
sweep_names = {s.get_name() for s in sweeps}
sweep_keys = {(s.get_directory(), s.get_template(),
s.get_image_range())
for s in sweeps}
for sweep in sweeps_new:
if (
sweep.get_directory(),
sweep.get_template(),
sweep.get_image_range(),
) not in sweep_keys:
if sweep.get_name() in sweep_names:
i = 1
while "SWEEEP%i" % i in sweep_names:
i += 1
sweep._name = "SWEEP%i" % i
break
wavelength.add_sweep(
name=sweep.get_name(),
sample=sweep.sample,
directory=sweep.get_directory(),
image=sweep.get_image(),
beam=sweep.get_beam_centre(),
reversephi=sweep.get_reversephi(),
distance=sweep.get_distance(),
gain=sweep.get_gain(),
dmin=sweep.get_resolution_high(),
dmax=sweep.get_resolution_low(),
polarization=sweep.get_polarization(),
frames_to_process=sweep.get_frames_to_process(),
user_lattice=sweep.get_user_lattice(),
user_cell=sweep.get_user_cell(),
epoch=sweep._epoch,
ice=sweep._ice,
excluded_regions=sweep._excluded_regions,
)
sweep_names.add(sweep.get_name())
crystals = xinfo.get_crystals()
failover = params.xia2.settings.failover
with cleanup(xinfo.path):
if mp_params.mode == "parallel" and njob > 1:
driver_type = mp_params.type
command_line_args = CommandLine.get_argv()[1:]
jobs = []
for crystal_id in crystals:
for wavelength_id in crystals[crystal_id].get_wavelength_names(
):
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
sweep._get_indexer()
sweep._get_refiner()
sweep._get_integrater()
jobs.append((group_args(
driver_type=driver_type,
stop_after=stop_after,
failover=failover,
command_line_args=command_line_args,
nproc=mp_params.nproc,
crystal_id=crystal_id,
wavelength_id=wavelength_id,
sweep_id=sweep.get_name(),
), ))
from xia2.Driver.DriverFactory import DriverFactory
default_driver_type = DriverFactory.get_driver_type()
# run every nth job on the current computer (no need to submit to qsub)
for i_job, arg in enumerate(jobs):
if (i_job % njob) == 0:
arg[0].driver_type = default_driver_type
nproc = mp_params.nproc
qsub_command = mp_params.qsub_command or "qsub"
qsub_command = "%s -V -cwd -pe smp %d" % (qsub_command, nproc)
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_sweep,
jobs,
processes=njob,
method="multiprocessing",
qsub_command=qsub_command,
preserve_order=True,
preserve_exception_message=True,
)
# Hack to update sweep with the serialized indexers/refiners/integraters
i_sweep = 0
for crystal_id in crystals:
for wavelength_id in crystals[crystal_id].get_wavelength_names(
):
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
success, output, xsweep_dict = results[i_sweep]
if output is not None:
logger.info(output)
if not success:
logger.info("Sweep failed: removing %s",
sweep.get_name())
remove_sweeps.append(sweep)
else:
assert xsweep_dict is not None
logger.info("Loading sweep: %s", sweep.get_name())
new_sweep = XSweep.from_dict(xsweep_dict)
sweep._indexer = new_sweep._indexer
sweep._refiner = new_sweep._refiner
sweep._integrater = new_sweep._integrater
i_sweep += 1
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.sample
sample.remove_sweep(sweep)
else:
for crystal_id in list(crystals.keys()):
for wavelength_id in crystals[crystal_id].get_wavelength_names(
):
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
from dials.command_line.show import show_experiments
from dxtbx.model.experiment_list import ExperimentListFactory
logger.debug(sweep.get_name())
logger.debug(
show_experiments(
ExperimentListFactory.
from_imageset_and_crystal(
sweep.get_imageset(), None)))
Citations.cite("dials")
try:
if stop_after == "index":
sweep.get_indexer_cell()
else:
sweep.get_integrater_intensities()
sweep.serialize()
except Exception as e:
if failover:
logger.info(
"Processing sweep %s failed: %s",
sweep.get_name(),
str(e),
)
remove_sweeps.append(sweep)
else:
raise
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.sample
sample.remove_sweep(sweep)
# save intermediate xia2.json file in case scaling step fails
xinfo.as_json(filename="xia2.json")
if stop_after not in ("index", "integrate"):
logger.info(xinfo.get_output())
for crystal in list(crystals.values()):
crystal.serialize()
# save final xia2.json file in case report generation fails
xinfo.as_json(filename="xia2.json")
if stop_after not in ("index", "integrate"):
# and the summary file
with open("xia2-summary.dat", "w") as fh:
for record in xinfo.summarise():
fh.write("%s\n" % record)
# looks like this import overwrites the initial command line
# Phil overrides so... for https://github.com/xia2/xia2/issues/150
from xia2.command_line.html import generate_xia2_html
if params.xia2.settings.small_molecule:
params.xia2.settings.report.xtriage_analysis = False
params.xia2.settings.report.include_radiation_damage = False
with xia2.Driver.timing.record_step("xia2.report"):
generate_xia2_html(xinfo,
filename="xia2.html",
params=params.xia2.settings.report)
duration = time.time() - start_time
# write out the time taken in a human readable way
logger.info("Processing took %s",
time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))
write_citations()
def exercise_dials_indexer(dials_data, tmp_dir, nproc=None):
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
template = dials_data("insulin").join("insulin_1_###.img").strpath
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
indexer = DialsIndexer()
indexer.set_working_directory(tmp_dir)
from dxtbx.model.experiment_list import ExperimentListTemplateImporter
importer = ExperimentListTemplateImporter([template])
experiments = importer.experiments
imageset = experiments.imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
indexer.index()
assert approx_equal(indexer.get_indexer_cell(),
(78.14, 78.14, 78.14, 90, 90, 90),
eps=1e-1)
solution = indexer.get_solution()
assert approx_equal(solution["rmsd"], 0.041, eps=1e-2)
assert approx_equal(solution["metric"], 0.027, eps=1e-2)
assert solution["number"] == 22
assert solution["lattice"] == "cI"
beam_centre = indexer.get_indexer_beam_centre()
assert approx_equal(beam_centre, (94.4223, 94.5097), eps=1e-2)
print(indexer.get_indexer_experiment_list()[0].crystal)
print(indexer.get_indexer_experiment_list()[0].detector)
# test serialization of indexer
json_str = indexer.as_json()
# print(json_str)
indexer2 = DialsIndexer.from_json(string=json_str)
indexer2.index()
assert approx_equal(indexer.get_indexer_cell(),
indexer2.get_indexer_cell())
assert approx_equal(indexer.get_indexer_beam_centre(),
indexer2.get_indexer_beam_centre())
indexer.eliminate()
indexer2.eliminate()
assert approx_equal(indexer.get_indexer_cell(),
indexer2.get_indexer_cell())
assert indexer.get_indexer_lattice() == "hR"
assert indexer2.get_indexer_lattice() == "hR"
def exercise_xds_indexer(xia2_regression_build, tmp_dir, nproc=None):
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
xia2_demo_data = os.path.join(xia2_regression_build, "test_data",
"insulin")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
indexer = XDSIndexer()
indexer.set_working_directory(tmp_dir)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
indexer.index()
assert approx_equal(indexer.get_indexer_cell(),
(78.076, 78.076, 78.076, 90, 90, 90),
eps=1), indexer.get_indexer_cell()
experiment = indexer.get_indexer_experiment_list()[0]
sgi = experiment.crystal.get_space_group().info()
assert sgi.type().number() == 197
beam_centre = indexer.get_indexer_beam_centre()
assert approx_equal(beam_centre, (94.4221, 94.5096), eps=1e-1)
assert indexer.get_indexer_images() == [(1, 5), (20, 24), (41, 45)]
print(indexer.get_indexer_experiment_list()[0].crystal)
print(indexer.get_indexer_experiment_list()[0].detector)
# test serialization of indexer
json_str = indexer.as_json()
print(json_str)
indexer2 = XDSIndexer.from_json(string=json_str)
indexer2.index()
assert approx_equal(indexer.get_indexer_cell(),
indexer2.get_indexer_cell())
assert approx_equal(indexer.get_indexer_beam_centre(),
indexer2.get_indexer_beam_centre())
assert approx_equal(indexer.get_indexer_images(),
indexer2.get_indexer_images())
indexer.eliminate()
indexer2.eliminate()
assert approx_equal(indexer.get_indexer_cell(),
indexer2.get_indexer_cell())
assert indexer.get_indexer_lattice() == 'hR'
assert indexer2.get_indexer_lattice() == 'hR'
def test_ccp4_scalerA(regression_test, ccp4, dials_data, run_in_tmpdir, nproc):
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
template = dials_data("insulin").join("insulin_1_###.img").strpath
tmpdir = run_in_tmpdir.strpath
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
from xia2.Modules.Refiner.DialsRefiner import DialsRefiner
from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater
from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA
indexer = DialsIndexer()
indexer.set_working_directory(tmpdir)
from dxtbx.model.experiment_list import ExperimentListTemplateImporter
importer = ExperimentListTemplateImporter([template])
experiments = importer.experiments
imageset = experiments.imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
with mock.patch.object(sys, "argv", []):
sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
refiner = DialsRefiner()
refiner.set_working_directory(tmpdir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
integrater = DialsIntegrater()
integrater.set_working_directory(tmpdir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
# integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.set_integrater_sweep_name("SWEEP1")
integrater.set_integrater_project_info("CRYST1", "WAVE1", "SWEEP1")
scaler = CCP4ScalerA()
scaler.add_scaler_integrater(integrater)
scaler.set_scaler_xcrystal(cryst)
scaler.set_scaler_project_info("CRYST1", "WAVE1")
check_scaler_files_exist(scaler)
# test serialization of scaler
json_str = scaler.as_json()
# print json_str
scaler2 = CCP4ScalerA.from_json(string=json_str)
scaler2.set_scaler_xcrystal(cryst)
check_scaler_files_exist(scaler2)
scaler2.set_scaler_finish_done(False)
check_scaler_files_exist(scaler2)
scaler2.set_scaler_done(False)
check_scaler_files_exist(scaler2)
scaler2._scalr_integraters = {} # XXX
scaler2.add_scaler_integrater(integrater)
scaler2.set_scaler_prepare_done(False)
check_scaler_files_exist(scaler2)
def exercise_serialization(dials_data, tmp_dir):
base_path = pathlib.Path(tmp_dir)
template = dials_data("insulin").join("insulin_1_###.img").strpath
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
from xia2.Modules.Refiner.DialsRefiner import DialsRefiner
from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater
from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA
from dxtbx.model.experiment_list import ExperimentListTemplateImporter
importer = ExperimentListTemplateImporter([template])
experiments = importer.experiments
imageset = experiments.imagesets()[0]
from xia2.Schema.XProject import XProject
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
proj = XProject(base_path=base_path)
proj._name = "PROJ1"
cryst = XCrystal("CRYST1", proj)
wav = XWavelength("WAVE1", cryst, wavelength=0.98)
samp = XSample("X1", cryst)
cryst.add_wavelength(wav)
cryst.set_ha_info({"atom": "S"})
cryst.add_sample(samp)
directory, image = os.path.split(imageset.get_path(1))
sweep = wav.add_sweep(name="SWEEP1", sample=samp, directory=directory, image=image)
samp.add_sweep(sweep)
from dxtbx.serialize.load import _decode_dict
indexer = DialsIndexer()
indexer.set_working_directory(tmp_dir)
indexer.add_indexer_imageset(imageset)
indexer.set_indexer_sweep(sweep)
sweep._indexer = indexer
refiner = DialsRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
refiner.add_refiner_sweep(sweep)
sweep._refiner = refiner
integrater = DialsIntegrater()
integrater.set_output_format("hkl")
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
# integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.set_integrater_epoch(sweep.get_epoch(1))
integrater.set_integrater_sweep_name(sweep.get_name())
integrater.set_integrater_project_info(
cryst.get_name(), wav.get_name(), sweep.get_name()
)
sweep._integrater = integrater
scaler = CCP4ScalerA(base_path=base_path)
scaler.add_scaler_integrater(integrater)
scaler.set_scaler_xcrystal(cryst)
scaler.set_scaler_project_info(cryst.get_name(), wav.get_name())
scaler._scalr_xcrystal = cryst
cryst._scaler = scaler
proj.add_crystal(cryst)
s_dict = sweep.to_dict()
s_str = json.dumps(s_dict, ensure_ascii=True)
s_dict = json.loads(s_str, object_hook=_decode_dict)
xsweep = XSweep.from_dict(s_dict)
assert xsweep
w_dict = wav.to_dict()
w_str = json.dumps(w_dict, ensure_ascii=True)
w_dict = json.loads(w_str, object_hook=_decode_dict)
xwav = XWavelength.from_dict(w_dict)
assert xwav.get_sweeps()[0].get_wavelength() is xwav
c_dict = cryst.to_dict()
c_str = json.dumps(c_dict, ensure_ascii=True)
c_dict = json.loads(c_str, object_hook=_decode_dict)
xcryst = XCrystal.from_dict(c_dict)
assert (
xcryst.get_xwavelength(xcryst.get_wavelength_names()[0]).get_crystal() is xcryst
)
p_dict = proj.to_dict()
p_str = json.dumps(p_dict, ensure_ascii=True)
p_dict = json.loads(p_str, object_hook=_decode_dict)
xproj = XProject.from_dict(p_dict)
assert xproj.path == base_path
assert list(xproj.get_crystals().values())[0].get_project() is xproj
assert list(xproj.get_crystals().values())[0]._scaler._base_path == base_path
json_str = proj.as_json()
xproj = XProject.from_json(string=json_str)
assert xproj.path == base_path
assert list(xproj.get_crystals().values())[0].get_project() is xproj
print(xproj.get_output())
print("\n".join(xproj.summarise()))
json_str = xproj.as_json()
xproj = XProject.from_json(string=json_str)
assert xproj.path == base_path
# Test that we can serialize to json and back again
xproj = XProject.from_json(string=xproj.as_json())
assert xproj.path == base_path
xcryst = list(xproj.get_crystals().values())[0]
assert xcryst.get_project() is xproj
intgr = xcryst._get_integraters()[0]
assert intgr.get_integrater_finish_done()
assert (
xcryst._get_scaler()
._sweep_handler.get_sweep_information(intgr.get_integrater_epoch())
.get_integrater()
is intgr
)
print(xproj.get_output())
print("\n".join(xproj.summarise()))
def exercise_serialization():
if not have_dials_regression:
print "Skipping exercise_serialization(): dials_regression not configured"
return
from xia2.Handlers.CommandLine import CommandLine
xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
cwd = os.path.abspath(os.curdir)
tmp_dir = os.path.abspath(open_tmp_directory())
os.chdir(tmp_dir)
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
from xia2.Modules.Refiner.DialsRefiner import DialsRefiner
from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater
from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
from xia2.Schema.XProject import XProject
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
proj = XProject()
proj._name = "PROJ1"
cryst = XCrystal("CRYST1", proj)
proj.add_crystal(cryst)
wav = XWavelength("WAVE1", cryst, wavelength=0.98)
samp = XSample("X1", cryst)
cryst.add_wavelength(wav)
cryst.set_ha_info({'atom': 'S'})
directory, image = os.path.split(imageset.get_path(1))
wav.add_sweep(name='SWEEP1', sample=samp, directory=directory, image=image)
import json
from dxtbx.serialize.load import _decode_dict
sweep = wav.get_sweeps()[0]
indexer = DialsIndexer()
indexer.set_working_directory(tmp_dir)
indexer.add_indexer_imageset(imageset)
indexer.set_indexer_sweep(sweep)
sweep._indexer = indexer
refiner = DialsRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
sweep._refiner = refiner
integrater = DialsIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
#integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.set_integrater_epoch(sweep.get_epoch(1))
integrater.set_integrater_sweep_name(sweep.get_name())
integrater.set_integrater_project_info(
cryst.get_name(), wav.get_name(), sweep.get_name())
sweep._integrater = integrater
scaler = CCP4ScalerA()
scaler.add_scaler_integrater(integrater)
scaler.set_scaler_xcrystal(cryst)
scaler.set_scaler_project_info(cryst.get_name(), wav.get_name())
scaler._scalr_xcrystal = cryst
cryst._scaler = scaler
s_dict = sweep.to_dict()
s_str = json.dumps(s_dict, ensure_ascii=True)
s_dict = json.loads(s_str, object_hook=_decode_dict)
xsweep = XSweep.from_dict(s_dict)
w_dict = wav.to_dict()
w_str = json.dumps(w_dict, ensure_ascii=True)
w_dict = json.loads(w_str, object_hook=_decode_dict)
xwav = XWavelength.from_dict(w_dict)
assert xwav.get_sweeps()[0].get_wavelength() is xwav
c_dict = cryst.to_dict()
c_str = json.dumps(c_dict, ensure_ascii=True)
c_dict = json.loads(c_str, object_hook=_decode_dict)
xcryst = XCrystal.from_dict(c_dict)
assert xcryst.get_xwavelength(xcryst.get_wavelength_names()[0]).get_crystal() is xcryst
p_dict = proj.to_dict()
p_str = json.dumps(p_dict, ensure_ascii=True)
p_dict = json.loads(p_str, object_hook=_decode_dict)
xproj = XProject.from_dict(p_dict)
assert xproj.get_crystals().values()[0].get_project() is xproj
json_str = proj.as_json()
xproj = XProject.from_json(string=json_str)
assert xproj.get_crystals().values()[0].get_project() is xproj
print xproj.get_output()
print "\n".join(xproj.summarise())
json_str = xproj.as_json()
xproj = XProject.from_json(string=json_str)
assert xproj.get_crystals().values()[0].get_project() is xproj
xcryst = xproj.get_crystals().values()[0]
intgr = xcryst._get_integraters()[0]
assert intgr.get_integrater_finish_done()
assert xcryst._get_scaler()._sweep_handler.get_sweep_information(
intgr.get_integrater_epoch()).get_integrater() is intgr
print xproj.get_output()
print "\n".join(xproj.summarise())
print
def exercise_mosflm_integrater(nproc=None):
if not have_dials_regression:
print "Skipping exercise_mosflm_integrater(): dials_regression not configured"
return
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
cwd = os.path.abspath(os.curdir)
tmp_dir = os.path.abspath(open_tmp_directory())
os.chdir(tmp_dir)
# otherwise if this test is running multiple times simultaneously two mosflm
# processes try to write to the same genfile
os.environ['CCP4_SCR'] = tmp_dir
from xia2.Modules.Indexer.MosflmIndexer import MosflmIndexer
from xia2.Modules.Integrater.MosflmIntegrater import MosflmIntegrater
from dxtbx.datablock import DataBlockTemplateImporter
indexer = MosflmIndexer()
indexer.set_working_directory(tmp_dir)
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.MosflmRefiner import MosflmRefiner
refiner = MosflmRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
nref_error = 500
integrater = MosflmIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
#integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.integrate()
integrater_intensities = integrater.get_integrater_intensities()
assert os.path.exists(integrater_intensities)
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(integrater_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 81116) < nref_error, \
mtz_object.n_reflections()
assert mtz_object.column_labels() == [
'H', 'K', 'L', 'M_ISYM', 'BATCH', 'I', 'SIGI', 'IPR', 'SIGIPR',
'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'WIDTH', 'LP', 'MPART', 'FLAG',
'BGPKRATIOS']
assert integrater.get_integrater_wedge() == (1, 45)
assert approx_equal(integrater.get_integrater_cell(),
(78.014, 78.014, 78.014, 90.0, 90.0, 90.0), eps=1e-2)
# test serialization of integrater
json_str = integrater.as_json()
#print json_str
integrater2 = MosflmIntegrater.from_json(string=json_str)
integrater2.set_integrater_sweep(sweep, reset=False)
integrater2_intensities = integrater.get_integrater_intensities()
assert integrater2_intensities == integrater_intensities
integrater2.set_integrater_finish_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 81116) < nref_error, \
mtz_object.n_reflections()
integrater2.set_integrater_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 81116) < nref_error, \
mtz_object.n_reflections()
integrater2.set_integrater_prepare_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 81116) < nref_error, \
mtz_object.n_reflections()
def exercise_xds_indexer(dials_data, tmp_dir, nproc=None):
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
template = dials_data("insulin").join("insulin_1_###.img").strpath
from xia2.Modules.Indexer.XDSIndexerII import XDSIndexerII
indexer = XDSIndexerII()
indexer.set_working_directory(tmp_dir)
from dxtbx.model.experiment_list import ExperimentListTemplateImporter
importer = ExperimentListTemplateImporter([template])
experiments = importer.experiments
imageset = experiments.imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
indexer.index()
assert approx_equal(indexer.get_indexer_cell(),
(78.054, 78.054, 78.054, 90, 90, 90),
eps=1), indexer.get_indexer_cell()
experiment = indexer.get_indexer_experiment_list()[0]
sgi = experiment.crystal.get_space_group().info()
assert sgi.type().number() == 197
beam_centre = indexer.get_indexer_beam_centre()
assert approx_equal(beam_centre, (94.4239, 94.5110), eps=1e-1)
assert indexer.get_indexer_images() == [(1, 45)]
print(indexer.get_indexer_experiment_list()[0].crystal)
print(indexer.get_indexer_experiment_list()[0].detector)
# test serialization of indexer
json_str = indexer.as_json()
print(json_str)
indexer2 = XDSIndexerII.from_json(string=json_str)
indexer2.index()
assert approx_equal(indexer.get_indexer_cell(),
indexer2.get_indexer_cell())
assert approx_equal(indexer.get_indexer_beam_centre(),
indexer2.get_indexer_beam_centre())
assert approx_equal(indexer.get_indexer_images(),
indexer2.get_indexer_images())
indexer.eliminate()
indexer2.eliminate()
assert approx_equal(indexer.get_indexer_cell(),
indexer2.get_indexer_cell())
assert indexer.get_indexer_lattice() == "hR"
assert indexer2.get_indexer_lattice() == "hR"
def xia2_main(stop_after=None):
'''Actually process something...'''
Citations.cite('xia2')
# print versions of related software
Chatter.write(dials_version())
ccp4_version = get_ccp4_version()
if ccp4_version is not None:
Chatter.write('CCP4 %s' % ccp4_version)
start_time = time.time()
CommandLine = get_command_line()
start_dir = Flags.get_starting_directory()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
no_images = True
for name in xtals.keys():
xtal = xtals[name]
if not xtal.get_all_image_names():
Chatter.write('-----------------------------------' + \
'-' * len(name))
Chatter.write('| No images assigned for crystal %s |' % name)
Chatter.write('-----------------------------------' + '-' \
* len(name))
else:
no_images = False
args = []
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
from libtbx import group_args
xinfo = CommandLine.get_xinfo()
if os.path.exists('xia2.json'):
from xia2.Schema.XProject import XProject
xinfo_new = xinfo
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
crystals_new = xinfo_new.get_crystals()
for crystal_id in crystals_new.keys():
if crystal_id not in crystals:
crystals[crystal_id] = crystals_new[crystal_id]
continue
crystals[crystal_id]._scaler = None # reset scaler
for wavelength_id in crystals_new[crystal_id].get_wavelength_names(
):
wavelength_new = crystals_new[crystal_id].get_xwavelength(
wavelength_id)
if wavelength_id not in crystals[
crystal_id].get_wavelength_names():
crystals[crystal_id].add_wavelength(
crystals_new[crystal_id].get_xwavelength(
wavelength_new))
continue
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps_new = wavelength_new.get_sweeps()
sweeps = wavelength.get_sweeps()
sweep_names = [s.get_name() for s in sweeps]
sweep_keys = [(s.get_directory(), s.get_template(),
s.get_image_range()) for s in sweeps]
for sweep in sweeps_new:
if ((sweep.get_directory(), sweep.get_template(),
sweep.get_image_range()) not in sweep_keys):
if sweep.get_name() in sweep_names:
i = 1
while 'SWEEEP%i' % i in sweep_names:
i += 1
sweep._name = 'SWEEP%i' % i
break
wavelength.add_sweep(
name=sweep.get_name(),
sample=sweep.get_xsample(),
directory=sweep.get_directory(),
image=sweep.get_image(),
beam=sweep.get_beam_centre(),
reversephi=sweep.get_reversephi(),
distance=sweep.get_distance(),
gain=sweep.get_gain(),
dmin=sweep.get_resolution_high(),
dmax=sweep.get_resolution_low(),
polarization=sweep.get_polarization(),
frames_to_process=sweep.get_frames_to_process(),
user_lattice=sweep.get_user_lattice(),
user_cell=sweep.get_user_cell(),
epoch=sweep._epoch,
ice=sweep._ice,
excluded_regions=sweep._excluded_regions,
)
sweep_names.append(sweep.get_name())
crystals = xinfo.get_crystals()
failover = params.xia2.settings.failover
if mp_params.mode == 'parallel' and njob > 1:
driver_type = mp_params.type
command_line_args = CommandLine.get_argv()[1:]
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
sweep._get_indexer()
sweep._get_refiner()
sweep._get_integrater()
args.append((group_args(
driver_type=driver_type,
stop_after=stop_after,
failover=failover,
command_line_args=command_line_args,
nproc=mp_params.nproc,
crystal_id=crystal_id,
wavelength_id=wavelength_id,
sweep_id=sweep.get_name(),
), ))
from xia2.Driver.DriverFactory import DriverFactory
default_driver_type = DriverFactory.get_driver_type()
# run every nth job on the current computer (no need to submit to qsub)
for i_job, arg in enumerate(args):
if (i_job % njob) == 0:
arg[0].driver_type = default_driver_type
if mp_params.type == "qsub":
method = "sge"
else:
method = "multiprocessing"
nproc = mp_params.nproc
qsub_command = mp_params.qsub_command
if not qsub_command:
qsub_command = 'qsub'
qsub_command = '%s -V -cwd -pe smp %d' % (qsub_command, nproc)
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_sweep,
args,
processes=njob,
#method=method,
method="multiprocessing",
qsub_command=qsub_command,
preserve_order=True,
preserve_exception_message=True)
# Hack to update sweep with the serialized indexers/refiners/integraters
i_sweep = 0
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
success, output, xsweep_dict = results[i_sweep]
if output is not None:
Chatter.write(output)
if not success:
Chatter.write('Sweep failed: removing %s' %
sweep.get_name())
remove_sweeps.append(sweep)
else:
assert xsweep_dict is not None
Chatter.write('Loading sweep: %s' % sweep.get_name())
from xia2.Schema.XSweep import XSweep
new_sweep = XSweep.from_dict(xsweep_dict)
sweep._indexer = new_sweep._indexer
sweep._refiner = new_sweep._refiner
sweep._integrater = new_sweep._integrater
i_sweep += 1
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
else:
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
from dials.command_line.show import show_datablocks
from dxtbx.datablock import DataBlock
Debug.write(sweep.get_name())
Debug.write(
show_datablocks([DataBlock([sweep.get_imageset()])]))
try:
if stop_after == 'index':
sweep.get_indexer_cell()
else:
sweep.get_integrater_intensities()
sweep.serialize()
except Exception as e:
if failover:
Chatter.write('Processing sweep %s failed: %s' % \
(sweep.get_name(), str(e)))
remove_sweeps.append(sweep)
else:
raise
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
# save intermediate xia2.json file in case scaling step fails
xinfo.as_json(filename='xia2.json')
if stop_after not in ('index', 'integrate'):
Chatter.write(xinfo.get_output(), strip=False)
for crystal in crystals.values():
crystal.serialize()
# save final xia2.json file in case report generation fails
xinfo.as_json(filename='xia2.json')
duration = time.time() - start_time
# write out the time taken in a human readable way
Chatter.write('Processing took %s' % \
time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))
if stop_after not in ('index', 'integrate'):
# and the summary file
with open('xia2-summary.dat', 'w') as fh:
for record in xinfo.summarise():
fh.write('%s\n' % record)
# looks like this import overwrites the initial command line
# Phil overrides so... for https://github.com/xia2/xia2/issues/150
from xia2.command_line.html import generate_xia2_html
if params.xia2.settings.small_molecule == True:
params.xia2.settings.report.xtriage_analysis = False
params.xia2.settings.report.include_radiation_damage = False
generate_xia2_html(xinfo,
filename='xia2.html',
params=params.xia2.settings.report)
write_citations()
# delete all of the temporary mtz files...
cleanup()
Environment.cleanup()
def exercise_xds_integrater(dials_data, tmp_dir, nproc=None):
if nproc:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
template = dials_data("insulin").join("insulin_1_###.img").strpath
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater
indexer = XDSIndexer()
indexer.set_working_directory(tmp_dir)
from dxtbx.model.experiment_list import ExperimentListTemplateImporter
importer = ExperimentListTemplateImporter([template])
experiments = importer.experiments
imageset = experiments.imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep("SWEEP1", wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.XDSRefiner import XDSRefiner
refiner = XDSRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
# refiner.refine()
integrater = XDSIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
integrater.set_integrater_sweep(sweep)
integrater.integrate()
from iotbx.reflection_file_reader import any_reflection_file
integrater_intensities = integrater.get_integrater_intensities()
assert os.path.exists(integrater_intensities)
reader = any_reflection_file(integrater_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=400)
assert mtz_object.column_labels() == [
"H",
"K",
"L",
"M_ISYM",
"BATCH",
"I",
"SIGI",
"FRACTIONCALC",
"XDET",
"YDET",
"ROT",
"LP",
"FLAG",
]
corrected_intensities = integrater.get_integrater_corrected_intensities()
assert os.path.exists(corrected_intensities)
reader = any_reflection_file(corrected_intensities)
assert reader.file_type() == "xds_ascii"
ma = reader.as_miller_arrays(merge_equivalents=False)[0]
assert approx_equal(ma.size(), 50000, eps=400)
assert integrater.get_integrater_wedge() == (1, 45)
assert approx_equal(
integrater.get_integrater_cell(), [78.066, 78.066, 78.066, 90, 90, 90], eps=1
)
assert approx_equal(
integrater.get_integrater_mosaic_min_mean_max(), (0.180, 0.180, 0.180), eps=1e-1
)
# test serialization of integrater
json_str = integrater.as_json()
# print(json_str)
integrater2 = XDSIntegrater.from_json(string=json_str)
integrater2.set_integrater_sweep(sweep, reset=False)
integrater2_intensities = integrater.get_integrater_intensities()
assert integrater2_intensities == integrater_intensities
integrater2.set_integrater_finish_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=400)
integrater2.set_integrater_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=450)
integrater2.set_integrater_prepare_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50100, eps=400)
def exercise_serialization(dials_regression, tmp_dir):
xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
os.chdir(tmp_dir)
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
from xia2.Modules.Refiner.DialsRefiner import DialsRefiner
from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater
from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
from xia2.Schema.XProject import XProject
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
proj = XProject()
proj._name = "PROJ1"
cryst = XCrystal("CRYST1", proj)
proj.add_crystal(cryst)
wav = XWavelength("WAVE1", cryst, wavelength=0.98)
samp = XSample("X1", cryst)
cryst.add_wavelength(wav)
cryst.set_ha_info({'atom': 'S'})
directory, image = os.path.split(imageset.get_path(1))
wav.add_sweep(name='SWEEP1', sample=samp, directory=directory, image=image)
import json
from dxtbx.serialize.load import _decode_dict
sweep = wav.get_sweeps()[0]
indexer = DialsIndexer()
indexer.set_working_directory(tmp_dir)
indexer.add_indexer_imageset(imageset)
indexer.set_indexer_sweep(sweep)
sweep._indexer = indexer
refiner = DialsRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
sweep._refiner = refiner
integrater = DialsIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
#integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.set_integrater_epoch(sweep.get_epoch(1))
integrater.set_integrater_sweep_name(sweep.get_name())
integrater.set_integrater_project_info(cryst.get_name(), wav.get_name(),
sweep.get_name())
sweep._integrater = integrater
scaler = CCP4ScalerA()
scaler.add_scaler_integrater(integrater)
scaler.set_scaler_xcrystal(cryst)
scaler.set_scaler_project_info(cryst.get_name(), wav.get_name())
scaler._scalr_xcrystal = cryst
cryst._scaler = scaler
s_dict = sweep.to_dict()
s_str = json.dumps(s_dict, ensure_ascii=True)
s_dict = json.loads(s_str, object_hook=_decode_dict)
xsweep = XSweep.from_dict(s_dict)
w_dict = wav.to_dict()
w_str = json.dumps(w_dict, ensure_ascii=True)
w_dict = json.loads(w_str, object_hook=_decode_dict)
xwav = XWavelength.from_dict(w_dict)
assert xwav.get_sweeps()[0].get_wavelength() is xwav
c_dict = cryst.to_dict()
c_str = json.dumps(c_dict, ensure_ascii=True)
c_dict = json.loads(c_str, object_hook=_decode_dict)
xcryst = XCrystal.from_dict(c_dict)
assert xcryst.get_xwavelength(
xcryst.get_wavelength_names()[0]).get_crystal() is xcryst
p_dict = proj.to_dict()
p_str = json.dumps(p_dict, ensure_ascii=True)
p_dict = json.loads(p_str, object_hook=_decode_dict)
xproj = XProject.from_dict(p_dict)
assert xproj.get_crystals().values()[0].get_project() is xproj
json_str = proj.as_json()
xproj = XProject.from_json(string=json_str)
assert xproj.get_crystals().values()[0].get_project() is xproj
print(xproj.get_output())
print("\n".join(xproj.summarise()))
json_str = xproj.as_json()
xproj = XProject.from_json(string=json_str)
assert xproj.get_crystals().values()[0].get_project() is xproj
xcryst = xproj.get_crystals().values()[0]
intgr = xcryst._get_integraters()[0]
assert intgr.get_integrater_finish_done()
assert xcryst._get_scaler()._sweep_handler.get_sweep_information(
intgr.get_integrater_epoch()).get_integrater() is intgr
print(xproj.get_output())
print("\n".join(xproj.summarise()))
def exercise_dials_integrater(dials_regression, tmp_dir, nproc=None):
if nproc:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater
from dxtbx.datablock import DataBlockTemplateImporter
indexer = DialsIndexer()
indexer.set_working_directory(tmp_dir)
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.DialsRefiner import DialsRefiner
refiner = DialsRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
#refiner.refine()
integrater = DialsIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
#integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.integrate()
integrater_intensities = integrater.get_integrater_intensities()
assert os.path.exists(integrater_intensities)
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(integrater_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
expected_reflections = 48456
assert abs(mtz_object.n_reflections() - expected_reflections) < 300, mtz_object.n_reflections()
assert mtz_object.column_labels() == [
'H', 'K', 'L', 'M_ISYM', 'BATCH', 'IPR', 'SIGIPR', 'I', 'SIGI',
'BG', 'SIGBG', 'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'LP', 'DQE']
assert integrater.get_integrater_wedge() == (1, 45)
assert approx_equal(integrater.get_integrater_cell(),
(78.14, 78.14, 78.14, 90, 90, 90), eps=1e-1)
# test serialization of integrater
json_str = integrater.as_json()
#print(json_str)
integrater2 = DialsIntegrater.from_json(string=json_str)
integrater2.set_integrater_sweep(sweep, reset=False)
integrater2_intensities = integrater.get_integrater_intensities()
assert integrater2_intensities == integrater_intensities
integrater2.set_integrater_finish_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - expected_reflections) < 300, mtz_object.n_reflections()
integrater2.set_integrater_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - expected_reflections) < 300, mtz_object.n_reflections()
integrater2.set_integrater_prepare_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - expected_reflections) < 300, mtz_object.n_reflections()
