def __init__(self, templates, **kwargs):
importer = DataBlockTemplateImporter(templates, **kwargs)
self.experiments = ExperimentList()
for db in importer.datablocks:
self.experiments.extend(
ExperimentListFactory.from_datablock_and_crystal(db, None)
)
def test_mosflm_indexer(ccp4, xia2_regression_build, tmpdir):
template = os.path.join(xia2_regression_build, "test_data", "insulin",
"insulin_1_###.img")
tmpdir.chdir()
from xia2.Modules.Indexer.MosflmIndexer import MosflmIndexer
indexer = MosflmIndexer()
indexer.set_working_directory(tmpdir.strpath)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
with mock.patch.object(
sys, 'argv',
[]): # otherwise indexing fails when running pytest with '--runslow'
indexer.index()
assert indexer.get_indexer_cell() == pytest.approx(
(78.6657, 78.6657, 78.6657, 90.0, 90.0, 90.0), abs=1e-3)
experiment = indexer.get_indexer_experiment_list()[0]
sgi = experiment.crystal.get_space_group().info()
assert sgi.type().number() == 197
beam_centre = indexer.get_indexer_beam_centre()
assert beam_centre == pytest.approx((94.34, 94.57), abs=1e-2)
assert indexer.get_indexer_images() == [(1, 1), (22, 22), (45, 45)]
print(indexer.get_indexer_experiment_list()[0].crystal)
print(indexer.get_indexer_experiment_list()[0].detector)
# test serialization of indexer
json_str = indexer.as_json()
print(json_str)
indexer2 = MosflmIndexer.from_json(string=json_str)
indexer2.index()
assert indexer.get_indexer_cell() == pytest.approx(
indexer2.get_indexer_cell(), abs=1e-6)
assert indexer.get_indexer_beam_centre() == pytest.approx(
indexer2.get_indexer_beam_centre(), abs=1e-6)
assert indexer2.get_indexer_images() == [[1, 1], [22, 22], [
45, 45
]] # coming from json these are now lists
indexer.eliminate()
indexer2.eliminate()
assert indexer.get_indexer_cell() == pytest.approx(
indexer2.get_indexer_cell(), abs=1e-6)
assert indexer.get_indexer_lattice() == 'hR'
assert indexer2.get_indexer_lattice() == 'hR'
def __call__(self):
'''
Import the datablocks
'''
from dxtbx.datablock import DataBlockTemplateImporter
from dxtbx.datablock import DataBlockFactory
from dials.util.options import flatten_datablocks
# Get the datablocks
datablocks = flatten_datablocks(self.params.input.datablock)
# Check we have some filenames
if len(datablocks) == 0:
# FIXME Should probably make this smarter since it requires editing here
# and in dials.import phil scope
try:
format_kwargs = {
'dynamic_shadowing': self.params.format.dynamic_shadowing,
'multi_panel': self.params.format.multi_panel,
}
except Exception:
format_kwargs = None
# Check if a template has been set and print help if not, otherwise try to
# import the images based on the template input
if len(self.params.input.template) > 0:
importer = DataBlockTemplateImporter(
self.params.input.template,
max(self.params.verbosity - 1, 0),
format_kwargs=format_kwargs)
datablocks = importer.datablocks
if len(datablocks) == 0:
raise Sorry('No datablocks found matching template %s' %
self.params.input.template)
elif len(self.params.input.directory) > 0:
datablocks = DataBlockFactory.from_filenames(
self.params.input.directory,
max(self.params.verbosity - 1, 0),
format_kwargs=format_kwargs)
if len(datablocks) == 0:
raise Sorry('No datablocks found in directories %s' %
self.params.input.directory)
else:
raise Sorry('No datablocks found')
# if len(datablocks) > 1:
# raise Sorry("More than 1 datablock found")
# Return the datablocks
return datablocks
def read_image_metadata_dxtbx(image):
"""Read the image header and send back the resulting metadata in a
dictionary. Read this using dxtbx - for a sequence of images use the
first image in the sequence to derive the metadata, for HDF5 files
just get on an read."""
check_file_readable(image)
if image.endswith(".h5"):
# XDS can literally only handle master files called (prefix)_master.h5
assert "master" in image
from dxtbx.datablock import DataBlockFactory
db = DataBlockFactory.from_filenames([image])[0]
else:
from dxtbx.datablock import DataBlockTemplateImporter
template, directory = image2template_directory(image)
full_template = os.path.join(directory, template)
importer = DataBlockTemplateImporter([full_template],
allow_incomplete_sweeps=True)
db = importer.datablocks[0]
if hasattr(db, "extract_sequences"):
db_extract = db.extract_sequences
else:
db_extract = db.extract_sweeps
sequences = db_extract()[0]
from dxtbx.serialize.xds import to_xds
XDS_INP = to_xds(sequences).XDS_INP()
params = XDS_INP_to_dict(XDS_INP)
# detector type specific parameters - minimum spot size, trusted region
# and so on.
# from dxtbx.model.detector_helpers import detector_helper_sensors
# d = sequences.get_detector()
# sensor_type = d[0].get_type()
# if sensor_type == detector_helper_sensors.SENSOR_PAD:
# params["MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT"] = "2"
# else:
# params["MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT"] = "4"
# remove things we will want to guarantee we set in fast_dp
for name in ["BACKGROUND_RANGE", "SPOT_RANGE", "JOB"]:
if name in params:
del params[name]
return params
def exercise_dials_multi_indexer(nproc=None):
template = "/Volumes/touro/data/i19/4sweep/56_Reza-H2-normalhem_100K_3.75mmAl/56-RezaH2-hem_%02i_#####.cbf"
cwd = os.path.abspath(os.curdir)
tmp_dir = os.path.abspath(open_tmp_directory())
os.chdir(tmp_dir)
cryst = None
wav = None
samp = None
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.trust_beam_centre = True
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
indexer = DialsIndexer()
indexer.set_working_directory(tmp_dir)
for i in range(4):
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template % (i + 1)])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
if cryst is None or wav is None:
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst,
imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.add_indexer_sweep(sweep)
indexer.index()
assert approx_equal(indexer.get_indexer_cell(),
(9.088, 12.415, 17.420, 90.000, 90.000, 90.000),
eps=1e-2)
def __call__(self):
'''
Import the datablocks
'''
from dxtbx.datablock import DataBlockTemplateImporter
from dxtbx.datablock import DataBlockFactory
from dials.util.options import flatten_datablocks
from libtbx.utils import Sorry
# Get the datablocks
datablocks = flatten_datablocks(self.params.input.datablock)
# Check we have some filenames
if len(datablocks) == 0:
format_kwargs = {
'dynamic_shadowing' : self.params.format.dynamic_shadowing
}
# Check if a template has been set and print help if not, otherwise try to
# import the images based on the template input
if len(self.params.input.template) > 0:
importer = DataBlockTemplateImporter(
self.params.input.template,
max(self.params.verbosity-1, 0),
format_kwargs=format_kwargs)
datablocks = importer.datablocks
if len(datablocks) == 0:
raise Sorry('No datablocks found matching template %s' % self.params.input.template)
elif len(self.params.input.directory) > 0:
datablocks = DataBlockFactory.from_filenames(
self.params.input.directory,
max(self.params.verbosity-1, 0),
format_kwargs=format_kwargs)
if len(datablocks) == 0:
raise Sorry('No datablocks found in directories %s' % self.params.input.directory)
else:
raise Sorry('No datablocks found')
if len(datablocks) > 1:
raise Sorry("More than 1 datablock found")
# Return the datablocks
return datablocks[0]
def test_labelit_indexer_II(dials_regression, tmpdir):
template = os.path.join(dials_regression, "xia2_demo_data",
"insulin_1_###.img")
tmpdir.chdir()
from xia2.Modules.Indexer.LabelitIndexerII import LabelitIndexerII
from xia2.DriverExceptions.NotAvailableError import NotAvailableError
try:
ls = LabelitIndexerII(indxr_print=True)
except NotAvailableError:
pytest.skip("labelit not found")
ls.set_working_directory(tmpdir.strpath)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
ls.add_indexer_imageset(imageset)
ls.set_indexer_input_cell((78, 78, 78, 90, 90, 90))
ls.set_indexer_user_input_lattice(True)
ls.set_indexer_input_lattice('cI')
ls.index()
assert ls.get_indexer_cell() == pytest.approx(
(78.52, 78.52, 78.52, 90, 90, 90), abs=1e-1)
solution = ls.get_solution()
assert solution['rmsd'] == pytest.approx(0.079, abs=1e-1)
assert solution['metric'] <= 0.1762
assert solution['number'] == 22
assert solution['lattice'] == 'cI'
assert solution['mosaic'] <= 0.2
assert solution['nspots'] == pytest.approx(5509, abs=50)
beam_centre = ls.get_indexer_beam_centre()
assert beam_centre == pytest.approx((94.3286, 94.4662), abs=5e-1)
assert ls.get_indexer_images() == [(1, 1), (3, 3), (5, 5), (7, 7), (9, 9),
(11, 11), (13, 13), (15, 15), (17, 17),
(19, 19), (21, 21), (23, 23), (25, 25),
(27, 27), (29, 29), (31, 31), (33, 33),
(35, 35), (37, 37), (39, 39)]
print(ls.get_indexer_experiment_list()[0].crystal)
print(ls.get_indexer_experiment_list()[0].detector)
def exercise_dials_indexer(xia2_regression_build, tmp_dir, nproc=None):
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
xia2_demo_data = os.path.join(xia2_regression_build, "test_data",
"insulin")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
indexer = DialsIndexer()
indexer.set_working_directory(tmp_dir)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
indexer.index()
assert approx_equal(indexer.get_indexer_cell(),
(78.14, 78.14, 78.14, 90, 90, 90),
eps=1e-1)
solution = indexer.get_solution()
assert approx_equal(solution['rmsd'], 0.041, eps=1e-2)
assert approx_equal(solution['metric'], 0.027, eps=1e-2)
assert solution['number'] == 22
assert solution['lattice'] == 'cI'
beam_centre = indexer.get_indexer_beam_centre()
assert approx_equal(beam_centre, (94.4223, 94.5097), eps=1e-2)
print(indexer.get_indexer_experiment_list()[0].crystal)
print(indexer.get_indexer_experiment_list()[0].detector)
# test serialization of indexer
json_str = indexer.as_json()
#print(json_str)
indexer2 = DialsIndexer.from_json(string=json_str)
indexer2.index()
assert approx_equal(indexer.get_indexer_cell(),
indexer2.get_indexer_cell())
assert approx_equal(indexer.get_indexer_beam_centre(),
indexer2.get_indexer_beam_centre())
indexer.eliminate()
indexer2.eliminate()
assert approx_equal(indexer.get_indexer_cell(),
indexer2.get_indexer_cell())
assert indexer.get_indexer_lattice() == 'hR'
assert indexer2.get_indexer_lattice() == 'hR'
def test_labelit_indexer(ccp4, xia2_regression_build, tmpdir):
template = os.path.join(xia2_regression_build, "test_data", "insulin",
"insulin_1_###.img")
tmpdir.chdir()
from xia2.Modules.Indexer.LabelitIndexer import LabelitIndexer
from xia2.DriverExceptions.NotAvailableError import NotAvailableError
try:
ls = LabelitIndexer(indxr_print=True)
except NotAvailableError:
pytest.skip("labelit not found")
ls.set_working_directory(tmpdir.strpath)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
ls.add_indexer_imageset(imageset)
ls.index()
assert ls.get_indexer_cell() == pytest.approx(
(78.58, 78.58, 78.58, 90, 90, 90), abs=0.5)
solution = ls.get_solution()
assert solution['rmsd'] <= 0.2
assert solution['metric'] <= 0.16
assert solution['number'] == 22
assert solution['lattice'] == 'cI'
assert solution['mosaic'] <= 0.25
assert solution['nspots'] == pytest.approx(860, abs=30)
beam_centre = ls.get_indexer_beam_centre()
assert beam_centre == pytest.approx((94.3416, 94.4994), abs=2e-1)
assert ls.get_indexer_images() == [(1, 1), (22, 22), (45, 45)]
print(ls.get_indexer_experiment_list()[0].crystal)
print(ls.get_indexer_experiment_list()[0].detector)
json_str = ls.as_json()
# print(json_str)
ls1 = LabelitIndexer.from_json(string=json_str)
ls1.index()
print(ls.get_indexer_experiment_list()[0].crystal)
assert ls.get_indexer_beam_centre() == ls1.get_indexer_beam_centre()
assert ls1.get_indexer_images() == [[1, 1], [22, 22],
[45,
45]] # in JSON tuples become lists
assert ls.get_distance() == ls1.get_distance()
ls.eliminate()
ls1.eliminate()
print(ls1.get_indexer_experiment_list()[0].crystal)
assert ls.get_indexer_beam_centre() == ls1.get_indexer_beam_centre()
assert ls.get_indexer_images() == [(1, 1), (22, 22), (45, 45)]
assert ls1.get_indexer_images() == [[1, 1], [22, 22],
[45,
45]] # in JSON tuples become lists
assert ls.get_distance() == ls1.get_distance()
print(ls1.get_indexer_cell())
print(ls1.get_solution())
assert ls.get_indexer_cell() == pytest.approx(
(111.11, 111.11, 68.08, 90.0, 90.0, 120.0), abs=5e-1)
solution = ls1.get_solution()
assert solution['rmsd'] >= 0.07, solution['rmsd']
assert solution['metric'] == pytest.approx(0.1291, abs=1e-1)
assert solution['lattice'] == 'hR', solution['lattice']
assert solution['mosaic'] <= 0.3, solution['mosaic']
assert solution['nspots'] == pytest.approx(856, abs=30)
def exercise_xds_indexer(xia2_regression_build, tmp_dir, nproc=None):
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
xia2_demo_data = os.path.join(xia2_regression_build, "test_data",
"insulin")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
indexer = XDSIndexer()
indexer.set_working_directory(tmp_dir)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
indexer.index()
assert approx_equal(indexer.get_indexer_cell(),
(78.076, 78.076, 78.076, 90, 90, 90),
eps=1), indexer.get_indexer_cell()
experiment = indexer.get_indexer_experiment_list()[0]
sgi = experiment.crystal.get_space_group().info()
assert sgi.type().number() == 197
beam_centre = indexer.get_indexer_beam_centre()
assert approx_equal(beam_centre, (94.4221, 94.5096), eps=1e-1)
assert indexer.get_indexer_images() == [(1, 5), (20, 24), (41, 45)]
print(indexer.get_indexer_experiment_list()[0].crystal)
print(indexer.get_indexer_experiment_list()[0].detector)
# test serialization of indexer
json_str = indexer.as_json()
print(json_str)
indexer2 = XDSIndexer.from_json(string=json_str)
indexer2.index()
assert approx_equal(indexer.get_indexer_cell(),
indexer2.get_indexer_cell())
assert approx_equal(indexer.get_indexer_beam_centre(),
indexer2.get_indexer_beam_centre())
assert approx_equal(indexer.get_indexer_images(),
indexer2.get_indexer_images())
indexer.eliminate()
indexer2.eliminate()
assert approx_equal(indexer.get_indexer_cell(),
indexer2.get_indexer_cell())
assert indexer.get_indexer_lattice() == 'hR'
assert indexer2.get_indexer_lattice() == 'hR'
def exercise_mosflm_integrater(dials_regression, tmp_dir, nproc):
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
# otherwise if this test is running multiple times simultaneously two mosflm
# processes try to write to the same genfile
os.environ['CCP4_SCR'] = tmp_dir
from xia2.Modules.Indexer.MosflmIndexer import MosflmIndexer
from xia2.Modules.Integrater.MosflmIntegrater import MosflmIntegrater
from dxtbx.datablock import DataBlockTemplateImporter
indexer = MosflmIndexer()
indexer.set_working_directory(tmp_dir)
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.MosflmRefiner import MosflmRefiner
refiner = MosflmRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
nref_error = 500
integrater = MosflmIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
#integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.integrate()
integrater_intensities = integrater.get_integrater_intensities()
assert os.path.exists(integrater_intensities)
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(integrater_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 81116) < nref_error, \
mtz_object.n_reflections()
assert mtz_object.column_labels() == [
'H', 'K', 'L', 'M_ISYM', 'BATCH', 'I', 'SIGI', 'IPR', 'SIGIPR',
'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'WIDTH', 'LP', 'MPART', 'FLAG',
'BGPKRATIOS']
assert integrater.get_integrater_wedge() == (1, 45)
assert approx_equal(integrater.get_integrater_cell(),
(78.014, 78.014, 78.014, 90.0, 90.0, 90.0), eps=1e-2)
# test serialization of integrater
json_str = integrater.as_json()
#print json_str
integrater2 = MosflmIntegrater.from_json(string=json_str)
integrater2.set_integrater_sweep(sweep, reset=False)
integrater2_intensities = integrater.get_integrater_intensities()
assert integrater2_intensities == integrater_intensities
integrater2.set_integrater_finish_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 81116) < nref_error, \
mtz_object.n_reflections()
integrater2.set_integrater_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 81116) < nref_error, \
mtz_object.n_reflections()
integrater2.set_integrater_prepare_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 81116) < nref_error, \
mtz_object.n_reflections()
def exercise_dials_integrater(dials_regression, tmp_dir, nproc=None):
if nproc:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater
from dxtbx.datablock import DataBlockTemplateImporter
indexer = DialsIndexer()
indexer.set_working_directory(tmp_dir)
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.DialsRefiner import DialsRefiner
refiner = DialsRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
#refiner.refine()
integrater = DialsIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
#integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.integrate()
integrater_intensities = integrater.get_integrater_intensities()
assert os.path.exists(integrater_intensities)
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(integrater_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
expected_reflections = 48456
assert abs(mtz_object.n_reflections() - expected_reflections) < 300, mtz_object.n_reflections()
assert mtz_object.column_labels() == [
'H', 'K', 'L', 'M_ISYM', 'BATCH', 'IPR', 'SIGIPR', 'I', 'SIGI',
'BG', 'SIGBG', 'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'LP', 'DQE']
assert integrater.get_integrater_wedge() == (1, 45)
assert approx_equal(integrater.get_integrater_cell(),
(78.14, 78.14, 78.14, 90, 90, 90), eps=1e-1)
# test serialization of integrater
json_str = integrater.as_json()
#print(json_str)
integrater2 = DialsIntegrater.from_json(string=json_str)
integrater2.set_integrater_sweep(sweep, reset=False)
integrater2_intensities = integrater.get_integrater_intensities()
assert integrater2_intensities == integrater_intensities
integrater2.set_integrater_finish_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - expected_reflections) < 300, mtz_object.n_reflections()
integrater2.set_integrater_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - expected_reflections) < 300, mtz_object.n_reflections()
integrater2.set_integrater_prepare_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - expected_reflections) < 300, mtz_object.n_reflections()
def run(self):
'''Execute the script.'''
from dxtbx.datablock import DataBlockTemplateImporter
from dials.util.options import flatten_datablocks
from dials.util import log
from logging import info
from time import time
from libtbx.utils import Abort
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
datablocks = flatten_datablocks(params.input.datablock)
# Check we have some filenames
if len(datablocks) == 0:
# Check if a template has been set and print help if not, otherwise try to
# import the images based on the template input
if len(params.input.template) == 0:
self.parser.print_help()
exit(0)
else:
importer = DataBlockTemplateImporter(params.input.template,
options.verbose)
datablocks = importer.datablocks
# Save the options
self.options = options
self.params = params
self.load_reference_geometry()
st = time()
# Import stuff
if len(datablocks) == 0:
raise Abort('No datablocks specified')
elif len(datablocks) > 1:
raise Abort('Only 1 datablock can be processed at a time.')
datablock = datablocks[0]
if self.reference_detector is not None:
for imageset in datablock.extract_imagesets():
imageset.set_detector(self.reference_detector)
# Configure logging
log.config(params.verbosity,
info='dials.process.log',
debug='dials.process.debug.log')
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
info('The following parameters have been modified:\n')
info(diff_phil)
if self.params.output.datablock_filename:
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(datablock)
dump.as_json(self.params.output.datablock_filename)
# Do the processing
observed = self.find_spots(datablock)
experiments, indexed = self.index(datablock, observed)
experiments, indexed = self.refine(experiments, indexed)
integrated = self.integrate(experiments, indexed)
# Total Time
info("")
info("Total Time Taken = %f seconds" % (time() - st))
def exercise_xds_integrater(xia2_regression_build, tmp_dir, nproc=None):
if nproc:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
xia2_demo_data = os.path.join(xia2_regression_build, "test_data",
"insulin")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater
indexer = XDSIndexer()
indexer.set_working_directory(tmp_dir)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, indexer.get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.XDSRefiner import XDSRefiner
refiner = XDSRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
#refiner.refine()
integrater = XDSIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
integrater.set_integrater_sweep(sweep)
integrater.integrate()
from iotbx.reflection_file_reader import any_reflection_file
integrater_intensities = integrater.get_integrater_intensities()
assert os.path.exists(integrater_intensities)
reader = any_reflection_file(integrater_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=400)
assert mtz_object.column_labels() == [
'H', 'K', 'L', 'M_ISYM', 'BATCH', 'I', 'SIGI', 'FRACTIONCALC', 'XDET',
'YDET', 'ROT', 'LP', 'FLAG'
]
corrected_intensities = integrater.get_integrater_corrected_intensities()
assert os.path.exists(corrected_intensities)
reader = any_reflection_file(corrected_intensities)
assert reader.file_type() == "xds_ascii"
ma = reader.as_miller_arrays(merge_equivalents=False)[0]
assert approx_equal(ma.size(), 50000, eps=400)
assert integrater.get_integrater_wedge() == (1, 45)
assert approx_equal(integrater.get_integrater_cell(),
[78.066, 78.066, 78.066, 90, 90, 90],
eps=1)
assert approx_equal(integrater.get_integrater_mosaic_min_mean_max(),
(0.180, 0.180, 0.180),
eps=1e-1)
# test serialization of integrater
json_str = integrater.as_json()
#print(json_str)
integrater2 = XDSIntegrater.from_json(string=json_str)
integrater2.set_integrater_sweep(sweep, reset=False)
integrater2_intensities = integrater.get_integrater_intensities()
assert integrater2_intensities == integrater_intensities
integrater2.set_integrater_finish_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=400)
integrater2.set_integrater_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50000, eps=450)
integrater2.set_integrater_prepare_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert approx_equal(mtz_object.n_reflections(), 50100, eps=400)
def test_xds_scalerA(ccp4, xia2_regression_build, tmpdir, nproc):
if nproc is not None:
from xia2.Handlers.Phil import PhilIndex
PhilIndex.params.xia2.settings.multiprocessing.nproc = nproc
template = os.path.join(xia2_regression_build, "test_data", "insulin",
"insulin_1_###.img")
tmpdir.chdir()
tmpdir = tmpdir.strpath
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
from xia2.Modules.Integrater.XDSIntegrater import XDSIntegrater
from xia2.Modules.Scaler.XDSScalerA import XDSScalerA
indexer = XDSIndexer()
indexer.set_working_directory(tmpdir)
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
with mock.patch.object(sys, 'argv', []):
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.XDSRefiner import XDSRefiner
refiner = XDSRefiner()
refiner.set_working_directory(tmpdir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
integrater = XDSIntegrater()
integrater.set_working_directory(tmpdir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
#integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.set_integrater_sweep_name('SWEEP1')
integrater.set_integrater_project_info('CRYST1', 'WAVE1', 'SWEEP1')
scaler = XDSScalerA()
scaler.add_scaler_integrater(integrater)
scaler.set_scaler_xcrystal(cryst)
scaler.set_scaler_project_info('CRYST1', 'WAVE1')
check_scaler_files_exist(scaler)
# test serialization of scaler
json_str = scaler.as_json()
#print json_str
scaler2 = XDSScalerA.from_json(string=json_str)
scaler2.set_scaler_xcrystal(cryst)
check_scaler_files_exist(scaler2)
scaler2.set_scaler_finish_done(False)
check_scaler_files_exist(scaler2)
scaler2.set_scaler_done(False)
check_scaler_files_exist(scaler2)
scaler2._scalr_integraters = {} # XXX
scaler2.add_scaler_integrater(integrater)
scaler2.set_scaler_prepare_done(False)
check_scaler_files_exist(scaler2)
def exercise_serialization(dials_regression, tmp_dir):
xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
os.chdir(tmp_dir)
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
from xia2.Modules.Refiner.DialsRefiner import DialsRefiner
from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater
from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
from xia2.Schema.XProject import XProject
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
proj = XProject()
proj._name = "PROJ1"
cryst = XCrystal("CRYST1", proj)
proj.add_crystal(cryst)
wav = XWavelength("WAVE1", cryst, wavelength=0.98)
samp = XSample("X1", cryst)
cryst.add_wavelength(wav)
cryst.set_ha_info({'atom': 'S'})
directory, image = os.path.split(imageset.get_path(1))
wav.add_sweep(name='SWEEP1', sample=samp, directory=directory, image=image)
import json
from dxtbx.serialize.load import _decode_dict
sweep = wav.get_sweeps()[0]
indexer = DialsIndexer()
indexer.set_working_directory(tmp_dir)
indexer.add_indexer_imageset(imageset)
indexer.set_indexer_sweep(sweep)
sweep._indexer = indexer
refiner = DialsRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
sweep._refiner = refiner
integrater = DialsIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
#integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.set_integrater_epoch(sweep.get_epoch(1))
integrater.set_integrater_sweep_name(sweep.get_name())
integrater.set_integrater_project_info(cryst.get_name(), wav.get_name(),
sweep.get_name())
sweep._integrater = integrater
scaler = CCP4ScalerA()
scaler.add_scaler_integrater(integrater)
scaler.set_scaler_xcrystal(cryst)
scaler.set_scaler_project_info(cryst.get_name(), wav.get_name())
scaler._scalr_xcrystal = cryst
cryst._scaler = scaler
s_dict = sweep.to_dict()
s_str = json.dumps(s_dict, ensure_ascii=True)
s_dict = json.loads(s_str, object_hook=_decode_dict)
xsweep = XSweep.from_dict(s_dict)
w_dict = wav.to_dict()
w_str = json.dumps(w_dict, ensure_ascii=True)
w_dict = json.loads(w_str, object_hook=_decode_dict)
xwav = XWavelength.from_dict(w_dict)
assert xwav.get_sweeps()[0].get_wavelength() is xwav
c_dict = cryst.to_dict()
c_str = json.dumps(c_dict, ensure_ascii=True)
c_dict = json.loads(c_str, object_hook=_decode_dict)
xcryst = XCrystal.from_dict(c_dict)
assert xcryst.get_xwavelength(
xcryst.get_wavelength_names()[0]).get_crystal() is xcryst
p_dict = proj.to_dict()
p_str = json.dumps(p_dict, ensure_ascii=True)
p_dict = json.loads(p_str, object_hook=_decode_dict)
xproj = XProject.from_dict(p_dict)
assert xproj.get_crystals().values()[0].get_project() is xproj
json_str = proj.as_json()
xproj = XProject.from_json(string=json_str)
assert xproj.get_crystals().values()[0].get_project() is xproj
print(xproj.get_output())
print("\n".join(xproj.summarise()))
json_str = xproj.as_json()
xproj = XProject.from_json(string=json_str)
assert xproj.get_crystals().values()[0].get_project() is xproj
xcryst = xproj.get_crystals().values()[0]
intgr = xcryst._get_integraters()[0]
assert intgr.get_integrater_finish_done()
assert xcryst._get_scaler()._sweep_handler.get_sweep_information(
intgr.get_integrater_epoch()).get_integrater() is intgr
print(xproj.get_output())
print("\n".join(xproj.summarise()))
def load_imagesets(template,
directory,
id_image=None,
image_range=None,
use_cache=True,
reversephi=False):
global imageset_cache
from dxtbx.datablock import DataBlockFactory
from xia2.Applications.xia2setup import known_hdf5_extensions
full_template_path = os.path.join(directory, template)
if full_template_path not in imageset_cache or not use_cache:
from dxtbx.datablock import BeamComparison
from dxtbx.datablock import DetectorComparison
from dxtbx.datablock import GoniometerComparison
params = PhilIndex.params.xia2.settings
compare_beam = BeamComparison(
wavelength_tolerance=params.input.tolerance.beam.wavelength,
direction_tolerance=params.input.tolerance.beam.direction,
polarization_normal_tolerance=params.input.tolerance.beam.
polarization_normal,
polarization_fraction_tolerance=params.input.tolerance.beam.
polarization_fraction)
compare_detector = DetectorComparison(
fast_axis_tolerance=params.input.tolerance.detector.fast_axis,
slow_axis_tolerance=params.input.tolerance.detector.slow_axis,
origin_tolerance=params.input.tolerance.detector.origin)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=params.input.tolerance.goniometer.
rotation_axis,
fixed_rotation_tolerance=params.input.tolerance.goniometer.
fixed_rotation,
setting_rotation_tolerance=params.input.tolerance.goniometer.
setting_rotation)
scan_tolerance = params.input.tolerance.scan.oscillation
format_kwargs = {
'dynamic_shadowing': params.input.format.dynamic_shadowing,
'multi_panel': params.input.format.multi_panel,
}
if os.path.splitext(full_template_path)[-1] in known_hdf5_extensions:
import glob
g = glob.glob(os.path.join(directory, '*_master.h5'))
master_file = None
for p in g:
substr = longest_common_substring(template, p)
if substr:
if (master_file is None or (len(substr) > len(
longest_common_substring(template, master_file)))):
master_file = p
if master_file is None and \
os.path.exists(full_template_path) and \
os.path.isfile(full_template_path):
master_file = full_template_path
if master_file is None:
raise RuntimeError("Can't find master file for %s" %
full_template_path)
unhandled = []
datablocks = DataBlockFactory.from_filenames(
[master_file],
verbose=False,
unhandled=unhandled,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs)
assert len(unhandled) == 0, "unhandled image files identified: %s" % \
unhandled
assert len(datablocks) == 1, "1 datablock expected, %d found" % \
len(datablocks)
else:
from dxtbx.sweep_filenames import locate_files_matching_template_string
params = PhilIndex.get_python_object()
read_all_image_headers = params.xia2.settings.read_all_image_headers
if read_all_image_headers:
paths = sorted(
locate_files_matching_template_string(full_template_path))
unhandled = []
datablocks = DataBlockFactory.from_filenames(
paths,
verbose=False,
unhandled=unhandled,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs)
assert len(unhandled) == 0, "unhandled image files identified: %s" % \
unhandled
assert len(datablocks) == 1, "1 datablock expected, %d found" % \
len(datablocks)
else:
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter(
[full_template_path], format_kwargs=format_kwargs)
datablocks = importer.datablocks
imagesets = datablocks[0].extract_sweeps()
assert len(imagesets) > 0, "no imageset found"
imageset_cache[full_template_path] = collections.OrderedDict()
if reversephi:
for imageset in imagesets:
goniometer = imageset.get_goniometer()
goniometer.set_rotation_axis(
tuple((-g for g in goniometer.get_rotation_axis())))
reference_geometry = PhilIndex.params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
update_with_reference_geometry(imagesets, reference_geometry)
# Update the geometry
params = PhilIndex.params.xia2.settings
update_geometry = []
from dials.command_line.dials_import import ManualGeometryUpdater
from dials.util.options import geometry_phil_scope
# Then add manual geometry
work_phil = geometry_phil_scope.format(params.input)
diff_phil = geometry_phil_scope.fetch_diff(source=work_phil)
if diff_phil.as_str() != "":
update_geometry.append(ManualGeometryUpdater(params.input))
imageset_list = []
for imageset in imagesets:
for updater in update_geometry:
imageset = updater(imageset)
imageset_list.append(imageset)
imagesets = imageset_list
from scitbx.array_family import flex
for imageset in imagesets:
scan = imageset.get_scan()
exposure_times = scan.get_exposure_times()
epochs = scan.get_epochs()
if exposure_times.all_eq(0) or exposure_times[0] == 0:
exposure_times = flex.double(exposure_times.size(), 1)
scan.set_exposure_times(exposure_times)
elif not exposure_times.all_gt(0):
exposure_times = flex.double(exposure_times.size(),
exposure_times[0])
scan.set_exposure_times(exposure_times)
if epochs.size() > 1 and not epochs.all_gt(0):
if epochs[0] == 0:
epochs[0] = 1
for i in range(1, epochs.size()):
epochs[i] = epochs[i - 1] + exposure_times[i - 1]
scan.set_epochs(epochs)
_id_image = scan.get_image_range()[0]
imageset_cache[full_template_path][_id_image] = imageset
if id_image is not None:
return [imageset_cache[full_template_path][id_image]]
elif image_range is not None:
for imageset in imageset_cache[full_template_path].values():
scan = imageset.get_scan()
scan_image_range = scan.get_image_range()
if (image_range[0] >= scan_image_range[0]
and image_range[1] <= scan_image_range[1]):
imagesets = [
imageset[image_range[0] -
scan_image_range[0]:image_range[1] + 1 -
scan_image_range[0]]
]
assert len(imagesets[0]) == image_range[1] - image_range[0] + 1, \
len(imagesets[0])
return imagesets
return imageset_cache[full_template_path].values()