def import_geometry(xds_inp=None, dials_json=None):
assert (xds_inp, dials_json).count(None) == 1
geom_kwds = set([
"DIRECTION_OF_DETECTOR_X-AXIS",
"DIRECTION_OF_DETECTOR_Y-AXIS",
"DETECTOR_DISTANCE",
"ORGX",
"ORGY",
"ROTATION_AXIS", # "X-RAY_WAVELENGTH",
"INCIDENT_BEAM_DIRECTION",
"SEGMENT",
"DIRECTION_OF_SEGMENT_X-AXIS",
"DIRECTION_OF_SEGMENT_Y-AXIS",
"SEGMENT_DISTANCE",
"SEGMENT_ORGX",
"SEGMENT_ORGY"
])
# FIXME in case of multi-segment detector..
if xds_inp:
inp = get_xdsinp_keyword(xds_inp)
inp = filter(lambda x: x[0] in geom_kwds, inp)
return map(lambda x: "%s= %s" % x, inp)
elif dials_json:
import dxtbx.imageset
from dxtbx.serialize.load import _decode_dict
from dxtbx.model import BeamFactory
from dxtbx.model import DetectorFactory
from dxtbx.model import GoniometerFactory
from dxtbx.model import ScanFactory
from dxtbx.serialize.xds import to_xds
j = json.loads(open(dials_json).read(), object_hook=_decode_dict)
# dummy
sweep = dxtbx.imageset.ImageSetFactory.from_template(
"####", image_range=[1, 1], check_format=False)[0]
sweep.set_detector(DetectorFactory.from_dict(j["detector"][0]))
sweep.set_beam(BeamFactory.from_dict(j["beam"][0]))
sweep.set_goniometer(GoniometerFactory.from_dict(j["goniometer"][0]))
sweep.set_scan(
ScanFactory.make_scan(image_range=[1, 1],
exposure_times=[1],
oscillation=[1, 2],
epochs=[0])) # dummy
sio = cStringIO.StringIO()
to_xds(sweep).XDS_INP(sio)
inp = get_xdsinp_keyword(inp_str=sio.getvalue())
inp = filter(lambda x: x[0] in geom_kwds, inp)
return map(lambda x: "%s= %s" % x, inp)
return []
def test_to_xds_multi_panel_i23(dials_regression, tmpdir):
tmpdir.chdir()
file_name = os.path.join(
dials_regression, "image_examples", "DLS_I23", "germ_13KeV_0001.cbf"
)
sweep = ImageSetFactory.new([file_name])[0]
to_xds = xds.to_xds(sweep)
s1 = to_xds.XDS_INP()
for expected_substr in (
"""\
!
! SEGMENT 1
!
SEGMENT= 1 2463 1 195
DIRECTION_OF_SEGMENT_X-AXIS= 1.00000 0.00000 0.00000
DIRECTION_OF_SEGMENT_Y-AXIS= 0.00000 -0.14347 0.98966
SEGMENT_DISTANCE= 250.000
SEGMENT_ORGX= 1075.00 SEGMENT_ORGY= 97.67""",
"""\
!
! SEGMENT 24
!
SEGMENT= 1 2463 4877 5071
DIRECTION_OF_SEGMENT_X-AXIS= 1.00000 0.00000 0.00000
DIRECTION_OF_SEGMENT_Y-AXIS= 0.00000 -0.06390 -0.99796
SEGMENT_DISTANCE= 250.000
SEGMENT_ORGX= 1075.00 SEGMENT_ORGY= 4973.67""",
):
assert expected_substr in s1
def dump(experiments, reflections, directory):
'''
Dump the files in XDS format
'''
import os
from dxtbx.serialize import xds
from scitbx import matrix
if len(experiments) > 0:
for i, experiment in enumerate(experiments):
suffix = ""
if len(experiments) > 1:
suffix = "_%i" %(i+1)
sub_dir = "%s%s" % (directory, suffix)
if not os.path.isdir(sub_dir):
os.makedirs(sub_dir)
# XXX imageset is getting the experimental geometry from the image files
# rather than the input experiments.json file
imageset = experiment.imageset
imageset.set_detector(experiment.detector)
imageset.set_beam(experiment.beam)
imageset.set_goniometer(experiment.goniometer)
imageset.set_scan(experiment.scan)
crystal_model = experiment.crystal
crystal_model = crystal_model.change_basis(
crystal_model.get_space_group().info()\
.change_of_basis_op_to_reference_setting())
A = matrix.sqr(crystal_model.get_A())
A_inv = A.inverse()
real_space_a = A_inv.elems[:3]
real_space_b = A_inv.elems[3:6]
real_space_c = A_inv.elems[6:9]
to_xds = xds.to_xds(imageset)
xds_inp = os.path.join(sub_dir, 'XDS.INP')
xparm_xds = os.path.join(sub_dir, 'XPARM.XDS')
print "Exporting experiment to %s and %s" %(xds_inp, xparm_xds)
with open(xds_inp, 'wb') as f:
to_xds.XDS_INP(
out=f,
space_group_number=crystal_model.get_space_group().type().number(),
real_space_a=real_space_a,
real_space_b=real_space_b,
real_space_c=real_space_c,
job_card="XYCORR INIT DEFPIX INTEGRATE CORRECT")
with open(xparm_xds, 'wb') as f:
to_xds.xparm_xds(
real_space_a, real_space_b, real_space_c,
crystal_model.get_space_group().type().number(),
out=f)
if reflections is not None and len(reflections) > 0:
ref_cryst = reflections.select(reflections['id'] == i)
export_spot_xds(ref_cryst, os.path.join(sub_dir, 'SPOT.XDS'))
else:
if not os.path.isdir(directory):
os.makedirs(directory)
export_spot_xds(reflections, os.path.join(directory, 'SPOT.XDS'))
def test_ii(self):
idxref = self.Idxref()
self._index_remove_masked_regions()
for file in ["SPOT.XDS"]:
idxref.set_input_data_file(file, self._indxr_payload[file])
idxref.set_data_range(self._indxr_images[0][0],
self._indxr_images[0][1])
idxref.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
for block in self._indxr_images[:1]:
starting_frame = block[0]
starting_angle = self.get_scan().get_angle_from_image_index(
starting_frame)
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
idxref.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1])
idxref.run()
return idxref.get_fraction_rmsd_rmsphi()
def test_vmxi_thaumatin(dials_data):
master_h5 = dials_data("vmxi_thaumatin") / "image_15799_master.h5"
expts = ExperimentListFactory.from_filenames([master_h5.strpath])
to_xds = xds.to_xds(expts[0].imageset)
s = to_xds.XDS_INP()
assert "DETECTOR=EIGER" in s
assert "SENSOR_THICKNESS= 0.450" in s
def test_ii(self):
idxref = self.Idxref()
self._index_remove_masked_regions()
for file in ['SPOT.XDS']:
idxref.set_input_data_file(file, self._indxr_payload[file])
idxref.set_data_range(self._indxr_images[0][0],
self._indxr_images[0][1])
idxref.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
for block in self._indxr_images[:1]:
starting_frame = block[0]
starting_angle = self.get_scan().get_angle_from_image_index(starting_frame)
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
#mosflm_beam_centre = self.get_beam_centre()
#xds_beam_centre = beam_centre_mosflm_to_xds(
#mosflm_beam_centre[0], mosflm_beam_centre[1], self.get_header())
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
idxref.set_beam_centre(xds_beam_centre[0],
xds_beam_centre[1])
idxref.run()
return idxref.get_fraction_rmsd_rmsphi()
def dump(experiments, reflections, directory):
'''
Dump the files in XDS format
'''
import os
from dxtbx.serialize import xds
if len(experiments) > 0:
for i in range(len(experiments)):
suffix = ""
if len(experiments) > 1:
suffix = "_%i" %(i+1)
sub_dir = "%s%s" % (directory, suffix)
if not os.path.isdir(sub_dir):
os.makedirs(sub_dir)
# XXX imageset is getting the experimental geometry from the image files
# rather than the input experiments.json file
imageset = experiments[i].imageset
imageset.set_detector(experiments[i].detector)
imageset.set_beam(experiments[i].beam)
imageset.set_goniometer(experiments[i].goniometer)
imageset.set_scan(experiments[i].scan)
crystal_model = experiments[i].crystal
crystal_model = crystal_model.change_basis(
crystal_model.get_space_group().info()\
.change_of_basis_op_to_reference_setting())
A = crystal_model.get_A()
A_inv = A.inverse()
real_space_a = A_inv.elems[:3]
real_space_b = A_inv.elems[3:6]
real_space_c = A_inv.elems[6:9]
to_xds = xds.to_xds(imageset)
xds_inp = os.path.join(sub_dir, 'XDS.INP')
xparm_xds = os.path.join(sub_dir, 'XPARM.XDS')
print "Exporting experiment to %s and %s" %(xds_inp, xparm_xds)
with open(xds_inp, 'wb') as f:
to_xds.XDS_INP(
out=f,
space_group_number=crystal_model.get_space_group().type().number(),
real_space_a=real_space_a,
real_space_b=real_space_b,
real_space_c=real_space_c,
job_card="XYCORR INIT DEFPIX INTEGRATE CORRECT")
with open(xparm_xds, 'wb') as f:
to_xds.xparm_xds(
real_space_a, real_space_b, real_space_c,
crystal_model.get_space_group().type().number(),
out=f)
if reflections is not None and len(reflections) > 0:
ref_cryst = reflections.select(reflections['id'] == i)
export_spot_xds(ref_cryst, os.path.join(sub_dir, 'SPOT.XDS'))
else:
if not os.path.isdir(directory):
os.makedirs(directory)
export_spot_xds(reflections, os.path.join(directory, 'SPOT.XDS'))
def run(file_names):
if len(file_names) == 1 and file_names[0].endswith('json'):
from dxtbx.serialize import load
datablock = load.datablock(file_names[0])
assert(len(datablock) == 1)
sweep = datablock[0].extract_sweeps()[0]
else:
from dxtbx.imageset import ImageSetFactory
sweep = ImageSetFactory.new(file_names)[0]
xsx = xds.to_xds(sweep)
xsx.XDS_INP()
def test_to_xds_multi_panel_i23(dials_regression, tmpdir, mocker):
tmpdir.chdir()
file_name = os.path.join(dials_regression, "image_examples", "DLS_I23",
"germ_13KeV_0001.cbf")
sequence = ImageSetFactory.new([file_name])[0]
to_xds = xds.to_xds(sequence)
s1 = to_xds.XDS_INP()
for expected_substr in (
"""\
!
! SEGMENT 1
!
SEGMENT= 1 2463 1 195
DIRECTION_OF_SEGMENT_X-AXIS= 1.00000 0.00000 0.00000
DIRECTION_OF_SEGMENT_Y-AXIS= 0.00000 -0.14347 0.98966
SEGMENT_DISTANCE= 250.000
SEGMENT_ORGX= 1075.00 SEGMENT_ORGY= 97.67""",
"""\
!
! SEGMENT 24
!
SEGMENT= 1 2463 4877 5071
DIRECTION_OF_SEGMENT_X-AXIS= 1.00000 0.00000 0.00000
DIRECTION_OF_SEGMENT_Y-AXIS= 0.00000 -0.06390 -0.99796
SEGMENT_DISTANCE= 250.000
SEGMENT_ORGX= 1075.00 SEGMENT_ORGY= 4973.67""",
):
assert expected_substr in s1
assert "UNTRUSTED_RECTANGLE" not in s1
# Fool the format class into masking out a couple of bad modules
mocked_timestamp = mocker.patch(
"dxtbx.format.FormatCBFMiniPilatusDLS12M.get_pilatus_timestamp")
mocked_timestamp.return_value = 1574857526.34
sequence = ImageSetFactory.new([file_name])[0]
to_xds = xds.to_xds(sequence)
s = to_xds.XDS_INP()
assert "UNTRUSTED_RECTANGLE= 0 488 3604 3800" in s
assert "UNTRUSTED_RECTANGLE= 1976 2464 3604 3800" in s
def read_image_metadata_dxtbx(image):
"""Read the image header and send back the resulting metadata in a
dictionary. Read this using dxtbx - for a sequence of images use the
first image in the sequence to derive the metadata, for HDF5 files
just get on an read."""
check_file_readable(image)
if image.endswith(".h5"):
# XDS can literally only handle master files called (prefix)_master.h5
assert "master" in image
from dxtbx.datablock import DataBlockFactory
db = DataBlockFactory.from_filenames([image])[0]
else:
from dxtbx.datablock import DataBlockTemplateImporter
template, directory = image2template_directory(image)
full_template = os.path.join(directory, template)
importer = DataBlockTemplateImporter([full_template],
allow_incomplete_sweeps=True)
db = importer.datablocks[0]
if hasattr(db, "extract_sequences"):
db_extract = db.extract_sequences
else:
db_extract = db.extract_sweeps
sequences = db_extract()[0]
from dxtbx.serialize.xds import to_xds
XDS_INP = to_xds(sequences).XDS_INP()
params = XDS_INP_to_dict(XDS_INP)
# detector type specific parameters - minimum spot size, trusted region
# and so on.
# from dxtbx.model.detector_helpers import detector_helper_sensors
# d = sequences.get_detector()
# sensor_type = d[0].get_type()
# if sensor_type == detector_helper_sensors.SENSOR_PAD:
# params["MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT"] = "2"
# else:
# params["MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT"] = "4"
# remove things we will want to guarantee we set in fast_dp
for name in ["BACKGROUND_RANGE", "SPOT_RANGE", "JOB"]:
if name in params:
del params[name]
return params
def exercise_to_xds():
if not libtbx.env.has_module("dials"):
print "Skipping test: dials not present"
return
if not libtbx.env.has_module("dials_regression"):
print "Skipping exercise_to_xds(): dials_regression not present"
return
data_dir = libtbx.env.find_in_repositories(
relative_path="dials_regression/centroid_test_data",
test=os.path.isdir)
template = os.path.join(data_dir, "centroid_00*.cbf")
file_names = glob.glob(template)
sweep = ImageSetFactory.new(file_names)[0]
to_xds = xds.to_xds(sweep)
s1 = StringIO()
to_xds.XDS_INP(out=s1)
s2 = StringIO()
real_space_a = (-5.327642, -39.034747, -4.988286)
real_space_b = (-35.253495, 7.596265, -22.127661)
real_space_c = (-22.673623, -1.486119, 35.793463)
to_xds.xparm_xds(real_space_a,
real_space_b,
real_space_c,
space_group=1,
out=s2)
# run coordinate frame converter on xparm.xds as a sanity check
f = open_tmp_file(suffix="XPARM.XDS", mode="wb")
s2.seek(0)
f.writelines(s2.readlines())
f.close()
from rstbx.cftbx import coordinate_frame_helpers
converter = coordinate_frame_helpers.import_xds_xparm(f.name)
scan = sweep.get_scan()
detector = sweep.get_detector()
goniometer = sweep.get_goniometer()
beam = sweep.get_beam()
assert approx_equal(real_space_a, converter.get_real_space_a())
assert approx_equal(real_space_b, converter.get_real_space_b())
assert approx_equal(real_space_c, converter.get_real_space_c())
assert approx_equal(goniometer.get_rotation_axis(),
converter.get_rotation_axis())
assert approx_equal(beam.get_direction(),
converter.get_sample_to_source().elems)
assert approx_equal(detector[0].get_fast_axis(),
converter.get_detector_fast())
assert approx_equal(detector[0].get_slow_axis(),
converter.get_detector_slow())
assert approx_equal(detector[0].get_origin(),
converter.get_detector_origin())
def test_to_xds(dials_regression, tmpdir):
tmpdir.chdir()
template = os.path.join(dials_regression, 'centroid_test_data',
"centroid_00*.cbf")
file_names = glob.glob(template)
sweep = ImageSetFactory.new(file_names)[0]
to_xds = xds.to_xds(sweep)
s1 = StringIO()
to_xds.XDS_INP(out=s1)
empty = StringIO()
s1a = to_xds.XDS_INP(as_str=True, out=empty)
assert empty.getvalue() == ''
assert s1.getvalue().strip() == s1a
s2 = StringIO()
real_space_a = (-5.327642, -39.034747, -4.988286)
real_space_b = (-35.253495, 7.596265, -22.127661)
real_space_c = (-22.673623, -1.486119, 35.793463)
to_xds.xparm_xds(real_space_a,
real_space_b,
real_space_c,
space_group=1,
out=s2)
# run coordinate frame converter on xparm.xds as a sanity check
with open('xparm.xds', mode="wb") as fh:
s2.seek(0)
fh.writelines(s2.readlines())
from rstbx.cftbx import coordinate_frame_helpers
converter = coordinate_frame_helpers.import_xds_xparm('xparm.xds')
scan = sweep.get_scan()
detector = sweep.get_detector()
goniometer = sweep.get_goniometer()
beam = sweep.get_beam()
assert approx_equal(real_space_a, converter.get_real_space_a())
assert approx_equal(real_space_b, converter.get_real_space_b())
assert approx_equal(real_space_c, converter.get_real_space_c())
assert approx_equal(goniometer.get_rotation_axis(),
converter.get_rotation_axis())
assert approx_equal(beam.get_direction(),
converter.get_sample_to_source().elems)
assert approx_equal(detector[0].get_fast_axis(),
converter.get_detector_fast())
assert approx_equal(detector[0].get_slow_axis(),
converter.get_detector_slow())
assert approx_equal(detector[0].get_origin(),
converter.get_detector_origin())
def test_i(self):
idxref = self.Idxref()
self._index_remove_masked_regions()
for file in ['SPOT.XDS']:
idxref.set_input_data_file(file, self._indxr_payload[file])
idxref.set_data_range(self._indxr_images[0][0],
self._indxr_images[0][1])
idxref.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
# set the phi start etc correctly
blocks = self._index_select_images_i()
for block in blocks[:1]:
starting_frame = block[0]
dd = Diffdump()
dd.set_image(self.get_image_name(starting_frame))
starting_angle = dd.readheader()['phi_start']
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
for block in blocks[1:]:
idxref.add_spot_range(block[0], block[1])
#mosflm_beam_centre = self.get_beam_centre()
#xds_beam_centre = beam_centre_mosflm_to_xds(
#mosflm_beam_centre[0], mosflm_beam_centre[1], self.get_header())
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
idxref.set_beam_centre(xds_beam_centre[0],
xds_beam_centre[1])
idxref.run()
return idxref.get_fraction_rmsd_rmsphi()
def run(file_names):
if len(file_names) == 1 and file_names[0].endswith('json'):
from dxtbx.serialize import load
try:
datablock = load.datablock(file_names[0])
assert len(datablock) == 1
sweep = datablock[0].extract_sweeps()[0]
except ValueError as e:
if str(e) == '"__id__" does not equal "imageset"':
experiments = load.experiment_list(file_names[0])
assert len(experiments) == 1
sweep = experiments[0].imageset
else:
raise
else:
from dxtbx.imageset import ImageSetFactory
sweep = ImageSetFactory.new(file_names)[0]
xsx = xds.to_xds(sweep)
xsx.XDS_INP()
def test_i(self):
idxref = self.Idxref()
self._index_remove_masked_regions()
for file in ["SPOT.XDS"]:
idxref.set_input_data_file(file, self._indxr_payload[file])
idxref.set_data_range(self._indxr_images[0][0],
self._indxr_images[0][1])
idxref.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
# set the phi start etc correctly
blocks = self._index_select_images_i()
for block in blocks[:1]:
starting_frame = block[0]
starting_angle = self.get_scan().get_angle_from_image_index(
starting_frame)
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
for block in blocks[1:]:
idxref.add_spot_range(block[0], block[1])
# mosflm_beam_centre = self.get_beam_centre()
# xds_beam_centre = beam_centre_mosflm_to_xds(
# mosflm_beam_centre[0], mosflm_beam_centre[1], self.get_header())
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
idxref.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1])
idxref.run()
return idxref.get_fraction_rmsd_rmsphi()
def exercise_to_xds():
if not libtbx.env.has_module("dials_regression"):
print "Skipping exercise_to_xds(): dials_regression not present"
return
data_dir = libtbx.env.find_in_repositories(
relative_path="dials_regression/centroid_test_data",
test=os.path.isdir)
template = os.path.join(data_dir, "centroid_00*.cbf")
file_names = glob.glob(template)
sweep = ImageSetFactory.new(file_names)[0]
to_xds = xds.to_xds(sweep)
s1 = StringIO()
to_xds.XDS_INP(out=s1)
s2 = StringIO()
real_space_a = (-5.327642, -39.034747, -4.988286)
real_space_b = (-35.253495, 7.596265, -22.127661)
real_space_c = (-22.673623, -1.486119, 35.793463)
to_xds.xparm_xds(real_space_a, real_space_b, real_space_c, space_group=1, out=s2)
# run coordinate frame converter on xparm.xds as a sanity check
f = open_tmp_file(suffix="XPARM.XDS", mode="wb")
s2.seek(0)
f.writelines(s2.readlines())
f.close()
from rstbx.cftbx import coordinate_frame_helpers
converter = coordinate_frame_helpers.import_xds_xparm(f.name)
scan = sweep.get_scan()
detector = sweep.get_detector()
goniometer = sweep.get_goniometer()
beam = sweep.get_beam()
assert approx_equal(real_space_a, converter.get_real_space_a())
assert approx_equal(real_space_b, converter.get_real_space_b())
assert approx_equal(real_space_c, converter.get_real_space_c())
assert approx_equal(goniometer.get_rotation_axis(),
converter.get_rotation_axis())
assert approx_equal(
beam.get_direction(), converter.get_sample_to_source().elems)
assert approx_equal(detector[0].get_fast_axis(), converter.get_detector_fast())
assert approx_equal(detector[0].get_slow_axis(), converter.get_detector_slow())
assert approx_equal(detector[0].get_origin(), converter.get_detector_origin())
def _index(self):
'''Actually do the autoindexing using the data prepared by the
previous method.'''
images_str = '%d to %d' % tuple(self._indxr_images[0])
for i in self._indxr_images[1:]:
images_str += ', %d to %d' % tuple(i)
cell_str = None
if self._indxr_input_cell:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
self._indxr_input_cell
# then this is a proper autoindexing run - describe this
# to the journal entry
#if len(self._fp_directory) <= 50:
#dirname = self._fp_directory
#else:
#dirname = '...%s' % self._fp_directory[-46:]
dirname = self.get_directory()
Journal.block('autoindexing', self._indxr_sweep_name, 'XDS',
{'images':images_str,
'target cell':cell_str,
'target lattice':self._indxr_input_lattice,
'template':self.get_template(),
'directory':dirname})
idxref = self.Idxref()
self._index_remove_masked_regions()
for file in ['SPOT.XDS']:
idxref.set_input_data_file(file, self._indxr_payload[file])
idxref.set_data_range(self._indxr_images[0][0],
self._indxr_images[0][1])
idxref.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
# set the phi start etc correctly
if self._i_or_ii == None:
self._i_or_ii = self.decide_i_or_ii()
Debug.write('Selecting I or II, chose %s' % self._i_or_ii)
if self._i_or_ii == 'i':
blocks = self._index_select_images_i()
for block in blocks[:1]:
starting_frame = block[0]
dd = Diffdump()
dd.set_image(self.get_image_name(starting_frame))
starting_angle = dd.readheader()['phi_start']
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
for block in blocks[1:]:
idxref.add_spot_range(block[0], block[1])
else:
for block in self._indxr_images[:1]:
starting_frame = block[0]
dd = Diffdump()
dd.set_image(self.get_image_name(starting_frame))
starting_angle = dd.readheader()['phi_start']
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
for block in self._indxr_images[1:]:
idxref.add_spot_range(block[0], block[1])
# FIXME need to also be able to pass in the known unit
# cell and lattice if already available e.g. from
# the helper... indirectly
if self._indxr_user_input_lattice:
idxref.set_indexer_user_input_lattice(True)
if self._indxr_input_lattice and self._indxr_input_cell:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
idxref.set_indexer_input_cell(self._indxr_input_cell)
Debug.write('Set lattice: %s' % self._indxr_input_lattice)
Debug.write('Set cell: %f %f %f %f %f %f' % \
self._indxr_input_cell)
original_cell = self._indxr_input_cell
elif self._indxr_input_lattice:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
original_cell = None
else:
original_cell = None
# FIXED need to set the beam centre here - this needs to come
# from the input .xinfo object or header, and be converted
# to the XDS frame... done.
#mosflm_beam_centre = self.get_beam_centre()
#xds_beam_centre = beam_centre_mosflm_to_xds(
#mosflm_beam_centre[0], mosflm_beam_centre[1], self.get_header())
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
idxref.set_beam_centre(xds_beam_centre[0],
xds_beam_centre[1])
# fixme need to check if the lattice, cell have been set already,
# and if they have, pass these in as input to the indexing job.
done = False
while not done:
try:
done = idxref.run()
# N.B. in here if the IDXREF step was being run in the first
# pass done is FALSE however there should be a refined
# P1 orientation matrix etc. available - so keep it!
except XDSException, e:
# inspect this - if we have complaints about not
# enough reflections indexed, and we have a target
# unit cell, and they are the same, well ignore it
if 'solution is inaccurate' in str(e):
Debug.write(
'XDS complains solution inaccurate - ignoring')
done = idxref.continue_from_error()
elif ('insufficient percentage (< 70%)' in str(e) or
'insufficient percentage (< 50%)' in str(e)) and \
original_cell:
done = idxref.continue_from_error()
lattice, cell, mosaic = \
idxref.get_indexing_solution()
# compare solutions
for j in range(3):
# allow two percent variation in unit cell length
if math.fabs((cell[j] - original_cell[j]) / \
original_cell[j]) > 0.02 and \
not Flags.get_relax():
Debug.write('XDS unhappy and solution wrong')
raise e
# and two degree difference in angle
if math.fabs(cell[j + 3] - original_cell[j + 3]) \
> 2.0 and not Flags.get_relax():
Debug.write('XDS unhappy and solution wrong')
raise e
Debug.write('XDS unhappy but solution ok')
elif 'insufficient percentage (< 70%)' in str(e) or \
'insufficient percentage (< 50%)' in str(e):
Debug.write('XDS unhappy but solution probably ok')
done = idxref.continue_from_error()
else:
raise e
def _index_prepare(self):
"""Prepare to do autoindexing - in XDS terms this will mean
calling xycorr, init and colspot on the input images."""
# decide on images to work with
logger.debug("XDS INDEX PREPARE:")
logger.debug("Wavelength: %.6f", self.get_wavelength())
logger.debug("Distance: %.2f", self.get_distance())
if self._indxr_images == []:
_select_images_function = getattr(
self, "_index_select_images_%s" % self._index_select_images
)
wedges = _select_images_function()
for wedge in wedges:
self.add_indexer_image_wedge(wedge)
self.set_indexer_prepare_done(True)
all_images = self.get_matching_images()
first = min(all_images)
last = max(all_images)
# next start to process these - first xycorr
xycorr = self.Xycorr()
xycorr.set_data_range(first, last)
xycorr.set_background_range(self._indxr_images[0][0], self._indxr_images[0][1])
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
xycorr.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1])
for block in self._indxr_images:
xycorr.add_spot_range(block[0], block[1])
# FIXME need to set the origin here
xycorr.run()
for file in ["X-CORRECTIONS.cbf", "Y-CORRECTIONS.cbf"]:
self._indxr_payload[file] = xycorr.get_output_data_file(file)
# next start to process these - then init
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
imageset = self._indxr_imagesets[0]
masker = (
imageset.get_format_class()
.get_instance(imageset.paths()[0])
.get_masker()
)
if masker is None:
# disable dynamic_shadowing
PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
# find the region of the scan with the least predicted shadow
# to use for background determination in XDS INIT step
from dxtbx.model.experiment_list import ExperimentListFactory
imageset = self._indxr_imagesets[0]
xsweep = self._indxr_sweeps[0]
sweep_filename = os.path.join(
self.get_working_directory(), "%s_indexed.expt" % xsweep.get_name()
)
ExperimentListFactory.from_imageset_and_crystal(imageset, None).as_file(
sweep_filename
)
from xia2.Wrappers.Dials.ShadowPlot import ShadowPlot
shadow_plot = ShadowPlot()
shadow_plot.set_working_directory(self.get_working_directory())
auto_logfiler(shadow_plot)
shadow_plot.set_sweep_filename(sweep_filename)
shadow_plot.set_json_filename(
os.path.join(
self.get_working_directory(),
"%s_shadow_plot.json" % shadow_plot.get_xpid(),
)
)
shadow_plot.run()
results = shadow_plot.get_results()
fraction_shadowed = flex.double(results["fraction_shadowed"])
if flex.max(fraction_shadowed) == 0:
PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False
else:
scan_points = flex.double(results["scan_points"])
scan = imageset.get_scan()
oscillation = scan.get_oscillation()
if self._background_images is not None:
bg_images = self._background_images
bg_range_deg = (
scan.get_angle_from_image_index(bg_images[0]),
scan.get_angle_from_image_index(bg_images[1]),
)
bg_range_width = bg_range_deg[1] - bg_range_deg[0]
min_shadow = 100
best_bg_range = bg_range_deg
from libtbx.utils import frange
for bg_range_start in frange(
flex.min(scan_points),
flex.max(scan_points) - bg_range_width,
step=oscillation[1],
):
bg_range_deg = (bg_range_start, bg_range_start + bg_range_width)
sel = (scan_points >= bg_range_deg[0]) & (
scan_points <= bg_range_deg[1]
)
mean_shadow = flex.mean(fraction_shadowed.select(sel))
if mean_shadow < min_shadow:
min_shadow = mean_shadow
best_bg_range = bg_range_deg
self._background_images = (
scan.get_image_index_from_angle(best_bg_range[0]),
scan.get_image_index_from_angle(best_bg_range[1]),
)
logger.debug(
"Setting background images: %s -> %s" % self._background_images
)
init = self.Init()
for file in ["X-CORRECTIONS.cbf", "Y-CORRECTIONS.cbf"]:
init.set_input_data_file(file, self._indxr_payload[file])
init.set_data_range(first, last)
if self._background_images:
init.set_background_range(
self._background_images[0], self._background_images[1]
)
else:
init.set_background_range(
self._indxr_images[0][0], self._indxr_images[0][1]
)
for block in self._indxr_images:
init.add_spot_range(block[0], block[1])
init.run()
# at this stage, need to (perhaps) modify the BKGINIT.cbf image
# to mark out the back stop
if PhilIndex.params.xds.backstop_mask:
logger.debug("Applying mask to BKGINIT.pck")
# copy the original file
cbf_old = os.path.join(init.get_working_directory(), "BKGINIT.cbf")
cbf_save = os.path.join(init.get_working_directory(), "BKGINIT.sav")
shutil.copyfile(cbf_old, cbf_save)
# modify the file to give the new mask
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.xds.backstop_mask)
mask.apply_mask_xds(self.get_header(), cbf_save, cbf_old)
init.reload()
for file in ["BLANK.cbf", "BKGINIT.cbf", "GAIN.cbf"]:
self._indxr_payload[file] = init.get_output_data_file(file)
if PhilIndex.params.xia2.settings.developmental.use_dials_spotfinder:
spotfinder = self.DialsSpotfinder()
for block in self._indxr_images:
spotfinder.add_spot_range(block[0], block[1])
spotfinder.run()
export = self.DialsExportSpotXDS()
export.set_input_data_file(
"observations.refl",
spotfinder.get_output_data_file("observations.refl"),
)
export.run()
for file in ["SPOT.XDS"]:
self._indxr_payload[file] = export.get_output_data_file(file)
else:
# next start to process these - then colspot
colspot = self.Colspot()
for file in (
"X-CORRECTIONS.cbf",
"Y-CORRECTIONS.cbf",
"BLANK.cbf",
"BKGINIT.cbf",
"GAIN.cbf",
):
colspot.set_input_data_file(file, self._indxr_payload[file])
colspot.set_data_range(first, last)
colspot.set_background_range(
self._indxr_images[0][0], self._indxr_images[0][1]
)
for block in self._indxr_images:
colspot.add_spot_range(block[0], block[1])
colspot.run()
for file in ["SPOT.XDS"]:
self._indxr_payload[file] = colspot.get_output_data_file(file)
def _index(self):
'''Actually do the autoindexing using the data prepared by the
previous method.'''
images_str = '%d to %d' % tuple(self._indxr_images[0])
for i in self._indxr_images[1:]:
images_str += ', %d to %d' % tuple(i)
cell_str = None
if self._indxr_input_cell:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
self._indxr_input_cell
# then this is a proper autoindexing run - describe this
# to the journal entry
#if len(self._fp_directory) <= 50:
#dirname = self._fp_directory
#else:
#dirname = '...%s' % self._fp_directory[-46:]
dirname = self.get_directory()
Journal.block(
'autoindexing', self._indxr_sweep_name, 'XDS', {
'images': images_str,
'target cell': cell_str,
'target lattice': self._indxr_input_lattice,
'template': self.get_template(),
'directory': dirname
})
self._index_remove_masked_regions()
if self._i_or_ii is None:
self._i_or_ii = self.decide_i_or_ii()
Debug.write('Selecting I or II, chose %s' % self._i_or_ii)
idxref = self.Idxref()
for file in ['SPOT.XDS']:
idxref.set_input_data_file(file, self._indxr_payload[file])
# set the phi start etc correctly
idxref.set_data_range(self._indxr_images[0][0],
self._indxr_images[0][1])
idxref.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
if self._i_or_ii == 'i':
blocks = self._index_select_images_i()
for block in blocks[:1]:
starting_frame = block[0]
starting_angle = self.get_scan().get_angle_from_image_index(
starting_frame)
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
for block in blocks[1:]:
idxref.add_spot_range(block[0], block[1])
else:
for block in self._indxr_images[:1]:
starting_frame = block[0]
starting_angle = self.get_scan().get_angle_from_image_index(
starting_frame)
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
for block in self._indxr_images[1:]:
idxref.add_spot_range(block[0], block[1])
# FIXME need to also be able to pass in the known unit
# cell and lattice if already available e.g. from
# the helper... indirectly
if self._indxr_user_input_lattice:
idxref.set_indexer_user_input_lattice(True)
if self._indxr_input_lattice and self._indxr_input_cell:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
idxref.set_indexer_input_cell(self._indxr_input_cell)
Debug.write('Set lattice: %s' % self._indxr_input_lattice)
Debug.write('Set cell: %f %f %f %f %f %f' % \
self._indxr_input_cell)
original_cell = self._indxr_input_cell
elif self._indxr_input_lattice:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
original_cell = None
else:
original_cell = None
# FIXED need to set the beam centre here - this needs to come
# from the input .xinfo object or header, and be converted
# to the XDS frame... done.
#mosflm_beam_centre = self.get_beam_centre()
#xds_beam_centre = beam_centre_mosflm_to_xds(
#mosflm_beam_centre[0], mosflm_beam_centre[1], self.get_header())
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
idxref.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1])
# fixme need to check if the lattice, cell have been set already,
# and if they have, pass these in as input to the indexing job.
done = False
while not done:
try:
done = idxref.run()
# N.B. in here if the IDXREF step was being run in the first
# pass done is FALSE however there should be a refined
# P1 orientation matrix etc. available - so keep it!
except XDSException as e:
# inspect this - if we have complaints about not
# enough reflections indexed, and we have a target
# unit cell, and they are the same, well ignore it
if 'solution is inaccurate' in str(e):
Debug.write('XDS complains solution inaccurate - ignoring')
done = idxref.continue_from_error()
elif ('insufficient percentage (< 70%)' in str(e) or
'insufficient percentage (< 50%)' in str(e)) and \
original_cell:
done = idxref.continue_from_error()
lattice, cell, mosaic = \
idxref.get_indexing_solution()
# compare solutions
check = PhilIndex.params.xia2.settings.xds_check_cell_deviation
for j in range(3):
# allow two percent variation in unit cell length
if math.fabs((cell[j] - original_cell[j]) / \
original_cell[j]) > 0.02 and check:
Debug.write('XDS unhappy and solution wrong')
raise e
# and two degree difference in angle
if math.fabs(cell[j + 3] - original_cell[j + 3]) \
> 2.0 and check:
Debug.write('XDS unhappy and solution wrong')
raise e
Debug.write('XDS unhappy but solution ok')
elif 'insufficient percentage (< 70%)' in str(e) or \
'insufficient percentage (< 50%)' in str(e):
Debug.write('XDS unhappy but solution probably ok')
done = idxref.continue_from_error()
else:
raise e
FileHandler.record_log_file(
'%s INDEX' % self.get_indexer_full_name(),
os.path.join(self.get_working_directory(), 'IDXREF.LP'))
for file in ['SPOT.XDS', 'XPARM.XDS']:
self._indxr_payload[file] = idxref.get_output_data_file(file)
# need to get the indexing solutions out somehow...
self._indxr_other_lattice_cell = idxref.get_indexing_solutions()
self._indxr_lattice, self._indxr_cell, self._indxr_mosaic = \
idxref.get_indexing_solution()
import dxtbx
from dxtbx.serialize.xds import to_crystal
xparm_file = os.path.join(self.get_working_directory(), 'XPARM.XDS')
models = dxtbx.load(xparm_file)
crystal_model = to_crystal(xparm_file)
from dxtbx.model import Experiment, ExperimentList
experiment = Experiment(
beam=models.get_beam(),
detector=models.get_detector(),
goniometer=models.get_goniometer(),
scan=models.get_scan(),
crystal=crystal_model,
#imageset=self.get_imageset(),
)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
# I will want this later on to check that the lattice was ok
self._idxref_subtree_problem = idxref.get_index_tree_problem()
return
def _index(self):
"""Actually do the autoindexing using the data prepared by the
previous method."""
images_str = "%d to %d" % tuple(self._indxr_images[0])
for i in self._indxr_images[1:]:
images_str += ", %d to %d" % tuple(i)
cell_str = None
if self._indxr_input_cell:
cell_str = "%.2f %.2f %.2f %.2f %.2f %.2f" % self._indxr_input_cell
# then this is a proper autoindexing run - describe this
# to the journal entry
dirname = self.get_directory()
Journal.block(
"autoindexing",
self._indxr_sweep_name,
"XDS",
{
"images": images_str,
"target cell": cell_str,
"target lattice": self._indxr_input_lattice,
"template": self.get_template(),
"directory": dirname,
},
)
idxref = self.Idxref()
self._index_remove_masked_regions()
for file in ["SPOT.XDS"]:
idxref.set_input_data_file(file, self._indxr_payload[file])
# edit SPOT.XDS to remove reflections in untrusted regions of the detector
idxref.set_data_range(self._indxr_images[0][0],
self._indxr_images[0][1])
idxref.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
# set the phi start etc correctly
for block in self._indxr_images[:1]:
starting_frame = block[0]
starting_angle = self.get_scan().get_angle_from_image_index(
starting_frame)
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
for block in self._indxr_images[1:]:
idxref.add_spot_range(block[0], block[1])
if self._indxr_user_input_lattice:
idxref.set_indexer_user_input_lattice(True)
if self._indxr_input_lattice and self._indxr_input_cell:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
idxref.set_indexer_input_cell(self._indxr_input_cell)
Debug.write("Set lattice: %s" % self._indxr_input_lattice)
Debug.write("Set cell: %f %f %f %f %f %f" % self._indxr_input_cell)
original_cell = self._indxr_input_cell
elif self._indxr_input_lattice:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
original_cell = None
else:
original_cell = None
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
idxref.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1])
# fixme need to check if the lattice, cell have been set already,
# and if they have, pass these in as input to the indexing job.
done = False
while not done:
try:
done = idxref.run()
# N.B. in here if the IDXREF step was being run in the first
# pass done is FALSE however there should be a refined
# P1 orientation matrix etc. available - so keep it!
except XDSException as e:
# inspect this - if we have complaints about not
# enough reflections indexed, and we have a target
# unit cell, and they are the same, well ignore it
if "solution is inaccurate" in str(e):
Debug.write("XDS complains solution inaccurate - ignoring")
done = idxref.continue_from_error()
elif ("insufficient percentage (< 70%)" in str(e)
or "insufficient percentage (< 50%)"
in str(e)) and original_cell:
done = idxref.continue_from_error()
lattice, cell, mosaic = idxref.get_indexing_solution()
# compare solutions FIXME should use xds_cell_deviation
check = PhilIndex.params.xia2.settings.xds_check_cell_deviation
for j in range(3):
# allow two percent variation in unit cell length
if (math.fabs(
(cell[j] - original_cell[j]) / original_cell[j]) >
0.02 and check):
Debug.write("XDS unhappy and solution wrong")
raise e
# and two degree difference in angle
if (math.fabs(cell[j + 3] - original_cell[j + 3]) > 2.0
and check):
Debug.write("XDS unhappy and solution wrong")
raise e
Debug.write("XDS unhappy but solution ok")
elif "insufficient percentage (< 70%)" in str(
e) or "insufficient percentage (< 50%)" in str(e):
Debug.write("XDS unhappy but solution probably ok")
done = idxref.continue_from_error()
else:
raise e
FileHandler.record_log_file(
"%s INDEX" % self.get_indexer_full_name(),
os.path.join(self.get_working_directory(), "IDXREF.LP"),
)
for file in ["SPOT.XDS", "XPARM.XDS"]:
self._indxr_payload[file] = idxref.get_output_data_file(file)
# need to get the indexing solutions out somehow...
self._indxr_other_lattice_cell = idxref.get_indexing_solutions()
self._indxr_lattice, self._indxr_cell, self._indxr_mosaic = (
idxref.get_indexing_solution())
import dxtbx
from dxtbx.serialize.xds import to_crystal
xparm_file = os.path.join(self.get_working_directory(), "XPARM.XDS")
models = dxtbx.load(xparm_file)
crystal_model = to_crystal(xparm_file)
from dxtbx.model import Experiment, ExperimentList
experiment = Experiment(
beam=models.get_beam(),
detector=models.get_detector(),
goniometer=models.get_goniometer(),
scan=models.get_scan(),
crystal=crystal_model,
# imageset=self.get_imageset(),
)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
# I will want this later on to check that the lattice was ok
self._idxref_subtree_problem = idxref.get_index_tree_problem()
return
def _index(self):
"""Actually do the autoindexing using the data prepared by the
previous method."""
self._index_remove_masked_regions()
if self._i_or_ii is None:
self._i_or_ii = self.decide_i_or_ii()
logger.debug("Selecting I or II, chose %s", self._i_or_ii)
idxref = self.Idxref()
for file in ["SPOT.XDS"]:
idxref.set_input_data_file(file, self._indxr_payload[file])
# set the phi start etc correctly
idxref.set_data_range(self._indxr_images[0][0],
self._indxr_images[0][1])
idxref.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
if self._i_or_ii == "i":
blocks = self._index_select_images_i()
for block in blocks[:1]:
starting_frame = block[0]
starting_angle = self.get_scan().get_angle_from_image_index(
starting_frame)
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
for block in blocks[1:]:
idxref.add_spot_range(block[0], block[1])
else:
for block in self._indxr_images[:1]:
starting_frame = block[0]
starting_angle = self.get_scan().get_angle_from_image_index(
starting_frame)
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
for block in self._indxr_images[1:]:
idxref.add_spot_range(block[0], block[1])
# FIXME need to also be able to pass in the known unit
# cell and lattice if already available e.g. from
# the helper... indirectly
if self._indxr_user_input_lattice:
idxref.set_indexer_user_input_lattice(True)
if self._indxr_input_lattice and self._indxr_input_cell:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
idxref.set_indexer_input_cell(self._indxr_input_cell)
logger.debug("Set lattice: %s", self._indxr_input_lattice)
logger.debug("Set cell: %f %f %f %f %f %f" %
self._indxr_input_cell)
original_cell = self._indxr_input_cell
elif self._indxr_input_lattice:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
original_cell = None
else:
original_cell = None
# FIXED need to set the beam centre here - this needs to come
# from the input .xinfo object or header, and be converted
# to the XDS frame... done.
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
idxref.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1])
# fixme need to check if the lattice, cell have been set already,
# and if they have, pass these in as input to the indexing job.
done = False
while not done:
try:
done = idxref.run()
# N.B. in here if the IDXREF step was being run in the first
# pass done is FALSE however there should be a refined
# P1 orientation matrix etc. available - so keep it!
except XDSException as e:
# inspect this - if we have complaints about not
# enough reflections indexed, and we have a target
# unit cell, and they are the same, well ignore it
if "solution is inaccurate" in str(e):
logger.debug(
"XDS complains solution inaccurate - ignoring")
done = idxref.continue_from_error()
elif ("insufficient percentage (< 70%)" in str(e)
or "insufficient percentage (< 50%)"
in str(e)) and original_cell:
done = idxref.continue_from_error()
lattice, cell, mosaic = idxref.get_indexing_solution()
# compare solutions
check = PhilIndex.params.xia2.settings.xds_check_cell_deviation
for j in range(3):
# allow two percent variation in unit cell length
if (math.fabs(
(cell[j] - original_cell[j]) / original_cell[j]) >
0.02 and check):
logger.debug("XDS unhappy and solution wrong")
raise e
# and two degree difference in angle
if (math.fabs(cell[j + 3] - original_cell[j + 3]) > 2.0
and check):
logger.debug("XDS unhappy and solution wrong")
raise e
logger.debug("XDS unhappy but solution ok")
elif "insufficient percentage (< 70%)" in str(
e) or "insufficient percentage (< 50%)" in str(e):
logger.debug("XDS unhappy but solution probably ok")
done = idxref.continue_from_error()
else:
raise e
FileHandler.record_log_file(
"%s INDEX" % self.get_indexer_full_name(),
os.path.join(self.get_working_directory(), "IDXREF.LP"),
)
for file in ["SPOT.XDS", "XPARM.XDS"]:
self._indxr_payload[file] = idxref.get_output_data_file(file)
# need to get the indexing solutions out somehow...
self._indxr_other_lattice_cell = idxref.get_indexing_solutions()
(
self._indxr_lattice,
self._indxr_cell,
self._indxr_mosaic,
) = idxref.get_indexing_solution()
xparm_file = os.path.join(self.get_working_directory(), "XPARM.XDS")
models = dxtbx.load(xparm_file)
crystal_model = to_crystal(xparm_file)
# this information gets lost when re-creating the models from the
# XDS results - however is not refined so can simply copy from the
# input - https://github.com/xia2/xia2/issues/372
models.get_detector()[0].set_thickness(
converter.get_detector()[0].get_thickness())
experiment = Experiment(
beam=models.get_beam(),
detector=models.get_detector(),
goniometer=models.get_goniometer(),
scan=models.get_scan(),
crystal=crystal_model,
# imageset=self.get_imageset(),
)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
# I will want this later on to check that the lattice was ok
self._idxref_subtree_problem = idxref.get_index_tree_problem()
def test_to_xds(dials_regression, tmpdir):
tmpdir.chdir()
template = os.path.join(dials_regression, "centroid_test_data", "centroid_00*.cbf")
file_names = glob.glob(template)
sweep = ImageSetFactory.new(file_names)[0]
to_xds = xds.to_xds(sweep)
s1 = to_xds.XDS_INP()
expected = """\
DETECTOR=PILATUS MINIMUM_VALID_PIXEL_VALUE=0 OVERLOAD=495976
SENSOR_THICKNESS= 0.320
!SENSOR_MATERIAL / THICKNESS Si 0.320
!SILICON= 3.960382
DIRECTION_OF_DETECTOR_X-AXIS= 1.00000 0.00000 0.00000
DIRECTION_OF_DETECTOR_Y-AXIS= 0.00000 1.00000 0.00000
NX=2463 NY=2527 QX=0.1720 QY=0.1720
DETECTOR_DISTANCE= 190.180000
ORGX= 1235.84 ORGY= 1279.58
ROTATION_AXIS= 1.00000 0.00000 0.00000
STARTING_ANGLE= 0.000
OSCILLATION_RANGE= 0.200
X-RAY_WAVELENGTH= 0.97950
INCIDENT_BEAM_DIRECTION= -0.000 -0.000 1.021
FRACTION_OF_POLARIZATION= 0.999
POLARIZATION_PLANE_NORMAL= 0.000 1.000 0.000
NAME_TEMPLATE_OF_DATA_FRAMES= %s
TRUSTED_REGION= 0.0 1.41
UNTRUSTED_RECTANGLE= 487 495 0 2528
UNTRUSTED_RECTANGLE= 981 989 0 2528
UNTRUSTED_RECTANGLE= 1475 1483 0 2528
UNTRUSTED_RECTANGLE= 1969 1977 0 2528
UNTRUSTED_RECTANGLE= 0 2464 195 213
UNTRUSTED_RECTANGLE= 0 2464 407 425
UNTRUSTED_RECTANGLE= 0 2464 619 637
UNTRUSTED_RECTANGLE= 0 2464 831 849
UNTRUSTED_RECTANGLE= 0 2464 1043 1061
UNTRUSTED_RECTANGLE= 0 2464 1255 1273
UNTRUSTED_RECTANGLE= 0 2464 1467 1485
UNTRUSTED_RECTANGLE= 0 2464 1679 1697
UNTRUSTED_RECTANGLE= 0 2464 1891 1909
UNTRUSTED_RECTANGLE= 0 2464 2103 2121
UNTRUSTED_RECTANGLE= 0 2464 2315 2333
DATA_RANGE= 1 9
JOB=XYCORR INIT COLSPOT IDXREF DEFPIX INTEGRATE CORRECT\
""" % os.path.join(
dials_regression, "centroid_test_data", "centroid_????.cbf"
)
assert s1 == expected
real_space_a = (-5.327642, -39.034747, -4.988286)
real_space_b = (-35.253495, 7.596265, -22.127661)
real_space_c = (-22.673623, -1.486119, 35.793463)
s2 = to_xds.xparm_xds(real_space_a, real_space_b, real_space_c, space_group=1)
# run coordinate frame converter on xparm.xds as a sanity check
with open("xparm.xds", mode="wb") as fh:
fh.write(s2.encode("ASCII"))
from rstbx.cftbx import coordinate_frame_helpers
converter = coordinate_frame_helpers.import_xds_xparm("xparm.xds")
detector = sweep.get_detector()
goniometer = sweep.get_goniometer()
beam = sweep.get_beam()
assert approx_equal(real_space_a, converter.get_real_space_a())
assert approx_equal(real_space_b, converter.get_real_space_b())
assert approx_equal(real_space_c, converter.get_real_space_c())
assert approx_equal(goniometer.get_rotation_axis(), converter.get_rotation_axis())
assert approx_equal(beam.get_direction(), converter.get_sample_to_source().elems)
assert approx_equal(detector[0].get_fast_axis(), converter.get_detector_fast())
assert approx_equal(detector[0].get_slow_axis(), converter.get_detector_slow())
assert approx_equal(detector[0].get_origin(), converter.get_detector_origin())
def imageset_to_xds(imageset, synchrotron = None, refined_beam_vector = None,
refined_rotation_axis = None, refined_distance = None):
'''A function to take an input header dictionary from Diffdump
and generate a list of records to start XDS - see Doc/INP.txt.'''
# decide if we are at a synchrotron if we don't know already...
# that is, the wavelength is around either the Copper or Chromium
# K-alpha edge and this is an image plate.
beam = imageset.get_beam()
from dxtbx.serialize.xds import to_xds, xds_detector_name
converter = to_xds(imageset)
detector_class_is_square = {
'adsc q4':True,
'adsc q4 2x2 binned':True,
'adsc q210':True,
'adsc q210 2x2 binned':True,
'adsc q270':True,
'adsc q270 2x2 binned':True,
'adsc q315':True,
'adsc q315 2x2 binned':True,
'adsc HF4M':True,
'holton fake 01':True,
'mar 345':False,
'mar 180':False,
'mar 240':False,
'mar 300 ccd':True,
'mar 325 ccd':True,
'mar 225 ccd':True,
'mar ccd 225 hs':True,
'rayonix ccd 165':False,
'rayonix ccd 135':False,
'rayonix ccd 300':True,
'rayonix ccd 325':True,
'rayonix ccd 225':True,
'rayonix ccd 225 hs':True,
'rayonix ccd 300 hs':True,
'mar 165 ccd':False,
'mar 135 ccd':False,
'pilatus 12M':True,
'pilatus 6M':True,
'pilatus 2M':True,
'pilatus 1M':True,
'pilatus 200K':True,
'pilatus 300K':True,
'eiger 4M':True,
'eiger 9M':True,
'eiger 16M':True,
'rigaku saturn 92 2x2 binned':True,
'rigaku saturn 944 2x2 binned':True,
'rigaku saturn 724 2x2 binned':True,
'rigaku saturn 92':True,
'rigaku saturn 944':True,
'rigaku saturn 724':True,
'rigaku saturn a200':True,
'raxis IV':True,
'NOIR1':True}
sensor = converter.get_detector()[0].get_type()
fast, slow = converter.detector_size
f, s = converter.pixel_size
df = int(1000 * f)
ds = int(1000 * s)
# FIXME probably need to rotate by pi about the X axis
result = []
from dxtbx.model.detector_helpers_types import detector_helpers_types
detector = xds_detector_name(
detector_helpers_types.get(sensor, fast, slow, df, ds))
trusted = converter.get_detector()[0].get_trusted_range()
# FIXME what follows below should perhaps be 0 for the really weak
# pilatus data sets?
result.append('DETECTOR=%s MINIMUM_VALID_PIXEL_VALUE=%d OVERLOAD=%d' %
(detector, trusted[0] + 1, trusted[1]))
result.append('DIRECTION_OF_DETECTOR_X-AXIS=%f %f %f' %
converter.detector_x_axis)
result.append('DIRECTION_OF_DETECTOR_Y-AXIS=%f %f %f' %
converter.detector_y_axis)
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
if params.xds.trusted_region:
result.append(
'TRUSTED_REGION= %.2f %.2f' % tuple(params.xds.trusted_region))
elif detector_class_is_square[
detector_helpers_types.get(sensor, fast, slow, df, ds).replace('-', ' ')]:
result.append('TRUSTED_REGION=0.0 1.41')
else:
result.append('TRUSTED_REGION=0.0 0.99')
result.append('NX=%d NY=%d QX=%.4f QY=%.4f' % (fast, slow, f, s))
# RAXIS detectors have the distance written negative - why????
# this is ONLY for XDS - SATURN are the same - probably left handed
# goniometer rotation on rigaku X-ray sets.
if refined_distance:
result.append('DETECTOR_DISTANCE=%7.3f' % refined_distance)
else:
result.append('DETECTOR_DISTANCE=%7.3f' % converter.detector_distance)
result.append('OSCILLATION_RANGE=%4.2f' % converter.oscillation_range)
result.append('X-RAY_WAVELENGTH=%8.6f' % converter.wavelength)
# if user specified reversephi and this was not picked up in the
# format class reverse phi: n.b. double-negative warning!
if refined_rotation_axis:
result.append('ROTATION_AXIS= %f %f %f' % \
refined_rotation_axis)
else:
result.append('ROTATION_AXIS= %.3f %.3f %.3f' % \
converter.rotation_axis)
if refined_beam_vector:
result.append('INCIDENT_BEAM_DIRECTION=%f %f %f' % \
refined_beam_vector)
else:
result.append(
'INCIDENT_BEAM_DIRECTION= %.3f %.3f %.3f' % converter.beam_vector)
if hasattr(beam, "get_polarization_fraction"):
R = converter.imagecif_to_xds_transformation_matrix
result.append('FRACTION_OF_POLARIZATION= %.3f' %
beam.get_polarization_fraction())
result.append('POLARIZATION_PLANE_NORMAL= %.3f %.3f %.3f' %
(R * matrix.col(beam.get_polarization_normal())).elems)
# 24/NOV/14 XDS determines the air absorption automatically
# based on wavelength. May be useful to override this for in vacuo exps
# result.append('AIR=0.001')
if detector == 'PILATUS':
try:
thickness = converter.get_detector()[0].get_thickness()
if not thickness:
thickness = 0.32
Debug.write('Could not determine sensor thickness. Assuming default PILATUS 0.32mm')
except e:
thickness = 0.32
Debug.write('Error occured during sensor thickness determination. Assuming default PILATUS 0.32mm')
result.append('SENSOR_THICKNESS=%f' % thickness)
# # FIXME: Sensor absorption coefficient calculation probably requires a more general solution
# if converter.get_detector()[0].get_material() == 'CdTe':
# print "CdTe detector detected. Beam wavelength is %8.6f Angstrom" % converter.wavelength
if len(converter.panel_x_axis) > 1:
for panel_id in range(len(converter.panel_x_axis)):
result.append('')
result.append('!')
result.append('! SEGMENT %d' %(panel_id+1))
result.append('!')
result.append('SEGMENT= %d %d %d %d' % converter.panel_limits[panel_id])
result.append('DIRECTION_OF_SEGMENT_X-AXIS= %.3f %.3f %.3f' % \
converter.panel_x_axis[panel_id])
result.append('DIRECTION_OF_SEGMENT_Y-AXIS= %.3f %.3f %.3f' % \
converter.panel_y_axis[panel_id])
result.append('SEGMENT_DISTANCE= %.3f' % converter.panel_distance[panel_id])
result.append(
'SEGMENT_ORGX= %.1f SEGMENT_ORGY= %.1f' % converter.panel_origin[panel_id])
result.append('')
for f0, s0, f1, s1 in converter.get_detector()[0].get_mask():
result.append('UNTRUSTED_RECTANGLE= %d %d %d %d' %
(f0 - 1, f1 + 1, s0 - 1, s1 + 1))
if params.xds.untrusted_ellipse:
for untrusted_ellipse in params.xds.untrusted_ellipse:
result.append(
'UNTRUSTED_ELLIPSE= %d %d %d %d' % tuple(untrusted_ellipse))
Debug.write(result[-1])
if params.xds.untrusted_rectangle:
for untrusted_rectangle in params.xds.untrusted_rectangle:
result.append(
'UNTRUSTED_RECTANGLE= %d %d %d %d' % tuple(untrusted_rectangle))
Debug.write(result[-1])
return result
def _index_prepare(self):
'''Prepare to do autoindexing - in XDS terms this will mean
calling xycorr, init and colspot on the input images.'''
# decide on images to work with
Debug.write('XDS INDEX PREPARE:')
Debug.write('Wavelength: %.6f' % self.get_wavelength())
Debug.write('Distance: %.2f' % self.get_distance())
if self._indxr_images == []:
_select_images_function = getattr(
self, '_index_select_images_%s' % (self._index_select_images))
wedges = _select_images_function()
for wedge in wedges:
self.add_indexer_image_wedge(wedge)
self.set_indexer_prepare_done(True)
all_images = self.get_matching_images()
first = min(all_images)
last = max(all_images)
# next start to process these - first xycorr
xycorr = self.Xycorr()
xycorr.set_data_range(first, last)
xycorr.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
xycorr.set_beam_centre(xds_beam_centre[0],
xds_beam_centre[1])
for block in self._indxr_images:
xycorr.add_spot_range(block[0], block[1])
# FIXME need to set the origin here
xycorr.run()
for file in ['X-CORRECTIONS.cbf',
'Y-CORRECTIONS.cbf']:
self._indxr_payload[file] = xycorr.get_output_data_file(file)
# next start to process these - then init
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
# find the region of the scan with the least predicted shadow
# to use for background determination in XDS INIT step
from dxtbx.serialize import dump
from dxtbx.datablock import DataBlock
imageset = self._indxr_imagesets[0]
xsweep = self._indxr_sweeps[0]
sweep_filename = os.path.join(
self.get_working_directory(), '%s_datablock.json' %xsweep.get_name())
dump.datablock(DataBlock([imageset]), sweep_filename)
from xia2.Wrappers.Dials.ShadowPlot import ShadowPlot
shadow_plot = ShadowPlot()
shadow_plot.set_working_directory(self.get_working_directory())
auto_logfiler(shadow_plot)
shadow_plot.set_sweep_filename(sweep_filename)
shadow_plot.set_json_filename(
os.path.join(
self.get_working_directory(),
'%s_shadow_plot.json' %shadow_plot.get_xpid()))
shadow_plot.run()
results = shadow_plot.get_results()
from scitbx.array_family import flex
fraction_shadowed = flex.double(results['fraction_shadowed'])
scan_points = flex.double(results['scan_points'])
phi_width = self.get_phi_width()
scan = imageset.get_scan()
oscillation_range = scan.get_oscillation_range()
oscillation = scan.get_oscillation()
bg_images = self._background_images
bg_range_deg = (scan.get_angle_from_image_index(bg_images[0]),
scan.get_angle_from_image_index(bg_images[1]))
bg_range_width = bg_range_deg[1] - bg_range_deg[0]
min_shadow = 100
best_bg_range = bg_range_deg
from libtbx.utils import frange
for bg_range_start in frange(flex.min(scan_points), flex.max(scan_points) - bg_range_width, step=oscillation[1]):
bg_range_deg = (bg_range_start, bg_range_start + bg_range_width)
sel = (scan_points >= bg_range_deg[0]) & (scan_points <= bg_range_deg[1])
mean_shadow = flex.mean(fraction_shadowed.select(sel))
if mean_shadow < min_shadow:
min_shadow = mean_shadow
best_bg_range = bg_range_deg
self._background_images = (
scan.get_image_index_from_angle(best_bg_range[0]),
scan.get_image_index_from_angle(best_bg_range[1]))
Debug.write('Setting background images: %s -> %s' %self._background_images)
init = self.Init()
for file in ['X-CORRECTIONS.cbf',
'Y-CORRECTIONS.cbf']:
init.set_input_data_file(file, self._indxr_payload[file])
init.set_data_range(first, last)
if self._background_images:
init.set_background_range(self._background_images[0],
self._background_images[1])
else:
init.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
for block in self._indxr_images:
init.add_spot_range(block[0], block[1])
init.run()
# at this stage, need to (perhaps) modify the BKGINIT.cbf image
# to mark out the back stop
if PhilIndex.params.general.backstop_mask:
Debug.write('Applying mask to BKGINIT.pck')
# copy the original file
cbf_old = os.path.join(init.get_working_directory(),
'BKGINIT.cbf')
cbf_save = os.path.join(init.get_working_directory(),
'BKGINIT.sav')
shutil.copyfile(cbf_old, cbf_save)
# modify the file to give the new mask
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask.apply_mask_xds(self.get_header(), cbf_save, cbf_old)
init.reload()
for file in ['BLANK.cbf',
'BKGINIT.cbf',
'GAIN.cbf']:
self._indxr_payload[file] = init.get_output_data_file(file)
if PhilIndex.params.xia2.settings.developmental.use_dials_spotfinder:
spotfinder = self.DialsSpotfinder()
for block in self._indxr_images:
spotfinder.add_spot_range(block[0], block[1])
spotfinder.run()
export = self.DialsExportSpotXDS()
export.set_input_data_file(
'reflections.pickle',
spotfinder.get_output_data_file('reflections.pickle'))
export.run()
for file in ['SPOT.XDS']:
self._indxr_payload[file] = export.get_output_data_file(file)
else:
# next start to process these - then colspot
colspot = self.Colspot()
for file in ['X-CORRECTIONS.cbf',
'Y-CORRECTIONS.cbf',
'BLANK.cbf',
'BKGINIT.cbf',
'GAIN.cbf']:
colspot.set_input_data_file(file, self._indxr_payload[file])
colspot.set_data_range(first, last)
colspot.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
for block in self._indxr_images:
colspot.add_spot_range(block[0], block[1])
colspot.run()
for file in ['SPOT.XDS']:
self._indxr_payload[file] = colspot.get_output_data_file(file)
# that should be everything prepared... all of the important
# files should be loaded into memory to be able to cope with
# integration happening somewhere else
return
def _index(self):
'''Actually do the autoindexing using the data prepared by the
previous method.'''
images_str = '%d to %d' % tuple(self._indxr_images[0])
for i in self._indxr_images[1:]:
images_str += ', %d to %d' % tuple(i)
cell_str = None
if self._indxr_input_cell:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
self._indxr_input_cell
# then this is a proper autoindexing run - describe this
# to the journal entry
dirname = self.get_directory()
Journal.block('autoindexing', self._indxr_sweep_name, 'XDS',
{'images':images_str,
'target cell':cell_str,
'target lattice':self._indxr_input_lattice,
'template':self.get_template(),
'directory':dirname})
idxref = self.Idxref()
self._index_remove_masked_regions()
for file in ['SPOT.XDS']:
idxref.set_input_data_file(file, self._indxr_payload[file])
# edit SPOT.XDS to remove reflections in untrusted regions of the detector
idxref.set_data_range(self._indxr_images[0][0],
self._indxr_images[0][1])
idxref.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
# set the phi start etc correctly
for block in self._indxr_images[:1]:
starting_frame = block[0]
starting_angle = self.get_scan().get_angle_from_image_index(starting_frame)
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
for block in self._indxr_images[1:]:
idxref.add_spot_range(block[0], block[1])
if self._indxr_user_input_lattice:
idxref.set_indexer_user_input_lattice(True)
if self._indxr_input_lattice and self._indxr_input_cell:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
idxref.set_indexer_input_cell(self._indxr_input_cell)
Debug.write('Set lattice: %s' % self._indxr_input_lattice)
Debug.write('Set cell: %f %f %f %f %f %f' % \
self._indxr_input_cell)
original_cell = self._indxr_input_cell
elif self._indxr_input_lattice:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
original_cell = None
else:
original_cell = None
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
idxref.set_beam_centre(xds_beam_centre[0],
xds_beam_centre[1])
# fixme need to check if the lattice, cell have been set already,
# and if they have, pass these in as input to the indexing job.
done = False
while not done:
try:
done = idxref.run()
# N.B. in here if the IDXREF step was being run in the first
# pass done is FALSE however there should be a refined
# P1 orientation matrix etc. available - so keep it!
except XDSException, e:
# inspect this - if we have complaints about not
# enough reflections indexed, and we have a target
# unit cell, and they are the same, well ignore it
if 'solution is inaccurate' in str(e):
Debug.write(
'XDS complains solution inaccurate - ignoring')
done = idxref.continue_from_error()
elif ('insufficient percentage (< 70%)' in str(e) or
'insufficient percentage (< 50%)' in str(e)) and \
original_cell:
done = idxref.continue_from_error()
lattice, cell, mosaic = \
idxref.get_indexing_solution()
# compare solutions FIXME should use xds_cell_deviation
check = PhilIndex.params.xia2.settings.xds_check_cell_deviation
for j in range(3):
# allow two percent variation in unit cell length
if math.fabs((cell[j] - original_cell[j]) / \
original_cell[j]) > 0.02 and check:
Debug.write('XDS unhappy and solution wrong')
raise e
# and two degree difference in angle
if math.fabs(cell[j + 3] - original_cell[j + 3]) \
> 2.0 and check:
Debug.write('XDS unhappy and solution wrong')
raise e
Debug.write('XDS unhappy but solution ok')
elif 'insufficient percentage (< 70%)' in str(e) or \
'insufficient percentage (< 50%)' in str(e):
Debug.write('XDS unhappy but solution probably ok')
done = idxref.continue_from_error()
else:
raise e
def imageset_to_xds(
imageset,
synchrotron=None,
refined_beam_vector=None,
refined_rotation_axis=None,
refined_distance=None,
):
"""A function to take an input header dictionary from Diffdump
and generate a list of records to start XDS - see Doc/INP.txt."""
# decide if we are at a synchrotron if we don't know already...
# that is, the wavelength is around either the Copper or Chromium
# K-alpha edge and this is an image plate.
beam = imageset.get_beam()
from dxtbx.serialize.xds import to_xds, xds_detector_name
converter = to_xds(imageset)
h5_names = ["h5", "nxs"]
if imageset.get_template().split(".")[-1] in h5_names:
if not check_xds_ok_with_h5():
raise RuntimeError("HDF5 input with no converter for XDS")
detector_class_is_square = {
"adsc q4": True,
"adsc q4 2x2 binned": True,
"adsc q210": True,
"adsc q210 2x2 binned": True,
"adsc q270": True,
"adsc q270 2x2 binned": True,
"adsc q315": True,
"adsc q315 2x2 binned": True,
"adsc HF4M": True,
"holton fake 01": True,
"unknown electron 57": True,
"mar 345": False,
"mar 180": False,
"mar 240": False,
"mar 300 ccd": True,
"mar 325 ccd": True,
"mar 225 ccd": True,
"mar ccd 225 hs": True,
"rayonix ccd 165": False,
"rayonix ccd 135": False,
"rayonix ccd 300": True,
"rayonix ccd 325": True,
"rayonix ccd 225": True,
"rayonix ccd 225 hs": True,
"rayonix ccd 300 hs": True,
"mar 165 ccd": False,
"mar 135 ccd": False,
"pilatus 12M": True,
"pilatus 6M": True,
"pilatus 2M": True,
"pilatus 1M": True,
"pilatus 200K": True,
"pilatus 300K": True,
"eiger 4M": True,
"eiger 9M": True,
"eiger 16M": True,
"rigaku saturn 92 2x2 binned": True,
"rigaku saturn 944 2x2 binned": True,
"rigaku saturn 724 2x2 binned": True,
"rigaku saturn 92": True,
"rigaku saturn 944": True,
"rigaku saturn 724": True,
"rigaku saturn a200": True,
"raxis IV": True,
"NOIR1": True,
}
sensor = converter.get_detector()[0].get_type()
fast, slow = converter.detector_size
f, s = converter.pixel_size
df = int(1000 * f)
ds = int(1000 * s)
# FIXME probably need to rotate by pi about the X axis
result = []
from dxtbx.model.detector_helpers_types import detector_helpers_types
detector = xds_detector_name(
detector_helpers_types.get(sensor, fast, slow, df, ds))
trusted = converter.get_detector()[0].get_trusted_range()
# if CCD; undo dxtbx pedestal offset, hard code minimum 1; else use trusted
# range verbatim (i.e. for PAD) (later in pipeline sensor is SENSOR_UNKNOWN
# so additional test)
if sensor == "SENSOR_CCD" or detector == "CCDCHESS":
trusted = 1, trusted[1] - trusted[0]
# XDS upset if we trust < 0 see #193
if trusted[0] < 0:
trusted = 0, trusted[1]
result.append("DETECTOR=%s MINIMUM_VALID_PIXEL_VALUE=%d OVERLOAD=%d" %
(detector, trusted[0], trusted[1]))
result.append("DIRECTION_OF_DETECTOR_X-AXIS=%f %f %f" %
converter.detector_x_axis)
result.append("DIRECTION_OF_DETECTOR_Y-AXIS=%f %f %f" %
converter.detector_y_axis)
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
if params.xds.trusted_region:
result.append("TRUSTED_REGION= %.2f %.2f" %
tuple(params.xds.trusted_region))
elif detector_class_is_square[detector_helpers_types.get(
sensor, fast, slow, df, ds).replace("-", " ")]:
result.append("TRUSTED_REGION=0.0 1.41")
else:
result.append("TRUSTED_REGION=0.0 0.99")
result.append("NX=%d NY=%d QX=%.4f QY=%.4f" % (fast, slow, f, s))
# RAXIS detectors have the distance written negative - why????
# this is ONLY for XDS - SATURN are the same - probably left handed
# goniometer rotation on rigaku X-ray sets.
if refined_distance:
result.append("DETECTOR_DISTANCE=%7.3f" % refined_distance)
else:
result.append("DETECTOR_DISTANCE=%7.3f" % converter.detector_distance)
result.append("OSCILLATION_RANGE=%4.2f" % converter.oscillation_range)
result.append("X-RAY_WAVELENGTH=%8.6f" % converter.wavelength)
# if user specified reversephi and this was not picked up in the
# format class reverse phi: n.b. double-negative warning!
if refined_rotation_axis:
result.append("ROTATION_AXIS= %f %f %f" % refined_rotation_axis)
else:
result.append("ROTATION_AXIS= %.3f %.3f %.3f" %
converter.rotation_axis)
if refined_beam_vector:
result.append("INCIDENT_BEAM_DIRECTION=%f %f %f" % refined_beam_vector)
else:
result.append("INCIDENT_BEAM_DIRECTION= %.3f %.3f %.3f" %
converter.beam_vector)
if hasattr(beam, "get_polarization_fraction"):
R = converter.imagecif_to_xds_transformation_matrix
result.append("FRACTION_OF_POLARIZATION= %.3f" %
beam.get_polarization_fraction())
result.append("POLARIZATION_PLANE_NORMAL= %.3f %.3f %.3f" %
(R * matrix.col(beam.get_polarization_normal())).elems)
# 24/NOV/14 XDS determines the air absorption automatically
# based on wavelength. May be useful to override this for in vacuo exps
# result.append('AIR=0.001')
if detector == "PILATUS":
try:
thickness = converter.get_detector()[0].get_thickness()
if not thickness:
thickness = 0.32
Debug.write(
"Could not determine sensor thickness. Assuming default PILATUS 0.32mm"
)
except e:
thickness = 0.32
Debug.write(
"Error occured during sensor thickness determination. Assuming default PILATUS 0.32mm"
)
result.append("SENSOR_THICKNESS=%f" % thickness)
# FIXME: Sensor absorption coefficient calculation probably requires a more general solution
# if converter.get_detector()[0].get_material() == 'CdTe':
# print "CdTe detector detected. Beam wavelength is %8.6f Angstrom" % converter.wavelength
if len(converter.panel_x_axis) > 1:
for panel_id in range(len(converter.panel_x_axis)):
result.append("")
result.append("!")
result.append("! SEGMENT %d" % (panel_id + 1))
result.append("!")
result.append("SEGMENT= %d %d %d %d" %
converter.panel_limits[panel_id])
result.append("DIRECTION_OF_SEGMENT_X-AXIS= %.3f %.3f %.3f" %
converter.panel_x_axis[panel_id])
result.append("DIRECTION_OF_SEGMENT_Y-AXIS= %.3f %.3f %.3f" %
converter.panel_y_axis[panel_id])
result.append("SEGMENT_DISTANCE= %.3f" %
converter.panel_distance[panel_id])
result.append("SEGMENT_ORGX= %.1f SEGMENT_ORGY= %.1f" %
converter.panel_origin[panel_id])
result.append("")
for f0, s0, f1, s1 in converter.get_detector()[0].get_mask():
result.append("UNTRUSTED_RECTANGLE= %d %d %d %d" %
(f0 - 1, f1 + 1, s0 - 1, s1 + 1))
if params.xds.untrusted_ellipse:
for untrusted_ellipse in params.xds.untrusted_ellipse:
result.append("UNTRUSTED_ELLIPSE= %d %d %d %d" %
tuple(untrusted_ellipse))
Debug.write(result[-1])
if params.xds.untrusted_rectangle:
for untrusted_rectangle in params.xds.untrusted_rectangle:
result.append("UNTRUSTED_RECTANGLE= %d %d %d %d" %
tuple(untrusted_rectangle))
Debug.write(result[-1])
return result
def _index_prepare(self):
'''Prepare to do autoindexing - in XDS terms this will mean
calling xycorr, init and colspot on the input images.'''
# decide on images to work with
Debug.write('XDS INDEX PREPARE:')
Debug.write('Wavelength: %.6f' % self.get_wavelength())
Debug.write('Distance: %.2f' % self.get_distance())
if self._indxr_images == []:
_select_images_function = getattr(
self, '_index_select_images_%s' % (self._index_select_images))
wedges = _select_images_function()
for wedge in wedges:
self.add_indexer_image_wedge(wedge)
self.set_indexer_prepare_done(True)
all_images = self.get_matching_images()
first = min(all_images)
last = max(all_images)
# next start to process these - first xycorr
xycorr = self.Xycorr()
xycorr.set_data_range(first, last)
xycorr.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
xycorr.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1])
for block in self._indxr_images:
xycorr.add_spot_range(block[0], block[1])
# FIXME need to set the origin here
xycorr.run()
for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']:
self._indxr_payload[file] = xycorr.get_output_data_file(file)
# next start to process these - then init
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
imageset = self._indxr_imagesets[0]
masker = imageset.masker().format_class(
imageset.paths()[0]).get_goniometer_shadow_masker()
if masker is None:
# disable dynamic_shadowing
PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
# find the region of the scan with the least predicted shadow
# to use for background determination in XDS INIT step
from dxtbx.serialize import dump
from dxtbx.datablock import DataBlock
imageset = self._indxr_imagesets[0]
xsweep = self._indxr_sweeps[0]
sweep_filename = os.path.join(
self.get_working_directory(),
'%s_datablock.json' % xsweep.get_name())
dump.datablock(DataBlock([imageset]), sweep_filename)
from xia2.Wrappers.Dials.ShadowPlot import ShadowPlot
shadow_plot = ShadowPlot()
shadow_plot.set_working_directory(self.get_working_directory())
auto_logfiler(shadow_plot)
shadow_plot.set_sweep_filename(sweep_filename)
shadow_plot.set_json_filename(
os.path.join(self.get_working_directory(),
'%s_shadow_plot.json' % shadow_plot.get_xpid()))
shadow_plot.run()
results = shadow_plot.get_results()
from scitbx.array_family import flex
fraction_shadowed = flex.double(results['fraction_shadowed'])
if flex.max(fraction_shadowed) == 0:
PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False
else:
scan_points = flex.double(results['scan_points'])
phi_width = self.get_phi_width()
scan = imageset.get_scan()
oscillation_range = scan.get_oscillation_range()
oscillation = scan.get_oscillation()
if self._background_images is not None:
bg_images = self._background_images
bg_range_deg = (scan.get_angle_from_image_index(
bg_images[0]),
scan.get_angle_from_image_index(
bg_images[1]))
bg_range_width = bg_range_deg[1] - bg_range_deg[0]
min_shadow = 100
best_bg_range = bg_range_deg
from libtbx.utils import frange
for bg_range_start in frange(flex.min(scan_points),
flex.max(scan_points) -
bg_range_width,
step=oscillation[1]):
bg_range_deg = (bg_range_start,
bg_range_start + bg_range_width)
sel = (scan_points >= bg_range_deg[0]) & (
scan_points <= bg_range_deg[1])
mean_shadow = flex.mean(fraction_shadowed.select(sel))
if mean_shadow < min_shadow:
min_shadow = mean_shadow
best_bg_range = bg_range_deg
self._background_images = (scan.get_image_index_from_angle(
best_bg_range[0]),
scan.get_image_index_from_angle(
best_bg_range[1]))
Debug.write('Setting background images: %s -> %s' %
self._background_images)
init = self.Init()
for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']:
init.set_input_data_file(file, self._indxr_payload[file])
init.set_data_range(first, last)
if self._background_images:
init.set_background_range(self._background_images[0],
self._background_images[1])
else:
init.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
for block in self._indxr_images:
init.add_spot_range(block[0], block[1])
init.run()
# at this stage, need to (perhaps) modify the BKGINIT.cbf image
# to mark out the back stop
if PhilIndex.params.general.backstop_mask:
Debug.write('Applying mask to BKGINIT.pck')
# copy the original file
cbf_old = os.path.join(init.get_working_directory(), 'BKGINIT.cbf')
cbf_save = os.path.join(init.get_working_directory(),
'BKGINIT.sav')
shutil.copyfile(cbf_old, cbf_save)
# modify the file to give the new mask
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask.apply_mask_xds(self.get_header(), cbf_save, cbf_old)
init.reload()
for file in ['BLANK.cbf', 'BKGINIT.cbf', 'GAIN.cbf']:
self._indxr_payload[file] = init.get_output_data_file(file)
if PhilIndex.params.xia2.settings.developmental.use_dials_spotfinder:
spotfinder = self.DialsSpotfinder()
for block in self._indxr_images:
spotfinder.add_spot_range(block[0], block[1])
spotfinder.run()
export = self.DialsExportSpotXDS()
export.set_input_data_file(
'reflections.pickle',
spotfinder.get_output_data_file('reflections.pickle'))
export.run()
for file in ['SPOT.XDS']:
self._indxr_payload[file] = export.get_output_data_file(file)
else:
# next start to process these - then colspot
colspot = self.Colspot()
for file in [
'X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf', 'BLANK.cbf',
'BKGINIT.cbf', 'GAIN.cbf'
]:
colspot.set_input_data_file(file, self._indxr_payload[file])
colspot.set_data_range(first, last)
colspot.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
for block in self._indxr_images:
colspot.add_spot_range(block[0], block[1])
colspot.run()
for file in ['SPOT.XDS']:
self._indxr_payload[file] = colspot.get_output_data_file(file)
# that should be everything prepared... all of the important
# files should be loaded into memory to be able to cope with
# integration happening somewhere else
return
def dump(experiments, reflections, directory):
"""Dump the files in XDS format"""
if len(experiments) > 0:
for i, experiment in enumerate(experiments):
suffix = ""
if len(experiments) > 1:
suffix = "_%i" % (i + 1)
sub_dir = f"{directory}{suffix}"
if not os.path.isdir(sub_dir):
os.makedirs(sub_dir)
# XXX imageset is getting the experimental geometry from the image files
# rather than the input models.expt file
imageset = experiment.imageset
imageset.set_detector(experiment.detector)
imageset.set_beam(experiment.beam)
imageset.set_goniometer(experiment.goniometer)
imageset.set_scan(experiment.scan)
if experiment.crystal is None:
space_group_number = None
real_space_a = None
real_space_b = None
real_space_c = None
job_card = "XYCORR INIT COLSPOT IDXREF DEFPIX INTEGRATE CORRECT"
else:
crystal_model = experiment.crystal
crystal_model = crystal_model.change_basis(
crystal_model.get_space_group()
.info()
.change_of_basis_op_to_reference_setting()
)
space_group_number = crystal_model.get_space_group().type().number()
A = matrix.sqr(crystal_model.get_A())
A_inv = A.inverse()
real_space_a = A_inv.elems[:3]
real_space_b = A_inv.elems[3:6]
real_space_c = A_inv.elems[6:9]
job_card = ("XYCORR INIT DEFPIX INTEGRATE CORRECT",)
to_xds = xds.to_xds(imageset)
xds_inp = os.path.join(sub_dir, "XDS.INP")
xparm_xds = os.path.join(sub_dir, "XPARM.XDS")
logger.info("Exporting experiment to %s", xds_inp)
with open(xds_inp, "w") as f:
f.write(
to_xds.XDS_INP(
space_group_number=space_group_number,
real_space_a=real_space_a,
real_space_b=real_space_b,
real_space_c=real_space_c,
job_card=job_card,
)
)
if space_group_number:
logger.info("Exporting crystal model to %s", xparm_xds)
with open(xparm_xds, "w") as f:
f.write(
to_xds.xparm_xds(
real_space_a, real_space_b, real_space_c, space_group_number
)
)
if reflections is not None and len(reflections) > 0:
ref_cryst = reflections.select(reflections["id"] == i)
export_spot_xds(ref_cryst, os.path.join(sub_dir, "SPOT.XDS"))
else:
if not os.path.isdir(directory):
os.makedirs(directory)
export_spot_xds(reflections, os.path.join(directory, "SPOT.XDS"))
# Print out the contents of the dxtbx understanding of a bunch of images to
# an example XDS.INP file. This should illustrate the usage of the dxtbx
# classes.
import sys
from dxtbx.serialize import xds
def run(file_names):
if len(file_names) == 1 and file_names[0].endswith('json'):
from dxtbx.serialize import load
try:
datablock = load.datablock(file_names[0])
assert len(datablock) == 1
sweep = datablock[0].extract_sweeps()[0]
except ValueError, e:
if str(e) == '"__id__" does not equal "imageset"':
experiments = load.experiment_list(file_names[0])
assert len(experiments) == 1
sweep = experiments[0].imageset
else:
raise
else:
from dxtbx.imageset import ImageSetFactory
sweep = ImageSetFactory.new(file_names)[0]
xsx = xds.to_xds(sweep)
xsx.XDS_INP()
if __name__ == '__main__':
run(sys.argv[1:])
# Print out the contents of the dxtbx understanding of a bunch of images to
# an example XDS.INP file. This should illustrate the usage of the dxtbx
# classes.
import sys
from dxtbx.serialize import xds
def run(file_names):
if len(file_names) == 1 and file_names[0].endswith('json'):
from dxtbx.serialize import load
try:
datablock = load.datablock(file_names[0])
assert len(datablock) == 1
sweep = datablock[0].extract_sweeps()[0]
except ValueError, e:
if str(e) == '"__id__" does not equal "imageset"':
experiments = load.experiment_list(file_names[0])
assert len(experiments) == 1
sweep = experiments[0].imageset
else:
raise
else:
from dxtbx.imageset import ImageSetFactory
sweep = ImageSetFactory.new(file_names)[0]
xsx = xds.to_xds(sweep)
xsx.XDS_INP()
if __name__ == '__main__':
run(sys.argv[1:])
