# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [2., 2., 2.])]
xlo_param.set_param_vals(p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = mycrystal.get_unit_cell().parameters()
cell_params = [a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0,
0.0, 0.1])]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(mycrystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
#############################
# Generate some reflections #
#############################
print "Reflections will be generated with the following geometry:"
class ModelState(object):
"""
A class to keep track of the model state
"""
def __init__(
self,
experiment,
mosaicity_parameterisation,
wavelength_parameterisation=None,
fix_mosaic_spread=False,
fix_wavelength_spread=True,
fix_unit_cell=False,
fix_orientation=False,
):
"""
Initialise the model state
"""
# Save the crystal model
self.experiment = experiment
self.crystal = experiment.crystal
# The U and P parameterisation
self._U_parameterisation = CrystalOrientationParameterisation(
self.crystal)
self._B_parameterisation = CrystalUnitCellParameterisation(
self.crystal)
# The M and L parameterisations
self._M_parameterisation = mosaicity_parameterisation
self._L_parameterisation = wavelength_parameterisation
# Set the flags to fix parameters
self._is_mosaic_spread_fixed = fix_mosaic_spread
self._is_wavelength_spread_fixed = fix_wavelength_spread
self._is_unit_cell_fixed = fix_unit_cell
self._is_orientation_fixed = fix_orientation
# Check wavelength parameterisation
if not self.is_wavelength_spread_fixed:
assert self._L_parameterisation is not None
@property
def is_orientation_fixed(self) -> bool:
return self._is_orientation_fixed
@property
def is_unit_cell_fixed(self) -> bool:
return self._is_unit_cell_fixed
@property
def is_mosaic_spread_fixed(self) -> bool:
return self._is_mosaic_spread_fixed
@property
def is_mosaic_spread_angular(self) -> bool:
return self._M_parameterisation.is_angular()
@property
def is_wavelength_spread_fixed(self) -> bool:
return self._is_wavelength_spread_fixed
@property
def unit_cell(self):
return self.crystal.get_unit_cell()
@property
def U_matrix(self) -> np.array:
return np.array([self.crystal.get_U()], dtype=np.float64).reshape(3, 3)
@property
def B_matrix(self) -> np.array:
return np.array([self.crystal.get_B()], dtype=np.float64).reshape(3, 3)
@property
def A_matrix(self) -> np.array:
return np.array([self.crystal.get_A()], dtype=np.float64).reshape(3, 3)
@property
def mosaicity_covariance_matrix(self) -> np.array:
return self._M_parameterisation.sigma()
@property
def wavelength_spread(self) -> flex.double:
if self._L_parameterisation is not None:
return self._L_parameterisation.sigma()
return flex.double()
@property
def U_params(self) -> np.array:
"""Get the parameters of the orientation parameterisation"""
return np.array(self._U_parameterisation.get_param_vals(),
dtype=np.float64)
@U_params.setter
def U_params(self, params) -> None:
self._U_parameterisation.set_param_vals(tuple(
float(i) for i in params))
@property
def B_params(self) -> np.array:
"""Get the parameters of the orientation parameterisation"""
return np.array(self._B_parameterisation.get_param_vals(),
dtype=np.float64)
@B_params.setter
def B_params(self, params) -> None:
self._B_parameterisation.set_param_vals(tuple(
float(i) for i in params))
@property
def M_params(self) -> np.array:
"Parameters of the mosaicity parameterisation"
return self._M_parameterisation.parameters
@M_params.setter
def M_params(self, params) -> None:
self._M_parameterisation.parameters = params
@property
def L_params(self) -> np.array:
"Parameters of the Lambda (wavelength) parameterisation"
if self._L_parameterisation is not None:
return np.array(self._L_parameterisation.parameters,
dtype=np.float64)
return np.array([])
@L_params.setter
def L_params(self, params: flex.double) -> None:
if self._L_parameterisation is not None:
self._L_parameterisation.parameters = params
@property
def dU_dp(self) -> np.array:
"""
Get the first derivatives of U w.r.t its parameters
"""
ds_dp = self._U_parameterisation.get_ds_dp()
n = len(ds_dp)
s = np.array([ds_dp[i] for i in range(n)],
dtype=np.float64).reshape(n, 3, 3)
return s
@property
def dB_dp(self) -> np.array:
"""
Get the first derivatives of B w.r.t its parameters
"""
ds_dp = self._B_parameterisation.get_ds_dp()
n = len(ds_dp)
s = np.array([ds_dp[i] for i in range(n)],
dtype=np.float64).reshape(n, 3, 3)
return s
@property
def dM_dp(self) -> np.array:
"""
Get the first derivatives of M w.r.t its parameters
"""
return self._M_parameterisation.first_derivatives()
@property
def dL_dp(self) -> flex.double:
"""
Get the first derivatives of L w.r.t its parameters
"""
if self._L_parameterisation is not None:
return self._L_parameterisation.first_derivatives()
return flex.double()
@property
def active_parameters(self) -> np.array:
"""
The active parameters in order: U, B, M, L, W
"""
active_params = []
if not self.is_orientation_fixed:
active_params.append(self.U_params)
if not self.is_unit_cell_fixed:
active_params.append(self.B_params)
if not self.is_mosaic_spread_fixed:
active_params.append(self.M_params)
if not self.is_wavelength_spread_fixed:
active_params.append(self.L_params)
active_params = np.concatenate(active_params)
assert len(active_params) > 0
return active_params
@active_parameters.setter
def active_parameters(self, params) -> None:
"""
Set the active parameters in order: U, B, M, L, W
"""
if not self.is_orientation_fixed:
n_U_params = len(self.U_params)
temp = params[:n_U_params]
params = params[n_U_params:]
self.U_params = temp
if not self.is_unit_cell_fixed:
n_B_params = len(self.B_params)
temp = params[:n_B_params]
params = params[n_B_params:]
self.B_params = temp
if not self.is_mosaic_spread_fixed:
n_M_params = self.M_params.size
temp = params[:n_M_params]
params = params[n_M_params:]
self.M_params = np.array(temp)
if not self.is_wavelength_spread_fixed:
n_L_params = self.L_params.size
temp = params[:n_L_params]
self.L_params = temp
@property
def parameter_labels(self) -> List[str]:
"""
Get the parameter labels
"""
labels = []
if not self.is_orientation_fixed:
labels += [f"Crystal_U_{i}" for i in range(self.U_params.size)]
if not self.is_unit_cell_fixed:
labels += [f"Crystal_B_{i}" for i in range(self.B_params.size)]
if not self.is_mosaic_spread_fixed:
labels += [f"Mosaicity_{i}" for i in range(self.M_params.size)]
if not self.is_wavelength_spread_fixed:
labels.append("Wavelength_Spread")
assert len(labels) > 0
return labels
def test_refinement():
'''Test a refinement run'''
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression",
test=os.path.isdir)
# Get a beam and detector from a datablock. This one has a CS-PAD, but that
# is irrelevant
data_dir = os.path.join(dials_regression, "refinement_test_data",
"hierarchy_test")
datablock_path = os.path.join(data_dir, "datablock.json")
assert os.path.exists(datablock_path)
# load models
from dxtbx.datablock import DataBlockFactory
datablock = DataBlockFactory.from_serialized_format(datablock_path, check_format=False)
im_set = datablock[0].extract_imagesets()[0]
from copy import deepcopy
detector = deepcopy(im_set.get_detector())
beam = im_set.get_beam()
# Invent a crystal, goniometer and scan for this test
from dxtbx.model.crystal import crystal_model
crystal = crystal_model((40.,0.,0.) ,(0.,40.,0.), (0.,0.,40.),
space_group_symbol = "P1")
orig_xl = deepcopy(crystal)
from dxtbx.model.experiment import goniometer_factory
goniometer = goniometer_factory.known_axis((1., 0., 0.))
# Build a mock scan for a 180 degree sweep
from dxtbx.model.scan import scan_factory
sf = scan_factory()
scan = sf.make_scan(image_range = (1,1800),
exposure_times = 0.1,
oscillation = (0, 0.1),
epochs = range(1800),
deg = True)
sweep_range = scan.get_oscillation_range(deg=False)
im_width = scan.get_oscillation(deg=False)[1]
assert sweep_range == (0., pi)
assert approx_equal(im_width, 0.1 * pi / 180.)
from dxtbx.model.experiment.experiment_list import ExperimentList, Experiment
# Build an experiment list
experiments = ExperimentList()
experiments.append(Experiment(
beam=beam, detector=detector, goniometer=goniometer,
scan=scan, crystal=crystal, imageset=None))
# simulate some reflections
refs, _ = generate_reflections(experiments)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# alpha and beta angles)
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalUnitCellParameterisation
xluc_param = CrystalUnitCellParameterisation(crystal)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = crystal.get_unit_cell().parameters()
cell_params = [a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1,
-0.1, 0.0])]
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import \
symmetrize_reduce_enlarge
from scitbx import matrix
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(crystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
# reparameterise the crystal at the perturbed geometry
xluc_param = CrystalUnitCellParameterisation(crystal)
# Dummy parameterisations for other models
beam_param = None
xlo_param = None
det_param = None
# parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.parameter_report import \
ParameterReporter
pred_param = TwoThetaPredictionParameterisation(experiments,
det_param, beam_param, xlo_param, [xluc_param])
param_reporter = ParameterReporter(det_param, beam_param,
xlo_param, [xluc_param])
# reflection manager
refman = TwoThetaReflectionManager(refs, experiments, nref_per_degree=20,
verbosity=2)
# reflection predictor
ref_predictor = TwoThetaExperimentsPredictor(experiments)
# target function
target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param)
# minimisation engine
from dials.algorithms.refinement.engine \
import LevenbergMarquardtIterations as Refinery
refinery = Refinery(target = target,
prediction_parameterisation = pred_param,
log = None,
verbosity = 0,
track_step = False,
track_gradient = False,
track_parameter_correlation = False,
max_iterations = 20)
# Refiner
from dials.algorithms.refinement.refiner import Refiner
refiner = Refiner(reflections=refs,
experiments=experiments,
pred_param=pred_param,
param_reporter=param_reporter,
refman=refman,
target=target,
refinery=refinery,
verbosity=1)
history = refiner.run()
# compare crystal with original crystal
refined_xl = refiner.get_experiments()[0].crystal
#print refined_xl
assert refined_xl.is_similar_to(orig_xl, uc_rel_length_tolerance=0.001,
uc_abs_angle_tolerance=0.01)
#print "Unit cell esds:"
#print refined_xl.get_cell_parameter_sd()
return
# shift beam by 4 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 4.
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=3 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [3., 3., 3.])]
xlo_param.set_param_vals(p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# alpha and beta angles)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = mycrystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1, -0.1, 0.0])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(mycrystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
#############################
# Generate some reflections #
#############################
def test():
# Python and cctbx imports
from math import pi
from scitbx import matrix
from scitbx.array_family import flex
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
# Get modules to build models and minimiser using PHIL
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
import dials.test.algorithms.refinement.setup_minimiser as setup_minimiser
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Model parameterisations
from dials.algorithms.refinement.parameterisation.detector_parameters import (
DetectorParameterisationSinglePanel, )
from dials.algorithms.refinement.parameterisation.beam_parameters import (
BeamParameterisation, )
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection
from dials.algorithms.refinement.prediction.managed_predictors import (
ScansRayPredictor,
ScansExperimentsPredictor,
)
from cctbx.sgtbx import space_group, space_group_symbols
# Parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import (
XYPhiPredictionParameterisation, )
# Imports for the target function
from dials.algorithms.refinement.target import (
LeastSquaresPositionalResidualWithRmsdCutoff, )
from dials.algorithms.refinement.reflection_manager import ReflectionManager
#############################
# Setup experimental models #
#############################
override = """geometry.parameters
{
beam.wavelength.random=False
beam.wavelength.value=1.0
beam.direction.inclination.random=False
crystal.a.length.random=False
crystal.a.length.value=12.0
crystal.a.direction.method=exactly
crystal.a.direction.exactly.direction=1.0 0.002 -0.004
crystal.b.length.random=False
crystal.b.length.value=14.0
crystal.b.direction.method=exactly
crystal.b.direction.exactly.direction=-0.002 1.0 0.002
crystal.c.length.random=False
crystal.c.length.value=13.0
crystal.c.direction.method=exactly
crystal.c.direction.exactly.direction=0.002 -0.004 1.0
detector.directions.method=exactly
detector.directions.exactly.dir1=0.99 0.002 -0.004
detector.directions.exactly.norm=0.002 -0.001 0.99
detector.centre.method=exactly
detector.centre.exactly.value=1.0 -0.5 199.0
}"""
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
models = setup_geometry.Extract(master_phil,
local_overrides=override,
verbose=False)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
# Build a mock scan for a 180 degree sweep
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(1800)),
deg=True,
)
# Build an ExperimentList
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
# Create the PredictionParameterisation
pred_param = XYPhiPredictionParameterisation(experiments, [det_param],
[s0_param], [xlo_param],
[xluc_param])
################################
# Apply known parameter shifts #
################################
# shift detector by 1.0 mm each translation and 4 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [
a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 4.0, 4.0, 4.0])
]
det_param.set_param_vals(p_vals)
# shift beam by 4 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 4.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=3 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [3.0, 3.0, 3.0])]
xlo_param.set_param_vals(p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# alpha and beta angles)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = mycrystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1, -0.1, 0.0])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(mycrystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sweep_range == (0.0, pi)
assert approx_equal(im_width, 0.1 * pi / 180.0)
# Predict rays within the sweep range
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs = ray_predictor(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ScansExperimentsPredictor(experiments)
obs_refs["id"] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2)
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# The total number of observations should be 1128
assert len(obs_refs) == 1128
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(obs_refs,
experiments,
outlier_detector=None,
close_to_spindle_cutoff=0.1)
##############################
# Set up the target function #
##############################
# The current 'achieved' criterion compares RMSD against 1/3 the pixel size and
# 1/3 the image width in radians. For the simulated data, these are just made up
mytarget = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments,
ref_predictor,
refman,
pred_param,
restraints_parameterisation=None)
######################################
# Set up the LSTBX refinement engine #
######################################
overrides = """minimiser.parameters.engine=GaussNewton
minimiser.parameters.logfile=None"""
refiner = setup_minimiser.Extract(master_phil,
mytarget,
pred_param,
local_overrides=overrides).refiner
refiner.run()
assert mytarget.achieved()
assert refiner.get_num_steps() == 1
assert approx_equal(
mytarget.rmsds(),
(0.00508252354876, 0.00420954552156, 8.97303428289e-05))
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
######################################################
# Set up the LBFGS with curvatures refinement engine #
######################################################
overrides = """minimiser.parameters.engine=LBFGScurvs
minimiser.parameters.logfile=None"""
refiner = setup_minimiser.Extract(master_phil,
mytarget,
pred_param,
local_overrides=overrides).refiner
refiner.run()
assert mytarget.achieved()
assert refiner.get_num_steps() == 9
assert approx_equal(mytarget.rmsds(),
(0.0558857700305, 0.0333446685335, 0.000347402754278))
def test(args=[]):
#############################
# Setup experimental models #
#############################
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
models = setup_geometry.Extract(master_phil, cmdline_args=args)
single_panel_detector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Make a 3x3 multi panel detector filling the same space as the existing
# single panel detector. Each panel of the multi-panel detector has pixels with
# 1/3 the length dimensions of the single panel.
multi_panel_detector = Detector()
for x in range(3):
for y in range(3):
new_panel = make_panel_in_array((x, y), single_panel_detector[0])
multi_panel_detector.add_panel(new_panel)
# Build a mock scan for a 180 degree sweep
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(1800)),
deg=True,
)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sweep_range == (0.0, pi)
assert approx_equal(im_width, 0.1 * pi / 180.0)
# Build ExperimentLists
experiments_single_panel = ExperimentList()
experiments_multi_panel = ExperimentList()
experiments_single_panel.append(
Experiment(
beam=mybeam,
detector=single_panel_detector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
experiments_multi_panel.append(
Experiment(
beam=mybeam,
detector=multi_panel_detector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(single_panel_detector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
multi_det_param = DetectorParameterisationMultiPanel(
multi_panel_detector, mybeam)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Fix crystal parameters
# xluc_param.set_fixed([True, True, True, True, True, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
pred_param = XYPhiPredictionParameterisation(experiments_single_panel,
[det_param], [s0_param],
[xlo_param], [xluc_param])
pred_param2 = XYPhiPredictionParameterisation(
experiments_multi_panel,
[multi_det_param],
[s0_param],
[xlo_param],
[xluc_param],
)
################################
# Apply known parameter shifts #
################################
# shift detectors by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [
a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])
]
det_param.set_param_vals(p_vals)
multi_det_p_vals = multi_det_param.get_param_vals()
p_vals = [
a + b for a, b in zip(multi_det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])
]
multi_det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])]
xlo_param.set_param_vals(p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = mycrystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(mycrystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# for the reflection predictor, it doesn't matter which experiment list is
# passed, as the detector is not used
ref_predictor = ScansRayPredictor(experiments_single_panel, sweep_range)
# get two sets of identical reflections
obs_refs = ref_predictor(indices)
obs_refs2 = ref_predictor(indices)
for r1, r2 in zip(obs_refs, obs_refs2):
assert r1["s1"] == r2["s1"]
# get the panel intersections
sel = ray_intersection(single_panel_detector, obs_refs)
obs_refs = obs_refs.select(sel)
sel = ray_intersection(multi_panel_detector, obs_refs2)
obs_refs2 = obs_refs2.select(sel)
assert len(obs_refs) == len(obs_refs2)
# Set 'observed' centroids from the predicted ones
obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"]
obs_refs2["xyzobs.mm.value"] = obs_refs2["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.0
px_size = single_panel_detector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2)
# set the variances and frame numbers
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
obs_refs2["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# Add in flags and ID columns by copying into standard reflection tables
tmp = flex.reflection_table.empty_standard(len(obs_refs))
tmp.update(obs_refs)
obs_refs = tmp
tmp = flex.reflection_table.empty_standard(len(obs_refs2))
tmp.update(obs_refs2)
obs_refs2 = tmp
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
multi_det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(obs_refs, experiments_single_panel)
refman2 = ReflectionManager(obs_refs, experiments_multi_panel)
###############################
# Set up the target functions #
###############################
mytarget = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments_single_panel,
ScansExperimentsPredictor(experiments_single_panel),
refman,
pred_param,
restraints_parameterisation=None,
)
mytarget2 = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments_multi_panel,
ScansExperimentsPredictor(experiments_multi_panel),
refman2,
pred_param2,
restraints_parameterisation=None,
)
#################################
# Set up the refinement engines #
#################################
refiner = setup_minimiser.Extract(master_phil,
mytarget,
pred_param,
cmdline_args=args).refiner
refiner2 = setup_minimiser.Extract(master_phil,
mytarget2,
pred_param2,
cmdline_args=args).refiner
refiner.run()
# reset parameters and run refinement with the multi panel detector
s0_param.set_param_vals(s0_p_vals)
multi_det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
refiner2.run()
# same number of steps
assert refiner.get_num_steps() == refiner2.get_num_steps()
# same rmsds
for rmsd, rmsd2 in zip(refiner.history["rmsd"], refiner2.history["rmsd"]):
assert approx_equal(rmsd, rmsd2)
# same parameter values each step
for params, params2 in zip(refiner.history["parameter_vector"],
refiner.history["parameter_vector"]):
assert approx_equal(params, params2)
def test(args=[]):
# Python and cctbx imports
from math import pi
from scitbx import matrix
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
# Import for surgery on reflection_tables
from dials.array_family import flex
# Get module to build models using PHIL
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Crystal parameterisations
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator
from dials.algorithms.refinement.prediction.managed_predictors import (
ScansRayPredictor,
StillsExperimentsPredictor,
)
from dials.algorithms.spot_prediction import ray_intersection
from cctbx.sgtbx import space_group, space_group_symbols
#############################
# Setup experimental models #
#############################
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
# build models, with a larger crystal than default in order to get enough
# reflections on the 'still' image
param = """
geometry.parameters.crystal.a.length.range=40 50;
geometry.parameters.crystal.b.length.range=40 50;
geometry.parameters.crystal.c.length.range=40 50;
geometry.parameters.random_seed = 42"""
models = setup_geometry.Extract(master_phil,
cmdline_args=args,
local_overrides=param)
crystal = models.crystal
mydetector = models.detector
mygonio = models.goniometer
mybeam = models.beam
# Build a mock scan for a 1.5 degree wedge. Only used for generating indices near
# the Ewald sphere
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1),
exposure_times=0.1,
oscillation=(0, 1.5),
epochs=list(range(1)),
deg=True,
)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert approx_equal(im_width, 1.5 * pi / 180.0)
# Build experiment lists
stills_experiments = ExperimentList()
stills_experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
crystal=crystal,
imageset=None))
scans_experiments = ExperimentList()
scans_experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
crystal=crystal,
goniometer=mygonio,
scan=myscan,
imageset=None,
))
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
################################
# Apply known parameter shifts #
################################
# rotate crystal (=5 mrad each rotation)
xlo_p_vals = []
p_vals = xlo_param.get_param_vals()
xlo_p_vals.append(p_vals)
new_p_vals = [a + b for a, b in zip(p_vals, [5.0, 5.0, 5.0])]
xlo_param.set_param_vals(new_p_vals)
# change unit cell (=1.0 Angstrom length upsets, 0.5 degree of
# gamma angle)
xluc_p_vals = []
p_vals = xluc_param.get_param_vals()
xluc_p_vals.append(p_vals)
cell_params = crystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [1.0, 1.0, -1.0, 0.0, 0.0, 0.5])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(crystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
# keep track of the target crystal model to compare with refined
from copy import deepcopy
target_crystal = deepcopy(crystal)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere for crystal
resolution = 2.0
index_generator = IndexGenerator(
crystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# Build a ray predictor and predict rays close to the Ewald sphere by using
# the narrow rotation scan
ref_predictor = ScansRayPredictor(scans_experiments, sweep_range)
obs_refs = ref_predictor(indices, experiment_id=0)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Add in flags and ID columns by copying into standard reflection table
tmp = flex.reflection_table.empty_standard(len(obs_refs))
tmp.update(obs_refs)
obs_refs = tmp
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2)
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# Re-predict using the stills reflection predictor
stills_ref_predictor = StillsExperimentsPredictor(stills_experiments)
obs_refs_stills = stills_ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs_stills["xyzobs.mm.value"] = obs_refs_stills["xyzcal.mm"]
###############################
# Undo known parameter shifts #
###############################
xlo_param.set_param_vals(xlo_p_vals[0])
xluc_param.set_param_vals(xluc_p_vals[0])
# make a refiner
from dials.algorithms.refinement.refiner import phil_scope
params = phil_scope.fetch(source=parse("")).extract()
# Change this to get a plot
do_plot = False
if do_plot:
params.refinement.refinery.journal.track_parameter_correlation = True
from dials.algorithms.refinement.refiner import RefinerFactory
# decrease bin_size_fraction to terminate on RMSD convergence
params.refinement.target.bin_size_fraction = 0.01
params.refinement.parameterisation.beam.fix = "all"
params.refinement.parameterisation.detector.fix = "all"
refiner = RefinerFactory.from_parameters_data_experiments(
params, obs_refs_stills, stills_experiments)
# run refinement
history = refiner.run()
# regression tests
assert len(history["rmsd"]) == 9
refined_crystal = refiner.get_experiments()[0].crystal
uc1 = refined_crystal.get_unit_cell()
uc2 = target_crystal.get_unit_cell()
assert uc1.is_similar_to(uc2)
if do_plot:
plt = refiner.parameter_correlation_plot(
len(history["parameter_correlation"]) - 1)
plt.show()
def test_refinement(dials_regression):
'''Test a refinement run'''
# Get a beam and detector from a datablock. This one has a CS-PAD, but that
# is irrelevant
data_dir = os.path.join(dials_regression, "refinement_test_data",
"hierarchy_test")
datablock_path = os.path.join(data_dir, "datablock.json")
assert os.path.exists(datablock_path)
# load models
from dxtbx.datablock import DataBlockFactory
datablock = DataBlockFactory.from_serialized_format(datablock_path, check_format=False)
im_set = datablock[0].extract_imagesets()[0]
detector = deepcopy(im_set.get_detector())
beam = im_set.get_beam()
# Invent a crystal, goniometer and scan for this test
from dxtbx.model import Crystal
crystal = Crystal((40.,0.,0.) ,(0.,40.,0.), (0.,0.,40.),
space_group_symbol = "P1")
orig_xl = deepcopy(crystal)
from dxtbx.model import GoniometerFactory
goniometer = GoniometerFactory.known_axis((1., 0., 0.))
# Build a mock scan for a 180 degree sweep
from dxtbx.model import ScanFactory
sf = ScanFactory()
scan = sf.make_scan(image_range = (1,1800),
exposure_times = 0.1,
oscillation = (0, 0.1),
epochs = range(1800),
deg = True)
sweep_range = scan.get_oscillation_range(deg=False)
im_width = scan.get_oscillation(deg=False)[1]
assert sweep_range == (0., pi)
assert approx_equal(im_width, 0.1 * pi / 180.)
# Build an experiment list
experiments = ExperimentList()
experiments.append(Experiment(
beam=beam, detector=detector, goniometer=goniometer,
scan=scan, crystal=crystal, imageset=None))
# simulate some reflections
refs, _ = generate_reflections(experiments)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# alpha and beta angles)
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalUnitCellParameterisation
xluc_param = CrystalUnitCellParameterisation(crystal)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = crystal.get_unit_cell().parameters()
cell_params = [a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1,
-0.1, 0.0])]
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import \
symmetrize_reduce_enlarge
from scitbx import matrix
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(crystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
# reparameterise the crystal at the perturbed geometry
xluc_param = CrystalUnitCellParameterisation(crystal)
# Dummy parameterisations for other models
beam_param = None
xlo_param = None
det_param = None
# parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.parameter_report import \
ParameterReporter
pred_param = TwoThetaPredictionParameterisation(experiments,
det_param, beam_param, xlo_param, [xluc_param])
param_reporter = ParameterReporter(det_param, beam_param,
xlo_param, [xluc_param])
# reflection manager
refman = TwoThetaReflectionManager(refs, experiments, nref_per_degree=20,
verbosity=2)
# reflection predictor
ref_predictor = TwoThetaExperimentsPredictor(experiments)
# target function
target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param)
# minimisation engine
from dials.algorithms.refinement.engine \
import LevenbergMarquardtIterations as Refinery
refinery = Refinery(target = target,
prediction_parameterisation = pred_param,
log = None,
verbosity = 0,
max_iterations = 20)
# Refiner
from dials.algorithms.refinement.refiner import Refiner
refiner = Refiner(reflections=refs,
experiments=experiments,
pred_param=pred_param,
param_reporter=param_reporter,
refman=refman,
target=target,
refinery=refinery,
verbosity=1)
history = refiner.run()
# compare crystal with original crystal
refined_xl = refiner.get_experiments()[0].crystal
#print refined_xl
assert refined_xl.is_similar_to(orig_xl, uc_rel_length_tolerance=0.001,
uc_abs_angle_tolerance=0.01)
class CrystalRefiner(object):
"""
A class to perform refinement of crystal parameters
"""
def __init__(self, experiment, reflections, mosaicity, params):
"""
Perform the refinement
:param experiment: The experiment
:param reflections: The reflections
:param mosaicity: The mosaicity
:param params: The parameters
"""
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation, )
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation, )
from dials_scratch.jmp.stills import Model
from dials.array_family import flex
from scitbx import simplex
from math import sqrt
# Store the input
self.experiment = experiment
self.reflections = reflections
self.mosaicity = mosaicity
self.params = params
# Get the data and image mask
data = self.experiment.imageset.get_raw_data(0)[0]
mask = self.experiment.imageset.get_mask(0)[0]
# Initialise the model
self.model = Model(
beam=self.experiment.beam,
detector=self.experiment.detector,
crystal=self.experiment.crystal,
reflections=self.reflections,
image_data=data,
image_mask=mask,
mosaicity=self.mosaicity,
bandpass=0.0,
foreground_limit=0.3,
background_limit=0.5,
num_samples=self.params.refinement.profile.num_samples,
)
# Get the crystal model and the parameterisation
self.crystal = self.experiment.crystal
self.cucp = CrystalUnitCellParameterisation(self.crystal)
self.cop = CrystalOrientationParameterisation(self.crystal)
# Get the current values and generate some offsets
values = flex.double(self.cucp.get_param_vals() +
self.cop.get_param_vals())
offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] +
[0.1, 0.1, 0.1])
# The optimization history
self.history = []
# Get the initial cell and initial score
initial_cell = self.crystal.get_unit_cell()
initial_score = self.target(values)
# Perform the optimization
optimizer = simple_simplex(values, offset, self, 2000)
result = optimizer.get_solution()
print("Initial cell:", initial_cell)
print("Final cell: ", self.crystal.get_unit_cell())
# Compute RMSD
xcal, ycal, _ = self.model.observed().parts()
xobs, yobs, _ = self.model.predicted().parts()
rmsd_x = sqrt(flex.sum((xcal - xobs)**2) / len(xcal))
rmsd_y = sqrt(flex.sum((ycal - yobs)**2) / len(ycal))
print("RMSD X, Y (px): %f, %f" % (rmsd_x, rmsd_y))
def target(self, vector):
"""
The target function
"""
from dials.array_family import flex
from math import sqrt
# Get the cell and orientation parameters
cell_parms = self.cucp.get_param_vals()
orientation_parms = self.cop.get_param_vals()
assert len(vector) == len(cell_parms) + len(orientation_parms)
# Update the cell and orientation parameters
tst_cell = vector[:len(cell_parms)]
tst_orientation = vector[len(cell_parms):len(cell_parms) +
len(orientation_parms)]
self.cucp.set_param_vals(tst_cell)
self.cop.set_param_vals(tst_orientation)
# Update the model
self.model.crystal = self.crystal
self.model.update(pixel_lookup=False)
# Get the observed and predicted position
T = self.model.success()
O = self.model.observed()
C = self.model.predicted()
# Select only those successes
selection = T == True
Xobs, Yobs, Zobs = O.select(selection).parts()
Xcal, Ycal, Zcal = C.select(selection).parts()
# Compute the rmsd between observed and calculated
score = flex.sum((Xobs - Xcal)**2 + (Yobs - Ycal)**2 +
(Zobs - Zcal)**2)
# Append to the history
self.history.append((tst_cell, tst_orientation, score))
# Print some info
print(
"Cell: %.3f %.3f %.3f %.3f %.3f %.3f; Phi: %.3f %.3f %.3f; RMSD: %.3f"
% (tuple(self.crystal.get_unit_cell().parameters()) +
tuple(tst_orientation) + tuple((sqrt(score / len(Xobs)), ))))
return score
def test(init_test):
single_panel_detector = init_test.experiments_single_panel.detectors()[0]
multi_panel_detector = init_test.experiments_multi_panel.detectors()[0]
beam = init_test.experiments_single_panel.beams()[0]
gonio = init_test.experiments_single_panel.goniometers()[0]
crystal = init_test.experiments_single_panel.crystals()[0]
# Parameterise the models
det_param = DetectorParameterisationSinglePanel(single_panel_detector)
s0_param = BeamParameterisation(beam, gonio)
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
multi_det_param = DetectorParameterisationMultiPanel(multi_panel_detector, beam)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Link model parameterisations together into a parameterisation of the
# prediction equation, first for the single panel detector
pred_param = XYPhiPredictionParameterisation(
init_test.experiments_single_panel,
[det_param],
[s0_param],
[xlo_param],
[xluc_param],
)
# ... and now for the multi-panel detector
pred_param2 = XYPhiPredictionParameterisation(
init_test.experiments_multi_panel,
[multi_det_param],
[s0_param],
[xlo_param],
[xluc_param],
)
################################
# Apply known parameter shifts #
################################
# shift detectors by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])]
det_param.set_param_vals(p_vals)
multi_det_p_vals = multi_det_param.get_param_vals()
p_vals = [a + b for a, b in zip(multi_det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])]
multi_det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])]
xlo_param.set_param_vals(p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = crystal.get_unit_cell().parameters()
cell_params = [a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(crystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
multi_det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(
init_test.observations_single_panel, init_test.experiments_single_panel
)
refman2 = ReflectionManager(
init_test.observations_multi_panel, init_test.experiments_multi_panel
)
###############################
# Set up the target functions #
###############################
target = LeastSquaresPositionalResidualWithRmsdCutoff(
init_test.experiments_single_panel,
ScansExperimentsPredictor(init_test.experiments_single_panel),
refman,
pred_param,
restraints_parameterisation=None,
)
target2 = LeastSquaresPositionalResidualWithRmsdCutoff(
init_test.experiments_multi_panel,
ScansExperimentsPredictor(init_test.experiments_multi_panel),
refman2,
pred_param2,
restraints_parameterisation=None,
)
#################################
# Set up the refinement engines #
#################################
refiner = setup_minimiser.Extract(master_phil, target, pred_param).refiner
refiner2 = setup_minimiser.Extract(master_phil, target2, pred_param2).refiner
refiner.run()
# reset parameters and run refinement with the multi panel detector
s0_param.set_param_vals(s0_p_vals)
multi_det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
refiner2.run()
# same number of steps
assert refiner.get_num_steps() == refiner2.get_num_steps()
# same rmsds
for rmsd, rmsd2 in zip(refiner.history["rmsd"], refiner2.history["rmsd"]):
assert approx_equal(rmsd, rmsd2)
# same parameter values each step
for params, params2 in zip(
refiner.history["parameter_vector"], refiner.history["parameter_vector"]
):
assert approx_equal(params, params2)
class Apple(object):
def __init__(self, reflection_file, experiment_file):
data = pickle.load(open(reflection_file, "rb"))
self.data = data.select(data["intensity.sum.variance"] > 0)
i = data["intensity.sum.value"]
v = data["intensity.sum.variance"]
s = flex.sqrt(v)
self.i_s = i / s
self.scale = 2
from dxtbx.model.experiment_list import ExperimentListFactory
expt = ExperimentListFactory.from_json_file(experiment_file)
panel = expt.detectors()[0][0]
crystal = expt.crystals()[0]
self.s0 = matrix.col(expt.beams()[0].get_s0())
wavelength = expt.beams()[0].get_wavelength()
# make a list of observed q positions
self.qobs = []
for j in range(data.size()):
x, y, z = data["xyzobs.px.value"][j]
p = matrix.col(panel.get_pixel_lab_coord((x, y)))
q = p.normalize() / wavelength - self.s0
self.qobs.append(q)
self.wavelength = wavelength
self.panel = panel
self.beam = expt.beams()[0]
self.crystal = crystal
# slurp data from $somewhere
imageset = expt.imagesets()[0]
self.raw_data = imageset.get_raw_data(0)[0]
self.imageset = imageset
return
def load(self, filename):
from dxtbx import load
i = load(filename)
self.raw_data = i.get_raw_data()
return
def refine(self, do_print=False):
crystal = self.crystal
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation,
CrystalOrientationParameterisation,
)
self.cucp = CrystalUnitCellParameterisation(crystal)
self.cop = CrystalOrientationParameterisation(crystal)
self.zero()
# 0-point and deltas
values = flex.double(self.cucp.get_param_vals() +
self.cop.get_param_vals())
offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] +
[0.1, 0.1, 0.1])
initial = crystal.get_unit_cell()
self.cells = []
self.best_score = 1e99
initial_score = self.target(values)
doohicky = simple_simplex(values, offset, self, 2000)
best = doohicky.get_solution()
if do_print:
print("Initial cell:", initial)
print("Final cell: ", crystal.get_unit_cell())
print("Score change", initial_score,
self.target(best, do_print=False))
self.best = best
def plot_map(self, map, filename):
import matplotlib
matplotlib.use("Agg")
from matplotlib import pyplot
data = map.as_numpy_array()
fig = pyplot.gcf()
pyplot.imshow(data, cmap="gray_r")
pyplot.colorbar()
pyplot.savefig(filename, dpi=400)
pyplot.clf()
return
def plot_log_map(self, map, filename):
import matplotlib
matplotlib.use("Agg")
from matplotlib import pyplot
negative = map.as_1d() <= 0
map.as_1d().set_selected(negative, 1)
logmap = flex.log10(map.as_double())
data = logmap.as_numpy_array()
fig = pyplot.gcf()
pyplot.imshow(data, cmap="gray_r")
pyplot.colorbar()
pyplot.savefig(filename, dpi=400)
pyplot.clf()
return
def render_distance(self):
distance_map = flex.double(flex.grid(self.raw_data.focus()))
origin = self.panel.get_origin()
fast = self.panel.get_fast_axis()
slow = self.panel.get_slow_axis()
nfast, nslow = self.panel.get_image_size()
UB = matrix.sqr(self.crystal.get_A())
UBi = UB.inverse()
from dials_scratch import q_map
distance_map = q_map(self.panel, self.beam, UB, 1)
return distance_map
def target(self, vector, do_print=False):
cell_parms = self.cucp.get_param_vals()
orientation_parms = self.cop.get_param_vals()
assert len(vector) == len(cell_parms) + len(orientation_parms)
tst_cell = vector[:len(cell_parms)]
tst_orientation = vector[len(cell_parms):len(cell_parms) +
len(orientation_parms)]
self.cucp.set_param_vals(tst_cell)
self.cop.set_param_vals(tst_orientation)
from scitbx import matrix
if do_print:
print("Cell: %.3f %.3f %.3f %.3f %.3f %.3f" %
tuple(self.crystal.get_unit_cell().parameters()))
print("Phi(1,2,3): %.3f %.3f %.3f" % tuple(tst_orientation))
UB = matrix.sqr(self.crystal.get_A())
score = self.score(UB)
if score < self.best_score:
self.best_score = score
self.cells.append(self.crystal.get_unit_cell().parameters())
return score
def score(self, UB):
score = 0.0
for j in range(self.data.size()):
hkl = self.data["miller_index"][j]
q = UB * hkl
qo = self.qobs[j]
score += (q - qo).length()**2
return score
def plotify(self):
vector = self.best
cell_parms = self.cucp.get_param_vals()
orientation_parms = self.cop.get_param_vals()
assert len(vector) == len(cell_parms) + len(orientation_parms)
tst_cell = vector[:len(cell_parms)]
tst_orientation = vector[len(cell_parms):len(cell_parms) +
len(orientation_parms)]
self.cucp.set_param_vals(tst_cell)
self.cop.set_param_vals(tst_orientation)
from scitbx import matrix
UB = matrix.sqr(self.crystal.get_A())
data = self.data
self.maxq = 0
for j in range(data.size()):
hkl = data["miller_index"][j]
q = UB * hkl
qo = self.qobs[j]
print((q - qo).length(), self.i_s[j], self.dq0[j])
if (q - qo).length() > self.maxq:
self.maxq = (q - qo).length()
return
def get_maxq(self):
vector = self.best
cell_parms = self.cucp.get_param_vals()
orientation_parms = self.cop.get_param_vals()
assert len(vector) == len(cell_parms) + len(orientation_parms)
tst_cell = vector[:len(cell_parms)]
tst_orientation = vector[len(cell_parms):len(cell_parms) +
len(orientation_parms)]
self.cucp.set_param_vals(tst_cell)
self.cop.set_param_vals(tst_orientation)
from scitbx import matrix
UB = matrix.sqr(self.crystal.get_A())
data = self.data
self.maxq = 0
for j in range(data.size()):
hkl = data["miller_index"][j]
q = UB * hkl
qo = self.qobs[j]
if (q - qo).length() > self.maxq:
self.maxq = (q - qo).length()
return self.maxq
def zero(self):
from scitbx import matrix
UB = matrix.sqr(self.crystal.get_A())
data = self.data
self.dq0 = []
for j in range(data.size()):
hkl = data["miller_index"][j]
q = UB * hkl
qo = self.qobs[j]
self.dq0.append((q - qo).length())
return
def get_signal_mask(self):
if hasattr(self, "signal_mask"):
return self.signal_mask
distance_map = self.render_distance()
maxq = self.get_maxq()
self.signal_mask = distance_map.as_1d() < (self.scale * maxq)
self.signal_mask.reshape(self.raw_data.accessor())
return self.signal_mask
def make_background(self):
import copy
if hasattr(self, "background"):
return self.background
# raw background data
background = copy.deepcopy(self.raw_data).as_double()
# mask out the signal areas
mask = self.get_signal_mask()
background.as_1d().set_selected(mask.as_1d(), 0.0)
inv_mask = (~mask).as_1d().as_int()
inv_mask.reshape(self.raw_data.accessor())
from dials.algorithms.image.filter import summed_area
from dials.array_family import flex
summed_background = summed_area(background, (5, 5))
summed_mask = summed_area(inv_mask, (5, 5))
mean_background = summed_background / summed_mask.as_double()
background.as_1d().set_selected(mask.as_1d(), mean_background.as_1d())
self.background = background
return background
def get_background_subtracted_spots(self):
if hasattr(self, "background_subtracted_spots"):
return self.background_subtracted_spots
mask = self.get_signal_mask()
background = self.make_background()
import copy
background_subtracted_spots = self.raw_data.as_double() - background
background_subtracted_spots.as_1d().set_selected(~mask.as_1d(), 0)
self.background_subtracted_spots = background_subtracted_spots
return background_subtracted_spots
def integrate(self):
from scitbx.array_family import flex
from scitbx import matrix
nslow, nfast = self.raw_data.focus()
binary_map = self.get_signal_mask().as_1d().as_int()
binary_map.reshape(flex.grid(1, nslow, nfast))
# find connected regions of spots - hacking code for density modification
# this is used to determine the integration masks for the reflections
from cctbx import masks
from cctbx import uctbx
uc = uctbx.unit_cell((1, nslow, nfast, 90, 90, 90))
flood_fill = masks.flood_fill(binary_map, uc)
binary_map = binary_map.as_1d()
coms = flood_fill.centres_of_mass()
# now iterate through these blobs, find the intensity and error, and
# find the Miller index
UB = matrix.sqr(self.crystal.get_A())
UBi = UB.inverse()
winv = 1 / self.beam.get_wavelength()
data = self.raw_data.as_double()
background = self.make_background()
from dials.array_family import flex
reflections = flex.reflection_table()
num_pixels_foreground = flex.int()
background_mean = flex.double()
background_sum_value = flex.double()
background_sum_variance = flex.double()
intensity_sum_value = flex.double()
intensity_sum_variance = flex.double()
miller_index = flex.miller_index()
xyzcal_px = flex.vec3_double()
bbox = flex.int6()
dq = flex.double()
fast = flex.int(self.raw_data.size(), -1)
fast.reshape(self.raw_data.accessor())
slow = flex.int(self.raw_data.size(), -1)
slow.reshape(self.raw_data.accessor())
nslow, nfast = fast.focus()
for j in range(nslow):
for i in range(nfast):
fast[(j, i)] = i
slow[(j, i)] = j
for j in range(flood_fill.n_voids()):
sel = binary_map == (j + 2)
pixels = data.select(sel)
if flex.min(pixels) < 0:
continue
bg_pixels = background.select(sel)
n = pixels.size()
d = flex.sum(pixels)
b = flex.sum(bg_pixels)
s = d - b
# FIXME is this the best centre of mass? if this spot is actually
# there, probably no, but if not there (i.e. no spot) then background
# subtracted centre of mass likely to be very noisy - keeping the
# background in likely to make this a little more stable
xy = coms[j][2], coms[j][1]
fs = fast.select(sel)
ss = slow.select(sel)
fd = fs.as_double()
sd = ss.as_double()
p = matrix.col(
self.panel.get_pixel_lab_coord(xy)).normalize() * winv
q = p - matrix.col(self.beam.get_s0())
hkl = UBi * q
ihkl = [int(round(h)) for h in hkl]
dq.append((q - UB * ihkl).length())
# puzzle out the bounding boxes - hack here, we have maps with the
# fast and slow positions in; select from these then find max, min of
# this selection
f_min = flex.min(fs)
f_max = flex.max(fs)
s_min = flex.min(ss)
s_max = flex.max(ss)
bbox.append((f_min, f_max + 1, s_min, s_max + 1, 0, 1))
num_pixels_foreground.append(n)
background_mean.append(b / n)
background_sum_value.append(b)
background_sum_variance.append(b)
intensity_sum_value.append(s)
intensity_sum_variance.append(d + b)
miller_index.append(ihkl)
xyzcal_px.append((xy[0], xy[1], 0.0))
reflections["num_pixels.foreground"] = num_pixels_foreground
reflections["background.mean"] = background_mean
reflections["background.sum.value"] = background_sum_value
reflections["background.sum.variance"] = background_sum_variance
reflections["intensity.sum.value"] = intensity_sum_value
reflections["intensity.sum.variance"] = intensity_sum_variance
reflections["miller_index"] = miller_index
reflections["xyzcal.px"] = xyzcal_px
reflections["id"] = flex.int(miller_index.size(), 0)
reflections["panel"] = flex.size_t(miller_index.size(), 0)
reflections["bbox"] = bbox
reflections["dq"] = dq
from dials.algorithms.shoebox import MaskCode
reflections["shoebox"] = flex.shoebox(reflections["panel"],
reflections["bbox"],
allocate=True)
reflections.extract_shoeboxes(self.imageset, verbose=True)
# now iterate through these (how?!) and assign mask values
fg = self.get_signal_mask()
for reflection in reflections:
s = reflection["shoebox"]
b = reflection["bbox"]
dz, dy, dx = s.mask.focus()
for j in range(dy):
for i in range(dx):
_i = b[0]
_j = b[2]
if fg[(j + _j, i + _i)]:
m = MaskCode.Valid | MaskCode.Foreground
else:
m = MaskCode.Valid | MaskCode.Background
s.mask[(0, j, i)] = m
return reflections
def find_spots(self, min_spot_size=2, max_spot_size=100):
from dials.algorithms.spot_finding.threshold import XDSThresholdStrategy
from dials.model.data import PixelList
from dials.model.data import PixelListLabeller
image = self.raw_data
mask = self.imageset.get_mask(0)[0]
threshold_image = XDSThresholdStrategy()
threshold_mask = threshold_image(image, mask)
plist = PixelList(0, image, threshold_mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(plist)
creator = flex.PixelListShoeboxCreator(pixel_labeller, 0, 0, True,
min_spot_size, max_spot_size,
False)
shoeboxes = creator.result()
# turns out we need to manually filter the list to get a sensible answer
size = creator.spot_size()
big = size > max_spot_size
small = size < min_spot_size
bad = big | small
shoeboxes = shoeboxes.select(~bad)
centroid = shoeboxes.centroid_valid()
intensity = shoeboxes.summed_intensity()
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
reflections = flex.reflection_table(observed, shoeboxes)
return reflections
def index(self, reflections):
# FIXME allow for the fact that there could be > 1 lattice on here to
# e.g. assign index over small spherical radius
miller_index = flex.miller_index()
UB = matrix.sqr(self.crystal.get_A())
UBi = UB.inverse()
self.qobs = []
for refl in reflections:
x, y, z = refl["xyzobs.px.value"]
p = matrix.col(self.panel.get_pixel_lab_coord((x, y)))
q = p.normalize() / self.wavelength - self.s0
self.qobs.append(q)
hkl = UBi * q
ihkl = [int(round(h)) for h in hkl]
miller_index.append(ihkl)
reflections["miller_index"] = miller_index
self.data = reflections
return reflections
class CrystalRefiner(object):
def __init__(self, experiment, reflections, parameters):
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation, )
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation, )
# Store the input
self.experiment = experiment
self.reflections = reflections
self.parameters = parameters
# Get the crystal model and the parameterisation
self.crystal = self.experiment.crystal
self.cucp = CrystalUnitCellParameterisation(self.crystal)
self.cop = CrystalOrientationParameterisation(self.crystal)
# Get the current values and generate some offsets
values = flex.double(self.cucp.get_param_vals() +
self.cop.get_param_vals())
offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] +
[0.1, 0.1, 0.1])
# The optimization history
self.history = []
# Get the initial cell and initial score
initial_cell = self.crystal.get_unit_cell()
initial_orientation = self.crystal.get_U()
initial_score = self.target(values)
# Perform the optimization
optimizer = simple_simplex(values, offset, self, 2000)
result = optimizer.get_solution()
# Print some information
fmt = "(%.3f, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f)"
print("Initial cell:", initial_cell)
print("Final cell: ", self.crystal.get_unit_cell())
print("Initial orientation: ", fmt % tuple(initial_orientation))
print("Final orientation: ", fmt % tuple(self.crystal.get_U()))
# Print RMSD
Xobs, Yobs, _ = self.reflections["xyzobs.px.value"].parts()
Xcal, Ycal, _ = self.reflections["xyzcal.px"].parts()
rmsd_x = sqrt(flex.sum((Xcal - Xobs)**2) / len(Xcal))
rmsd_y = sqrt(flex.sum((Ycal - Yobs)**2) / len(Ycal))
print("RMSD X, Y (px): %f, %f" % (rmsd_x, rmsd_y))
def target(self, vector):
"""
The target function
"""
from dials.array_family import flex
from math import sqrt
# Get the cell and orientation parameters
cell_parms = self.cucp.get_param_vals()
orientation_parms = self.cop.get_param_vals()
assert len(vector) == len(cell_parms) + len(orientation_parms)
# Update the cell and orientation parameters
tst_cell = vector[:len(cell_parms)]
tst_orientation = vector[len(cell_parms):len(cell_parms) +
len(orientation_parms)]
self.cucp.set_param_vals(tst_cell)
self.cop.set_param_vals(tst_orientation)
# Generate predicted positions
s1_cal, s2_cal = self.generate_predictions(self.experiment,
self.reflections,
self.parameters)
# Do the ray intersection
self.reflections["s1"] = s1_cal
self.reflections["s2"] = s2_cal
self.reflections["xyzcal.px"] = flex.vec3_double([
self.experiment.detector[0].get_ray_intersection_px(s1) + (0, )
for s1 in s1_cal
])
# Get predictions and observations
Xobs, Yobs, _ = self.reflections["xyzobs.px.value"].parts()
Xcal, Ycal, _ = self.reflections["xyzcal.px"].parts()
# Compute the rmsd between observed and calculated
score = flex.sum((Xobs - Xcal)**2 + (Yobs - Ycal)**2)
# Append to the history
self.history.append((tst_cell, tst_orientation, score))
# Print some info
print(
"Cell: %.3f %.3f %.3f %.3f %.3f %.3f; Phi: %.3f %.3f %.3f; RMSD: %.3f"
% (tuple(self.crystal.get_unit_cell().parameters()) +
tuple(tst_orientation) + tuple((sqrt(score / len(Xobs)), ))))
return score
def generate_predictions(self, experiment, reflections, parameters):
# Create the mosaicity model
parameterisation = MosaicityParameterisation(parameters)
# The crystal A and beam s0
A = matrix.sqr(experiment.crystal.get_A())
s0 = matrix.col(experiment.beam.get_s0())
s0_length = s0.length()
# Compute all the vectors
s1_cal = flex.vec3_double()
s2_cal = flex.vec3_double()
for i in range(len(reflections)):
# Compute the reciprocal lattice vector
h = matrix.col(reflections[i]["miller_index"])
r = A * h
s2 = s0 + r
# Rotate the covariance matrix
R = compute_change_of_basis_operation(s0, s2)
S = R * parameterisation.sigma() * R.transpose()
mu = R * s2
assert abs(1 - mu.normalize().dot(matrix.col((0, 0, 1)))) < 1e-7
# Partition the mean vector
mu1 = matrix.col((mu[0], mu[1]))
mu2 = mu[2]
# Partition the covariance matrix
S11 = matrix.sqr((S[0], S[1], S[3], S[4]))
S12 = matrix.col((S[2], S[5]))
S21 = matrix.col((S[6], S[7])).transpose()
S22 = S[8]
# Compute the conditional mean
mubar = mu1 + S12 * (1 / S22) * (s0_length - mu2)
# Compute the vector and rotate
v = matrix.col(
(mubar[0], mubar[1], s0_length)).normalize() * s0_length
s1 = R.transpose() * v
# Append the 2 vectors
s1_cal.append(s1)
s2_cal.append(s2)
# Return the predicted vectors
return s1_cal, s2_cal
class ModelState(object):
"""
A class to keep track of the model state
"""
def __init__(
self,
experiment,
mosaicity_parameterisation,
fix_mosaic_spread=False,
fix_wavelength_spread=False,
fix_unit_cell=False,
fix_orientation=False,
):
"""
Initialise the model state
"""
# Save the crystal model
self.experiment = experiment
self.crystal = experiment.crystal
# The U and P parameterisation
self.U_parameterisation = CrystalOrientationParameterisation(
self.crystal)
self.B_parameterisation = CrystalUnitCellParameterisation(self.crystal)
# The M, L and W parameterisations
self.M_parameterisation = mosaicity_parameterisation
# Set the flags to fix parameters
self._is_mosaic_spread_fixed = fix_mosaic_spread
self._is_wavelength_spread_fixed = fix_wavelength_spread
self._is_unit_cell_fixed = fix_unit_cell
self._is_orientation_fixed = fix_orientation
def is_orientation_fixed(self):
"""
Return whether the orientation is fixed
"""
return self._is_orientation_fixed
def is_unit_cell_fixed(self):
"""
Return whether the unit cell is fixed
"""
return self._is_unit_cell_fixed
def is_mosaic_spread_fixed(self):
"""
Return whether the mosaic spread is fixed
"""
return self._is_mosaic_spread_fixed
def is_mosaic_spread_angular(self):
"""
Return whether the mosaic spread is angular
"""
return self.M_parameterisation.is_angular()
def is_wavelength_spread_fixed(self):
"""
Return whether the wavelength spread is fixed
"""
return self._is_wavelength_spread_fixed
def get_unit_cell(self):
"""
Get the crystal unit cell
"""
return self.crystal.get_unit_cell()
def get_U(self):
"""
Get the crystal U matrix
"""
return matrix.sqr(self.crystal.get_U())
def get_B(self):
"""
Get the crystal B matrix
"""
return matrix.sqr(self.crystal.get_B())
def get_A(self):
"""
Get the crystal A matrix
"""
return matrix.sqr(self.crystal.get_A())
def get_M(self):
"""
Get the Sigma M matrix
"""
return self.M_parameterisation.sigma()
def get_U_params(self):
"""
Get the U parameters
"""
return flex.double(self.U_parameterisation.get_param_vals())
def get_B_params(self):
"""
Get the B parameters
"""
return flex.double(self.B_parameterisation.get_param_vals())
def get_M_params(self):
"""
Get the M parameters
"""
return self.M_parameterisation.parameters()
def set_U_params(self, params):
"""
Set the U parameters
"""
return self.U_parameterisation.set_param_vals(params)
def set_B_params(self, params):
"""
Set the B parameters
"""
return self.B_parameterisation.set_param_vals(params)
def set_M_params(self, params):
"""
Set the M parameters
"""
return self.M_parameterisation.set_parameters(params)
def num_U_params(self):
"""
Get the number of U parameters
"""
return len(self.get_U_params())
def num_B_params(self):
"""
Get the number of B parameters
"""
return len(self.get_B_params())
def num_M_params(self):
"""
Get the number of M parameters
"""
return len(self.get_M_params())
def get_dU_dp(self):
"""
Get the first derivatives of U w.r.t its parameters
"""
return flex.mat3_double(self.U_parameterisation.get_ds_dp())
def get_dB_dp(self):
"""
Get the first derivatives of B w.r.t its parameters
"""
return flex.mat3_double(self.B_parameterisation.get_ds_dp())
def get_dM_dp(self):
"""
Get the first derivatives of M w.r.t its parameters
"""
return self.M_parameterisation.first_derivatives()
def get_active_parameters(self):
"""
Get the active parameters in order: U, B, M, L, W
"""
active_params = flex.double()
if not self._is_orientation_fixed:
active_params.extend(self.get_U_params())
if not self._is_unit_cell_fixed:
active_params.extend(self.get_B_params())
if not self._is_mosaic_spread_fixed:
active_params.extend(self.get_M_params())
if not self._is_wavelength_spread_fixed:
active_params.extend(self.get_L_params())
assert len(active_params) > 0
return active_params
def set_active_parameters(self, params):
"""
Set the active parameters in order: U, B, M, L, W
"""
if not self._is_orientation_fixed:
temp = params[:self.num_U_params()]
params = params[self.num_U_params():]
self.set_U_params(temp)
if not self._is_unit_cell_fixed:
temp = params[:self.num_B_params()]
params = params[self.num_B_params():]
self.set_B_params(temp)
if not self._is_mosaic_spread_fixed:
temp = params[:self.num_M_params()]
params = params[self.num_M_params():]
self.set_M_params(temp)
if not self._is_wavelength_spread_fixed:
temp = params[:self.num_L_params()]
params = params[self.num_L_params():]
self.set_L_params(temp)
def get_labels(self):
"""
Get the parameter labels
"""
labels = []
if not self._is_orientation_fixed:
for i in range(len(self.get_U_params())):
labels.append("Crystal_U_%d" % i)
if not self._is_unit_cell_fixed:
for i in range(len(self.get_B_params())):
labels.append("Crystal_B_%d" % i)
if not self._is_mosaic_spread_fixed:
for i in range(len(self.get_M_params())):
labels.append("Mosaicity_%d" % i)
if not self._is_wavelength_spread_fixed:
labels.append("Wavelength_Spread")
assert len(labels) > 0
return labels
def test(args=[]):
from math import pi
from cctbx.sgtbx import space_group, space_group_symbols
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import Experiment, ExperimentList
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge
from scitbx import matrix
from scitbx.array_family import flex
# Get modules to build models and minimiser using PHIL
import dials.tests.algorithms.refinement.setup_geometry as setup_geometry
import dials.tests.algorithms.refinement.setup_minimiser as setup_minimiser
from dials.algorithms.refinement.parameterisation.beam_parameters import (
BeamParameterisation,
)
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
# Model parameterisations
from dials.algorithms.refinement.parameterisation.detector_parameters import (
DetectorParameterisationSinglePanel,
)
# Parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import (
XYPhiPredictionParameterisation,
)
from dials.algorithms.refinement.prediction.managed_predictors import (
ScansExperimentsPredictor,
ScansRayPredictor,
)
from dials.algorithms.refinement.reflection_manager import ReflectionManager
# Imports for the target function
from dials.algorithms.refinement.target import (
LeastSquaresPositionalResidualWithRmsdCutoff,
)
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection
#############################
# Setup experimental models #
#############################
master_phil = parse(
"""
include scope dials.tests.algorithms.refinement.geometry_phil
include scope dials.tests.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
models = setup_geometry.Extract(master_phil, cmdline_args=args)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Build a mock scan for a 180 degree sequence
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(1800)),
deg=True,
)
sequence_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sequence_range == (0.0, pi)
assert approx_equal(im_width, 0.1 * pi / 180.0)
# Build an experiment list
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
)
)
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Fix crystal parameters
# xluc_param.set_fixed([True, True, True, True, True, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
pred_param = XYPhiPredictionParameterisation(
experiments, [det_param], [s0_param], [xlo_param], [xluc_param]
)
################################
# Apply known parameter shifts #
################################
# shift detector by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])]
det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])]
xlo_param.set_param_vals(p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = mycrystal.get_unit_cell().parameters()
cell_params = [a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(mycrystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
#############################
# Generate some reflections #
#############################
print("Reflections will be generated with the following geometry:")
print(mybeam)
print(mydetector)
print(mycrystal)
print("Target values of parameters are")
msg = "Parameters: " + "%.5f " * len(pred_param)
print(msg % tuple(pred_param.get_param_vals()))
print()
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# Predict rays within the sequence range
ray_predictor = ScansRayPredictor(experiments, sequence_range)
obs_refs = ray_predictor(indices)
print("Total number of reflections excited", len(obs_refs))
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ScansExperimentsPredictor(experiments)
obs_refs["id"] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0) ** 2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0) ** 2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0) ** 2)
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
print("Total number of observations made", len(obs_refs))
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
print("Initial values of parameters are")
msg = "Parameters: " + "%.5f " * len(pred_param)
print(msg % tuple(pred_param.get_param_vals()))
print()
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(obs_refs, experiments)
##############################
# Set up the target function #
##############################
# The current 'achieved' criterion compares RMSD against 1/3 the pixel size and
# 1/3 the image width in radians. For the simulated data, these are just made up
mytarget = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments, ref_predictor, refman, pred_param, restraints_parameterisation=None
)
################################
# Set up the refinement engine #
################################
refiner = setup_minimiser.Extract(
master_phil, mytarget, pred_param, cmdline_args=args
).refiner
print("Prior to refinement the experimental model is:")
print(mybeam)
print(mydetector)
print(mycrystal)
refiner.run()
print()
print("Refinement has completed with the following geometry:")
print(mybeam)
print(mydetector)
print(mycrystal)
class Blueberry(object):
def __init__(self, reflection_file, experiment_file):
# make a pie
data = pickle.load(open(reflection_file, "rb"))
print("%d reflections" % data.size())
self.data = data.select(data["intensity.sum.variance"] > 0)
i = data["intensity.sum.value"]
v = data["intensity.sum.variance"]
s = flex.sqrt(v)
self.i_s = i / s
from dxtbx.model.experiment_list import ExperimentListFactory
expt = ExperimentListFactory.from_json_file(experiment_file)
crystal = expt.crystals()[0]
self.s0 = matrix.col(expt.beams()[0].get_s0())
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation,
CrystalOrientationParameterisation,
)
self.crystal = crystal
self.cucp = CrystalUnitCellParameterisation(crystal)
self.cop = CrystalOrientationParameterisation(crystal)
# 0-point and deltas
values = flex.double(self.cucp.get_param_vals() +
self.cop.get_param_vals())
offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] +
[0.1, 0.1, 0.1])
initial = crystal.get_unit_cell()
initial_score = self.target(values)
doohicky = simple_simplex(values, offset, self, 2000)
best = doohicky.get_solution()
print("Initial cell:", initial)
print("Final cell: ", crystal.get_unit_cell())
print("Score change", initial_score, self.target(best, do_print=False))
self.best = best
def target(self, vector, do_print=False):
cell_parms = self.cucp.get_param_vals()
orientation_parms = self.cop.get_param_vals()
assert len(vector) == len(cell_parms) + len(orientation_parms)
tst_cell = vector[:len(cell_parms)]
tst_orientation = vector[len(cell_parms):len(cell_parms) +
len(orientation_parms)]
self.cucp.set_param_vals(tst_cell)
self.cop.set_param_vals(tst_orientation)
from scitbx import matrix
if do_print:
print("Cell: %.3f %.3f %.3f %.3f %.3f %.3f" %
tuple(self.crystal.get_unit_cell().parameters()))
print("Phi(1,2,3): %.3f %.3f %.3f" % tuple(tst_orientation))
UB = matrix.sqr(self.crystal.get_A())
return self.score(UB)
def score_old(self, RUB):
score = 0.0
for j in range(self.data.size()):
hkl = self.data["miller_index"][j]
q = RUB * hkl
score += self.i_s[j] * abs((q + self.s0).length() -
matrix.col(self.data["s1"][j]).length())
return score
def score(self, RUB):
score = 0.0
for j in range(self.data.size()):
hkl = self.data["miller_index"][j]
q = RUB * hkl
score += abs((q + self.s0).length() - self.s0.length())
return score
def plotify(self):
vector = self.best
cell_parms = self.cucp.get_param_vals()
orientation_parms = self.cop.get_param_vals()
assert len(vector) == len(cell_parms) + len(orientation_parms)
tst_cell = vector[:len(cell_parms)]
tst_orientation = vector[len(cell_parms):len(cell_parms) +
len(orientation_parms)]
self.cucp.set_param_vals(tst_cell)
self.cop.set_param_vals(tst_orientation)
from scitbx import matrix
UB = matrix.sqr(self.crystal.get_A())
data = self.data
for j in range(data.size()):
hkl = data["miller_index"][j]
q = UB * hkl
print((q + self.s0).length() - matrix.col(data["s1"][j]).length(),
self.i_s[j])
return