def test():
# Build models, with a larger crystal than default in order to get plenty of
# reflections on the 'still' image
overrides = """
geometry.parameters.crystal.a.length.range=40 50;
geometry.parameters.crystal.b.length.range=40 50;
geometry.parameters.crystal.c.length.range=40 50;
geometry.parameters.random_seed = 42"""
master_phil = parse("""
include scope dials.test.algorithms.refinement.geometry_phil
""",
process_includes=True)
models = Extract(master_phil, overrides)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Build a mock scan for a 3 degree sweep
from dxtbx.model import ScanFactory
sf = ScanFactory()
myscan = sf.make_scan(image_range=(1, 1),
exposure_times=0.1,
oscillation=(0, 3.0),
epochs=range(1),
deg=True)
sweep_range = myscan.get_oscillation_range(deg=False)
# Create parameterisations of these models
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
# Create a scans ExperimentList, only for generating reflections
experiments = ExperimentList()
experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None))
# Create a stills ExperimentList
stills_experiments = ExperimentList()
stills_experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
crystal=mycrystal,
imageset=None))
# Generate rays - only to work out which hkls are predicted
ray_predictor = ScansRayPredictor(experiments, sweep_range)
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), resolution)
indices = index_generator.to_array()
rays = ray_predictor(indices)
# Make a standard reflection_table and copy in the ray data
reflections = flex.reflection_table.empty_standard(len(rays))
reflections.update(rays)
# Build a standard prediction parameterisation for the stills experiment to do
# FD calculation (not used for its analytical gradients)
pred_param = StillsPredictionParameterisation(
stills_experiments,
detector_parameterisations=[det_param],
beam_parameterisations=[s0_param],
xl_orientation_parameterisations=[xlo_param],
xl_unit_cell_parameterisations=[xluc_param])
# Make a managed SphericalRelpStillsReflectionPredictor reflection predictor
# for the first (only) experiment
ref_predictor = Predictor(stills_experiments)
# Predict these reflections in place. Must do this ahead of calculating
# the analytical gradients so quantities like s1 are correct
ref_predictor.update()
ref_predictor.predict(reflections)
# calculate analytical gradients
ag = AnalyticalGradients(stills_experiments,
detector_parameterisation=det_param,
beam_parameterisation=s0_param,
xl_orientation_parameterisation=xlo_param,
xl_unit_cell_parameterisation=xluc_param)
an_grads = ag.get_beam_gradients(reflections)
an_grads.update(ag.get_crystal_orientation_gradients(reflections))
an_grads.update(ag.get_crystal_unit_cell_gradients(reflections))
# get finite difference gradients
p_vals = pred_param.get_param_vals()
deltas = [1.e-7] * len(p_vals)
fd_grads = []
p_names = pred_param.get_param_names()
for i, delta in enumerate(deltas):
# save parameter value
val = p_vals[i]
# calc reverse state
p_vals[i] -= delta / 2.
pred_param.set_param_vals(p_vals)
ref_predictor.update()
ref_predictor.predict(reflections)
x, y, _ = reflections['xyzcal.mm'].deep_copy().parts()
delpsi = reflections['delpsical.rad'].deep_copy()
s1 = reflections['s1'].deep_copy()
rev_state = s1
# calc forward state
p_vals[i] += delta
pred_param.set_param_vals(p_vals)
ref_predictor.update()
ref_predictor.predict(reflections)
x, y, _ = reflections['xyzcal.mm'].deep_copy().parts()
delpsi = reflections['delpsical.rad'].deep_copy()
s1 = reflections['s1'].deep_copy()
fwd_state = s1
# reset parameter to saved value
p_vals[i] = val
# finite difference - currently for s1 only
fd = (fwd_state - rev_state)
inv_delta = 1. / delta
s1_grads = fd * inv_delta
# store gradients
fd_grads.append({'name': p_names[i], 'ds1': s1_grads})
# return to the initial state
pred_param.set_param_vals(p_vals)
for i, fd_grad in enumerate(fd_grads):
## compare FD with analytical calculations
print("\n\nParameter {0}: {1}".format(i, fd_grad['name']))
print("d[s1]/dp for the first reflection")
print('finite diff', fd_grad['ds1'][0])
try:
an_grad = an_grads[fd_grad['name']]
except KeyError:
continue
print('checking analytical vs finite difference gradients for s1')
for a, b in zip(fd_grad['ds1'], an_grad['ds1']):
assert a == pytest.approx(b, abs=1e-7)
def test(dials_regression):
from scitbx import matrix
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
from scitbx.array_family import flex
# Get modules to build models and minimiser using PHIL
from dials.test.algorithms.refinement import setup_geometry
from dials.test.algorithms.refinement import setup_minimiser
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalOrientationParameterisation, CrystalUnitCellParameterisation
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import \
symmetrize_reduce_enlarge
DEG2RAD = math.pi/180.0
RAD2DEG = 180.0/math.pi
master_phil = parse("""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""", process_includes=True)
# make cell more oblique
args=["a.direction.close_to.sd=5","b.direction.close_to.sd=5","c.direction.close_to.sd=5"]
models = setup_geometry.Extract(master_phil, cmdline_args = args)
crystal = models.crystal
# a hexagonal crystal is a good test case for behaviour of oblique cells
do_hexagonal = True
if do_hexagonal:
from dxtbx.model.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(
os.path.join(dials_regression, "refinement_test_data", "multi_stills", "combined_experiments.json"),
check_format=False)
crystal = experiments[0].crystal
# derive finite difference gradients of various quantities wrt each param
def check_fd_gradients(parameterisation):
mp = parameterisation
p_vals = mp.get_param_vals()
deltas = [1.e-7 for p in p_vals]
assert len(deltas) == len(p_vals)
fd_grad = []
# get matrix to unset rotations of unit cell vectors
Ut = matrix.sqr(mp.get_model().get_U()).transpose()
for i, delta in enumerate(deltas):
val = p_vals[i]
p_vals[i] -= delta / 2.
mp.set_param_vals(p_vals)
rev_uc = mp.get_model().get_unit_cell().parameters()
rev_vec = mp.get_model().get_real_space_vectors()
rev_vec = [Ut * vec for vec in rev_vec]
rev_B = matrix.sqr(mp.get_model().get_B())
rev_O = rev_B.transpose().inverse()
p_vals[i] += delta
mp.set_param_vals(p_vals)
fwd_uc = mp.get_model().get_unit_cell().parameters()
fwd_vec = mp.get_model().get_real_space_vectors()
fwd_vec = [Ut * vec for vec in fwd_vec]
fwd_B = matrix.sqr(mp.get_model().get_B())
fwd_O = fwd_B.transpose().inverse()
fd_uc = [(f - r) / delta for f,r in zip(fwd_uc, rev_uc)]
fd_vec = [(f - r) / delta for f,r in zip(fwd_vec, rev_vec)]
fd_B = (fwd_B - rev_B) / delta
fd_O = (fwd_O - rev_O) / delta
fd_grad.append({'da_dp':fd_uc[0],
'db_dp':fd_uc[1],
'dc_dp':fd_uc[2],
'daa_dp':fd_uc[3],
'dbb_dp':fd_uc[4],
'dcc_dp':fd_uc[5],
'davec_dp':fd_vec[0],
'dbvec_dp':fd_vec[1],
'dcvec_dp':fd_vec[2],
'dB_dp':fd_B,
'dO_dp':fd_O})
p_vals[i] = val
# return to the initial state
mp.set_param_vals(p_vals)
return fd_grad
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
from dials.algorithms.refinement.restraints.restraints import SingleUnitCellTie
uct = SingleUnitCellTie(xluc_param, [None]*6, [None]*6)
from scitbx.math import angle_derivative_wrt_vectors
B = matrix.sqr(crystal.get_B())
O = (B.transpose()).inverse()
a, b, c, aa, bb, cc = crystal.get_unit_cell().parameters()
aa *= DEG2RAD
bb *= DEG2RAD
cc *= DEG2RAD
Ut = matrix.sqr(crystal.get_U()).transpose()
avec, bvec, cvec = [Ut * vec for vec in crystal.get_real_space_vectors()]
# calculate d[B^T]/dp
dB_dp = xluc_param.get_ds_dp()
dBT_dp = [dB.transpose() for dB in dB_dp]
# calculate d[O]/dp
dO_dp = [-O * dBT * O for dBT in dBT_dp]
# function to get analytical derivative of angles wrt vectors
def dangle(u, v):
return [matrix.col(e) for e in angle_derivative_wrt_vectors(u,v)]
dalpha_db, dalpha_dc = dangle(bvec, cvec)
dbeta_da, dbeta_dc = dangle(avec, cvec)
dgamma_da, dgamma_db = dangle(avec, bvec)
# get all FD derivatives
fd_grad = check_fd_gradients(xluc_param)
# look at each parameter
for i, dO in enumerate(dO_dp):
#print
#print "***** PARAMETER {0} *****".format(i)
#print "dB_dp analytical"
#print dB_dp[i]
#print "dB_dp FD"
#print fd_grad[i]['dB_dp']
#print
# dB_dp is good. What about dO_dp?
#print "O MATRIX"
#print "dO_dp analytical"
#print dO.round(6)
#print "dO_dp FD"
#print fd_grad[i]['dO_dp'].round(6)
#print
assert approx_equal(dO, fd_grad[i]['dO_dp'])
# extract derivatives of each unit cell vector wrt p
dav_dp, dbv_dp, dcv_dp = dO.transpose().as_list_of_lists()
dav_dp = matrix.col(dav_dp)
dbv_dp = matrix.col(dbv_dp)
dcv_dp = matrix.col(dcv_dp)
# check these are correct vs FD
#print "CELL VECTORS"
#diff = dav_dp - fd_grad[i]['davec_dp']
#print 2 * diff.length() / (dav_dp.length() + fd_grad[i]['davec_dp'].length()) * 100
#print 'davec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dav_dp.elems)
#print 'davec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['davec_dp'].elems)
assert approx_equal(dav_dp, fd_grad[i]['davec_dp'])
#diff = dbv_dp - fd_grad[i]['dbvec_dp']
#print 2 * diff.length() / (dbv_dp.length() + fd_grad[i]['dbvec_dp'].length()) * 100
#print 'dbvec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dbv_dp.elems)
#print 'dbvec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['dbvec_dp'].elems)
assert approx_equal(dbv_dp, fd_grad[i]['dbvec_dp'])
#diff = dcv_dp - fd_grad[i]['dcvec_dp']
#print 2 * diff.length() / (dcv_dp.length() + fd_grad[i]['dcvec_dp'].length()) * 100
#print 'dcvec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dcv_dp.elems)
#print 'dcvec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['dcvec_dp'].elems)
#print
assert approx_equal(dcv_dp, fd_grad[i]['dcvec_dp'])
#print "CELL LENGTHS"
da_dp = 1./a * avec.dot(dav_dp)
#print "d[a]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(da_dp, fd_grad[i]['da_dp'], i)
assert approx_equal(da_dp, fd_grad[i]['da_dp'])
db_dp = 1./b * bvec.dot(dbv_dp)
#print "d[b]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(db_dp, fd_grad[i]['db_dp'], i)
assert approx_equal(db_dp, fd_grad[i]['db_dp'])
dc_dp = 1./c * cvec.dot(dcv_dp)
#print "d[c]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dc_dp, fd_grad[i]['dc_dp'], i)
assert approx_equal(dc_dp, fd_grad[i]['dc_dp'])
#print
#print "CELL ANGLES"
daa_dp = RAD2DEG * (dbv_dp.dot(dalpha_db) + dcv_dp.dot(dalpha_dc))
dbb_dp = RAD2DEG * (dav_dp.dot(dbeta_da) + dcv_dp.dot(dbeta_dc))
dcc_dp = RAD2DEG * (dav_dp.dot(dgamma_da) + dbv_dp.dot(dgamma_db))
#print "d[alpha]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(daa_dp, fd_grad[i]['daa_dp'], i)
#print "d[beta]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dbb_dp, fd_grad[i]['dbb_dp'], i)
#print "d[gamma]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dcc_dp, fd_grad[i]['dcc_dp'], i)
assert approx_equal(daa_dp, fd_grad[i]['daa_dp'])
assert approx_equal(dbb_dp, fd_grad[i]['dbb_dp'])
assert approx_equal(dcc_dp, fd_grad[i]['dcc_dp'])
def test():
from cctbx.sgtbx import space_group, space_group_symbols
from dxtbx.model.experiment_list import Experiment, ExperimentList
from libtbx.phil import parse
from scitbx.array_family import flex
from dials.algorithms.refinement.parameterisation.beam_parameters import (
BeamParameterisation, )
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
from dials.algorithms.refinement.parameterisation.detector_parameters import (
DetectorParameterisationSinglePanel, )
from dials.algorithms.refinement.parameterisation.goniometer_parameters import (
GoniometerParameterisation, )
#### Import model parameterisations
from dials.algorithms.refinement.parameterisation.prediction_parameters import (
XYPhiPredictionParameterisation, )
from dials.algorithms.refinement.prediction.managed_predictors import (
ScansExperimentsPredictor,
ScansRayPredictor,
)
##### Imports for reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection
##### Import model builder
from dials.test.algorithms.refinement.setup_geometry import Extract
#### Create models
overrides = """geometry.parameters.crystal.a.length.range = 10 50
geometry.parameters.crystal.b.length.range = 10 50
geometry.parameters.crystal.c.length.range = 10 50"""
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
""",
process_includes=True,
)
models = Extract(master_phil, overrides)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Build a mock scan for a 72 degree sequence
sequence_range = (0.0, math.pi / 5.0)
from dxtbx.model import ScanFactory
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 720),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(720)),
deg=True,
)
#### Create parameterisations of these models
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
gon_param = GoniometerParameterisation(mygonio, mybeam)
# Create an ExperimentList
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
#### Unit tests
# Build a prediction parameterisation
pred_param = XYPhiPredictionParameterisation(
experiments,
detector_parameterisations=[det_param],
beam_parameterisations=[s0_param],
xl_orientation_parameterisations=[xlo_param],
xl_unit_cell_parameterisations=[xluc_param],
goniometer_parameterisations=[gon_param],
)
# Generate reflections
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# Predict rays within the sequence range
ray_predictor = ScansRayPredictor(experiments, sequence_range)
obs_refs = ray_predictor(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ScansExperimentsPredictor(experiments)
obs_refs["id"] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * math.pi / 180.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2)
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# use a ReflectionManager to exclude reflections too close to the spindle
from dials.algorithms.refinement.reflection_manager import ReflectionManager
refman = ReflectionManager(obs_refs, experiments, outlier_detector=None)
refman.finalise()
# Redefine the reflection predictor to use the type expected by the Target class
ref_predictor = ScansExperimentsPredictor(experiments)
# keep only those reflections that pass inclusion criteria and have predictions
reflections = refman.get_matches()
# get analytical gradients
an_grads = pred_param.get_gradients(reflections)
# get finite difference gradients
p_vals = pred_param.get_param_vals()
deltas = [1.0e-7] * len(p_vals)
for i, delta in enumerate(deltas):
val = p_vals[i]
p_vals[i] -= delta / 2.0
pred_param.set_param_vals(p_vals)
ref_predictor(reflections)
rev_state = reflections["xyzcal.mm"].deep_copy()
p_vals[i] += delta
pred_param.set_param_vals(p_vals)
ref_predictor(reflections)
fwd_state = reflections["xyzcal.mm"].deep_copy()
p_vals[i] = val
fd = fwd_state - rev_state
x_grads, y_grads, phi_grads = fd.parts()
x_grads /= delta
y_grads /= delta
phi_grads /= delta
# compare with analytical calculation
assert x_grads == pytest.approx(an_grads[i]["dX_dp"], abs=5.0e-6)
assert y_grads == pytest.approx(an_grads[i]["dY_dp"], abs=5.5e-6)
assert phi_grads == pytest.approx(an_grads[i]["dphi_dp"], abs=5.0e-6)
# return to the initial state
pred_param.set_param_vals(p_vals)
def test(args=[]):
# Python and cctbx imports
import random
from math import pi
from cctbx.sgtbx import space_group, space_group_symbols
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import Experiment, ExperimentList
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
from scitbx import matrix
from scitbx.array_family import flex
from dials.algorithms.refinement.parameterisation.beam_parameters import (
BeamParameterisation,
)
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
# Model parameterisations
from dials.algorithms.refinement.parameterisation.detector_parameters import (
DetectorParameterisationSinglePanel,
)
# Parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import (
XYPhiPredictionParameterisation,
)
from dials.algorithms.refinement.prediction.managed_predictors import (
ScansExperimentsPredictor,
ScansRayPredictor,
)
from dials.algorithms.refinement.reflection_manager import ReflectionManager
# Imports for the target function
from dials.algorithms.refinement.target import (
LeastSquaresPositionalResidualWithRmsdCutoff,
)
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection
# Experimental model builder
from dials.test.algorithms.refinement.setup_geometry import Extract
# Local functions
def random_direction_close_to(vector, sd=0.5):
return vector.rotate_around_origin(
matrix.col((random.random(), random.random(), random.random())).normalize(),
random.gauss(0, sd),
deg=True,
)
#############################
# Setup experimental models #
#############################
# make a small cell to speed up calculations
overrides = """geometry.parameters.crystal.a.length.range = 10 15
geometry.parameters.crystal.b.length.range = 10 15
geometry.parameters.crystal.c.length.range = 10 15"""
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
""",
process_includes=True,
)
models = Extract(master_phil, overrides, cmdline_args=args)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Build a mock scan for a 180 degree sequence of 0.1 degree images
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(1800)),
deg=True,
)
sequence_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sequence_range == (0.0, pi)
assert approx_equal(im_width, 0.1 * pi / 180.0)
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
)
)
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
pred_param = XYPhiPredictionParameterisation(
experiments, [det_param], [s0_param], [xlo_param], [xluc_param]
)
################################
# Apply known parameter shifts #
################################
# shift detector by 0.2 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [a + b for a, b in zip(det_p_vals, [2.0, 2.0, 2.0, 2.0, 2.0, 2.0])]
det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in one axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[1] += 2.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])]
xlo_param.set_param_vals(p_vals)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# Predict rays within the sequence range
ray_predictor = ScansRayPredictor(experiments, sequence_range)
obs_refs = ray_predictor(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ScansExperimentsPredictor(experiments)
obs_refs["id"] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0) ** 2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0) ** 2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0) ** 2)
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(obs_refs, experiments)
##############################
# Set up the target function #
##############################
# Redefine the reflection predictor to use the type expected by the Target class
ref_predictor = ScansExperimentsPredictor(experiments)
mytarget = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments, ref_predictor, refman, pred_param, restraints_parameterisation=None
)
# get the functional and gradients
mytarget.predict()
L, dL_dp, curvs = mytarget.compute_functional_gradients_and_curvatures()
####################################
# Do FD calculation for comparison #
####################################
# function for calculating finite difference gradients of the target function
def get_fd_gradients(target, pred_param, deltas):
"""Calculate centered finite difference gradients for each of the
parameters of the target function.
"deltas" must be a sequence of the same length as the parameter list, and
contains the step size for the difference calculations for each parameter.
"""
p_vals = pred_param.get_param_vals()
assert len(deltas) == len(p_vals)
fd_grad = []
fd_curvs = []
for i in range(len(deltas)):
val = p_vals[i]
p_vals[i] -= deltas[i] / 2.0
pred_param.set_param_vals(p_vals)
target.predict()
rev_state = target.compute_functional_gradients_and_curvatures()
p_vals[i] += deltas[i]
pred_param.set_param_vals(p_vals)
target.predict()
fwd_state = target.compute_functional_gradients_and_curvatures()
# finite difference estimation of first derivatives
fd_grad.append((fwd_state[0] - rev_state[0]) / deltas[i])
# finite difference estimation of curvatures, using the analytical
# first derivatives
fd_curvs.append((fwd_state[1][i] - rev_state[1][i]) / deltas[i])
# set parameter back to centred value
p_vals[i] = val
# return to the initial state
pred_param.set_param_vals(p_vals)
return fd_grad, fd_curvs
# test normalised differences between FD and analytical calculations
fdgrads = get_fd_gradients(mytarget, pred_param, [1.0e-7] * len(pred_param))
diffs = [a - b for a, b in zip(dL_dp, fdgrads[0])]
norm_diffs = tuple([a / b for a, b in zip(diffs, fdgrads[0])])
for e in norm_diffs:
assert abs(e) < 0.001 # check differences less than 0.1%
# test normalised differences between FD curvatures and analytical least
# squares approximation. We don't expect this to be especially close
if curvs:
diffs = [a - b for a, b in zip(curvs, fdgrads[1])]
norm_diffs = tuple([a / b for a, b in zip(diffs, fdgrads[1])])
for e in norm_diffs:
assert abs(e) < 0.1 # check differences less than 10%
experiments = ExperimentList()
experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None))
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Fix crystal parameters
#xluc_param.set_fixed([True, True, True, True, True, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
pred_param = XYPhiPredictionParameterisation(experiments, [det_param],
[s0_param], [xlo_param],
def test(init_test):
single_panel_detector = init_test.experiments_single_panel.detectors()[0]
multi_panel_detector = init_test.experiments_multi_panel.detectors()[0]
beam = init_test.experiments_single_panel.beams()[0]
gonio = init_test.experiments_single_panel.goniometers()[0]
crystal = init_test.experiments_single_panel.crystals()[0]
# Parameterise the models
det_param = DetectorParameterisationSinglePanel(single_panel_detector)
s0_param = BeamParameterisation(beam, gonio)
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
multi_det_param = DetectorParameterisationMultiPanel(multi_panel_detector, beam)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Link model parameterisations together into a parameterisation of the
# prediction equation, first for the single panel detector
pred_param = XYPhiPredictionParameterisation(
init_test.experiments_single_panel,
[det_param],
[s0_param],
[xlo_param],
[xluc_param],
)
# ... and now for the multi-panel detector
pred_param2 = XYPhiPredictionParameterisation(
init_test.experiments_multi_panel,
[multi_det_param],
[s0_param],
[xlo_param],
[xluc_param],
)
################################
# Apply known parameter shifts #
################################
# shift detectors by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])]
det_param.set_param_vals(p_vals)
multi_det_p_vals = multi_det_param.get_param_vals()
p_vals = [a + b for a, b in zip(multi_det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])]
multi_det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])]
xlo_param.set_param_vals(p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = crystal.get_unit_cell().parameters()
cell_params = [a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(crystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
multi_det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(
init_test.observations_single_panel, init_test.experiments_single_panel
)
refman2 = ReflectionManager(
init_test.observations_multi_panel, init_test.experiments_multi_panel
)
###############################
# Set up the target functions #
###############################
target = LeastSquaresPositionalResidualWithRmsdCutoff(
init_test.experiments_single_panel,
ScansExperimentsPredictor(init_test.experiments_single_panel),
refman,
pred_param,
restraints_parameterisation=None,
)
target2 = LeastSquaresPositionalResidualWithRmsdCutoff(
init_test.experiments_multi_panel,
ScansExperimentsPredictor(init_test.experiments_multi_panel),
refman2,
pred_param2,
restraints_parameterisation=None,
)
#################################
# Set up the refinement engines #
#################################
refiner = setup_minimiser.Extract(master_phil, target, pred_param).refiner
refiner2 = setup_minimiser.Extract(master_phil, target2, pred_param2).refiner
refiner.run()
# reset parameters and run refinement with the multi panel detector
s0_param.set_param_vals(s0_p_vals)
multi_det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
refiner2.run()
# same number of steps
assert refiner.get_num_steps() == refiner2.get_num_steps()
# same rmsds
for rmsd, rmsd2 in zip(refiner.history["rmsd"], refiner2.history["rmsd"]):
assert approx_equal(rmsd, rmsd2)
# same parameter values each step
for params, params2 in zip(
refiner.history["parameter_vector"], refiner.history["parameter_vector"]
):
assert approx_equal(params, params2)
def test(args=[]):
#############################
# Setup experimental models #
#############################
master_phil = parse(
"""
include scope dials.tests.algorithms.refinement.geometry_phil
include scope dials.tests.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
models = setup_geometry.Extract(
master_phil,
cmdline_args=args,
local_overrides="geometry.parameters.random_seed = 1",
)
crystal1 = models.crystal
models = setup_geometry.Extract(
master_phil,
cmdline_args=args,
local_overrides="geometry.parameters.random_seed = 2",
)
mydetector = models.detector
mygonio = models.goniometer
crystal2 = models.crystal
mybeam = models.beam
# Build a mock scan for an 18 degree sequence
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 180),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(180)),
deg=True,
)
sequence_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sequence_range == (0.0, pi / 10)
assert approx_equal(im_width, 0.1 * pi / 180.0)
# Build an experiment list
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal1,
imageset=None,
))
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal2,
imageset=None,
))
assert len(experiments.detectors()) == 1
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xl1o_param = CrystalOrientationParameterisation(crystal1)
xl1uc_param = CrystalUnitCellParameterisation(crystal1)
xl2o_param = CrystalOrientationParameterisation(crystal2)
xl2uc_param = CrystalUnitCellParameterisation(crystal2)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
################################
# Apply known parameter shifts #
################################
# shift detector by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [
a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])
]
det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = []
for xlo in (xl1o_param, xl2o_param):
p_vals = xlo.get_param_vals()
xlo_p_vals.append(p_vals)
new_p_vals = [a + b for a, b in zip(p_vals, [2.0, 2.0, 2.0])]
xlo.set_param_vals(new_p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = []
for xluc, xl in ((xl1uc_param, crystal1), ((xl2uc_param, crystal2))):
p_vals = xluc.get_param_vals()
xluc_p_vals.append(p_vals)
cell_params = xl.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(xl.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc.set_param_vals(X)
#############################
# Generate some reflections #
#############################
# All indices in a 2.5 Angstrom sphere for crystal1
resolution = 2.5
index_generator = IndexGenerator(
crystal1.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices1 = index_generator.to_array()
# All indices in a 2.5 Angstrom sphere for crystal2
resolution = 2.5
index_generator = IndexGenerator(
crystal2.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices2 = index_generator.to_array()
# Predict rays within the sequence range. Set experiment IDs
ray_predictor = ScansRayPredictor(experiments, sequence_range)
obs_refs1 = ray_predictor(indices1, experiment_id=0)
obs_refs1["id"] = flex.int(len(obs_refs1), 0)
obs_refs2 = ray_predictor(indices2, experiment_id=1)
obs_refs2["id"] = flex.int(len(obs_refs2), 1)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs1)
obs_refs1 = obs_refs1.select(intersects)
intersects = ray_intersection(mydetector, obs_refs2)
obs_refs2 = obs_refs2.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ScansExperimentsPredictor(experiments)
obs_refs1 = ref_predictor(obs_refs1)
obs_refs2 = ref_predictor(obs_refs2)
# Set 'observed' centroids from the predicted ones
obs_refs1["xyzobs.mm.value"] = obs_refs1["xyzcal.mm"]
obs_refs2["xyzobs.mm.value"] = obs_refs2["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 18.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs1), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs1), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs1), (im_width / 2.0)**2)
obs_refs1["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
var_x = flex.double(len(obs_refs2), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs2), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs2), (im_width / 2.0)**2)
obs_refs2["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# concatenate reflection lists
obs_refs1.extend(obs_refs2)
obs_refs = obs_refs1
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xl1o_param.set_param_vals(xlo_p_vals[0])
xl2o_param.set_param_vals(xlo_p_vals[1])
xl1uc_param.set_param_vals(xluc_p_vals[0])
xl2uc_param.set_param_vals(xluc_p_vals[1])
# scan static first
params = phil_scope.fetch(source=parse("")).extract()
refiner = RefinerFactory.from_parameters_data_experiments(
params, obs_refs, experiments)
refiner.run()
# scan varying
params.refinement.parameterisation.scan_varying = True
refiner = RefinerFactory.from_parameters_data_experiments(
params, obs_refs, experiments)
refiner.run()
# Ensure all models have scan-varying state set
# (https://github.com/dials/dials/issues/798)
refined_experiments = refiner.get_experiments()
sp = [xl.get_num_scan_points() for xl in refined_experiments.crystals()]
assert sp.count(181) == 2
def test(args=[]):
from math import pi
from cctbx.sgtbx import space_group, space_group_symbols
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import Experiment, ExperimentList
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge
from scitbx import matrix
from scitbx.array_family import flex
# Get modules to build models and minimiser using PHIL
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
import dials.test.algorithms.refinement.setup_minimiser as setup_minimiser
from dials.algorithms.refinement.parameterisation.beam_parameters import (
BeamParameterisation,
)
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
# Model parameterisations
from dials.algorithms.refinement.parameterisation.detector_parameters import (
DetectorParameterisationSinglePanel,
)
# Parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import (
XYPhiPredictionParameterisation,
)
from dials.algorithms.refinement.prediction.managed_predictors import (
ScansExperimentsPredictor,
ScansRayPredictor,
)
from dials.algorithms.refinement.reflection_manager import ReflectionManager
# Imports for the target function
from dials.algorithms.refinement.target import (
LeastSquaresPositionalResidualWithRmsdCutoff,
)
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection
#############################
# Setup experimental models #
#############################
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
models = setup_geometry.Extract(master_phil, cmdline_args=args)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Build a mock scan for a 180 degree sequence
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(1800)),
deg=True,
)
sequence_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sequence_range == (0.0, pi)
assert approx_equal(im_width, 0.1 * pi / 180.0)
# Build an experiment list
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
)
)
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Fix crystal parameters
# xluc_param.set_fixed([True, True, True, True, True, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
pred_param = XYPhiPredictionParameterisation(
experiments, [det_param], [s0_param], [xlo_param], [xluc_param]
)
################################
# Apply known parameter shifts #
################################
# shift detector by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])]
det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])]
xlo_param.set_param_vals(p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = mycrystal.get_unit_cell().parameters()
cell_params = [a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(mycrystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
#############################
# Generate some reflections #
#############################
print("Reflections will be generated with the following geometry:")
print(mybeam)
print(mydetector)
print(mycrystal)
print("Target values of parameters are")
msg = "Parameters: " + "%.5f " * len(pred_param)
print(msg % tuple(pred_param.get_param_vals()))
print()
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# Predict rays within the sequence range
ray_predictor = ScansRayPredictor(experiments, sequence_range)
obs_refs = ray_predictor(indices)
print("Total number of reflections excited", len(obs_refs))
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ScansExperimentsPredictor(experiments)
obs_refs["id"] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0) ** 2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0) ** 2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0) ** 2)
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
print("Total number of observations made", len(obs_refs))
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
print("Initial values of parameters are")
msg = "Parameters: " + "%.5f " * len(pred_param)
print(msg % tuple(pred_param.get_param_vals()))
print()
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(obs_refs, experiments)
##############################
# Set up the target function #
##############################
# The current 'achieved' criterion compares RMSD against 1/3 the pixel size and
# 1/3 the image width in radians. For the simulated data, these are just made up
mytarget = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments, ref_predictor, refman, pred_param, restraints_parameterisation=None
)
################################
# Set up the refinement engine #
################################
refiner = setup_minimiser.Extract(
master_phil, mytarget, pred_param, cmdline_args=args
).refiner
print("Prior to refinement the experimental model is:")
print(mybeam)
print(mydetector)
print(mycrystal)
refiner.run()
print()
print("Refinement has completed with the following geometry:")
print(mybeam)
print(mydetector)
print(mycrystal)
def test():
# Python and cctbx imports
from math import pi
from cctbx.sgtbx import space_group, space_group_symbols
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import Experiment, ExperimentList
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge
from scitbx import matrix
from scitbx.array_family import flex
# Get modules to build models and minimiser using PHIL
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
import dials.test.algorithms.refinement.setup_minimiser as setup_minimiser
from dials.algorithms.refinement.parameterisation.beam_parameters import (
BeamParameterisation,
)
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
# Model parameterisations
from dials.algorithms.refinement.parameterisation.detector_parameters import (
DetectorParameterisationSinglePanel,
)
# Parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import (
XYPhiPredictionParameterisation,
)
from dials.algorithms.refinement.prediction.managed_predictors import (
ScansExperimentsPredictor,
ScansRayPredictor,
)
from dials.algorithms.refinement.reflection_manager import ReflectionManager
# Imports for the target function
from dials.algorithms.refinement.target import (
LeastSquaresPositionalResidualWithRmsdCutoff,
)
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection
#############################
# Setup experimental models #
#############################
override = """geometry.parameters
{
beam.wavelength.random=False
beam.wavelength.value=1.0
beam.direction.inclination.random=False
crystal.a.length.random=False
crystal.a.length.value=12.0
crystal.a.direction.method=exactly
crystal.a.direction.exactly.direction=1.0 0.002 -0.004
crystal.b.length.random=False
crystal.b.length.value=14.0
crystal.b.direction.method=exactly
crystal.b.direction.exactly.direction=-0.002 1.0 0.002
crystal.c.length.random=False
crystal.c.length.value=13.0
crystal.c.direction.method=exactly
crystal.c.direction.exactly.direction=0.002 -0.004 1.0
detector.directions.method=exactly
detector.directions.exactly.dir1=0.99 0.002 -0.004
detector.directions.exactly.norm=0.002 -0.001 0.99
detector.centre.method=exactly
detector.centre.exactly.value=1.0 -0.5 199.0
}"""
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
models = setup_geometry.Extract(
master_phil, local_overrides=override, verbose=False
)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
# Build a mock scan for a 180 degree sequence
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(1800)),
deg=True,
)
# Build an ExperimentList
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
)
)
# Create the PredictionParameterisation
pred_param = XYPhiPredictionParameterisation(
experiments, [det_param], [s0_param], [xlo_param], [xluc_param]
)
################################
# Apply known parameter shifts #
################################
# shift detector by 1.0 mm each translation and 4 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 4.0, 4.0, 4.0])]
det_param.set_param_vals(p_vals)
# shift beam by 4 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 4.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=3 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [3.0, 3.0, 3.0])]
xlo_param.set_param_vals(p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# alpha and beta angles)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = mycrystal.get_unit_cell().parameters()
cell_params = [a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1, -0.1, 0.0])]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(mycrystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
sequence_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sequence_range == (0.0, pi)
assert approx_equal(im_width, 0.1 * pi / 180.0)
# Predict rays within the sequence range
ray_predictor = ScansRayPredictor(experiments, sequence_range)
obs_refs = ray_predictor(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ScansExperimentsPredictor(experiments)
obs_refs["id"] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0) ** 2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0) ** 2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0) ** 2)
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# The total number of observations should be 1128
assert len(obs_refs) == 1128
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(
obs_refs, experiments, outlier_detector=None, close_to_spindle_cutoff=0.1
)
##############################
# Set up the target function #
##############################
# The current 'achieved' criterion compares RMSD against 1/3 the pixel size and
# 1/3 the image width in radians. For the simulated data, these are just made up
mytarget = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments, ref_predictor, refman, pred_param, restraints_parameterisation=None
)
######################################
# Set up the LSTBX refinement engine #
######################################
overrides = """minimiser.parameters.engine=GaussNewton
minimiser.parameters.logfile=None"""
refiner = setup_minimiser.Extract(
master_phil, mytarget, pred_param, local_overrides=overrides
).refiner
refiner.run()
assert mytarget.achieved()
assert refiner.get_num_steps() == 1
assert approx_equal(
mytarget.rmsds(), (0.00508252354876, 0.00420954552156, 8.97303428289e-05)
)
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
######################################################
# Set up the LBFGS with curvatures refinement engine #
######################################################
overrides = """minimiser.parameters.engine=LBFGScurvs
minimiser.parameters.logfile=None"""
refiner = setup_minimiser.Extract(
master_phil, mytarget, pred_param, local_overrides=overrides
).refiner
refiner.run()
assert mytarget.achieved()
assert refiner.get_num_steps() == 9
assert approx_equal(
mytarget.rmsds(), (0.0558857700305, 0.0333446685335, 0.000347402754278)
)
def tst_use_in_stills_parameterisation_for_crystal(crystal_param=0):
# test use of analytical expression in stills prediction parameterisation
from scitbx import matrix
from math import pi, sqrt, atan2
import random
print()
print("Test use of analytical expressions in stills prediction " +
"parameterisation for crystal parameters")
# crystal model
from dxtbx.model.crystal import crystal_model
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
crystal = crystal_model((20, 0, 0), (0, 30, 0), (0, 0, 40),
space_group_symbol="P 1")
# random reorientation
e = matrix.col(
(random.random(), random.random(), random.random())).normalize()
angle = random.random() * 180
crystal.rotate_around_origin(e, angle)
wl = 1.1
s0 = matrix.col((0, 0, 1 / wl))
s0u = s0.normalize()
# these are stills, but need a rotation axis for the Reeke algorithm
axis = matrix.col((1, 0, 0))
# crystal parameterisations
xlop = CrystalOrientationParameterisation(crystal)
xlucp = CrystalUnitCellParameterisation(crystal)
# Find some reflections close to the Ewald sphere
from dials.algorithms.spot_prediction.reeke import reeke_model
U = crystal.get_U()
B = crystal.get_B()
UB = U * B
dmin = 4
hkl = reeke_model(UB, UB, axis, s0, dmin, margin=1).generate_indices()
# choose first reflection for now, calc quantities relating to q
h = matrix.col(hkl[0])
q = UB * h
q0 = q.normalize()
q_scalar = q.length()
qq = q_scalar * q_scalar
# calculate the axis of closest rotation
e1 = q0.cross(s0u).normalize()
# calculate c0, a vector orthogonal to s0u and e1
c0 = s0u.cross(e1).normalize()
# calculate q1
q1 = q0.cross(e1).normalize()
# calculate DeltaPsi
a = 0.5 * qq * wl
b = sqrt(qq - a * a)
r = -1.0 * a * s0u + b * c0
DeltaPsi = -1.0 * atan2(r.dot(q1), r.dot(q0))
# Checks on the reflection prediction
from libtbx.test_utils import approx_equal
# 1. check r is on the Ewald sphere
s1 = s0 + r
assert approx_equal(s1.length(), s0.length())
# 2. check DeltaPsi is correct
tst = q.rotate_around_origin(e1, DeltaPsi)
assert approx_equal(tst, r)
# choose the derivative with respect to a particular parameter.
if crystal_param < 3:
dU_dp = xlop.get_ds_dp()[crystal_param]
dq = dU_dp * B * h
else:
dB_dp = xlucp.get_ds_dp()[crystal_param - 3]
dq = U * dB_dp * h
# NKS method of calculating d[q0]/dp
q_dot_dq = q.dot(dq)
dqq = 2.0 * q_dot_dq
dq_scalar = dqq / q_scalar
dq0_dp = (q_scalar * dq - (q_dot_dq * q0)) / qq
# orthogonal to q0, as expected.
print("NKS [q0].(d[q0]/dp) = {0} (should be 0.0)".format(q0.dot(dq0_dp)))
# intuitive method of calculating d[q0]/dp, based on the fact that
# it must be orthogonal to q0, i.e. in the plane containing q1 and e1
scaled = dq / q.length()
dq0_dp = scaled.dot(q1) * q1 + scaled.dot(e1) * e1
# orthogonal to q0, as expected.
print("DGW [q0].(d[q0]/dp) = {0} (should be 0.0)".format(q0.dot(dq0_dp)))
# So it doesn't matter which method I use to calculate d[q0]/dp, as
# both methods give the same results
# use the fact that -e1 == q0.cross(q1) to redefine the derivative d[e1]/dp
# from Sauter et al. (2014) (A.22)
de1_dp = -1.0 * dq0_dp.cross(q1)
# this *is* orthogonal to e1, as expected.
print("[e1].(d[e1]/dp) = {0} (should be 0.0)".format(e1.dot(de1_dp)))
# calculate (d[r]/d[e1])(d[e1]/dp) analytically
from scitbx.array_family import flex
from dials_refinement_helpers_ext import dRq_de
dr_de1 = matrix.sqr(
dRq_de(flex.double([DeltaPsi]), flex.vec3_double([e1]),
flex.vec3_double([q]))[0])
print("Analytical calculation for (d[r]/d[e1])(d[e1]/dp):")
print(dr_de1 * de1_dp)
# now calculate using finite differences.
dp = 1.0e-8
del_e1 = de1_dp * dp
e1f = e1 + del_e1 * 0.5
rfwd = q.rotate_around_origin(e1f, DeltaPsi)
e1r = e1 - del_e1 * 0.5
rrev = q.rotate_around_origin(e1r, DeltaPsi)
print("Finite difference estimate for (d[r]/d[e1])(d[e1]/dp):")
print((rfwd - rrev) * (1 / dp))
print("These are essentially the same :-)")
