def refine(self, do_print=False):
crystal = self.crystal
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation,
CrystalOrientationParameterisation,
)
self.cucp = CrystalUnitCellParameterisation(crystal)
self.cop = CrystalOrientationParameterisation(crystal)
self.zero()
# 0-point and deltas
values = flex.double(self.cucp.get_param_vals() +
self.cop.get_param_vals())
offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] +
[0.1, 0.1, 0.1])
initial = crystal.get_unit_cell()
self.cells = []
self.best_score = 1e99
initial_score = self.target(values)
doohicky = simple_simplex(values, offset, self, 2000)
best = doohicky.get_solution()
if do_print:
print("Initial cell:", initial)
print("Final cell: ", crystal.get_unit_cell())
print("Score change", initial_score,
self.target(best, do_print=False))
self.best = best
def __init__(
self,
experiment,
mosaicity_parameterisation,
fix_mosaic_spread=False,
fix_wavelength_spread=False,
fix_unit_cell=False,
fix_orientation=False,
):
"""
Initialise the model state
"""
# Save the crystal model
self.experiment = experiment
self.crystal = experiment.crystal
# The U and P parameterisation
self.U_parameterisation = CrystalOrientationParameterisation(
self.crystal)
self.B_parameterisation = CrystalUnitCellParameterisation(self.crystal)
# The M, L and W parameterisations
self.M_parameterisation = mosaicity_parameterisation
# Set the flags to fix parameters
self._is_mosaic_spread_fixed = fix_mosaic_spread
self._is_wavelength_spread_fixed = fix_wavelength_spread
self._is_unit_cell_fixed = fix_unit_cell
self._is_orientation_fixed = fix_orientation
def create_models(self):
# build models, with a larger crystal than default in order to get plenty of
# reflections on the 'still' image
overrides = """
geometry.parameters.crystal.a.length.range=40 50;
geometry.parameters.crystal.b.length.range=40 50;
geometry.parameters.crystal.c.length.range=40 50;
geometry.parameters.random_seed = 42"""
master_phil = parse("""
include scope dials.test.algorithms.refinement.geometry_phil
""",
process_includes=True)
models = Extract(master_phil, overrides)
# keep track of the models
self.detector = models.detector
self.gonio = models.goniometer
self.crystal = models.crystal
self.beam = models.beam
# Create a stills ExperimentList
self.stills_experiments = ExperimentList()
self.stills_experiments.append(
Experiment(beam=self.beam,
detector=self.detector,
crystal=self.crystal,
imageset=None))
# keep track of the parameterisation of the models
self.det_param = DetectorParameterisationSinglePanel(self.detector)
self.s0_param = BeamParameterisation(self.beam, self.gonio)
self.xlo_param = CrystalOrientationParameterisation(self.crystal)
self.xluc_param = CrystalUnitCellParameterisation(self.crystal)
def create_refiner(params, reflections, experiments):
# Only parameterise the crystal unit cell
det_params = None
beam_params = None
xlo_params = None
xluc_params = []
for crystal in experiments.crystals():
exp_ids = experiments.indices(crystal)
xluc_params.append(
CrystalUnitCellParameterisation(crystal, experiment_ids=exp_ids)
)
# Two theta prediction equation parameterisation
pred_param = TwoThetaPredictionParameterisation(
experiments, det_params, beam_params, xlo_params, xluc_params
)
param_reporter = ParameterReporter(
det_params, beam_params, xlo_params, xluc_params
)
# ReflectionManager, currently without outlier rejection
# Note: If not all reflections are used, then the filtering must be
# communicated to generate_cif/mmcif() to be included in the CIF file!
refman = TwoThetaReflectionManager(
reflections, experiments, outlier_detector=None
)
# Reflection predictor
ref_predictor = TwoThetaExperimentsPredictor(experiments)
# Two theta target
target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param)
# Switch on correlation matrix tracking if a correlation plot is requested
journal = None
if params.output.correlation_plot.filename is not None:
journal = refinery_phil_scope.extract().refinery.journal
journal.track_parameter_correlation = True
# Minimisation engine - hardcoded to LevMar for now.
refinery = Refinery(
target=target,
prediction_parameterisation=pred_param,
log=None,
tracking=journal,
max_iterations=20,
)
# Refiner
refiner = Refiner(
experiments=experiments,
pred_param=pred_param,
param_reporter=param_reporter,
refman=refman,
target=target,
refinery=refinery,
)
return refiner
def __init__(self, experiment, reflections, parameters):
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation, )
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation, )
# Store the input
self.experiment = experiment
self.reflections = reflections
self.parameters = parameters
# Get the crystal model and the parameterisation
self.crystal = self.experiment.crystal
self.cucp = CrystalUnitCellParameterisation(self.crystal)
self.cop = CrystalOrientationParameterisation(self.crystal)
# Get the current values and generate some offsets
values = flex.double(self.cucp.get_param_vals() +
self.cop.get_param_vals())
offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] +
[0.1, 0.1, 0.1])
# The optimization history
self.history = []
# Get the initial cell and initial score
initial_cell = self.crystal.get_unit_cell()
initial_orientation = self.crystal.get_U()
initial_score = self.target(values)
# Perform the optimization
optimizer = simple_simplex(values, offset, self, 2000)
result = optimizer.get_solution()
# Print some information
fmt = "(%.3f, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f)"
print("Initial cell:", initial_cell)
print("Final cell: ", self.crystal.get_unit_cell())
print("Initial orientation: ", fmt % tuple(initial_orientation))
print("Final orientation: ", fmt % tuple(self.crystal.get_U()))
# Print RMSD
Xobs, Yobs, _ = self.reflections["xyzobs.px.value"].parts()
Xcal, Ycal, _ = self.reflections["xyzcal.px"].parts()
rmsd_x = sqrt(flex.sum((Xcal - Xobs)**2) / len(Xcal))
rmsd_y = sqrt(flex.sum((Ycal - Yobs)**2) / len(Ycal))
print("RMSD X, Y (px): %f, %f" % (rmsd_x, rmsd_y))
def generate_data(experiments, reflections):
from random import seed
seed(0)
index = randint(0, len(reflections))
h = reflections[index]["miller_index"]
s0 = matrix.col(experiments[0].beam.get_s0())
U_param = CrystalOrientationParameterisation(experiments[0].crystal)
B_param = CrystalUnitCellParameterisation(experiments[0].crystal)
U = matrix.sqr(experiments[0].crystal.get_U())
B = matrix.sqr(experiments[0].crystal.get_B())
r = U * B * matrix.col(h)
s2 = s0 + r
mobs = (s2 + matrix.col((uniform(0, 1e-3), uniform(
0, 1e-3), uniform(0, 1e-3)))).normalize() * s0.length()
b1, b2, b3, b4, b5, b6 = (
uniform(1e-3, 3e-3),
uniform(0.0, 1e-3),
uniform(1e-3, 3e-3),
uniform(0.0, 1e-3),
uniform(0.0, 1e-3),
uniform(1e-3, 3e-3),
)
params = (b1, b2, b3, b4, b5, b6)
S_param = SimpleMosaicityParameterisation(params)
L_param = (uniform(1e-3, 2e-3), )
W_param = (uniform(1e-3, 2e-3), uniform(0, 1e-3), uniform(1e-3, 2e-3))
ctot = randint(100, 1000)
T = matrix.sqr(
(uniform(1e-3, 2e-3), 0, uniform(1e-6, 2e-6), uniform(1e-3, 2e-3)))
Sobs = T * T.transpose()
params = [S_param, U_param, B_param, L_param, W_param]
return params, s0, h, ctot, mobs, Sobs
def generate_data(experiments, reflections):
from random import seed
seed(0)
index = randint(0, len(reflections))
h = reflections[index]["miller_index"]
s0 = matrix.col(experiments[0].beam.get_s0())
U_param = CrystalOrientationParameterisation(experiments[0].crystal)
B_param = CrystalUnitCellParameterisation(experiments[0].crystal)
U = matrix.sqr(experiments[0].crystal.get_U())
B = matrix.sqr(experiments[0].crystal.get_B())
r = U * B * matrix.col(h)
s2 = s0 + r
mobs = (s2 + matrix.col((uniform(0, 1e-3), uniform(
0, 1e-3), uniform(0, 1e-3)))).normalize() * s0.length()
mobs = np.array([uniform(0, 1e-3), uniform(0, 1e-3)])
sp = s2.normalize() * s0.length()
b1, b2, b3, b4, b5, b6 = (
uniform(1e-3, 3e-3),
uniform(0.0, 1e-3),
uniform(1e-3, 3e-3),
uniform(0.0, 1e-3),
uniform(0.0, 1e-3),
uniform(1e-3, 3e-3),
)
S_param = (b1, b2, b3, b4, b5, b6)
L_param = (uniform(1e-3, 2e-3), )
ctot = randint(100, 1000)
T = np.array((uniform(1e-3, 2e-3), 0, uniform(1e-6, 2e-6),
uniform(1e-3, 2e-3))).reshape(2, 2)
Sobs = np.matmul(T, T.T)
params = [S_param, U_param, B_param, L_param]
return params, s0, sp, h, ctot, mobs, Sobs
def __init__(self, reflection_file, experiment_file):
# make a pie
data = pickle.load(open(reflection_file, "rb"))
print("%d reflections" % data.size())
self.data = data.select(data["intensity.sum.variance"] > 0)
i = data["intensity.sum.value"]
v = data["intensity.sum.variance"]
s = flex.sqrt(v)
self.i_s = i / s
from dxtbx.model.experiment_list import ExperimentListFactory
expt = ExperimentListFactory.from_json_file(experiment_file)
crystal = expt.crystals()[0]
self.s0 = matrix.col(expt.beams()[0].get_s0())
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation,
CrystalOrientationParameterisation,
)
self.crystal = crystal
self.cucp = CrystalUnitCellParameterisation(crystal)
self.cop = CrystalOrientationParameterisation(crystal)
# 0-point and deltas
values = flex.double(self.cucp.get_param_vals() +
self.cop.get_param_vals())
offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] +
[0.1, 0.1, 0.1])
initial = crystal.get_unit_cell()
initial_score = self.target(values)
doohicky = simple_simplex(values, offset, self, 2000)
best = doohicky.get_solution()
print("Initial cell:", initial)
print("Final cell: ", crystal.get_unit_cell())
print("Score change", initial_score, self.target(best, do_print=False))
self.best = best
def __init__(
self,
experiment,
mosaicity_parameterisation,
wavelength_parameterisation=None,
fix_mosaic_spread=False,
fix_wavelength_spread=True,
fix_unit_cell=False,
fix_orientation=False,
):
"""
Initialise the model state
"""
# Save the crystal model
self.experiment = experiment
self.crystal = experiment.crystal
# The U and P parameterisation
self._U_parameterisation = CrystalOrientationParameterisation(
self.crystal)
self._B_parameterisation = CrystalUnitCellParameterisation(
self.crystal)
# The M and L parameterisations
self._M_parameterisation = mosaicity_parameterisation
self._L_parameterisation = wavelength_parameterisation
# Set the flags to fix parameters
self._is_mosaic_spread_fixed = fix_mosaic_spread
self._is_wavelength_spread_fixed = fix_wavelength_spread
self._is_unit_cell_fixed = fix_unit_cell
self._is_orientation_fixed = fix_orientation
# Check wavelength parameterisation
if not self.is_wavelength_spread_fixed:
assert self._L_parameterisation is not None
def test():
from cctbx.sgtbx import space_group, space_group_symbols
from dxtbx.model.experiment_list import Experiment, ExperimentList
from libtbx.phil import parse
from scitbx.array_family import flex
from dials.algorithms.refinement.parameterisation.beam_parameters import (
BeamParameterisation, )
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
from dials.algorithms.refinement.parameterisation.detector_parameters import (
DetectorParameterisationSinglePanel, )
from dials.algorithms.refinement.parameterisation.goniometer_parameters import (
GoniometerParameterisation, )
#### Import model parameterisations
from dials.algorithms.refinement.parameterisation.prediction_parameters import (
XYPhiPredictionParameterisation, )
from dials.algorithms.refinement.prediction.managed_predictors import (
ScansExperimentsPredictor,
ScansRayPredictor,
)
##### Imports for reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection
##### Import model builder
from dials.tests.algorithms.refinement.setup_geometry import Extract
#### Create models
overrides = """geometry.parameters.crystal.a.length.range = 10 50
geometry.parameters.crystal.b.length.range = 10 50
geometry.parameters.crystal.c.length.range = 10 50"""
master_phil = parse(
"""
include scope dials.tests.algorithms.refinement.geometry_phil
""",
process_includes=True,
)
models = Extract(master_phil, overrides)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Build a mock scan for a 72 degree sequence
sequence_range = (0.0, math.pi / 5.0)
from dxtbx.model import ScanFactory
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 720),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(720)),
deg=True,
)
#### Create parameterisations of these models
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
gon_param = GoniometerParameterisation(mygonio, mybeam)
# Create an ExperimentList
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
#### Unit tests
# Build a prediction parameterisation
pred_param = XYPhiPredictionParameterisation(
experiments,
detector_parameterisations=[det_param],
beam_parameterisations=[s0_param],
xl_orientation_parameterisations=[xlo_param],
xl_unit_cell_parameterisations=[xluc_param],
goniometer_parameterisations=[gon_param],
)
# Generate reflections
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# Predict rays within the sequence range
ray_predictor = ScansRayPredictor(experiments, sequence_range)
obs_refs = ray_predictor(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ScansExperimentsPredictor(experiments)
obs_refs["id"] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * math.pi / 180.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2)
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# use a ReflectionManager to exclude reflections too close to the spindle
from dials.algorithms.refinement.reflection_manager import ReflectionManager
refman = ReflectionManager(obs_refs, experiments, outlier_detector=None)
refman.finalise()
# Redefine the reflection predictor to use the type expected by the Target class
ref_predictor = ScansExperimentsPredictor(experiments)
# keep only those reflections that pass inclusion criteria and have predictions
reflections = refman.get_matches()
# get analytical gradients
an_grads = pred_param.get_gradients(reflections)
# get finite difference gradients
p_vals = pred_param.get_param_vals()
deltas = [1.0e-7] * len(p_vals)
for i, delta in enumerate(deltas):
val = p_vals[i]
p_vals[i] -= delta / 2.0
pred_param.set_param_vals(p_vals)
ref_predictor(reflections)
rev_state = reflections["xyzcal.mm"].deep_copy()
p_vals[i] += delta
pred_param.set_param_vals(p_vals)
ref_predictor(reflections)
fwd_state = reflections["xyzcal.mm"].deep_copy()
p_vals[i] = val
fd = fwd_state - rev_state
x_grads, y_grads, phi_grads = fd.parts()
x_grads /= delta
y_grads /= delta
phi_grads /= delta
# compare with analytical calculation
assert x_grads == pytest.approx(an_grads[i]["dX_dp"], abs=5.0e-6)
assert y_grads == pytest.approx(an_grads[i]["dY_dp"], abs=5.5e-6)
assert phi_grads == pytest.approx(an_grads[i]["dphi_dp"], abs=5.0e-6)
# return to the initial state
pred_param.set_param_vals(p_vals)
def test(args=[]):
#############################
# Setup experimental models #
#############################
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
models = setup_geometry.Extract(master_phil, cmdline_args=args)
single_panel_detector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Make a 3x3 multi panel detector filling the same space as the existing
# single panel detector. Each panel of the multi-panel detector has pixels with
# 1/3 the length dimensions of the single panel.
multi_panel_detector = Detector()
for x in range(3):
for y in range(3):
new_panel = make_panel_in_array((x, y), single_panel_detector[0])
multi_panel_detector.add_panel(new_panel)
# Build a mock scan for a 180 degree sweep
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(1800)),
deg=True,
)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sweep_range == (0.0, pi)
assert approx_equal(im_width, 0.1 * pi / 180.0)
# Build ExperimentLists
experiments_single_panel = ExperimentList()
experiments_multi_panel = ExperimentList()
experiments_single_panel.append(
Experiment(
beam=mybeam,
detector=single_panel_detector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
experiments_multi_panel.append(
Experiment(
beam=mybeam,
detector=multi_panel_detector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(single_panel_detector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
multi_det_param = DetectorParameterisationMultiPanel(
multi_panel_detector, mybeam)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Fix crystal parameters
# xluc_param.set_fixed([True, True, True, True, True, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
pred_param = XYPhiPredictionParameterisation(experiments_single_panel,
[det_param], [s0_param],
[xlo_param], [xluc_param])
pred_param2 = XYPhiPredictionParameterisation(
experiments_multi_panel,
[multi_det_param],
[s0_param],
[xlo_param],
[xluc_param],
)
################################
# Apply known parameter shifts #
################################
# shift detectors by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [
a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])
]
det_param.set_param_vals(p_vals)
multi_det_p_vals = multi_det_param.get_param_vals()
p_vals = [
a + b for a, b in zip(multi_det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])
]
multi_det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])]
xlo_param.set_param_vals(p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = mycrystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(mycrystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# for the reflection predictor, it doesn't matter which experiment list is
# passed, as the detector is not used
ref_predictor = ScansRayPredictor(experiments_single_panel, sweep_range)
# get two sets of identical reflections
obs_refs = ref_predictor(indices)
obs_refs2 = ref_predictor(indices)
for r1, r2 in zip(obs_refs, obs_refs2):
assert r1["s1"] == r2["s1"]
# get the panel intersections
sel = ray_intersection(single_panel_detector, obs_refs)
obs_refs = obs_refs.select(sel)
sel = ray_intersection(multi_panel_detector, obs_refs2)
obs_refs2 = obs_refs2.select(sel)
assert len(obs_refs) == len(obs_refs2)
# Set 'observed' centroids from the predicted ones
obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"]
obs_refs2["xyzobs.mm.value"] = obs_refs2["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.0
px_size = single_panel_detector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2)
# set the variances and frame numbers
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
obs_refs2["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# Add in flags and ID columns by copying into standard reflection tables
tmp = flex.reflection_table.empty_standard(len(obs_refs))
tmp.update(obs_refs)
obs_refs = tmp
tmp = flex.reflection_table.empty_standard(len(obs_refs2))
tmp.update(obs_refs2)
obs_refs2 = tmp
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
multi_det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(obs_refs, experiments_single_panel)
refman2 = ReflectionManager(obs_refs, experiments_multi_panel)
###############################
# Set up the target functions #
###############################
mytarget = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments_single_panel,
ScansExperimentsPredictor(experiments_single_panel),
refman,
pred_param,
restraints_parameterisation=None,
)
mytarget2 = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments_multi_panel,
ScansExperimentsPredictor(experiments_multi_panel),
refman2,
pred_param2,
restraints_parameterisation=None,
)
#################################
# Set up the refinement engines #
#################################
refiner = setup_minimiser.Extract(master_phil,
mytarget,
pred_param,
cmdline_args=args).refiner
refiner2 = setup_minimiser.Extract(master_phil,
mytarget2,
pred_param2,
cmdline_args=args).refiner
refiner.run()
# reset parameters and run refinement with the multi panel detector
s0_param.set_param_vals(s0_p_vals)
multi_det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
refiner2.run()
# same number of steps
assert refiner.get_num_steps() == refiner2.get_num_steps()
# same rmsds
for rmsd, rmsd2 in zip(refiner.history["rmsd"], refiner2.history["rmsd"]):
assert approx_equal(rmsd, rmsd2)
# same parameter values each step
for params, params2 in zip(refiner.history["parameter_vector"],
refiner.history["parameter_vector"]):
assert approx_equal(params, params2)
def test():
# Python and cctbx imports
from math import pi
from scitbx import matrix
from scitbx.array_family import flex
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
# Get modules to build models and minimiser using PHIL
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
import dials.test.algorithms.refinement.setup_minimiser as setup_minimiser
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Model parameterisations
from dials.algorithms.refinement.parameterisation.detector_parameters import (
DetectorParameterisationSinglePanel, )
from dials.algorithms.refinement.parameterisation.beam_parameters import (
BeamParameterisation, )
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection
from dials.algorithms.refinement.prediction.managed_predictors import (
ScansRayPredictor,
ScansExperimentsPredictor,
)
from cctbx.sgtbx import space_group, space_group_symbols
# Parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import (
XYPhiPredictionParameterisation, )
# Imports for the target function
from dials.algorithms.refinement.target import (
LeastSquaresPositionalResidualWithRmsdCutoff, )
from dials.algorithms.refinement.reflection_manager import ReflectionManager
#############################
# Setup experimental models #
#############################
override = """geometry.parameters
{
beam.wavelength.random=False
beam.wavelength.value=1.0
beam.direction.inclination.random=False
crystal.a.length.random=False
crystal.a.length.value=12.0
crystal.a.direction.method=exactly
crystal.a.direction.exactly.direction=1.0 0.002 -0.004
crystal.b.length.random=False
crystal.b.length.value=14.0
crystal.b.direction.method=exactly
crystal.b.direction.exactly.direction=-0.002 1.0 0.002
crystal.c.length.random=False
crystal.c.length.value=13.0
crystal.c.direction.method=exactly
crystal.c.direction.exactly.direction=0.002 -0.004 1.0
detector.directions.method=exactly
detector.directions.exactly.dir1=0.99 0.002 -0.004
detector.directions.exactly.norm=0.002 -0.001 0.99
detector.centre.method=exactly
detector.centre.exactly.value=1.0 -0.5 199.0
}"""
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
models = setup_geometry.Extract(master_phil,
local_overrides=override,
verbose=False)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
# Build a mock scan for a 180 degree sweep
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(1800)),
deg=True,
)
# Build an ExperimentList
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
# Create the PredictionParameterisation
pred_param = XYPhiPredictionParameterisation(experiments, [det_param],
[s0_param], [xlo_param],
[xluc_param])
################################
# Apply known parameter shifts #
################################
# shift detector by 1.0 mm each translation and 4 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [
a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 4.0, 4.0, 4.0])
]
det_param.set_param_vals(p_vals)
# shift beam by 4 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 4.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=3 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [3.0, 3.0, 3.0])]
xlo_param.set_param_vals(p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# alpha and beta angles)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = mycrystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1, -0.1, 0.0])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(mycrystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sweep_range == (0.0, pi)
assert approx_equal(im_width, 0.1 * pi / 180.0)
# Predict rays within the sweep range
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs = ray_predictor(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ScansExperimentsPredictor(experiments)
obs_refs["id"] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2)
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# The total number of observations should be 1128
assert len(obs_refs) == 1128
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(obs_refs,
experiments,
outlier_detector=None,
close_to_spindle_cutoff=0.1)
##############################
# Set up the target function #
##############################
# The current 'achieved' criterion compares RMSD against 1/3 the pixel size and
# 1/3 the image width in radians. For the simulated data, these are just made up
mytarget = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments,
ref_predictor,
refman,
pred_param,
restraints_parameterisation=None)
######################################
# Set up the LSTBX refinement engine #
######################################
overrides = """minimiser.parameters.engine=GaussNewton
minimiser.parameters.logfile=None"""
refiner = setup_minimiser.Extract(master_phil,
mytarget,
pred_param,
local_overrides=overrides).refiner
refiner.run()
assert mytarget.achieved()
assert refiner.get_num_steps() == 1
assert approx_equal(
mytarget.rmsds(),
(0.00508252354876, 0.00420954552156, 8.97303428289e-05))
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
######################################################
# Set up the LBFGS with curvatures refinement engine #
######################################################
overrides = """minimiser.parameters.engine=LBFGScurvs
minimiser.parameters.logfile=None"""
refiner = setup_minimiser.Extract(master_phil,
mytarget,
pred_param,
local_overrides=overrides).refiner
refiner.run()
assert mytarget.achieved()
assert refiner.get_num_steps() == 9
assert approx_equal(mytarget.rmsds(),
(0.0558857700305, 0.0333446685335, 0.000347402754278))
def test(args=[]):
# Python and cctbx imports
from math import pi
import random
from scitbx import matrix
from scitbx.array_family import flex
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
# Experimental model builder
from dials.test.algorithms.refinement.setup_geometry import Extract
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Model parameterisations
from dials.algorithms.refinement.parameterisation.detector_parameters import (
DetectorParameterisationSinglePanel, )
from dials.algorithms.refinement.parameterisation.beam_parameters import (
BeamParameterisation, )
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection
from dials.algorithms.refinement.prediction.managed_predictors import (
ScansRayPredictor,
ScansExperimentsPredictor,
)
from cctbx.sgtbx import space_group, space_group_symbols
# Parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import (
XYPhiPredictionParameterisation, )
# Imports for the target function
from dials.algorithms.refinement.target import (
LeastSquaresPositionalResidualWithRmsdCutoff, )
from dials.algorithms.refinement.reflection_manager import ReflectionManager
# Local functions
def random_direction_close_to(vector, sd=0.5):
return vector.rotate_around_origin(
matrix.col((random.random(), random.random(),
random.random())).normalize(),
random.gauss(0, sd),
deg=True,
)
#############################
# Setup experimental models #
#############################
# make a small cell to speed up calculations
overrides = """geometry.parameters.crystal.a.length.range = 10 15
geometry.parameters.crystal.b.length.range = 10 15
geometry.parameters.crystal.c.length.range = 10 15"""
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
""",
process_includes=True,
)
models = Extract(master_phil, overrides, cmdline_args=args)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Build a mock scan for a 180 degree sweep of 0.1 degree images
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(1800)),
deg=True,
)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sweep_range == (0.0, pi)
assert approx_equal(im_width, 0.1 * pi / 180.0)
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
pred_param = XYPhiPredictionParameterisation(experiments, [det_param],
[s0_param], [xlo_param],
[xluc_param])
################################
# Apply known parameter shifts #
################################
# shift detector by 0.2 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [
a + b for a, b in zip(det_p_vals, [2.0, 2.0, 2.0, 2.0, 2.0, 2.0])
]
det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in one axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[1] += 2.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])]
xlo_param.set_param_vals(p_vals)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# Predict rays within the sweep range
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs = ray_predictor(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ScansExperimentsPredictor(experiments)
obs_refs["id"] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2)
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(obs_refs, experiments)
##############################
# Set up the target function #
##############################
# Redefine the reflection predictor to use the type expected by the Target class
ref_predictor = ScansExperimentsPredictor(experiments)
mytarget = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments,
ref_predictor,
refman,
pred_param,
restraints_parameterisation=None)
# get the functional and gradients
mytarget.predict()
L, dL_dp, curvs = mytarget.compute_functional_gradients_and_curvatures()
####################################
# Do FD calculation for comparison #
####################################
# function for calculating finite difference gradients of the target function
def get_fd_gradients(target, pred_param, deltas):
"""Calculate centered finite difference gradients for each of the
parameters of the target function.
"deltas" must be a sequence of the same length as the parameter list, and
contains the step size for the difference calculations for each parameter.
"""
p_vals = pred_param.get_param_vals()
assert len(deltas) == len(p_vals)
fd_grad = []
fd_curvs = []
for i in range(len(deltas)):
val = p_vals[i]
p_vals[i] -= deltas[i] / 2.0
pred_param.set_param_vals(p_vals)
target.predict()
rev_state = target.compute_functional_gradients_and_curvatures()
p_vals[i] += deltas[i]
pred_param.set_param_vals(p_vals)
target.predict()
fwd_state = target.compute_functional_gradients_and_curvatures()
# finite difference estimation of first derivatives
fd_grad.append((fwd_state[0] - rev_state[0]) / deltas[i])
# finite difference estimation of curvatures, using the analytical
# first derivatives
fd_curvs.append((fwd_state[1][i] - rev_state[1][i]) / deltas[i])
# set parameter back to centred value
p_vals[i] = val
# return to the initial state
pred_param.set_param_vals(p_vals)
return fd_grad, fd_curvs
# test normalised differences between FD and analytical calculations
fdgrads = get_fd_gradients(mytarget, pred_param,
[1.0e-7] * len(pred_param))
diffs = [a - b for a, b in zip(dL_dp, fdgrads[0])]
norm_diffs = tuple([a / b for a, b in zip(diffs, fdgrads[0])])
for e in norm_diffs:
assert abs(e) < 0.001 # check differences less than 0.1%
# test normalised differences between FD curvatures and analytical least
# squares approximation. We don't expect this to be especially close
if curvs:
diffs = [a - b for a, b in zip(curvs, fdgrads[1])]
norm_diffs = tuple([a / b for a, b in zip(diffs, fdgrads[1])])
for e in norm_diffs:
assert abs(e) < 0.1 # check differences less than 10%
def test_fd_derivatives():
"""Test derivatives of the prediction equation"""
from libtbx.phil import parse
# Import model builder
from dials.test.algorithms.refinement.setup_geometry import Extract
# Create models
overrides = """geometry.parameters.crystal.a.length.range = 10 50
geometry.parameters.crystal.b.length.range = 10 50
geometry.parameters.crystal.c.length.range = 10 50"""
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
""",
process_includes=True,
)
models = Extract(master_phil, overrides)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Build a mock scan for a 72 degree sequence
from dxtbx.model import ScanFactory
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 720),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(720)),
deg=True,
)
# Create a parameterisation of the crystal unit cell
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation, )
xluc_param = CrystalUnitCellParameterisation(mycrystal)
# Create an ExperimentList
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
# Build a prediction parameterisation for two theta prediction
pred_param = TwoThetaPredictionParameterisation(
experiments,
detector_parameterisations=None,
beam_parameterisations=None,
xl_orientation_parameterisations=None,
xl_unit_cell_parameterisations=[xluc_param],
)
# Generate some reflections
obs_refs, ref_predictor = generate_reflections(experiments)
# Build a ReflectionManager with overloads for handling 2theta residuals
refman = TwoThetaReflectionManager(obs_refs,
experiments,
outlier_detector=None)
# Build a TwoThetaExperimentsPredictor
ref_predictor = TwoThetaExperimentsPredictor(experiments)
# Make a target for the least squares 2theta residual
target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param)
# Keep only reflections that pass inclusion criteria and have predictions
reflections = refman.get_matches()
# Get analytical gradients
an_grads = pred_param.get_gradients(reflections)
# Get finite difference gradients
p_vals = pred_param.get_param_vals()
deltas = [1.0e-7] * len(p_vals)
for i in range(len(deltas)):
val = p_vals[i]
p_vals[i] -= deltas[i] / 2.0
pred_param.set_param_vals(p_vals)
target.predict()
reflections = refman.get_matches()
rev_state = reflections["2theta_resid"].deep_copy()
p_vals[i] += deltas[i]
pred_param.set_param_vals(p_vals)
target.predict()
reflections = refman.get_matches()
fwd_state = reflections["2theta_resid"].deep_copy()
p_vals[i] = val
fd = fwd_state - rev_state
fd /= deltas[i]
# compare with analytical calculation
assert approx_equal(fd, an_grads[i]["d2theta_dp"], eps=1.0e-6)
# return to the initial state
pred_param.set_param_vals(p_vals)
def test(dials_regression):
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
from scitbx import matrix
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
# Get modules to build models and minimiser using PHIL
from dials.tests.algorithms.refinement import setup_geometry
# Symmetry constrained parameterisation for the unit cell
DEG2RAD = math.pi / 180.0
RAD2DEG = 180.0 / math.pi
master_phil = parse(
"""
include scope dials.tests.algorithms.refinement.geometry_phil
include scope dials.tests.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
# make cell more oblique
args = [
"a.direction.close_to.sd=5",
"b.direction.close_to.sd=5",
"c.direction.close_to.sd=5",
]
models = setup_geometry.Extract(master_phil, cmdline_args=args)
crystal = models.crystal
# a hexagonal crystal is a good test case for behaviour of oblique cells
do_hexagonal = True
if do_hexagonal:
from dxtbx.model.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(
os.path.join(
dials_regression,
"refinement_test_data",
"multi_stills",
"combined_experiments.json",
),
check_format=False,
)
crystal = experiments[0].crystal
# derive finite difference gradients of various quantities wrt each param
def check_fd_gradients(parameterisation):
mp = parameterisation
p_vals = mp.get_param_vals()
deltas = [1.0e-7 for p in p_vals]
assert len(deltas) == len(p_vals)
fd_grad = []
# get matrix to unset rotations of unit cell vectors
Ut = matrix.sqr(mp.get_model().get_U()).transpose()
for i, delta in enumerate(deltas):
val = p_vals[i]
p_vals[i] -= delta / 2.0
mp.set_param_vals(p_vals)
rev_uc = mp.get_model().get_unit_cell().parameters()
rev_vec = mp.get_model().get_real_space_vectors()
rev_vec = [Ut * vec for vec in rev_vec]
rev_B = matrix.sqr(mp.get_model().get_B())
rev_O = rev_B.transpose().inverse()
p_vals[i] += delta
mp.set_param_vals(p_vals)
fwd_uc = mp.get_model().get_unit_cell().parameters()
fwd_vec = mp.get_model().get_real_space_vectors()
fwd_vec = [Ut * vec for vec in fwd_vec]
fwd_B = matrix.sqr(mp.get_model().get_B())
fwd_O = fwd_B.transpose().inverse()
fd_uc = [(f - r) / delta for f, r in zip(fwd_uc, rev_uc)]
fd_vec = [(f - r) / delta for f, r in zip(fwd_vec, rev_vec)]
fd_B = (fwd_B - rev_B) / delta
fd_O = (fwd_O - rev_O) / delta
fd_grad.append(
{
"da_dp": fd_uc[0],
"db_dp": fd_uc[1],
"dc_dp": fd_uc[2],
"daa_dp": fd_uc[3],
"dbb_dp": fd_uc[4],
"dcc_dp": fd_uc[5],
"davec_dp": fd_vec[0],
"dbvec_dp": fd_vec[1],
"dcvec_dp": fd_vec[2],
"dB_dp": fd_B,
"dO_dp": fd_O,
}
)
p_vals[i] = val
# return to the initial state
mp.set_param_vals(p_vals)
return fd_grad
assert CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
from dials.algorithms.refinement.restraints.restraints import SingleUnitCellTie
assert SingleUnitCellTie(xluc_param, [0] * 6, [0] * 6)
from scitbx.math import angle_derivative_wrt_vectors
B = matrix.sqr(crystal.get_B())
O = (B.transpose()).inverse()
a, b, c, aa, bb, cc = crystal.get_unit_cell().parameters()
aa *= DEG2RAD
bb *= DEG2RAD
cc *= DEG2RAD
Ut = matrix.sqr(crystal.get_U()).transpose()
avec, bvec, cvec = [Ut * vec for vec in crystal.get_real_space_vectors()]
# calculate d[B^T]/dp
dB_dp = xluc_param.get_ds_dp()
dBT_dp = [dB.transpose() for dB in dB_dp]
# calculate d[O]/dp
dO_dp = [-O * dBT * O for dBT in dBT_dp]
# function to get analytical derivative of angles wrt vectors
def dangle(u, v):
return [matrix.col(e) for e in angle_derivative_wrt_vectors(u, v)]
dalpha_db, dalpha_dc = dangle(bvec, cvec)
dbeta_da, dbeta_dc = dangle(avec, cvec)
dgamma_da, dgamma_db = dangle(avec, bvec)
# get all FD derivatives
fd_grad = check_fd_gradients(xluc_param)
# look at each parameter
for i, dO in enumerate(dO_dp):
# print
# print "***** PARAMETER {0} *****".format(i)
# print "dB_dp analytical"
# print dB_dp[i]
# print "dB_dp FD"
# print fd_grad[i]['dB_dp']
# print
# dB_dp is good. What about dO_dp?
# print "O MATRIX"
# print "dO_dp analytical"
# print dO.round(6)
# print "dO_dp FD"
# print fd_grad[i]['dO_dp'].round(6)
# print
assert approx_equal(dO, fd_grad[i]["dO_dp"])
# extract derivatives of each unit cell vector wrt p
dav_dp, dbv_dp, dcv_dp = dO.transpose().as_list_of_lists()
dav_dp = matrix.col(dav_dp)
dbv_dp = matrix.col(dbv_dp)
dcv_dp = matrix.col(dcv_dp)
# check these are correct vs FD
# print "CELL VECTORS"
# diff = dav_dp - fd_grad[i]['davec_dp']
# print 2 * diff.length() / (dav_dp.length() + fd_grad[i]['davec_dp'].length()) * 100
# print 'davec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dav_dp.elems)
# print 'davec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['davec_dp'].elems)
assert approx_equal(dav_dp, fd_grad[i]["davec_dp"])
# diff = dbv_dp - fd_grad[i]['dbvec_dp']
# print 2 * diff.length() / (dbv_dp.length() + fd_grad[i]['dbvec_dp'].length()) * 100
# print 'dbvec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dbv_dp.elems)
# print 'dbvec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['dbvec_dp'].elems)
assert approx_equal(dbv_dp, fd_grad[i]["dbvec_dp"])
# diff = dcv_dp - fd_grad[i]['dcvec_dp']
# print 2 * diff.length() / (dcv_dp.length() + fd_grad[i]['dcvec_dp'].length()) * 100
# print 'dcvec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dcv_dp.elems)
# print 'dcvec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['dcvec_dp'].elems)
# print
assert approx_equal(dcv_dp, fd_grad[i]["dcvec_dp"])
# print "CELL LENGTHS"
da_dp = 1.0 / a * avec.dot(dav_dp)
# print "d[a]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(da_dp, fd_grad[i]['da_dp'], i)
assert approx_equal(da_dp, fd_grad[i]["da_dp"])
db_dp = 1.0 / b * bvec.dot(dbv_dp)
# print "d[b]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(db_dp, fd_grad[i]['db_dp'], i)
assert approx_equal(db_dp, fd_grad[i]["db_dp"])
dc_dp = 1.0 / c * cvec.dot(dcv_dp)
# print "d[c]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dc_dp, fd_grad[i]['dc_dp'], i)
assert approx_equal(dc_dp, fd_grad[i]["dc_dp"])
# print
# print "CELL ANGLES"
daa_dp = RAD2DEG * (dbv_dp.dot(dalpha_db) + dcv_dp.dot(dalpha_dc))
dbb_dp = RAD2DEG * (dav_dp.dot(dbeta_da) + dcv_dp.dot(dbeta_dc))
dcc_dp = RAD2DEG * (dav_dp.dot(dgamma_da) + dbv_dp.dot(dgamma_db))
# print "d[alpha]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(daa_dp, fd_grad[i]['daa_dp'], i)
# print "d[beta]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dbb_dp, fd_grad[i]['dbb_dp'], i)
# print "d[gamma]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dcc_dp, fd_grad[i]['dcc_dp'], i)
assert approx_equal(daa_dp, fd_grad[i]["daa_dp"])
assert approx_equal(dbb_dp, fd_grad[i]["dbb_dp"])
assert approx_equal(dcc_dp, fd_grad[i]["dcc_dp"])
def test():
import random
import textwrap
from cctbx.uctbx import unit_cell
from libtbx.test_utils import approx_equal
def random_direction_close_to(vector):
return vector.rotate_around_origin(
matrix.col((random.random(), random.random(),
random.random())).normalize(),
random.gauss(0, 1.0),
deg=True,
)
# make a random P1 crystal and parameterise it
a = random.uniform(10, 50) * random_direction_close_to(
matrix.col((1, 0, 0)))
b = random.uniform(10, 50) * random_direction_close_to(
matrix.col((0, 1, 0)))
c = random.uniform(10, 50) * random_direction_close_to(
matrix.col((0, 0, 1)))
xl = Crystal(a, b, c, space_group_symbol="P 1")
xl_op = CrystalOrientationParameterisation(xl)
xl_ucp = CrystalUnitCellParameterisation(xl)
null_mat = matrix.sqr((0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0))
# compare analytical and finite difference derivatives
an_ds_dp = xl_op.get_ds_dp()
fd_ds_dp = get_fd_gradients(xl_op, [1.0e-6 * pi / 180] * 3)
for e, f in zip(an_ds_dp, fd_ds_dp):
assert approx_equal((e - f), null_mat, eps=1.0e-6)
an_ds_dp = xl_ucp.get_ds_dp()
fd_ds_dp = get_fd_gradients(xl_ucp, [1.0e-7] * xl_ucp.num_free())
for e, f in zip(an_ds_dp, fd_ds_dp):
assert approx_equal((e - f), null_mat, eps=1.0e-6)
# random initial orientations with a random parameter shift at each
attempts = 100
for i in range(attempts):
# make a random P1 crystal and parameterise it
a = random.uniform(10, 50) * random_direction_close_to(
matrix.col((1, 0, 0)))
b = random.uniform(10, 50) * random_direction_close_to(
matrix.col((0, 1, 0)))
c = random.uniform(10, 50) * random_direction_close_to(
matrix.col((0, 0, 1)))
xl = Crystal(a, b, c, space_group_symbol="P 1")
xl_op = CrystalOrientationParameterisation(xl)
xl_uc = CrystalUnitCellParameterisation(xl)
# apply a random parameter shift to the orientation
p_vals = xl_op.get_param_vals()
p_vals = random_param_shift(
p_vals, [1000 * pi / 9, 1000 * pi / 9, 1000 * pi / 9])
xl_op.set_param_vals(p_vals)
# compare analytical and finite difference derivatives
xl_op_an_ds_dp = xl_op.get_ds_dp()
xl_op_fd_ds_dp = get_fd_gradients(xl_op, [1.0e-5 * pi / 180] * 3)
# apply a random parameter shift to the unit cell. We have to
# do this in a way that is respectful to metrical constraints,
# so don't modify the parameters directly; modify the cell
# constants and extract the new parameters
cell_params = xl.get_unit_cell().parameters()
cell_params = random_param_shift(cell_params, [1.0] * 6)
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(xl.get_space_group())
S.set_orientation(orientation=newB)
X = S.forward_independent_parameters()
xl_uc.set_param_vals(X)
xl_uc_an_ds_dp = xl_ucp.get_ds_dp()
# now doing finite differences about each parameter in turn
xl_uc_fd_ds_dp = get_fd_gradients(xl_ucp, [1.0e-7] * xl_ucp.num_free())
for j in range(3):
assert approx_equal((xl_op_fd_ds_dp[j] - xl_op_an_ds_dp[j]),
null_mat,
eps=1.0e-6), textwrap.dedent("""\
Failure in try {i}
failure for parameter number {j}
of the orientation parameterisation
with fd_ds_dp =
{fd}
and an_ds_dp =
{an}
so that difference fd_ds_dp - an_ds_dp =
{diff}
""").format(
i=i,
j=j,
fd=xl_op_fd_ds_dp[j],
an=xl_op_an_ds_dp[j],
diff=xl_op_fd_ds_dp[j] - xl_op_an_ds_dp[j],
)
for j in range(xl_ucp.num_free()):
assert approx_equal((xl_uc_fd_ds_dp[j] - xl_uc_an_ds_dp[j]),
null_mat,
eps=1.0e-6), textwrap.dedent("""\
Failure in try {i}
failure for parameter number {j}
of the unit cell parameterisation
with fd_ds_dp =
{fd}
and an_ds_dp =
{an}
so that difference fd_ds_dp - an_ds_dp =
{diff}
""").format(
i=i,
j=j,
fd=xl_uc_fd_ds_dp[j],
an=xl_uc_an_ds_dp[j],
diff=xl_uc_fd_ds_dp[j] - xl_uc_an_ds_dp[j],
)
def test_single_crystal_restraints_gradients():
"""Simple test with a single triclinic crystal restrained to a target unit cell"""
from dxtbx.model.experiment_list import Experiment, ExperimentList
from dials.algorithms.refinement.parameterisation.beam_parameters import (
BeamParameterisation, )
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
from dials.algorithms.refinement.parameterisation.detector_parameters import (
DetectorParameterisationSinglePanel, )
from dials.algorithms.refinement.parameterisation.prediction_parameters import (
XYPhiPredictionParameterisation, )
from dials.test.algorithms.refinement.setup_geometry import Extract
overrides = """geometry.parameters.crystal.a.length.range = 10 50
geometry.parameters.crystal.b.length.range = 10 50
geometry.parameters.crystal.c.length.range = 10 50"""
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
""",
process_includes=True,
)
models = Extract(master_phil, overrides)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Build a mock scan for a 72 degree sequence
from dxtbx.model import ScanFactory
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 720),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(720)),
deg=True,
)
# Create parameterisations of these models
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
# Create an ExperimentList
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
# Build a prediction parameterisation
pred_param = XYPhiPredictionParameterisation(
experiments,
detector_parameterisations=[det_param],
beam_parameterisations=[s0_param],
xl_orientation_parameterisations=[xlo_param],
xl_unit_cell_parameterisations=[xluc_param],
)
# Build a restraints parameterisation
rp = RestraintsParameterisation(
detector_parameterisations=[det_param],
beam_parameterisations=[s0_param],
xl_orientation_parameterisations=[xlo_param],
xl_unit_cell_parameterisations=[xluc_param],
)
# make a unit cell target
sigma = 1.0
uc = mycrystal.get_unit_cell().parameters()
target_uc = [random.gauss(e, sigma) for e in uc]
rp.add_restraints_to_target_xl_unit_cell(experiment_id=0,
values=target_uc,
sigma=[sigma] * 6)
# get analytical values and gradients
vals, grads, weights = rp.get_residuals_gradients_and_weights()
assert len(vals) == rp.num_residuals
# get finite difference gradients
p_vals = pred_param.get_param_vals()
deltas = [1.0e-7] * len(p_vals)
fd_grad = []
for i, delta in enumerate(deltas):
val = p_vals[i]
p_vals[i] -= delta / 2.0
pred_param.set_param_vals(p_vals)
rev_state, foo, bar = rp.get_residuals_gradients_and_weights()
rev_state = flex.double(rev_state)
p_vals[i] += delta
pred_param.set_param_vals(p_vals)
fwd_state, foo, bar = rp.get_residuals_gradients_and_weights()
fwd_state = flex.double(fwd_state)
p_vals[i] = val
fd = (fwd_state - rev_state) / delta
fd_grad.append(fd)
# for comparison, fd_grad is a list of flex.doubles, each of which corresponds
# to a column of the sparse matrix grads.
for i, fd in enumerate(fd_grad):
# extract dense column from the sparse matrix
an = grads.col(i).as_dense_vector()
assert an == pytest.approx(fd, abs=1e-5)
def test(init_test):
single_panel_detector = init_test.experiments_single_panel.detectors()[0]
multi_panel_detector = init_test.experiments_multi_panel.detectors()[0]
beam = init_test.experiments_single_panel.beams()[0]
gonio = init_test.experiments_single_panel.goniometers()[0]
crystal = init_test.experiments_single_panel.crystals()[0]
# Parameterise the models
det_param = DetectorParameterisationSinglePanel(single_panel_detector)
s0_param = BeamParameterisation(beam, gonio)
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
multi_det_param = DetectorParameterisationMultiPanel(multi_panel_detector, beam)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Link model parameterisations together into a parameterisation of the
# prediction equation, first for the single panel detector
pred_param = XYPhiPredictionParameterisation(
init_test.experiments_single_panel,
[det_param],
[s0_param],
[xlo_param],
[xluc_param],
)
# ... and now for the multi-panel detector
pred_param2 = XYPhiPredictionParameterisation(
init_test.experiments_multi_panel,
[multi_det_param],
[s0_param],
[xlo_param],
[xluc_param],
)
################################
# Apply known parameter shifts #
################################
# shift detectors by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])]
det_param.set_param_vals(p_vals)
multi_det_p_vals = multi_det_param.get_param_vals()
p_vals = [a + b for a, b in zip(multi_det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])]
multi_det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])]
xlo_param.set_param_vals(p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = crystal.get_unit_cell().parameters()
cell_params = [a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(crystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
multi_det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(
init_test.observations_single_panel, init_test.experiments_single_panel
)
refman2 = ReflectionManager(
init_test.observations_multi_panel, init_test.experiments_multi_panel
)
###############################
# Set up the target functions #
###############################
target = LeastSquaresPositionalResidualWithRmsdCutoff(
init_test.experiments_single_panel,
ScansExperimentsPredictor(init_test.experiments_single_panel),
refman,
pred_param,
restraints_parameterisation=None,
)
target2 = LeastSquaresPositionalResidualWithRmsdCutoff(
init_test.experiments_multi_panel,
ScansExperimentsPredictor(init_test.experiments_multi_panel),
refman2,
pred_param2,
restraints_parameterisation=None,
)
#################################
# Set up the refinement engines #
#################################
refiner = setup_minimiser.Extract(master_phil, target, pred_param).refiner
refiner2 = setup_minimiser.Extract(master_phil, target2, pred_param2).refiner
refiner.run()
# reset parameters and run refinement with the multi panel detector
s0_param.set_param_vals(s0_p_vals)
multi_det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
refiner2.run()
# same number of steps
assert refiner.get_num_steps() == refiner2.get_num_steps()
# same rmsds
for rmsd, rmsd2 in zip(refiner.history["rmsd"], refiner2.history["rmsd"]):
assert approx_equal(rmsd, rmsd2)
# same parameter values each step
for params, params2 in zip(
refiner.history["parameter_vector"], refiner.history["parameter_vector"]
):
assert approx_equal(params, params2)
def create_refiner(params, reflections, experiments):
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalUnitCellParameterisation
from dials.algorithms.refinement.parameterisation.parameter_report import \
ParameterReporter
from dials.algorithms.refinement.two_theta_refiner import (
TwoThetaReflectionManager, TwoThetaTarget,
TwoThetaExperimentsPredictor, TwoThetaPredictionParameterisation)
verb = params.refinement.verbosity
# Only parameterise the crystal unit cell
det_params = None
beam_params = None
xlo_params = None
xluc_params = []
for icrystal, crystal in enumerate(experiments.crystals()):
exp_ids = experiments.indices(crystal)
xluc_params.append(
CrystalUnitCellParameterisation(crystal,
experiment_ids=exp_ids))
# Two theta prediction equation parameterisation
pred_param = TwoThetaPredictionParameterisation(
experiments, det_params, beam_params, xlo_params, xluc_params)
param_reporter = ParameterReporter(det_params, beam_params, xlo_params,
xluc_params)
# ReflectionManager, currently without outlier rejection
# Note: If not all reflections are used, then the filtering must be
# communicated to generate_cif/mmcif() to be included in the CIF file!
refman = TwoThetaReflectionManager(reflections,
experiments,
outlier_detector=None,
verbosity=verb)
# Reflection predictor
ref_predictor = TwoThetaExperimentsPredictor(experiments)
# Two theta target
target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param)
# Do a correlation plot?
tpc = params.output.correlation_plot.filename is not None
# Minimisation engine - FIXME not many choices exposed yet. Do we want
# more cowbell?
from dials.algorithms.refinement.engine \
import LevenbergMarquardtIterations as Refinery
refinery = Refinery(target=target,
prediction_parameterisation=pred_param,
log=None,
verbosity=verb,
track_step=False,
track_gradient=False,
track_parameter_correlation=tpc,
max_iterations=20)
# Refiner
from dials.algorithms.refinement.refiner import Refiner
refiner = Refiner(reflections=reflections,
experiments=experiments,
pred_param=pred_param,
param_reporter=param_reporter,
refman=refman,
target=target,
refinery=refinery,
verbosity=verb)
return refiner
def create_refiner(params, reflections, experiments):
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalUnitCellParameterisation
from dials.algorithms.refinement.parameterisation.parameter_report import \
ParameterReporter
from dials.algorithms.refinement.two_theta_refiner import \
TwoThetaReflectionManager, TwoThetaTarget, TwoThetaExperimentsPredictor, \
TwoThetaPredictionParameterisation
verb = params.refinement.verbosity
# Only parameterise the crystal unit cell
det_params = None
beam_params = None
xlo_params = None
xluc_params = []
for icrystal, crystal in enumerate(experiments.crystals()):
exp_ids = experiments.indices(crystal)
xluc_params.append(
CrystalUnitCellParameterisation(crystal,
experiment_ids=exp_ids))
# Two theta prediction equation parameterisation
pred_param = TwoThetaPredictionParameterisation(
experiments, det_params, beam_params, xlo_params, xluc_params)
param_reporter = ParameterReporter(det_params, beam_params, xlo_params,
xluc_params)
# ReflectionManager, currently without outlier rejection
# Note: If not all reflections are used, then the filtering must be
# communicated to generate_cif/mmcif() to be included in the CIF file!
refman = TwoThetaReflectionManager(reflections,
experiments,
outlier_detector=None,
verbosity=verb)
# Reflection predictor
ref_predictor = TwoThetaExperimentsPredictor(experiments)
# Two theta target
target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param)
# Switch on correlation matrix tracking if a correlation plot is requested
journal = None
if params.output.correlation_plot.filename is not None:
from dials.algorithms.refinement.engine import refinery_phil_scope
journal = refinery_phil_scope.extract().refinery.journal
journal.track_parameter_correlation = True
# Minimisation engine - hardcoded to LevMar for now.
from dials.algorithms.refinement.engine \
import LevenbergMarquardtIterations as Refinery
refinery = Refinery(target=target,
prediction_parameterisation=pred_param,
log=None,
verbosity=verb,
tracking=journal,
max_iterations=20)
# Refiner
from dials.algorithms.refinement.refiner import Refiner
refiner = Refiner(reflections=reflections,
experiments=experiments,
pred_param=pred_param,
param_reporter=param_reporter,
refman=refman,
target=target,
refinery=refinery,
verbosity=verb)
return refiner
def __init__(self, experiment, reflections, mosaicity, params):
"""
Perform the refinement
:param experiment: The experiment
:param reflections: The reflections
:param mosaicity: The mosaicity
:param params: The parameters
"""
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation, )
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation, )
from dials_scratch.jmp.stills import Model
from dials.array_family import flex
from scitbx import simplex
from math import sqrt
# Store the input
self.experiment = experiment
self.reflections = reflections
self.mosaicity = mosaicity
self.params = params
# Get the data and image mask
data = self.experiment.imageset.get_raw_data(0)[0]
mask = self.experiment.imageset.get_mask(0)[0]
# Initialise the model
self.model = Model(
beam=self.experiment.beam,
detector=self.experiment.detector,
crystal=self.experiment.crystal,
reflections=self.reflections,
image_data=data,
image_mask=mask,
mosaicity=self.mosaicity,
bandpass=0.0,
foreground_limit=0.3,
background_limit=0.5,
num_samples=self.params.refinement.profile.num_samples,
)
# Get the crystal model and the parameterisation
self.crystal = self.experiment.crystal
self.cucp = CrystalUnitCellParameterisation(self.crystal)
self.cop = CrystalOrientationParameterisation(self.crystal)
# Get the current values and generate some offsets
values = flex.double(self.cucp.get_param_vals() +
self.cop.get_param_vals())
offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] +
[0.1, 0.1, 0.1])
# The optimization history
self.history = []
# Get the initial cell and initial score
initial_cell = self.crystal.get_unit_cell()
initial_score = self.target(values)
# Perform the optimization
optimizer = simple_simplex(values, offset, self, 2000)
result = optimizer.get_solution()
print("Initial cell:", initial_cell)
print("Final cell: ", self.crystal.get_unit_cell())
# Compute RMSD
xcal, ycal, _ = self.model.observed().parts()
xobs, yobs, _ = self.model.predicted().parts()
rmsd_x = sqrt(flex.sum((xcal - xobs)**2) / len(xcal))
rmsd_y = sqrt(flex.sum((ycal - yobs)**2) / len(ycal))
print("RMSD X, Y (px): %f, %f" % (rmsd_x, rmsd_y))
def test1():
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression", test=os.path.isdir)
# use a datablock that contains a CS-PAD detector description
data_dir = os.path.join(dials_regression, "refinement_test_data",
"hierarchy_test")
datablock_path = os.path.join(data_dir, "datablock.json")
assert os.path.exists(datablock_path)
# load models
from dxtbx.datablock import DataBlockFactory
datablock = DataBlockFactory.from_serialized_format(datablock_path,
check_format=False)
im_set = datablock[0].extract_imagesets()[0]
from copy import deepcopy
detector = deepcopy(im_set.get_detector())
beam = im_set.get_beam()
# we'll invent a crystal, goniometer and scan for this test
from dxtbx.model import Crystal
crystal = Crystal((40., 0., 0.), (0., 40., 0.), (0., 0., 40.),
space_group_symbol="P1")
from dxtbx.model import GoniometerFactory
goniometer = GoniometerFactory.known_axis((1., 0., 0.))
# Build a mock scan for a 180 degree sweep
from dxtbx.model import ScanFactory
sf = ScanFactory()
scan = sf.make_scan(image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=range(1800),
deg=True)
sweep_range = scan.get_oscillation_range(deg=False)
im_width = scan.get_oscillation(deg=False)[1]
assert sweep_range == (0., pi)
assert approx_equal(im_width, 0.1 * pi / 180.)
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Build an experiment list
experiments = ExperimentList()
experiments.append(
Experiment(beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
crystal=crystal,
imageset=None))
# simulate some reflections
refs, ref_predictor = generate_reflections(experiments)
# move the detector quadrants apart by 2mm both horizontally and vertically
from dials.algorithms.refinement.parameterisation \
import DetectorParameterisationHierarchical
det_param = DetectorParameterisationHierarchical(detector, level=1)
det_p_vals = det_param.get_param_vals()
p_vals = list(det_p_vals)
p_vals[1] += 2
p_vals[2] -= 2
p_vals[7] += 2
p_vals[8] += 2
p_vals[13] -= 2
p_vals[14] += 2
p_vals[19] -= 2
p_vals[20] -= 2
det_param.set_param_vals(p_vals)
# reparameterise the detector at the new perturbed geometry
det_param = DetectorParameterisationHierarchical(detector, level=1)
# parameterise other models
from dials.algorithms.refinement.parameterisation.beam_parameters import \
BeamParameterisation
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalOrientationParameterisation, CrystalUnitCellParameterisation
beam_param = BeamParameterisation(beam, goniometer)
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
# fix beam
beam_param.set_fixed([True] * 3)
# fix crystal
xluc_param.set_fixed([True] * 6)
xlo_param.set_fixed([True] * 3)
# parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import \
XYPhiPredictionParameterisation
from dials.algorithms.refinement.parameterisation.parameter_report import \
ParameterReporter
pred_param = XYPhiPredictionParameterisation(experiments, [det_param],
[beam_param], [xlo_param],
[xluc_param])
param_reporter = ParameterReporter([det_param], [beam_param], [xlo_param],
[xluc_param])
# reflection manager and target function
from dials.algorithms.refinement.target import \
LeastSquaresPositionalResidualWithRmsdCutoff
from dials.algorithms.refinement.reflection_manager import ReflectionManager
refman = ReflectionManager(refs, experiments, nref_per_degree=20)
# set a very tight rmsd target of 1/10000 of a pixel
target = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments,
ref_predictor,
refman,
pred_param,
restraints_parameterisation=None,
frac_binsize_cutoff=0.0001)
# minimisation engine
from dials.algorithms.refinement.engine \
import LevenbergMarquardtIterations as Refinery
refinery = Refinery(target=target,
prediction_parameterisation=pred_param,
log=None,
verbosity=0,
max_iterations=20)
# Refiner
from dials.algorithms.refinement.refiner import Refiner
refiner = Refiner(reflections=refs,
experiments=experiments,
pred_param=pred_param,
param_reporter=param_reporter,
refman=refman,
target=target,
refinery=refinery,
verbosity=0)
history = refiner.run()
assert history.reason_for_termination == "RMSD target achieved"
#compare detector with original detector
orig_det = im_set.get_detector()
refined_det = refiner.get_experiments()[0].detector
from scitbx import matrix
import math
for op, rp in zip(orig_det, refined_det):
# compare the origin vectors by...
o1 = matrix.col(op.get_origin())
o2 = matrix.col(rp.get_origin())
# ...their relative lengths
assert approx_equal(math.fabs(o1.length() - o2.length()) / o1.length(),
0,
eps=1e-5)
# ...the angle between them
assert approx_equal(o1.accute_angle(o2), 0, eps=1e-5)
print "OK"
return
def test(args=[]):
# Python and cctbx imports
from math import pi
from scitbx import matrix
from scitbx.array_family import flex
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
# Get module to build models using PHIL
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Model parameterisations
from dials.algorithms.refinement.parameterisation.detector_parameters import \
DetectorParameterisationSinglePanel
from dials.algorithms.refinement.parameterisation.beam_parameters import \
BeamParameterisation
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalOrientationParameterisation, CrystalUnitCellParameterisation
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import \
symmetrize_reduce_enlarge
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator
from dials.algorithms.refinement.prediction import ScansRayPredictor, \
ExperimentsPredictor
from dials.algorithms.spot_prediction import ray_intersection
from cctbx.sgtbx import space_group, space_group_symbols
# Parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import \
XYPhiPredictionParameterisation # implicit import
# Imports for the target function
from dials.algorithms.refinement.target import \
LeastSquaresPositionalResidualWithRmsdCutoff # implicit import
#############################
# Setup experimental models #
#############################
master_phil = parse("""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True)
models = setup_geometry.Extract(
master_phil,
cmdline_args=args,
local_overrides="geometry.parameters.random_seed = 1")
crystal1 = models.crystal
models = setup_geometry.Extract(
master_phil,
cmdline_args=args,
local_overrides="geometry.parameters.random_seed = 2")
mydetector = models.detector
mygonio = models.goniometer
crystal2 = models.crystal
mybeam = models.beam
# Build a mock scan for a 180 degree sweep
sf = ScanFactory()
myscan = sf.make_scan(image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=range(1800),
deg=True)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sweep_range == (0., pi)
assert approx_equal(im_width, 0.1 * pi / 180.)
# Build an experiment list
experiments = ExperimentList()
experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal1,
imageset=None))
experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal2,
imageset=None))
assert len(experiments.detectors()) == 1
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xl1o_param = CrystalOrientationParameterisation(crystal1)
xl1uc_param = CrystalUnitCellParameterisation(crystal1)
xl2o_param = CrystalOrientationParameterisation(crystal2)
xl2uc_param = CrystalUnitCellParameterisation(crystal2)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Fix crystal parameters
#xluc_param.set_fixed([True, True, True, True, True, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
#pred_param = XYPhiPredictionParameterisation(experiments,
# [det_param], [s0_param], [xlo_param], [xluc_param])
################################
# Apply known parameter shifts #
################################
# shift detector by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2., 2., 2.])]
det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = []
for xlo in (xl1o_param, xl2o_param):
p_vals = xlo.get_param_vals()
xlo_p_vals.append(p_vals)
new_p_vals = [a + b for a, b in zip(p_vals, [2., 2., 2.])]
xlo.set_param_vals(new_p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = []
for xluc, xl in ((xl1uc_param, crystal1), ((xl2uc_param, crystal2))):
p_vals = xluc.get_param_vals()
xluc_p_vals.append(p_vals)
cell_params = xl.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(xl.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.e5 for e in S.forward_independent_parameters()])
xluc.set_param_vals(X)
#############################
# Generate some reflections #
#############################
#print "Reflections will be generated with the following geometry:"
#print mybeam
#print mydetector
#print crystal1
#print crystal2
# All indices in a 2.0 Angstrom sphere for crystal1
resolution = 2.0
index_generator = IndexGenerator(
crystal1.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), resolution)
indices1 = index_generator.to_array()
# All indices in a 2.0 Angstrom sphere for crystal2
resolution = 2.0
index_generator = IndexGenerator(
crystal2.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), resolution)
indices2 = index_generator.to_array()
# Predict rays within the sweep range. Set experiment IDs
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs1 = ray_predictor(indices1, experiment_id=0)
obs_refs1['id'] = flex.int(len(obs_refs1), 0)
obs_refs2 = ray_predictor(indices1, experiment_id=1)
obs_refs2['id'] = flex.int(len(obs_refs2), 1)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs1)
obs_refs1 = obs_refs1.select(intersects)
intersects = ray_intersection(mydetector, obs_refs2)
obs_refs2 = obs_refs2.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ExperimentsPredictor(experiments)
obs_refs1 = ref_predictor(obs_refs1)
obs_refs2 = ref_predictor(obs_refs2)
# Set 'observed' centroids from the predicted ones
obs_refs1['xyzobs.mm.value'] = obs_refs1['xyzcal.mm']
obs_refs2['xyzobs.mm.value'] = obs_refs2['xyzcal.mm']
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs1), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs1), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs1), (im_width / 2.)**2)
obs_refs1['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
var_x = flex.double(len(obs_refs2), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs2), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs2), (im_width / 2.)**2)
obs_refs2['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
#print "Total number of reflections excited for crystal1", len(obs_refs1)
#print "Total number of reflections excited for crystal2", len(obs_refs2)
# concatenate reflection lists
obs_refs1.extend(obs_refs2)
obs_refs = obs_refs1
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xl1o_param.set_param_vals(xlo_p_vals[0])
xl2o_param.set_param_vals(xlo_p_vals[1])
xl1uc_param.set_param_vals(xluc_p_vals[0])
xl2uc_param.set_param_vals(xluc_p_vals[1])
#print "Initial values of parameters are"
#msg = "Parameters: " + "%.5f " * len(pred_param)
#print msg % tuple(pred_param.get_param_vals())
#print
# make a refiner
from dials.algorithms.refinement.refiner import phil_scope
params = phil_scope.fetch(source=parse('')).extract()
# in case we want a plot
params.refinement.refinery.journal.track_parameter_correlation = True
# scan static first
from dials.algorithms.refinement.refiner import RefinerFactory
refiner = RefinerFactory.from_parameters_data_experiments(params,
obs_refs,
experiments,
verbosity=0)
history = refiner.run()
# scan varying
params.refinement.parameterisation.scan_varying = True
refiner = RefinerFactory.from_parameters_data_experiments(params,
obs_refs,
experiments,
verbosity=0)
history = refiner.run()
def test(args=[]):
# Python and cctbx imports
from math import pi
from scitbx import matrix
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
# Import for surgery on reflection_tables
from dials.array_family import flex
# Get module to build models using PHIL
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Crystal parameterisations
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator
from dials.algorithms.refinement.prediction.managed_predictors import (
ScansRayPredictor,
StillsExperimentsPredictor,
)
from dials.algorithms.spot_prediction import ray_intersection
from cctbx.sgtbx import space_group, space_group_symbols
#############################
# Setup experimental models #
#############################
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
# build models, with a larger crystal than default in order to get enough
# reflections on the 'still' image
param = """
geometry.parameters.crystal.a.length.range=40 50;
geometry.parameters.crystal.b.length.range=40 50;
geometry.parameters.crystal.c.length.range=40 50;
geometry.parameters.random_seed = 42"""
models = setup_geometry.Extract(master_phil,
cmdline_args=args,
local_overrides=param)
crystal = models.crystal
mydetector = models.detector
mygonio = models.goniometer
mybeam = models.beam
# Build a mock scan for a 1.5 degree wedge. Only used for generating indices near
# the Ewald sphere
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1),
exposure_times=0.1,
oscillation=(0, 1.5),
epochs=list(range(1)),
deg=True,
)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert approx_equal(im_width, 1.5 * pi / 180.0)
# Build experiment lists
stills_experiments = ExperimentList()
stills_experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
crystal=crystal,
imageset=None))
scans_experiments = ExperimentList()
scans_experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
crystal=crystal,
goniometer=mygonio,
scan=myscan,
imageset=None,
))
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
################################
# Apply known parameter shifts #
################################
# rotate crystal (=5 mrad each rotation)
xlo_p_vals = []
p_vals = xlo_param.get_param_vals()
xlo_p_vals.append(p_vals)
new_p_vals = [a + b for a, b in zip(p_vals, [5.0, 5.0, 5.0])]
xlo_param.set_param_vals(new_p_vals)
# change unit cell (=1.0 Angstrom length upsets, 0.5 degree of
# gamma angle)
xluc_p_vals = []
p_vals = xluc_param.get_param_vals()
xluc_p_vals.append(p_vals)
cell_params = crystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [1.0, 1.0, -1.0, 0.0, 0.0, 0.5])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(crystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
# keep track of the target crystal model to compare with refined
from copy import deepcopy
target_crystal = deepcopy(crystal)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere for crystal
resolution = 2.0
index_generator = IndexGenerator(
crystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# Build a ray predictor and predict rays close to the Ewald sphere by using
# the narrow rotation scan
ref_predictor = ScansRayPredictor(scans_experiments, sweep_range)
obs_refs = ref_predictor(indices, experiment_id=0)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Add in flags and ID columns by copying into standard reflection table
tmp = flex.reflection_table.empty_standard(len(obs_refs))
tmp.update(obs_refs)
obs_refs = tmp
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2)
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# Re-predict using the stills reflection predictor
stills_ref_predictor = StillsExperimentsPredictor(stills_experiments)
obs_refs_stills = stills_ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs_stills["xyzobs.mm.value"] = obs_refs_stills["xyzcal.mm"]
###############################
# Undo known parameter shifts #
###############################
xlo_param.set_param_vals(xlo_p_vals[0])
xluc_param.set_param_vals(xluc_p_vals[0])
# make a refiner
from dials.algorithms.refinement.refiner import phil_scope
params = phil_scope.fetch(source=parse("")).extract()
# Change this to get a plot
do_plot = False
if do_plot:
params.refinement.refinery.journal.track_parameter_correlation = True
from dials.algorithms.refinement.refiner import RefinerFactory
# decrease bin_size_fraction to terminate on RMSD convergence
params.refinement.target.bin_size_fraction = 0.01
params.refinement.parameterisation.beam.fix = "all"
params.refinement.parameterisation.detector.fix = "all"
refiner = RefinerFactory.from_parameters_data_experiments(
params, obs_refs_stills, stills_experiments)
# run refinement
history = refiner.run()
# regression tests
assert len(history["rmsd"]) == 9
refined_crystal = refiner.get_experiments()[0].crystal
uc1 = refined_crystal.get_unit_cell()
uc2 = target_crystal.get_unit_cell()
assert uc1.is_similar_to(uc2)
if do_plot:
plt = refiner.parameter_correlation_plot(
len(history["parameter_correlation"]) - 1)
plt.show()
def test():
# Build models, with a larger crystal than default in order to get plenty of
# reflections on the 'still' image
overrides = """
geometry.parameters.crystal.a.length.range=40 50;
geometry.parameters.crystal.b.length.range=40 50;
geometry.parameters.crystal.c.length.range=40 50;
geometry.parameters.random_seed = 42"""
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
""",
process_includes=True,
)
models = Extract(master_phil, overrides)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Build a mock scan for a 3 degree sweep
from dxtbx.model import ScanFactory
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1),
exposure_times=0.1,
oscillation=(0, 3.0),
epochs=list(range(1)),
deg=True,
)
sweep_range = myscan.get_oscillation_range(deg=False)
# Create parameterisations of these models
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
# Create a scans ExperimentList, only for generating reflections
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
# Create a stills ExperimentList
stills_experiments = ExperimentList()
stills_experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
crystal=mycrystal,
imageset=None))
# Generate rays - only to work out which hkls are predicted
ray_predictor = ScansRayPredictor(experiments, sweep_range)
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
rays = ray_predictor(indices)
# Make a standard reflection_table and copy in the ray data
reflections = flex.reflection_table.empty_standard(len(rays))
reflections.update(rays)
# Build a standard prediction parameterisation for the stills experiment to do
# FD calculation (not used for its analytical gradients)
pred_param = StillsPredictionParameterisation(
stills_experiments,
detector_parameterisations=[det_param],
beam_parameterisations=[s0_param],
xl_orientation_parameterisations=[xlo_param],
xl_unit_cell_parameterisations=[xluc_param],
)
# Make a managed SphericalRelpStillsReflectionPredictor reflection predictor
# for the first (only) experiment
ref_predictor = Predictor(stills_experiments)
# Predict these reflections in place. Must do this ahead of calculating
# the analytical gradients so quantities like s1 are correct
ref_predictor.update()
ref_predictor.predict(reflections)
# calculate analytical gradients
ag = AnalyticalGradients(
stills_experiments,
detector_parameterisation=det_param,
beam_parameterisation=s0_param,
xl_orientation_parameterisation=xlo_param,
xl_unit_cell_parameterisation=xluc_param,
)
an_grads = ag.get_beam_gradients(reflections)
an_grads.update(ag.get_crystal_orientation_gradients(reflections))
an_grads.update(ag.get_crystal_unit_cell_gradients(reflections))
# get finite difference gradients
p_vals = pred_param.get_param_vals()
deltas = [1.0e-7] * len(p_vals)
fd_grads = []
p_names = pred_param.get_param_names()
for i, delta in enumerate(deltas):
# save parameter value
val = p_vals[i]
# calc reverse state
p_vals[i] -= delta / 2.0
pred_param.set_param_vals(p_vals)
ref_predictor.update()
ref_predictor.predict(reflections)
x, y, _ = reflections["xyzcal.mm"].deep_copy().parts()
s1 = reflections["s1"].deep_copy()
rev_state = s1
# calc forward state
p_vals[i] += delta
pred_param.set_param_vals(p_vals)
ref_predictor.update()
ref_predictor.predict(reflections)
x, y, _ = reflections["xyzcal.mm"].deep_copy().parts()
s1 = reflections["s1"].deep_copy()
fwd_state = s1
# reset parameter to saved value
p_vals[i] = val
# finite difference - currently for s1 only
fd = fwd_state - rev_state
inv_delta = 1.0 / delta
s1_grads = fd * inv_delta
# store gradients
fd_grads.append({"name": p_names[i], "ds1": s1_grads})
# return to the initial state
pred_param.set_param_vals(p_vals)
for i, fd_grad in enumerate(fd_grads):
## compare FD with analytical calculations
print("\n\nParameter {0}: {1}".format(i, fd_grad["name"]))
print("d[s1]/dp for the first reflection")
print("finite diff", fd_grad["ds1"][0])
try:
an_grad = an_grads[fd_grad["name"]]
except KeyError:
continue
print("checking analytical vs finite difference gradients for s1")
for a, b in zip(fd_grad["ds1"], an_grad["ds1"]):
assert a == pytest.approx(b, abs=1e-7)
def crystal_parameters():
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
from dials.array_family import flex
from scitbx import matrix
experiments = ExperimentListFactory.from_json_file("experiments.json")
reflections = flex.reflection_table.from_pickle("reflections.pickle")
beam = experiments[0].beam
detector = experiments[0].detector
goniometer = experiments[0].goniometer
scan = experiments[0].scan
crystal = experiments[0].crystal
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation, )
parameters = CrystalUnitCellParameterisation(crystal)
print("")
print("Parameters")
for param in parameters.get_params():
print(param.name, param.value)
print("")
print("db/dp")
for db_dp in parameters.get_ds_dp():
print(db_dp.round(5))
print("")
print("B matrix")
B = crystal.get_B().round(7)
print(B)
print(B.transpose() * B)
# from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge
# temp = symmetrize_reduce_enlarge(crystal.get_space_group())
# temp.set_orientation(crystal.get_B())
# independent = temp.forward_independent_parameters()
# print independent
# new_mm = temp.enlarge(independent)
# temp.Bconverter.validate_and_setG(new_mm)
# B = temp.Bconverter.back_as_orientation()
# print matrix.sqr(B.reciprocal_matrix()).round(5)
# from cctbx import sgtbx
# from rstbx.symmetry.constraints import AGconvert
# constraints = sgtbx.tensor_rank_2_constraints(space_group=crystal.get_space_group(), reciprocal_space=True)
# params = constraints.all_params(independent_params=tuple(p.value for p in
# parameters.get_params()))
# from cctbx.crystal_orientation import crystal_orientation
# converter = AGconvert()
# converter.forward(crystal_orientation(crystal.get_B(), False))
# converter.validate_and_setG(params)
# orientation = converter.back_as_orientation()
# print orientation
# print ""
print("SUM(p * dp/dp)")
MAT = [
p.value * db_dp
for p, db_dp in zip(parameters.get_params(), parameters.get_ds_dp())
]
COV = sum(MAT[1:], MAT[0])
print(2 * COV.round(7))
print((2 * COV - crystal.get_B()).round(7))
def test_refinement(dials_regression):
"""Test a refinement run"""
# Get a beam and detector from a experiments. This one has a CS-PAD, but that
# is irrelevant
data_dir = os.path.join(dials_regression, "refinement_test_data",
"hierarchy_test")
experiments_path = os.path.join(data_dir, "datablock.json")
assert os.path.exists(experiments_path)
# load models
from dxtbx.model.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_serialized_format(
experiments_path, check_format=False)
im_set = experiments.imagesets()[0]
detector = deepcopy(im_set.get_detector())
beam = im_set.get_beam()
# Invent a crystal, goniometer and scan for this test
from dxtbx.model import Crystal
crystal = Crystal((40.0, 0.0, 0.0), (0.0, 40.0, 0.0), (0.0, 0.0, 40.0),
space_group_symbol="P1")
orig_xl = deepcopy(crystal)
from dxtbx.model import GoniometerFactory
goniometer = GoniometerFactory.known_axis((1.0, 0.0, 0.0))
# Build a mock scan for a 180 degree sequence
from dxtbx.model import ScanFactory
sf = ScanFactory()
scan = sf.make_scan(
image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(1800)),
deg=True,
)
sequence_range = scan.get_oscillation_range(deg=False)
im_width = scan.get_oscillation(deg=False)[1]
assert sequence_range == (0.0, pi)
assert approx_equal(im_width, 0.1 * pi / 180.0)
# Build an experiment list
experiments = ExperimentList()
experiments.append(
Experiment(
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
crystal=crystal,
imageset=None,
))
# simulate some reflections
refs, _ = generate_reflections(experiments)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# alpha and beta angles)
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation, )
xluc_param = CrystalUnitCellParameterisation(crystal)
cell_params = crystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1, -0.1, 0.0])
]
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge
from scitbx import matrix
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(crystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
# reparameterise the crystal at the perturbed geometry
xluc_param = CrystalUnitCellParameterisation(crystal)
# Dummy parameterisations for other models
beam_param = None
xlo_param = None
det_param = None
# parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.parameter_report import (
ParameterReporter, )
pred_param = TwoThetaPredictionParameterisation(experiments, det_param,
beam_param, xlo_param,
[xluc_param])
param_reporter = ParameterReporter(det_param, beam_param, xlo_param,
[xluc_param])
# reflection manager
refman = TwoThetaReflectionManager(refs, experiments, nref_per_degree=20)
# reflection predictor
ref_predictor = TwoThetaExperimentsPredictor(experiments)
# target function
target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param)
# minimisation engine
from dials.algorithms.refinement.engine import (
LevenbergMarquardtIterations as Refinery, )
refinery = Refinery(
target=target,
prediction_parameterisation=pred_param,
log=None,
max_iterations=20,
)
# Refiner
from dials.algorithms.refinement.refiner import Refiner
refiner = Refiner(
experiments=experiments,
pred_param=pred_param,
param_reporter=param_reporter,
refman=refman,
target=target,
refinery=refinery,
)
refiner.run()
# compare crystal with original crystal
refined_xl = refiner.get_experiments()[0].crystal
# print refined_xl
assert refined_xl.is_similar_to(orig_xl,
uc_rel_length_tolerance=0.001,
uc_abs_angle_tolerance=0.01)
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal2,
imageset=None))
assert len(experiments.detectors()) == 1
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xl1o_param = CrystalOrientationParameterisation(crystal1)
xl1uc_param = CrystalUnitCellParameterisation(crystal1)
xl2o_param = CrystalOrientationParameterisation(crystal2)
xl2uc_param = CrystalUnitCellParameterisation(crystal2)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Fix crystal parameters
#xluc_param.set_fixed([True, True, True, True, True, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
#pred_param = XYPhiPredictionParameterisation(experiments,
'dbvec_dp': fd_vec[1],
'dcvec_dp': fd_vec[2],
'dB_dp': fd_B,
'dO_dp': fd_O
})
p_vals[i] = val
# return to the initial state
mp.set_param_vals(p_vals)
return fd_grad
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
from dials.algorithms.refinement.restraints.restraints import SingleUnitCellTie
uct = SingleUnitCellTie(xluc_param, [None] * 6, [None] * 6)
from scitbx.math import angle_derivative_wrt_vectors
B = matrix.sqr(crystal.get_B())
O = (B.transpose()).inverse()
a, b, c, aa, bb, cc = crystal.get_unit_cell().parameters()
aa *= DEG2RAD
bb *= DEG2RAD
cc *= DEG2RAD
Ut = matrix.sqr(crystal.get_U()).transpose()
avec, bvec, cvec = [Ut * vec for vec in crystal.get_real_space_vectors()]
def test(args=[]):
#############################
# Setup experimental models #
#############################
master_phil = parse(
"""
include scope dials.tests.algorithms.refinement.geometry_phil
include scope dials.tests.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
models = setup_geometry.Extract(
master_phil,
cmdline_args=args,
local_overrides="geometry.parameters.random_seed = 1",
)
crystal1 = models.crystal
models = setup_geometry.Extract(
master_phil,
cmdline_args=args,
local_overrides="geometry.parameters.random_seed = 2",
)
mydetector = models.detector
mygonio = models.goniometer
crystal2 = models.crystal
mybeam = models.beam
# Build a mock scan for an 18 degree sequence
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 180),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(180)),
deg=True,
)
sequence_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sequence_range == (0.0, pi / 10)
assert approx_equal(im_width, 0.1 * pi / 180.0)
# Build an experiment list
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal1,
imageset=None,
))
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal2,
imageset=None,
))
assert len(experiments.detectors()) == 1
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xl1o_param = CrystalOrientationParameterisation(crystal1)
xl1uc_param = CrystalUnitCellParameterisation(crystal1)
xl2o_param = CrystalOrientationParameterisation(crystal2)
xl2uc_param = CrystalUnitCellParameterisation(crystal2)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
################################
# Apply known parameter shifts #
################################
# shift detector by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [
a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])
]
det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = []
for xlo in (xl1o_param, xl2o_param):
p_vals = xlo.get_param_vals()
xlo_p_vals.append(p_vals)
new_p_vals = [a + b for a, b in zip(p_vals, [2.0, 2.0, 2.0])]
xlo.set_param_vals(new_p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = []
for xluc, xl in ((xl1uc_param, crystal1), ((xl2uc_param, crystal2))):
p_vals = xluc.get_param_vals()
xluc_p_vals.append(p_vals)
cell_params = xl.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(xl.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc.set_param_vals(X)
#############################
# Generate some reflections #
#############################
# All indices in a 2.5 Angstrom sphere for crystal1
resolution = 2.5
index_generator = IndexGenerator(
crystal1.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices1 = index_generator.to_array()
# All indices in a 2.5 Angstrom sphere for crystal2
resolution = 2.5
index_generator = IndexGenerator(
crystal2.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices2 = index_generator.to_array()
# Predict rays within the sequence range. Set experiment IDs
ray_predictor = ScansRayPredictor(experiments, sequence_range)
obs_refs1 = ray_predictor(indices1, experiment_id=0)
obs_refs1["id"] = flex.int(len(obs_refs1), 0)
obs_refs2 = ray_predictor(indices2, experiment_id=1)
obs_refs2["id"] = flex.int(len(obs_refs2), 1)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs1)
obs_refs1 = obs_refs1.select(intersects)
intersects = ray_intersection(mydetector, obs_refs2)
obs_refs2 = obs_refs2.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ScansExperimentsPredictor(experiments)
obs_refs1 = ref_predictor(obs_refs1)
obs_refs2 = ref_predictor(obs_refs2)
# Set 'observed' centroids from the predicted ones
obs_refs1["xyzobs.mm.value"] = obs_refs1["xyzcal.mm"]
obs_refs2["xyzobs.mm.value"] = obs_refs2["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 18.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs1), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs1), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs1), (im_width / 2.0)**2)
obs_refs1["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
var_x = flex.double(len(obs_refs2), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs2), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs2), (im_width / 2.0)**2)
obs_refs2["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# concatenate reflection lists
obs_refs1.extend(obs_refs2)
obs_refs = obs_refs1
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xl1o_param.set_param_vals(xlo_p_vals[0])
xl2o_param.set_param_vals(xlo_p_vals[1])
xl1uc_param.set_param_vals(xluc_p_vals[0])
xl2uc_param.set_param_vals(xluc_p_vals[1])
# scan static first
params = phil_scope.fetch(source=parse("")).extract()
refiner = RefinerFactory.from_parameters_data_experiments(
params, obs_refs, experiments)
refiner.run()
# scan varying
params.refinement.parameterisation.scan_varying = True
refiner = RefinerFactory.from_parameters_data_experiments(
params, obs_refs, experiments)
refiner.run()
# Ensure all models have scan-varying state set
# (https://github.com/dials/dials/issues/798)
refined_experiments = refiner.get_experiments()
sp = [xl.get_num_scan_points() for xl in refined_experiments.crystals()]
assert sp.count(181) == 2
