def __init__(
self,
experiment,
mosaicity_parameterisation,
fix_mosaic_spread=False,
fix_wavelength_spread=False,
fix_unit_cell=False,
fix_orientation=False,
):
"""
Initialise the model state
"""
# Save the crystal model
self.experiment = experiment
self.crystal = experiment.crystal
# The U and P parameterisation
self.U_parameterisation = CrystalOrientationParameterisation(
self.crystal)
self.B_parameterisation = CrystalUnitCellParameterisation(self.crystal)
# The M, L and W parameterisations
self.M_parameterisation = mosaicity_parameterisation
# Set the flags to fix parameters
self._is_mosaic_spread_fixed = fix_mosaic_spread
self._is_wavelength_spread_fixed = fix_wavelength_spread
self._is_unit_cell_fixed = fix_unit_cell
self._is_orientation_fixed = fix_orientation
def refine(self, do_print=False):
crystal = self.crystal
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation,
CrystalOrientationParameterisation,
)
self.cucp = CrystalUnitCellParameterisation(crystal)
self.cop = CrystalOrientationParameterisation(crystal)
self.zero()
# 0-point and deltas
values = flex.double(self.cucp.get_param_vals() +
self.cop.get_param_vals())
offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] +
[0.1, 0.1, 0.1])
initial = crystal.get_unit_cell()
self.cells = []
self.best_score = 1e99
initial_score = self.target(values)
doohicky = simple_simplex(values, offset, self, 2000)
best = doohicky.get_solution()
if do_print:
print("Initial cell:", initial)
print("Final cell: ", crystal.get_unit_cell())
print("Score change", initial_score,
self.target(best, do_print=False))
self.best = best
def create_models(self):
# build models, with a larger crystal than default in order to get plenty of
# reflections on the 'still' image
overrides = """
geometry.parameters.crystal.a.length.range=40 50;
geometry.parameters.crystal.b.length.range=40 50;
geometry.parameters.crystal.c.length.range=40 50;
geometry.parameters.random_seed = 42"""
master_phil = parse("""
include scope dials.test.algorithms.refinement.geometry_phil
""",
process_includes=True)
models = Extract(master_phil, overrides)
# keep track of the models
self.detector = models.detector
self.gonio = models.goniometer
self.crystal = models.crystal
self.beam = models.beam
# Create a stills ExperimentList
self.stills_experiments = ExperimentList()
self.stills_experiments.append(
Experiment(beam=self.beam,
detector=self.detector,
crystal=self.crystal,
imageset=None))
# keep track of the parameterisation of the models
self.det_param = DetectorParameterisationSinglePanel(self.detector)
self.s0_param = BeamParameterisation(self.beam, self.gonio)
self.xlo_param = CrystalOrientationParameterisation(self.crystal)
self.xluc_param = CrystalUnitCellParameterisation(self.crystal)
def __init__(self, experiment, reflections, parameters):
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation, )
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation, )
# Store the input
self.experiment = experiment
self.reflections = reflections
self.parameters = parameters
# Get the crystal model and the parameterisation
self.crystal = self.experiment.crystal
self.cucp = CrystalUnitCellParameterisation(self.crystal)
self.cop = CrystalOrientationParameterisation(self.crystal)
# Get the current values and generate some offsets
values = flex.double(self.cucp.get_param_vals() +
self.cop.get_param_vals())
offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] +
[0.1, 0.1, 0.1])
# The optimization history
self.history = []
# Get the initial cell and initial score
initial_cell = self.crystal.get_unit_cell()
initial_orientation = self.crystal.get_U()
initial_score = self.target(values)
# Perform the optimization
optimizer = simple_simplex(values, offset, self, 2000)
result = optimizer.get_solution()
# Print some information
fmt = "(%.3f, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f)"
print("Initial cell:", initial_cell)
print("Final cell: ", self.crystal.get_unit_cell())
print("Initial orientation: ", fmt % tuple(initial_orientation))
print("Final orientation: ", fmt % tuple(self.crystal.get_U()))
# Print RMSD
Xobs, Yobs, _ = self.reflections["xyzobs.px.value"].parts()
Xcal, Ycal, _ = self.reflections["xyzcal.px"].parts()
rmsd_x = sqrt(flex.sum((Xcal - Xobs)**2) / len(Xcal))
rmsd_y = sqrt(flex.sum((Ycal - Yobs)**2) / len(Ycal))
print("RMSD X, Y (px): %f, %f" % (rmsd_x, rmsd_y))
def __init__(self, reflection_file, experiment_file):
# make a pie
data = pickle.load(open(reflection_file, "rb"))
print("%d reflections" % data.size())
self.data = data.select(data["intensity.sum.variance"] > 0)
i = data["intensity.sum.value"]
v = data["intensity.sum.variance"]
s = flex.sqrt(v)
self.i_s = i / s
from dxtbx.model.experiment_list import ExperimentListFactory
expt = ExperimentListFactory.from_json_file(experiment_file)
crystal = expt.crystals()[0]
self.s0 = matrix.col(expt.beams()[0].get_s0())
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation,
CrystalOrientationParameterisation,
)
self.crystal = crystal
self.cucp = CrystalUnitCellParameterisation(crystal)
self.cop = CrystalOrientationParameterisation(crystal)
# 0-point and deltas
values = flex.double(self.cucp.get_param_vals() +
self.cop.get_param_vals())
offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] +
[0.1, 0.1, 0.1])
initial = crystal.get_unit_cell()
initial_score = self.target(values)
doohicky = simple_simplex(values, offset, self, 2000)
best = doohicky.get_solution()
print("Initial cell:", initial)
print("Final cell: ", crystal.get_unit_cell())
print("Score change", initial_score, self.target(best, do_print=False))
self.best = best
def __init__(
self,
experiment,
mosaicity_parameterisation,
wavelength_parameterisation=None,
fix_mosaic_spread=False,
fix_wavelength_spread=True,
fix_unit_cell=False,
fix_orientation=False,
):
"""
Initialise the model state
"""
# Save the crystal model
self.experiment = experiment
self.crystal = experiment.crystal
# The U and P parameterisation
self._U_parameterisation = CrystalOrientationParameterisation(
self.crystal)
self._B_parameterisation = CrystalUnitCellParameterisation(
self.crystal)
# The M and L parameterisations
self._M_parameterisation = mosaicity_parameterisation
self._L_parameterisation = wavelength_parameterisation
# Set the flags to fix parameters
self._is_mosaic_spread_fixed = fix_mosaic_spread
self._is_wavelength_spread_fixed = fix_wavelength_spread
self._is_unit_cell_fixed = fix_unit_cell
self._is_orientation_fixed = fix_orientation
# Check wavelength parameterisation
if not self.is_wavelength_spread_fixed:
assert self._L_parameterisation is not None
def generate_data(experiments, reflections):
from random import seed
seed(0)
index = randint(0, len(reflections))
h = reflections[index]["miller_index"]
s0 = matrix.col(experiments[0].beam.get_s0())
U_param = CrystalOrientationParameterisation(experiments[0].crystal)
B_param = CrystalUnitCellParameterisation(experiments[0].crystal)
U = matrix.sqr(experiments[0].crystal.get_U())
B = matrix.sqr(experiments[0].crystal.get_B())
r = U * B * matrix.col(h)
s2 = s0 + r
mobs = (s2 + matrix.col((uniform(0, 1e-3), uniform(
0, 1e-3), uniform(0, 1e-3)))).normalize() * s0.length()
b1, b2, b3, b4, b5, b6 = (
uniform(1e-3, 3e-3),
uniform(0.0, 1e-3),
uniform(1e-3, 3e-3),
uniform(0.0, 1e-3),
uniform(0.0, 1e-3),
uniform(1e-3, 3e-3),
)
params = (b1, b2, b3, b4, b5, b6)
S_param = SimpleMosaicityParameterisation(params)
L_param = (uniform(1e-3, 2e-3), )
W_param = (uniform(1e-3, 2e-3), uniform(0, 1e-3), uniform(1e-3, 2e-3))
ctot = randint(100, 1000)
T = matrix.sqr(
(uniform(1e-3, 2e-3), 0, uniform(1e-6, 2e-6), uniform(1e-3, 2e-3)))
Sobs = T * T.transpose()
params = [S_param, U_param, B_param, L_param, W_param]
return params, s0, h, ctot, mobs, Sobs
def generate_data(experiments, reflections):
from random import seed
seed(0)
index = randint(0, len(reflections))
h = reflections[index]["miller_index"]
s0 = matrix.col(experiments[0].beam.get_s0())
U_param = CrystalOrientationParameterisation(experiments[0].crystal)
B_param = CrystalUnitCellParameterisation(experiments[0].crystal)
U = matrix.sqr(experiments[0].crystal.get_U())
B = matrix.sqr(experiments[0].crystal.get_B())
r = U * B * matrix.col(h)
s2 = s0 + r
mobs = (s2 + matrix.col((uniform(0, 1e-3), uniform(
0, 1e-3), uniform(0, 1e-3)))).normalize() * s0.length()
mobs = np.array([uniform(0, 1e-3), uniform(0, 1e-3)])
sp = s2.normalize() * s0.length()
b1, b2, b3, b4, b5, b6 = (
uniform(1e-3, 3e-3),
uniform(0.0, 1e-3),
uniform(1e-3, 3e-3),
uniform(0.0, 1e-3),
uniform(0.0, 1e-3),
uniform(1e-3, 3e-3),
)
S_param = (b1, b2, b3, b4, b5, b6)
L_param = (uniform(1e-3, 2e-3), )
ctot = randint(100, 1000)
T = np.array((uniform(1e-3, 2e-3), 0, uniform(1e-6, 2e-6),
uniform(1e-3, 2e-3))).reshape(2, 2)
Sobs = np.matmul(T, T.T)
params = [S_param, U_param, B_param, L_param]
return params, s0, sp, h, ctot, mobs, Sobs
def test():
# Python and cctbx imports
from math import pi
from scitbx import matrix
from scitbx.array_family import flex
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
# Get modules to build models and minimiser using PHIL
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
import dials.test.algorithms.refinement.setup_minimiser as setup_minimiser
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Model parameterisations
from dials.algorithms.refinement.parameterisation.detector_parameters import (
DetectorParameterisationSinglePanel, )
from dials.algorithms.refinement.parameterisation.beam_parameters import (
BeamParameterisation, )
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection
from dials.algorithms.refinement.prediction.managed_predictors import (
ScansRayPredictor,
ScansExperimentsPredictor,
)
from cctbx.sgtbx import space_group, space_group_symbols
# Parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import (
XYPhiPredictionParameterisation, )
# Imports for the target function
from dials.algorithms.refinement.target import (
LeastSquaresPositionalResidualWithRmsdCutoff, )
from dials.algorithms.refinement.reflection_manager import ReflectionManager
#############################
# Setup experimental models #
#############################
override = """geometry.parameters
{
beam.wavelength.random=False
beam.wavelength.value=1.0
beam.direction.inclination.random=False
crystal.a.length.random=False
crystal.a.length.value=12.0
crystal.a.direction.method=exactly
crystal.a.direction.exactly.direction=1.0 0.002 -0.004
crystal.b.length.random=False
crystal.b.length.value=14.0
crystal.b.direction.method=exactly
crystal.b.direction.exactly.direction=-0.002 1.0 0.002
crystal.c.length.random=False
crystal.c.length.value=13.0
crystal.c.direction.method=exactly
crystal.c.direction.exactly.direction=0.002 -0.004 1.0
detector.directions.method=exactly
detector.directions.exactly.dir1=0.99 0.002 -0.004
detector.directions.exactly.norm=0.002 -0.001 0.99
detector.centre.method=exactly
detector.centre.exactly.value=1.0 -0.5 199.0
}"""
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
models = setup_geometry.Extract(master_phil,
local_overrides=override,
verbose=False)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
# Build a mock scan for a 180 degree sweep
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(1800)),
deg=True,
)
# Build an ExperimentList
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
# Create the PredictionParameterisation
pred_param = XYPhiPredictionParameterisation(experiments, [det_param],
[s0_param], [xlo_param],
[xluc_param])
################################
# Apply known parameter shifts #
################################
# shift detector by 1.0 mm each translation and 4 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [
a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 4.0, 4.0, 4.0])
]
det_param.set_param_vals(p_vals)
# shift beam by 4 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 4.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=3 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [3.0, 3.0, 3.0])]
xlo_param.set_param_vals(p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# alpha and beta angles)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = mycrystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1, -0.1, 0.0])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(mycrystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sweep_range == (0.0, pi)
assert approx_equal(im_width, 0.1 * pi / 180.0)
# Predict rays within the sweep range
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs = ray_predictor(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ScansExperimentsPredictor(experiments)
obs_refs["id"] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2)
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# The total number of observations should be 1128
assert len(obs_refs) == 1128
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(obs_refs,
experiments,
outlier_detector=None,
close_to_spindle_cutoff=0.1)
##############################
# Set up the target function #
##############################
# The current 'achieved' criterion compares RMSD against 1/3 the pixel size and
# 1/3 the image width in radians. For the simulated data, these are just made up
mytarget = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments,
ref_predictor,
refman,
pred_param,
restraints_parameterisation=None)
######################################
# Set up the LSTBX refinement engine #
######################################
overrides = """minimiser.parameters.engine=GaussNewton
minimiser.parameters.logfile=None"""
refiner = setup_minimiser.Extract(master_phil,
mytarget,
pred_param,
local_overrides=overrides).refiner
refiner.run()
assert mytarget.achieved()
assert refiner.get_num_steps() == 1
assert approx_equal(
mytarget.rmsds(),
(0.00508252354876, 0.00420954552156, 8.97303428289e-05))
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
######################################################
# Set up the LBFGS with curvatures refinement engine #
######################################################
overrides = """minimiser.parameters.engine=LBFGScurvs
minimiser.parameters.logfile=None"""
refiner = setup_minimiser.Extract(master_phil,
mytarget,
pred_param,
local_overrides=overrides).refiner
refiner.run()
assert mytarget.achieved()
assert refiner.get_num_steps() == 9
assert approx_equal(mytarget.rmsds(),
(0.0558857700305, 0.0333446685335, 0.000347402754278))
Experiment(beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal2,
imageset=None))
assert len(experiments.detectors()) == 1
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xl1o_param = CrystalOrientationParameterisation(crystal1)
xl1uc_param = CrystalUnitCellParameterisation(crystal1)
xl2o_param = CrystalOrientationParameterisation(crystal2)
xl2uc_param = CrystalUnitCellParameterisation(crystal2)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Fix crystal parameters
#xluc_param.set_fixed([True, True, True, True, True, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
def test(args=[]):
# Python and cctbx imports
from math import pi
from scitbx import matrix
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
# Import for surgery on reflection_tables
from dials.array_family import flex
# Get module to build models using PHIL
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Crystal parameterisations
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator
from dials.algorithms.refinement.prediction.managed_predictors import (
ScansRayPredictor,
StillsExperimentsPredictor,
)
from dials.algorithms.spot_prediction import ray_intersection
from cctbx.sgtbx import space_group, space_group_symbols
#############################
# Setup experimental models #
#############################
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
# build models, with a larger crystal than default in order to get enough
# reflections on the 'still' image
param = """
geometry.parameters.crystal.a.length.range=40 50;
geometry.parameters.crystal.b.length.range=40 50;
geometry.parameters.crystal.c.length.range=40 50;
geometry.parameters.random_seed = 42"""
models = setup_geometry.Extract(master_phil,
cmdline_args=args,
local_overrides=param)
crystal = models.crystal
mydetector = models.detector
mygonio = models.goniometer
mybeam = models.beam
# Build a mock scan for a 1.5 degree wedge. Only used for generating indices near
# the Ewald sphere
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1),
exposure_times=0.1,
oscillation=(0, 1.5),
epochs=list(range(1)),
deg=True,
)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert approx_equal(im_width, 1.5 * pi / 180.0)
# Build experiment lists
stills_experiments = ExperimentList()
stills_experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
crystal=crystal,
imageset=None))
scans_experiments = ExperimentList()
scans_experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
crystal=crystal,
goniometer=mygonio,
scan=myscan,
imageset=None,
))
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
################################
# Apply known parameter shifts #
################################
# rotate crystal (=5 mrad each rotation)
xlo_p_vals = []
p_vals = xlo_param.get_param_vals()
xlo_p_vals.append(p_vals)
new_p_vals = [a + b for a, b in zip(p_vals, [5.0, 5.0, 5.0])]
xlo_param.set_param_vals(new_p_vals)
# change unit cell (=1.0 Angstrom length upsets, 0.5 degree of
# gamma angle)
xluc_p_vals = []
p_vals = xluc_param.get_param_vals()
xluc_p_vals.append(p_vals)
cell_params = crystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [1.0, 1.0, -1.0, 0.0, 0.0, 0.5])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(crystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
# keep track of the target crystal model to compare with refined
from copy import deepcopy
target_crystal = deepcopy(crystal)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere for crystal
resolution = 2.0
index_generator = IndexGenerator(
crystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# Build a ray predictor and predict rays close to the Ewald sphere by using
# the narrow rotation scan
ref_predictor = ScansRayPredictor(scans_experiments, sweep_range)
obs_refs = ref_predictor(indices, experiment_id=0)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Add in flags and ID columns by copying into standard reflection table
tmp = flex.reflection_table.empty_standard(len(obs_refs))
tmp.update(obs_refs)
obs_refs = tmp
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2)
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# Re-predict using the stills reflection predictor
stills_ref_predictor = StillsExperimentsPredictor(stills_experiments)
obs_refs_stills = stills_ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs_stills["xyzobs.mm.value"] = obs_refs_stills["xyzcal.mm"]
###############################
# Undo known parameter shifts #
###############################
xlo_param.set_param_vals(xlo_p_vals[0])
xluc_param.set_param_vals(xluc_p_vals[0])
# make a refiner
from dials.algorithms.refinement.refiner import phil_scope
params = phil_scope.fetch(source=parse("")).extract()
# Change this to get a plot
do_plot = False
if do_plot:
params.refinement.refinery.journal.track_parameter_correlation = True
from dials.algorithms.refinement.refiner import RefinerFactory
# decrease bin_size_fraction to terminate on RMSD convergence
params.refinement.target.bin_size_fraction = 0.01
params.refinement.parameterisation.beam.fix = "all"
params.refinement.parameterisation.detector.fix = "all"
refiner = RefinerFactory.from_parameters_data_experiments(
params, obs_refs_stills, stills_experiments)
# run refinement
history = refiner.run()
# regression tests
assert len(history["rmsd"]) == 9
refined_crystal = refiner.get_experiments()[0].crystal
uc1 = refined_crystal.get_unit_cell()
uc2 = target_crystal.get_unit_cell()
assert uc1.is_similar_to(uc2)
if do_plot:
plt = refiner.parameter_correlation_plot(
len(history["parameter_correlation"]) - 1)
plt.show()
class ModelState(object):
"""
A class to keep track of the model state
"""
def __init__(
self,
experiment,
mosaicity_parameterisation,
fix_mosaic_spread=False,
fix_wavelength_spread=False,
fix_unit_cell=False,
fix_orientation=False,
):
"""
Initialise the model state
"""
# Save the crystal model
self.experiment = experiment
self.crystal = experiment.crystal
# The U and P parameterisation
self.U_parameterisation = CrystalOrientationParameterisation(
self.crystal)
self.B_parameterisation = CrystalUnitCellParameterisation(self.crystal)
# The M, L and W parameterisations
self.M_parameterisation = mosaicity_parameterisation
# Set the flags to fix parameters
self._is_mosaic_spread_fixed = fix_mosaic_spread
self._is_wavelength_spread_fixed = fix_wavelength_spread
self._is_unit_cell_fixed = fix_unit_cell
self._is_orientation_fixed = fix_orientation
def is_orientation_fixed(self):
"""
Return whether the orientation is fixed
"""
return self._is_orientation_fixed
def is_unit_cell_fixed(self):
"""
Return whether the unit cell is fixed
"""
return self._is_unit_cell_fixed
def is_mosaic_spread_fixed(self):
"""
Return whether the mosaic spread is fixed
"""
return self._is_mosaic_spread_fixed
def is_mosaic_spread_angular(self):
"""
Return whether the mosaic spread is angular
"""
return self.M_parameterisation.is_angular()
def is_wavelength_spread_fixed(self):
"""
Return whether the wavelength spread is fixed
"""
return self._is_wavelength_spread_fixed
def get_unit_cell(self):
"""
Get the crystal unit cell
"""
return self.crystal.get_unit_cell()
def get_U(self):
"""
Get the crystal U matrix
"""
return matrix.sqr(self.crystal.get_U())
def get_B(self):
"""
Get the crystal B matrix
"""
return matrix.sqr(self.crystal.get_B())
def get_A(self):
"""
Get the crystal A matrix
"""
return matrix.sqr(self.crystal.get_A())
def get_M(self):
"""
Get the Sigma M matrix
"""
return self.M_parameterisation.sigma()
def get_U_params(self):
"""
Get the U parameters
"""
return flex.double(self.U_parameterisation.get_param_vals())
def get_B_params(self):
"""
Get the B parameters
"""
return flex.double(self.B_parameterisation.get_param_vals())
def get_M_params(self):
"""
Get the M parameters
"""
return self.M_parameterisation.parameters()
def set_U_params(self, params):
"""
Set the U parameters
"""
return self.U_parameterisation.set_param_vals(params)
def set_B_params(self, params):
"""
Set the B parameters
"""
return self.B_parameterisation.set_param_vals(params)
def set_M_params(self, params):
"""
Set the M parameters
"""
return self.M_parameterisation.set_parameters(params)
def num_U_params(self):
"""
Get the number of U parameters
"""
return len(self.get_U_params())
def num_B_params(self):
"""
Get the number of B parameters
"""
return len(self.get_B_params())
def num_M_params(self):
"""
Get the number of M parameters
"""
return len(self.get_M_params())
def get_dU_dp(self):
"""
Get the first derivatives of U w.r.t its parameters
"""
return flex.mat3_double(self.U_parameterisation.get_ds_dp())
def get_dB_dp(self):
"""
Get the first derivatives of B w.r.t its parameters
"""
return flex.mat3_double(self.B_parameterisation.get_ds_dp())
def get_dM_dp(self):
"""
Get the first derivatives of M w.r.t its parameters
"""
return self.M_parameterisation.first_derivatives()
def get_active_parameters(self):
"""
Get the active parameters in order: U, B, M, L, W
"""
active_params = flex.double()
if not self._is_orientation_fixed:
active_params.extend(self.get_U_params())
if not self._is_unit_cell_fixed:
active_params.extend(self.get_B_params())
if not self._is_mosaic_spread_fixed:
active_params.extend(self.get_M_params())
if not self._is_wavelength_spread_fixed:
active_params.extend(self.get_L_params())
assert len(active_params) > 0
return active_params
def set_active_parameters(self, params):
"""
Set the active parameters in order: U, B, M, L, W
"""
if not self._is_orientation_fixed:
temp = params[:self.num_U_params()]
params = params[self.num_U_params():]
self.set_U_params(temp)
if not self._is_unit_cell_fixed:
temp = params[:self.num_B_params()]
params = params[self.num_B_params():]
self.set_B_params(temp)
if not self._is_mosaic_spread_fixed:
temp = params[:self.num_M_params()]
params = params[self.num_M_params():]
self.set_M_params(temp)
if not self._is_wavelength_spread_fixed:
temp = params[:self.num_L_params()]
params = params[self.num_L_params():]
self.set_L_params(temp)
def get_labels(self):
"""
Get the parameter labels
"""
labels = []
if not self._is_orientation_fixed:
for i in range(len(self.get_U_params())):
labels.append("Crystal_U_%d" % i)
if not self._is_unit_cell_fixed:
for i in range(len(self.get_B_params())):
labels.append("Crystal_B_%d" % i)
if not self._is_mosaic_spread_fixed:
for i in range(len(self.get_M_params())):
labels.append("Mosaicity_%d" % i)
if not self._is_wavelength_spread_fixed:
labels.append("Wavelength_Spread")
assert len(labels) > 0
return labels
models = setup_geometry.Extract(master_phil,
local_overrides=override,
verbose=False)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
# Build a mock scan for a 180 degree sweep
sf = scan_factory()
myscan = sf.make_scan(image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
assert sweep_range == (0., pi)
assert approx_equal(im_width, 0.1 * pi / 180.)
# Build an experiment list
experiments = ExperimentList()
experiments.append(Experiment(
beam=mybeam, detector=mydetector, goniometer=mygonio,
scan=myscan, crystal=mycrystal, imageset=None))
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
# TEMPORARY TESTING HERE
from dials.algorithms.refinement.restraints.restraints import SingleUnitCellTie
uct = SingleUnitCellTie(xluc_param, [None]*6, [None]*6)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Fix crystal parameters
#xluc_param.set_fixed([True, True, True, True, True, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
crystal=crystal,
imageset=None))
scans_experiments = ExperimentList()
scans_experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
crystal=crystal,
goniometer=mygonio,
scan=myscan,
imageset=None))
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
################################
# Apply known parameter shifts #
################################
# rotate crystal (=5 mrad each rotation)
xlo_p_vals = []
p_vals = xlo_param.get_param_vals()
xlo_p_vals.append(p_vals)
new_p_vals = [a + b for a, b in zip(p_vals, [5., 5., 5.])]
xlo_param.set_param_vals(new_p_vals)
# change unit cell (=1.0 Angstrom length upsets, 0.5 degree of
# gamma angle)
def test(dials_regression):
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
from scitbx import matrix
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
# Get modules to build models and minimiser using PHIL
from dials.tests.algorithms.refinement import setup_geometry
# Symmetry constrained parameterisation for the unit cell
DEG2RAD = math.pi / 180.0
RAD2DEG = 180.0 / math.pi
master_phil = parse(
"""
include scope dials.tests.algorithms.refinement.geometry_phil
include scope dials.tests.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
# make cell more oblique
args = [
"a.direction.close_to.sd=5",
"b.direction.close_to.sd=5",
"c.direction.close_to.sd=5",
]
models = setup_geometry.Extract(master_phil, cmdline_args=args)
crystal = models.crystal
# a hexagonal crystal is a good test case for behaviour of oblique cells
do_hexagonal = True
if do_hexagonal:
from dxtbx.model.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(
os.path.join(
dials_regression,
"refinement_test_data",
"multi_stills",
"combined_experiments.json",
),
check_format=False,
)
crystal = experiments[0].crystal
# derive finite difference gradients of various quantities wrt each param
def check_fd_gradients(parameterisation):
mp = parameterisation
p_vals = mp.get_param_vals()
deltas = [1.0e-7 for p in p_vals]
assert len(deltas) == len(p_vals)
fd_grad = []
# get matrix to unset rotations of unit cell vectors
Ut = matrix.sqr(mp.get_model().get_U()).transpose()
for i, delta in enumerate(deltas):
val = p_vals[i]
p_vals[i] -= delta / 2.0
mp.set_param_vals(p_vals)
rev_uc = mp.get_model().get_unit_cell().parameters()
rev_vec = mp.get_model().get_real_space_vectors()
rev_vec = [Ut * vec for vec in rev_vec]
rev_B = matrix.sqr(mp.get_model().get_B())
rev_O = rev_B.transpose().inverse()
p_vals[i] += delta
mp.set_param_vals(p_vals)
fwd_uc = mp.get_model().get_unit_cell().parameters()
fwd_vec = mp.get_model().get_real_space_vectors()
fwd_vec = [Ut * vec for vec in fwd_vec]
fwd_B = matrix.sqr(mp.get_model().get_B())
fwd_O = fwd_B.transpose().inverse()
fd_uc = [(f - r) / delta for f, r in zip(fwd_uc, rev_uc)]
fd_vec = [(f - r) / delta for f, r in zip(fwd_vec, rev_vec)]
fd_B = (fwd_B - rev_B) / delta
fd_O = (fwd_O - rev_O) / delta
fd_grad.append(
{
"da_dp": fd_uc[0],
"db_dp": fd_uc[1],
"dc_dp": fd_uc[2],
"daa_dp": fd_uc[3],
"dbb_dp": fd_uc[4],
"dcc_dp": fd_uc[5],
"davec_dp": fd_vec[0],
"dbvec_dp": fd_vec[1],
"dcvec_dp": fd_vec[2],
"dB_dp": fd_B,
"dO_dp": fd_O,
}
)
p_vals[i] = val
# return to the initial state
mp.set_param_vals(p_vals)
return fd_grad
assert CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
from dials.algorithms.refinement.restraints.restraints import SingleUnitCellTie
assert SingleUnitCellTie(xluc_param, [0] * 6, [0] * 6)
from scitbx.math import angle_derivative_wrt_vectors
B = matrix.sqr(crystal.get_B())
O = (B.transpose()).inverse()
a, b, c, aa, bb, cc = crystal.get_unit_cell().parameters()
aa *= DEG2RAD
bb *= DEG2RAD
cc *= DEG2RAD
Ut = matrix.sqr(crystal.get_U()).transpose()
avec, bvec, cvec = [Ut * vec for vec in crystal.get_real_space_vectors()]
# calculate d[B^T]/dp
dB_dp = xluc_param.get_ds_dp()
dBT_dp = [dB.transpose() for dB in dB_dp]
# calculate d[O]/dp
dO_dp = [-O * dBT * O for dBT in dBT_dp]
# function to get analytical derivative of angles wrt vectors
def dangle(u, v):
return [matrix.col(e) for e in angle_derivative_wrt_vectors(u, v)]
dalpha_db, dalpha_dc = dangle(bvec, cvec)
dbeta_da, dbeta_dc = dangle(avec, cvec)
dgamma_da, dgamma_db = dangle(avec, bvec)
# get all FD derivatives
fd_grad = check_fd_gradients(xluc_param)
# look at each parameter
for i, dO in enumerate(dO_dp):
# print
# print "***** PARAMETER {0} *****".format(i)
# print "dB_dp analytical"
# print dB_dp[i]
# print "dB_dp FD"
# print fd_grad[i]['dB_dp']
# print
# dB_dp is good. What about dO_dp?
# print "O MATRIX"
# print "dO_dp analytical"
# print dO.round(6)
# print "dO_dp FD"
# print fd_grad[i]['dO_dp'].round(6)
# print
assert approx_equal(dO, fd_grad[i]["dO_dp"])
# extract derivatives of each unit cell vector wrt p
dav_dp, dbv_dp, dcv_dp = dO.transpose().as_list_of_lists()
dav_dp = matrix.col(dav_dp)
dbv_dp = matrix.col(dbv_dp)
dcv_dp = matrix.col(dcv_dp)
# check these are correct vs FD
# print "CELL VECTORS"
# diff = dav_dp - fd_grad[i]['davec_dp']
# print 2 * diff.length() / (dav_dp.length() + fd_grad[i]['davec_dp'].length()) * 100
# print 'davec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dav_dp.elems)
# print 'davec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['davec_dp'].elems)
assert approx_equal(dav_dp, fd_grad[i]["davec_dp"])
# diff = dbv_dp - fd_grad[i]['dbvec_dp']
# print 2 * diff.length() / (dbv_dp.length() + fd_grad[i]['dbvec_dp'].length()) * 100
# print 'dbvec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dbv_dp.elems)
# print 'dbvec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['dbvec_dp'].elems)
assert approx_equal(dbv_dp, fd_grad[i]["dbvec_dp"])
# diff = dcv_dp - fd_grad[i]['dcvec_dp']
# print 2 * diff.length() / (dcv_dp.length() + fd_grad[i]['dcvec_dp'].length()) * 100
# print 'dcvec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dcv_dp.elems)
# print 'dcvec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['dcvec_dp'].elems)
# print
assert approx_equal(dcv_dp, fd_grad[i]["dcvec_dp"])
# print "CELL LENGTHS"
da_dp = 1.0 / a * avec.dot(dav_dp)
# print "d[a]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(da_dp, fd_grad[i]['da_dp'], i)
assert approx_equal(da_dp, fd_grad[i]["da_dp"])
db_dp = 1.0 / b * bvec.dot(dbv_dp)
# print "d[b]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(db_dp, fd_grad[i]['db_dp'], i)
assert approx_equal(db_dp, fd_grad[i]["db_dp"])
dc_dp = 1.0 / c * cvec.dot(dcv_dp)
# print "d[c]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dc_dp, fd_grad[i]['dc_dp'], i)
assert approx_equal(dc_dp, fd_grad[i]["dc_dp"])
# print
# print "CELL ANGLES"
daa_dp = RAD2DEG * (dbv_dp.dot(dalpha_db) + dcv_dp.dot(dalpha_dc))
dbb_dp = RAD2DEG * (dav_dp.dot(dbeta_da) + dcv_dp.dot(dbeta_dc))
dcc_dp = RAD2DEG * (dav_dp.dot(dgamma_da) + dbv_dp.dot(dgamma_db))
# print "d[alpha]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(daa_dp, fd_grad[i]['daa_dp'], i)
# print "d[beta]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dbb_dp, fd_grad[i]['dbb_dp'], i)
# print "d[gamma]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dcc_dp, fd_grad[i]['dcc_dp'], i)
assert approx_equal(daa_dp, fd_grad[i]["daa_dp"])
assert approx_equal(dbb_dp, fd_grad[i]["dbb_dp"])
assert approx_equal(dcc_dp, fd_grad[i]["dcc_dp"])
def test1():
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression", test=os.path.isdir)
# use a datablock that contains a CS-PAD detector description
data_dir = os.path.join(dials_regression, "refinement_test_data",
"hierarchy_test")
datablock_path = os.path.join(data_dir, "datablock.json")
assert os.path.exists(datablock_path)
# load models
from dxtbx.datablock import DataBlockFactory
datablock = DataBlockFactory.from_serialized_format(datablock_path,
check_format=False)
im_set = datablock[0].extract_imagesets()[0]
from copy import deepcopy
detector = deepcopy(im_set.get_detector())
beam = im_set.get_beam()
# we'll invent a crystal, goniometer and scan for this test
from dxtbx.model import Crystal
crystal = Crystal((40., 0., 0.), (0., 40., 0.), (0., 0., 40.),
space_group_symbol="P1")
from dxtbx.model import GoniometerFactory
goniometer = GoniometerFactory.known_axis((1., 0., 0.))
# Build a mock scan for a 180 degree sweep
from dxtbx.model import ScanFactory
sf = ScanFactory()
scan = sf.make_scan(image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=range(1800),
deg=True)
sweep_range = scan.get_oscillation_range(deg=False)
im_width = scan.get_oscillation(deg=False)[1]
assert sweep_range == (0., pi)
assert approx_equal(im_width, 0.1 * pi / 180.)
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Build an experiment list
experiments = ExperimentList()
experiments.append(
Experiment(beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
crystal=crystal,
imageset=None))
# simulate some reflections
refs, ref_predictor = generate_reflections(experiments)
# move the detector quadrants apart by 2mm both horizontally and vertically
from dials.algorithms.refinement.parameterisation \
import DetectorParameterisationHierarchical
det_param = DetectorParameterisationHierarchical(detector, level=1)
det_p_vals = det_param.get_param_vals()
p_vals = list(det_p_vals)
p_vals[1] += 2
p_vals[2] -= 2
p_vals[7] += 2
p_vals[8] += 2
p_vals[13] -= 2
p_vals[14] += 2
p_vals[19] -= 2
p_vals[20] -= 2
det_param.set_param_vals(p_vals)
# reparameterise the detector at the new perturbed geometry
det_param = DetectorParameterisationHierarchical(detector, level=1)
# parameterise other models
from dials.algorithms.refinement.parameterisation.beam_parameters import \
BeamParameterisation
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalOrientationParameterisation, CrystalUnitCellParameterisation
beam_param = BeamParameterisation(beam, goniometer)
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
# fix beam
beam_param.set_fixed([True] * 3)
# fix crystal
xluc_param.set_fixed([True] * 6)
xlo_param.set_fixed([True] * 3)
# parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import \
XYPhiPredictionParameterisation
from dials.algorithms.refinement.parameterisation.parameter_report import \
ParameterReporter
pred_param = XYPhiPredictionParameterisation(experiments, [det_param],
[beam_param], [xlo_param],
[xluc_param])
param_reporter = ParameterReporter([det_param], [beam_param], [xlo_param],
[xluc_param])
# reflection manager and target function
from dials.algorithms.refinement.target import \
LeastSquaresPositionalResidualWithRmsdCutoff
from dials.algorithms.refinement.reflection_manager import ReflectionManager
refman = ReflectionManager(refs, experiments, nref_per_degree=20)
# set a very tight rmsd target of 1/10000 of a pixel
target = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments,
ref_predictor,
refman,
pred_param,
restraints_parameterisation=None,
frac_binsize_cutoff=0.0001)
# minimisation engine
from dials.algorithms.refinement.engine \
import LevenbergMarquardtIterations as Refinery
refinery = Refinery(target=target,
prediction_parameterisation=pred_param,
log=None,
verbosity=0,
max_iterations=20)
# Refiner
from dials.algorithms.refinement.refiner import Refiner
refiner = Refiner(reflections=refs,
experiments=experiments,
pred_param=pred_param,
param_reporter=param_reporter,
refman=refman,
target=target,
refinery=refinery,
verbosity=0)
history = refiner.run()
assert history.reason_for_termination == "RMSD target achieved"
#compare detector with original detector
orig_det = im_set.get_detector()
refined_det = refiner.get_experiments()[0].detector
from scitbx import matrix
import math
for op, rp in zip(orig_det, refined_det):
# compare the origin vectors by...
o1 = matrix.col(op.get_origin())
o2 = matrix.col(rp.get_origin())
# ...their relative lengths
assert approx_equal(math.fabs(o1.length() - o2.length()) / o1.length(),
0,
eps=1e-5)
# ...the angle between them
assert approx_equal(o1.accute_angle(o2), 0, eps=1e-5)
print "OK"
return
def test(args=[]):
#############################
# Setup experimental models #
#############################
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
models = setup_geometry.Extract(master_phil, cmdline_args=args)
single_panel_detector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Make a 3x3 multi panel detector filling the same space as the existing
# single panel detector. Each panel of the multi-panel detector has pixels with
# 1/3 the length dimensions of the single panel.
multi_panel_detector = Detector()
for x in range(3):
for y in range(3):
new_panel = make_panel_in_array((x, y), single_panel_detector[0])
multi_panel_detector.add_panel(new_panel)
# Build a mock scan for a 180 degree sweep
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(1800)),
deg=True,
)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sweep_range == (0.0, pi)
assert approx_equal(im_width, 0.1 * pi / 180.0)
# Build ExperimentLists
experiments_single_panel = ExperimentList()
experiments_multi_panel = ExperimentList()
experiments_single_panel.append(
Experiment(
beam=mybeam,
detector=single_panel_detector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
experiments_multi_panel.append(
Experiment(
beam=mybeam,
detector=multi_panel_detector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(single_panel_detector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
multi_det_param = DetectorParameterisationMultiPanel(
multi_panel_detector, mybeam)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Fix crystal parameters
# xluc_param.set_fixed([True, True, True, True, True, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
pred_param = XYPhiPredictionParameterisation(experiments_single_panel,
[det_param], [s0_param],
[xlo_param], [xluc_param])
pred_param2 = XYPhiPredictionParameterisation(
experiments_multi_panel,
[multi_det_param],
[s0_param],
[xlo_param],
[xluc_param],
)
################################
# Apply known parameter shifts #
################################
# shift detectors by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [
a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])
]
det_param.set_param_vals(p_vals)
multi_det_p_vals = multi_det_param.get_param_vals()
p_vals = [
a + b for a, b in zip(multi_det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])
]
multi_det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])]
xlo_param.set_param_vals(p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = mycrystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(mycrystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# for the reflection predictor, it doesn't matter which experiment list is
# passed, as the detector is not used
ref_predictor = ScansRayPredictor(experiments_single_panel, sweep_range)
# get two sets of identical reflections
obs_refs = ref_predictor(indices)
obs_refs2 = ref_predictor(indices)
for r1, r2 in zip(obs_refs, obs_refs2):
assert r1["s1"] == r2["s1"]
# get the panel intersections
sel = ray_intersection(single_panel_detector, obs_refs)
obs_refs = obs_refs.select(sel)
sel = ray_intersection(multi_panel_detector, obs_refs2)
obs_refs2 = obs_refs2.select(sel)
assert len(obs_refs) == len(obs_refs2)
# Set 'observed' centroids from the predicted ones
obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"]
obs_refs2["xyzobs.mm.value"] = obs_refs2["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.0
px_size = single_panel_detector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2)
# set the variances and frame numbers
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
obs_refs2["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# Add in flags and ID columns by copying into standard reflection tables
tmp = flex.reflection_table.empty_standard(len(obs_refs))
tmp.update(obs_refs)
obs_refs = tmp
tmp = flex.reflection_table.empty_standard(len(obs_refs2))
tmp.update(obs_refs2)
obs_refs2 = tmp
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
multi_det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(obs_refs, experiments_single_panel)
refman2 = ReflectionManager(obs_refs, experiments_multi_panel)
###############################
# Set up the target functions #
###############################
mytarget = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments_single_panel,
ScansExperimentsPredictor(experiments_single_panel),
refman,
pred_param,
restraints_parameterisation=None,
)
mytarget2 = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments_multi_panel,
ScansExperimentsPredictor(experiments_multi_panel),
refman2,
pred_param2,
restraints_parameterisation=None,
)
#################################
# Set up the refinement engines #
#################################
refiner = setup_minimiser.Extract(master_phil,
mytarget,
pred_param,
cmdline_args=args).refiner
refiner2 = setup_minimiser.Extract(master_phil,
mytarget2,
pred_param2,
cmdline_args=args).refiner
refiner.run()
# reset parameters and run refinement with the multi panel detector
s0_param.set_param_vals(s0_p_vals)
multi_det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
refiner2.run()
# same number of steps
assert refiner.get_num_steps() == refiner2.get_num_steps()
# same rmsds
for rmsd, rmsd2 in zip(refiner.history["rmsd"], refiner2.history["rmsd"]):
assert approx_equal(rmsd, rmsd2)
# same parameter values each step
for params, params2 in zip(refiner.history["parameter_vector"],
refiner.history["parameter_vector"]):
assert approx_equal(params, params2)
def test(args=[]):
# Python and cctbx imports
from math import pi
import random
from scitbx import matrix
from scitbx.array_family import flex
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
# Experimental model builder
from dials.test.algorithms.refinement.setup_geometry import Extract
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Model parameterisations
from dials.algorithms.refinement.parameterisation.detector_parameters import (
DetectorParameterisationSinglePanel, )
from dials.algorithms.refinement.parameterisation.beam_parameters import (
BeamParameterisation, )
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection
from dials.algorithms.refinement.prediction.managed_predictors import (
ScansRayPredictor,
ScansExperimentsPredictor,
)
from cctbx.sgtbx import space_group, space_group_symbols
# Parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import (
XYPhiPredictionParameterisation, )
# Imports for the target function
from dials.algorithms.refinement.target import (
LeastSquaresPositionalResidualWithRmsdCutoff, )
from dials.algorithms.refinement.reflection_manager import ReflectionManager
# Local functions
def random_direction_close_to(vector, sd=0.5):
return vector.rotate_around_origin(
matrix.col((random.random(), random.random(),
random.random())).normalize(),
random.gauss(0, sd),
deg=True,
)
#############################
# Setup experimental models #
#############################
# make a small cell to speed up calculations
overrides = """geometry.parameters.crystal.a.length.range = 10 15
geometry.parameters.crystal.b.length.range = 10 15
geometry.parameters.crystal.c.length.range = 10 15"""
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
""",
process_includes=True,
)
models = Extract(master_phil, overrides, cmdline_args=args)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Build a mock scan for a 180 degree sweep of 0.1 degree images
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(1800)),
deg=True,
)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sweep_range == (0.0, pi)
assert approx_equal(im_width, 0.1 * pi / 180.0)
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
pred_param = XYPhiPredictionParameterisation(experiments, [det_param],
[s0_param], [xlo_param],
[xluc_param])
################################
# Apply known parameter shifts #
################################
# shift detector by 0.2 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [
a + b for a, b in zip(det_p_vals, [2.0, 2.0, 2.0, 2.0, 2.0, 2.0])
]
det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in one axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[1] += 2.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])]
xlo_param.set_param_vals(p_vals)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# Predict rays within the sweep range
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs = ray_predictor(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ScansExperimentsPredictor(experiments)
obs_refs["id"] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2)
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(obs_refs, experiments)
##############################
# Set up the target function #
##############################
# Redefine the reflection predictor to use the type expected by the Target class
ref_predictor = ScansExperimentsPredictor(experiments)
mytarget = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments,
ref_predictor,
refman,
pred_param,
restraints_parameterisation=None)
# get the functional and gradients
mytarget.predict()
L, dL_dp, curvs = mytarget.compute_functional_gradients_and_curvatures()
####################################
# Do FD calculation for comparison #
####################################
# function for calculating finite difference gradients of the target function
def get_fd_gradients(target, pred_param, deltas):
"""Calculate centered finite difference gradients for each of the
parameters of the target function.
"deltas" must be a sequence of the same length as the parameter list, and
contains the step size for the difference calculations for each parameter.
"""
p_vals = pred_param.get_param_vals()
assert len(deltas) == len(p_vals)
fd_grad = []
fd_curvs = []
for i in range(len(deltas)):
val = p_vals[i]
p_vals[i] -= deltas[i] / 2.0
pred_param.set_param_vals(p_vals)
target.predict()
rev_state = target.compute_functional_gradients_and_curvatures()
p_vals[i] += deltas[i]
pred_param.set_param_vals(p_vals)
target.predict()
fwd_state = target.compute_functional_gradients_and_curvatures()
# finite difference estimation of first derivatives
fd_grad.append((fwd_state[0] - rev_state[0]) / deltas[i])
# finite difference estimation of curvatures, using the analytical
# first derivatives
fd_curvs.append((fwd_state[1][i] - rev_state[1][i]) / deltas[i])
# set parameter back to centred value
p_vals[i] = val
# return to the initial state
pred_param.set_param_vals(p_vals)
return fd_grad, fd_curvs
# test normalised differences between FD and analytical calculations
fdgrads = get_fd_gradients(mytarget, pred_param,
[1.0e-7] * len(pred_param))
diffs = [a - b for a, b in zip(dL_dp, fdgrads[0])]
norm_diffs = tuple([a / b for a, b in zip(diffs, fdgrads[0])])
for e in norm_diffs:
assert abs(e) < 0.001 # check differences less than 0.1%
# test normalised differences between FD curvatures and analytical least
# squares approximation. We don't expect this to be especially close
if curvs:
diffs = [a - b for a, b in zip(curvs, fdgrads[1])]
norm_diffs = tuple([a / b for a, b in zip(diffs, fdgrads[1])])
for e in norm_diffs:
assert abs(e) < 0.1 # check differences less than 10%
class CrystalRefiner(object):
def __init__(self, experiment, reflections, parameters):
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation, )
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation, )
# Store the input
self.experiment = experiment
self.reflections = reflections
self.parameters = parameters
# Get the crystal model and the parameterisation
self.crystal = self.experiment.crystal
self.cucp = CrystalUnitCellParameterisation(self.crystal)
self.cop = CrystalOrientationParameterisation(self.crystal)
# Get the current values and generate some offsets
values = flex.double(self.cucp.get_param_vals() +
self.cop.get_param_vals())
offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] +
[0.1, 0.1, 0.1])
# The optimization history
self.history = []
# Get the initial cell and initial score
initial_cell = self.crystal.get_unit_cell()
initial_orientation = self.crystal.get_U()
initial_score = self.target(values)
# Perform the optimization
optimizer = simple_simplex(values, offset, self, 2000)
result = optimizer.get_solution()
# Print some information
fmt = "(%.3f, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f)"
print("Initial cell:", initial_cell)
print("Final cell: ", self.crystal.get_unit_cell())
print("Initial orientation: ", fmt % tuple(initial_orientation))
print("Final orientation: ", fmt % tuple(self.crystal.get_U()))
# Print RMSD
Xobs, Yobs, _ = self.reflections["xyzobs.px.value"].parts()
Xcal, Ycal, _ = self.reflections["xyzcal.px"].parts()
rmsd_x = sqrt(flex.sum((Xcal - Xobs)**2) / len(Xcal))
rmsd_y = sqrt(flex.sum((Ycal - Yobs)**2) / len(Ycal))
print("RMSD X, Y (px): %f, %f" % (rmsd_x, rmsd_y))
def target(self, vector):
"""
The target function
"""
from dials.array_family import flex
from math import sqrt
# Get the cell and orientation parameters
cell_parms = self.cucp.get_param_vals()
orientation_parms = self.cop.get_param_vals()
assert len(vector) == len(cell_parms) + len(orientation_parms)
# Update the cell and orientation parameters
tst_cell = vector[:len(cell_parms)]
tst_orientation = vector[len(cell_parms):len(cell_parms) +
len(orientation_parms)]
self.cucp.set_param_vals(tst_cell)
self.cop.set_param_vals(tst_orientation)
# Generate predicted positions
s1_cal, s2_cal = self.generate_predictions(self.experiment,
self.reflections,
self.parameters)
# Do the ray intersection
self.reflections["s1"] = s1_cal
self.reflections["s2"] = s2_cal
self.reflections["xyzcal.px"] = flex.vec3_double([
self.experiment.detector[0].get_ray_intersection_px(s1) + (0, )
for s1 in s1_cal
])
# Get predictions and observations
Xobs, Yobs, _ = self.reflections["xyzobs.px.value"].parts()
Xcal, Ycal, _ = self.reflections["xyzcal.px"].parts()
# Compute the rmsd between observed and calculated
score = flex.sum((Xobs - Xcal)**2 + (Yobs - Ycal)**2)
# Append to the history
self.history.append((tst_cell, tst_orientation, score))
# Print some info
print(
"Cell: %.3f %.3f %.3f %.3f %.3f %.3f; Phi: %.3f %.3f %.3f; RMSD: %.3f"
% (tuple(self.crystal.get_unit_cell().parameters()) +
tuple(tst_orientation) + tuple((sqrt(score / len(Xobs)), ))))
return score
def generate_predictions(self, experiment, reflections, parameters):
# Create the mosaicity model
parameterisation = MosaicityParameterisation(parameters)
# The crystal A and beam s0
A = matrix.sqr(experiment.crystal.get_A())
s0 = matrix.col(experiment.beam.get_s0())
s0_length = s0.length()
# Compute all the vectors
s1_cal = flex.vec3_double()
s2_cal = flex.vec3_double()
for i in range(len(reflections)):
# Compute the reciprocal lattice vector
h = matrix.col(reflections[i]["miller_index"])
r = A * h
s2 = s0 + r
# Rotate the covariance matrix
R = compute_change_of_basis_operation(s0, s2)
S = R * parameterisation.sigma() * R.transpose()
mu = R * s2
assert abs(1 - mu.normalize().dot(matrix.col((0, 0, 1)))) < 1e-7
# Partition the mean vector
mu1 = matrix.col((mu[0], mu[1]))
mu2 = mu[2]
# Partition the covariance matrix
S11 = matrix.sqr((S[0], S[1], S[3], S[4]))
S12 = matrix.col((S[2], S[5]))
S21 = matrix.col((S[6], S[7])).transpose()
S22 = S[8]
# Compute the conditional mean
mubar = mu1 + S12 * (1 / S22) * (s0_length - mu2)
# Compute the vector and rotate
v = matrix.col(
(mubar[0], mubar[1], s0_length)).normalize() * s0_length
s1 = R.transpose() * v
# Append the 2 vectors
s1_cal.append(s1)
s2_cal.append(s2)
# Return the predicted vectors
return s1_cal, s2_cal
def test(args=[]):
#############################
# Setup experimental models #
#############################
master_phil = parse(
"""
include scope dials.tests.algorithms.refinement.geometry_phil
include scope dials.tests.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
models = setup_geometry.Extract(
master_phil,
cmdline_args=args,
local_overrides="geometry.parameters.random_seed = 1",
)
crystal1 = models.crystal
models = setup_geometry.Extract(
master_phil,
cmdline_args=args,
local_overrides="geometry.parameters.random_seed = 2",
)
mydetector = models.detector
mygonio = models.goniometer
crystal2 = models.crystal
mybeam = models.beam
# Build a mock scan for an 18 degree sequence
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 180),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(180)),
deg=True,
)
sequence_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sequence_range == (0.0, pi / 10)
assert approx_equal(im_width, 0.1 * pi / 180.0)
# Build an experiment list
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal1,
imageset=None,
))
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal2,
imageset=None,
))
assert len(experiments.detectors()) == 1
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xl1o_param = CrystalOrientationParameterisation(crystal1)
xl1uc_param = CrystalUnitCellParameterisation(crystal1)
xl2o_param = CrystalOrientationParameterisation(crystal2)
xl2uc_param = CrystalUnitCellParameterisation(crystal2)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
################################
# Apply known parameter shifts #
################################
# shift detector by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [
a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])
]
det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = []
for xlo in (xl1o_param, xl2o_param):
p_vals = xlo.get_param_vals()
xlo_p_vals.append(p_vals)
new_p_vals = [a + b for a, b in zip(p_vals, [2.0, 2.0, 2.0])]
xlo.set_param_vals(new_p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = []
for xluc, xl in ((xl1uc_param, crystal1), ((xl2uc_param, crystal2))):
p_vals = xluc.get_param_vals()
xluc_p_vals.append(p_vals)
cell_params = xl.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(xl.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc.set_param_vals(X)
#############################
# Generate some reflections #
#############################
# All indices in a 2.5 Angstrom sphere for crystal1
resolution = 2.5
index_generator = IndexGenerator(
crystal1.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices1 = index_generator.to_array()
# All indices in a 2.5 Angstrom sphere for crystal2
resolution = 2.5
index_generator = IndexGenerator(
crystal2.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices2 = index_generator.to_array()
# Predict rays within the sequence range. Set experiment IDs
ray_predictor = ScansRayPredictor(experiments, sequence_range)
obs_refs1 = ray_predictor(indices1, experiment_id=0)
obs_refs1["id"] = flex.int(len(obs_refs1), 0)
obs_refs2 = ray_predictor(indices2, experiment_id=1)
obs_refs2["id"] = flex.int(len(obs_refs2), 1)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs1)
obs_refs1 = obs_refs1.select(intersects)
intersects = ray_intersection(mydetector, obs_refs2)
obs_refs2 = obs_refs2.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ScansExperimentsPredictor(experiments)
obs_refs1 = ref_predictor(obs_refs1)
obs_refs2 = ref_predictor(obs_refs2)
# Set 'observed' centroids from the predicted ones
obs_refs1["xyzobs.mm.value"] = obs_refs1["xyzcal.mm"]
obs_refs2["xyzobs.mm.value"] = obs_refs2["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 18.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs1), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs1), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs1), (im_width / 2.0)**2)
obs_refs1["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
var_x = flex.double(len(obs_refs2), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs2), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs2), (im_width / 2.0)**2)
obs_refs2["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# concatenate reflection lists
obs_refs1.extend(obs_refs2)
obs_refs = obs_refs1
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xl1o_param.set_param_vals(xlo_p_vals[0])
xl2o_param.set_param_vals(xlo_p_vals[1])
xl1uc_param.set_param_vals(xluc_p_vals[0])
xl2uc_param.set_param_vals(xluc_p_vals[1])
# scan static first
params = phil_scope.fetch(source=parse("")).extract()
refiner = RefinerFactory.from_parameters_data_experiments(
params, obs_refs, experiments)
refiner.run()
# scan varying
params.refinement.parameterisation.scan_varying = True
refiner = RefinerFactory.from_parameters_data_experiments(
params, obs_refs, experiments)
refiner.run()
# Ensure all models have scan-varying state set
# (https://github.com/dials/dials/issues/798)
refined_experiments = refiner.get_experiments()
sp = [xl.get_num_scan_points() for xl in refined_experiments.crystals()]
assert sp.count(181) == 2
class Apple(object):
def __init__(self, reflection_file, experiment_file):
data = pickle.load(open(reflection_file, "rb"))
self.data = data.select(data["intensity.sum.variance"] > 0)
i = data["intensity.sum.value"]
v = data["intensity.sum.variance"]
s = flex.sqrt(v)
self.i_s = i / s
self.scale = 2
from dxtbx.model.experiment_list import ExperimentListFactory
expt = ExperimentListFactory.from_json_file(experiment_file)
panel = expt.detectors()[0][0]
crystal = expt.crystals()[0]
self.s0 = matrix.col(expt.beams()[0].get_s0())
wavelength = expt.beams()[0].get_wavelength()
# make a list of observed q positions
self.qobs = []
for j in range(data.size()):
x, y, z = data["xyzobs.px.value"][j]
p = matrix.col(panel.get_pixel_lab_coord((x, y)))
q = p.normalize() / wavelength - self.s0
self.qobs.append(q)
self.wavelength = wavelength
self.panel = panel
self.beam = expt.beams()[0]
self.crystal = crystal
# slurp data from $somewhere
imageset = expt.imagesets()[0]
self.raw_data = imageset.get_raw_data(0)[0]
self.imageset = imageset
return
def load(self, filename):
from dxtbx import load
i = load(filename)
self.raw_data = i.get_raw_data()
return
def refine(self, do_print=False):
crystal = self.crystal
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation,
CrystalOrientationParameterisation,
)
self.cucp = CrystalUnitCellParameterisation(crystal)
self.cop = CrystalOrientationParameterisation(crystal)
self.zero()
# 0-point and deltas
values = flex.double(self.cucp.get_param_vals() +
self.cop.get_param_vals())
offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] +
[0.1, 0.1, 0.1])
initial = crystal.get_unit_cell()
self.cells = []
self.best_score = 1e99
initial_score = self.target(values)
doohicky = simple_simplex(values, offset, self, 2000)
best = doohicky.get_solution()
if do_print:
print("Initial cell:", initial)
print("Final cell: ", crystal.get_unit_cell())
print("Score change", initial_score,
self.target(best, do_print=False))
self.best = best
def plot_map(self, map, filename):
import matplotlib
matplotlib.use("Agg")
from matplotlib import pyplot
data = map.as_numpy_array()
fig = pyplot.gcf()
pyplot.imshow(data, cmap="gray_r")
pyplot.colorbar()
pyplot.savefig(filename, dpi=400)
pyplot.clf()
return
def plot_log_map(self, map, filename):
import matplotlib
matplotlib.use("Agg")
from matplotlib import pyplot
negative = map.as_1d() <= 0
map.as_1d().set_selected(negative, 1)
logmap = flex.log10(map.as_double())
data = logmap.as_numpy_array()
fig = pyplot.gcf()
pyplot.imshow(data, cmap="gray_r")
pyplot.colorbar()
pyplot.savefig(filename, dpi=400)
pyplot.clf()
return
def render_distance(self):
distance_map = flex.double(flex.grid(self.raw_data.focus()))
origin = self.panel.get_origin()
fast = self.panel.get_fast_axis()
slow = self.panel.get_slow_axis()
nfast, nslow = self.panel.get_image_size()
UB = matrix.sqr(self.crystal.get_A())
UBi = UB.inverse()
from dials_scratch import q_map
distance_map = q_map(self.panel, self.beam, UB, 1)
return distance_map
def target(self, vector, do_print=False):
cell_parms = self.cucp.get_param_vals()
orientation_parms = self.cop.get_param_vals()
assert len(vector) == len(cell_parms) + len(orientation_parms)
tst_cell = vector[:len(cell_parms)]
tst_orientation = vector[len(cell_parms):len(cell_parms) +
len(orientation_parms)]
self.cucp.set_param_vals(tst_cell)
self.cop.set_param_vals(tst_orientation)
from scitbx import matrix
if do_print:
print("Cell: %.3f %.3f %.3f %.3f %.3f %.3f" %
tuple(self.crystal.get_unit_cell().parameters()))
print("Phi(1,2,3): %.3f %.3f %.3f" % tuple(tst_orientation))
UB = matrix.sqr(self.crystal.get_A())
score = self.score(UB)
if score < self.best_score:
self.best_score = score
self.cells.append(self.crystal.get_unit_cell().parameters())
return score
def score(self, UB):
score = 0.0
for j in range(self.data.size()):
hkl = self.data["miller_index"][j]
q = UB * hkl
qo = self.qobs[j]
score += (q - qo).length()**2
return score
def plotify(self):
vector = self.best
cell_parms = self.cucp.get_param_vals()
orientation_parms = self.cop.get_param_vals()
assert len(vector) == len(cell_parms) + len(orientation_parms)
tst_cell = vector[:len(cell_parms)]
tst_orientation = vector[len(cell_parms):len(cell_parms) +
len(orientation_parms)]
self.cucp.set_param_vals(tst_cell)
self.cop.set_param_vals(tst_orientation)
from scitbx import matrix
UB = matrix.sqr(self.crystal.get_A())
data = self.data
self.maxq = 0
for j in range(data.size()):
hkl = data["miller_index"][j]
q = UB * hkl
qo = self.qobs[j]
print((q - qo).length(), self.i_s[j], self.dq0[j])
if (q - qo).length() > self.maxq:
self.maxq = (q - qo).length()
return
def get_maxq(self):
vector = self.best
cell_parms = self.cucp.get_param_vals()
orientation_parms = self.cop.get_param_vals()
assert len(vector) == len(cell_parms) + len(orientation_parms)
tst_cell = vector[:len(cell_parms)]
tst_orientation = vector[len(cell_parms):len(cell_parms) +
len(orientation_parms)]
self.cucp.set_param_vals(tst_cell)
self.cop.set_param_vals(tst_orientation)
from scitbx import matrix
UB = matrix.sqr(self.crystal.get_A())
data = self.data
self.maxq = 0
for j in range(data.size()):
hkl = data["miller_index"][j]
q = UB * hkl
qo = self.qobs[j]
if (q - qo).length() > self.maxq:
self.maxq = (q - qo).length()
return self.maxq
def zero(self):
from scitbx import matrix
UB = matrix.sqr(self.crystal.get_A())
data = self.data
self.dq0 = []
for j in range(data.size()):
hkl = data["miller_index"][j]
q = UB * hkl
qo = self.qobs[j]
self.dq0.append((q - qo).length())
return
def get_signal_mask(self):
if hasattr(self, "signal_mask"):
return self.signal_mask
distance_map = self.render_distance()
maxq = self.get_maxq()
self.signal_mask = distance_map.as_1d() < (self.scale * maxq)
self.signal_mask.reshape(self.raw_data.accessor())
return self.signal_mask
def make_background(self):
import copy
if hasattr(self, "background"):
return self.background
# raw background data
background = copy.deepcopy(self.raw_data).as_double()
# mask out the signal areas
mask = self.get_signal_mask()
background.as_1d().set_selected(mask.as_1d(), 0.0)
inv_mask = (~mask).as_1d().as_int()
inv_mask.reshape(self.raw_data.accessor())
from dials.algorithms.image.filter import summed_area
from dials.array_family import flex
summed_background = summed_area(background, (5, 5))
summed_mask = summed_area(inv_mask, (5, 5))
mean_background = summed_background / summed_mask.as_double()
background.as_1d().set_selected(mask.as_1d(), mean_background.as_1d())
self.background = background
return background
def get_background_subtracted_spots(self):
if hasattr(self, "background_subtracted_spots"):
return self.background_subtracted_spots
mask = self.get_signal_mask()
background = self.make_background()
import copy
background_subtracted_spots = self.raw_data.as_double() - background
background_subtracted_spots.as_1d().set_selected(~mask.as_1d(), 0)
self.background_subtracted_spots = background_subtracted_spots
return background_subtracted_spots
def integrate(self):
from scitbx.array_family import flex
from scitbx import matrix
nslow, nfast = self.raw_data.focus()
binary_map = self.get_signal_mask().as_1d().as_int()
binary_map.reshape(flex.grid(1, nslow, nfast))
# find connected regions of spots - hacking code for density modification
# this is used to determine the integration masks for the reflections
from cctbx import masks
from cctbx import uctbx
uc = uctbx.unit_cell((1, nslow, nfast, 90, 90, 90))
flood_fill = masks.flood_fill(binary_map, uc)
binary_map = binary_map.as_1d()
coms = flood_fill.centres_of_mass()
# now iterate through these blobs, find the intensity and error, and
# find the Miller index
UB = matrix.sqr(self.crystal.get_A())
UBi = UB.inverse()
winv = 1 / self.beam.get_wavelength()
data = self.raw_data.as_double()
background = self.make_background()
from dials.array_family import flex
reflections = flex.reflection_table()
num_pixels_foreground = flex.int()
background_mean = flex.double()
background_sum_value = flex.double()
background_sum_variance = flex.double()
intensity_sum_value = flex.double()
intensity_sum_variance = flex.double()
miller_index = flex.miller_index()
xyzcal_px = flex.vec3_double()
bbox = flex.int6()
dq = flex.double()
fast = flex.int(self.raw_data.size(), -1)
fast.reshape(self.raw_data.accessor())
slow = flex.int(self.raw_data.size(), -1)
slow.reshape(self.raw_data.accessor())
nslow, nfast = fast.focus()
for j in range(nslow):
for i in range(nfast):
fast[(j, i)] = i
slow[(j, i)] = j
for j in range(flood_fill.n_voids()):
sel = binary_map == (j + 2)
pixels = data.select(sel)
if flex.min(pixels) < 0:
continue
bg_pixels = background.select(sel)
n = pixels.size()
d = flex.sum(pixels)
b = flex.sum(bg_pixels)
s = d - b
# FIXME is this the best centre of mass? if this spot is actually
# there, probably no, but if not there (i.e. no spot) then background
# subtracted centre of mass likely to be very noisy - keeping the
# background in likely to make this a little more stable
xy = coms[j][2], coms[j][1]
fs = fast.select(sel)
ss = slow.select(sel)
fd = fs.as_double()
sd = ss.as_double()
p = matrix.col(
self.panel.get_pixel_lab_coord(xy)).normalize() * winv
q = p - matrix.col(self.beam.get_s0())
hkl = UBi * q
ihkl = [int(round(h)) for h in hkl]
dq.append((q - UB * ihkl).length())
# puzzle out the bounding boxes - hack here, we have maps with the
# fast and slow positions in; select from these then find max, min of
# this selection
f_min = flex.min(fs)
f_max = flex.max(fs)
s_min = flex.min(ss)
s_max = flex.max(ss)
bbox.append((f_min, f_max + 1, s_min, s_max + 1, 0, 1))
num_pixels_foreground.append(n)
background_mean.append(b / n)
background_sum_value.append(b)
background_sum_variance.append(b)
intensity_sum_value.append(s)
intensity_sum_variance.append(d + b)
miller_index.append(ihkl)
xyzcal_px.append((xy[0], xy[1], 0.0))
reflections["num_pixels.foreground"] = num_pixels_foreground
reflections["background.mean"] = background_mean
reflections["background.sum.value"] = background_sum_value
reflections["background.sum.variance"] = background_sum_variance
reflections["intensity.sum.value"] = intensity_sum_value
reflections["intensity.sum.variance"] = intensity_sum_variance
reflections["miller_index"] = miller_index
reflections["xyzcal.px"] = xyzcal_px
reflections["id"] = flex.int(miller_index.size(), 0)
reflections["panel"] = flex.size_t(miller_index.size(), 0)
reflections["bbox"] = bbox
reflections["dq"] = dq
from dials.algorithms.shoebox import MaskCode
reflections["shoebox"] = flex.shoebox(reflections["panel"],
reflections["bbox"],
allocate=True)
reflections.extract_shoeboxes(self.imageset, verbose=True)
# now iterate through these (how?!) and assign mask values
fg = self.get_signal_mask()
for reflection in reflections:
s = reflection["shoebox"]
b = reflection["bbox"]
dz, dy, dx = s.mask.focus()
for j in range(dy):
for i in range(dx):
_i = b[0]
_j = b[2]
if fg[(j + _j, i + _i)]:
m = MaskCode.Valid | MaskCode.Foreground
else:
m = MaskCode.Valid | MaskCode.Background
s.mask[(0, j, i)] = m
return reflections
def find_spots(self, min_spot_size=2, max_spot_size=100):
from dials.algorithms.spot_finding.threshold import XDSThresholdStrategy
from dials.model.data import PixelList
from dials.model.data import PixelListLabeller
image = self.raw_data
mask = self.imageset.get_mask(0)[0]
threshold_image = XDSThresholdStrategy()
threshold_mask = threshold_image(image, mask)
plist = PixelList(0, image, threshold_mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(plist)
creator = flex.PixelListShoeboxCreator(pixel_labeller, 0, 0, True,
min_spot_size, max_spot_size,
False)
shoeboxes = creator.result()
# turns out we need to manually filter the list to get a sensible answer
size = creator.spot_size()
big = size > max_spot_size
small = size < min_spot_size
bad = big | small
shoeboxes = shoeboxes.select(~bad)
centroid = shoeboxes.centroid_valid()
intensity = shoeboxes.summed_intensity()
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
reflections = flex.reflection_table(observed, shoeboxes)
return reflections
def index(self, reflections):
# FIXME allow for the fact that there could be > 1 lattice on here to
# e.g. assign index over small spherical radius
miller_index = flex.miller_index()
UB = matrix.sqr(self.crystal.get_A())
UBi = UB.inverse()
self.qobs = []
for refl in reflections:
x, y, z = refl["xyzobs.px.value"]
p = matrix.col(self.panel.get_pixel_lab_coord((x, y)))
q = p.normalize() / self.wavelength - self.s0
self.qobs.append(q)
hkl = UBi * q
ihkl = [int(round(h)) for h in hkl]
miller_index.append(ihkl)
reflections["miller_index"] = miller_index
self.data = reflections
return reflections
def tst_use_in_stills_parameterisation_for_crystal(crystal_param=0):
# test use of analytical expression in stills prediction parameterisation
from scitbx import matrix
from math import pi, sqrt, atan2
import random
print
print "Test use of analytical expressions in stills prediction " + "parameterisation for crystal parameters"
# crystal model
from dxtbx.model.crystal import crystal_model
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
crystal = crystal_model((20, 0, 0), (0, 30, 0), (0, 0, 40), space_group_symbol="P 1")
# random reorientation
e = matrix.col((random.random(), random.random(), random.random())).normalize()
angle = random.random() * 180
crystal.rotate_around_origin(e, angle)
wl = 1.1
s0 = matrix.col((0, 0, 1 / wl))
s0u = s0.normalize()
# these are stills, but need a rotation axis for the Reeke algorithm
axis = matrix.col((1, 0, 0))
# crystal parameterisations
xlop = CrystalOrientationParameterisation(crystal)
xlucp = CrystalUnitCellParameterisation(crystal)
# Find some reflections close to the Ewald sphere
from dials.algorithms.spot_prediction.reeke import reeke_model
U = crystal.get_U()
B = crystal.get_B()
UB = U * B
dmin = 4
hkl = reeke_model(UB, UB, axis, s0, dmin, margin=1).generate_indices()
# choose first reflection for now, calc quantities relating to q
h = matrix.col(hkl[0])
q = UB * h
q0 = q.normalize()
q_scalar = q.length()
qq = q_scalar * q_scalar
# calculate the axis of closest rotation
e1 = q0.cross(s0u).normalize()
# calculate c0, a vector orthogonal to s0u and e1
c0 = s0u.cross(e1).normalize()
# calculate q1
q1 = q0.cross(e1).normalize()
# calculate DeltaPsi
a = 0.5 * qq * wl
b = sqrt(qq - a * a)
r = -1.0 * a * s0u + b * c0
DeltaPsi = -1.0 * atan2(r.dot(q1), r.dot(q0))
# Checks on the reflection prediction
from libtbx.test_utils import approx_equal
# 1. check r is on the Ewald sphere
s1 = s0 + r
assert approx_equal(s1.length(), s0.length())
# 2. check DeltaPsi is correct
tst = q.rotate_around_origin(e1, DeltaPsi)
assert approx_equal(tst, r)
# choose the derivative with respect to a particular parameter.
if crystal_param < 3:
dU_dp = xlop.get_ds_dp()[crystal_param]
dq = dU_dp * B * h
else:
dB_dp = xlucp.get_ds_dp()[crystal_param - 3]
dq = U * dB_dp * h
# NKS method of calculating d[q0]/dp
q_dot_dq = q.dot(dq)
dqq = 2.0 * q_dot_dq
dq_scalar = dqq / q_scalar
dq0_dp = (q_scalar * dq - (q_dot_dq * q0)) / qq
# orthogonal to q0, as expected.
print "NKS [q0].(d[q0]/dp) = {0} (should be 0.0)".format(q0.dot(dq0_dp))
# intuitive method of calculating d[q0]/dp, based on the fact that
# it must be orthogonal to q0, i.e. in the plane containing q1 and e1
scaled = dq / q.length()
dq0_dp = scaled.dot(q1) * q1 + scaled.dot(e1) * e1
# orthogonal to q0, as expected.
print "DGW [q0].(d[q0]/dp) = {0} (should be 0.0)".format(q0.dot(dq0_dp))
# So it doesn't matter which method I use to calculate d[q0]/dp, as
# both methods give the same results
# use the fact that -e1 == q0.cross(q1) to redefine the derivative d[e1]/dp
# from Sauter et al. (2014) (A.22)
de1_dp = -1.0 * dq0_dp.cross(q1)
# this *is* orthogonal to e1, as expected.
print "[e1].(d[e1]/dp) = {0} (should be 0.0)".format(e1.dot(de1_dp))
# calculate (d[r]/d[e1])(d[e1]/dp) analytically
from scitbx.array_family import flex
from dials_refinement_helpers_ext import dRq_de
dr_de1 = matrix.sqr(dRq_de(flex.double([DeltaPsi]), flex.vec3_double([e1]), flex.vec3_double([q]))[0])
print "Analytical calculation for (d[r]/d[e1])(d[e1]/dp):"
print dr_de1 * de1_dp
# now calculate using finite differences.
dp = 1.0e-8
del_e1 = de1_dp * dp
e1f = e1 + del_e1 * 0.5
rfwd = q.rotate_around_origin(e1f, DeltaPsi)
e1r = e1 - del_e1 * 0.5
rrev = q.rotate_around_origin(e1r, DeltaPsi)
print "Finite difference estimate for (d[r]/d[e1])(d[e1]/dp):"
print (rfwd - rrev) * (1 / dp)
print "These are essentially the same :-)"
def test(args=[]):
# Python and cctbx imports
from math import pi
from scitbx import matrix
from scitbx.array_family import flex
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
# Get module to build models using PHIL
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Model parameterisations
from dials.algorithms.refinement.parameterisation.detector_parameters import \
DetectorParameterisationSinglePanel
from dials.algorithms.refinement.parameterisation.beam_parameters import \
BeamParameterisation
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalOrientationParameterisation, CrystalUnitCellParameterisation
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import \
symmetrize_reduce_enlarge
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator
from dials.algorithms.refinement.prediction import ScansRayPredictor, \
ExperimentsPredictor
from dials.algorithms.spot_prediction import ray_intersection
from cctbx.sgtbx import space_group, space_group_symbols
# Parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import \
XYPhiPredictionParameterisation # implicit import
# Imports for the target function
from dials.algorithms.refinement.target import \
LeastSquaresPositionalResidualWithRmsdCutoff # implicit import
#############################
# Setup experimental models #
#############################
master_phil = parse("""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True)
models = setup_geometry.Extract(
master_phil,
cmdline_args=args,
local_overrides="geometry.parameters.random_seed = 1")
crystal1 = models.crystal
models = setup_geometry.Extract(
master_phil,
cmdline_args=args,
local_overrides="geometry.parameters.random_seed = 2")
mydetector = models.detector
mygonio = models.goniometer
crystal2 = models.crystal
mybeam = models.beam
# Build a mock scan for a 180 degree sweep
sf = ScanFactory()
myscan = sf.make_scan(image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=range(1800),
deg=True)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sweep_range == (0., pi)
assert approx_equal(im_width, 0.1 * pi / 180.)
# Build an experiment list
experiments = ExperimentList()
experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal1,
imageset=None))
experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal2,
imageset=None))
assert len(experiments.detectors()) == 1
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xl1o_param = CrystalOrientationParameterisation(crystal1)
xl1uc_param = CrystalUnitCellParameterisation(crystal1)
xl2o_param = CrystalOrientationParameterisation(crystal2)
xl2uc_param = CrystalUnitCellParameterisation(crystal2)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Fix crystal parameters
#xluc_param.set_fixed([True, True, True, True, True, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
#pred_param = XYPhiPredictionParameterisation(experiments,
# [det_param], [s0_param], [xlo_param], [xluc_param])
################################
# Apply known parameter shifts #
################################
# shift detector by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2., 2., 2.])]
det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = []
for xlo in (xl1o_param, xl2o_param):
p_vals = xlo.get_param_vals()
xlo_p_vals.append(p_vals)
new_p_vals = [a + b for a, b in zip(p_vals, [2., 2., 2.])]
xlo.set_param_vals(new_p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = []
for xluc, xl in ((xl1uc_param, crystal1), ((xl2uc_param, crystal2))):
p_vals = xluc.get_param_vals()
xluc_p_vals.append(p_vals)
cell_params = xl.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(xl.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.e5 for e in S.forward_independent_parameters()])
xluc.set_param_vals(X)
#############################
# Generate some reflections #
#############################
#print "Reflections will be generated with the following geometry:"
#print mybeam
#print mydetector
#print crystal1
#print crystal2
# All indices in a 2.0 Angstrom sphere for crystal1
resolution = 2.0
index_generator = IndexGenerator(
crystal1.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), resolution)
indices1 = index_generator.to_array()
# All indices in a 2.0 Angstrom sphere for crystal2
resolution = 2.0
index_generator = IndexGenerator(
crystal2.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), resolution)
indices2 = index_generator.to_array()
# Predict rays within the sweep range. Set experiment IDs
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs1 = ray_predictor(indices1, experiment_id=0)
obs_refs1['id'] = flex.int(len(obs_refs1), 0)
obs_refs2 = ray_predictor(indices1, experiment_id=1)
obs_refs2['id'] = flex.int(len(obs_refs2), 1)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs1)
obs_refs1 = obs_refs1.select(intersects)
intersects = ray_intersection(mydetector, obs_refs2)
obs_refs2 = obs_refs2.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ExperimentsPredictor(experiments)
obs_refs1 = ref_predictor(obs_refs1)
obs_refs2 = ref_predictor(obs_refs2)
# Set 'observed' centroids from the predicted ones
obs_refs1['xyzobs.mm.value'] = obs_refs1['xyzcal.mm']
obs_refs2['xyzobs.mm.value'] = obs_refs2['xyzcal.mm']
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs1), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs1), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs1), (im_width / 2.)**2)
obs_refs1['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
var_x = flex.double(len(obs_refs2), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs2), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs2), (im_width / 2.)**2)
obs_refs2['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
#print "Total number of reflections excited for crystal1", len(obs_refs1)
#print "Total number of reflections excited for crystal2", len(obs_refs2)
# concatenate reflection lists
obs_refs1.extend(obs_refs2)
obs_refs = obs_refs1
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xl1o_param.set_param_vals(xlo_p_vals[0])
xl2o_param.set_param_vals(xlo_p_vals[1])
xl1uc_param.set_param_vals(xluc_p_vals[0])
xl2uc_param.set_param_vals(xluc_p_vals[1])
#print "Initial values of parameters are"
#msg = "Parameters: " + "%.5f " * len(pred_param)
#print msg % tuple(pred_param.get_param_vals())
#print
# make a refiner
from dials.algorithms.refinement.refiner import phil_scope
params = phil_scope.fetch(source=parse('')).extract()
# in case we want a plot
params.refinement.refinery.journal.track_parameter_correlation = True
# scan static first
from dials.algorithms.refinement.refiner import RefinerFactory
refiner = RefinerFactory.from_parameters_data_experiments(params,
obs_refs,
experiments,
verbosity=0)
history = refiner.run()
# scan varying
params.refinement.parameterisation.scan_varying = True
refiner = RefinerFactory.from_parameters_data_experiments(params,
obs_refs,
experiments,
verbosity=0)
history = refiner.run()
class CrystalRefiner(object):
"""
A class to perform refinement of crystal parameters
"""
def __init__(self, experiment, reflections, mosaicity, params):
"""
Perform the refinement
:param experiment: The experiment
:param reflections: The reflections
:param mosaicity: The mosaicity
:param params: The parameters
"""
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation, )
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation, )
from dials_scratch.jmp.stills import Model
from dials.array_family import flex
from scitbx import simplex
from math import sqrt
# Store the input
self.experiment = experiment
self.reflections = reflections
self.mosaicity = mosaicity
self.params = params
# Get the data and image mask
data = self.experiment.imageset.get_raw_data(0)[0]
mask = self.experiment.imageset.get_mask(0)[0]
# Initialise the model
self.model = Model(
beam=self.experiment.beam,
detector=self.experiment.detector,
crystal=self.experiment.crystal,
reflections=self.reflections,
image_data=data,
image_mask=mask,
mosaicity=self.mosaicity,
bandpass=0.0,
foreground_limit=0.3,
background_limit=0.5,
num_samples=self.params.refinement.profile.num_samples,
)
# Get the crystal model and the parameterisation
self.crystal = self.experiment.crystal
self.cucp = CrystalUnitCellParameterisation(self.crystal)
self.cop = CrystalOrientationParameterisation(self.crystal)
# Get the current values and generate some offsets
values = flex.double(self.cucp.get_param_vals() +
self.cop.get_param_vals())
offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] +
[0.1, 0.1, 0.1])
# The optimization history
self.history = []
# Get the initial cell and initial score
initial_cell = self.crystal.get_unit_cell()
initial_score = self.target(values)
# Perform the optimization
optimizer = simple_simplex(values, offset, self, 2000)
result = optimizer.get_solution()
print("Initial cell:", initial_cell)
print("Final cell: ", self.crystal.get_unit_cell())
# Compute RMSD
xcal, ycal, _ = self.model.observed().parts()
xobs, yobs, _ = self.model.predicted().parts()
rmsd_x = sqrt(flex.sum((xcal - xobs)**2) / len(xcal))
rmsd_y = sqrt(flex.sum((ycal - yobs)**2) / len(ycal))
print("RMSD X, Y (px): %f, %f" % (rmsd_x, rmsd_y))
def target(self, vector):
"""
The target function
"""
from dials.array_family import flex
from math import sqrt
# Get the cell and orientation parameters
cell_parms = self.cucp.get_param_vals()
orientation_parms = self.cop.get_param_vals()
assert len(vector) == len(cell_parms) + len(orientation_parms)
# Update the cell and orientation parameters
tst_cell = vector[:len(cell_parms)]
tst_orientation = vector[len(cell_parms):len(cell_parms) +
len(orientation_parms)]
self.cucp.set_param_vals(tst_cell)
self.cop.set_param_vals(tst_orientation)
# Update the model
self.model.crystal = self.crystal
self.model.update(pixel_lookup=False)
# Get the observed and predicted position
T = self.model.success()
O = self.model.observed()
C = self.model.predicted()
# Select only those successes
selection = T == True
Xobs, Yobs, Zobs = O.select(selection).parts()
Xcal, Ycal, Zcal = C.select(selection).parts()
# Compute the rmsd between observed and calculated
score = flex.sum((Xobs - Xcal)**2 + (Yobs - Ycal)**2 +
(Zobs - Zcal)**2)
# Append to the history
self.history.append((tst_cell, tst_orientation, score))
# Print some info
print(
"Cell: %.3f %.3f %.3f %.3f %.3f %.3f; Phi: %.3f %.3f %.3f; RMSD: %.3f"
% (tuple(self.crystal.get_unit_cell().parameters()) +
tuple(tst_orientation) + tuple((sqrt(score / len(Xobs)), ))))
return score
def test1():
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression",
test=os.path.isdir)
# use a datablock that contains a CS-PAD detector description
data_dir = os.path.join(dials_regression, "refinement_test_data",
"hierarchy_test")
datablock_path = os.path.join(data_dir, "datablock.json")
assert os.path.exists(datablock_path)
# load models
from dxtbx.datablock import DataBlockFactory
datablock = DataBlockFactory.from_serialized_format(datablock_path, check_format=False)
im_set = datablock[0].extract_imagesets()[0]
from copy import deepcopy
detector = deepcopy(im_set.get_detector())
beam = im_set.get_beam()
# we'll invent a crystal, goniometer and scan for this test
from dxtbx.model.crystal import crystal_model
crystal = crystal_model((40.,0.,0.) ,(0.,40.,0.), (0.,0.,40.),
space_group_symbol = "P1")
from dxtbx.model.experiment import goniometer_factory
goniometer = goniometer_factory.known_axis((1., 0., 0.))
# Build a mock scan for a 180 degree sweep
from dxtbx.model.scan import scan_factory
sf = scan_factory()
scan = sf.make_scan(image_range = (1,1800),
exposure_times = 0.1,
oscillation = (0, 0.1),
epochs = range(1800),
deg = True)
sweep_range = scan.get_oscillation_range(deg=False)
im_width = scan.get_oscillation(deg=False)[1]
assert sweep_range == (0., pi)
assert approx_equal(im_width, 0.1 * pi / 180.)
from dxtbx.model.experiment.experiment_list import ExperimentList, Experiment
# Build an experiment list
experiments = ExperimentList()
experiments.append(Experiment(
beam=beam, detector=detector, goniometer=goniometer,
scan=scan, crystal=crystal, imageset=None))
# simulate some reflections
refs, ref_predictor = generate_reflections(experiments)
# move the detector quadrants apart by 2mm both horizontally and vertically
from dials.algorithms.refinement.parameterisation \
import DetectorParameterisationHierarchical
det_param = DetectorParameterisationHierarchical(detector, level=1)
det_p_vals = det_param.get_param_vals()
p_vals = list(det_p_vals)
p_vals[1] += 2
p_vals[2] -= 2
p_vals[7] += 2
p_vals[8] += 2
p_vals[13] -= 2
p_vals[14] += 2
p_vals[19] -= 2
p_vals[20] -= 2
det_param.set_param_vals(p_vals)
# reparameterise the detector at the new perturbed geometry
det_param = DetectorParameterisationHierarchical(detector, level=1)
# parameterise other models
from dials.algorithms.refinement.parameterisation.beam_parameters import \
BeamParameterisation
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalOrientationParameterisation, CrystalUnitCellParameterisation
beam_param = BeamParameterisation(beam, goniometer)
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
# fix beam
beam_param.set_fixed([True]*3)
# fix crystal
xluc_param.set_fixed([True]*6)
xlo_param.set_fixed([True]*3)
# parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import \
XYPhiPredictionParameterisation
from dials.algorithms.refinement.parameterisation.parameter_report import \
ParameterReporter
pred_param = XYPhiPredictionParameterisation(experiments,
[det_param], [beam_param], [xlo_param], [xluc_param])
param_reporter = ParameterReporter([det_param], [beam_param],
[xlo_param], [xluc_param])
# reflection manager and target function
from dials.algorithms.refinement.target import \
LeastSquaresPositionalResidualWithRmsdCutoff
from dials.algorithms.refinement.reflection_manager import ReflectionManager
refman = ReflectionManager(refs, experiments, nref_per_degree=20)
# set a very tight rmsd target of 1/10000 of a pixel
target = LeastSquaresPositionalResidualWithRmsdCutoff(experiments,
ref_predictor, refman, pred_param, restraints_parameterisation=None,
frac_binsize_cutoff=0.0001)
# minimisation engine
from dials.algorithms.refinement.engine \
import LevenbergMarquardtIterations as Refinery
refinery = Refinery(target = target,
prediction_parameterisation = pred_param,
log = None,
verbosity = 0,
track_step = False,
track_gradient = False,
track_parameter_correlation = False,
max_iterations = 20)
# Refiner
from dials.algorithms.refinement.refiner import Refiner
refiner = Refiner(reflections=refs,
experiments=experiments,
pred_param=pred_param,
param_reporter=param_reporter,
refman=refman,
target=target,
refinery=refinery,
verbosity=0)
history = refiner.run()
assert history.reason_for_termination == "RMSD target achieved"
#compare detector with original detector
orig_det = im_set.get_detector()
refined_det = refiner.get_experiments()[0].detector
from scitbx import matrix
import math
for op, rp in zip(orig_det, refined_det):
# compare the origin vectors by...
o1 = matrix.col(op.get_origin())
o2 = matrix.col(rp.get_origin())
# ...their relative lengths
assert approx_equal(
math.fabs(o1.length() - o2.length()) / o1.length(), 0, eps=1e-5)
# ...the angle between them
assert approx_equal(o1.accute_angle(o2), 0, eps=1e-5)
print "OK"
return
def __init__(self, experiment, reflections, mosaicity, params):
"""
Perform the refinement
:param experiment: The experiment
:param reflections: The reflections
:param mosaicity: The mosaicity
:param params: The parameters
"""
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation, )
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation, )
from dials_scratch.jmp.stills import Model
from dials.array_family import flex
from scitbx import simplex
from math import sqrt
# Store the input
self.experiment = experiment
self.reflections = reflections
self.mosaicity = mosaicity
self.params = params
# Get the data and image mask
data = self.experiment.imageset.get_raw_data(0)[0]
mask = self.experiment.imageset.get_mask(0)[0]
# Initialise the model
self.model = Model(
beam=self.experiment.beam,
detector=self.experiment.detector,
crystal=self.experiment.crystal,
reflections=self.reflections,
image_data=data,
image_mask=mask,
mosaicity=self.mosaicity,
bandpass=0.0,
foreground_limit=0.3,
background_limit=0.5,
num_samples=self.params.refinement.profile.num_samples,
)
# Get the crystal model and the parameterisation
self.crystal = self.experiment.crystal
self.cucp = CrystalUnitCellParameterisation(self.crystal)
self.cop = CrystalOrientationParameterisation(self.crystal)
# Get the current values and generate some offsets
values = flex.double(self.cucp.get_param_vals() +
self.cop.get_param_vals())
offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] +
[0.1, 0.1, 0.1])
# The optimization history
self.history = []
# Get the initial cell and initial score
initial_cell = self.crystal.get_unit_cell()
initial_score = self.target(values)
# Perform the optimization
optimizer = simple_simplex(values, offset, self, 2000)
result = optimizer.get_solution()
print("Initial cell:", initial_cell)
print("Final cell: ", self.crystal.get_unit_cell())
# Compute RMSD
xcal, ycal, _ = self.model.observed().parts()
xobs, yobs, _ = self.model.predicted().parts()
rmsd_x = sqrt(flex.sum((xcal - xobs)**2) / len(xcal))
rmsd_y = sqrt(flex.sum((ycal - yobs)**2) / len(ycal))
print("RMSD X, Y (px): %f, %f" % (rmsd_x, rmsd_y))
def test():
# Build models, with a larger crystal than default in order to get plenty of
# reflections on the 'still' image
overrides = """
geometry.parameters.crystal.a.length.range=40 50;
geometry.parameters.crystal.b.length.range=40 50;
geometry.parameters.crystal.c.length.range=40 50;
geometry.parameters.random_seed = 42"""
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
""",
process_includes=True,
)
models = Extract(master_phil, overrides)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Build a mock scan for a 3 degree sweep
from dxtbx.model import ScanFactory
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1),
exposure_times=0.1,
oscillation=(0, 3.0),
epochs=list(range(1)),
deg=True,
)
sweep_range = myscan.get_oscillation_range(deg=False)
# Create parameterisations of these models
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
# Create a scans ExperimentList, only for generating reflections
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
# Create a stills ExperimentList
stills_experiments = ExperimentList()
stills_experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
crystal=mycrystal,
imageset=None))
# Generate rays - only to work out which hkls are predicted
ray_predictor = ScansRayPredictor(experiments, sweep_range)
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
rays = ray_predictor(indices)
# Make a standard reflection_table and copy in the ray data
reflections = flex.reflection_table.empty_standard(len(rays))
reflections.update(rays)
# Build a standard prediction parameterisation for the stills experiment to do
# FD calculation (not used for its analytical gradients)
pred_param = StillsPredictionParameterisation(
stills_experiments,
detector_parameterisations=[det_param],
beam_parameterisations=[s0_param],
xl_orientation_parameterisations=[xlo_param],
xl_unit_cell_parameterisations=[xluc_param],
)
# Make a managed SphericalRelpStillsReflectionPredictor reflection predictor
# for the first (only) experiment
ref_predictor = Predictor(stills_experiments)
# Predict these reflections in place. Must do this ahead of calculating
# the analytical gradients so quantities like s1 are correct
ref_predictor.update()
ref_predictor.predict(reflections)
# calculate analytical gradients
ag = AnalyticalGradients(
stills_experiments,
detector_parameterisation=det_param,
beam_parameterisation=s0_param,
xl_orientation_parameterisation=xlo_param,
xl_unit_cell_parameterisation=xluc_param,
)
an_grads = ag.get_beam_gradients(reflections)
an_grads.update(ag.get_crystal_orientation_gradients(reflections))
an_grads.update(ag.get_crystal_unit_cell_gradients(reflections))
# get finite difference gradients
p_vals = pred_param.get_param_vals()
deltas = [1.0e-7] * len(p_vals)
fd_grads = []
p_names = pred_param.get_param_names()
for i, delta in enumerate(deltas):
# save parameter value
val = p_vals[i]
# calc reverse state
p_vals[i] -= delta / 2.0
pred_param.set_param_vals(p_vals)
ref_predictor.update()
ref_predictor.predict(reflections)
x, y, _ = reflections["xyzcal.mm"].deep_copy().parts()
s1 = reflections["s1"].deep_copy()
rev_state = s1
# calc forward state
p_vals[i] += delta
pred_param.set_param_vals(p_vals)
ref_predictor.update()
ref_predictor.predict(reflections)
x, y, _ = reflections["xyzcal.mm"].deep_copy().parts()
s1 = reflections["s1"].deep_copy()
fwd_state = s1
# reset parameter to saved value
p_vals[i] = val
# finite difference - currently for s1 only
fd = fwd_state - rev_state
inv_delta = 1.0 / delta
s1_grads = fd * inv_delta
# store gradients
fd_grads.append({"name": p_names[i], "ds1": s1_grads})
# return to the initial state
pred_param.set_param_vals(p_vals)
for i, fd_grad in enumerate(fd_grads):
## compare FD with analytical calculations
print("\n\nParameter {0}: {1}".format(i, fd_grad["name"]))
print("d[s1]/dp for the first reflection")
print("finite diff", fd_grad["ds1"][0])
try:
an_grad = an_grads[fd_grad["name"]]
except KeyError:
continue
print("checking analytical vs finite difference gradients for s1")
for a, b in zip(fd_grad["ds1"], an_grad["ds1"]):
assert a == pytest.approx(b, abs=1e-7)
def test(init_test):
single_panel_detector = init_test.experiments_single_panel.detectors()[0]
multi_panel_detector = init_test.experiments_multi_panel.detectors()[0]
beam = init_test.experiments_single_panel.beams()[0]
gonio = init_test.experiments_single_panel.goniometers()[0]
crystal = init_test.experiments_single_panel.crystals()[0]
# Parameterise the models
det_param = DetectorParameterisationSinglePanel(single_panel_detector)
s0_param = BeamParameterisation(beam, gonio)
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
multi_det_param = DetectorParameterisationMultiPanel(multi_panel_detector, beam)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Link model parameterisations together into a parameterisation of the
# prediction equation, first for the single panel detector
pred_param = XYPhiPredictionParameterisation(
init_test.experiments_single_panel,
[det_param],
[s0_param],
[xlo_param],
[xluc_param],
)
# ... and now for the multi-panel detector
pred_param2 = XYPhiPredictionParameterisation(
init_test.experiments_multi_panel,
[multi_det_param],
[s0_param],
[xlo_param],
[xluc_param],
)
################################
# Apply known parameter shifts #
################################
# shift detectors by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])]
det_param.set_param_vals(p_vals)
multi_det_p_vals = multi_det_param.get_param_vals()
p_vals = [a + b for a, b in zip(multi_det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])]
multi_det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])]
xlo_param.set_param_vals(p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = crystal.get_unit_cell().parameters()
cell_params = [a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(crystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
multi_det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(
init_test.observations_single_panel, init_test.experiments_single_panel
)
refman2 = ReflectionManager(
init_test.observations_multi_panel, init_test.experiments_multi_panel
)
###############################
# Set up the target functions #
###############################
target = LeastSquaresPositionalResidualWithRmsdCutoff(
init_test.experiments_single_panel,
ScansExperimentsPredictor(init_test.experiments_single_panel),
refman,
pred_param,
restraints_parameterisation=None,
)
target2 = LeastSquaresPositionalResidualWithRmsdCutoff(
init_test.experiments_multi_panel,
ScansExperimentsPredictor(init_test.experiments_multi_panel),
refman2,
pred_param2,
restraints_parameterisation=None,
)
#################################
# Set up the refinement engines #
#################################
refiner = setup_minimiser.Extract(master_phil, target, pred_param).refiner
refiner2 = setup_minimiser.Extract(master_phil, target2, pred_param2).refiner
refiner.run()
# reset parameters and run refinement with the multi panel detector
s0_param.set_param_vals(s0_p_vals)
multi_det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
refiner2.run()
# same number of steps
assert refiner.get_num_steps() == refiner2.get_num_steps()
# same rmsds
for rmsd, rmsd2 in zip(refiner.history["rmsd"], refiner2.history["rmsd"]):
assert approx_equal(rmsd, rmsd2)
# same parameter values each step
for params, params2 in zip(
refiner.history["parameter_vector"], refiner.history["parameter_vector"]
):
assert approx_equal(params, params2)
experiments.append(Experiment(
beam=mybeam, detector=mydetector, goniometer=mygonio,
scan=myscan, crystal=crystal1, imageset=None))
experiments.append(Experiment(
beam=mybeam, detector=mydetector, goniometer=mygonio,
scan=myscan, crystal=crystal2, imageset=None))
assert len(experiments.detectors()) == 1
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xl1o_param = CrystalOrientationParameterisation(crystal1)
xl1uc_param = CrystalUnitCellParameterisation(crystal1)
xl2o_param = CrystalOrientationParameterisation(crystal2)
xl2uc_param = CrystalUnitCellParameterisation(crystal2)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Fix crystal parameters
#xluc_param.set_fixed([True, True, True, True, True, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
def test_single_crystal_restraints_gradients():
"""Simple test with a single triclinic crystal restrained to a target unit cell"""
from dxtbx.model.experiment_list import Experiment, ExperimentList
from dials.algorithms.refinement.parameterisation.beam_parameters import (
BeamParameterisation, )
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
from dials.algorithms.refinement.parameterisation.detector_parameters import (
DetectorParameterisationSinglePanel, )
from dials.algorithms.refinement.parameterisation.prediction_parameters import (
XYPhiPredictionParameterisation, )
from dials.test.algorithms.refinement.setup_geometry import Extract
overrides = """geometry.parameters.crystal.a.length.range = 10 50
geometry.parameters.crystal.b.length.range = 10 50
geometry.parameters.crystal.c.length.range = 10 50"""
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
""",
process_includes=True,
)
models = Extract(master_phil, overrides)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Build a mock scan for a 72 degree sequence
from dxtbx.model import ScanFactory
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 720),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(720)),
deg=True,
)
# Create parameterisations of these models
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
# Create an ExperimentList
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None,
))
# Build a prediction parameterisation
pred_param = XYPhiPredictionParameterisation(
experiments,
detector_parameterisations=[det_param],
beam_parameterisations=[s0_param],
xl_orientation_parameterisations=[xlo_param],
xl_unit_cell_parameterisations=[xluc_param],
)
# Build a restraints parameterisation
rp = RestraintsParameterisation(
detector_parameterisations=[det_param],
beam_parameterisations=[s0_param],
xl_orientation_parameterisations=[xlo_param],
xl_unit_cell_parameterisations=[xluc_param],
)
# make a unit cell target
sigma = 1.0
uc = mycrystal.get_unit_cell().parameters()
target_uc = [random.gauss(e, sigma) for e in uc]
rp.add_restraints_to_target_xl_unit_cell(experiment_id=0,
values=target_uc,
sigma=[sigma] * 6)
# get analytical values and gradients
vals, grads, weights = rp.get_residuals_gradients_and_weights()
assert len(vals) == rp.num_residuals
# get finite difference gradients
p_vals = pred_param.get_param_vals()
deltas = [1.0e-7] * len(p_vals)
fd_grad = []
for i, delta in enumerate(deltas):
val = p_vals[i]
p_vals[i] -= delta / 2.0
pred_param.set_param_vals(p_vals)
rev_state, foo, bar = rp.get_residuals_gradients_and_weights()
rev_state = flex.double(rev_state)
p_vals[i] += delta
pred_param.set_param_vals(p_vals)
fwd_state, foo, bar = rp.get_residuals_gradients_and_weights()
fwd_state = flex.double(fwd_state)
p_vals[i] = val
fd = (fwd_state - rev_state) / delta
fd_grad.append(fd)
# for comparison, fd_grad is a list of flex.doubles, each of which corresponds
# to a column of the sparse matrix grads.
for i, fd in enumerate(fd_grad):
# extract dense column from the sparse matrix
an = grads.col(i).as_dense_vector()
assert an == pytest.approx(fd, abs=1e-5)
class ModelState(object):
"""
A class to keep track of the model state
"""
def __init__(
self,
experiment,
mosaicity_parameterisation,
wavelength_parameterisation=None,
fix_mosaic_spread=False,
fix_wavelength_spread=True,
fix_unit_cell=False,
fix_orientation=False,
):
"""
Initialise the model state
"""
# Save the crystal model
self.experiment = experiment
self.crystal = experiment.crystal
# The U and P parameterisation
self._U_parameterisation = CrystalOrientationParameterisation(
self.crystal)
self._B_parameterisation = CrystalUnitCellParameterisation(
self.crystal)
# The M and L parameterisations
self._M_parameterisation = mosaicity_parameterisation
self._L_parameterisation = wavelength_parameterisation
# Set the flags to fix parameters
self._is_mosaic_spread_fixed = fix_mosaic_spread
self._is_wavelength_spread_fixed = fix_wavelength_spread
self._is_unit_cell_fixed = fix_unit_cell
self._is_orientation_fixed = fix_orientation
# Check wavelength parameterisation
if not self.is_wavelength_spread_fixed:
assert self._L_parameterisation is not None
@property
def is_orientation_fixed(self) -> bool:
return self._is_orientation_fixed
@property
def is_unit_cell_fixed(self) -> bool:
return self._is_unit_cell_fixed
@property
def is_mosaic_spread_fixed(self) -> bool:
return self._is_mosaic_spread_fixed
@property
def is_mosaic_spread_angular(self) -> bool:
return self._M_parameterisation.is_angular()
@property
def is_wavelength_spread_fixed(self) -> bool:
return self._is_wavelength_spread_fixed
@property
def unit_cell(self):
return self.crystal.get_unit_cell()
@property
def U_matrix(self) -> np.array:
return np.array([self.crystal.get_U()], dtype=np.float64).reshape(3, 3)
@property
def B_matrix(self) -> np.array:
return np.array([self.crystal.get_B()], dtype=np.float64).reshape(3, 3)
@property
def A_matrix(self) -> np.array:
return np.array([self.crystal.get_A()], dtype=np.float64).reshape(3, 3)
@property
def mosaicity_covariance_matrix(self) -> np.array:
return self._M_parameterisation.sigma()
@property
def wavelength_spread(self) -> flex.double:
if self._L_parameterisation is not None:
return self._L_parameterisation.sigma()
return flex.double()
@property
def U_params(self) -> np.array:
"""Get the parameters of the orientation parameterisation"""
return np.array(self._U_parameterisation.get_param_vals(),
dtype=np.float64)
@U_params.setter
def U_params(self, params) -> None:
self._U_parameterisation.set_param_vals(tuple(
float(i) for i in params))
@property
def B_params(self) -> np.array:
"""Get the parameters of the orientation parameterisation"""
return np.array(self._B_parameterisation.get_param_vals(),
dtype=np.float64)
@B_params.setter
def B_params(self, params) -> None:
self._B_parameterisation.set_param_vals(tuple(
float(i) for i in params))
@property
def M_params(self) -> np.array:
"Parameters of the mosaicity parameterisation"
return self._M_parameterisation.parameters
@M_params.setter
def M_params(self, params) -> None:
self._M_parameterisation.parameters = params
@property
def L_params(self) -> np.array:
"Parameters of the Lambda (wavelength) parameterisation"
if self._L_parameterisation is not None:
return np.array(self._L_parameterisation.parameters,
dtype=np.float64)
return np.array([])
@L_params.setter
def L_params(self, params: flex.double) -> None:
if self._L_parameterisation is not None:
self._L_parameterisation.parameters = params
@property
def dU_dp(self) -> np.array:
"""
Get the first derivatives of U w.r.t its parameters
"""
ds_dp = self._U_parameterisation.get_ds_dp()
n = len(ds_dp)
s = np.array([ds_dp[i] for i in range(n)],
dtype=np.float64).reshape(n, 3, 3)
return s
@property
def dB_dp(self) -> np.array:
"""
Get the first derivatives of B w.r.t its parameters
"""
ds_dp = self._B_parameterisation.get_ds_dp()
n = len(ds_dp)
s = np.array([ds_dp[i] for i in range(n)],
dtype=np.float64).reshape(n, 3, 3)
return s
@property
def dM_dp(self) -> np.array:
"""
Get the first derivatives of M w.r.t its parameters
"""
return self._M_parameterisation.first_derivatives()
@property
def dL_dp(self) -> flex.double:
"""
Get the first derivatives of L w.r.t its parameters
"""
if self._L_parameterisation is not None:
return self._L_parameterisation.first_derivatives()
return flex.double()
@property
def active_parameters(self) -> np.array:
"""
The active parameters in order: U, B, M, L, W
"""
active_params = []
if not self.is_orientation_fixed:
active_params.append(self.U_params)
if not self.is_unit_cell_fixed:
active_params.append(self.B_params)
if not self.is_mosaic_spread_fixed:
active_params.append(self.M_params)
if not self.is_wavelength_spread_fixed:
active_params.append(self.L_params)
active_params = np.concatenate(active_params)
assert len(active_params) > 0
return active_params
@active_parameters.setter
def active_parameters(self, params) -> None:
"""
Set the active parameters in order: U, B, M, L, W
"""
if not self.is_orientation_fixed:
n_U_params = len(self.U_params)
temp = params[:n_U_params]
params = params[n_U_params:]
self.U_params = temp
if not self.is_unit_cell_fixed:
n_B_params = len(self.B_params)
temp = params[:n_B_params]
params = params[n_B_params:]
self.B_params = temp
if not self.is_mosaic_spread_fixed:
n_M_params = self.M_params.size
temp = params[:n_M_params]
params = params[n_M_params:]
self.M_params = np.array(temp)
if not self.is_wavelength_spread_fixed:
n_L_params = self.L_params.size
temp = params[:n_L_params]
self.L_params = temp
@property
def parameter_labels(self) -> List[str]:
"""
Get the parameter labels
"""
labels = []
if not self.is_orientation_fixed:
labels += [f"Crystal_U_{i}" for i in range(self.U_params.size)]
if not self.is_unit_cell_fixed:
labels += [f"Crystal_B_{i}" for i in range(self.B_params.size)]
if not self.is_mosaic_spread_fixed:
labels += [f"Mosaicity_{i}" for i in range(self.M_params.size)]
if not self.is_wavelength_spread_fixed:
labels.append("Wavelength_Spread")
assert len(labels) > 0
return labels
def test():
import random
import textwrap
from cctbx.uctbx import unit_cell
from libtbx.test_utils import approx_equal
def random_direction_close_to(vector):
return vector.rotate_around_origin(
matrix.col((random.random(), random.random(),
random.random())).normalize(),
random.gauss(0, 1.0),
deg=True,
)
# make a random P1 crystal and parameterise it
a = random.uniform(10, 50) * random_direction_close_to(
matrix.col((1, 0, 0)))
b = random.uniform(10, 50) * random_direction_close_to(
matrix.col((0, 1, 0)))
c = random.uniform(10, 50) * random_direction_close_to(
matrix.col((0, 0, 1)))
xl = Crystal(a, b, c, space_group_symbol="P 1")
xl_op = CrystalOrientationParameterisation(xl)
xl_ucp = CrystalUnitCellParameterisation(xl)
null_mat = matrix.sqr((0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0))
# compare analytical and finite difference derivatives
an_ds_dp = xl_op.get_ds_dp()
fd_ds_dp = get_fd_gradients(xl_op, [1.0e-6 * pi / 180] * 3)
for e, f in zip(an_ds_dp, fd_ds_dp):
assert approx_equal((e - f), null_mat, eps=1.0e-6)
an_ds_dp = xl_ucp.get_ds_dp()
fd_ds_dp = get_fd_gradients(xl_ucp, [1.0e-7] * xl_ucp.num_free())
for e, f in zip(an_ds_dp, fd_ds_dp):
assert approx_equal((e - f), null_mat, eps=1.0e-6)
# random initial orientations with a random parameter shift at each
attempts = 100
for i in range(attempts):
# make a random P1 crystal and parameterise it
a = random.uniform(10, 50) * random_direction_close_to(
matrix.col((1, 0, 0)))
b = random.uniform(10, 50) * random_direction_close_to(
matrix.col((0, 1, 0)))
c = random.uniform(10, 50) * random_direction_close_to(
matrix.col((0, 0, 1)))
xl = Crystal(a, b, c, space_group_symbol="P 1")
xl_op = CrystalOrientationParameterisation(xl)
xl_uc = CrystalUnitCellParameterisation(xl)
# apply a random parameter shift to the orientation
p_vals = xl_op.get_param_vals()
p_vals = random_param_shift(
p_vals, [1000 * pi / 9, 1000 * pi / 9, 1000 * pi / 9])
xl_op.set_param_vals(p_vals)
# compare analytical and finite difference derivatives
xl_op_an_ds_dp = xl_op.get_ds_dp()
xl_op_fd_ds_dp = get_fd_gradients(xl_op, [1.0e-5 * pi / 180] * 3)
# apply a random parameter shift to the unit cell. We have to
# do this in a way that is respectful to metrical constraints,
# so don't modify the parameters directly; modify the cell
# constants and extract the new parameters
cell_params = xl.get_unit_cell().parameters()
cell_params = random_param_shift(cell_params, [1.0] * 6)
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(xl.get_space_group())
S.set_orientation(orientation=newB)
X = S.forward_independent_parameters()
xl_uc.set_param_vals(X)
xl_uc_an_ds_dp = xl_ucp.get_ds_dp()
# now doing finite differences about each parameter in turn
xl_uc_fd_ds_dp = get_fd_gradients(xl_ucp, [1.0e-7] * xl_ucp.num_free())
for j in range(3):
assert approx_equal((xl_op_fd_ds_dp[j] - xl_op_an_ds_dp[j]),
null_mat,
eps=1.0e-6), textwrap.dedent("""\
Failure in try {i}
failure for parameter number {j}
of the orientation parameterisation
with fd_ds_dp =
{fd}
and an_ds_dp =
{an}
so that difference fd_ds_dp - an_ds_dp =
{diff}
""").format(
i=i,
j=j,
fd=xl_op_fd_ds_dp[j],
an=xl_op_an_ds_dp[j],
diff=xl_op_fd_ds_dp[j] - xl_op_an_ds_dp[j],
)
for j in range(xl_ucp.num_free()):
assert approx_equal((xl_uc_fd_ds_dp[j] - xl_uc_an_ds_dp[j]),
null_mat,
eps=1.0e-6), textwrap.dedent("""\
Failure in try {i}
failure for parameter number {j}
of the unit cell parameterisation
with fd_ds_dp =
{fd}
and an_ds_dp =
{an}
so that difference fd_ds_dp - an_ds_dp =
{diff}
""").format(
i=i,
j=j,
fd=xl_uc_fd_ds_dp[j],
an=xl_uc_an_ds_dp[j],
diff=xl_uc_fd_ds_dp[j] - xl_uc_an_ds_dp[j],
)
'davec_dp': fd_vec[0],
'dbvec_dp': fd_vec[1],
'dcvec_dp': fd_vec[2],
'dB_dp': fd_B,
'dO_dp': fd_O
})
p_vals[i] = val
# return to the initial state
mp.set_param_vals(p_vals)
return fd_grad
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
from dials.algorithms.refinement.restraints.restraints import SingleUnitCellTie
uct = SingleUnitCellTie(xluc_param, [None] * 6, [None] * 6)
from scitbx.math import angle_derivative_wrt_vectors
B = matrix.sqr(crystal.get_B())
O = (B.transpose()).inverse()
a, b, c, aa, bb, cc = crystal.get_unit_cell().parameters()
aa *= DEG2RAD
bb *= DEG2RAD
cc *= DEG2RAD
Ut = matrix.sqr(crystal.get_U()).transpose()
avec, bvec, cvec = [Ut * vec for vec in crystal.get_real_space_vectors()]
assert approx_equal(im_width, 1.5 * pi / 180.)
# Build experiment lists
stills_experiments = ExperimentList()
stills_experiments.append(Experiment(
beam=mybeam, detector=mydetector, crystal=crystal, imageset=None))
scans_experiments = ExperimentList()
scans_experiments.append(Experiment(
beam=mybeam, detector=mydetector, crystal=crystal, goniometer = mygonio,
scan=myscan, imageset=None))
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
################################
# Apply known parameter shifts #
################################
# rotate crystal (=5 mrad each rotation)
xlo_p_vals = []
p_vals = xlo_param.get_param_vals()
xlo_p_vals.append(p_vals)
new_p_vals = [a + b for a, b in zip(p_vals, [5., 5., 5.])]
xlo_param.set_param_vals(new_p_vals)
# change unit cell (=1.0 Angstrom length upsets, 0.5 degree of
# gamma angle)
