def ref_gen_static(experiments):
"""Generate some reflections using the static predictor"""
beam = experiments[0].beam
crystal = experiments[0].crystal
goniometer = experiments[0].goniometer
detector = experiments[0].detector
scan = experiments[0].scan
# All indices to the detector max resolution
dmin = detector.get_max_resolution(beam.get_s0())
index_generator = IndexGenerator(crystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), dmin)
indices = index_generator.to_array()
# Predict rays within the sweep range
sweep_range = scan.get_oscillation_range(deg=False)
ray_predictor = ScansRayPredictor(experiments, sweep_range)
refs = ray_predictor.predict(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(detector, refs)
refs = refs.select(intersects)
# Make a reflection predictor and re-predict for these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ExperimentsPredictor(experiments)
refs['id'] = flex.int(len(refs), 0)
refs = ref_predictor.predict(refs)
return refs
def ref_gen_static(experiments):
"""Generate some reflections using the static predictor"""
beam = experiments[0].beam
crystal = experiments[0].crystal
goniometer = experiments[0].goniometer
detector = experiments[0].detector
scan = experiments[0].scan
# All indices to the detector max resolution
dmin = detector.get_max_resolution(beam.get_s0())
index_generator = IndexGenerator(
crystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), dmin)
indices = index_generator.to_array()
# Predict rays within the sweep range
sweep_range = scan.get_oscillation_range(deg=False)
ray_predictor = ScansRayPredictor(experiments, sweep_range)
refs = ray_predictor(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(detector, refs)
refs = refs.select(intersects)
# Make a reflection predictor and re-predict for these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ExperimentsPredictor(experiments)
refs['id'] = flex.int(len(refs), 0)
refs = ref_predictor(refs)
return refs
def generate_reflections(experiments):
from dials.algorithms.spot_prediction import IndexGenerator
from dials.algorithms.refinement.prediction import \
ScansRayPredictor, ExperimentsPredictor
from dials.algorithms.spot_prediction import ray_intersection
from cctbx.sgtbx import space_group, space_group_symbols
from scitbx.array_family import flex
detector = experiments[0].detector
crystal = experiments[0].crystal
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(crystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), resolution)
indices = index_generator.to_array()
# Predict rays within the sweep range
scan = experiments[0].scan
sweep_range = scan.get_oscillation_range(deg=False)
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs = ray_predictor.predict(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(detector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ExperimentsPredictor(experiments)
obs_refs['id'] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor.predict(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm']
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.
px_size = detector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.)**2)
obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
return obs_refs, ref_predictor
def run_spherical_relp_stills_pred_param(self, verbose=True):
if verbose:
print 'Testing derivatives for SphericalRelpStillsPredictionParameterisation'
print '====================================================================='
# Build a prediction parameterisation for the stills experiment
pred_param = SphericalRelpStillsPredictionParameterisation(
self.stills_experiments,
detector_parameterisations = [self.det_param],
beam_parameterisations = [self.s0_param],
xl_orientation_parameterisations = [self.xlo_param],
xl_unit_cell_parameterisations = [self.xluc_param])
# Predict the reflections in place. Must do this ahead of calculating
# the analytical gradients so quantities like s1 are correct
from dials.algorithms.refinement.prediction import ExperimentsPredictor
ref_predictor = ExperimentsPredictor(self.stills_experiments,
spherical_relp=True)
ref_predictor.update()
ref_predictor.predict(self.reflections)
# get analytical gradients
an_grads = pred_param.get_gradients(self.reflections)
fd_grads = self.get_fd_gradients(pred_param, ref_predictor)
# compare FD with analytical calculations
for i, (an_grad, fd_grad) in enumerate(zip(an_grads, fd_grads)):
if verbose: print "\nParameter {0}: {1}". format(i, fd_grad['name'])
for idx, name in enumerate(["dX_dp", "dY_dp", "dDeltaPsi_dp"]):
if verbose: print name
for a, b in zip(an_grad[name], fd_grad[name]):
if name == 'dDeltaPsi_dp':
# DeltaPsi errors are much worse than X, Y errors!
# FIXME, look into this further
assert approx_equal(a,b, eps=5e-3)
else:
assert approx_equal(a,b, eps=5e-6)
if verbose: print "OK"
if verbose: print
def create_models(self, cmdline_overrides=None):
from dials.test.algorithms.refinement.setup_geometry import Extract
from dxtbx.model import ScanFactory
from libtbx.phil import parse
if cmdline_overrides is None:
cmdline_overrides = []
overrides = """geometry.parameters.crystal.a.length.range = 10 50
geometry.parameters.crystal.b.length.range = 10 50
geometry.parameters.crystal.c.length.range = 10 50"""
master_phil = parse("""
include scope dials.test.algorithms.refinement.geometry_phil
""",
process_includes=True)
# Extract models
models = Extract(master_phil,
overrides,
cmdline_args=cmdline_overrides)
self.detector = models.detector
self.goniometer = models.goniometer
self.crystal = models.crystal
self.beam = models.beam
# Make a scan of 1-360 * 0.5 deg images
sf = ScanFactory()
self.scan = sf.make_scan((1, 360), 0.5, (0, 0.5), range(360))
# Generate an ExperimentList
self.experiments = ExperimentList()
self.experiments.append(
Experiment(beam=self.beam,
detector=self.detector,
goniometer=self.goniometer,
scan=self.scan,
crystal=self.crystal,
imageset=None))
# Create a reflection predictor for the experiments
self.ref_predictor = ExperimentsPredictor(self.experiments)
# Create scan-varying parameterisations of these models, with 5 samples
self.det_param = ScanVaryingDetectorParameterisationSinglePanel(
self.detector, self.scan.get_array_range(), 5)
self.s0_param = ScanVaryingBeamParameterisation(
self.beam, self.scan.get_array_range(), 5, self.goniometer)
self.xlo_param = ScanVaryingCrystalOrientationParameterisation(
self.crystal, self.scan.get_array_range(), 5)
self.xluc_param = ScanVaryingCrystalUnitCellParameterisation(
self.crystal, self.scan.get_array_range(), 5)
self.gon_param = ScanVaryingGoniometerParameterisation(
self.goniometer, self.scan.get_array_range(), 5, self.beam)
def generate_reflections(experiments):
from dials.algorithms.spot_prediction import IndexGenerator
from dials.algorithms.refinement.prediction import \
ScansRayPredictor, ExperimentsPredictor
from dials.algorithms.spot_prediction import ray_intersection
from cctbx.sgtbx import space_group, space_group_symbols
from scitbx.array_family import flex
detector = experiments[0].detector
crystal = experiments[0].crystal
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(
crystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), resolution)
indices = index_generator.to_array()
# Predict rays within the sweep range
scan = experiments[0].scan
sweep_range = scan.get_oscillation_range(deg=False)
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs = ray_predictor(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(detector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ExperimentsPredictor(experiments)
obs_refs['id'] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm']
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.
px_size = detector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.)**2)
obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
return obs_refs, ref_predictor
def test_spherical_relp_stills_pred_param(tc):
print(
'Testing derivatives for SphericalRelpStillsPredictionParameterisation'
)
print(
'====================================================================='
)
# Build a prediction parameterisation for the stills experiment
pred_param = SphericalRelpStillsPredictionParameterisation(
tc.stills_experiments,
detector_parameterisations=[tc.det_param],
beam_parameterisations=[tc.s0_param],
xl_orientation_parameterisations=[tc.xlo_param],
xl_unit_cell_parameterisations=[tc.xluc_param])
# Predict the reflections in place. Must do this ahead of calculating
# the analytical gradients so quantities like s1 are correct
from dials.algorithms.refinement.prediction import ExperimentsPredictor
ref_predictor = ExperimentsPredictor(tc.stills_experiments,
spherical_relp=True)
ref_predictor(tc.reflections)
# get analytical gradients
an_grads = pred_param.get_gradients(tc.reflections)
fd_grads = tc.get_fd_gradients(pred_param, ref_predictor)
# compare FD with analytical calculations
for i, (an_grad, fd_grad) in enumerate(zip(an_grads, fd_grads)):
print("\nParameter {0}: {1}".format(i, fd_grad['name']))
for name in ["dX_dp", "dY_dp", "dDeltaPsi_dp"]:
print(name)
for a, b in zip(an_grad[name], fd_grad[name]):
if name == 'dDeltaPsi_dp':
# DeltaPsi errors are much worse than X, Y errors!
# FIXME, look into this further
assert a == pytest.approx(b, abs=5e-3)
else:
assert a == pytest.approx(b, abs=5e-6)
print("OK")
def run_spherical_relp_stills_pred_param(self, verbose=True):
if verbose:
print 'Testing derivatives for SphericalRelpStillsPredictionParameterisation'
print '====================================================================='
# Build a prediction parameterisation for the stills experiment
pred_param = SphericalRelpStillsPredictionParameterisation(
self.stills_experiments,
detector_parameterisations=[self.det_param],
beam_parameterisations=[self.s0_param],
xl_orientation_parameterisations=[self.xlo_param],
xl_unit_cell_parameterisations=[self.xluc_param])
# Predict the reflections in place. Must do this ahead of calculating
# the analytical gradients so quantities like s1 are correct
from dials.algorithms.refinement.prediction import ExperimentsPredictor
ref_predictor = ExperimentsPredictor(self.stills_experiments,
spherical_relp=True)
ref_predictor(self.reflections)
# get analytical gradients
an_grads = pred_param.get_gradients(self.reflections)
fd_grads = self.get_fd_gradients(pred_param, ref_predictor)
# compare FD with analytical calculations
for i, (an_grad, fd_grad) in enumerate(zip(an_grads, fd_grads)):
if verbose: print "\nParameter {0}: {1}".format(i, fd_grad['name'])
for idx, name in enumerate(["dX_dp", "dY_dp", "dDeltaPsi_dp"]):
if verbose: print name
for a, b in zip(an_grad[name], fd_grad[name]):
if name == 'dDeltaPsi_dp':
# DeltaPsi errors are much worse than X, Y errors!
# FIXME, look into this further
assert approx_equal(a, b, eps=5e-3)
else:
assert approx_equal(a, b, eps=5e-6)
if verbose: print "OK"
if verbose: print
def create_indexed(self, experiments, filename):
print("Simulating indexed observations for {0}".format(filename))
# Extract the experiment
exp = experiments[0]
# print "Experiment scan range is {0},{1}".format(*exp.scan.get_oscillation_range(deg=True))
# Copy essential columns of the original reflections into a new table
# (imageset_id is required for reciprocal_lattice_viewer)
obs_refs = flex.reflection_table()
cols = [
"id",
"imageset_id",
"miller_index",
"panel",
"s1",
"flags",
"entering",
"xyzobs.mm.value",
"xyzobs.mm.variance",
"xyzcal.mm",
"xyzobs.px.value",
"xyzobs.px.variance",
"xyzcal.px",
]
for k in cols:
if k in self.original_reflections.keys():
obs_refs[k] = self.original_reflections[k]
x_obs, y_obs, phi_obs = obs_refs["xyzobs.mm.value"].parts()
# print "Original observation scan range is {0},{1}".format(
# flex.min(phi_obs) * RAD_TO_DEG,
# flex.max(phi_obs) * RAD_TO_DEG)
# Reset panel number to zero prior to prediction
obs_refs["panel"] = obs_refs["panel"] * 0
# Predict new centroid positions
from dials.algorithms.refinement.prediction import ExperimentsPredictor
predictor = ExperimentsPredictor(experiments)
predictor(obs_refs)
# Get vectors to add as errors to the predicted centroid positions to form
# new 'observations'
shift_px = flex.vec3_double(self.shiftX_px, self.shiftY_px,
self.shiftZ_px)
px_size_mm = exp.detector[0].get_pixel_size()
image_width_rad = exp.scan.get_oscillation(deg=False)[1]
shift = flex.vec3_double(
px_size_mm[0] * self.shiftX_px,
px_size_mm[1] * self.shiftY_px,
image_width_rad * self.shiftZ_px,
)
obs_refs["xyzobs.px.value"] = obs_refs["xyzcal.px"] + shift_px
obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"] + shift
x, y, z = obs_refs["xyzobs.mm.value"].parts()
# print "Simulated observation scan range is {0},{1}".format(
# flex.min(z) * RAD_TO_DEG,
# flex.max(z) * RAD_TO_DEG)
# Keep original variance estimates from spot-finding for centroids in
# pixels/images, but optionally rescale for centroids in mm/rad.
# Note that an overall scale factor for the variance is irrelevant to
# refinement. What matters are differences in the relative scale between
# the detector space and rotation parts.
if self.params.recalculate_centroid_variances:
var_x_px, var_y_px, var_z_px = obs_refs[
"xyzobs.px.variance"].parts()
var_x_mm = var_x_px * px_size_mm[0]**2
var_y_mm = var_y_px * px_size_mm[1]**2
var_z_rd = var_z_px * image_width_rad**2
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(
var_x_mm, var_y_mm, var_z_rd)
# Reset observed s1 vectors
from dials.algorithms.refinement.refinement_helpers import set_obs_s1
obs_refs["s1"] = flex.vec3_double(len(obs_refs))
set_obs_s1(obs_refs, experiments)
return obs_refs
def test(args=[]):
# Python and cctbx imports
from math import pi
from scitbx import matrix
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
# Import for surgery on reflection_tables
from dials.array_family import flex
# Get module to build models using PHIL
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Crystal parameterisations
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalOrientationParameterisation, CrystalUnitCellParameterisation
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import \
symmetrize_reduce_enlarge
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator
from dials.algorithms.refinement.prediction import ScansRayPredictor, \
ExperimentsPredictor
from dials.algorithms.spot_prediction import ray_intersection
from cctbx.sgtbx import space_group, space_group_symbols
#############################
# Setup experimental models #
#############################
master_phil = parse("""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True)
# build models, with a larger crystal than default in order to get enough
# reflections on the 'still' image
param = """
geometry.parameters.crystal.a.length.range=40 50;
geometry.parameters.crystal.b.length.range=40 50;
geometry.parameters.crystal.c.length.range=40 50;
geometry.parameters.random_seed = 42"""
models = setup_geometry.Extract(master_phil,
cmdline_args=args,
local_overrides=param)
crystal = models.crystal
mydetector = models.detector
mygonio = models.goniometer
mybeam = models.beam
# Build a mock scan for a 1.5 degree wedge. Only used for generating indices near
# the Ewald sphere
sf = ScanFactory()
myscan = sf.make_scan(image_range=(1, 1),
exposure_times=0.1,
oscillation=(0, 1.5),
epochs=range(1),
deg=True)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert approx_equal(im_width, 1.5 * pi / 180.)
# Build experiment lists
stills_experiments = ExperimentList()
stills_experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
crystal=crystal,
imageset=None))
scans_experiments = ExperimentList()
scans_experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
crystal=crystal,
goniometer=mygonio,
scan=myscan,
imageset=None))
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
################################
# Apply known parameter shifts #
################################
# rotate crystal (=5 mrad each rotation)
xlo_p_vals = []
p_vals = xlo_param.get_param_vals()
xlo_p_vals.append(p_vals)
new_p_vals = [a + b for a, b in zip(p_vals, [5., 5., 5.])]
xlo_param.set_param_vals(new_p_vals)
# change unit cell (=1.0 Angstrom length upsets, 0.5 degree of
# gamma angle)
xluc_p_vals = []
p_vals = xluc_param.get_param_vals()
xluc_p_vals.append(p_vals)
cell_params = crystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [1.0, 1.0, -1.0, 0.0, 0.0, 0.5])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(crystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
# keep track of the target crystal model to compare with refined
from copy import deepcopy
target_crystal = deepcopy(crystal)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere for crystal
resolution = 2.0
index_generator = IndexGenerator(
crystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), resolution)
indices = index_generator.to_array()
# Build a ray predictor and predict rays close to the Ewald sphere by using
# the narrow rotation scan
ref_predictor = ScansRayPredictor(scans_experiments, sweep_range)
obs_refs = ref_predictor(indices, experiment_id=0)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Add in flags and ID columns by copying into standard reflection table
tmp = flex.reflection_table.empty_standard(len(obs_refs))
tmp.update(obs_refs)
obs_refs = tmp
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.)**2)
obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
# Re-predict using the stills reflection predictor
stills_ref_predictor = ExperimentsPredictor(stills_experiments)
obs_refs_stills = stills_ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs_stills['xyzobs.mm.value'] = obs_refs_stills['xyzcal.mm']
###############################
# Undo known parameter shifts #
###############################
xlo_param.set_param_vals(xlo_p_vals[0])
xluc_param.set_param_vals(xluc_p_vals[0])
# make a refiner
from dials.algorithms.refinement.refiner import phil_scope
params = phil_scope.fetch(source=parse('')).extract()
# Change this to get a plot
do_plot = False
if do_plot:
params.refinement.refinery.journal.track_parameter_correlation = True
from dials.algorithms.refinement.refiner import RefinerFactory
# decrease bin_size_fraction to terminate on RMSD convergence
params.refinement.target.bin_size_fraction = 0.01
params.refinement.parameterisation.beam.fix = "all"
params.refinement.parameterisation.detector.fix = "all"
refiner = RefinerFactory.from_parameters_data_experiments(
params, obs_refs_stills, stills_experiments, verbosity=0)
# run refinement
history = refiner.run()
# regression tests
assert len(history["rmsd"]) == 9
refined_crystal = refiner.get_experiments()[0].crystal
uc1 = refined_crystal.get_unit_cell()
uc2 = target_crystal.get_unit_cell()
assert uc1.is_similar_to(uc2)
if do_plot:
plt = refiner.parameter_correlation_plot(
len(history["parameter_correlation"]) - 1)
plt.show()
def run_stills_pred_param(self, verbose = False):
if verbose:
print 'Testing derivatives for StillsPredictionParameterisation'
print '========================================================'
# Build a prediction parameterisation for the stills experiment
pred_param = StillsPredictionParameterisation(self.stills_experiments,
detector_parameterisations = [self.det_param],
beam_parameterisations = [self.s0_param],
xl_orientation_parameterisations = [self.xlo_param],
xl_unit_cell_parameterisations = [self.xluc_param])
# Predict the reflections in place. Must do this ahead of calculating
# the analytical gradients so quantities like s1 are correct
from dials.algorithms.refinement.prediction import ExperimentsPredictor
ref_predictor = ExperimentsPredictor(self.stills_experiments)
ref_predictor.update()
ref_predictor.predict(self.reflections)
# get analytical gradients
an_grads = pred_param.get_gradients(self.reflections)
fd_grads = self.get_fd_gradients(pred_param, ref_predictor)
for i, (an_grad, fd_grad) in enumerate(zip(an_grads, fd_grads)):
# compare FD with analytical calculations
if verbose: print "\nParameter {0}: {1}". format(i, fd_grad['name'])
for idx, name in enumerate(["dX_dp", "dY_dp", "dDeltaPsi_dp"]):
if verbose: print name
a = fd_grad[name]
b = an_grad[name]
abs_error = a - b
denom = a + b
fns = five_number_summary(abs_error)
if verbose: print (" summary of absolute errors: %9.6f %9.6f %9.6f " + \
"%9.6f %9.6f") % fns
assert flex.max(flex.abs(abs_error)) < 0.0003
# largest absolute error found to be about 0.00025 for dY/dp of
# Crystal0g_param_3. Reject outlying absolute errors and test again.
iqr = fns[3] - fns[1]
# skip further stats on errors with an iqr of near zero, e.g. dDeltaPsi_dp
# for detector parameters, which are all equal to zero
if iqr < 1.e-10:
continue
sel1 = abs_error < fns[3] + 1.5 * iqr
sel2 = abs_error > fns[1] - 1.5 * iqr
sel = sel1 & sel2
tst = flex.max_index(flex.abs(abs_error.select(sel)))
tst_val = abs_error.select(sel)[tst]
n_outliers = sel.count(False)
if verbose: print (" {0} outliers rejected, leaving greatest " + \
"absolute error: {1:9.6f}").format(n_outliers, tst_val)
# largest absolute error now 0.000086 for dX/dp of Beam0Mu2
assert abs(tst_val) < 0.00009
# Completely skip parameters with FD gradients all zero (e.g. gradients of
# DeltaPsi for detector parameters)
sel1 = flex.abs(a) < 1.e-10
if sel1.all_eq(True):
continue
# otherwise calculate normalised errors, by dividing absolute errors by
# the IQR (more stable than relative error calculation)
norm_error = abs_error / iqr
fns = five_number_summary(norm_error)
if verbose: print (" summary of normalised errors: %9.6f %9.6f %9.6f " + \
"%9.6f %9.6f") % fns
# largest normalised error found to be about 25.7 for dY/dp of
# Crystal0g_param_3.
try:
assert flex.max(flex.abs(norm_error)) < 30
except AssertionError as e:
e.args += ("extreme normalised error value: {0}".format(
flex.max(flex.abs(norm_error))),)
raise e
# Reject outlying normalised errors and test again
iqr = fns[3] - fns[1]
if iqr > 0.:
sel1 = norm_error < fns[3] + 1.5 * iqr
sel2 = norm_error > fns[1] - 1.5 * iqr
sel = sel1 & sel2
tst = flex.max_index(flex.abs(norm_error.select(sel)))
tst_val = norm_error.select(sel)[tst]
n_outliers = sel.count(False)
# most outliers found for for dY/dp of Crystal0g_param_3 (which had
# largest errors, so no surprise there).
try:
assert n_outliers < 250
except AssertionError as e:
e.args += ("too many outliers rejected: {0}".format(n_outliers),)
raise e
if verbose: print (" {0} outliers rejected, leaving greatest " + \
"normalised error: {1:9.6f}").format(n_outliers, tst_val)
# largest normalied error now about -4. for dX/dp of Detector0Tau1
assert abs(tst_val) < 4.5
if verbose: print
return
def run_stills_pred_param(self, verbose=False):
if verbose:
print 'Testing derivatives for StillsPredictionParameterisation'
print '========================================================'
# Build a prediction parameterisation for the stills experiment
pred_param = StillsPredictionParameterisation(
self.stills_experiments,
detector_parameterisations=[self.det_param],
beam_parameterisations=[self.s0_param],
xl_orientation_parameterisations=[self.xlo_param],
xl_unit_cell_parameterisations=[self.xluc_param])
# Predict the reflections in place. Must do this ahead of calculating
# the analytical gradients so quantities like s1 are correct
from dials.algorithms.refinement.prediction import ExperimentsPredictor
ref_predictor = ExperimentsPredictor(self.stills_experiments)
ref_predictor(self.reflections)
# get analytical gradients
an_grads = pred_param.get_gradients(self.reflections)
fd_grads = self.get_fd_gradients(pred_param, ref_predictor)
for i, (an_grad, fd_grad) in enumerate(zip(an_grads, fd_grads)):
# compare FD with analytical calculations
if verbose: print "\nParameter {0}: {1}".format(i, fd_grad['name'])
for idx, name in enumerate(["dX_dp", "dY_dp", "dDeltaPsi_dp"]):
if verbose: print name
a = fd_grad[name]
b = an_grad[name]
abs_error = a - b
denom = a + b
fns = five_number_summary(abs_error)
if verbose: print (" summary of absolute errors: %9.6f %9.6f %9.6f " + \
"%9.6f %9.6f") % fns
assert flex.max(flex.abs(abs_error)) < 0.0003
# largest absolute error found to be about 0.00025 for dY/dp of
# Crystal0g_param_3. Reject outlying absolute errors and test again.
iqr = fns[3] - fns[1]
# skip further stats on errors with an iqr of near zero, e.g. dDeltaPsi_dp
# for detector parameters, which are all equal to zero
if iqr < 1.e-10:
continue
sel1 = abs_error < fns[3] + 1.5 * iqr
sel2 = abs_error > fns[1] - 1.5 * iqr
sel = sel1 & sel2
tst = flex.max_index(flex.abs(abs_error.select(sel)))
tst_val = abs_error.select(sel)[tst]
n_outliers = sel.count(False)
if verbose: print (" {0} outliers rejected, leaving greatest " + \
"absolute error: {1:9.6f}").format(n_outliers, tst_val)
# largest absolute error now 0.000086 for dX/dp of Beam0Mu2
assert abs(tst_val) < 0.00009
# Completely skip parameters with FD gradients all zero (e.g. gradients of
# DeltaPsi for detector parameters)
sel1 = flex.abs(a) < 1.e-10
if sel1.all_eq(True):
continue
# otherwise calculate normalised errors, by dividing absolute errors by
# the IQR (more stable than relative error calculation)
norm_error = abs_error / iqr
fns = five_number_summary(norm_error)
if verbose: print (" summary of normalised errors: %9.6f %9.6f %9.6f " + \
"%9.6f %9.6f") % fns
# largest normalised error found to be about 25.7 for dY/dp of
# Crystal0g_param_3.
try:
assert flex.max(flex.abs(norm_error)) < 30
except AssertionError as e:
e.args += ("extreme normalised error value: {0}".format(
flex.max(flex.abs(norm_error))), )
raise e
# Reject outlying normalised errors and test again
iqr = fns[3] - fns[1]
if iqr > 0.:
sel1 = norm_error < fns[3] + 1.5 * iqr
sel2 = norm_error > fns[1] - 1.5 * iqr
sel = sel1 & sel2
tst = flex.max_index(flex.abs(norm_error.select(sel)))
tst_val = norm_error.select(sel)[tst]
n_outliers = sel.count(False)
# most outliers found for for dY/dp of Crystal0g_param_3 (which had
# largest errors, so no surprise there).
try:
assert n_outliers < 250
except AssertionError as e:
e.args += ("too many outliers rejected: {0}".format(
n_outliers), )
raise e
if verbose: print (" {0} outliers rejected, leaving greatest " + \
"normalised error: {1:9.6f}").format(n_outliers, tst_val)
# largest normalied error now about -4. for dX/dp of Detector0Tau1
assert abs(
tst_val) < 4.5, 'should be about 4 not %s' % tst_val
if verbose: print
return
# Predict rays within the sweep range. Set experiment IDs ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs1 = ray_predictor.predict(indices1, experiment_id=0) obs_refs1['id'] = flex.int(len(obs_refs1), 0) obs_refs2 = ray_predictor.predict(indices1, experiment_id=1) obs_refs2['id'] = flex.int(len(obs_refs2), 1) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs1) obs_refs1 = obs_refs1.select(intersects) intersects = ray_intersection(mydetector, obs_refs2) obs_refs2 = obs_refs2.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs1 = ref_predictor.predict(obs_refs1) obs_refs2 = ref_predictor.predict(obs_refs2) # Set 'observed' centroids from the predicted ones obs_refs1['xyzobs.mm.value'] = obs_refs1['xyzcal.mm'] obs_refs2['xyzobs.mm.value'] = obs_refs2['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs1), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs1), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs1), (im_width / 2.)**2) obs_refs1['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) var_x = flex.double(len(obs_refs2), (px_size[0] / 2.)**2)
def choose_best_orientation_matrix(self, candidate_orientation_matrices):
from dxtbx.model.experiment_list import Experiment, ExperimentList
import copy
logger.info('*' * 80)
logger.info('Selecting the best orientation matrix')
logger.info('*' * 80)
from libtbx import group_args
class candidate_info(group_args):
pass
candidates = []
params = copy.deepcopy(self.all_params)
n_cand = len(candidate_orientation_matrices)
for icm,cm in enumerate(candidate_orientation_matrices):
sel = ((self.reflections['id'] == -1))
#(1/self.reflections['rlp'].norms() > self.d_min))
refl = self.reflections.select(sel)
experiments = self.experiment_list_for_crystal(cm)
self.index_reflections(experiments, refl)
indexed = refl.select(refl['id'] >= 0)
indexed = indexed.select(indexed.get_flags(indexed.flags.indexed))
if params.indexing.stills.refine_all_candidates:
try:
print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d initial outlier identification"%(icm, n_cand)
acceptance_flags = self.identify_outliers(params, experiments, indexed)
#create a new "indexed" list with outliers thrown out:
indexed = indexed.select(acceptance_flags)
print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d refinement before outlier rejection"%(icm, n_cand)
R = e_refine(params = params, experiments=experiments, reflections=indexed, graph_verbose=False)
ref_experiments = R.get_experiments()
# try to improve the outcome with a second round of outlier rejection post-initial refinement:
acceptance_flags = self.identify_outliers(params, ref_experiments, indexed)
# insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection
nv0 = nave_parameters(params = params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False)
crystal_model_nv0 = nv0()
acceptance_flags_nv0 = nv0.nv_acceptance_flags
indexed = indexed.select(acceptance_flags & acceptance_flags_nv0)
print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d after positional and delta-psi outlier rejection"%(icm, n_cand)
R = e_refine(params = params, experiments=ref_experiments, reflections=indexed, graph_verbose=False)
ref_experiments = R.get_experiments()
nv = nave_parameters(params = params, experiments=ref_experiments, reflections=indexed, refinery=R, graph_verbose=False)
crystal_model = nv()
# Drop candidates that after refinement can no longer be converted to the known target space group
if self.params.known_symmetry.space_group is not None:
target_space_group = self.target_symmetry_primitive.space_group()
new_crystal, cb_op_to_primitive = self.apply_symmetry(crystal_model, target_space_group)
if new_crystal is None:
print "P1 refinement yielded model diverged from target, candidate %d/%d"%(icm, n_cand)
continue
rmsd, _ = calc_2D_rmsd_and_displacements(R.predict_for_reflection_table(indexed))
except Exception, e:
print "Couldn't refine candiate %d/%d, %s"%(icm, n_cand, str(e))
else:
print "$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d done"%(icm, n_cand)
candidates.append(candidate_info(crystal = crystal_model,
green_curve_area = nv.green_curve_area,
ewald_proximal_volume = nv.ewald_proximal_volume(),
n_indexed = len(indexed),
rmsd = rmsd,
indexed = indexed,
experiments = ref_experiments))
else:
from dials.algorithms.refinement.prediction import ExperimentsPredictor
ref_predictor = ExperimentsPredictor(experiments, force_stills=True,
spherical_relp=params.refinement.parameterisation.spherical_relp_model)
rmsd, _ = calc_2D_rmsd_and_displacements(ref_predictor(indexed))
candidates.append(candidate_info(crystal = cm,
n_indexed = len(indexed),
rmsd = rmsd,
indexed = indexed,
experiments = experiments))
def test2():
"""Test on simulated data"""
# Get models for reflection prediction
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
from libtbx.phil import parse
overrides = """geometry.parameters.crystal.a.length.value = 77
geometry.parameters.crystal.b.length.value = 77
geometry.parameters.crystal.c.length.value = 37"""
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
""",
process_includes=True,
)
from dxtbx.model import Crystal
models = setup_geometry.Extract(master_phil)
crystal = Crystal(
real_space_a=(2.62783398111729, -63.387215823567125,
-45.751375737456975),
real_space_b=(15.246640559660356, -44.48254330406616,
62.50501032727026),
real_space_c=(-76.67246874451074, -11.01804131886244,
10.861322446352226),
space_group_symbol="I 2 3",
)
detector = models.detector
goniometer = models.goniometer
beam = models.beam
# Build a mock scan for a 180 degree sweep
from dxtbx.model import ScanFactory
sf = ScanFactory()
scan = sf.make_scan(
image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=range(1800),
deg=True,
)
# Build an experiment list
from dxtbx.model.experiment_list import ExperimentList, Experiment
experiments = ExperimentList()
experiments.append(
Experiment(
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
crystal=crystal,
imageset=None,
))
# Generate all indices in a 1.5 Angstrom sphere
from dials.algorithms.spot_prediction import IndexGenerator
from cctbx.sgtbx import space_group, space_group_symbols
resolution = 1.5
index_generator = IndexGenerator(
crystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# Predict rays within the sweep range
from dials.algorithms.refinement.prediction import ScansRayPredictor
sweep_range = scan.get_oscillation_range(deg=False)
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs = ray_predictor(indices)
# Take only those rays that intersect the detector
from dials.algorithms.spot_prediction import ray_intersection
intersects = ray_intersection(detector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
from dials.algorithms.refinement.prediction import ExperimentsPredictor
ref_predictor = ExperimentsPredictor(experiments)
obs_refs["id"] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor(obs_refs)
# Copy 'observed' centroids from the predicted ones, applying sinusoidal
# offsets
obs_x, obs_y, obs_z = obs_refs["xyzcal.mm"].parts()
# obs_z is in range (0, pi). Calculate offsets for phi at twice that
# frequency
im_width = scan.get_oscillation(deg=False)[1]
z_off = flex.sin(2 * obs_z) * im_width
obs_z += z_off
# Calculate offsets for x
pixel_size = detector[0].get_pixel_size()
x_off = flex.sin(20 * obs_z) * pixel_size[0]
# Calculate offsets for y with a phase-shifted sine wave
from math import pi
y_off = flex.sin(4 * obs_z + pi / 6) * pixel_size[1]
# Incorporate the offsets into the 'observed' centroids
obs_z += z_off
obs_x += x_off
obs_y += y_off
obs_refs["xyzobs.mm.value"] = flex.vec3_double(obs_x, obs_y, obs_z)
# Now do centroid analysis of the residuals
results = CentroidAnalyser(obs_refs, debug=True)()
# FIXME this test shows that the suggested interval width heuristic is not
# yet robust. This simulation function seems a useful direction to proceed
# in though
raise RuntimeError("test2 failed")
print("OK")
return
def test(args=[]):
# Python and cctbx imports
from math import pi
import random
from scitbx import matrix
from scitbx.array_family import flex
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
# Experimental model builder
from dials.test.algorithms.refinement.setup_geometry import Extract
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Model parameterisations
from dials.algorithms.refinement.parameterisation.detector_parameters import \
DetectorParameterisationSinglePanel
from dials.algorithms.refinement.parameterisation.beam_parameters import \
BeamParameterisation
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalOrientationParameterisation, CrystalUnitCellParameterisation
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection
from dials.algorithms.refinement.prediction import ScansRayPredictor, \
ExperimentsPredictor
from cctbx.sgtbx import space_group, space_group_symbols
# Parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import \
XYPhiPredictionParameterisation
# Imports for the target function
from dials.algorithms.refinement.target import \
LeastSquaresPositionalResidualWithRmsdCutoff
from dials.algorithms.refinement.reflection_manager import ReflectionManager
# Local functions
def random_direction_close_to(vector, sd = 0.5):
return vector.rotate_around_origin(matrix.col(
(random.random(),
random.random(),
random.random())).normalize(),
random.gauss(0, sd), deg = True)
#############################
# Setup experimental models #
#############################
# make a small cell to speed up calculations
overrides = """geometry.parameters.crystal.a.length.range = 10 15
geometry.parameters.crystal.b.length.range = 10 15
geometry.parameters.crystal.c.length.range = 10 15"""
master_phil = parse("""
include scope dials.test.algorithms.refinement.geometry_phil
""", process_includes=True)
models = Extract(master_phil, overrides, cmdline_args = args)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Build a mock scan for a 180 degree sweep of 0.1 degree images
sf = ScanFactory()
myscan = sf.make_scan(image_range = (1,1800),
exposure_times = 0.1,
oscillation = (0, 0.1),
epochs = range(1800),
deg = True)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sweep_range == (0., pi)
assert approx_equal(im_width, 0.1 * pi / 180.)
experiments = ExperimentList()
experiments.append(Experiment(
beam=mybeam, detector=mydetector, goniometer=mygonio,
scan=myscan, crystal=mycrystal, imageset=None))
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
pred_param = XYPhiPredictionParameterisation(experiments,
[det_param], [s0_param], [xlo_param], [xluc_param])
################################
# Apply known parameter shifts #
################################
# shift detector by 0.2 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [a + b for a, b in zip(det_p_vals,
[2.0, 2.0, 2.0, 2.0, 2.0, 2.0])]
det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in one axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[1] += 2.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [2.0, 2.0, 2.0])]
xlo_param.set_param_vals(p_vals)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), resolution)
indices = index_generator.to_array()
# Predict rays within the sweep range
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs = ray_predictor(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ExperimentsPredictor(experiments)
obs_refs['id'] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm']
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.)**2)
obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(obs_refs, experiments)
##############################
# Set up the target function #
##############################
# Redefine the reflection predictor to use the type expected by the Target class
ref_predictor = ExperimentsPredictor(experiments)
mytarget = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments, ref_predictor, refman, pred_param,
restraints_parameterisation=None)
# get the functional and gradients
mytarget.predict()
L, dL_dp, curvs = mytarget.compute_functional_gradients_and_curvatures()
####################################
# Do FD calculation for comparison #
####################################
# function for calculating finite difference gradients of the target function
def get_fd_gradients(target, pred_param, deltas):
"""Calculate centered finite difference gradients for each of the
parameters of the target function.
"deltas" must be a sequence of the same length as the parameter list, and
contains the step size for the difference calculations for each parameter.
"""
p_vals = pred_param.get_param_vals()
assert len(deltas) == len(p_vals)
fd_grad = []
fd_curvs = []
for i in range(len(deltas)):
val = p_vals[i]
p_vals[i] -= deltas[i] / 2.
pred_param.set_param_vals(p_vals)
target.predict()
rev_state = target.compute_functional_gradients_and_curvatures()
p_vals[i] += deltas[i]
pred_param.set_param_vals(p_vals)
target.predict()
fwd_state = target.compute_functional_gradients_and_curvatures()
# finite difference estimation of first derivatives
fd_grad.append((fwd_state[0] - rev_state[0]) / deltas[i])
# finite difference estimation of curvatures, using the analytical
# first derivatives
fd_curvs.append((fwd_state[1][i] - rev_state[1][i]) / deltas[i])
# set parameter back to centred value
p_vals[i] = val
# return to the initial state
pred_param.set_param_vals(p_vals)
return fd_grad, fd_curvs
# test normalised differences between FD and analytical calculations
fdgrads = get_fd_gradients(mytarget, pred_param, [1.e-7] * len(pred_param))
diffs = [a - b for a, b in zip(dL_dp, fdgrads[0])]
norm_diffs = tuple([a / b for a, b in zip(diffs, fdgrads[0])])
for e in norm_diffs: assert abs(e) < 0.001 # check differences less than 0.1%
# test normalised differences between FD curvatures and analytical least
# squares approximation. We don't expect this to be especially close
if curvs:
diffs = [a - b for a, b in zip(curvs, fdgrads[1])]
norm_diffs = tuple([a / b for a, b in zip(diffs, fdgrads[1])])
for e in norm_diffs: assert abs(e) < 0.1 # check differences less than 10%
sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) # Predict rays within the sweep range ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor.predict(indices) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs['id'] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor.predict(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) # The total number of observations should be 1128
def test():
from libtbx.phil import parse
from scitbx import matrix
from scitbx.array_family import flex
# Building experimental models
from dials.test.algorithms.refinement.setup_geometry import Extract
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator
from dials.algorithms.refinement.prediction import ScansRayPredictor, \
ExperimentsPredictor
from cctbx.sgtbx import space_group, space_group_symbols
# We will set up a mock scan
from dxtbx.model import ScanFactory
master_phil = parse("""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True)
overrides = """geometry.parameters.crystal.a.length.range = 10 50
geometry.parameters.crystal.b.length.range = 10 50
geometry.parameters.crystal.c.length.range = 10 50"""
models = Extract(master_phil, local_overrides=overrides)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), resolution)
indices = index_generator.to_array()
# Build a mock scan for a 30 degree sweep
sf = ScanFactory()
myscan = sf.make_scan(image_range=(1, 300),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=range(300),
deg=True)
sweep_range = myscan.get_oscillation_range(deg=False)
assert sweep_range == pytest.approx((0., math.pi / 6.))
im_width = myscan.get_oscillation(deg=False)[1]
assert im_width == pytest.approx(0.1 * math.pi / 180.)
# Create an ExperimentList for ScansRayPredictor
experiments = ExperimentList()
experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None))
# Select those that are excited in a 30 degree sweep and get angles
UB = mycrystal.get_A()
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs = ray_predictor(indices)
# Set the experiment number
obs_refs['id'] = flex.int(len(obs_refs), 0)
# Calculate intersections
ref_predictor = ExperimentsPredictor(experiments)
obs_refs = ref_predictor(obs_refs)
print("Total number of observations made", len(obs_refs))
mypanel = mydetector[0]
s0 = matrix.col(mybeam.get_s0())
spindle = matrix.col(mygonio.get_rotation_axis())
for ref in obs_refs:
# get the s1 vector of this reflection
s1 = matrix.col(ref['s1'])
r = s1 - s0
r_orig = r.rotate_around_origin(spindle, -1., deg=True)
# is it outside the Ewald sphere (i.e. entering)?
test = (s0 + r_orig).length() > s0.length()
assert ref['entering'] == test
def test(args=[]):
#############################
# Setup experimental models #
#############################
master_phil = parse("""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True)
models = setup_geometry.Extract(master_phil, cmdline_args=args)
single_panel_detector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Make a 3x3 multi panel detector filling the same space as the existing
# single panel detector. Each panel of the multi-panel detector has pixels with
# 1/3 the length dimensions of the single panel.
multi_panel_detector = Detector()
for x in range(3):
for y in range(3):
new_panel = make_panel_in_array((x, y), single_panel_detector[0])
multi_panel_detector.add_panel(new_panel)
# Build a mock scan for a 180 degree sweep
sf = ScanFactory()
myscan = sf.make_scan(image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=range(1800),
deg=True)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sweep_range == (0., pi)
assert approx_equal(im_width, 0.1 * pi / 180.)
# Build ExperimentLists
experiments_single_panel = ExperimentList()
experiments_multi_panel = ExperimentList()
experiments_single_panel.append(
Experiment(beam=mybeam,
detector=single_panel_detector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None))
experiments_multi_panel.append(
Experiment(beam=mybeam,
detector=multi_panel_detector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None))
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(single_panel_detector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
multi_det_param = DetectorParameterisationMultiPanel(
multi_panel_detector, mybeam)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Fix crystal parameters
#xluc_param.set_fixed([True, True, True, True, True, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
pred_param = XYPhiPredictionParameterisation(experiments_single_panel,
[det_param], [s0_param],
[xlo_param], [xluc_param])
pred_param2 = XYPhiPredictionParameterisation(experiments_multi_panel,
[multi_det_param],
[s0_param], [xlo_param],
[xluc_param])
################################
# Apply known parameter shifts #
################################
# shift detectors by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2., 2., 2.])]
det_param.set_param_vals(p_vals)
multi_det_p_vals = multi_det_param.get_param_vals()
p_vals = [
a + b for a, b in zip(multi_det_p_vals, [1.0, 1.0, 1.0, 2., 2., 2.])
]
multi_det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [2., 2., 2.])]
xlo_param.set_param_vals(p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = mycrystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(mycrystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), resolution)
indices = index_generator.to_array()
# for the reflection predictor, it doesn't matter which experiment list is
# passed, as the detector is not used
ref_predictor = ScansRayPredictor(experiments_single_panel, sweep_range)
# get two sets of identical reflections
obs_refs = ref_predictor(indices)
obs_refs2 = ref_predictor(indices)
for r1, r2 in zip(obs_refs, obs_refs2):
assert r1['s1'] == r2['s1']
# get the panel intersections
sel = ray_intersection(single_panel_detector, obs_refs)
obs_refs = obs_refs.select(sel)
sel = ray_intersection(multi_panel_detector, obs_refs2)
obs_refs2 = obs_refs2.select(sel)
assert len(obs_refs) == len(obs_refs2)
# Set 'observed' centroids from the predicted ones
obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm']
obs_refs2['xyzobs.mm.value'] = obs_refs2['xyzcal.mm']
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.
px_size = single_panel_detector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.)**2)
# set the variances and frame numbers
obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
obs_refs2['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
# Add in flags and ID columns by copying into standard reflection tables
tmp = flex.reflection_table.empty_standard(len(obs_refs))
tmp.update(obs_refs)
obs_refs = tmp
tmp = flex.reflection_table.empty_standard(len(obs_refs2))
tmp.update(obs_refs2)
obs_refs2 = tmp
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
multi_det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(obs_refs, experiments_single_panel)
refman2 = ReflectionManager(obs_refs, experiments_multi_panel)
###############################
# Set up the target functions #
###############################
mytarget = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments_single_panel,
ExperimentsPredictor(experiments_single_panel),
refman,
pred_param,
restraints_parameterisation=None)
mytarget2 = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments_multi_panel,
ExperimentsPredictor(experiments_multi_panel),
refman2,
pred_param2,
restraints_parameterisation=None)
#################################
# Set up the refinement engines #
#################################
refiner = setup_minimiser.Extract(master_phil,
mytarget,
pred_param,
cmdline_args=args).refiner
refiner2 = setup_minimiser.Extract(master_phil,
mytarget2,
pred_param2,
cmdline_args=args).refiner
refiner.run()
# reset parameters and run refinement with the multi panel detector
s0_param.set_param_vals(s0_p_vals)
multi_det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
refiner2.run()
# same number of steps
assert refiner.get_num_steps() == refiner2.get_num_steps()
# same rmsds
for rmsd, rmsd2 in zip(refiner.history["rmsd"], refiner2.history["rmsd"]):
assert approx_equal(rmsd, rmsd2)
# same parameter values each step
for params, params2 in zip(refiner.history["parameter_vector"],
refiner.history["parameter_vector"]):
assert approx_equal(params, params2)
def test():
from cctbx.sgtbx import space_group, space_group_symbols
from libtbx.phil import parse
from scitbx.array_family import flex
##### Import model builder
from dials.test.algorithms.refinement.setup_geometry import Extract
##### Imports for reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection
from dxtbx.model.experiment_list import ExperimentList, Experiment
from dials.algorithms.refinement.prediction import ScansRayPredictor, \
ExperimentsPredictor
#### Import model parameterisations
from dials.algorithms.refinement.parameterisation.prediction_parameters import \
XYPhiPredictionParameterisation
from dials.algorithms.refinement.parameterisation.detector_parameters import \
DetectorParameterisationSinglePanel
from dials.algorithms.refinement.parameterisation.beam_parameters import \
BeamParameterisation
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalOrientationParameterisation, \
CrystalUnitCellParameterisation
from dials.algorithms.refinement.parameterisation.goniometer_parameters import \
GoniometerParameterisation
#### Create models
overrides = """geometry.parameters.crystal.a.length.range = 10 50
geometry.parameters.crystal.b.length.range = 10 50
geometry.parameters.crystal.c.length.range = 10 50"""
master_phil = parse("""
include scope dials.test.algorithms.refinement.geometry_phil
""", process_includes=True)
models = Extract(master_phil, overrides)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
# Build a mock scan for a 72 degree sweep
sweep_range = (0., math.pi/5.)
from dxtbx.model import ScanFactory
sf = ScanFactory()
myscan = sf.make_scan(image_range = (1,720),
exposure_times = 0.1,
oscillation = (0, 0.1),
epochs = range(720),
deg = True)
#### Create parameterisations of these models
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
gon_param = GoniometerParameterisation(mygonio, mybeam)
# Create an ExperimentList
experiments = ExperimentList()
experiments.append(Experiment(
beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan,
crystal=mycrystal, imageset=None))
#### Unit tests
# Build a prediction parameterisation
pred_param = XYPhiPredictionParameterisation(experiments,
detector_parameterisations = [det_param],
beam_parameterisations = [s0_param],
xl_orientation_parameterisations = [xlo_param],
xl_unit_cell_parameterisations = [xluc_param],
goniometer_parameterisations = [gon_param])
# Generate reflections
resolution = 2.0
index_generator = IndexGenerator(mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution)
indices = index_generator.to_array()
# Predict rays within the sweep range
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs = ray_predictor(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ExperimentsPredictor(experiments)
obs_refs['id'] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm']
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * math.pi / 180.
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.)**2)
obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
# use a ReflectionManager to exclude reflections too close to the spindle
from dials.algorithms.refinement.reflection_manager import ReflectionManager
refman = ReflectionManager(obs_refs, experiments, outlier_detector=None)
# Redefine the reflection predictor to use the type expected by the Target class
ref_predictor = ExperimentsPredictor(experiments)
# make a target to ensure reflections are predicted and refman is finalised
from dials.algorithms.refinement.target import \
LeastSquaresPositionalResidualWithRmsdCutoff
target = LeastSquaresPositionalResidualWithRmsdCutoff(experiments,
ref_predictor, refman, pred_param, restraints_parameterisation=None)
# keep only those reflections that pass inclusion criteria and have predictions
reflections = refman.get_matches()
# get analytical gradients
an_grads = pred_param.get_gradients(reflections)
# get finite difference gradients
p_vals = pred_param.get_param_vals()
deltas = [1.e-7] * len(p_vals)
for i, delta in enumerate(deltas):
val = p_vals[i]
p_vals[i] -= delta / 2.
pred_param.set_param_vals(p_vals)
ref_predictor(reflections)
rev_state = reflections['xyzcal.mm'].deep_copy()
p_vals[i] += delta
pred_param.set_param_vals(p_vals)
ref_predictor(reflections)
fwd_state = reflections['xyzcal.mm'].deep_copy()
p_vals[i] = val
fd = (fwd_state - rev_state)
x_grads, y_grads, phi_grads = fd.parts()
x_grads /= delta
y_grads /= delta
phi_grads /= delta
# compare with analytical calculation
assert x_grads == pytest.approx(an_grads[i]["dX_dp"], abs=5.e-6)
assert y_grads == pytest.approx(an_grads[i]["dY_dp"], abs=5.e-6)
assert phi_grads == pytest.approx(an_grads[i]["dphi_dp"], abs=5.e-6)
# return to the initial state
pred_param.set_param_vals(p_vals)
# Add in flags and ID columns by copying into standard reflection table tmp = flex.reflection_table.empty_standard(len(obs_refs)) tmp.update(obs_refs) obs_refs = tmp # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) # Re-predict using the stills reflection predictor stills_ref_predictor = ExperimentsPredictor(stills_experiments) stills_ref_predictor.update() obs_refs_stills = stills_ref_predictor.predict(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs_stills['xyzobs.mm.value'] = obs_refs_stills['xyzcal.mm'] ############################### # Undo known parameter shifts # ############################### xlo_param.set_param_vals(xlo_p_vals[0]) xluc_param.set_param_vals(xluc_p_vals[0]) # make a refiner from dials.algorithms.refinement.refiner import phil_scope
def test(args=[]):
# Python and cctbx imports
from math import pi
from scitbx import matrix
from scitbx.array_family import flex
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
# Get module to build models using PHIL
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Model parameterisations
from dials.algorithms.refinement.parameterisation.detector_parameters import \
DetectorParameterisationSinglePanel
from dials.algorithms.refinement.parameterisation.beam_parameters import \
BeamParameterisation
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalOrientationParameterisation, CrystalUnitCellParameterisation
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import \
symmetrize_reduce_enlarge
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator
from dials.algorithms.refinement.prediction import ScansRayPredictor, \
ExperimentsPredictor
from dials.algorithms.spot_prediction import ray_intersection
from cctbx.sgtbx import space_group, space_group_symbols
# Parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import \
XYPhiPredictionParameterisation # implicit import
# Imports for the target function
from dials.algorithms.refinement.target import \
LeastSquaresPositionalResidualWithRmsdCutoff # implicit import
#############################
# Setup experimental models #
#############################
master_phil = parse("""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True)
models = setup_geometry.Extract(
master_phil,
cmdline_args=args,
local_overrides="geometry.parameters.random_seed = 1")
crystal1 = models.crystal
models = setup_geometry.Extract(
master_phil,
cmdline_args=args,
local_overrides="geometry.parameters.random_seed = 2")
mydetector = models.detector
mygonio = models.goniometer
crystal2 = models.crystal
mybeam = models.beam
# Build a mock scan for a 180 degree sweep
sf = ScanFactory()
myscan = sf.make_scan(image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=range(1800),
deg=True)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sweep_range == (0., pi)
assert approx_equal(im_width, 0.1 * pi / 180.)
# Build an experiment list
experiments = ExperimentList()
experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal1,
imageset=None))
experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal2,
imageset=None))
assert len(experiments.detectors()) == 1
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xl1o_param = CrystalOrientationParameterisation(crystal1)
xl1uc_param = CrystalUnitCellParameterisation(crystal1)
xl2o_param = CrystalOrientationParameterisation(crystal2)
xl2uc_param = CrystalUnitCellParameterisation(crystal2)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Fix crystal parameters
#xluc_param.set_fixed([True, True, True, True, True, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
#pred_param = XYPhiPredictionParameterisation(experiments,
# [det_param], [s0_param], [xlo_param], [xluc_param])
################################
# Apply known parameter shifts #
################################
# shift detector by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2., 2., 2.])]
det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = []
for xlo in (xl1o_param, xl2o_param):
p_vals = xlo.get_param_vals()
xlo_p_vals.append(p_vals)
new_p_vals = [a + b for a, b in zip(p_vals, [2., 2., 2.])]
xlo.set_param_vals(new_p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = []
for xluc, xl in ((xl1uc_param, crystal1), ((xl2uc_param, crystal2))):
p_vals = xluc.get_param_vals()
xluc_p_vals.append(p_vals)
cell_params = xl.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(xl.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.e5 for e in S.forward_independent_parameters()])
xluc.set_param_vals(X)
#############################
# Generate some reflections #
#############################
#print "Reflections will be generated with the following geometry:"
#print mybeam
#print mydetector
#print crystal1
#print crystal2
# All indices in a 2.0 Angstrom sphere for crystal1
resolution = 2.0
index_generator = IndexGenerator(
crystal1.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), resolution)
indices1 = index_generator.to_array()
# All indices in a 2.0 Angstrom sphere for crystal2
resolution = 2.0
index_generator = IndexGenerator(
crystal2.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), resolution)
indices2 = index_generator.to_array()
# Predict rays within the sweep range. Set experiment IDs
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs1 = ray_predictor(indices1, experiment_id=0)
obs_refs1['id'] = flex.int(len(obs_refs1), 0)
obs_refs2 = ray_predictor(indices1, experiment_id=1)
obs_refs2['id'] = flex.int(len(obs_refs2), 1)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs1)
obs_refs1 = obs_refs1.select(intersects)
intersects = ray_intersection(mydetector, obs_refs2)
obs_refs2 = obs_refs2.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ExperimentsPredictor(experiments)
obs_refs1 = ref_predictor(obs_refs1)
obs_refs2 = ref_predictor(obs_refs2)
# Set 'observed' centroids from the predicted ones
obs_refs1['xyzobs.mm.value'] = obs_refs1['xyzcal.mm']
obs_refs2['xyzobs.mm.value'] = obs_refs2['xyzcal.mm']
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs1), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs1), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs1), (im_width / 2.)**2)
obs_refs1['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
var_x = flex.double(len(obs_refs2), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs2), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs2), (im_width / 2.)**2)
obs_refs2['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
#print "Total number of reflections excited for crystal1", len(obs_refs1)
#print "Total number of reflections excited for crystal2", len(obs_refs2)
# concatenate reflection lists
obs_refs1.extend(obs_refs2)
obs_refs = obs_refs1
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xl1o_param.set_param_vals(xlo_p_vals[0])
xl2o_param.set_param_vals(xlo_p_vals[1])
xl1uc_param.set_param_vals(xluc_p_vals[0])
xl2uc_param.set_param_vals(xluc_p_vals[1])
#print "Initial values of parameters are"
#msg = "Parameters: " + "%.5f " * len(pred_param)
#print msg % tuple(pred_param.get_param_vals())
#print
# make a refiner
from dials.algorithms.refinement.refiner import phil_scope
params = phil_scope.fetch(source=parse('')).extract()
# in case we want a plot
params.refinement.refinery.journal.track_parameter_correlation = True
# scan static first
from dials.algorithms.refinement.refiner import RefinerFactory
refiner = RefinerFactory.from_parameters_data_experiments(params,
obs_refs,
experiments,
verbosity=0)
history = refiner.run()
# scan varying
params.refinement.parameterisation.scan_varying = True
refiner = RefinerFactory.from_parameters_data_experiments(params,
obs_refs,
experiments,
verbosity=0)
history = refiner.run()
def test():
# Python and cctbx imports
from math import pi
from scitbx import matrix
from scitbx.array_family import flex
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
# Get modules to build models and minimiser using PHIL
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
import dials.test.algorithms.refinement.setup_minimiser as setup_minimiser
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Model parameterisations
from dials.algorithms.refinement.parameterisation.detector_parameters import \
DetectorParameterisationSinglePanel
from dials.algorithms.refinement.parameterisation.beam_parameters import \
BeamParameterisation
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalOrientationParameterisation, CrystalUnitCellParameterisation
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import \
symmetrize_reduce_enlarge
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection
from dials.algorithms.refinement.prediction import ScansRayPredictor, \
ExperimentsPredictor
from cctbx.sgtbx import space_group, space_group_symbols
# Parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import \
XYPhiPredictionParameterisation
# Imports for the target function
from dials.algorithms.refinement.target import \
LeastSquaresPositionalResidualWithRmsdCutoff
from dials.algorithms.refinement.reflection_manager import ReflectionManager
#############################
# Setup experimental models #
#############################
override = """geometry.parameters
{
beam.wavelength.random=False
beam.wavelength.value=1.0
beam.direction.inclination.random=False
crystal.a.length.random=False
crystal.a.length.value=12.0
crystal.a.direction.method=exactly
crystal.a.direction.exactly.direction=1.0 0.002 -0.004
crystal.b.length.random=False
crystal.b.length.value=14.0
crystal.b.direction.method=exactly
crystal.b.direction.exactly.direction=-0.002 1.0 0.002
crystal.c.length.random=False
crystal.c.length.value=13.0
crystal.c.direction.method=exactly
crystal.c.direction.exactly.direction=0.002 -0.004 1.0
detector.directions.method=exactly
detector.directions.exactly.dir1=0.99 0.002 -0.004
detector.directions.exactly.norm=0.002 -0.001 0.99
detector.centre.method=exactly
detector.centre.exactly.value=1.0 -0.5 199.0
}"""
master_phil = parse("""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True)
models = setup_geometry.Extract(master_phil,
local_overrides=override,
verbose=False)
mydetector = models.detector
mygonio = models.goniometer
mycrystal = models.crystal
mybeam = models.beam
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
xluc_param = CrystalUnitCellParameterisation(mycrystal)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
# Build a mock scan for a 180 degree sweep
sf = ScanFactory()
myscan = sf.make_scan(image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=range(1800),
deg=True)
# Build an ExperimentList
experiments = ExperimentList()
experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=mycrystal,
imageset=None))
# Create the PredictionParameterisation
pred_param = XYPhiPredictionParameterisation(experiments, [det_param],
[s0_param], [xlo_param],
[xluc_param])
################################
# Apply known parameter shifts #
################################
# shift detector by 1.0 mm each translation and 4 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 4., 4., 4.])]
det_param.set_param_vals(p_vals)
# shift beam by 4 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 4.
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=3 mrad each rotation)
xlo_p_vals = xlo_param.get_param_vals()
p_vals = [a + b for a, b in zip(xlo_p_vals, [3., 3., 3.])]
xlo_param.set_param_vals(p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# alpha and beta angles)
xluc_p_vals = xluc_param.get_param_vals()
cell_params = mycrystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1, -0.1, 0.0])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(mycrystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere
resolution = 2.0
index_generator = IndexGenerator(
mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), resolution)
indices = index_generator.to_array()
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sweep_range == (0., pi)
assert approx_equal(im_width, 0.1 * pi / 180.)
# Predict rays within the sweep range
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs = ray_predictor(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ExperimentsPredictor(experiments)
obs_refs['id'] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm']
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.)**2)
obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
# The total number of observations should be 1128
assert len(obs_refs) == 1128
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(obs_refs,
experiments,
outlier_detector=None,
close_to_spindle_cutoff=0.1)
##############################
# Set up the target function #
##############################
# The current 'achieved' criterion compares RMSD against 1/3 the pixel size and
# 1/3 the image width in radians. For the simulated data, these are just made up
mytarget = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments,
ref_predictor,
refman,
pred_param,
restraints_parameterisation=None)
######################################
# Set up the LSTBX refinement engine #
######################################
overrides = """minimiser.parameters.engine=GaussNewton
minimiser.parameters.verbosity=0
minimiser.parameters.logfile=None"""
refiner = setup_minimiser.Extract(master_phil,
mytarget,
pred_param,
local_overrides=overrides).refiner
refiner.run()
assert mytarget.achieved()
assert refiner.get_num_steps() == 1
assert approx_equal(
mytarget.rmsds(),
(0.00508252354876, 0.00420954552156, 8.97303428289e-05))
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
######################################################
# Set up the LBFGS with curvatures refinement engine #
######################################################
overrides = """minimiser.parameters.engine=LBFGScurvs
minimiser.parameters.verbosity=0
minimiser.parameters.logfile=None"""
refiner = setup_minimiser.Extract(master_phil,
mytarget,
pred_param,
local_overrides=overrides).refiner
refiner.run()
assert mytarget.achieved()
assert refiner.get_num_steps() == 9
assert approx_equal(mytarget.rmsds(),
(0.0558857700305, 0.0333446685335, 0.000347402754278))
# Add in flags and ID columns by copying into standard reflection table
tmp = flex.reflection_table.empty_standard(len(obs_refs))
tmp.update(obs_refs)
obs_refs = tmp
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.)**2)
obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
# Re-predict using the stills reflection predictor
stills_ref_predictor = ExperimentsPredictor(stills_experiments)
obs_refs_stills = stills_ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs_stills['xyzobs.mm.value'] = obs_refs_stills['xyzcal.mm']
###############################
# Undo known parameter shifts #
###############################
xlo_param.set_param_vals(xlo_p_vals[0])
xluc_param.set_param_vals(xluc_p_vals[0])
# make a refiner
from dials.algorithms.refinement.refiner import phil_scope
params = phil_scope.fetch(source=parse('')).extract()
index_generator = IndexGenerator(mycrystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution)
indices = index_generator.to_array()
# Predict rays within the sweep range
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs = ray_predictor.predict(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ExperimentsPredictor(experiments)
obs_refs['id'] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor.predict(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm']
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.)**2)
obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
# set the flex random seed to an 'uninteresting' number
def config_target(params, experiments, refman, do_stills):
"""Given a set of parameters, configure a factory to build a
target function
Params:
params The input parameters
Returns:
The target factory instance
"""
# Shorten parameter paths
options = params.refinement.target
sparse = params.refinement.parameterisation.sparse
if options.rmsd_cutoff == "fraction_of_bin_size":
absolute_cutoffs = None
elif options.rmsd_cutoff == "absolute":
absolute_cutoffs = options.absolute_cutoffs
else:
raise RuntimeError("Target function rmsd_cutoff option" +
options.rmsd_cutoff + " not recognised")
# all experiments have the same (or no) goniometer
goniometer = experiments[0].goniometer
for e in experiments:
assert e.goniometer is goniometer
# build managed reflection predictors
from dials.algorithms.refinement.prediction import ExperimentsPredictor
ref_predictor = ExperimentsPredictor(experiments, do_stills)
# Determine whether the target is in X, Y, Phi space or just X, Y.
if do_stills:
if sparse:
targ = LeastSquaresStillsDetectorSparse
else:
targ = LeastSquaresStillsDetector
else:
raise NotImplementedError("currently only for stills")
#if sparse:
# raise NotImplementedError("currently only for stills")
# #from dials.algorithms.refinement.target \
# # import LeastSquaresPositionalResidualWithRmsdCutoffSparse as targ
#else:
# raise NotImplementedError("currently only for stills")
# #from dials.algorithms.refinement.target \
# # import LeastSquaresPositionalResidualWithRmsdCutoff as targ
# Here we pass in None for prediction_parameterisation and
# restraints_parameterisation, as these will be linked to the object later
target = targ(experiments=experiments,
reflection_predictor=ref_predictor,
ref_man=refman,
prediction_parameterisation=None,
restraints_parameterisation=None,
frac_binsize_cutoff=options.bin_size_fraction,
absolute_cutoffs=absolute_cutoffs,
gradient_calculation_blocksize=options.
gradient_calculation_blocksize)
return target
xlo_param.set_param_vals(xlo_p_vals)
xluc_param.set_param_vals(xluc_p_vals)
#####################################
# Select reflections for refinement #
#####################################
refman = ReflectionManager(obs_refs, experiments_single_panel)
refman2 = ReflectionManager(obs_refs, experiments_multi_panel)
###############################
# Set up the target functions #
###############################
mytarget = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments_single_panel, ExperimentsPredictor(experiments_single_panel),
refman, pred_param, restraints_parameterisation=None)
mytarget2 = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments_multi_panel, ExperimentsPredictor(experiments_multi_panel),
refman2, pred_param2, restraints_parameterisation=None)
#################################
# Set up the refinement engines #
#################################
refiner = setup_minimiser.Extract(master_phil,
mytarget,
pred_param,
cmdline_args = args).refiner
refiner2 = setup_minimiser.Extract(master_phil,
mytarget2,
sweep_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) # Predict rays within the sweep range ray_predictor = ScansRayPredictor(experiments, sweep_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ExperimentsPredictor(experiments) obs_refs['id'] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) # The total number of observations should be 1128
def generate_reflections(experiments, xyzvar=(0.0, 0.0, 0.0)):
"""Generate synthetic reflection centroids using the supplied experiments,
with normally-distributed errors applied the variances in xyzvar"""
# check input
if [e >= 0.0 for e in xyzvar].count(False) > 0:
msg = "negative variance requested in " + str(xyzvar) + "!"
raise RuntimeError(msg)
refs = []
for iexp, exp in enumerate(experiments):
info("Generating reflections for experiment {0}".format(iexp))
# All indices in a 1.5 Angstrom sphere
resolution = 1.5
index_generator = IndexGenerator(
exp.crystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# Predict rays within the sweep range
ray_predictor = ScansRayPredictor(
experiments, exp.scan.get_oscillation_range(deg=False)
)
obs_refs = ray_predictor.predict(indices, experiment_id=iexp)
info("Total number of reflections excited: {0}".format(len(obs_refs)))
# Take only those rays that intersect the detector
intersects = ray_intersection(exp.detector, obs_refs)
obs_refs = obs_refs.select(intersects)
obs_refs["id"] = flex.size_t(len(obs_refs), iexp)
refs.append(obs_refs)
info("Total number of impacts: {0}".format(len(obs_refs)))
# Concatenate reflections
obs_refs = reduce(lambda x, y: x.extend(y), refs)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ExperimentsPredictor(experiments)
obs_refs = ref_predictor.predict(obs_refs)
# calculate (uncorrelated) errors to offset the centroids
# this is safe as elts of xyzvar are already tested to be > 0
sigX, sigY, sigZ = [sqrt(e) for e in xyzvar]
shift = [
(random.gauss(0, sigX), random.gauss(0, sigY), random.gauss(0, sigZ))
for _ in xrange(len(obs_refs))
]
shift = flex.vec3_double(shift)
# Set 'observed' centroids from the predicted ones
obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"] + shift
# Store variances for the centroid positions of the simulated data. If errors
# are zero, invent some variances
if tuple(xyzvar) == (0.0, 0.0, 0.0):
im_width = exp.scan.get_oscillation()[1] * pi / 180.0
px_size = exp.detector[0].get_pixel_size()
xyzvar = (
(px_size[0] / 2.0) ** 2,
(px_size[1] / 2.0) ** 2,
(im_width / 2.0) ** 2,
)
var_x = flex.double(len(obs_refs), xyzvar[0])
var_y = flex.double(len(obs_refs), xyzvar[1])
var_phi = flex.double(len(obs_refs), xyzvar[2])
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
info("Total number of observations made: {0}".format(len(obs_refs)))
return obs_refs
def choose_best_orientation_matrix(self, candidate_orientation_matrices):
from dxtbx.model.experiment_list import Experiment, ExperimentList
import copy
logger.info('*' * 80)
logger.info('Selecting the best orientation matrix')
logger.info('*' * 80)
from libtbx import group_args
class candidate_info(group_args):
pass
candidates = []
params = copy.deepcopy(self.all_params)
n_cand = len(candidate_orientation_matrices)
for icm, cm in enumerate(candidate_orientation_matrices):
# Index reflections in P1
sel = ((self.reflections['id'] == -1))
refl = self.reflections.select(sel)
experiments = self.experiment_list_for_crystal(cm)
self.index_reflections(experiments, refl)
indexed = refl.select(refl['id'] >= 0)
indexed = indexed.select(indexed.get_flags(indexed.flags.indexed))
# If target symmetry supplied, try to apply it. Then, apply the change of basis to the reflections
# indexed in P1 to the target setting
if self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None:
target_space_group = self.target_symmetry_primitive.space_group(
)
new_crystal, cb_op_to_primitive = self.apply_symmetry(
cm, target_space_group)
if new_crystal is None:
print(
"Cannot convert to target symmetry, candidate %d/%d" %
(icm, n_cand))
continue
new_crystal = new_crystal.change_basis(
self.cb_op_primitive_inp)
cm = candidate_orientation_matrices[icm] = new_crystal
experiments = self.experiment_list_for_crystal(cm)
if not cb_op_to_primitive.is_identity_op():
indexed['miller_index'] = cb_op_to_primitive.apply(
indexed['miller_index'])
if self.cb_op_primitive_inp is not None:
indexed['miller_index'] = self.cb_op_primitive_inp.apply(
indexed['miller_index'])
if params.indexing.stills.refine_all_candidates:
try:
print(
"$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d initial outlier identification"
% (icm, n_cand))
acceptance_flags = self.identify_outliers(
params, experiments, indexed)
#create a new "indexed" list with outliers thrown out:
indexed = indexed.select(acceptance_flags)
print(
"$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d refinement before outlier rejection"
% (icm, n_cand))
R = e_refine(params=params,
experiments=experiments,
reflections=indexed,
graph_verbose=False)
ref_experiments = R.get_experiments()
# try to improve the outcome with a second round of outlier rejection post-initial refinement:
acceptance_flags = self.identify_outliers(
params, ref_experiments, indexed)
# insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection
nv0 = nave_parameters(params=params,
experiments=ref_experiments,
reflections=indexed,
refinery=R,
graph_verbose=False)
crystal_model_nv0 = nv0()
acceptance_flags_nv0 = nv0.nv_acceptance_flags
indexed = indexed.select(acceptance_flags
& acceptance_flags_nv0)
print(
"$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d after positional and delta-psi outlier rejection"
% (icm, n_cand))
R = e_refine(params=params,
experiments=ref_experiments,
reflections=indexed,
graph_verbose=False)
ref_experiments = R.get_experiments()
nv = nave_parameters(params=params,
experiments=ref_experiments,
reflections=indexed,
refinery=R,
graph_verbose=False)
crystal_model = nv()
# Drop candidates that after refinement can no longer be converted to the known target space group
if not self.params.stills.refine_candidates_with_known_symmetry and self.params.known_symmetry.space_group is not None:
target_space_group = self.target_symmetry_primitive.space_group(
)
new_crystal, cb_op_to_primitive = self.apply_symmetry(
crystal_model, target_space_group)
if new_crystal is None:
print(
"P1 refinement yielded model diverged from target, candidate %d/%d"
% (icm, n_cand))
continue
rmsd, _ = calc_2D_rmsd_and_displacements(
R.predict_for_reflection_table(indexed))
except Exception as e:
print("Couldn't refine candiate %d/%d, %s" %
(icm, n_cand, str(e)))
else:
print(
"$$$ stills_indexer::choose_best_orientation_matrix, candidate %d/%d done"
% (icm, n_cand))
candidates.append(
candidate_info(
crystal=crystal_model,
green_curve_area=nv.green_curve_area,
ewald_proximal_volume=nv.ewald_proximal_volume(),
n_indexed=len(indexed),
rmsd=rmsd,
indexed=indexed,
experiments=ref_experiments))
else:
from dials.algorithms.refinement.prediction import ExperimentsPredictor
ref_predictor = ExperimentsPredictor(
experiments,
force_stills=True,
spherical_relp=params.refinement.parameterisation.
spherical_relp_model)
rmsd, _ = calc_2D_rmsd_and_displacements(
ref_predictor(indexed))
candidates.append(
candidate_info(crystal=cm,
n_indexed=len(indexed),
rmsd=rmsd,
indexed=indexed,
experiments=experiments))
if len(candidates) == 0:
raise Sorry("No suitable indexing solution found")
print("**** ALL CANDIDATES:")
for i, XX in enumerate(candidates):
print("\n****Candidate %d" % i, XX)
cc = XX.crystal
if hasattr(cc, 'get_half_mosaicity_deg'):
print(" half mosaicity %5.2f deg." %
(cc.get_half_mosaicity_deg()))
print(" domain size %.0f Ang." % (cc.get_domain_size_ang()))
print("\n**** BEST CANDIDATE:")
results = flex.double([c.rmsd for c in candidates])
best = candidates[flex.min_index(results)]
print(best)
if params.indexing.stills.refine_all_candidates:
if best.rmsd > params.indexing.stills.rmsd_min_px:
raise Sorry("RMSD too high, %f" % best.rmsd)
if best.ewald_proximal_volume > params.indexing.stills.ewald_proximal_volume_max:
raise Sorry("Ewald proximity volume too high, %f" %
best.ewald_proximal_volume)
if len(candidates) > 1:
for i in xrange(len(candidates)):
if i == flex.min_index(results):
continue
if best.ewald_proximal_volume > candidates[
i].ewald_proximal_volume:
print(
"Couldn't figure out which candidate is best; picked the one with the best RMSD."
)
best.indexed['entering'] = flex.bool(best.n_indexed, False)
return best.crystal, best.n_indexed
def generate_reflections(experiments, xyzvar=(0., 0., 0.)):
'''Generate synthetic reflection centroids using the supplied experiments,
with normally-distributed errors applied the variances in xyzvar'''
# check input
if [e >= 0. for e in xyzvar].count(False) > 0:
msg = "negative variance requested in " + str(xyzvar) + "!"
raise RuntimeError(msg)
refs = []
for iexp, exp in enumerate(experiments):
info("Generating reflections for experiment {0}".format(iexp))
# All indices in a 1.5 Angstrom sphere
resolution = 1.5
index_generator = IndexGenerator(exp.crystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), resolution)
indices = index_generator.to_array()
# Predict rays within the sweep range
ray_predictor = ScansRayPredictor(experiments,
exp.scan.get_oscillation_range(deg=False))
obs_refs = ray_predictor.predict(indices, experiment_id=iexp)
info("Total number of reflections excited: {0}".format(len(obs_refs)))
# Take only those rays that intersect the detector
intersects = ray_intersection(exp.detector, obs_refs)
obs_refs = obs_refs.select(intersects)
obs_refs['id'] = flex.size_t(len(obs_refs), iexp)
refs.append(obs_refs)
info("Total number of impacts: {0}".format(len(obs_refs)))
# Concatenate reflections
obs_refs = reduce(lambda x, y: x.extend(y), refs)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ExperimentsPredictor(experiments)
obs_refs = ref_predictor.predict(obs_refs)
# calculate (uncorrelated) errors to offset the centroids
# this is safe as elts of xyzvar are already tested to be > 0
sigX, sigY, sigZ = [sqrt(e) for e in xyzvar]
shift = [(random.gauss(0, sigX),
random.gauss(0, sigY),
random.gauss(0, sigZ)) for _ in xrange(len(obs_refs))]
shift = flex.vec3_double(shift)
# Set 'observed' centroids from the predicted ones
obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] + shift
# Store variances for the centroid positions of the simulated data. If errors
# are zero, invent some variances
if tuple(xyzvar) == (0., 0., 0.):
im_width = exp.scan.get_oscillation()[1] * pi / 180.
px_size = exp.detector[0].get_pixel_size()
xyzvar = ((px_size[0] / 2.)**2, (px_size[1] / 2.)**2, (im_width / 2.)**2)
var_x = flex.double(len(obs_refs), xyzvar[0])
var_y = flex.double(len(obs_refs), xyzvar[1])
var_phi = flex.double(len(obs_refs), xyzvar[2])
obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
info("Total number of observations made: {0}".format(len(obs_refs)))
return obs_refs