def e_refine(params, experiments, reflections, graph_verbose=False):
# Stills-specific parameters we always want
assert params.refinement.reflections.outlier.algorithm in (
None,
"null",
), "Cannot index, set refinement.reflections.outlier.algorithm=null" # we do our own outlier rejection
from dials.algorithms.refinement.refiner import RefinerFactory
refiner = RefinerFactory.from_parameters_data_experiments(
params, reflections, experiments
)
refiner.run()
ref_sel = refiner.selection_used_for_refinement()
assert ref_sel.count(True) == len(reflections)
if not graph_verbose:
return refiner
RR = refiner.predict_for_reflection_table(reflections)
plot_displacements(reflections, RR, experiments)
return refiner
def refine(self, experiments, reflections, isoform=None):
from dials.algorithms.refinement.refiner import phil_scope
from libtbx.phil import parse
params = phil_scope.fetch(source=parse('')).extract()
params.refinement.reflections.weighting_strategy.delpsi_constant = 100000.
params.refinement.reflections.weighting_strategy.override = "stills"
params.refinement.parameterisation.auto_reduction.action = "fix"
#params.refinement.reflections.do_outlier_rejection=True
#params.refinement.reflections.iqr_multiplier=0.5
#params.refinement.reflections.minimum_sample_size=50
#params.refinement.reflections.maximum_sample_size=50
#params.refinement.reflections.random_seed=1
if self.horizons_phil.integration.dials_refinement.strategy == "distance":
params.refinement.parameterisation.beam.fix = "all"
params.refinement.parameterisation.detector.fix_list = [
"Tau1"
] # fix detector rotz, allow distance to refine
elif self.horizons_phil.integration.dials_refinement.strategy == "wavelength":
params.refinement.parameterisation.beam.fix = "in_spindle_plane,out_spindle_plane"
params.refinement.parameterisation.detector.fix_list = [
"Dist", "Tau1"
] # fix detector rotz and distance
elif self.horizons_phil.integration.dials_refinement.strategy == "fix":
params.refinement.parameterisation.beam.fix = "all"
params.refinement.parameterisation.detector.fix_list = [
"Dist", "Tau1"
] # fix detector rotz and distance
if isoform is not None:
params.refinement.reflections.outlier.algorithm = "null"
params.refinement.parameterisation.crystal.fix = "cell"
from dials.algorithms.refinement.refiner import RefinerFactory
refiner = RefinerFactory.from_parameters_data_experiments(params,
reflections,
experiments,
verbosity=1)
history = refiner.run()
print(history.keys())
for item in history["rmsd"]:
print("%5.2f %5.2f %8.5f" %
(item[0], item[1], 180. * item[2] / math.pi))
#for item in history["parameter_vector"]:
# print ["%8.5f"%a for a in item]
print(refiner.selection_used_for_refinement().count(True),
"spots used for refinement")
print(refiner.get_experiments()[0].beam)
print(refiner.get_experiments()[0].detector)
print(
"Distance:",
-refiner.get_experiments()[0].detector[0].get_beam_centre_lab(
refiner.get_experiments()[0].beam.get_s0())[2])
print(refiner.get_experiments()[0].crystal)
return refiner
def e_refine(params, experiments, reflections, graph_verbose=False):
# Stills-specific parameters we always want
assert params.refinement.reflections.outlier.algorithm in (None, "null"), \
"Cannot index, set refinement.reflections.outlier.algorithm=null" # we do our own outlier rejection
from dials.algorithms.refinement.refiner import RefinerFactory
refiner = RefinerFactory.from_parameters_data_experiments(params,
reflections, experiments, verbosity=1)
history = refiner.run()
ref_sel = refiner.selection_used_for_refinement()
assert ref_sel.count(True) == len(reflections)
if not graph_verbose: return refiner
RR = refiner.predict_for_reflection_table(reflections)
plot_displacements(reflections, RR, experiments)
return refiner
def refine(self, experiments, reflections, isoform=None):
from dials.algorithms.refinement.refiner import phil_scope
from libtbx.phil import parse
params = phil_scope.fetch(source=parse('')).extract()
params.refinement.reflections.weighting_strategy.delpsi_constant=100000.
params.refinement.reflections.weighting_strategy.override="stills"
params.refinement.parameterisation.auto_reduction.action="fix"
#params.refinement.reflections.do_outlier_rejection=True
#params.refinement.reflections.iqr_multiplier=0.5
#params.refinement.reflections.minimum_sample_size=50
#params.refinement.reflections.maximum_sample_size=50
#params.refinement.reflections.random_seed=1
if self.horizons_phil.integration.dials_refinement.strategy=="distance":
params.refinement.parameterisation.beam.fix="all"
params.refinement.parameterisation.detector.fix_list=["Tau1"] # fix detector rotz, allow distance to refine
elif self.horizons_phil.integration.dials_refinement.strategy=="wavelength":
params.refinement.parameterisation.beam.fix="in_spindle_plane,out_spindle_plane"
params.refinement.parameterisation.detector.fix_list=["Dist","Tau1"] # fix detector rotz and distance
if isoform is not None:
params.refinement.reflections.outlier.algorithm="null"
params.refinement.parameterisation.crystal.fix="cell"
from dials.algorithms.refinement.refiner import RefinerFactory
refiner = RefinerFactory.from_parameters_data_experiments(params,
reflections, experiments, verbosity=1)
history = refiner.run()
print history.keys()
for item in history["rmsd"]:
print "%5.2f %5.2f %8.5f"%(item[0],item[1],180.*item[2]/math.pi)
#for item in history["parameter_vector"]:
# print ["%8.5f"%a for a in item]
print refiner.selection_used_for_refinement().count(True),"spots used for refinement"
print refiner.get_experiments()[0].beam
print refiner.get_experiments()[0].detector
print "Distance:", -refiner.get_experiments()[0].detector[0].get_beam_centre_lab(refiner.get_experiments()[0].beam.get_s0())[2]
print refiner.get_experiments()[0].crystal
return refiner
def test_run(dials_regression, tmpdir):
expected_unit_cell = uctbx.unit_cell(
(11.624, 13.550, 30.103, 89.964, 93.721, 90.132))
expected_rmsds = (0.039, 0.035, 0.002)
experiments_old = os.path.join(dials_regression, "indexing_test_data",
"phi_scan", "datablock_old.json")
experiments_new = os.path.join(dials_regression, "indexing_test_data",
"phi_scan", "datablock.json")
strong_pickle = os.path.join(dials_regression, "indexing_test_data",
"phi_scan", "strong.pickle")
from dxtbx.serialize import load
imageset_old = load.experiment_list(experiments_old,
check_format=False).imagesets()[0]
imageset_new = load.experiment_list(experiments_new,
check_format=False).imagesets()[0]
gonio_old = imageset_old.get_goniometer()
gonio_new = imageset_new.get_goniometer()
assert gonio_old.get_rotation_axis() == pytest.approx(
(0.7497646259807715, -0.5517923303436749, 0.36520984351713554))
assert gonio_old.get_setting_rotation() == pytest.approx(
(1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0))
assert gonio_old.get_fixed_rotation() == pytest.approx((
0.7497646259807748,
-0.20997265900532208,
-0.6275065641872948,
-0.5517923303436731,
0.3250014637526764,
-0.7680490041218182,
0.3652098435171313,
0.9221092836691605,
0.12781329809272568,
))
assert gonio_new.get_rotation_axis() == pytest.approx(
gonio_old.get_rotation_axis())
assert gonio_new.get_rotation_axis_datum() == pytest.approx((1, 0, 0))
assert gonio_new.get_setting_rotation() == pytest.approx((
0.7497646259807705,
-0.20997265900532142,
-0.6275065641873,
-0.5517923303436786,
0.3250014637526763,
-0.768049004121814,
0.3652098435171315,
0.9221092836691607,
0.12781329809272335,
))
assert gonio_new.get_fixed_rotation() == pytest.approx(
(1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0))
result_old = run_indexing(
strong_pickle,
experiments_old,
tmpdir,
extra_args=[],
expected_unit_cell=expected_unit_cell,
expected_rmsds=expected_rmsds,
expected_hall_symbol=" P 1",
)
result_new = run_indexing(
strong_pickle,
experiments_new,
tmpdir,
extra_args=[],
expected_unit_cell=expected_unit_cell,
expected_rmsds=expected_rmsds,
expected_hall_symbol=" P 1",
)
assert result_old.rmsds == pytest.approx(result_new.rmsds, abs=1e-6)
assert result_old.experiments[0].crystal.get_unit_cell().parameters(
) == pytest.approx(
result_new.experiments[0].crystal.get_unit_cell().parameters(),
abs=1e-6)
# Now test refinement gradients are correct
from dxtbx.model.experiment_list import Experiment, ExperimentList
old_exps = ExperimentList([
Experiment(
beam=imageset_old.get_beam(),
detector=imageset_old.get_detector(),
goniometer=gonio_old,
scan=imageset_old.get_scan(),
crystal=result_old.experiments[0].crystal,
imageset=None,
)
])
new_exps = ExperimentList([
Experiment(
beam=imageset_new.get_beam(),
detector=imageset_new.get_detector(),
goniometer=gonio_new,
scan=imageset_new.get_scan(),
crystal=result_new.experiments[0].crystal,
imageset=None,
)
])
from libtbx.phil import parse
from dials.algorithms.refinement.refiner import phil_scope
params = phil_scope.fetch(source=parse("")).extract()
from dials.algorithms.refinement.refiner import RefinerFactory
refiner_old = RefinerFactory.from_parameters_data_experiments(
params, result_old.indexed_reflections, old_exps)
refiner_new = RefinerFactory.from_parameters_data_experiments(
params, result_new.indexed_reflections, new_exps)
# Analytical gradients should be approximately the same in either case
an_grads_old = refiner_old._pred_param.get_gradients(
refiner_old.get_matches())
an_grads_new = refiner_new._pred_param.get_gradients(
refiner_new.get_matches())
for g1, g2 in zip(an_grads_old, an_grads_new):
assert g1["dX_dp"] == pytest.approx(g2["dX_dp"], abs=1.0e-6)
assert g1["dY_dp"] == pytest.approx(g2["dY_dp"], abs=1.0e-6)
assert g1["dphi_dp"] == pytest.approx(g2["dphi_dp"], abs=1.0e-6)
# Analytical gradients should be approximately equal to finite difference
# gradients in either case
fd_grads_old = calc_fd_grads(refiner_old)
for g1, g2 in zip(fd_grads_old, an_grads_old):
assert g1["dX_dp"] == pytest.approx(g2["dX_dp"], abs=5.0e-6)
assert g1["dY_dp"] == pytest.approx(g2["dY_dp"], abs=5.0e-6)
assert g1["dphi_dp"] == pytest.approx(g2["dphi_dp"], abs=5.0e-6)
fd_grads_new = calc_fd_grads(refiner_new)
for g1, g2 in zip(fd_grads_new, an_grads_new):
assert g1["dX_dp"] == pytest.approx(g2["dX_dp"], abs=5.0e-6)
assert g1["dY_dp"] == pytest.approx(g2["dY_dp"], abs=5.0e-6)
assert g1["dphi_dp"] == pytest.approx(g2["dphi_dp"], abs=5.0e-6)
#print "Initial values of parameters are"
#msg = "Parameters: " + "%.5f " * len(pred_param)
#print msg % tuple(pred_param.get_param_vals())
#print
# make a refiner
from dials.algorithms.refinement.refiner import phil_scope
from libtbx.phil import parse
params = phil_scope.fetch(source=parse('')).extract()
# in case we want a plot
params.refinement.refinery.track_parameter_correlation=True
# DEBUG scan varying by uncommenting the following line
#params.refinement.parameterisation.crystal.scan_varying=True
from dials.algorithms.refinement.refiner import RefinerFactory
refiner = RefinerFactory.from_parameters_data_experiments(params, obs_refs,
experiments, verbosity=0)
history = refiner.run()
#plt = refiner.parameter_correlation_plot(len(history["parameter_correlation"])-1)
#plt.show()
#print "Refinement has completed with the following geometry:"
#expts = refiner.get_experiments()
#for beam in expts.beams(): print beam
#for detector in expts.detectors(): print detector
#for crystal in expts.crystals(): print crystal
print "OK"
def run():
have_dials_regression = libtbx.env.has_module("dials_regression")
if not have_dials_regression:
print "Skipped: dials_regression not available"
return
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression", test=os.path.isdir)
from dials.test.algorithms.indexing.tst_index import run_one_indexing
expected_unit_cell = uctbx.unit_cell(
(11.624, 13.550, 30.103, 89.964, 93.721, 90.132))
expected_rmsds = (0.039, 0.035, 0.002)
datablock_old = os.path.join(
dials_regression, "indexing_test_data/phi_scan/datablock_old.json")
datablock_new = os.path.join(dials_regression,
"indexing_test_data/phi_scan/datablock.json")
strong_pickle = os.path.join(dials_regression,
"indexing_test_data/phi_scan/strong.pickle")
from dxtbx.serialize import load
imageset_old = load.datablock(datablock_old,
check_format=False)[0].extract_imagesets()[0]
imageset_new = load.datablock(datablock_new,
check_format=False)[0].extract_imagesets()[0]
gonio_old = imageset_old.get_goniometer()
gonio_new = imageset_new.get_goniometer()
assert approx_equal(
gonio_old.get_rotation_axis(),
(0.7497646259807715, -0.5517923303436749, 0.36520984351713554))
assert approx_equal(gonio_old.get_setting_rotation(),
(1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0))
assert approx_equal(
gonio_old.get_fixed_rotation(),
(0.7497646259807748, -0.20997265900532208, -0.6275065641872948,
-0.5517923303436731, 0.3250014637526764, -0.7680490041218182,
0.3652098435171313, 0.9221092836691605, 0.12781329809272568))
assert approx_equal(gonio_new.get_rotation_axis(),
gonio_old.get_rotation_axis())
assert approx_equal(gonio_new.get_rotation_axis_datum(), (1, 0, 0))
assert approx_equal(
gonio_new.get_setting_rotation(),
(0.7497646259807705, -0.20997265900532142, -0.6275065641873,
-0.5517923303436786, 0.3250014637526763, -0.768049004121814,
0.3652098435171315, 0.9221092836691607, 0.12781329809272335))
assert approx_equal(gonio_new.get_fixed_rotation(),
(1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0))
result_old = run_one_indexing(
pickle_path=strong_pickle,
sweep_path=datablock_old,
extra_args=[],
expected_unit_cell=expected_unit_cell,
expected_rmsds=expected_rmsds,
expected_hall_symbol=' P 1',
)
result_new = run_one_indexing(
pickle_path=strong_pickle,
sweep_path=datablock_new,
extra_args=[],
expected_unit_cell=expected_unit_cell,
expected_rmsds=expected_rmsds,
expected_hall_symbol=' P 1',
)
assert approx_equal(result_old.rmsds, result_new.rmsds)
assert approx_equal(result_old.crystal_model.get_unit_cell().parameters(),
result_new.crystal_model.get_unit_cell().parameters())
# Now test refinement gradients are correct
from dxtbx.model.experiment_list import ExperimentList, Experiment
old_exps = ExperimentList([
Experiment(beam=imageset_old.get_beam(),
detector=imageset_old.get_detector(),
goniometer=gonio_old,
scan=imageset_old.get_scan(),
crystal=result_old.crystal_model,
imageset=None)
])
new_exps = ExperimentList([
Experiment(beam=imageset_new.get_beam(),
detector=imageset_new.get_detector(),
goniometer=gonio_new,
scan=imageset_new.get_scan(),
crystal=result_new.crystal_model,
imageset=None)
])
from libtbx.phil import parse
from dials.algorithms.refinement.refiner import phil_scope
params = phil_scope.fetch(source=parse('')).extract()
from dials.algorithms.refinement.refiner import RefinerFactory
refiner_old = RefinerFactory.from_parameters_data_experiments(
params, result_old.indexed_reflections, old_exps, verbosity=0)
refiner_new = RefinerFactory.from_parameters_data_experiments(
params, result_new.indexed_reflections, new_exps, verbosity=0)
# Analytical gradients should be approximately the same in either case
an_grads_old = refiner_old._pred_param.get_gradients(
refiner_old.get_matches())
an_grads_new = refiner_new._pred_param.get_gradients(
refiner_new.get_matches())
for g1, g2 in zip(an_grads_old, an_grads_new):
assert approx_equal(g1["dX_dp"], g2["dX_dp"], eps=1.e-6)
assert approx_equal(g1["dY_dp"], g2["dY_dp"], eps=1.e-6)
assert approx_equal(g1["dphi_dp"], g2["dphi_dp"], eps=1.e-6)
# Analytical gradients should be approximately equal to finite difference
# gradients in either case
fd_grads_old = calc_fd_grads(refiner_old)
for g1, g2 in zip(fd_grads_old, an_grads_old):
assert approx_equal(g1["dX_dp"], g2["dX_dp"], eps=5.e-6)
assert approx_equal(g1["dY_dp"], g2["dY_dp"], eps=5.e-6)
assert approx_equal(g1["dphi_dp"], g2["dphi_dp"], eps=5.e-6)
fd_grads_new = calc_fd_grads(refiner_new)
for g1, g2 in zip(fd_grads_new, an_grads_new):
assert approx_equal(g1["dX_dp"], g2["dX_dp"], eps=5.e-6)
assert approx_equal(g1["dY_dp"], g2["dY_dp"], eps=5.e-6)
assert approx_equal(g1["dphi_dp"], g2["dphi_dp"], eps=5.e-6)
# make a refiner
from dials.algorithms.refinement.refiner import phil_scope
params = phil_scope.fetch(source=parse('')).extract()
# Change this to get a plot
do_plot = False
if do_plot:
params.refinement.refinery.journal.track_parameter_correlation = True
from dials.algorithms.refinement.refiner import RefinerFactory
# decrease bin_size_fraction to terminate on RMSD convergence
params.refinement.target.bin_size_fraction = 0.01
params.refinement.parameterisation.beam.fix = "all"
params.refinement.parameterisation.detector.fix = "all"
refiner = RefinerFactory.from_parameters_data_experiments(params,
obs_refs_stills,
stills_experiments,
verbosity=0)
# run refinement
history = refiner.run()
# regression tests
assert len(history["rmsd"]) == 9
refined_crystal = refiner.get_experiments()[0].crystal
uc1 = refined_crystal.get_unit_cell()
uc2 = target_crystal.get_unit_cell()
assert uc1.is_similar_to(uc2)
if do_plot:
plt = refiner.parameter_correlation_plot(
def test(args=[]):
# Python and cctbx imports
from math import pi
from scitbx import matrix
from scitbx.array_family import flex
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
# Get module to build models using PHIL
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Model parameterisations
from dials.algorithms.refinement.parameterisation.detector_parameters import \
DetectorParameterisationSinglePanel
from dials.algorithms.refinement.parameterisation.beam_parameters import \
BeamParameterisation
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalOrientationParameterisation, CrystalUnitCellParameterisation
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import \
symmetrize_reduce_enlarge
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator
from dials.algorithms.refinement.prediction import ScansRayPredictor, \
ExperimentsPredictor
from dials.algorithms.spot_prediction import ray_intersection
from cctbx.sgtbx import space_group, space_group_symbols
# Parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import \
XYPhiPredictionParameterisation # implicit import
# Imports for the target function
from dials.algorithms.refinement.target import \
LeastSquaresPositionalResidualWithRmsdCutoff # implicit import
#############################
# Setup experimental models #
#############################
master_phil = parse("""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True)
models = setup_geometry.Extract(
master_phil,
cmdline_args=args,
local_overrides="geometry.parameters.random_seed = 1")
crystal1 = models.crystal
models = setup_geometry.Extract(
master_phil,
cmdline_args=args,
local_overrides="geometry.parameters.random_seed = 2")
mydetector = models.detector
mygonio = models.goniometer
crystal2 = models.crystal
mybeam = models.beam
# Build a mock scan for a 180 degree sweep
sf = ScanFactory()
myscan = sf.make_scan(image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=range(1800),
deg=True)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sweep_range == (0., pi)
assert approx_equal(im_width, 0.1 * pi / 180.)
# Build an experiment list
experiments = ExperimentList()
experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal1,
imageset=None))
experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal2,
imageset=None))
assert len(experiments.detectors()) == 1
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xl1o_param = CrystalOrientationParameterisation(crystal1)
xl1uc_param = CrystalUnitCellParameterisation(crystal1)
xl2o_param = CrystalOrientationParameterisation(crystal2)
xl2uc_param = CrystalUnitCellParameterisation(crystal2)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
# Fix crystal parameters
#xluc_param.set_fixed([True, True, True, True, True, True])
########################################################################
# Link model parameterisations together into a parameterisation of the #
# prediction equation #
########################################################################
#pred_param = XYPhiPredictionParameterisation(experiments,
# [det_param], [s0_param], [xlo_param], [xluc_param])
################################
# Apply known parameter shifts #
################################
# shift detector by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2., 2., 2.])]
det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = []
for xlo in (xl1o_param, xl2o_param):
p_vals = xlo.get_param_vals()
xlo_p_vals.append(p_vals)
new_p_vals = [a + b for a, b in zip(p_vals, [2., 2., 2.])]
xlo.set_param_vals(new_p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = []
for xluc, xl in ((xl1uc_param, crystal1), ((xl2uc_param, crystal2))):
p_vals = xluc.get_param_vals()
xluc_p_vals.append(p_vals)
cell_params = xl.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(xl.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.e5 for e in S.forward_independent_parameters()])
xluc.set_param_vals(X)
#############################
# Generate some reflections #
#############################
#print "Reflections will be generated with the following geometry:"
#print mybeam
#print mydetector
#print crystal1
#print crystal2
# All indices in a 2.0 Angstrom sphere for crystal1
resolution = 2.0
index_generator = IndexGenerator(
crystal1.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), resolution)
indices1 = index_generator.to_array()
# All indices in a 2.0 Angstrom sphere for crystal2
resolution = 2.0
index_generator = IndexGenerator(
crystal2.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), resolution)
indices2 = index_generator.to_array()
# Predict rays within the sweep range. Set experiment IDs
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs1 = ray_predictor(indices1, experiment_id=0)
obs_refs1['id'] = flex.int(len(obs_refs1), 0)
obs_refs2 = ray_predictor(indices1, experiment_id=1)
obs_refs2['id'] = flex.int(len(obs_refs2), 1)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs1)
obs_refs1 = obs_refs1.select(intersects)
intersects = ray_intersection(mydetector, obs_refs2)
obs_refs2 = obs_refs2.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ExperimentsPredictor(experiments)
obs_refs1 = ref_predictor(obs_refs1)
obs_refs2 = ref_predictor(obs_refs2)
# Set 'observed' centroids from the predicted ones
obs_refs1['xyzobs.mm.value'] = obs_refs1['xyzcal.mm']
obs_refs2['xyzobs.mm.value'] = obs_refs2['xyzcal.mm']
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs1), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs1), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs1), (im_width / 2.)**2)
obs_refs1['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
var_x = flex.double(len(obs_refs2), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs2), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs2), (im_width / 2.)**2)
obs_refs2['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
#print "Total number of reflections excited for crystal1", len(obs_refs1)
#print "Total number of reflections excited for crystal2", len(obs_refs2)
# concatenate reflection lists
obs_refs1.extend(obs_refs2)
obs_refs = obs_refs1
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xl1o_param.set_param_vals(xlo_p_vals[0])
xl2o_param.set_param_vals(xlo_p_vals[1])
xl1uc_param.set_param_vals(xluc_p_vals[0])
xl2uc_param.set_param_vals(xluc_p_vals[1])
#print "Initial values of parameters are"
#msg = "Parameters: " + "%.5f " * len(pred_param)
#print msg % tuple(pred_param.get_param_vals())
#print
# make a refiner
from dials.algorithms.refinement.refiner import phil_scope
params = phil_scope.fetch(source=parse('')).extract()
# in case we want a plot
params.refinement.refinery.journal.track_parameter_correlation = True
# scan static first
from dials.algorithms.refinement.refiner import RefinerFactory
refiner = RefinerFactory.from_parameters_data_experiments(params,
obs_refs,
experiments,
verbosity=0)
history = refiner.run()
# scan varying
params.refinement.parameterisation.scan_varying = True
refiner = RefinerFactory.from_parameters_data_experiments(params,
obs_refs,
experiments,
verbosity=0)
history = refiner.run()
def test(args=[]):
# Python and cctbx imports
from math import pi
from scitbx import matrix
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
# Import for surgery on reflection_tables
from dials.array_family import flex
# Get module to build models using PHIL
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
# We will set up a mock scan and a mock experiment list
from dxtbx.model import ScanFactory
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Crystal parameterisations
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalOrientationParameterisation,
CrystalUnitCellParameterisation,
)
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge
# Reflection prediction
from dials.algorithms.spot_prediction import IndexGenerator
from dials.algorithms.refinement.prediction.managed_predictors import (
ScansRayPredictor,
StillsExperimentsPredictor,
)
from dials.algorithms.spot_prediction import ray_intersection
from cctbx.sgtbx import space_group, space_group_symbols
#############################
# Setup experimental models #
#############################
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
# build models, with a larger crystal than default in order to get enough
# reflections on the 'still' image
param = """
geometry.parameters.crystal.a.length.range=40 50;
geometry.parameters.crystal.b.length.range=40 50;
geometry.parameters.crystal.c.length.range=40 50;
geometry.parameters.random_seed = 42"""
models = setup_geometry.Extract(master_phil,
cmdline_args=args,
local_overrides=param)
crystal = models.crystal
mydetector = models.detector
mygonio = models.goniometer
mybeam = models.beam
# Build a mock scan for a 1.5 degree wedge. Only used for generating indices near
# the Ewald sphere
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 1),
exposure_times=0.1,
oscillation=(0, 1.5),
epochs=list(range(1)),
deg=True,
)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert approx_equal(im_width, 1.5 * pi / 180.0)
# Build experiment lists
stills_experiments = ExperimentList()
stills_experiments.append(
Experiment(beam=mybeam,
detector=mydetector,
crystal=crystal,
imageset=None))
scans_experiments = ExperimentList()
scans_experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
crystal=crystal,
goniometer=mygonio,
scan=myscan,
imageset=None,
))
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
################################
# Apply known parameter shifts #
################################
# rotate crystal (=5 mrad each rotation)
xlo_p_vals = []
p_vals = xlo_param.get_param_vals()
xlo_p_vals.append(p_vals)
new_p_vals = [a + b for a, b in zip(p_vals, [5.0, 5.0, 5.0])]
xlo_param.set_param_vals(new_p_vals)
# change unit cell (=1.0 Angstrom length upsets, 0.5 degree of
# gamma angle)
xluc_p_vals = []
p_vals = xluc_param.get_param_vals()
xluc_p_vals.append(p_vals)
cell_params = crystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [1.0, 1.0, -1.0, 0.0, 0.0, 0.5])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(crystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
# keep track of the target crystal model to compare with refined
from copy import deepcopy
target_crystal = deepcopy(crystal)
#############################
# Generate some reflections #
#############################
# All indices in a 2.0 Angstrom sphere for crystal
resolution = 2.0
index_generator = IndexGenerator(
crystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# Build a ray predictor and predict rays close to the Ewald sphere by using
# the narrow rotation scan
ref_predictor = ScansRayPredictor(scans_experiments, sweep_range)
obs_refs = ref_predictor(indices, experiment_id=0)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Add in flags and ID columns by copying into standard reflection table
tmp = flex.reflection_table.empty_standard(len(obs_refs))
tmp.update(obs_refs)
obs_refs = tmp
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.0)**2)
obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# Re-predict using the stills reflection predictor
stills_ref_predictor = StillsExperimentsPredictor(stills_experiments)
obs_refs_stills = stills_ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs_stills["xyzobs.mm.value"] = obs_refs_stills["xyzcal.mm"]
###############################
# Undo known parameter shifts #
###############################
xlo_param.set_param_vals(xlo_p_vals[0])
xluc_param.set_param_vals(xluc_p_vals[0])
# make a refiner
from dials.algorithms.refinement.refiner import phil_scope
params = phil_scope.fetch(source=parse("")).extract()
# Change this to get a plot
do_plot = False
if do_plot:
params.refinement.refinery.journal.track_parameter_correlation = True
from dials.algorithms.refinement.refiner import RefinerFactory
# decrease bin_size_fraction to terminate on RMSD convergence
params.refinement.target.bin_size_fraction = 0.01
params.refinement.parameterisation.beam.fix = "all"
params.refinement.parameterisation.detector.fix = "all"
refiner = RefinerFactory.from_parameters_data_experiments(
params, obs_refs_stills, stills_experiments)
# run refinement
history = refiner.run()
# regression tests
assert len(history["rmsd"]) == 9
refined_crystal = refiner.get_experiments()[0].crystal
uc1 = refined_crystal.get_unit_cell()
uc2 = target_crystal.get_unit_cell()
assert uc1.is_similar_to(uc2)
if do_plot:
plt = refiner.parameter_correlation_plot(
len(history["parameter_correlation"]) - 1)
plt.show()
def test(args=[]):
#############################
# Setup experimental models #
#############################
master_phil = parse(
"""
include scope dials.tests.algorithms.refinement.geometry_phil
include scope dials.tests.algorithms.refinement.minimiser_phil
""",
process_includes=True,
)
models = setup_geometry.Extract(
master_phil,
cmdline_args=args,
local_overrides="geometry.parameters.random_seed = 1",
)
crystal1 = models.crystal
models = setup_geometry.Extract(
master_phil,
cmdline_args=args,
local_overrides="geometry.parameters.random_seed = 2",
)
mydetector = models.detector
mygonio = models.goniometer
crystal2 = models.crystal
mybeam = models.beam
# Build a mock scan for an 18 degree sequence
sf = ScanFactory()
myscan = sf.make_scan(
image_range=(1, 180),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(180)),
deg=True,
)
sequence_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sequence_range == (0.0, pi / 10)
assert approx_equal(im_width, 0.1 * pi / 180.0)
# Build an experiment list
experiments = ExperimentList()
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal1,
imageset=None,
))
experiments.append(
Experiment(
beam=mybeam,
detector=mydetector,
goniometer=mygonio,
scan=myscan,
crystal=crystal2,
imageset=None,
))
assert len(experiments.detectors()) == 1
##########################################################
# Parameterise the models (only for perturbing geometry) #
##########################################################
det_param = DetectorParameterisationSinglePanel(mydetector)
s0_param = BeamParameterisation(mybeam, mygonio)
xl1o_param = CrystalOrientationParameterisation(crystal1)
xl1uc_param = CrystalUnitCellParameterisation(crystal1)
xl2o_param = CrystalOrientationParameterisation(crystal2)
xl2uc_param = CrystalUnitCellParameterisation(crystal2)
# Fix beam to the X-Z plane (imgCIF geometry), fix wavelength
s0_param.set_fixed([True, False, True])
################################
# Apply known parameter shifts #
################################
# shift detector by 1.0 mm each translation and 2 mrad each rotation
det_p_vals = det_param.get_param_vals()
p_vals = [
a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 2.0, 2.0, 2.0])
]
det_param.set_param_vals(p_vals)
# shift beam by 2 mrad in free axis
s0_p_vals = s0_param.get_param_vals()
p_vals = list(s0_p_vals)
p_vals[0] += 2.0
s0_param.set_param_vals(p_vals)
# rotate crystal a bit (=2 mrad each rotation)
xlo_p_vals = []
for xlo in (xl1o_param, xl2o_param):
p_vals = xlo.get_param_vals()
xlo_p_vals.append(p_vals)
new_p_vals = [a + b for a, b in zip(p_vals, [2.0, 2.0, 2.0])]
xlo.set_param_vals(new_p_vals)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# gamma angle)
xluc_p_vals = []
for xluc, xl in ((xl1uc_param, crystal1), ((xl2uc_param, crystal2))):
p_vals = xluc.get_param_vals()
xluc_p_vals.append(p_vals)
cell_params = xl.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, 0.1, 0.1, 0.0, 0.0, 0.1])
]
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(xl.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc.set_param_vals(X)
#############################
# Generate some reflections #
#############################
# All indices in a 2.5 Angstrom sphere for crystal1
resolution = 2.5
index_generator = IndexGenerator(
crystal1.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices1 = index_generator.to_array()
# All indices in a 2.5 Angstrom sphere for crystal2
resolution = 2.5
index_generator = IndexGenerator(
crystal2.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices2 = index_generator.to_array()
# Predict rays within the sequence range. Set experiment IDs
ray_predictor = ScansRayPredictor(experiments, sequence_range)
obs_refs1 = ray_predictor(indices1, experiment_id=0)
obs_refs1["id"] = flex.int(len(obs_refs1), 0)
obs_refs2 = ray_predictor(indices2, experiment_id=1)
obs_refs2["id"] = flex.int(len(obs_refs2), 1)
# Take only those rays that intersect the detector
intersects = ray_intersection(mydetector, obs_refs1)
obs_refs1 = obs_refs1.select(intersects)
intersects = ray_intersection(mydetector, obs_refs2)
obs_refs2 = obs_refs2.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ScansExperimentsPredictor(experiments)
obs_refs1 = ref_predictor(obs_refs1)
obs_refs2 = ref_predictor(obs_refs2)
# Set 'observed' centroids from the predicted ones
obs_refs1["xyzobs.mm.value"] = obs_refs1["xyzcal.mm"]
obs_refs2["xyzobs.mm.value"] = obs_refs2["xyzcal.mm"]
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 18.0
px_size = mydetector[0].get_pixel_size()
var_x = flex.double(len(obs_refs1), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs1), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs1), (im_width / 2.0)**2)
obs_refs1["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
var_x = flex.double(len(obs_refs2), (px_size[0] / 2.0)**2)
var_y = flex.double(len(obs_refs2), (px_size[1] / 2.0)**2)
var_phi = flex.double(len(obs_refs2), (im_width / 2.0)**2)
obs_refs2["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi)
# concatenate reflection lists
obs_refs1.extend(obs_refs2)
obs_refs = obs_refs1
###############################
# Undo known parameter shifts #
###############################
s0_param.set_param_vals(s0_p_vals)
det_param.set_param_vals(det_p_vals)
xl1o_param.set_param_vals(xlo_p_vals[0])
xl2o_param.set_param_vals(xlo_p_vals[1])
xl1uc_param.set_param_vals(xluc_p_vals[0])
xl2uc_param.set_param_vals(xluc_p_vals[1])
# scan static first
params = phil_scope.fetch(source=parse("")).extract()
refiner = RefinerFactory.from_parameters_data_experiments(
params, obs_refs, experiments)
refiner.run()
# scan varying
params.refinement.parameterisation.scan_varying = True
refiner = RefinerFactory.from_parameters_data_experiments(
params, obs_refs, experiments)
refiner.run()
# Ensure all models have scan-varying state set
# (https://github.com/dials/dials/issues/798)
refined_experiments = refiner.get_experiments()
sp = [xl.get_num_scan_points() for xl in refined_experiments.crystals()]
assert sp.count(181) == 2
