def create_refiner(params, reflections, experiments):
# Only parameterise the crystal unit cell
det_params = None
beam_params = None
xlo_params = None
xluc_params = []
for crystal in experiments.crystals():
exp_ids = experiments.indices(crystal)
xluc_params.append(
CrystalUnitCellParameterisation(crystal, experiment_ids=exp_ids)
)
# Two theta prediction equation parameterisation
pred_param = TwoThetaPredictionParameterisation(
experiments, det_params, beam_params, xlo_params, xluc_params
)
param_reporter = ParameterReporter(
det_params, beam_params, xlo_params, xluc_params
)
# ReflectionManager, currently without outlier rejection
# Note: If not all reflections are used, then the filtering must be
# communicated to generate_cif/mmcif() to be included in the CIF file!
refman = TwoThetaReflectionManager(
reflections, experiments, outlier_detector=None
)
# Reflection predictor
ref_predictor = TwoThetaExperimentsPredictor(experiments)
# Two theta target
target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param)
# Switch on correlation matrix tracking if a correlation plot is requested
journal = None
if params.output.correlation_plot.filename is not None:
journal = refinery_phil_scope.extract().refinery.journal
journal.track_parameter_correlation = True
# Minimisation engine - hardcoded to LevMar for now.
refinery = Refinery(
target=target,
prediction_parameterisation=pred_param,
log=None,
tracking=journal,
max_iterations=20,
)
# Refiner
refiner = Refiner(
experiments=experiments,
pred_param=pred_param,
param_reporter=param_reporter,
refman=refman,
target=target,
refinery=refinery,
)
return refiner
def config_parameterisation(params, experiments, refman, do_stills=False):
from dials.algorithms.refinement.parameterisation import (
build_prediction_parameterisation, )
pred_param = build_prediction_parameterisation(params, experiments,
refman, do_stills)
# Parameter reporting
from dials.algorithms.refinement.parameterisation.parameter_report import (
ParameterReporter, )
param_reporter = ParameterReporter(
pred_param.get_detector_parameterisations(),
pred_param.get_beam_parameterisations(),
pred_param.get_crystal_orientation_parameterisations(),
pred_param.get_crystal_unit_cell_parameterisations(),
pred_param.get_goniometer_parameterisations(),
)
return pred_param, param_reporter
def test_refinement(dials_regression):
"""Test a refinement run"""
# Get a beam and detector from a experiments. This one has a CS-PAD, but that
# is irrelevant
data_dir = os.path.join(dials_regression, "refinement_test_data",
"hierarchy_test")
experiments_path = os.path.join(data_dir, "datablock.json")
assert os.path.exists(experiments_path)
# load models
from dxtbx.model.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_serialized_format(
experiments_path, check_format=False)
im_set = experiments.imagesets()[0]
detector = deepcopy(im_set.get_detector())
beam = im_set.get_beam()
# Invent a crystal, goniometer and scan for this test
from dxtbx.model import Crystal
crystal = Crystal((40.0, 0.0, 0.0), (0.0, 40.0, 0.0), (0.0, 0.0, 40.0),
space_group_symbol="P1")
orig_xl = deepcopy(crystal)
from dxtbx.model import GoniometerFactory
goniometer = GoniometerFactory.known_axis((1.0, 0.0, 0.0))
# Build a mock scan for a 180 degree sequence
from dxtbx.model import ScanFactory
sf = ScanFactory()
scan = sf.make_scan(
image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(1800)),
deg=True,
)
sequence_range = scan.get_oscillation_range(deg=False)
im_width = scan.get_oscillation(deg=False)[1]
assert sequence_range == (0.0, pi)
assert approx_equal(im_width, 0.1 * pi / 180.0)
# Build an experiment list
experiments = ExperimentList()
experiments.append(
Experiment(
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
crystal=crystal,
imageset=None,
))
# simulate some reflections
refs, _ = generate_reflections(experiments)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# alpha and beta angles)
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation, )
xluc_param = CrystalUnitCellParameterisation(crystal)
cell_params = crystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1, -0.1, 0.0])
]
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge
from scitbx import matrix
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(crystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
# reparameterise the crystal at the perturbed geometry
xluc_param = CrystalUnitCellParameterisation(crystal)
# Dummy parameterisations for other models
beam_param = None
xlo_param = None
det_param = None
# parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.parameter_report import (
ParameterReporter, )
pred_param = TwoThetaPredictionParameterisation(experiments, det_param,
beam_param, xlo_param,
[xluc_param])
param_reporter = ParameterReporter(det_param, beam_param, xlo_param,
[xluc_param])
# reflection manager
refman = TwoThetaReflectionManager(refs, experiments, nref_per_degree=20)
# reflection predictor
ref_predictor = TwoThetaExperimentsPredictor(experiments)
# target function
target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param)
# minimisation engine
from dials.algorithms.refinement.engine import (
LevenbergMarquardtIterations as Refinery, )
refinery = Refinery(
target=target,
prediction_parameterisation=pred_param,
log=None,
max_iterations=20,
)
# Refiner
from dials.algorithms.refinement.refiner import Refiner
refiner = Refiner(
experiments=experiments,
pred_param=pred_param,
param_reporter=param_reporter,
refman=refman,
target=target,
refinery=refinery,
)
refiner.run()
# compare crystal with original crystal
refined_xl = refiner.get_experiments()[0].crystal
# print refined_xl
assert refined_xl.is_similar_to(orig_xl,
uc_rel_length_tolerance=0.001,
uc_abs_angle_tolerance=0.01)
def _build_components(cls, params, reflections, experiments):
"""low level build"""
# Currently a refinement job can only have one parameterisation of the
# prediction equation. This can either be of the XYDelPsi (stills) type, the
# XYPhi (scans) type or the scan-varying XYPhi type with a varying crystal
# model
single_as_still = params.refinement.parameterisation.treat_single_image_as_still
exps_are_stills = []
for exp in experiments:
if exp.scan is None:
exps_are_stills.append(True)
elif exp.scan.get_num_images() == 1:
if single_as_still:
exps_are_stills.append(True)
elif exp.scan.is_still():
exps_are_stills.append(True)
else:
exps_are_stills.append(False)
else:
if exp.scan.get_oscillation()[1] <= 0.0:
raise DialsRefineConfigError(
"Cannot refine a zero-width scan")
exps_are_stills.append(False)
# check experiment types are consistent
if not all(exps_are_stills[0] == e for e in exps_are_stills):
raise DialsRefineConfigError(
"Cannot refine a mixture of stills and scans")
do_stills = exps_are_stills[0]
# If experiments are stills, ensure scan-varying refinement won't be attempted
if do_stills:
params.refinement.parameterisation.scan_varying = False
# Refiner does not accept scan_varying=Auto. This is a special case for
# doing macrocycles of refinement in dials.refine.
if params.refinement.parameterisation.scan_varying is libtbx.Auto:
params.refinement.parameterisation.scan_varying = False
# Trim scans and calculate reflection block_width if required for scan-varying refinement
if (params.refinement.parameterisation.scan_varying and
params.refinement.parameterisation.trim_scan_to_observations):
experiments = _trim_scans_to_observations(experiments, reflections)
from dials.algorithms.refinement.reflection_manager import BlockCalculator
block_calculator = BlockCalculator(experiments, reflections)
if params.refinement.parameterisation.compose_model_per == "block":
reflections = block_calculator.per_width(
params.refinement.parameterisation.block_width, deg=True)
elif params.refinement.parameterisation.compose_model_per == "image":
reflections = block_calculator.per_image()
logger.debug("\nBuilding reflection manager")
logger.debug("Input reflection list size = %d observations",
len(reflections))
# create reflection manager
refman = ReflectionManagerFactory.from_parameters_reflections_experiments(
params.refinement.reflections, reflections, experiments, do_stills)
logger.debug(
"Number of observations that pass initial inclusion criteria = %d",
refman.get_accepted_refs_size(),
)
sample_size = refman.get_sample_size()
if sample_size > 0:
logger.debug("Working set size = %d observations", sample_size)
logger.debug("Reflection manager built\n")
# configure use of sparse data types
params = cls.config_sparse(params, experiments)
do_sparse = params.refinement.parameterisation.sparse
# create managed reflection predictor
ref_predictor = ExperimentsPredictorFactory.from_experiments(
experiments,
force_stills=do_stills,
spherical_relp=params.refinement.parameterisation.
spherical_relp_model,
)
# Predict for the managed observations, set columns for residuals and set
# the used_in_refinement flag to the predictions
obs = refman.get_obs()
ref_predictor(obs)
x_obs, y_obs, phi_obs = obs["xyzobs.mm.value"].parts()
x_calc, y_calc, phi_calc = obs["xyzcal.mm"].parts()
obs["x_resid"] = x_calc - x_obs
obs["y_resid"] = y_calc - y_obs
obs["phi_resid"] = phi_calc - phi_obs
# determine whether to do basic centroid analysis to automatically
# determine outlier rejection block
if params.refinement.reflections.outlier.block_width is libtbx.Auto:
ca = refman.get_centroid_analyser()
analysis = ca(calc_average_residuals=False,
calc_periodograms=False)
else:
analysis = None
# Now predictions and centroid analysis are available, so we can finalise
# the reflection manager
refman.finalise(analysis)
# Create model parameterisations
logger.debug("Building prediction equation parameterisation")
pred_param = build_prediction_parameterisation(
params.refinement.parameterisation, experiments, refman, do_stills)
# Build a constraints manager, if requested
cmf = ConstraintManagerFactory(params, pred_param)
constraints_manager = cmf()
# Test for parameters that have too little data to refine and act accordingly
autoreduce = AutoReduce(
params.refinement.parameterisation.auto_reduction,
pred_param,
refman,
constraints_manager,
cmf,
)
autoreduce()
# if reduction was done, constraints_manager will have changed
constraints_manager = autoreduce.constraints_manager
# Build a restraints parameterisation (if requested).
# Only unit cell restraints are supported at the moment.
restraints_parameterisation = cls.config_restraints(
params.refinement.parameterisation, pred_param)
# Parameter reporting
logger.debug("Prediction equation parameterisation built")
logger.debug("Parameter order : name mapping")
for i, e in enumerate(pred_param.get_param_names()):
logger.debug("Parameter %03d : %s", i + 1, e)
param_reporter = ParameterReporter(
pred_param.get_detector_parameterisations(),
pred_param.get_beam_parameterisations(),
pred_param.get_crystal_orientation_parameterisations(),
pred_param.get_crystal_unit_cell_parameterisations(),
pred_param.get_goniometer_parameterisations(),
)
# Create target function
logger.debug("Building target function")
target = cls.config_target(
params.refinement.target,
experiments,
refman,
ref_predictor,
pred_param,
restraints_parameterisation,
do_stills,
do_sparse,
)
logger.debug("Target function built")
# create refinery
logger.debug("Building refinement engine")
refinery = cls.config_refinery(params, target, pred_param,
constraints_manager)
logger.debug("Refinement engine built")
nparam = len(pred_param)
ndim = target.dim
nref = len(refman.get_matches())
logger.info(
"There are %s parameters to refine against %s reflections in %s dimensions",
nparam,
nref,
ndim,
)
if not params.refinement.parameterisation.sparse and isinstance(
refinery, AdaptLstbx):
dense_jacobian_gigabytes = (nparam * nref * ndim *
flex.double.element_size()) / 1e9
avail_memory_gigabytes = psutil.virtual_memory().available / 1e9
# Report if the Jacobian requires a large amount of storage
if (dense_jacobian_gigabytes > 0.2 * avail_memory_gigabytes
or dense_jacobian_gigabytes > 0.5):
logger.info(
"Storage of the Jacobian matrix requires %.1f GB",
dense_jacobian_gigabytes,
)
# build refiner interface and return
if params.refinement.parameterisation.scan_varying:
refiner = ScanVaryingRefiner
else:
refiner = Refiner
return refiner(experiments, pred_param, param_reporter, refman, target,
refinery)
def test1():
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression", test=os.path.isdir)
# use a datablock that contains a CS-PAD detector description
data_dir = os.path.join(dials_regression, "refinement_test_data",
"hierarchy_test")
datablock_path = os.path.join(data_dir, "datablock.json")
assert os.path.exists(datablock_path)
# load models
from dxtbx.datablock import DataBlockFactory
datablock = DataBlockFactory.from_serialized_format(datablock_path,
check_format=False)
im_set = datablock[0].extract_imagesets()[0]
from copy import deepcopy
detector = deepcopy(im_set.get_detector())
beam = im_set.get_beam()
# we'll invent a crystal, goniometer and scan for this test
from dxtbx.model import Crystal
crystal = Crystal((40., 0., 0.), (0., 40., 0.), (0., 0., 40.),
space_group_symbol="P1")
from dxtbx.model import GoniometerFactory
goniometer = GoniometerFactory.known_axis((1., 0., 0.))
# Build a mock scan for a 180 degree sweep
from dxtbx.model import ScanFactory
sf = ScanFactory()
scan = sf.make_scan(image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=range(1800),
deg=True)
sweep_range = scan.get_oscillation_range(deg=False)
im_width = scan.get_oscillation(deg=False)[1]
assert sweep_range == (0., pi)
assert approx_equal(im_width, 0.1 * pi / 180.)
from dxtbx.model.experiment_list import ExperimentList, Experiment
# Build an experiment list
experiments = ExperimentList()
experiments.append(
Experiment(beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
crystal=crystal,
imageset=None))
# simulate some reflections
refs, ref_predictor = generate_reflections(experiments)
# move the detector quadrants apart by 2mm both horizontally and vertically
from dials.algorithms.refinement.parameterisation \
import DetectorParameterisationHierarchical
det_param = DetectorParameterisationHierarchical(detector, level=1)
det_p_vals = det_param.get_param_vals()
p_vals = list(det_p_vals)
p_vals[1] += 2
p_vals[2] -= 2
p_vals[7] += 2
p_vals[8] += 2
p_vals[13] -= 2
p_vals[14] += 2
p_vals[19] -= 2
p_vals[20] -= 2
det_param.set_param_vals(p_vals)
# reparameterise the detector at the new perturbed geometry
det_param = DetectorParameterisationHierarchical(detector, level=1)
# parameterise other models
from dials.algorithms.refinement.parameterisation.beam_parameters import \
BeamParameterisation
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalOrientationParameterisation, CrystalUnitCellParameterisation
beam_param = BeamParameterisation(beam, goniometer)
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
# fix beam
beam_param.set_fixed([True] * 3)
# fix crystal
xluc_param.set_fixed([True] * 6)
xlo_param.set_fixed([True] * 3)
# parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import \
XYPhiPredictionParameterisation
from dials.algorithms.refinement.parameterisation.parameter_report import \
ParameterReporter
pred_param = XYPhiPredictionParameterisation(experiments, [det_param],
[beam_param], [xlo_param],
[xluc_param])
param_reporter = ParameterReporter([det_param], [beam_param], [xlo_param],
[xluc_param])
# reflection manager and target function
from dials.algorithms.refinement.target import \
LeastSquaresPositionalResidualWithRmsdCutoff
from dials.algorithms.refinement.reflection_manager import ReflectionManager
refman = ReflectionManager(refs, experiments, nref_per_degree=20)
# set a very tight rmsd target of 1/10000 of a pixel
target = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments,
ref_predictor,
refman,
pred_param,
restraints_parameterisation=None,
frac_binsize_cutoff=0.0001)
# minimisation engine
from dials.algorithms.refinement.engine \
import LevenbergMarquardtIterations as Refinery
refinery = Refinery(target=target,
prediction_parameterisation=pred_param,
log=None,
verbosity=0,
max_iterations=20)
# Refiner
from dials.algorithms.refinement.refiner import Refiner
refiner = Refiner(reflections=refs,
experiments=experiments,
pred_param=pred_param,
param_reporter=param_reporter,
refman=refman,
target=target,
refinery=refinery,
verbosity=0)
history = refiner.run()
assert history.reason_for_termination == "RMSD target achieved"
#compare detector with original detector
orig_det = im_set.get_detector()
refined_det = refiner.get_experiments()[0].detector
from scitbx import matrix
import math
for op, rp in zip(orig_det, refined_det):
# compare the origin vectors by...
o1 = matrix.col(op.get_origin())
o2 = matrix.col(rp.get_origin())
# ...their relative lengths
assert approx_equal(math.fabs(o1.length() - o2.length()) / o1.length(),
0,
eps=1e-5)
# ...the angle between them
assert approx_equal(o1.accute_angle(o2), 0, eps=1e-5)
print "OK"
return
def create_refiner(params, reflections, experiments):
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalUnitCellParameterisation
from dials.algorithms.refinement.parameterisation.parameter_report import \
ParameterReporter
from dials.algorithms.refinement.two_theta_refiner import (
TwoThetaReflectionManager, TwoThetaTarget,
TwoThetaExperimentsPredictor, TwoThetaPredictionParameterisation)
verb = params.refinement.verbosity
# Only parameterise the crystal unit cell
det_params = None
beam_params = None
xlo_params = None
xluc_params = []
for icrystal, crystal in enumerate(experiments.crystals()):
exp_ids = experiments.indices(crystal)
xluc_params.append(
CrystalUnitCellParameterisation(crystal,
experiment_ids=exp_ids))
# Two theta prediction equation parameterisation
pred_param = TwoThetaPredictionParameterisation(
experiments, det_params, beam_params, xlo_params, xluc_params)
param_reporter = ParameterReporter(det_params, beam_params, xlo_params,
xluc_params)
# ReflectionManager, currently without outlier rejection
# Note: If not all reflections are used, then the filtering must be
# communicated to generate_cif/mmcif() to be included in the CIF file!
refman = TwoThetaReflectionManager(reflections,
experiments,
outlier_detector=None,
verbosity=verb)
# Reflection predictor
ref_predictor = TwoThetaExperimentsPredictor(experiments)
# Two theta target
target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param)
# Do a correlation plot?
tpc = params.output.correlation_plot.filename is not None
# Minimisation engine - FIXME not many choices exposed yet. Do we want
# more cowbell?
from dials.algorithms.refinement.engine \
import LevenbergMarquardtIterations as Refinery
refinery = Refinery(target=target,
prediction_parameterisation=pred_param,
log=None,
verbosity=verb,
track_step=False,
track_gradient=False,
track_parameter_correlation=tpc,
max_iterations=20)
# Refiner
from dials.algorithms.refinement.refiner import Refiner
refiner = Refiner(reflections=reflections,
experiments=experiments,
pred_param=pred_param,
param_reporter=param_reporter,
refman=refman,
target=target,
refinery=refinery,
verbosity=verb)
return refiner
def create_refiner(params, reflections, experiments):
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalUnitCellParameterisation
from dials.algorithms.refinement.parameterisation.parameter_report import \
ParameterReporter
from dials.algorithms.refinement.two_theta_refiner import \
TwoThetaReflectionManager, TwoThetaTarget, TwoThetaExperimentsPredictor, \
TwoThetaPredictionParameterisation
verb = params.refinement.verbosity
# Only parameterise the crystal unit cell
det_params = None
beam_params = None
xlo_params = None
xluc_params = []
for icrystal, crystal in enumerate(experiments.crystals()):
exp_ids = experiments.indices(crystal)
xluc_params.append(
CrystalUnitCellParameterisation(crystal,
experiment_ids=exp_ids))
# Two theta prediction equation parameterisation
pred_param = TwoThetaPredictionParameterisation(
experiments, det_params, beam_params, xlo_params, xluc_params)
param_reporter = ParameterReporter(det_params, beam_params, xlo_params,
xluc_params)
# ReflectionManager, currently without outlier rejection
# Note: If not all reflections are used, then the filtering must be
# communicated to generate_cif/mmcif() to be included in the CIF file!
refman = TwoThetaReflectionManager(reflections,
experiments,
outlier_detector=None,
verbosity=verb)
# Reflection predictor
ref_predictor = TwoThetaExperimentsPredictor(experiments)
# Two theta target
target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param)
# Switch on correlation matrix tracking if a correlation plot is requested
journal = None
if params.output.correlation_plot.filename is not None:
from dials.algorithms.refinement.engine import refinery_phil_scope
journal = refinery_phil_scope.extract().refinery.journal
journal.track_parameter_correlation = True
# Minimisation engine - hardcoded to LevMar for now.
from dials.algorithms.refinement.engine \
import LevenbergMarquardtIterations as Refinery
refinery = Refinery(target=target,
prediction_parameterisation=pred_param,
log=None,
verbosity=verb,
tracking=journal,
max_iterations=20)
# Refiner
from dials.algorithms.refinement.refiner import Refiner
refiner = Refiner(reflections=reflections,
experiments=experiments,
pred_param=pred_param,
param_reporter=param_reporter,
refman=refman,
target=target,
refinery=refinery,
verbosity=verb)
return refiner
