def predict_reflections(self):
from dials.algorithms import shoebox
from dials.array_family import flex
from dxtbx.model.experiment.experiment_list import ExperimentList
from dxtbx.model.experiment.experiment_list import Experiment
from dials.algorithms.profile_model.gaussian_rs import Model
# Get models from the sweep
self.beam = self.sweep.get_beam()
self.detector = self.sweep.get_detector()
self.gonio = self.sweep.get_goniometer()
self.scan = self.sweep.get_scan()
sigma_b = self.beam.get_sigma_divergence(deg=True)
sigma_m = self.crystal.get_mosaicity(deg=True)
exlist = ExperimentList()
exlist.append(
Experiment(imageset=self.sweep,
beam=self.beam,
detector=self.detector,
goniometer=self.gonio,
scan=self.scan,
crystal=self.crystal,
profile=Model(None, 3, sigma_b, sigma_m, deg=True)))
predicted = flex.reflection_table.from_predictions(exlist[0])
predicted['id'] = flex.int(len(predicted), 0)
predicted.compute_bbox(exlist)
# Find overlapping reflections
overlaps = shoebox.find_overlapping(predicted['bbox'])
# Return the reflections and overlaps
return predicted, overlaps
def experiment_list_for_crystal(self, crystal):
experiments = ExperimentList()
for imageset in self.imagesets:
experiments.append(
Experiment(imageset=imageset,
beam=imageset.get_beam(),
detector=imageset.get_detector(),
goniometer=imageset.get_goniometer(),
scan=imageset.get_scan(),
crystal=crystal))
return experiments
def __init__(self, experiment, reflections, expected_miller_indices):
from dials.algorithms.indexing \
import index_reflections, index_reflections_local
import copy
# index reflections using simple "global" method
self.reflections_global = copy.deepcopy(reflections)
self.reflections_global['id'] = flex.int(len(self.reflections_global),
-1)
self.reflections_global['imageset_id'] = flex.int(
len(self.reflections_global), 0)
index_reflections(self.reflections_global,
ExperimentList([experiment]))
non_zero_sel = (self.reflections_global['miller_index'] != (0, 0, 0))
assert self.reflections_global['id'].select(~non_zero_sel).all_eq(-1)
self.misindexed_global = (
expected_miller_indices == self.reflections_global['miller_index']
).select(non_zero_sel).count(False)
self.correct_global = (expected_miller_indices == self.
reflections_global['miller_index']).count(True)
# index reflections using xds-style "local" method
self.reflections_local = copy.deepcopy(reflections)
self.reflections_local['id'] = flex.int(len(self.reflections_local),
-1)
index_reflections_local(self.reflections_local,
ExperimentList([experiment]))
non_zero_sel = (self.reflections_local['miller_index'] != (0, 0, 0))
assert self.reflections_local['id'].select(~non_zero_sel).all_eq(-1)
self.misindexed_local = (
expected_miller_indices == self.reflections_local['miller_index']
).select(non_zero_sel).count(False)
self.correct_local = (expected_miller_indices == self.
reflections_local['miller_index']).count(True)
print self.misindexed_global, self.correct_global, len(
self.reflections_global)
print self.misindexed_local, self.correct_local, len(
self.reflections_local)
def run(self):
'''Execute the script.'''
from dials.util.options import flatten_reflections, flatten_experiments
from libtbx.utils import Sorry
from dials.array_family import flex
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
# Try to load the models and data
if not params.input.experiments:
print "No Experiments found in the input"
self.parser.print_help()
return
if params.input.reflections:
if len(params.input.reflections) != len(params.input.experiments):
raise Sorry(
"The number of input reflections files does not match the "
"number of input experiments")
experiments = flatten_experiments(params.input.experiments)
if params.input.reflections:
reflections = flatten_reflections(params.input.reflections)[0]
else:
reflections = None
import math
experiments_template = "%s_%%0%sd.json" % (
params.output.experiments_prefix,
int(math.floor(math.log10(len(experiments))) + 1))
reflections_template = "%s_%%0%sd.pickle" % (
params.output.reflections_prefix,
int(math.floor(math.log10(len(experiments))) + 1))
for i, experiment in enumerate(experiments):
from dxtbx.model.experiment.experiment_list import ExperimentList
from dxtbx.serialize import dump
experiment_filename = experiments_template % i
print 'Saving experiment %d to %s' % (i, experiment_filename)
dump.experiment_list(ExperimentList([experiment]),
experiment_filename)
if reflections is not None:
reflections_filename = reflections_template % i
print 'Saving reflections for experiment %d to %s' % (
i, reflections_filename)
ref_sel = reflections.select(reflections['id'] == i)
ref_sel['id'] = flex.int(len(ref_sel), 0)
ref_sel.as_pickle(reflections_filename)
return
def find_lattices(self):
experiments = ExperimentList()
for cm in self.known_orientations:
# indexer expects crystals to be in primitive setting
space_group = cm.get_space_group()
cb_op_to_primitive \
= space_group.info().change_of_basis_op_to_primitive_setting()
cm = cm.change_basis(cb_op_to_primitive)
for imageset in self.imagesets:
experiments.append(
Experiment(imageset=imageset,
beam=imageset.get_beam(),
detector=imageset.get_detector(),
goniometer=imageset.get_goniometer(),
scan=imageset.get_scan(),
crystal=cm))
return experiments
def create_models(self, cmdline_overrides=None):
if cmdline_overrides is None:
cmdline_overrides = []
overrides = """geometry.parameters.crystal.a.length.range = 10 50
geometry.parameters.crystal.b.length.range = 10 50
geometry.parameters.crystal.c.length.range = 10 50"""
master_phil = parse("""
include scope dials.test.algorithms.refinement.geometry_phil
""", process_includes=True)
# Extract models
models = Extract(master_phil, overrides, cmdline_args = cmdline_overrides)
self.detector = models.detector
self.goniometer = models.goniometer
self.crystal = models.crystal
self.beam = models.beam
# Make a scan of 1-360 * 0.5 deg images
sf = scan_factory()
self.scan = sf.make_scan((1,360), 0.5, (0, 0.5), range(360))
# Generate an ExperimentList
self.experiments = ExperimentList()
self.experiments.append(Experiment(
beam=self.beam, detector=self.detector, goniometer=self.goniometer,
scan=self.scan, crystal=self.crystal, imageset=None))
# Create a reflection predictor for the experiments
self.ref_predictor = ExperimentsPredictor(self.experiments)
# Create scan-varying parameterisations of these models, with 5 samples
self.det_param = ScanVaryingDetectorParameterisationSinglePanel(
self.detector, self.scan.get_array_range(), 5)
self.s0_param = ScanVaryingBeamParameterisation(
self.beam, self.scan.get_array_range(), 5, self.goniometer)
self.xlo_param = ScanVaryingCrystalOrientationParameterisation(
self.crystal, self.scan.get_array_range(), 5)
self.xluc_param = ScanVaryingCrystalUnitCellParameterisation(
self.crystal, self.scan.get_array_range(), 5)
return
def find_lattices(self):
self.d_min = self.params.refinement_protocol.d_min_start
from rstbx.phil.phil_preferences import indexing_api_defs
import iotbx.phil
hardcoded_phil = iotbx.phil.parse(
input_string=indexing_api_defs).extract()
sel = (self.reflections['id'] == -1)
if self.d_min is not None:
sel &= (1 / self.reflections['rlp'].norms() > self.d_min)
reflections = self.reflections.select(sel)
solutions = candidate_basis_vectors_fft1d(
reflections['rlp'], hardcoded_phil, max_cell=self.params.max_cell)
self.candidate_basis_vectors = solutions[0]
self.debug_show_candidate_basis_vectors()
if self.params.debug_plots:
self.debug_plot_candidate_basis_vectors()
self.candidate_crystal_models = self.find_candidate_orientation_matrices(
self.candidate_basis_vectors,
max_combinations=self.params.basis_vector_combinations.max_try)
crystal_model, n_indexed = self.choose_best_orientation_matrix(
self.candidate_crystal_models)
if crystal_model is not None:
crystal_models = [crystal_model]
else:
crystal_models = []
experiments = ExperimentList()
for cm in crystal_models:
for imageset in self.imagesets:
experiments.append(
Experiment(imageset=imageset,
beam=imageset.get_beam(),
detector=imageset.get_detector(),
goniometer=imageset.get_goniometer(),
scan=imageset.get_scan(),
crystal=cm))
return experiments
def __call__(self, experiments, reflections):
self.working_phil.show()
params = self.working_phil.extract()
for iexp, exp in enumerate(experiments):
print("Refining crystal", iexp)
# reflection subset for a single experiment
refs = reflections.select(reflections["id"] == iexp)
refs["id"] = flex.size_t(len(refs), 0)
# experiment list for a single experiment
exps = ExperimentList()
exps.append(exp)
refiner = RefinerFactory.from_parameters_data_experiments(
params, refs, exps, verbosity=1
)
# do refinement
refiner.run()
refined_exps = refiner.get_experiments()
# replace this experiment with the refined one
experiments[iexp] = refined_exps[0]
return experiments
def test1():
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression", test=os.path.isdir)
# use a datablock that contains a CS-PAD detector description
data_dir = os.path.join(dials_regression, "refinement_test_data",
"hierarchy_test")
datablock_path = os.path.join(data_dir, "datablock.json")
assert os.path.exists(datablock_path)
# load models
from dxtbx.datablock import DataBlockFactory
datablock = DataBlockFactory.from_serialized_format(datablock_path,
check_format=False)
im_set = datablock[0].extract_imagesets()[0]
from copy import deepcopy
detector = deepcopy(im_set.get_detector())
beam = im_set.get_beam()
# we'll invent a crystal, goniometer and scan for this test
from dxtbx.model.crystal import crystal_model
crystal = crystal_model((40., 0., 0.), (0., 40., 0.), (0., 0., 40.),
space_group_symbol="P1")
from dxtbx.model.experiment import goniometer_factory
goniometer = goniometer_factory.known_axis((1., 0., 0.))
# Build a mock scan for a 180 degree sweep
from dxtbx.model.scan import scan_factory
sf = scan_factory()
scan = sf.make_scan(image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=range(1800),
deg=True)
sweep_range = scan.get_oscillation_range(deg=False)
im_width = scan.get_oscillation(deg=False)[1]
assert sweep_range == (0., pi)
assert approx_equal(im_width, 0.1 * pi / 180.)
from dxtbx.model.experiment.experiment_list import ExperimentList, Experiment
# Build an experiment list
experiments = ExperimentList()
experiments.append(
Experiment(beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
crystal=crystal,
imageset=None))
# simulate some reflections
refs, ref_predictor = generate_reflections(experiments)
# move the detector quadrants apart by 2mm both horizontally and vertically
from dials.algorithms.refinement.parameterisation \
import DetectorParameterisationHierarchical
det_param = DetectorParameterisationHierarchical(detector, level=1)
det_p_vals = det_param.get_param_vals()
p_vals = list(det_p_vals)
p_vals[1] += 2
p_vals[2] -= 2
p_vals[7] += 2
p_vals[8] += 2
p_vals[13] -= 2
p_vals[14] += 2
p_vals[19] -= 2
p_vals[20] -= 2
det_param.set_param_vals(p_vals)
# reparameterise the detector at the new perturbed geometry
det_param = DetectorParameterisationHierarchical(detector, level=1)
# parameterise other models
from dials.algorithms.refinement.parameterisation.beam_parameters import \
BeamParameterisation
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalOrientationParameterisation, CrystalUnitCellParameterisation
beam_param = BeamParameterisation(beam, goniometer)
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
# fix beam
beam_param.set_fixed([True] * 3)
# fix crystal
xluc_param.set_fixed([True] * 6)
xlo_param.set_fixed([True] * 3)
# parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.prediction_parameters import \
XYPhiPredictionParameterisation
from dials.algorithms.refinement.parameterisation.parameter_report import \
ParameterReporter
pred_param = XYPhiPredictionParameterisation(experiments, [det_param],
[beam_param], [xlo_param],
[xluc_param])
param_reporter = ParameterReporter([det_param], [beam_param], [xlo_param],
[xluc_param])
# reflection manager and target function
from dials.algorithms.refinement.target import \
LeastSquaresPositionalResidualWithRmsdCutoff
from dials.algorithms.refinement.reflection_manager import ReflectionManager
refman = ReflectionManager(refs, experiments, nref_per_degree=20)
# set a very tight rmsd target of 1/10000 of a pixel
target = LeastSquaresPositionalResidualWithRmsdCutoff(
experiments,
ref_predictor,
refman,
pred_param,
restraints_parameterisation=None,
frac_binsize_cutoff=0.0001)
# minimisation engine
from dials.algorithms.refinement.engine \
import LevenbergMarquardtIterations as Refinery
refinery = Refinery(target=target,
prediction_parameterisation=pred_param,
log=None,
verbosity=0,
track_step=False,
track_gradient=False,
track_parameter_correlation=False,
max_iterations=20)
# Refiner
from dials.algorithms.refinement.refiner import Refiner
refiner = Refiner(reflections=refs,
experiments=experiments,
pred_param=pred_param,
param_reporter=param_reporter,
refman=refman,
target=target,
refinery=refinery,
verbosity=0)
history = refiner.run()
assert history.reason_for_termination == "RMSD target achieved"
#compare detector with original detector
orig_det = im_set.get_detector()
refined_det = refiner.get_experiments()[0].detector
from scitbx import matrix
import math
for op, rp in zip(orig_det, refined_det):
# compare the origin vectors by...
o1 = matrix.col(op.get_origin())
o2 = matrix.col(rp.get_origin())
# ...their relative lengths
assert approx_equal(math.fabs(o1.length() - o2.length()) / o1.length(),
0,
eps=1e-5)
# ...the angle between them
assert approx_equal(o1.accute_angle(o2), 0, eps=1e-5)
print "OK"
return
def run(args):
from dials.util import log
import libtbx.load_env
usage = "%s experiments.json indexed.pickle [options]" % libtbx.env.dispatcher_name
parser = OptionParser(usage=usage,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=False)
# Configure the logging
log.config(info=params.output.log, debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(reflections) == 0 or len(experiments) == 0:
parser.print_help()
return
assert (len(reflections) == 1)
reflections = reflections[0]
if len(experiments) == 0:
parser.print_help()
return
elif len(experiments.crystals()) > 1:
if params.crystal_id is not None:
assert params.crystal_id < len(experiments.crystals())
experiment_ids = experiments.where(
crystal=experiments.crystals()[params.crystal_id])
from dxtbx.model.experiment.experiment_list import ExperimentList
experiments = ExperimentList(
[experiments[i] for i in experiment_ids])
refl_selections = [reflections['id'] == i for i in experiment_ids]
reflections['id'] = flex.int(len(reflections), -1)
for i, sel in enumerate(refl_selections):
reflections['id'].set_selected(sel, i)
reflections = reflections.select(reflections['id'] > -1)
else:
raise Sorry(
"Only one crystal can be processed at a time: set crystal_id to choose experiment."
)
if params.refinement.reflections.outlier.algorithm in ('auto',
libtbx.Auto):
if experiments[0].goniometer is None:
params.refinement.reflections.outlier.algorithm = 'sauter_poon'
else:
# different default to dials.refine
# tukey is faster and more appropriate at the indexing step
params.refinement.reflections.outlier.algorithm = 'tukey'
from dials.algorithms.indexing.symmetry \
import refined_settings_factory_from_refined_triclinic
cb_op_to_primitive = experiments[0].crystal.get_space_group().info()\
.change_of_basis_op_to_primitive_setting()
if experiments[0].crystal.get_space_group().n_ltr() > 1:
effective_group = experiments[0].crystal.get_space_group()\
.build_derived_reflection_intensity_group(anomalous_flag=True)
sys_absent_flags = effective_group.is_sys_absent(
reflections['miller_index'])
reflections = reflections.select(~sys_absent_flags)
experiments[0].crystal.update(
experiments[0].crystal.change_basis(cb_op_to_primitive))
miller_indices = reflections['miller_index']
miller_indices = cb_op_to_primitive.apply(miller_indices)
reflections['miller_index'] = miller_indices
Lfat = refined_settings_factory_from_refined_triclinic(
params,
experiments,
reflections,
lepage_max_delta=params.lepage_max_delta,
nproc=params.nproc,
refiner_verbosity=params.verbosity)
s = StringIO()
possible_bravais_settings = set(solution['bravais'] for solution in Lfat)
bravais_lattice_to_space_group_table(possible_bravais_settings)
Lfat.labelit_printout(out=s)
logger.info(s.getvalue())
from json import dumps
from os.path import join
open(join(params.output.directory, 'bravais_summary.json'),
'wb').write(dumps(Lfat.as_dict()))
from dxtbx.serialize import dump
import copy
for subgroup in Lfat:
expts = copy.deepcopy(experiments)
for expt in expts:
expt.crystal.update(subgroup.refined_crystal)
expt.detector = subgroup.detector
expt.beam = subgroup.beam
dump.experiment_list(
expts,
join(params.output.directory,
'bravais_setting_%i.json' % (int(subgroup.setting_number))))
return
def run(self):
'''Execute the script.'''
from dials.util.options import flatten_reflections, flatten_experiments, \
flatten_datablocks
import cPickle as pickle
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
datablocks = flatten_datablocks(params.input.datablock)
# Try to load the models and data
slice_exps = len(experiments) > 0
slice_refs = len(reflections) > 0
slice_dbs = len(datablocks) > 0
# Catch case of nothing to do
if not any([slice_exps, slice_refs, slice_dbs]):
print "No suitable input provided"
self.parser.print_help()
return
if reflections:
if len(reflections) > 1:
raise Sorry(
"Only one reflections list can be imported at present")
reflections = reflections[0]
# calculate frame numbers if needed
if experiments:
reflections = calculate_frame_numbers(reflections, experiments)
# if we still don't have the right column give up
if 'xyzobs.px.value' not in reflections:
raise Sorry(
"These reflections do not have frame numbers set, and "
"there are no experiments provided to calculate these.")
# set trivial case where no scan range is provided at all
if not params.scan_range:
params.scan_range = [None]
# check if slicing into blocks
if params.block_size is not None:
# in this case for simplicity, ensure that there is either an
# an experiment list or datablocks, but not both. Ensure there is only
# a single scan contained within.
if [slice_exps, slice_dbs].count(True) != 1:
raise Sorry(
"For slicing into blocks please provide either datablocks"
" or experiments, but not both.")
if slice_exps:
if len(experiments) > 1:
raise Sorry(
"For slicing into blocks please provide a single "
"scan only")
scan = experiments[0].scan
if slice_dbs:
scans = datablocks[0].unique_scans()
if len(scans) > 1 or len(datablocks) > 1:
raise Sorry(
"For slicing into blocks please provide a single "
"scan only")
scan = scans[0]
# Having extracted the scan, calculate the blocks
params.scan_range = calculate_block_ranges(scan, params.block_size)
# Do the slicing then recombine
if slice_exps:
sliced = [slice_experiments(experiments, [sr])[0] \
for sr in params.scan_range]
sliced_experiments = ExperimentList()
for exp in sliced:
sliced_experiments.append(exp)
if slice_dbs:
sliced = [slice_datablocks(datablocks, [sr])[0] \
for sr in params.scan_range]
imagesets = [db.extract_imagesets()[0] for db in sliced]
sliced_datablocks = DataBlock(imagesets)
# slice reflections if present
if slice_refs:
sliced = [slice_reflections(reflections, [sr]) \
for sr in params.scan_range]
sliced_reflections = sliced[0]
for i, rt in enumerate(sliced[1:]):
rt['id'] += (i + 1) # set id
sliced_reflections.extend(rt)
else:
# slice each dataset into the requested subset
if slice_exps:
sliced_experiments = slice_experiments(experiments,
params.scan_range)
if slice_refs:
sliced_reflections = slice_reflections(reflections,
params.scan_range)
if slice_dbs:
sliced_datablocks = slice_datablocks(datablocks,
params.scan_range)
# Save sliced experiments
if slice_exps:
output_experiments_filename = params.output.experiments_filename
if output_experiments_filename is None:
# take first filename as template
bname = basename(params.input.experiments[0].filename)
bname = splitext(bname)[0]
if not bname: bname = "experiments"
if len(params.scan_range
) == 1 and params.scan_range[0] is not None:
ext = "_{0}_{1}.json".format(*params.scan_range[0])
else:
ext = "_sliced.json"
output_experiments_filename = bname + ext
print 'Saving sliced experiments to {0}'.format(
output_experiments_filename)
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(sliced_experiments)
dump.as_json(output_experiments_filename)
# Save sliced reflections
if slice_refs:
output_reflections_filename = params.output.reflections_filename
if output_reflections_filename is None:
# take first filename as template
bname = basename(params.input.reflections[0].filename)
bname = splitext(bname)[0]
if not bname: bname = "reflections"
if len(params.scan_range
) == 1 and params.scan_range[0] is not None:
ext = "_{0}_{1}.pickle".format(*params.scan_range[0])
else:
ext = "_sliced.pickle"
output_reflections_filename = bname + ext
print 'Saving sliced reflections to {0}'.format(
output_reflections_filename)
sliced_reflections.as_pickle(output_reflections_filename)
# Save sliced datablocks
if slice_dbs:
output_datablocks_filename = params.output.datablocks_filename
if output_datablocks_filename is None:
# take first filename as template
bname = basename(params.input.datablock[0].filename)
bname = splitext(bname)[0]
if not bname: bname = "datablock"
if len(params.scan_range
) == 1 and params.scan_range[0] is not None:
ext = "_{0}_{1}.json".format(*params.scan_range[0])
else:
ext = "_sliced.json"
output_datablocks_filename = bname + ext
print 'Saving sliced datablocks to {0}'.format(
output_datablocks_filename)
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(sliced_datablocks)
dump.as_file(output_datablocks_filename)
return
multi_panel_detector.add_panel(new_panel)
# Build a mock scan for a 180 degree sweep
sf = scan_factory()
myscan = sf.make_scan(image_range = (1,1800),
exposure_times = 0.1,
oscillation = (0, 0.1),
epochs = range(1800),
deg = True)
sweep_range = myscan.get_oscillation_range(deg=False)
im_width = myscan.get_oscillation(deg=False)[1]
assert sweep_range == (0., pi)
assert approx_equal(im_width, 0.1 * pi / 180.)
# Build ExperimentLists
experiments_single_panel = ExperimentList()
experiments_multi_panel = ExperimentList()
experiments_single_panel.append(Experiment(
beam=mybeam, detector=single_panel_detector, goniometer=mygonio,
scan=myscan, crystal=mycrystal, imageset=None))
experiments_multi_panel.append(Experiment(
beam=mybeam, detector=multi_panel_detector, goniometer=mygonio,
scan=myscan, crystal=mycrystal, imageset=None))
###########################
# Parameterise the models #
###########################
det_param = DetectorParameterisationSinglePanel(single_panel_detector)
s0_param = BeamParameterisation(mybeam, mygonio)
xlo_param = CrystalOrientationParameterisation(mycrystal)
