def refine(self):
# From Aaron Brewster: refinement step skipped as it's done in indexing
# This writes out experiments to disc
if self.phil.output.refined_experiments_filename:
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(self.experiments)
dump.as_json(self.phil.output.refined_experiments_filename)
def tst_dump_empty_sweep(self):
from dxtbx.imageset import ImageSweep, NullReader, SweepFileList
from dxtbx.model import Beam, Detector, Goniometer, Scan
from dxtbx.model.crystal import crystal_model
from uuid import uuid4
imageset = ImageSweep(NullReader(SweepFileList("filename%01d.cbf", (0, 3))))
imageset.set_beam(Beam((1, 0, 0)))
imageset.set_detector(Detector())
imageset.set_goniometer(Goniometer())
imageset.set_scan(Scan((1, 3), (0.0, 1.0)))
crystal = crystal_model((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol=1)
experiments = ExperimentListFactory.from_imageset_and_crystal(
imageset, crystal)
dump = ExperimentListDumper(experiments)
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename)
experiments2 = ExperimentListFactory.from_json_file(filename,
check_format=False)
self.check(experiments, experiments2)
print 'OK'
def refine(self, experiments, centroids):
print "Skipping refinement because the crystal orientation is refined during indexing"
# TODO add dispatch.refine as option and use this code
# from dials.algorithms.refinement import RefinerFactory
# from time import time
# st = time()
#
# logger.info('*' * 80)
# logger.info('Refining Model')
# logger.info('*' * 80)
#
# refiner = RefinerFactory.from_parameters_data_experiments(
# self.params, centroids, experiments)
#
# refiner.run()
# experiments = refiner.get_experiments()
# Dump experiments to disk
if self.params.output.refined_experiments_filename:
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(experiments)
dump.as_json(self.params.output.refined_experiments_filename)
# logger.info('')
# logger.info('Time Taken = %f seconds' % (time() - st))
return experiments
def tst_dump_empty_sweep(self):
from dxtbx.imageset import ImageSweep, NullReader, SweepFileList
from dxtbx.model import Beam, Detector, Goniometer, Scan
from dxtbx.model.crystal import crystal_model
from uuid import uuid4
imageset = ImageSweep(NullReader(SweepFileList("filename%01d.cbf", (0, 3))))
imageset.set_beam(Beam((1, 0, 0)))
imageset.set_detector(Detector())
imageset.set_goniometer(Goniometer())
imageset.set_scan(Scan((1, 3), (0.0, 1.0)))
crystal = crystal_model((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol=1)
experiments = ExperimentListFactory.from_imageset_and_crystal(
imageset, crystal)
dump = ExperimentListDumper(experiments)
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename)
experiments2 = ExperimentListFactory.from_json_file(filename,
check_format=False)
self.check(experiments, experiments2)
print 'OK'
def refine(self, experiments, centroids):
print "Not refining here because the crystal orientation is refined during indexing"
# TODO add dispatch.refine as option and use this code
# from dials.algorithms.refinement import RefinerFactory
# from time import time
# st = time()
#
# logger.info('*' * 80)
# logger.info('Refining Model')
# logger.info('*' * 80)
#
# refiner = RefinerFactory.from_parameters_data_experiments(
# self.params, centroids, experiments)
#
# refiner.run()
# experiments = refiner.get_experiments()
# Dump experiments to disk
if self.params.output.refined_experiments_filename:
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(experiments)
dump.as_json(self.params.output.refined_experiments_filename)
# logger.info('')
# logger.info('Time Taken = %f seconds' % (time() - st))
return experiments
def refine(self):
# From Aaron Brewster: refinement step skipped as it's done in indexing
# This writes out experiments to disc
if self.phil.output.refined_experiments_filename:
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(self.experiments)
dump.as_json(self.phil.output.refined_experiments_filename)
def refine(self, experiments, centroids):
print "Skipping refinement because the crystal orientation is refined during indexing"
# Dump experiments to disk
if self.params.output.refined_experiments_filename:
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(experiments)
dump.as_json(self.params.output.refined_experiments_filename)
return experiments
def save_experiments(self, experiments, filename):
''' Save the profile model parameters. '''
from time import time
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
st = time()
logger.info('Saving the experiments to %s' % filename)
dump = ExperimentListDumper(experiments)
with open(filename, "w") as outfile:
outfile.write(dump.as_json())
logger.info(' time taken: %g' % (time() - st))
def save_experiments(self, experiments, filename):
''' Save the profile model parameters. '''
from time import time
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
st = time()
logger.info('Saving the experiments to %s' % filename)
dump = ExperimentListDumper(experiments)
with open(filename, "w") as outfile:
outfile.write(dump.as_json())
logger.info(' time taken: %g' % (time() - st))
def save_experiments(self, experiments, filename):
""" Save the profile model parameters. """
from logging import info
from time import time
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
st = time()
info("Saving the experiments to %s" % filename)
dump = ExperimentListDumper(experiments)
with open(filename, "w") as outfile:
outfile.write(dump.as_json())
info(" time taken: %g" % (time() - st))
def tst_dump_formats(self):
from uuid import uuid4
from os.path import join
import os
os.environ['DIALS_REGRESSION'] = self.path
# Get all the filenames
filename1 = join(self.path, 'experiment_test_data', 'experiment_1.json')
# Read all the experiment lists in
elist1 = ExperimentListFactory.from_json_file(filename1)
# Create the experiment list dumper
dump = ExperimentListDumper(elist1)
# Dump as JSON file and reload
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename)
elist2 = ExperimentListFactory.from_json_file(filename)
self.check(elist1, elist2)
# Dump as split JSON file and reload
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename, split=True)
elist2 = ExperimentListFactory.from_json_file(filename)
self.check(elist1, elist2)
# Dump as pickle and reload
filename = 'temp%s.pickle' % uuid4().hex
dump.as_pickle(filename)
elist2 = ExperimentListFactory.from_pickle_file(filename)
self.check(elist1, elist2)
def tst_dump_formats(self):
from uuid import uuid4
from os.path import join
import os
os.environ['DIALS_REGRESSION'] = self.path
# Get all the filenames
filename1 = join(self.path, 'experiment_test_data', 'experiment_1.json')
# Read all the experiment lists in
elist1 = ExperimentListFactory.from_json_file(filename1)
# Create the experiment list dumper
dump = ExperimentListDumper(elist1)
# Dump as JSON file and reload
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename)
elist2 = ExperimentListFactory.from_json_file(filename)
self.check(elist1, elist2)
# Dump as split JSON file and reload
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename, split=True)
elist2 = ExperimentListFactory.from_json_file(filename)
self.check(elist1, elist2)
# Dump as pickle and reload
filename = 'temp%s.pickle' % uuid4().hex
dump.as_pickle(filename)
elist2 = ExperimentListFactory.from_pickle_file(filename)
self.check(elist1, elist2)
}
target.gradient_calculation_blocksize=10000
reflections.outlier.algorithm=tukey
refinery.engine=LBFGScurvs
}""")
from dials.data.refinement import phil_scope as master_phil
working_phil = master_phil.fetch(
sources=[user_phil])
working_phil.show()
params = working_phil.extract()
from dials.algorithms.refinement import RefinerFactory
refiner = RefinerFactory.from_parameters_data_experiments(
params, reflections, experiments,
verbosity=2)
refiner.run()
# save the refined experiments
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
experiments = refiner.get_experiments()
dump = ExperimentListDumper(experiments)
experiments_filename = "refined_experiments.json"
dump.as_json(experiments_filename)
print "refined geometry written to {0}".format(experiments_filename)
# save reflections used in refinement
matches = refiner.get_matches()
reflections_filename = "refined_reflections.json"
matches.as_pickle(reflections_filename)
print "reflections used in refinement written to {0}".format(reflections_filename)
filename = join(dials_regression, "centroid_test_data",
"experiments_with_bad_lookup.json")
experiments = ExperimentListFactory.from_json_file(
filename,
check_format=False)
imageset = experiments[0].imageset
assert imageset.external_lookup.mask.data is None
assert imageset.external_lookup.gain.data is None
assert imageset.external_lookup.pedestal.data is None
assert imageset.external_lookup.mask.filename is not None
assert imageset.external_lookup.gain.filename is not None
assert imageset.external_lookup.pedestal.filename is not None
dump = ExperimentListDumper(experiments)
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename)
experiments = ExperimentListFactory.from_json_file(
filename,
check_format=False)
imageset = experiments[0].imageset
assert imageset.external_lookup.mask.data is None
assert imageset.external_lookup.gain.data is None
assert imageset.external_lookup.pedestal.data is None
assert imageset.external_lookup.mask.filename is not None
assert imageset.external_lookup.gain.filename is not None
assert imageset.external_lookup.pedestal.filename is not None
def run(self):
''' Run the script. '''
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.util.command_line import Command
from dials.array_family import flex
from dials.util.options import flatten_reflections, flatten_experiments
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
from libtbx.utils import Sorry
from dials.util import log
log.config()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reflections) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reflections) != 1:
raise Sorry('exactly 1 reflection table must be specified')
if len(experiments) == 0:
raise Sorry('no experiments were specified')
if (not 'background.mean' in reflections[0]) and params.subtract_background:
raise Sorry('for subtract_background need background.mean in reflections')
reflections, _ = self.process_reference(reflections[0], params)
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static)
# Match with predicted
matched, reflections, unmatched = predicted.match_with_reference(reflections)
assert(len(matched) == len(predicted))
assert(matched.count(True) <= len(reflections))
if matched.count(True) == 0:
raise Sorry('''
Invalid input for reference reflections.
Zero reference spots were matched to predictions
''')
elif len(unmatched) != 0:
logger.info('')
logger.info('*' * 80)
logger.info('Warning: %d reference spots were not matched to predictions' % (
len(unmatched)))
logger.info('*' * 80)
logger.info('')
# Create the profile model
experiments = ProfileModelFactory.create(params, experiments, reflections)
for model in experiments:
sigma_b = model.profile.sigma_b(deg=True)
sigma_m = model.profile.sigma_m(deg=True)
if type(sigma_b) == type(1.0):
logger.info('Sigma B: %f' % sigma_b)
logger.info('Sigma M: %f' % sigma_m)
else: # scan varying
mean_sigma_b = sum(sigma_b) / len(sigma_b)
mean_sigma_m = sum(sigma_m) / len(sigma_m)
logger.info('Sigma B: %f' % mean_sigma_b)
logger.info('Sigma M: %f' % mean_sigma_m)
# Wrtie the parameters
Command.start("Writing experiments to %s" % params.output)
dump = ExperimentListDumper(experiments)
with open(params.output, "w") as outfile:
outfile.write(dump.as_json())
Command.end("Wrote experiments to %s" % params.output)
def run(self):
print "Parsing input"
params, options = self.parser.parse_args(show_diff_phil=True)
#Configure the logging
log.config(params.detector_phase.refinement.verbosity,
info='dials.refine.log',
debug='dials.refine.debug.log')
# Try to obtain the models and data
if not params.input.experiments:
raise Sorry("No Experiments found in the input")
if not params.input.reflections:
raise Sorry("No reflection data found in the input")
try:
assert len(params.input.reflections) == len(
params.input.experiments)
except AssertionError:
raise Sorry(
"The number of input reflections files does not match the "
"number of input experiments")
# set up global experiments and reflections lists
from dials.array_family import flex
reflections = flex.reflection_table()
global_id = 0
from dxtbx.model.experiment.experiment_list import ExperimentList
experiments = ExperimentList()
if params.reference_detector == "first":
# Use the first experiment of the first experiment list as the reference detector
ref_exp = params.input.experiments[0].data[0]
else:
# Average all the detectors to generate a reference detector
assert params.detector_phase.refinement.parameterisation.detector.hierarchy_level == 0
from scitbx.matrix import col
panel_fasts = []
panel_slows = []
panel_oris = []
for exp_wrapper in params.input.experiments:
exp = exp_wrapper.data[0]
if panel_oris:
for i, panel in enumerate(exp.detector):
panel_fasts[i] += col(panel.get_fast_axis())
panel_slows[i] += col(panel.get_slow_axis())
panel_oris[i] += col(panel.get_origin())
else:
for i, panel in enumerate(exp.detector):
panel_fasts.append(col(panel.get_fast_axis()))
panel_slows.append(col(panel.get_slow_axis()))
panel_oris.append(col(panel.get_origin()))
ref_exp = copy.deepcopy(params.input.experiments[0].data[0])
for i, panel in enumerate(ref_exp.detector):
# Averaging the fast and slow axes can make them be non-orthagonal. Fix by finding
# the vector that goes exactly between them and rotate
# around their cross product 45 degrees from that vector in either direction
vf = panel_fasts[i] / len(params.input.experiments)
vs = panel_slows[i] / len(params.input.experiments)
c = vf.cross(vs)
angle = vf.angle(vs, deg=True)
v45 = vf.rotate(c, angle / 2, deg=True)
vf = v45.rotate(c, -45, deg=True)
vs = v45.rotate(c, 45, deg=True)
panel.set_frame(vf, vs,
panel_oris[i] / len(params.input.experiments))
print "Reference detector (averaged):", str(ref_exp.detector)
# set the experiment factory that combines a crystal with the reference beam
# and the reference detector
experiment_from_crystal = ExperimentFromCrystal(
ref_exp.beam, ref_exp.detector)
# keep track of the number of refl per accepted experiment for a table
nrefs_per_exp = []
# loop through the input, building up the global lists
for ref_wrapper, exp_wrapper in zip(params.input.reflections,
params.input.experiments):
refs = ref_wrapper.data
exps = exp_wrapper.data
# there might be multiple experiments already here. Loop through them
for i, exp in enumerate(exps):
# select the relevant reflections
sel = refs['id'] == i
sub_ref = refs.select(sel)
## DGW commented out as reflections.minimum_number_of_reflections no longer exists
#if len(sub_ref) < params.crystals_phase.refinement.reflections.minimum_number_of_reflections:
# print "skipping experiment", i, "in", exp_wrapper.filename, "due to insufficient strong reflections in", ref_wrapper.filename
# continue
# build an experiment with this crystal plus the reference models
combined_exp = experiment_from_crystal(exp.crystal)
# next experiment ID in series
exp_id = len(experiments)
# check this experiment
if not check_experiment(combined_exp, sub_ref):
print "skipping experiment", i, "in", exp_wrapper.filename, "due to poor RMSDs"
continue
# set reflections ID
sub_ref['id'] = flex.int(len(sub_ref), exp_id)
# keep number of reflections for the table
nrefs_per_exp.append(len(sub_ref))
# obtain mm positions on the reference detector
sub_ref = indexer_base.map_spots_pixel_to_mm_rad(
sub_ref, combined_exp.detector, combined_exp.scan)
# extend refl and experiments lists
reflections.extend(sub_ref)
experiments.append(combined_exp)
# print number of reflections per accepted experiment
from libtbx.table_utils import simple_table
header = ["Experiment", "Nref"]
rows = [(str(i), str(n)) for (i, n) in enumerate(nrefs_per_exp)]
st = simple_table(rows, header)
print "Number of reflections per experiment"
print st.format()
for cycle in range(params.n_macrocycles):
print "MACROCYCLE %02d" % (cycle + 1)
print "=============\n"
# first run: multi experiment joint refinement of detector with fixed beam and
# crystals
print "PHASE 1"
# SET THIS TEST TO FALSE TO REFINE WHOLE DETECTOR AS SINGLE JOB
if params.detector_phase.refinement.parameterisation.detector.hierarchy_level > 0:
experiments = detector_parallel_refiners(
params.detector_phase, experiments, reflections)
else:
experiments = detector_refiner(params.detector_phase,
experiments, reflections)
# second run
print "PHASE 2"
experiments = crystals_refiner(params.crystals_phase, experiments,
reflections)
# Save the refined experiments to file
output_experiments_filename = params.output.experiments_filename
print 'Saving refined experiments to {0}'.format(
output_experiments_filename)
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(experiments)
dump.as_json(output_experiments_filename)
# Write out refined reflections, if requested
if params.output.reflections_filename:
print 'Saving refined reflections to {0}'.format(
params.output.reflections_filename)
reflections.as_pickle(params.output.reflections_filename)
return
def run(self):
'''Execute the script.'''
from dials.algorithms.refinement.two_theta_refiner import \
TwoThetaReflectionManager, TwoThetaTarget, \
TwoThetaPredictionParameterisation
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
# set up global experiments and reflections lists
from dials.array_family import flex
reflections = flex.reflection_table()
global_id = 0
from dxtbx.model.experiment.experiment_list import ExperimentList
experiments = ExperimentList()
# loop through the input, building up the global lists
nrefs_per_exp = []
for ref_wrapper, exp_wrapper in zip(params.input.reflections,
params.input.experiments):
refs = ref_wrapper.data
exps = exp_wrapper.data
for i, exp in enumerate(exps):
sel = refs['id'] == i
sub_ref = refs.select(sel)
nrefs_per_exp.append(len(sub_ref))
sub_ref['id'] = flex.int(len(sub_ref), global_id)
reflections.extend(sub_ref)
experiments.append(exp)
global_id += 1
# Try to load the models and data
nexp = len(experiments)
if nexp == 0:
print "No Experiments found in the input"
self.parser.print_help()
return
if len(reflections) == 0:
print "No reflection data found in the input"
self.parser.print_help()
return
self.check_input(reflections)
# Configure the logging
log.config(info=params.output.log, debug=params.output.debug_log)
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
# Convert to P 1?
if params.refinement.triclinic:
reflections, experiments = self.convert_to_P1(
reflections, experiments)
# Combine crystals?
if params.refinement.combine_crystal_models and len(experiments) > 1:
logger.info('Combining {0} crystal models'.format(
len(experiments)))
experiments = self.combine_crystals(experiments)
# Filter integrated centroids?
if params.refinement.filter_integrated_centroids:
reflections = self.filter_integrated_centroids(reflections)
# Get the refiner
logger.info('Configuring refiner')
refiner = self.create_refiner(params, reflections, experiments)
# Refine the geometry
if nexp == 1:
logger.info('Performing refinement of a single Experiment...')
else:
logger.info(
'Performing refinement of {0} Experiments...'.format(nexp))
# Refine and get the refinement history
history = refiner.run()
# get the refined experiments
experiments = refiner.get_experiments()
crystals = experiments.crystals()
if len(crystals) == 1:
# output the refined model for information
logger.info('')
logger.info('Final refined crystal model:')
logger.info(crystals[0])
logger.info(self.cell_param_table(crystals[0]))
# Save the refined experiments to file
output_experiments_filename = params.output.experiments
logger.info('Saving refined experiments to {0}'.format(
output_experiments_filename))
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(experiments)
dump.as_json(output_experiments_filename)
# Correlation plot
if params.output.correlation_plot.filename is not None:
from os.path import splitext
root, ext = splitext(params.output.correlation_plot.filename)
if not ext: ext = ".pdf"
steps = params.output.correlation_plot.steps
if steps is None: steps = [history.get_nrows() - 1]
# extract individual column names or indices
col_select = params.output.correlation_plot.col_select
num_plots = 0
for step in steps:
fname_base = root
if len(steps) > 1: fname_base += "_step%02d" % step
plot_fname = fname_base + ext
corrmat, labels = refiner.get_parameter_correlation_matrix(
step, col_select)
if [corrmat, labels].count(None) == 0:
from dials.algorithms.refinement.refinement_helpers import corrgram
plt = corrgram(corrmat, labels)
if plt is not None:
logger.info(
'Saving parameter correlation plot to {}'.format(
plot_fname))
plt.savefig(plot_fname)
num_plots += 1
mat_fname = fname_base + ".pickle"
with open(mat_fname, 'wb') as handle:
py_mat = corrmat.as_scitbx_matrix(
) #convert to pickle-friendly form
logger.info(
'Saving parameter correlation matrix to {0}'.
format(mat_fname))
pickle.dump({
'corrmat': py_mat,
'labels': labels
}, handle)
if num_plots == 0:
msg = "Sorry, no parameter correlation plots were produced. Please set " \
"track_parameter_correlation=True to ensure correlations are " \
"tracked, and make sure correlation_plot.col_select is valid."
logger.info(msg)
if params.output.cif is not None:
self.generate_cif(crystals[0], refiner, file=params.output.cif)
if params.output.p4p is not None:
self.generate_p4p(crystals[0],
experiments[0].beam,
file=params.output.p4p)
if params.output.mmcif is not None:
self.generate_mmcif(crystals[0], refiner, file=params.output.mmcif)
# Log the total time taken
logger.info("\nTotal time taken: {0:.2f}s".format(time() - start_time))
def run(self):
print "Parsing input"
params, options = self.parser.parse_args(show_diff_phil=True)
#Configure the logging
log.config(params.detector_phase.refinement.verbosity,
info='dials.refine.log', debug='dials.refine.debug.log')
# Try to obtain the models and data
if not params.input.experiments:
raise Sorry("No Experiments found in the input")
if not params.input.reflections:
raise Sorry("No reflection data found in the input")
try:
assert len(params.input.reflections) == len(params.input.experiments)
except AssertionError:
raise Sorry("The number of input reflections files does not match the "
"number of input experiments")
# set up global experiments and reflections lists
from dials.array_family import flex
reflections = flex.reflection_table()
global_id = 0
from dxtbx.model.experiment.experiment_list import ExperimentList
experiments=ExperimentList()
if params.reference_detector == "first":
# Use the first experiment of the first experiment list as the reference detector
ref_exp = params.input.experiments[0].data[0]
else:
# Average all the detectors to generate a reference detector
assert params.detector_phase.refinement.parameterisation.detector.hierarchy_level == 0
from scitbx.matrix import col
panel_fasts = []
panel_slows = []
panel_oris = []
for exp_wrapper in params.input.experiments:
exp = exp_wrapper.data[0]
if panel_oris:
for i, panel in enumerate(exp.detector):
panel_fasts[i] += col(panel.get_fast_axis())
panel_slows[i] += col(panel.get_slow_axis())
panel_oris[i] += col(panel.get_origin())
else:
for i, panel in enumerate(exp.detector):
panel_fasts.append(col(panel.get_fast_axis()))
panel_slows.append(col(panel.get_slow_axis()))
panel_oris.append(col(panel.get_origin()))
ref_exp = copy.deepcopy(params.input.experiments[0].data[0])
for i, panel in enumerate(ref_exp.detector):
# Averaging the fast and slow axes can make them be non-orthagonal. Fix by finding
# the vector that goes exactly between them and rotate
# around their cross product 45 degrees from that vector in either direction
vf = panel_fasts[i]/len(params.input.experiments)
vs = panel_slows[i]/len(params.input.experiments)
c = vf.cross(vs)
angle = vf.angle(vs, deg=True)
v45 = vf.rotate(c, angle/2, deg=True)
vf = v45.rotate(c, -45, deg=True)
vs = v45.rotate(c, 45, deg=True)
panel.set_frame(vf, vs,
panel_oris[i]/len(params.input.experiments))
print "Reference detector (averaged):", str(ref_exp.detector)
# set the experiment factory that combines a crystal with the reference beam
# and the reference detector
experiment_from_crystal=ExperimentFromCrystal(ref_exp.beam, ref_exp.detector)
# keep track of the number of refl per accepted experiment for a table
nrefs_per_exp = []
# loop through the input, building up the global lists
for ref_wrapper, exp_wrapper in zip(params.input.reflections,
params.input.experiments):
refs = ref_wrapper.data
exps = exp_wrapper.data
# there might be multiple experiments already here. Loop through them
for i, exp in enumerate(exps):
# select the relevant reflections
sel = refs['id'] == i
sub_ref = refs.select(sel)
## DGW commented out as reflections.minimum_number_of_reflections no longer exists
#if len(sub_ref) < params.crystals_phase.refinement.reflections.minimum_number_of_reflections:
# print "skipping experiment", i, "in", exp_wrapper.filename, "due to insufficient strong reflections in", ref_wrapper.filename
# continue
# build an experiment with this crystal plus the reference models
combined_exp = experiment_from_crystal(exp.crystal)
# next experiment ID in series
exp_id = len(experiments)
# check this experiment
if not check_experiment(combined_exp, sub_ref):
print "skipping experiment", i, "in", exp_wrapper.filename, "due to poor RMSDs"
continue
# set reflections ID
sub_ref['id'] = flex.int(len(sub_ref), exp_id)
# keep number of reflections for the table
nrefs_per_exp.append(len(sub_ref))
# obtain mm positions on the reference detector
sub_ref = indexer_base.map_spots_pixel_to_mm_rad(sub_ref,
combined_exp.detector, combined_exp.scan)
# extend refl and experiments lists
reflections.extend(sub_ref)
experiments.append(combined_exp)
# print number of reflections per accepted experiment
from libtbx.table_utils import simple_table
header = ["Experiment", "Nref"]
rows = [(str(i), str(n)) for (i, n) in enumerate(nrefs_per_exp)]
st = simple_table(rows, header)
print "Number of reflections per experiment"
print st.format()
for cycle in range(params.n_macrocycles):
print "MACROCYCLE %02d" % (cycle + 1)
print "=============\n"
# first run: multi experiment joint refinement of detector with fixed beam and
# crystals
print "PHASE 1"
# SET THIS TEST TO FALSE TO REFINE WHOLE DETECTOR AS SINGLE JOB
if params.detector_phase.refinement.parameterisation.detector.hierarchy_level > 0:
experiments = detector_parallel_refiners(params.detector_phase, experiments, reflections)
else:
experiments = detector_refiner(params.detector_phase, experiments, reflections)
# second run
print "PHASE 2"
experiments = crystals_refiner(params.crystals_phase, experiments, reflections)
# Save the refined experiments to file
output_experiments_filename = params.output.experiments_filename
print 'Saving refined experiments to {0}'.format(output_experiments_filename)
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(experiments)
dump.as_json(output_experiments_filename)
# Write out refined reflections, if requested
if params.output.reflections_filename:
print 'Saving refined reflections to {0}'.format(
params.output.reflections_filename)
reflections.as_pickle(params.output.reflections_filename)
return
def run(self):
'''Execute the script.'''
from dials.algorithms.refinement.two_theta_refiner import \
TwoThetaReflectionManager, TwoThetaTarget, \
TwoThetaPredictionParameterisation
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
# set up global experiments and reflections lists
from dials.array_family import flex
reflections = flex.reflection_table()
global_id = 0
from dxtbx.model.experiment.experiment_list import ExperimentList
experiments=ExperimentList()
# loop through the input, building up the global lists
nrefs_per_exp = []
for ref_wrapper, exp_wrapper in zip(params.input.reflections,
params.input.experiments):
refs = ref_wrapper.data
exps = exp_wrapper.data
for i, exp in enumerate(exps):
sel = refs['id'] == i
sub_ref = refs.select(sel)
nrefs_per_exp.append(len(sub_ref))
sub_ref['id'] = flex.int(len(sub_ref), global_id)
reflections.extend(sub_ref)
experiments.append(exp)
global_id += 1
# Try to load the models and data
nexp = len(experiments)
if nexp == 0:
print "No Experiments found in the input"
self.parser.print_help()
return
if len(reflections) == 0:
print "No reflection data found in the input"
self.parser.print_help()
return
self.check_input(reflections)
# Configure the logging
log.config(info=params.output.log,
debug=params.output.debug_log)
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
# Convert to P 1?
if params.refinement.triclinic:
reflections, experiments = self.convert_to_P1(reflections, experiments)
# Combine crystals?
if params.refinement.combine_crystal_models and len(experiments) > 1:
logger.info('Combining {0} crystal models'.format(len(experiments)))
experiments = self.combine_crystals(experiments)
# Filter integrated centroids?
if params.refinement.filter_integrated_centroids:
reflections = self.filter_integrated_centroids(reflections)
# Get the refiner
logger.info('Configuring refiner')
refiner = self.create_refiner(params, reflections, experiments)
# Refine the geometry
if nexp == 1:
logger.info('Performing refinement of a single Experiment...')
else:
logger.info('Performing refinement of {0} Experiments...'.format(nexp))
# Refine and get the refinement history
history = refiner.run()
# get the refined experiments
experiments = refiner.get_experiments()
crystals = experiments.crystals()
if len(crystals) == 1:
# output the refined model for information
logger.info('')
logger.info('Final refined crystal model:')
logger.info(crystals[0])
logger.info(self.cell_param_table(crystals[0]))
# Save the refined experiments to file
output_experiments_filename = params.output.experiments
logger.info('Saving refined experiments to {0}'.format(output_experiments_filename))
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(experiments)
dump.as_json(output_experiments_filename)
# Correlation plot
if params.output.correlation_plot.filename is not None:
from os.path import splitext
root, ext = splitext(params.output.correlation_plot.filename)
if not ext: ext = ".pdf"
steps = params.output.correlation_plot.steps
if steps is None: steps = [history.get_nrows()-1]
# extract individual column names or indices
col_select = params.output.correlation_plot.col_select
num_plots = 0
for step in steps:
fname_base = root
if len(steps) > 1: fname_base += "_step%02d" % step
plot_fname = fname_base + ext
corrmat, labels = refiner.get_parameter_correlation_matrix(step, col_select)
if [corrmat, labels].count(None) == 0:
from dials.algorithms.refinement.refinement_helpers import corrgram
plt = corrgram(corrmat, labels)
if plt is not None:
logger.info('Saving parameter correlation plot to {}'.format(plot_fname))
plt.savefig(plot_fname)
num_plots += 1
mat_fname = fname_base + ".pickle"
with open(mat_fname, 'wb') as handle:
py_mat = corrmat.as_scitbx_matrix() #convert to pickle-friendly form
logger.info('Saving parameter correlation matrix to {0}'.format(mat_fname))
pickle.dump({'corrmat':py_mat, 'labels':labels}, handle)
if num_plots == 0:
msg = "Sorry, no parameter correlation plots were produced. Please set " \
"track_parameter_correlation=True to ensure correlations are " \
"tracked, and make sure correlation_plot.col_select is valid."
logger.info(msg)
if params.output.cif is not None:
self.generate_cif(crystals[0], refiner, file=params.output.cif)
if params.output.p4p is not None:
self.generate_p4p(crystals[0], experiments[0].beam,
file=params.output.p4p)
if params.output.mmcif is not None:
self.generate_mmcif(crystals[0], refiner, file=params.output.mmcif)
# Log the total time taken
logger.info("\nTotal time taken: {0:.2f}s".format(time() - start_time))
slow = matrix.col((-float(ss_x.rstrip('x')),float(ss_y.rstrip('y')), 0.0))
origin = matrix.col((-float(geom[key]['corner_x']) * params.pixel_size,
float(geom[key]['corner_y']) * params.pixel_size,
0.0))
# OBS! you need to set the panel to a root before set local frame...
p = root.add_panel()
p.set_name('panel-%s' % key)
p.set_image_size((512, 1024))
p.set_trusted_range((-1, 1000000))
p.set_pixel_size((params.pixel_size, params.pixel_size))
p.set_local_frame(
fast.elems,
slow.elems,
origin.elems)
from dxtbx.model.beam import beam_factory
wavelength = params.wavelength
beam = beam_factory.simple(wavelength)
from dxtbx.model.experiment.experiment_list import Experiment, ExperimentList, ExperimentListDumper
experiments = ExperimentList()
experiment = Experiment(detector = detector, beam = beam)
experiments.append(experiment)
dump = ExperimentListDumper(experiments)
dump.as_json("geometry.json")
if __name__ == "__main__":
run(sys.argv[1:])
def run(self):
'''Execute the script.'''
from dials.util.options import flatten_reflections, flatten_experiments, \
flatten_datablocks
import cPickle as pickle
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
datablocks = flatten_datablocks(params.input.datablock)
# Try to load the models and data
slice_exps = len(experiments) > 0
slice_refs = len(reflections) > 0
slice_dbs = len(datablocks) > 0
# Catch case of nothing to do
if not any([slice_exps, slice_refs, slice_dbs]):
print "No suitable input provided"
self.parser.print_help()
return
if reflections:
if len(reflections) > 1:
raise Sorry(
"Only one reflections list can be imported at present")
reflections = reflections[0]
# calculate frame numbers if needed
if experiments:
reflections = calculate_frame_numbers(reflections, experiments)
# if we still don't have the right column give up
if 'xyzobs.px.value' not in reflections:
raise Sorry(
"These reflections do not have frame numbers set, and "
"there are no experiments provided to calculate these.")
# set trivial case where no scan range is provided at all
if not params.scan_range:
params.scan_range = [None]
# check if slicing into blocks
if params.block_size is not None:
# in this case for simplicity, ensure that there is either an
# an experiment list or datablocks, but not both. Ensure there is only
# a single scan contained within.
if [slice_exps, slice_dbs].count(True) != 1:
raise Sorry(
"For slicing into blocks please provide either datablocks"
" or experiments, but not both.")
if slice_exps:
if len(experiments) > 1:
raise Sorry(
"For slicing into blocks please provide a single "
"scan only")
scan = experiments[0].scan
if slice_dbs:
scans = datablocks[0].unique_scans()
if len(scans) > 1 or len(datablocks) > 1:
raise Sorry(
"For slicing into blocks please provide a single "
"scan only")
scan = scans[0]
# Having extracted the scan, calculate the blocks
params.scan_range = calculate_block_ranges(scan, params.block_size)
# Do the slicing then recombine
if slice_exps:
sliced = [slice_experiments(experiments, [sr])[0] \
for sr in params.scan_range]
sliced_experiments = ExperimentList()
for exp in sliced:
sliced_experiments.append(exp)
if slice_dbs:
sliced = [slice_datablocks(datablocks, [sr])[0] \
for sr in params.scan_range]
imagesets = [db.extract_imagesets()[0] for db in sliced]
sliced_datablocks = DataBlock(imagesets)
# slice reflections if present
if slice_refs:
sliced = [slice_reflections(reflections, [sr]) \
for sr in params.scan_range]
sliced_reflections = sliced[0]
for i, rt in enumerate(sliced[1:]):
rt['id'] += (i + 1) # set id
sliced_reflections.extend(rt)
else:
# slice each dataset into the requested subset
if slice_exps:
sliced_experiments = slice_experiments(experiments,
params.scan_range)
if slice_refs:
sliced_reflections = slice_reflections(reflections,
params.scan_range)
if slice_dbs:
sliced_datablocks = slice_datablocks(datablocks,
params.scan_range)
# Save sliced experiments
if slice_exps:
output_experiments_filename = params.output.experiments_filename
if output_experiments_filename is None:
# take first filename as template
bname = basename(params.input.experiments[0].filename)
bname = splitext(bname)[0]
if not bname: bname = "experiments"
if len(params.scan_range
) == 1 and params.scan_range[0] is not None:
ext = "_{0}_{1}.json".format(*params.scan_range[0])
else:
ext = "_sliced.json"
output_experiments_filename = bname + ext
print 'Saving sliced experiments to {0}'.format(
output_experiments_filename)
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(sliced_experiments)
dump.as_json(output_experiments_filename)
# Save sliced reflections
if slice_refs:
output_reflections_filename = params.output.reflections_filename
if output_reflections_filename is None:
# take first filename as template
bname = basename(params.input.reflections[0].filename)
bname = splitext(bname)[0]
if not bname: bname = "reflections"
if len(params.scan_range
) == 1 and params.scan_range[0] is not None:
ext = "_{0}_{1}.pickle".format(*params.scan_range[0])
else:
ext = "_sliced.pickle"
output_reflections_filename = bname + ext
print 'Saving sliced reflections to {0}'.format(
output_reflections_filename)
sliced_reflections.as_pickle(output_reflections_filename)
# Save sliced datablocks
if slice_dbs:
output_datablocks_filename = params.output.datablocks_filename
if output_datablocks_filename is None:
# take first filename as template
bname = basename(params.input.datablock[0].filename)
bname = splitext(bname)[0]
if not bname: bname = "datablock"
if len(params.scan_range
) == 1 and params.scan_range[0] is not None:
ext = "_{0}_{1}.json".format(*params.scan_range[0])
else:
ext = "_sliced.json"
output_datablocks_filename = bname + ext
print 'Saving sliced datablocks to {0}'.format(
output_datablocks_filename)
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(sliced_datablocks)
dump.as_file(output_datablocks_filename)
return
else: print "no beam!"
if exp.detector: print exp.detector
else: print "no detector!"
if exp.goniometer: print exp.goniometer
else: print "no goniometer!"
if exp.scan: print exp.scan
else: print "no scan!"
if exp.crystal: print exp.crystal
else: print "no crystal!"
# Write the experiment list to a JSON or pickle file
if params.output.filename is None:
params.output.filename = 'experiments.json'
print "-" * 80
print 'Writing experiments to %s' % params.output.filename
dump = ExperimentListDumper(experiments)
dump.as_file(params.output.filename)
# Optionally save as a data block
if params.output.xds_datablock:
print "-" * 80
print "Writing data block to %s" % params.output.xds_datablock
dump = DataBlockDumper(experiments.to_datablocks())
dump.as_file(params.output.xds_datablock)
@staticmethod
def find_best_xds_file(xds_dir):
''' Find the best available file.'''
from os.path import exists, join
# The possible files to check
def run(self):
'''Execute the script.'''
from dials.util.options import flatten_experiments
from libtbx.utils import Sorry
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
# Try to load the models and data
if len(params.input.experiments) == 0:
print "No Experiments found in the input"
self.parser.print_help()
return
if len(params.input.reflections) == 0:
print "No reflection data found in the input"
self.parser.print_help()
return
try:
assert len(params.input.reflections) == len(
params.input.experiments)
except AssertionError:
raise Sorry(
"The number of input reflections files does not match the "
"number of input experiments")
flat_exps = flatten_experiments(params.input.experiments)
ref_beam = params.reference_from_experiment.beam
ref_goniometer = params.reference_from_experiment.goniometer
ref_scan = params.reference_from_experiment.scan
ref_crystal = params.reference_from_experiment.crystal
ref_detector = params.reference_from_experiment.detector
if ref_beam is not None:
try:
ref_beam = flat_exps[ref_beam].beam
except IndexError:
raise Sorry(
"{0} is not a valid experiment ID".format(ref_beam))
if ref_goniometer is not None:
try:
ref_goniometer = flat_exps[ref_goniometer].goniometer
except IndexError:
raise Sorry(
"{0} is not a valid experiment ID".format(ref_goniometer))
if ref_scan is not None:
try:
ref_scan = flat_exps[ref_scan].scan
except IndexError:
raise Sorry(
"{0} is not a valid experiment ID".format(ref_scan))
if ref_crystal is not None:
try:
ref_crystal = flat_exps[ref_crystal].crystal
except IndexError:
raise Sorry(
"{0} is not a valid experiment ID".format(ref_crystal))
if ref_detector is not None:
assert not params.reference_from_experiment.average_detector
try:
ref_detector = flat_exps[ref_detector].detector
except IndexError:
raise Sorry(
"{0} is not a valid experiment ID".format(ref_detector))
elif params.reference_from_experiment.average_detector:
# Average all of the detectors together
from scitbx.matrix import col
def average_detectors(target, panelgroups, depth):
# Recursive function to do the averaging
if params.reference_from_experiment.average_hierarchy_level is None or \
depth == params.reference_from_experiment.average_hierarchy_level:
n = len(panelgroups)
sum_fast = col((0.0, 0.0, 0.0))
sum_slow = col((0.0, 0.0, 0.0))
sum_ori = col((0.0, 0.0, 0.0))
# Average the d matrix vectors
for pg in panelgroups:
sum_fast += col(pg.get_local_fast_axis())
sum_slow += col(pg.get_local_slow_axis())
sum_ori += col(pg.get_local_origin())
sum_fast /= n
sum_slow /= n
sum_ori /= n
# Re-orthagonalize the slow and the fast vectors by rotating around the cross product
c = sum_fast.cross(sum_slow)
a = sum_fast.angle(sum_slow, deg=True) / 2
sum_fast = sum_fast.rotate(c, a - 45, deg=True)
sum_slow = sum_slow.rotate(c, -(a - 45), deg=True)
target.set_local_frame(sum_fast, sum_slow, sum_ori)
if target.is_group():
# Recurse
for i, target_pg in enumerate(target):
average_detectors(target_pg,
[pg[i] for pg in panelgroups],
depth + 1)
ref_detector = flat_exps[0].detector
average_detectors(ref_detector.hierarchy(),
[e.detector.hierarchy() for e in flat_exps], 0)
combine = CombineWithReference(beam=ref_beam,
goniometer=ref_goniometer,
scan=ref_scan,
crystal=ref_crystal,
detector=ref_detector,
params=params)
# set up global experiments and reflections lists
from dials.array_family import flex
reflections = flex.reflection_table()
global_id = 0
from dxtbx.model.experiment.experiment_list import ExperimentList
experiments = ExperimentList()
# loop through the input, building up the global lists
nrefs_per_exp = []
for ref_wrapper, exp_wrapper in zip(params.input.reflections,
params.input.experiments):
refs = ref_wrapper.data
exps = exp_wrapper.data
for i, exp in enumerate(exps):
sel = refs['id'] == i
sub_ref = refs.select(sel)
nrefs_per_exp.append(len(sub_ref))
sub_ref['id'] = flex.int(len(sub_ref), global_id)
reflections.extend(sub_ref)
experiments.append(combine(exp))
global_id += 1
# print number of reflections per experiment
from libtbx.table_utils import simple_table
header = ["Experiment", "Nref"]
rows = [(str(i), str(n)) for (i, n) in enumerate(nrefs_per_exp)]
st = simple_table(rows, header)
print st.format()
# save output
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
print 'Saving combined experiments to {0}'.format(
params.output.experiments_filename)
dump = ExperimentListDumper(experiments)
dump.as_json(params.output.experiments_filename)
print 'Saving combined reflections to {0}'.format(
params.output.reflections_filename)
reflections.as_pickle(params.output.reflections_filename)
return
def run(self):
'''Execute the script.'''
from time import time
import cPickle as pickle
from dials.util import log
from dials.algorithms.refinement import RefinerFactory
from dials.util.options import flatten_reflections, flatten_experiments
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
# Try to load the models and data
nexp = len(experiments)
if nexp == 0:
print "No Experiments found in the input"
self.parser.print_help()
return
if len(reflections) == 0:
print "No reflection data found in the input"
self.parser.print_help()
return
if len(reflections) > 1:
raise Sorry("Only one reflections list can be imported at present")
reflections = reflections[0]
self.check_input(reflections)
# Configure the logging
log.config(info=params.output.log, debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
# Modify options if necessary
if params.output.correlation_plot.filename is not None:
params.refinement.refinery.track_parameter_correlation = True
# Warn about potentially unhelpful options
if params.refinement.mp.nproc > 1:
logger.warning(
"WARNING: setting nproc > 1 is only helpful in rare "
"circumstances. It is not recommended for typical data processing "
"tasks.\n")
# Get the refiner
logger.info('Configuring refiner')
refiner = RefinerFactory.from_parameters_data_experiments(
params, reflections, experiments)
# Refine the geometry
if nexp == 1:
logger.info('Performing refinement of a single Experiment...')
else:
logger.info(
'Performing refinement of {0} Experiments...'.format(nexp))
# Refine and get the refinement history
history = refiner.run()
if params.output.centroids:
logger.info("Writing table of centroids to '{0}'".format(
params.output.centroids))
self.write_centroids_table(refiner, params.output.centroids)
# Get the refined experiments
experiments = refiner.get_experiments()
# Write scan-varying parameters to file, if there were any
if params.output.parameter_table:
scans = experiments.scans()
if len(scans) > 1:
logger.info(
"Writing a scan-varying parameter table is only supported "
"for refinement of a single scan")
else:
scan = scans[0]
text = refiner.get_param_reporter(
).varying_params_vs_image_number(scan.get_array_range())
if text:
logger.info(
"Writing scan-varying parameter table to {0}".format(
params.output.parameter_table))
f = open(params.output.parameter_table, "w")
f.write(text)
f.close()
else:
logger.info("No scan-varying parameter table to write")
crystals = experiments.crystals()
if len(crystals) == 1:
# output the refined model for information
logger.info('')
logger.info('Final refined crystal model:')
logger.info(crystals[0])
# Save the refined experiments to file
output_experiments_filename = params.output.experiments
logger.info('Saving refined experiments to {0}'.format(
output_experiments_filename))
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(experiments)
dump.as_json(output_experiments_filename)
# Save reflections with updated predictions if requested (allow to switch
# this off if it is a time-consuming step)
if params.output.reflections:
# Update predictions for all indexed reflections
logger.info('Updating predictions for indexed reflections')
preds = refiner.predict_for_indexed()
# just copy over the columns of interest, leaving behind things
# added by e.g. scan-varying refinement such as 'block', the
# U, B and UB matrices and gradients.
reflections['s1'] = preds['s1']
reflections['xyzcal.mm'] = preds['xyzcal.mm']
reflections['xyzcal.px'] = preds['xyzcal.px']
if 'entering' in preds:
reflections['entering'] = preds['entering']
# set used_in_refinement and centroid_outlier flags
assert len(preds) == len(reflections)
reflections.unset_flags(
flex.size_t_range(len(reflections)),
reflections.flags.used_in_refinement
| reflections.flags.centroid_outlier)
mask = preds.get_flags(preds.flags.centroid_outlier)
reflections.set_flags(mask, reflections.flags.centroid_outlier)
mask = preds.get_flags(preds.flags.used_in_refinement)
reflections.set_flags(mask, reflections.flags.used_in_refinement)
logger.info(
'Saving reflections with updated predictions to {0}'.format(
params.output.reflections))
if params.output.include_unused_reflections:
reflections.as_pickle(params.output.reflections)
else:
sel = reflections.get_flags(
reflections.flags.used_in_refinement)
reflections.select(sel).as_pickle(params.output.reflections)
# For debugging, if requested save matches to file
if params.output.matches:
matches = refiner.get_matches()
logger.info(
'Saving matches (use for debugging purposes) to {0}'.format(
params.output.matches))
matches.as_pickle(params.output.matches)
# Correlation plot
if params.output.correlation_plot.filename is not None:
from os.path import splitext
root, ext = splitext(params.output.correlation_plot.filename)
if not ext: ext = ".pdf"
steps = params.output.correlation_plot.steps
if steps is None: steps = [history.get_nrows() - 1]
# extract individual column names or indices
col_select = params.output.correlation_plot.col_select
num_plots = 0
for step in steps:
fname_base = root
if len(steps) > 1: fname_base += "_step%02d" % step
plot_fname = fname_base + ext
corrmat, labels = refiner.get_parameter_correlation_matrix(
step, col_select)
if [corrmat, labels].count(None) == 0:
from dials.algorithms.refinement.refinement_helpers import corrgram
plt = corrgram(corrmat, labels)
if plt is not None:
logger.info(
'Saving parameter correlation plot to {}'.format(
plot_fname))
plt.savefig(plot_fname)
num_plots += 1
mat_fname = fname_base + ".pickle"
with open(mat_fname, 'wb') as handle:
py_mat = corrmat.as_scitbx_matrix(
) #convert to pickle-friendly form
logger.info(
'Saving parameter correlation matrix to {0}'.
format(mat_fname))
pickle.dump({
'corrmat': py_mat,
'labels': labels
}, handle)
if num_plots == 0:
msg = "Sorry, no parameter correlation plots were produced. Please set " \
"track_parameter_correlation=True to ensure correlations are " \
"tracked, and make sure correlation_plot.col_select is valid."
logger.info(msg)
# Write out refinement history, if requested
if params.output.history:
with open(params.output.history, 'wb') as handle:
logger.info('Saving refinement step history to {0}'.format(
params.output.history))
pickle.dump(history, handle)
# Log the total time taken
logger.info("\nTotal time taken: {0:.2f}s".format(time() - start_time))
return
def experiment_list(obj, outfile): ''' Dump an experiment list. ''' from dxtbx.model.experiment.experiment_list import ExperimentListDumper dumper = ExperimentListDumper(obj) dumper.as_file(outfile)
class TestExperimentListDumper(object):
def __init__(self, path):
self.path = path
def run(self):
self.tst_dump_formats()
self.tst_dump_empty_sweep()
self.tst_dump_with_lookup()
self.tst_dump_with_bad_lookup()
def tst_dump_formats(self):
from uuid import uuid4
from os.path import join
import os
os.environ['DIALS_REGRESSION'] = self.path
# Get all the filenames
filename1 = join(self.path, 'experiment_test_data', 'experiment_1.json')
# Read all the experiment lists in
elist1 = ExperimentListFactory.from_json_file(filename1)
# Create the experiment list dumper
dump = ExperimentListDumper(elist1)
# Dump as JSON file and reload
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename)
elist2 = ExperimentListFactory.from_json_file(filename)
self.check(elist1, elist2)
# Dump as split JSON file and reload
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename, split=True)
elist2 = ExperimentListFactory.from_json_file(filename)
self.check(elist1, elist2)
# Dump as pickle and reload
filename = 'temp%s.pickle' % uuid4().hex
dump.as_pickle(filename)
elist2 = ExperimentListFactory.from_pickle_file(filename)
self.check(elist1, elist2)
def tst_dump_empty_sweep(self):
from dxtbx.imageset import ImageSweep, NullReader, SweepFileList
from dxtbx.model import Beam, Detector, Goniometer, Scan
from dxtbx.model.crystal import crystal_model
from uuid import uuid4
imageset = ImageSweep(NullReader(SweepFileList("filename%01d.cbf", (0, 3))))
imageset.set_beam(Beam((1, 0, 0)))
imageset.set_detector(Detector())
imageset.set_goniometer(Goniometer())
imageset.set_scan(Scan((1, 3), (0.0, 1.0)))
crystal = crystal_model((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol=1)
experiments = ExperimentListFactory.from_imageset_and_crystal(
imageset, crystal)
dump = ExperimentListDumper(experiments)
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename)
experiments2 = ExperimentListFactory.from_json_file(filename,
check_format=False)
self.check(experiments, experiments2)
print 'OK'
def tst_dump_with_lookup(self):
from dxtbx.imageset import ImageSweep, NullReader, SweepFileList
from dxtbx.model import Beam, Detector, Goniometer, Scan
from dxtbx.model.crystal import crystal_model
from uuid import uuid4
import libtbx.load_env
import os
from os.path import join
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'FAIL: dials_regression not configured'
exit(0)
filename = join(dials_regression, "centroid_test_data",
"experiments_with_lookup.json")
experiments = ExperimentListFactory.from_json_file(
filename,
check_format=True)
imageset = experiments[0].imageset
assert imageset.external_lookup.mask.data is not None
assert imageset.external_lookup.gain.data is not None
assert imageset.external_lookup.pedestal.data is not None
assert imageset.external_lookup.mask.filename is not None
assert imageset.external_lookup.gain.filename is not None
assert imageset.external_lookup.pedestal.filename is not None
assert imageset.external_lookup.mask.data.all_eq(True)
assert imageset.external_lookup.gain.data.all_eq(1)
assert imageset.external_lookup.pedestal.data.all_eq(0)
dump = ExperimentListDumper(experiments)
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename)
experiments = ExperimentListFactory.from_json_file(
filename,
check_format=True)
imageset = experiments[0].imageset
assert imageset.external_lookup.mask.data is not None
assert imageset.external_lookup.gain.data is not None
assert imageset.external_lookup.pedestal.data is not None
assert imageset.external_lookup.mask.filename is not None
assert imageset.external_lookup.gain.filename is not None
assert imageset.external_lookup.pedestal.filename is not None
assert imageset.external_lookup.mask.data.all_eq(True)
assert imageset.external_lookup.gain.data.all_eq(1)
assert imageset.external_lookup.pedestal.data.all_eq(0)
(-float(fs_x.rstrip('x')), float(fs_y.rstrip('y')), 0.0))
slow = matrix.col(
(-float(ss_x.rstrip('x')), float(ss_y.rstrip('y')), 0.0))
origin = matrix.col(
(-float(geom[key]['corner_x']) * params.pixel_size,
float(geom[key]['corner_y']) * params.pixel_size, 0.0))
# OBS! you need to set the panel to a root before set local frame...
p = root.add_panel()
p.set_name('panel-%s' % key)
p.set_image_size((512, 1024))
p.set_trusted_range((-1, 1000000))
p.set_pixel_size((params.pixel_size, params.pixel_size))
p.set_local_frame(fast.elems, slow.elems, origin.elems)
from dxtbx.model.beam import beam_factory
wavelength = params.wavelength
beam = beam_factory.simple(wavelength)
from dxtbx.model.experiment.experiment_list import Experiment, ExperimentList, ExperimentListDumper
experiments = ExperimentList()
experiment = Experiment(detector=detector, beam=beam)
experiments.append(experiment)
dump = ExperimentListDumper(experiments)
dump.as_json("geometry.json")
if __name__ == "__main__":
run(sys.argv[1:])
def run(self):
'''Execute the script.'''
from dials.util.options import flatten_experiments
from libtbx.utils import Sorry
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
# Try to load the models and data
if len(params.input.experiments) == 0:
print "No Experiments found in the input"
self.parser.print_help()
return
if len(params.input.reflections) == 0:
print "No reflection data found in the input"
self.parser.print_help()
return
try:
assert len(params.input.reflections) == len(params.input.experiments)
except AssertionError:
raise Sorry("The number of input reflections files does not match the "
"number of input experiments")
flat_exps = flatten_experiments(params.input.experiments)
ref_beam = params.reference_from_experiment.beam
ref_goniometer = params.reference_from_experiment.goniometer
ref_scan = params.reference_from_experiment.scan
ref_crystal = params.reference_from_experiment.crystal
ref_detector = params.reference_from_experiment.detector
if ref_beam is not None:
try:
ref_beam = flat_exps[ref_beam].beam
except IndexError:
raise Sorry("{0} is not a valid experiment ID".format(ref_beam))
if ref_goniometer is not None:
try:
ref_goniometer = flat_exps[ref_goniometer].goniometer
except IndexError:
raise Sorry("{0} is not a valid experiment ID".format(ref_goniometer))
if ref_scan is not None:
try:
ref_scan = flat_exps[ref_scan].scan
except IndexError:
raise Sorry("{0} is not a valid experiment ID".format(ref_scan))
if ref_crystal is not None:
try:
ref_crystal = flat_exps[ref_crystal].crystal
except IndexError:
raise Sorry("{0} is not a valid experiment ID".format(ref_crystal))
if ref_detector is not None:
assert not params.reference_from_experiment.average_detector
try:
ref_detector = flat_exps[ref_detector].detector
except IndexError:
raise Sorry("{0} is not a valid experiment ID".format(ref_detector))
elif params.reference_from_experiment.average_detector:
# Average all of the detectors together
from scitbx.matrix import col
def average_detectors(target, panelgroups, depth):
# Recursive function to do the averaging
if params.reference_from_experiment.average_hierarchy_level is None or \
depth == params.reference_from_experiment.average_hierarchy_level:
n = len(panelgroups)
sum_fast = col((0.0,0.0,0.0))
sum_slow = col((0.0,0.0,0.0))
sum_ori = col((0.0,0.0,0.0))
# Average the d matrix vectors
for pg in panelgroups:
sum_fast += col(pg.get_local_fast_axis())
sum_slow += col(pg.get_local_slow_axis())
sum_ori += col(pg.get_local_origin())
sum_fast /= n
sum_slow /= n
sum_ori /= n
# Re-orthagonalize the slow and the fast vectors by rotating around the cross product
c = sum_fast.cross(sum_slow)
a = sum_fast.angle(sum_slow, deg=True)/2
sum_fast = sum_fast.rotate(c, a-45, deg=True)
sum_slow = sum_slow.rotate(c, -(a-45), deg=True)
target.set_local_frame(sum_fast,sum_slow,sum_ori)
if target.is_group():
# Recurse
for i, target_pg in enumerate(target):
average_detectors(target_pg, [pg[i] for pg in panelgroups], depth+1)
ref_detector = flat_exps[0].detector
average_detectors(ref_detector.hierarchy(), [e.detector.hierarchy() for e in flat_exps], 0)
combine = CombineWithReference(beam=ref_beam, goniometer=ref_goniometer,
scan=ref_scan, crystal=ref_crystal, detector=ref_detector,
params=params)
# set up global experiments and reflections lists
from dials.array_family import flex
reflections = flex.reflection_table()
global_id = 0
from dxtbx.model.experiment.experiment_list import ExperimentList
experiments=ExperimentList()
# loop through the input, building up the global lists
nrefs_per_exp = []
for ref_wrapper, exp_wrapper in zip(params.input.reflections,
params.input.experiments):
refs = ref_wrapper.data
exps = exp_wrapper.data
for i, exp in enumerate(exps):
sel = refs['id'] == i
sub_ref = refs.select(sel)
nrefs_per_exp.append(len(sub_ref))
sub_ref['id'] = flex.int(len(sub_ref), global_id)
reflections.extend(sub_ref)
experiments.append(combine(exp))
global_id += 1
# print number of reflections per experiment
from libtbx.table_utils import simple_table
header = ["Experiment", "Nref"]
rows = [(str(i), str(n)) for (i, n) in enumerate(nrefs_per_exp)]
st = simple_table(rows, header)
print st.format()
# save a random subset if requested
if params.output.n_subset is not None and len(experiments) > params.output.n_subset:
import random
subset_exp = ExperimentList()
subset_refls = flex.reflection_table()
n_picked = 0
indices = range(len(experiments))
while n_picked < params.output.n_subset:
idx = indices.pop(random.randint(0, len(indices)-1))
subset_exp.append(experiments[idx])
refls = reflections.select(reflections['id'] == idx)
refls['id'] = flex.int(len(refls), n_picked)
subset_refls.extend(refls)
n_picked += 1
experiments = subset_exp
reflections = subset_refls
# save output
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
print 'Saving combined experiments to {0}'.format(
params.output.experiments_filename)
dump = ExperimentListDumper(experiments)
dump.as_json(params.output.experiments_filename)
print 'Saving combined reflections to {0}'.format(
params.output.reflections_filename)
reflections.as_pickle(params.output.reflections_filename)
return
def run(self):
'''Execute the script.'''
from time import time
import cPickle as pickle
from logging import info
from dials.util import log
from dials.algorithms.refinement import RefinerFactory
from dials.util.options import flatten_reflections, flatten_experiments
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
# Try to load the models and data
nexp = len(experiments)
if nexp == 0:
print "No Experiments found in the input"
self.parser.print_help()
return
if len(reflections) == 0:
print "No reflection data found in the input"
self.parser.print_help()
return
if len(reflections) > 1:
raise Sorry("Only one reflections list can be imported at present")
reflections = reflections[0]
self.check_input(reflections)
# Configure the logging
log.config(info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
info('The following parameters have been modified:\n')
info(diff_phil)
# Modify options if necessary
if params.output.correlation_plot.filename is not None:
params.refinement.refinery.track_parameter_correlation = True
# Get the refiner
info('Configuring refiner')
refiner = RefinerFactory.from_parameters_data_experiments(params,
reflections, experiments)
# Refine the geometry
if nexp == 1:
info('Performing refinement of a single Experiment...')
else:
info('Performing refinement of {0} Experiments...'.format(nexp))
# Refine and get the refinement history
history = refiner.run()
if params.output.centroids:
info("Writing table of centroids to '{0}'".format(
params.output.centroids))
self.write_centroids_table(refiner, params.output.centroids)
# Write scan-varying parameters to file, if there were any
if params.output.parameter_table:
scan = refiner.get_scan()
if scan:
text = refiner.get_param_reporter().varying_params_vs_image_number(
scan.get_array_range())
if text:
info("Writing scan-varying parameter table to {0}".format(
params.output.parameter_table))
f = open(params.output.parameter_table,"w")
f.write(text)
f.close()
else:
info("No scan-varying parameter table to write")
# get the refined experiments
experiments = refiner.get_experiments()
crystals = experiments.crystals()
if len(crystals) == 1:
# output the refined model for information
info('')
info('Final refined crystal model:')
info(crystals[0])
# Save the refined experiments to file
output_experiments_filename = params.output.experiments
info('Saving refined experiments to {0}'.format(output_experiments_filename))
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(experiments)
dump.as_json(output_experiments_filename)
# Save reflections with updated predictions if requested (allow to switch
# this off if it is a time-consuming step)
if params.output.reflections:
# Update predictions for all indexed reflections
info('Updating predictions for indexed reflections')
preds = refiner.predict_for_indexed()
# just copy over the columns of interest, leaving behind things
# added by e.g. scan-varying refinement such as 'block', the
# U, B and UB matrices and gradients.
reflections['s1'] = preds['s1']
reflections['xyzcal.mm'] = preds['xyzcal.mm']
reflections['xyzcal.px'] = preds['xyzcal.px']
if preds.has_key('entering'):
reflections['entering'] = preds['entering']
# set used_in_refinement and centroid_outlier flags
assert len(preds) == len(reflections)
reflections.unset_flags(flex.size_t_range(len(reflections)),
reflections.flags.used_in_refinement | reflections.flags.centroid_outlier)
mask = preds.get_flags(preds.flags.centroid_outlier)
reflections.set_flags(mask, reflections.flags.centroid_outlier)
mask = preds.get_flags(preds.flags.used_in_refinement)
reflections.set_flags(mask, reflections.flags.used_in_refinement)
info('Saving reflections with updated predictions to {0}'.format(
params.output.reflections))
if params.output.include_unused_reflections:
reflections.as_pickle(params.output.reflections)
else:
sel = reflections.get_flags(reflections.flags.used_in_refinement)
reflections.select(sel).as_pickle(params.output.reflections)
# For debugging, if requested save matches to file
if params.output.matches:
matches = refiner.get_matches()
info('Saving matches (use for debugging purposes) to {0}'.format(
params.output.matches))
matches.as_pickle(params.output.matches)
# Correlation plot
if params.output.correlation_plot.filename is not None:
from os.path import splitext
root, ext = splitext(params.output.correlation_plot.filename)
if not ext: ext = ".pdf"
steps = params.output.correlation_plot.steps
if steps is None: steps = [history.get_nrows()-1]
# extract individual column names or indices
col_select = params.output.correlation_plot.col_select
num_plots = 0
for step in steps:
fname_base = root + "_step%02d" % step
plot_fname = fname_base + ext
corrmat, labels = refiner.get_parameter_correlation_matrix(step, col_select)
if [corrmat, labels].count(None) == 0:
from dials.algorithms.refinement.refinement_helpers import corrgram
plt = corrgram(corrmat, labels)
if plt is not None:
info('Saving parameter correlation plot to {}'.format(plot_fname))
plt.savefig(plot_fname)
num_plots += 1
mat_fname = fname_base + ".pickle"
with open(mat_fname, 'wb') as handle:
py_mat = corrmat.as_scitbx_matrix() #convert to pickle-friendly form
info('Saving parameter correlation matrix to {0}'.format(mat_fname))
pickle.dump({'corrmat':py_mat, 'labels':labels}, handle)
if num_plots == 0:
msg = "Sorry, no parameter correlation plots were produced. Please set " \
"track_parameter_correlation=True to ensure correlations are " \
"tracked, and make sure correlation_plot.col_select is valid."
info(msg)
# Write out refinement history, if requested
if params.output.history:
with open(params.output.history, 'wb') as handle:
info('Saving refinement step history to {0}'.format(
params.output.history))
pickle.dump(history, handle)
# Log the total time taken
info("\nTotal time taken: {0:.2f}s".format(time() - start_time))
return
dr = DetectorRefiner()
else:
experiments = None
reflections = None
dr = None
cr = CrystalRefiners()
for cycle in range(working_params.n_macrocycles):
if rank == 0:
print("MACROCYCLE %02d" % (cycle + 1))
print("=============\n")
# first run: multi experiment joint refinement of detector with fixed beam and
# crystals
experiments = dr(experiments, reflections)
else:
experiments = None
# second run
experiments = cr(experiments, reflections)
if rank == 0:
# save the refined experiments
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(experiments)
experiments_filename = "refined_experiments.json"
dump.as_json(experiments_filename)
print("refined geometry written to {0}".format(experiments_filename))
def run(self):
''' Parse the options. '''
from dials.util.options import flatten_experiments, flatten_reflections
from dxtbx.model.experiment.experiment_list import ExperimentList
from scitbx.math import five_number_summary
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
self.params = params
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
assert len(reflections) == 1
reflections = reflections[0]
print "Found", len(reflections), "reflections", "and", len(
experiments), "experiments"
difference_vector_norms = (reflections['xyzcal.mm'] -
reflections['xyzobs.mm.value']).norms()
data = flex.double()
counts = flex.double()
for i in xrange(len(experiments)):
dvns = difference_vector_norms.select(reflections['id'] == i)
counts.append(len(dvns))
if len(dvns) == 0:
data.append(0)
continue
rmsd = math.sqrt(flex.sum_sq(dvns) / len(dvns))
data.append(rmsd)
data *= 1000
subset = data.select(counts > 0)
print len(subset), "experiments with > 0 reflections"
if params.show_plots:
h = flex.histogram(subset, n_slots=40)
fig = plt.figure()
ax = fig.add_subplot('111')
ax.plot(h.slot_centers().as_numpy_array(),
h.slots().as_numpy_array(), '-')
plt.title("Histogram of %d image RMSDs" % len(subset))
fig = plt.figure()
plt.boxplot(subset, vert=False)
plt.title("Boxplot of %d image RMSDs" % len(subset))
plt.show()
outliers = counts == 0
min_x, q1_x, med_x, q3_x, max_x = five_number_summary(subset)
print "Five number summary of RMSDs (microns): min %.1f, q1 %.1f, med %.1f, q3 %.1f, max %.1f" % (
min_x, q1_x, med_x, q3_x, max_x)
iqr_x = q3_x - q1_x
cut_x = params.iqr_multiplier * iqr_x
outliers.set_selected(data > q3_x + cut_x, True)
#outliers.set_selected(col < q1_x - cut_x, True) # Don't throw away the images that are outliers in the 'good' direction!
filtered_reflections = flex.reflection_table()
filtered_experiments = ExperimentList()
for i in xrange(len(experiments)):
if outliers[i]:
continue
refls = reflections.select(reflections['id'] == i)
refls['id'] = flex.int(len(refls), len(filtered_experiments))
filtered_reflections.extend(refls)
filtered_experiments.append(experiments[i])
zeroes = counts == 0
n_zero = len(counts.select(zeroes))
print "Removed %d bad experiments and %d experiments with zero reflections, out of %d (%%%.1f)" % (
len(experiments) - len(filtered_experiments) - n_zero, n_zero,
len(experiments), 100 *
((len(experiments) - len(filtered_experiments)) /
len(experiments)))
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(filtered_experiments)
dump.as_json(params.output.filtered_experiments)
filtered_reflections.as_pickle(params.output.filtered_reflections)
