def main():
from dials.util.options import OptionParser
from dials.util.options import flatten_datablocks
import libtbx.load_env
usage = "%s [options] image_*.cbf" % (
libtbx.env.dispatcher_name)
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_datablocks=True,
read_datablocks_from_images=True,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
if len(datablocks) == 0:
parser.print_help()
exit()
datablock = datablocks[0]
imageset = datablock.extract_imagesets()[0]
stability_fft(imageset, params)
def run(args):
import libtbx.load_env
from dials.util.options import OptionParser
from dials.util.options import flatten_reflections
from libtbx.utils import Sorry
from libtbx.phil import parse
phil_scope = parse(
"""
hklout = hklout.pickle
.type = str
.help = "The output pickle file"
"""
)
usage = "%s integrated.pickle [hklout=hklout.pickle]" % (libtbx.env.dispatcher_name)
parser = OptionParser(usage=usage, read_reflections=True, check_format=False, phil=phil_scope)
params, options = parser.parse_args(show_diff_phil=True)
reflections = flatten_reflections(params.input.reflections)
if len(reflections) != 1:
raise Sorry("exactly 1 reflection table must be specified")
integrated_data = reflections[0]
filter_good_reflections(integrated_data).as_pickle(params.hklout)
class Script(object):
def __init__(self):
# Create the parser
self.parser = OptionParser(
read_experiments=True,
read_datablocks=True,
read_reflections=True,
read_datablocks_from_images=True,
check_format=False,
)
def run(self):
params, options = self.parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(reflections) > 0:
reflections = reflections[0]
else:
reflections = None
all_detectors = []
for db in datablocks:
all_detectors.extend(db.unique_detectors())
all_detectors.extend(experiments.detectors())
display_detectors(all_detectors[: min(len(all_detectors), 10)], reflections=reflections)
def run(args):
import libtbx.load_env
usage = "%s [options]" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
check_format=False,
epilog=help_message)
params, options, args = parser.parse_args(
show_diff_phil=True, return_unhandled=True)
results = []
for mtz in args:
print mtz
assert os.path.isfile(mtz), mtz
results.append(get_merging_stats(
mtz, anomalous=params.anomalous,
n_bins=params.n_bins,
use_internal_variance=params.use_internal_variance,
eliminate_sys_absent=params.eliminate_sys_absent))
plot_merging_stats(results, labels=params.labels,
size_inches=params.size_inches,
image_dir=params.image_dir)
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
from dials.util.options import flatten_reflections
import libtbx.load_env
usage = "%s [options] integrated.pickle experiments.json" % (
libtbx.env.dispatcher_name)
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=False)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(experiments) != 1 or len(reflections) != 1:
parser.print_help()
exit()
if not 'shoebox' in reflections[0]:
print 'Please add shoeboxes to reflection pickle'
exit()
results = main(reflections[0], experiments[0], params)
if results:
print 'mean result: %.3f' % (sum(results) / len(results))
def run(args):
from dials.util.options import OptionParser
import libtbx.load_env
usage = "%s [options] find_spots.json" %(
libtbx.env.dispatcher_name)
parser = OptionParser(
usage=usage,
phil=phil_scope,
epilog=help_message)
filenames = [arg for arg in args if os.path.isfile(arg)]
args = [arg for arg in args if not arg in filenames]
params, options, args = parser.parse_args(
show_diff_phil=True, return_unhandled=True)
if params.nproc is libtbx.Auto:
from libtbx.introspection import number_of_processors
params.nproc = number_of_processors(return_value_if_unknown=-1)
print 'nproc: %i' %params.nproc
results = work_all(filenames, args, nproc=params.nproc)
print results
if params.json is not None:
import json
with open(params.json, 'wb') as f:
json.dump(results, f)
class Script(object):
''' The debugging visualization program. '''
def __init__(self):
'''Initialise the script.'''
from dials.util.options import OptionParser
import libtbx.load_env
# The script usage
usage = "usage: %s [options] experiment.json" \
% libtbx.env.dispatcher_name
# Create the parser
self.parser = OptionParser(
usage=usage,
epilog=help_message,
read_reflections=True)
def run(self):
from dials.util.options import flatten_reflections
from dials.viewer.viewer_interface import extract_n_show
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
table = flatten_reflections(params.input.reflections)
if len(table) == 0:
self.parser.print_help()
return
extract_n_show(table[0])
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_datablocks
import libtbx.load_env
usage = "%s [options] image_*.cbf" % (
libtbx.env.dispatcher_name)
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_datablocks=True,
read_datablocks_from_images=True,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
if len(datablocks) == 0 and len(experiments) == 0 and len(reflections) == 0:
parser.print_help()
exit()
assert(len(datablocks) == 1)
datablock = datablocks[0]
imagesets = datablock.extract_imagesets()
assert(len(imagesets) == 1)
imageset = imagesets[0]
images = imageset.indices()
if params.frames:
images = params.frames
d_spacings = []
intensities = []
sigmas = []
for indx in images:
print 'For frame %d:' % indx
d, I, sig = background(imageset, indx, n_bins=params.n_bins)
print '%8s %8s %8s' % ('d', 'I', 'sig')
for j in range(len(I)):
print '%8.3f %8.3f %8.3f' % (d[j], I[j], sig[j])
d_spacings.append(d)
intensities.append(I)
sigmas.append(sig)
if params.plot:
from matplotlib import pyplot
fig = pyplot.figure()
for d, I, sig in zip(d_spacings, intensities, sigmas):
ds2 = 1/flex.pow2(d)
pyplot.plot(ds2, I)
pyplot.show()
def run(args):
import libtbx.load_env
usage = "%s [options] experiment.json indexed.pickle" % libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message,
)
params, options = parser.parse_args(show_diff_phil=True)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reflections) == 0 or len(experiments) == 0:
parser.print_help()
return
assert len(reflections) == 1
assert len(experiments) == 1
experiment = experiments[0]
reflections = reflections[0]
test_P1_crystal_indexing(reflections, experiment, params)
test_crystal_pointgroup_symmetry(reflections, experiment, params)
def read_experiment_file(self, experiment_file):
### open DIALS json file
phil_scope_str='''
experiments = 'example_refined_experiments.json'
'''
phil_scope = parse(phil_scope_str, process_includes=True)
parser = OptionParser(
phil=phil_scope,
check_format=False,
read_experiments=True)
params, options = parser.parse_args(args=[experiment_file], show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
exp_xtal = experiments.crystals()[0]
### define useful attributes
self.crystal = exp_xtal
uc = self.crystal.get_unit_cell()
uc_nospace = str(uc).replace(" ", "")
uc_nospace_noparen = uc_nospace[1:-1]
self.unit_cell = uc_nospace_noparen
self.space_group = self.crystal.get_space_group()
self.laue_group = self.space_group.laue_group_type()
# self.a_matrix = crystal.get_A()
self.experiments = experiments
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_datablocks
parser = OptionParser(
read_datablocks=True,
read_datablocks_from_images=True,
phil=phil_scope,
check_format=True)
params, options = parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
assert len(datablocks) == 1
imagesets = datablocks[0].extract_imagesets()
img_count = 0
import time
t0 = time.time()
for imgset in imagesets:
for i in range(len(imgset)):
if params.data == 'raw':
imgset.get_raw_data(i)
else:
imgset.get_corrected_data(i)
img_count += 1
print "Read %i images" %img_count
t1 = time.time()
t = t1 - t0
print "Read %i images in %.2fs (%.1f images/s)" %(
img_count, t, img_count/t)
return
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
from libtbx.utils import Sorry
import libtbx.load_env
usage = "%s [options] experiments.json" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) <= 1:
parser.print_help()
return
hkl = flex.miller_index(params.hkl)
from dials.algorithms.indexing.compare_orientation_matrices import \
show_rotation_matrix_differences
show_rotation_matrix_differences(experiments.crystals(),
miller_indices=hkl)
def test_function2():
from dials.util.options import OptionParser
from time import time
from omptbx import omp_get_max_threads
args = [
"/media/upc86896/Betelgeuse/Data/xia2_test_data/i19_sucrose/processed/dataset1/experiments.json",
"/media/upc86896/Betelgeuse/Data/xia2_test_data/i19_sucrose/processed/dataset1/shoeboxes_0.pickle",
]
parser = OptionParser(read_experiments=True, read_reflections=True)
params, options = parser.parse_args(args=args)
experiment = params.input.experiments[0].data[0]
reflections = params.input.reflections[0].data
print "N Threads: ", omp_get_max_threads()
print "N Refl: ", len(reflections)
from dials.algorithms.integration.fitrs import test_function
st = time()
test_function(
experiment.beam, experiment.detector, experiment.goniometer, experiment.scan, 0.071016, 0.390601, 5, reflections
)
print "Time: ", time() - st
def run(args):
import libtbx.load_env
from libtbx.utils import Sorry
from dials.util import log
usage = "%s [options] datablock.json strong.pickle" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_datablocks=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=False)
datablocks = flatten_datablocks(params.input.datablock)
reflections = flatten_reflections(params.input.reflections)
if len(datablocks) == 0 or len(reflections) == 0:
parser.print_help()
exit(0)
# Configure the logging
log.config(
info=params.output.log,
debug=params.output.debug_log)
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil is not '':
info('The following parameters have been modified:\n')
info(diff_phil)
imagesets = []
for datablock in datablocks:
imagesets.extend(datablock.extract_imagesets())
assert len(imagesets) > 0
assert len(reflections) == len(imagesets)
if params.scan_range is not None and len(params.scan_range) > 0:
reflections = [
filter_reflections_by_scan_range(refl, params.scan_range)
for refl in reflections]
dps_params = dps_phil_scope.extract()
# for development, we want an exhaustive plot of beam probability map:
dps_params.indexing.plot_search_scope = params.plot_search_scope
dps_params.indexing.mm_search_scope = params.mm_search_scope
new_detector, new_beam = discover_better_experimental_model(
imagesets, reflections, params, dps_params, nproc=params.nproc,
wide_search_binning=params.wide_search_binning)
for imageset in imagesets:
imageset.set_detector(new_detector)
imageset.set_beam(new_beam)
from dxtbx.serialize import dump
dump.datablock(datablock, params.output.datablock)
def run(args):
from dials.util import log
import libtbx.load_env
usage = "%s experiments.json indexed.pickle [options]" %libtbx.env.dispatcher_name
from dials.util.options import OptionParser
from dials.util.options import flatten_reflections
from dials.util.options import flatten_experiments
from dials.array_family import flex
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=False)
# Configure the logging
#log.config(info=params.output.log, debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
assert(len(reflections) == 1)
reflections = reflections[0]
if len(experiments) == 0:
parser.print_help()
return
#from dials.command_line import refine
#params = refine.phil_scope.extract()
indexed_reflections = reflections.select(reflections['id'] > -1)
from dials.algorithms.refinement import RefinerFactory
refiner = RefinerFactory.from_parameters_data_experiments(
params, indexed_reflections, experiments)
#refiner.run()
rmsds = refiner.rmsds()
import math
xy_rmsds = math.sqrt(rmsds[0]**2 + rmsds[1]**2)
print rmsds
return
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_datablocks
from dials.util.options import flatten_experiments
import libtbx.load_env
usage = "%s [options] datablock.json" %(
libtbx.env.dispatcher_name)
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_datablocks=True,
read_experiments=True,
check_format=True,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
experiments = flatten_experiments(params.input.experiments)
if (len(datablocks) == 0 and len(experiments) == 0):
parser.print_help()
exit(0)
if len(datablocks) == 0 and len(experiments) > 0:
imagesets = experiments.imagesets()
else:
imagesets = []
for datablock in datablocks:
imagesets.extend(datablock.extract_imagesets())
assert len(imagesets) == 1
imageset = imagesets[0]
gonio = imageset.get_goniometer()
if not params.detector_distance:
detector = imageset.get_detector()
if len(detector) > 1:
params.detector_distance = detector.hierarchy().get_directed_distance()
else:
params.detector_distance = detector[0].get_directed_distance()
if params.angle:
assert len(params.angle) == len(gonio.get_angles())
else:
for angle in gonio.get_angles():
params.angle.append(angle)
import wxtbx.app
a = wxtbx.app.CCTBXApp(0)
a.settings = params
f = ExperimentViewer(
None, -1, "Experiment viewer", size=(1024,768))
f.load_imageset(imageset)
f.Show()
a.SetTopWindow(f)
#a.Bind(wx.EVT_WINDOW_DESTROY, lambda evt: tb_icon.Destroy(), f)
a.MainLoop()
class Script(object):
''' The integration program. '''
def __init__(self):
'''Initialise the script.'''
from dials.util.options import OptionParser
import libtbx.load_env
# The script usage
usage = "usage: %s [options] experiment.json" % libtbx.env.dispatcher_name
# Create the parser
self.parser = OptionParser(
usage=usage,
phil=phil_scope,
epilog=help_message,
read_experiments=True)
def run(self):
''' Analyse the background '''
from dials.util.command_line import heading
from dials.util.options import flatten_experiments
from dials.util import log
from logging import info, debug
from time import time
from libtbx.utils import Sorry
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0:
self.parser.print_help()
return
assert len(experiments) == 1
# Get the imageset
imageset = experiments[0].imageset
total_image = None
total_mask = None
for i in range(len(imageset)):
print i
image = imageset.get_raw_data(i)
mask = imageset.get_mask(i)
if total_image is None:
total_image = image[0]
total_mask = mask[0]
else:
total_image += image[0]
total_image /= len(imageset)
print min(total_image)
print max(total_image)
print sum(total_image) / len(total_image)
from matplotlib import pylab
pylab.imshow(total_image.as_numpy_array(), vmin=0,vmax=2)
pylab.show()
def run(args):
import libtbx.load_env
from libtbx.utils import Sorry
from dials.util import log
from logging import info
import cPickle as pickle
usage = "%s [options] datablock.json strong.pickle" % \
libtbx.env.dispatcher_name
# Create the option parser
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_datablocks=True,
check_format=False,
epilog=help_message)
# Get the parameters
params, options = parser.parse_args(show_diff_phil=False)
# Configure the log
log.config(
params.verbosity,
info='dials.find_hot_pixels.log',
debug='dials.find_hot_pixels.debug.log')
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil is not '':
info('The following parameters have been modified:\n')
info(diff_phil)
datablocks = flatten_datablocks(params.input.datablock)
reflections = flatten_reflections(params.input.reflections)
if len(datablocks) == 0 and len(reflections) == 0:
parser.print_help()
exit(0)
if len(datablocks) > 1:
raise Sorry("Only one DataBlock can be processed at a time")
else:
imagesets = datablocks[0].extract_imagesets()
if len(reflections) == 0:
raise Sorry("No reflection lists found in input")
if len(reflections) > 1:
raise Sorry("Multiple reflections lists provided in input")
assert(len(reflections) == 1)
reflections = reflections[0]
mask = hot_pixel_mask(imagesets[0], reflections)
pickle.dump(mask, open(params.output.mask, 'w'), pickle.HIGHEST_PROTOCOL)
print 'Wrote hot pixel mask to %s' % params.output.mask
return
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_datablocks
from dials.util.options import flatten_experiments
from dials.util.options import flatten_reflections
parser = OptionParser(
phil=master_phil,
read_datablocks=True,
read_experiments=True,
read_reflections=True,
check_format=False)
params, options = parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)[0]
if len(params.input.reflections) == 2:
reflections2 = flatten_reflections(params.input.reflections)[1]
else:
reflections2 = None
# find the reflections in the second set that DO NOT match those in the
# first set
mask, _ = reflections2.match_with_reference(reflections)
reflections2 = reflections2.select(~mask)
print "{0} reflections from the second set do not match the first". \
format(len(reflections2))
#reflections2 = reflections2.select(reflections2["miller_index"] == (-7,2,-25))
if len(datablocks) == 0:
if len(experiments) > 0:
imagesets = experiments.imagesets()
else:
parser.print_help()
return
elif len(datablocks) > 1:
raise Sorry("Only one DataBlock can be processed at a time")
else:
imagesets = datablocks[0].extract_imagesets()
if len(imagesets) > 1:
raise Sorry("Only one ImageSet can be processed at a time")
imageset = imagesets[0]
import wxtbx.app
a = wxtbx.app.CCTBXApp(0)
a.settings = params
f = PredRelpViewer(
None, -1, "Prediction reciprocal lattice viewer", size=(1024,768))
f.load_reflections2(reflections2)
f.load_models(imageset, reflections)
f.Show()
a.SetTopWindow(f)
#a.Bind(wx.EVT_WINDOW_DESTROY, lambda evt: tb_icon.Destroy(), f)
a.MainLoop()
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_datablocks
from dials.util.options import flatten_experiments
from dials.util.options import flatten_reflections
import libtbx.load_env
usage = "%s [options] datablock.json reflections.pickle" %(
libtbx.env.dispatcher_name)
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_datablocks=True,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if (len(datablocks) == 0 and len(experiments) == 0) or len(reflections) == 0:
parser.print_help()
exit(0)
if len(datablocks) == 0 and len(experiments) > 0:
imagesets = experiments.imagesets()
else:
imagesets = []
for datablock in datablocks:
imagesets.extend(datablock.extract_imagesets())
if len(reflections) > 1:
assert len(reflections) == len(imagesets)
from scitbx.array_family import flex
for i in range(len(reflections)):
reflections[i]['imageset_id'] = flex.int(len(reflections[i]), i)
if i > 0:
reflections[0].extend(reflections[i])
reflections = reflections[0]
import wxtbx.app
a = wxtbx.app.CCTBXApp(0)
a.settings = params
f = ReciprocalLatticeViewer(
None, -1, "Reflection data viewer", size=(1024,768))
f.load_models(imagesets, reflections)
f.Show()
a.SetTopWindow(f)
#a.Bind(wx.EVT_WINDOW_DESTROY, lambda evt: tb_icon.Destroy(), f)
a.MainLoop()
class Script(object):
''' A class to encapsulate the script. '''
def __init__(self):
''' Initialise the script. '''
from dials.util.options import OptionParser
import libtbx.load_env
# Create the parser
usage = "usage: %s [options] datablock.json" % libtbx.env.dispatcher_name
self.parser = OptionParser(
usage=usage,
epilog=help_message,
phil=phil_scope,
read_datablocks=True)
def run(self):
''' Run the script. '''
from dials.util.options import flatten_datablocks
from dxtbx.datablock import DataBlockDumper
from libtbx.utils import Sorry
import cPickle as pickle
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
# Check number of args
if len(datablocks) == 0:
self.parser.print_help()
return
# Check the mask file is given
if params.input.mask is None:
self.parser.print_help()
return
# Check nbumber of datablocks
if len(datablocks) != 1:
raise Sorry('exactly 1 datablock must be specified')
# Get the imageset
datablock = datablocks[0]
imagesets = datablock.extract_imagesets()
if len(imagesets) != 1:
raise Sorry('datablock must contain exactly 1 imageset')
imageset = imagesets[0]
# Set the lookup
imageset.external_lookup.mask.filename = params.input.mask
# Dump the datablock
print "Writing datablock to %s" % params.output.datablock
dump = DataBlockDumper(datablock)
dump.as_json(filename=params.output.datablock)
def run(args):
import libtbx.load_env
usage = "%s experiments.json [options]" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0:
parser.print_help()
return
elif len(experiments) > 1:
raise Sorry("More than one experiment present")
assert len(params.miller_index), "Must specify at least one miller_index to predict."
experiment = experiments[0]
reflections = flex.reflection_table()
miller_indices = flex.miller_index()
entering_flags = flex.bool()
for mi in params.miller_index:
miller_indices.append(mi)
miller_indices.append(mi)
entering_flags.append(True)
entering_flags.append(False)
reflections['miller_index'] = miller_indices
reflections['entering'] = entering_flags
reflections['id'] = flex.size_t(len(reflections), 0)
if params.expand_to_p1:
from cctbx.miller import expand_to_p1_iselection
proxy = expand_to_p1_iselection(
experiment.crystal.get_space_group(),
anomalous_flag=True,
indices=miller_indices,
build_iselection=True)
reflections = reflections.select(proxy.iselection)
reflections['miller_index'] = proxy.indices
from dials.algorithms.refinement.prediction.managed_predictors import ExperimentsPredictor
predictor = ExperimentsPredictor([experiment])
predicted = predictor.predict(reflections)
zmin, zmax = experiment.scan.get_array_range()
z = predicted['xyzcal.px'].parts()[2]
predicted = predicted.select((z >= zmin) & (z <= zmax))
show_predictions(predicted)
class Script(object):
''' A class to encapsulate the script. '''
def __init__(self):
''' Initialise the script. '''
from dials.util.options import OptionParser
import libtbx.load_env
# The script usage
usage = "usage: %s [options] /path/to/image/reflection/files" % libtbx.env.dispatcher_name
self.parser = OptionParser(
epilog=help_message,
usage=usage,
phil=phil_scope,
read_reflections=True)
def run(self):
''' Run the script. '''
from dials.array_family import flex
from dials.util.command_line import Command
from libtbx.utils import Sorry
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
if len(params.input.reflections) == 0:
self.parser.print_help()
return
if len(params.input.reflections) <= 1:
raise Sorry('more than 1 reflection table must be specified')
tables = [p.data for p in params.input.reflections]
# Get the number of rows and columns
nrows = [t.nrows() for t in tables]
ncols = [t.ncols() for t in tables]
# Merge the reflection lists
if params.method == "update":
assert(all(n == nrows[0] for n in nrows[1:]))
table = tables[0]
for t in tables[1:]:
table.update(t)
elif params.method == "extend":
assert(all(n == ncols[0] for n in ncols[1:]))
table = tables[0]
for t in tables[1:]:
table.extend(t)
else:
raise RuntimeError('unknown method, %s' % params.method)
# Write the reflections to the file
Command.start('Writing %d reflections to %s' % (len(table), params.output))
table.as_pickle(params.output)
Command.end('Wrote %d reflections to %s' % (len(table), params.output))
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_datablocks
from dials.util.options import flatten_experiments
from dials.util.options import flatten_reflections
from dials.util import log
usage = "%s [options] datablock.json reflections.pickle" %(
libtbx.env.dispatcher_name)
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_datablocks=True,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args()
datablocks = flatten_datablocks(params.input.datablock)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if (len(datablocks) == 0 and len(experiments) == 0) or len(reflections) == 0:
parser.print_help()
exit(0)
## Configure the logging
#log.config(info='dials.rl_png.log')
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
reflections = reflections[0]
if len(datablocks) == 0 and len(experiments) > 0:
imagesets = experiments.imagesets()
else:
imagesets = []
for datablock in datablocks:
imagesets.extend(datablock.extract_imagesets())
f = ReciprocalLatticeJson(settings=params)
f.load_models(imagesets, reflections)
f.as_json(filename=params.output.json, compact=params.output.compact)
print
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_datablocks
from dials.util.options import flatten_experiments
import libtbx.load_env
usage = "%s [options] datablock.json | experiments.json" %(
libtbx.env.dispatcher_name)
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_datablocks=True,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
datablocks = flatten_datablocks(params.input.datablock)
if len(experiments) == 0 and len(datablocks) == 0:
parser.print_help()
exit(0)
from dials.command_line.dials_import import ManualGeometryUpdater
update_geometry = ManualGeometryUpdater(params)
if len(experiments):
imagesets = experiments.imagesets()
elif len(datablocks):
assert len(datablocks) == 1
imagesets = datablocks[0].extract_imageset()
for imageset in imagesets:
imageset_new = update_geometry(imageset)
imageset.set_detector(imageset_new.get_detector())
imageset.set_beam(imageset_new.get_beam())
imageset.set_goniometer(imageset_new.get_goniometer())
imageset.set_scan(imageset_new.get_scan())
from dxtbx.serialize import dump
if len(experiments):
print "Saving modified experiments to %s" %params.output.experiments
dump.experiment_list(experiments, params.output.experiments)
elif len(datablocks):
raise NotImplemented
def read_reflection_file(self, reflection_file):
### open DIALS pickle file
phil_scope_str='''
reflections = 'example_refined.pickle'
'''
phil_scope = parse(phil_scope_str, process_includes=True)
parser = OptionParser(
phil=phil_scope,
check_format=False,
read_reflections=True)
params, options = parser.parse_args(args=[reflection_file], show_diff_phil=True)
reflections = flatten_reflections(params.input.reflections)
self.reflections = reflections
def run(args):
import libtbx.load_env
from dials.array_family import flex
from dials.util import log
from dials.util.version import dials_version
usage = "%s [options] experiment.json indexed.pickle" % \
libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
# Configure the logging
log.config(info=params.output.log, debug=params.output.debug_log)
logger.info(dials_version())
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reflections) == 0 or len(experiments) == 0:
parser.print_help()
return
assert(len(reflections) == 1)
assert(len(experiments) == 1)
experiment = experiments[0]
reflections = reflections[0]
# remove reflections with 0, 0, 0 index
zero = (reflections['miller_index'] == (0, 0, 0))
logger.info('Removing %d unindexed reflections' % zero.count(True))
reflections = reflections.select(~zero)
h, k, l = reflections['miller_index'].as_vec3_double().parts()
h = h.iround()
k = k.iround()
l = l.iround()
logger.info('Range on h: %d to %d' % (flex.min(h), flex.max(h)))
logger.info('Range on k: %d to %d' % (flex.min(k), flex.max(k)))
logger.info('Range on l: %d to %d' % (flex.min(l), flex.max(l)))
test_P1_crystal_indexing(reflections, experiment, params)
test_crystal_pointgroup_symmetry(reflections, experiment, params)
class Script(object):
def __init__(self):
# Create the parser
self.parser = OptionParser(
read_experiments=True,
read_datablocks=True,
read_reflections=True,
read_datablocks_from_images=True,
check_format=False)
def run(self):
# load at least two detectors from the command line
params, options = self.parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
experiments = flatten_experiments(params.input.experiments)
# collect all detectors found
all_detectors = []
for db in datablocks:
all_detectors.extend(db.unique_detectors())
all_detectors.extend(experiments.detectors())
assert len(all_detectors) >= 2
a = all_detectors[0]
b = all_detectors[1]
level = 0
pga = a.hierarchy()
pgb = b.hierarchy()
while True:
# starting at the top of the hierarchy, show diffs in local origins at each level
print "Level", level
oa = col(pga.get_local_origin())
ob = col(pgb.get_local_origin())
print " Detector a", oa.elems
print " Detector b", ob.elems
print " Diffs", (ob-oa).elems
if hasattr(pga, 'children'):
pga = pga[0]
pgb = pgb[0]
level += 1
else:
break
def run(args):
import libtbx.load_env
usage = "%s [options]" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
check_format=False,
epilog=help_message)
params, options, args = parser.parse_args(show_diff_phil=True,
return_unhandled=True)
assert len(args) == 1
from iotbx.reflection_file_reader import any_reflection_file
intensities = None
f = args[0]
arrays = any_reflection_file(f).as_miller_arrays(merge_equivalents=False)
for ma in arrays:
print ma.info().labels
if ma.info().labels == ['I', 'SIGI']:
intensities = ma
elif ma.info().labels == ['IMEAN', 'SIGIMEAN']:
intensities = ma
elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']:
intensities = ma
assert intensities is not None
i_sigi = intensities.data()/intensities.sigmas()
# set backend before importing pyplot
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot
pyplot.scatter(intensities.data(), i_sigi, marker='+', s=2, alpha=0.5, c='black')
pyplot.gca().set_xscale('log',basex=10)
xlim = pyplot.xlim()
ylim = pyplot.ylim()
pyplot.ylim(0, ylim[1])
pyplot.xlim(1, xlim[1])
pyplot.savefig('i_sigi_vs_i.png')
pyplot.clf()
class Script(object):
''' Class to parse the command line options. '''
def __init__(self):
''' Set the expected options. '''
from dials.util.options import OptionParser
import libtbx.load_env
# Create the option parser
usage = "usage: %s [options] /path/to/refined/json/file" % libtbx.env.dispatcher_name
self.parser = OptionParser(
usage=usage,
sort_options=True,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
epilog=help_message)
def run(self):
''' Parse the options. '''
from dials.util.options import flatten_experiments, flatten_reflections
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
self.params = params
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
assert len(reflections) == len(experiments) == 1
reflections = reflections[0]
exp = experiments[0]
from dials.algorithms.indexing import index_reflections
from dials.algorithms.indexing.indexer import indexer_base
reflections['id'] = flex.int(len(reflections), -1)
reflections['imageset_id'] = flex.int(len(reflections), 0)
reflections = indexer_base.map_spots_pixel_to_mm_rad(reflections, exp.detector, exp.scan)
indexer_base.map_centroids_to_reciprocal_space(
reflections, exp.detector, exp.beam, exp.goniometer,)
index_reflections(reflections,
experiments, params.d_min,
tolerance=0.3)
indexed_reflections = reflections.select(reflections['miller_index'] != (0,0,0))
print "Indexed %d reflections out of %d"%(len(indexed_reflections), len(reflections))
easy_pickle.dump("indexedstrong.pickle", indexed_reflections)
def run(args=None):
usage = "dials.background [options] image_*.cbf"
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
read_experiments_from_images=True,
epilog=help_message,
)
params, options = parser.parse_args(args, show_diff_phil=True)
# Ensure we have either a data block or an experiment list
experiments = flatten_experiments(params.input.experiments)
imagesets = experiments.imagesets()
if params.output.plot:
import matplotlib
matplotlib.use("agg")
import matplotlib.ticker as mticker
from matplotlib import pyplot
fig = pyplot.figure(figsize=params.output.size_inches)
ax = fig.add_subplot(111)
for i_imgset, imageset in enumerate(imagesets):
first, last = imageset.get_scan().get_image_range()
images = range(first, last + 1)
if params.images:
if min(params.images) < first or max(params.images) > last:
raise Sorry("image outside of scan range")
images = params.images
d_spacings = []
intensities = []
sigmas = []
for indx in images:
print(f"For imageset {i_imgset} image {indx}:")
d, I, sig = background(
imageset,
indx - first, # indices passed to imageset.get_raw_data start from zero
n_bins=params.n_bins,
corrected=params.corrected,
mask_params=params.masking,
)
print(f"{'d':>8} {'I':>8} {'sig':>8}")
for j in range(len(I)):
print(f"{d[j]:8.3f} {I[j]:8.3f} {sig[j]:8.3f}")
d_spacings.append(d)
intensities.append(I)
sigmas.append(sig)
if params.output.plot:
ax.set_xlabel(r"resolution ($\AA$)")
ax.set_ylabel(r"$\langle I_b \rangle$")
for indx, d, I, sig in zip(images, d_spacings, intensities, sigmas):
filenames = imageset.reader().paths()
if len(imagesets) > 1:
label = (
f"{filenames[indx - first]}"
if len(filenames) > 1
else f"{filenames[0]} image {indx}"
)
else:
label = f"image {indx}" if len(images) > 1 else f""
ds2 = 1 / flex.pow2(d)
ax.plot(ds2, I, label=label)
xticks = ax.get_xticks().tolist()
ax.xaxis.set_major_locator(mticker.FixedLocator(xticks))
x_tick_labs = [
"" if e <= 0.0 else f"{math.sqrt(1.0 / e):.2f}" for e in xticks
]
ax.set_xticklabels(x_tick_labs)
if params.output.plot:
try:
if len(imagesets) > 1 or len(images) > 1:
# Plot a legend if there are fewer lines than the number of colours
# in the colour cycle
if len(ax.lines) <= len(
pyplot.rcParams["axes.prop_cycle"].by_key()["color"]
):
pyplot.gca().legend()
pyplot.savefig(params.output.plot)
except ValueError:
raise Sorry(f"Unable to save plot to {params.output.plot}")
class Script(object):
''' Class to parse the command line options. '''
def __init__(self):
''' Set the expected options. '''
from dials.util.options import OptionParser
import libtbx.load_env
# Create the option parser
usage = "usage: %s experiment1.json experiment2.json reflections1.pickle reflections2.pickle" % libtbx.env.dispatcher_name
self.parser = OptionParser(usage=usage,
sort_options=True,
phil=phil_scope,
read_experiments=True,
read_datablocks=True,
read_reflections=True,
epilog=help_message)
def run(self):
''' Parse the options. '''
from dials.util.options import flatten_experiments, flatten_datablocks, flatten_reflections
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
self.params = params
experiments = flatten_experiments(params.input.experiments)
datablocks = flatten_datablocks(params.input.datablock)
reflections = flatten_reflections(params.input.reflections)
# Find all detector objects
detectors = []
detectors.extend(experiments.detectors())
dbs = []
for datablock in datablocks:
dbs.extend(datablock.unique_detectors())
detectors.extend(dbs)
# Verify inputs
if len(detectors) != 2:
print "Please provide two experiments and or datablocks for comparison"
return
# These lines exercise the iterate_detector_at_level and iterate_panels functions
# for a detector with 4 hierarchy levels
"""
print "Testing iterate_detector_at_level"
for level in xrange(4):
print "iterating at level", level
for panelg in iterate_detector_at_level(detectors[0].hierarchy(), 0, level):
print panelg.get_name()
print "Testing iterate_panels"
for level in xrange(4):
print "iterating at level", level
for panelg in iterate_detector_at_level(detectors[0].hierarchy(), 0, level):
for panel in iterate_panels(panelg):
print panel.get_name()
"""
tmp = []
for refls in reflections:
print "N reflections total:", len(refls)
sel = refls.get_flags(refls.flags.used_in_refinement)
if sel.count(True) > 0:
refls = refls.select(sel)
print "N reflections used in refinement", len(refls)
print "Reporting only on those reflections used in refinement"
refls['difference_vector_norms'] = (
refls['xyzcal.mm'] - refls['xyzobs.mm.value']).norms()
tmp.append(refls)
reflections = tmp
s0 = col(
flex.vec3_double([col(b.get_s0())
for b in experiments.beams()]).mean())
# Compute a set of radial and transverse displacements for each reflection
print "Setting up stats..."
tmp_refls = []
for refls, expts in zip(
reflections,
[wrapper.data for wrapper in params.input.experiments]):
tmp = flex.reflection_table()
assert len(expts.detectors()) == 1
dect = expts.detectors()[0]
# Need to construct a variety of vectors
for panel_id, panel in enumerate(dect):
panel_refls = refls.select(refls['panel'] == panel_id)
bcl = flex.vec3_double()
# Compute the beam center in lab space (a vector pointing from the origin to where the beam would intersect
# the panel, if it did intersect the panel)
for expt_id in set(panel_refls['id']):
beam = expts[expt_id].beam
s0_ = beam.get_s0()
expt_refls = panel_refls.select(
panel_refls['id'] == expt_id)
beam_centre = panel.get_beam_centre_lab(s0_)
bcl.extend(flex.vec3_double(len(expt_refls), beam_centre))
panel_refls['beam_centre_lab'] = bcl
# Compute obs in lab space
x, y, _ = panel_refls['xyzobs.mm.value'].parts()
c = flex.vec2_double(x, y)
panel_refls['obs_lab_coords'] = panel.get_lab_coord(c)
# Compute deltaXY in panel space. This vector is relative to the panel origin
x, y, _ = (panel_refls['xyzcal.mm'] -
panel_refls['xyzobs.mm.value']).parts()
# Convert deltaXY to lab space, subtracting off of the panel origin
panel_refls['delta_lab_coords'] = panel.get_lab_coord(
flex.vec2_double(x, y)) - panel.get_origin()
tmp.extend(panel_refls)
refls = tmp
# The radial vector points from the center of the reflection to the beam center
radial_vectors = (refls['obs_lab_coords'] -
refls['beam_centre_lab']).each_normalize()
# The transverse vector is orthogonal to the radial vector and the beam vector
transverse_vectors = radial_vectors.cross(
refls['beam_centre_lab']).each_normalize()
# Compute the raidal and transverse components of each deltaXY
refls['radial_displacements'] = refls['delta_lab_coords'].dot(
radial_vectors)
refls['transverse_displacements'] = refls['delta_lab_coords'].dot(
transverse_vectors)
tmp_refls.append(refls)
reflections = tmp_refls
# storage for plots
refl_counts = {}
# Data for all tables
pg_bc_dists = flex.double()
root1 = detectors[0].hierarchy()
root2 = detectors[1].hierarchy()
all_weights = flex.double()
all_refls_count = flex.int()
# Data for lab space table
lab_table_data = []
lab_delta_table_data = []
all_lab_x = flex.double()
all_lab_y = flex.double()
all_lab_z = flex.double()
pg_lab_x_sigmas = flex.double()
pg_lab_y_sigmas = flex.double()
pg_lab_z_sigmas = flex.double()
all_rotX = flex.double()
all_rotY = flex.double()
all_rotZ = flex.double()
pg_rotX_sigmas = flex.double()
pg_rotY_sigmas = flex.double()
pg_rotZ_sigmas = flex.double()
all_delta_x = flex.double()
all_delta_y = flex.double()
all_delta_z = flex.double()
all_delta_xy = flex.double()
all_delta_xyz = flex.double()
all_delta_r = flex.double()
all_delta_t = flex.double()
all_delta_norm = flex.double()
if params.hierarchy_level > 0:
# Data for local table
local_table_data = []
local_delta_table_data = []
all_local_x = flex.double()
all_local_y = flex.double()
all_local_z = flex.double()
pg_local_x_sigmas = flex.double()
pg_local_y_sigmas = flex.double()
pg_local_z_sigmas = flex.double()
all_local_rotX = flex.double()
all_local_rotY = flex.double()
all_local_rotZ = flex.double()
pg_local_rotX_sigmas = flex.double()
pg_local_rotY_sigmas = flex.double()
pg_local_rotZ_sigmas = flex.double()
all_local_delta_x = flex.double()
all_local_delta_y = flex.double()
all_local_delta_z = flex.double()
all_local_delta_xy = flex.double()
all_local_delta_xyz = flex.double()
# Data for RMSD table
rmsds_table_data = []
for pg_id, (pg1, pg2) in enumerate(
zip(
iterate_detector_at_level(root1, 0,
params.hierarchy_level),
iterate_detector_at_level(root2, 0,
params.hierarchy_level))):
# Count up the number of reflections in this panel group pair for use as a weighting scheme
total_refls = 0
pg1_refls = 0
pg2_refls = 0
for p1, p2 in zip(iterate_panels(pg1), iterate_panels(pg2)):
r1 = len(reflections[0].select(
reflections[0]['panel'] == id_from_name(
detectors[0], p1.get_name())))
r2 = len(reflections[1].select(
reflections[1]['panel'] == id_from_name(
detectors[1], p2.get_name())))
total_refls += r1 + r2
pg1_refls += r1
pg2_refls += r2
if pg1_refls == 0 and pg2_refls == 0:
print "No reflections on panel group", pg_id
continue
all_refls_count.append(total_refls)
all_weights.append(pg1_refls)
all_weights.append(pg2_refls)
assert pg1.get_name() == pg2.get_name()
refl_counts[pg1.get_name()] = total_refls
# Compute RMSDs
row = ["%d" % pg_id]
for pg, refls, det in zip([pg1, pg2], reflections, detectors):
pg_refls = flex.reflection_table()
for p in iterate_panels(pg):
pg_refls.extend(
refls.select(
refls['panel'] == id_from_name(det, p.get_name())))
if len(pg_refls) == 0:
rmsd = r_rmsd = t_rmsd = 0
else:
rmsd = math.sqrt(
flex.sum_sq(pg_refls['difference_vector_norms']) /
len(pg_refls)) * 1000
r_rmsd = math.sqrt(
flex.sum_sq(pg_refls['radial_displacements']) /
len(pg_refls)) * 1000
t_rmsd = math.sqrt(
flex.sum_sq(pg_refls['transverse_displacements']) /
len(pg_refls)) * 1000
row.extend([
"%6.1f" % rmsd,
"%6.1f" % r_rmsd,
"%6.1f" % t_rmsd,
"%8d" % len(pg_refls)
])
rmsds_table_data.append(row)
dists = flex.double()
lab_x = flex.double()
lab_y = flex.double()
lab_z = flex.double()
rot_X = flex.double()
rot_Y = flex.double()
rot_Z = flex.double()
for pg in [pg1, pg2]:
bc = col(pg.get_beam_centre_lab(s0))
ori = get_center(pg)
dists.append((ori - bc).length())
ori_lab = pg.get_origin()
lab_x.append(ori_lab[0])
lab_y.append(ori_lab[1])
lab_z.append(ori_lab[2])
f = col(pg.get_fast_axis())
s = col(pg.get_slow_axis())
n = col(pg.get_normal())
basis = sqr(
[f[0], s[0], n[0], f[1], s[1], n[1], f[2], s[2], n[2]])
rotX, rotY, rotZ = basis.r3_rotation_matrix_as_x_y_z_angles(
deg=True)
rot_X.append(rotX)
rot_Y.append(rotY)
rot_Z.append(rotZ)
all_lab_x.extend(lab_x)
all_lab_y.extend(lab_y)
all_lab_z.extend(lab_z)
all_rotX.extend(rot_X)
all_rotY.extend(rot_Y)
all_rotZ.extend(rot_Z)
pg_weights = flex.double([pg1_refls, pg2_refls])
if 0 in pg_weights:
dist_m = dist_s = 0
lx_m = lx_s = ly_m = ly_s = lz_m = lz_s = 0
lrx_m = lrx_s = lry_m = lry_s = lrz_m = lrz_s = 0
dx = dy = dz = dxy = dxyz = dr = dt = dnorm = 0
else:
stats = flex.mean_and_variance(dists, pg_weights)
dist_m = stats.mean()
dist_s = stats.gsl_stats_wsd()
stats = flex.mean_and_variance(lab_x, pg_weights)
lx_m = stats.mean()
lx_s = stats.gsl_stats_wsd()
stats = flex.mean_and_variance(lab_y, pg_weights)
ly_m = stats.mean()
ly_s = stats.gsl_stats_wsd()
stats = flex.mean_and_variance(lab_z, pg_weights)
lz_m = stats.mean()
lz_s = stats.gsl_stats_wsd()
stats = flex.mean_and_variance(rot_X, pg_weights)
lrx_m = stats.mean()
lrx_s = stats.gsl_stats_wsd()
stats = flex.mean_and_variance(rot_Y, pg_weights)
lry_m = stats.mean()
lry_s = stats.gsl_stats_wsd()
stats = flex.mean_and_variance(rot_Z, pg_weights)
lrz_m = stats.mean()
lrz_s = stats.gsl_stats_wsd()
dx = lab_x[0] - lab_x[1]
dy = lab_y[0] - lab_y[1]
dz = lab_z[0] - lab_z[1]
dxy = math.sqrt(dx**2 + dy**2)
dxyz = math.sqrt(dx**2 + dy**2 + dz**2)
delta = col([lab_x[0], lab_y[0], lab_z[0]]) - col(
[lab_x[1], lab_y[1], lab_z[1]])
pg1_center = get_center_lab(pg1).normalize()
transverse = s0.cross(pg1_center).normalize()
radial = transverse.cross(s0).normalize()
dr = delta.dot(radial)
dt = delta.dot(transverse)
dnorm = col(pg1.get_normal()).angle(col(pg2.get_normal()),
deg=True)
pg_bc_dists.append(dist_m)
pg_lab_x_sigmas.append(lx_s)
pg_lab_y_sigmas.append(ly_s)
pg_lab_z_sigmas.append(lz_s)
pg_rotX_sigmas.append(lrx_s)
pg_rotY_sigmas.append(lry_s)
pg_rotZ_sigmas.append(lrz_s)
all_delta_x.append(dx)
all_delta_y.append(dy)
all_delta_z.append(dz)
all_delta_xy.append(dxy)
all_delta_xyz.append(dxyz)
all_delta_r.append(dr)
all_delta_t.append(dt)
all_delta_norm.append(dnorm)
lab_table_data.append([
"%d" % pg_id,
"%5.1f" % dist_m,
"%9.3f" % lx_m,
"%9.3f" % lx_s,
"%9.3f" % ly_m,
"%9.3f" % ly_s,
"%9.3f" % lz_m,
"%9.3f" % lz_s,
"%9.3f" % lrx_m,
"%9.3f" % lrx_s,
"%9.3f" % lry_m,
"%9.3f" % lry_s,
"%9.3f" % lrz_m,
"%9.3f" % lrz_s,
"%6d" % total_refls
])
lab_delta_table_data.append([
"%d" % pg_id,
"%5.1f" % dist_m,
"%9.1f" % (dx * 1000),
"%9.1f" % (dy * 1000),
"%9.3f" % dz,
"%9.1f" % (dxy * 1000),
"%9.3f" % dxyz,
"%9.1f" % (dr * 1000),
"%9.1f" % (dt * 1000),
"%9.3f" % dnorm,
"%6d" % total_refls
])
if params.hierarchy_level > 0:
local_x = flex.double()
local_y = flex.double()
local_z = flex.double()
l_rot_X = flex.double()
l_rot_Y = flex.double()
l_rot_Z = flex.double()
l_dx = flex.double()
l_dy = flex.double()
l_dz = flex.double()
l_dxy = flex.double()
l_dxyz = flex.double()
for pg in [pg1, pg2]:
l_ori = pg.get_local_origin()
local_x.append(l_ori[0])
local_y.append(l_ori[1])
local_z.append(l_ori[2])
f = col(pg.get_local_fast_axis())
s = col(pg.get_local_slow_axis())
n = f.cross(s)
basis = sqr(
[f[0], s[0], n[0], f[1], s[1], n[1], f[2], s[2], n[2]])
rotX, rotY, rotZ = basis.r3_rotation_matrix_as_x_y_z_angles(
deg=True)
l_rot_X.append(rotX)
l_rot_Y.append(rotY)
l_rot_Z.append(rotZ)
all_local_x.extend(local_x)
all_local_y.extend(local_y)
all_local_z.extend(local_z)
all_local_rotX.extend(l_rot_X)
all_local_rotY.extend(l_rot_Y)
all_local_rotZ.extend(l_rot_Z)
pg_weights = flex.double([pg1_refls, pg2_refls])
if 0 in pg_weights:
lx_m = lx_s = ly_m = ly_s = lz_m = lz_s = 0
lrx_m = lrx_s = lry_m = lry_s = lrz_m = lrz_s = 0
ldx = ldy = ldz = ldxy = ldxyz = 0
else:
stats = flex.mean_and_variance(local_x, pg_weights)
lx_m = stats.mean()
lx_s = stats.gsl_stats_wsd()
stats = flex.mean_and_variance(local_y, pg_weights)
ly_m = stats.mean()
ly_s = stats.gsl_stats_wsd()
stats = flex.mean_and_variance(local_z, pg_weights)
lz_m = stats.mean()
lz_s = stats.gsl_stats_wsd()
stats = flex.mean_and_variance(l_rot_X, pg_weights)
lrx_m = stats.mean()
lrx_s = stats.gsl_stats_wsd()
stats = flex.mean_and_variance(l_rot_Y, pg_weights)
lry_m = stats.mean()
lry_s = stats.gsl_stats_wsd()
stats = flex.mean_and_variance(l_rot_Z, pg_weights)
lrz_m = stats.mean()
lrz_s = stats.gsl_stats_wsd()
ldx = local_x[0] - local_x[1]
ldy = local_y[0] - local_y[1]
ldz = local_z[0] - local_z[1]
ldxy = math.sqrt(ldx**2 + ldy**2)
ldxyz = math.sqrt(ldx**2 + ldy**2 + ldz**2)
pg_local_x_sigmas.append(lx_s)
pg_local_y_sigmas.append(ly_s)
pg_local_z_sigmas.append(lz_s)
pg_local_rotX_sigmas.append(lrx_s)
pg_local_rotY_sigmas.append(lry_s)
pg_local_rotZ_sigmas.append(lrz_s)
all_local_delta_x.append(ldx)
all_local_delta_y.append(ldy)
all_local_delta_z.append(ldz)
all_local_delta_xy.append(ldxy)
all_local_delta_xyz.append(ldxyz)
local_table_data.append([
"%d" % pg_id,
"%5.1f" % dist_m,
"%9.3f" % lx_m,
"%9.3f" % lx_s,
"%9.3f" % ly_m,
"%9.3f" % ly_s,
"%9.3f" % lz_m,
"%9.3f" % lz_s,
"%9.3f" % lrx_m,
"%9.3f" % lrx_s,
"%9.3f" % lry_m,
"%9.3f" % lry_s,
"%9.3f" % lrz_m,
"%9.3f" % lrz_s,
"%6d" % total_refls
])
local_delta_table_data.append([
"%d" % pg_id,
"%5.1f" % dist_m,
"%9.1f" % (ldx * 1000),
"%9.1f" % (ldy * 1000),
"%9.3f" % ldz,
"%9.1f" % (ldxy * 1000),
"%9.3f" % ldxyz,
"%6d" % total_refls
])
# Set up table output, starting with lab table
table_d = {d: row for d, row in zip(pg_bc_dists, lab_table_data)}
table_header = [
"PanelG", "Radial", "Lab X", "Lab X", "Lab Y", "Lab Y", "Lab Z",
"Lab Z", "Rot X", "Rot X", "Rot Y", "Rot Y", "Rot Z", "Rot Z", "N"
]
table_header2 = [
"Id", "Dist", "", "Sigma", "", "Sigma", "", "Sigma", "", "Sigma",
"", "Sigma", "", "Sigma", "Refls"
]
table_header3 = [
"", "(mm)", "(mm)", "(mm)", "(mm)", "(mm)", "(mm)", "(mm)",
"(deg)", "(deg)", "(deg)", "(deg)", "(deg)", "(deg)", ""
]
lab_table_data = [table_header, table_header2, table_header3]
lab_table_data.extend([table_d[key] for key in sorted(table_d)])
if len(all_weights) > 1:
r1 = ["All"]
r2 = ["Mean"]
for data, weights, fmt in [
[None, None, None],
[all_lab_x, all_weights.as_double(), "%9.3f"],
[pg_lab_x_sigmas,
all_refls_count.as_double(), "%9.3f"],
[all_lab_y, all_weights.as_double(), "%9.3f"],
[pg_lab_y_sigmas,
all_refls_count.as_double(), "%9.3f"],
[all_lab_z, all_weights.as_double(), "%9.3f"],
[pg_lab_z_sigmas,
all_refls_count.as_double(), "%9.3f"],
[all_rotX, all_weights.as_double(), "%9.3f"],
[pg_rotX_sigmas,
all_refls_count.as_double(), "%9.3f"],
[all_rotY, all_weights.as_double(), "%9.3f"],
[pg_rotY_sigmas,
all_refls_count.as_double(), "%9.3f"],
[all_rotZ, all_weights.as_double(), "%9.3f"],
[pg_rotZ_sigmas,
all_refls_count.as_double(), "%9.3f"]
]:
r2.append("")
if data is None and weights is None:
r1.append("")
continue
stats = flex.mean_and_variance(data, weights)
r1.append(fmt % stats.mean())
r1.append("")
r2.append("%6.1f" % flex.mean(all_refls_count.as_double()))
lab_table_data.append(r1)
lab_table_data.append(r2)
from libtbx import table_utils
print "Detector statistics relative to lab origin"
print table_utils.format(lab_table_data,
has_header=3,
justify='center',
delim=" ")
print "PanelG Id: panel group id or panel id, depending on hierarchy_level. For each panel group, weighted means and weighted standard deviations (Sigmas) for the properties listed below are computed using the matching panel groups between the input experiments."
print "Radial dist: distance from center of panel group to the beam center"
print "Lab X, Y and Z: mean coordinate in lab space"
print "Rot X, Y and Z: rotation of panel group around lab X, Y and Z axes"
print "N refls: number of reflections summed between both matching panel groups. This number is used as a weight when computing means and standard deviations."
print "All: weighted mean of the values shown"
print
# Next, deltas in lab space
table_d = {d: row for d, row in zip(pg_bc_dists, lab_delta_table_data)}
table_header = [
"PanelG", "Radial", "Lab dX", "Lab dY", "Lab dZ", "Lab dXY",
"Lab dXYZ", "Lab dR", "Lab dT", "Lab dNorm", "N"
]
table_header2 = ["Id", "Dist", "", "", "", "", "", "", "", "", "Refls"]
table_header3 = [
"", "(mm)", "(microns)", "(microns)", "(mm)", "(microns)", "(mm)",
"(microns)", "(microns)", "(deg)", ""
]
lab_delta_table_data = [table_header, table_header2, table_header3]
lab_delta_table_data.extend([table_d[key] for key in sorted(table_d)])
if len(all_weights) > 1:
r1 = ["WMean"]
r2 = ["WStddev"]
r3 = ["Mean"]
for data, weights, fmt in [
[None, None, None],
[all_delta_x * 1000,
all_refls_count.as_double(), "%9.1f"],
[all_delta_y * 1000,
all_refls_count.as_double(), "%9.1f"],
[all_delta_z,
all_refls_count.as_double(), "%9.3f"],
[all_delta_xy * 1000,
all_refls_count.as_double(), "%9.1f"],
[all_delta_xyz,
all_refls_count.as_double(), "%9.3f"],
[all_delta_r * 1000,
all_refls_count.as_double(), "%9.1f"],
[all_delta_t * 1000,
all_refls_count.as_double(), "%9.1f"],
[all_delta_norm,
all_refls_count.as_double(), "%9.3f"]
]:
r3.append("")
if data is None and weights is None:
r1.append("")
r2.append("")
continue
stats = flex.mean_and_variance(data, weights)
r1.append(fmt % stats.mean())
if len(data) > 1:
r2.append(fmt % stats.gsl_stats_wsd())
else:
r2.append("-")
r1.append("")
r2.append("")
r3.append("%6.1f" % flex.mean(all_refls_count.as_double()))
lab_delta_table_data.append(r1)
lab_delta_table_data.append(r2)
lab_delta_table_data.append(r3)
print "Detector deltas in lab space"
print table_utils.format(lab_delta_table_data,
has_header=3,
justify='center',
delim=" ")
print "PanelG Id: panel group id or panel id, depending on hierarchy_level. For each panel group, weighted means and weighted standard deviations (Sigmas) for the properties listed below are computed using the matching panel groups between the input experiments."
print "Radial dist: distance from center of panel group to the beam center"
print "Lab dX, dY and dZ: delta between X, Y and Z coordinates in lab space"
print "Lab dR, dT and dZ: radial and transverse components of dXY in lab space"
print "Lab dNorm: angle between normal vectors in lab space"
print "N refls: number of reflections summed between both matching panel groups. This number is used as a weight when computing means and standard deviations."
print "WMean: weighted mean of the values shown"
print "WStddev: weighted standard deviation of the values shown"
print "Mean: mean of the values shown"
print
if params.hierarchy_level > 0:
# Local table
table_d = {d: row for d, row in zip(pg_bc_dists, local_table_data)}
table_header = [
"PanelG", "Radial", "Local X", "Local X", "Local Y", "Local Y",
"Local Z", "Local Z", "Rot X", "Rot X", "Rot Y", "Rot Y",
"Rot Z", "Rot Z", "N"
]
table_header2 = [
"Id", "Dist", "", "Sigma", "", "Sigma", "", "Sigma", "",
"Sigma", "", "Sigma", "", "Sigma", "Refls"
]
table_header3 = [
"", "(mm)", "(mm)", "(mm)", "(mm)", "(mm)", "(mm)", "(mm)",
"(deg)", "(deg)", "(deg)", "(deg)", "(deg)", "(deg)", ""
]
local_table_data = [table_header, table_header2, table_header3]
local_table_data.extend([table_d[key] for key in sorted(table_d)])
if len(all_weights) > 1:
r1 = ["All"]
r2 = ["Mean"]
for data, weights, fmt in [
[None, None, None],
[all_local_x,
all_weights.as_double(), "%9.3f"],
[pg_local_x_sigmas,
all_refls_count.as_double(), "%9.3f"],
[all_local_y,
all_weights.as_double(), "%9.3f"],
[pg_local_y_sigmas,
all_refls_count.as_double(), "%9.3f"],
[all_local_z,
all_weights.as_double(), "%9.3f"],
[pg_local_z_sigmas,
all_refls_count.as_double(), "%9.3f"],
[all_local_rotX,
all_weights.as_double(), "%9.3f"],
[
pg_local_rotX_sigmas,
all_refls_count.as_double(), "%9.3f"
], [all_local_rotY,
all_weights.as_double(), "%9.3f"],
[
pg_local_rotY_sigmas,
all_refls_count.as_double(), "%9.3f"
], [all_local_rotZ,
all_weights.as_double(), "%9.3f"],
[
pg_local_rotZ_sigmas,
all_refls_count.as_double(), "%9.3f"
]
]:
r2.append("")
if data is None and weights is None:
r1.append("")
continue
stats = flex.mean_and_variance(data, weights)
r1.append(fmt % stats.mean())
r1.append("")
r2.append("%6.1f" % flex.mean(all_refls_count.as_double()))
local_table_data.append(r1)
local_table_data.append(r2)
print "Detector statistics in local frame of each panel group"
print table_utils.format(local_table_data,
has_header=3,
justify='center',
delim=" ")
print "PanelG Id: panel group id or panel id, depending on hierarchy_level. For each panel group, weighted means and weighted standard deviations (Sigmas) for the properties listed below are computed using the matching panel groups between the input experiments."
print "Radial dist: distance from center of panel group to the beam center"
print "Lab X, Y and Z: mean coordinate in relative to parent panel group"
print "Rot X, Y and Z: rotation of panel group around parent panel group X, Y and Z axes"
print "N refls: number of reflections summed between both matching panel groups. This number is used as a weight when computing means and standard deviations."
print "All: weighted mean of the values shown"
print
# Next, deltas in local space
table_d = {
d: row
for d, row in zip(pg_bc_dists, local_delta_table_data)
}
table_header = [
"PanelG", "Radial", "Local dX", "Local dY", "Local dZ",
"Local dXY", "Local dXYZ", "N"
]
table_header2 = ["Id", "Dist", "", "", "", "", "", "Refls"]
table_header3 = [
"", "(mm)", "(microns)", "(microns)", "(mm)", "(microns)",
"(mm)", ""
]
local_delta_table_data = [
table_header, table_header2, table_header3
]
local_delta_table_data.extend(
[table_d[key] for key in sorted(table_d)])
if len(all_weights) > 1:
r1 = ["WMean"]
r2 = ["WStddev"]
r3 = ["Mean"]
for data, weights, fmt in [
[None, None, None],
[
all_local_delta_x * 1000,
all_refls_count.as_double(), "%9.1f"
],
[
all_local_delta_y * 1000,
all_refls_count.as_double(), "%9.1f"
],
[all_local_delta_z,
all_refls_count.as_double(), "%9.3f"],
[
all_local_delta_xy * 1000,
all_refls_count.as_double(), "%9.1f"
],
[
all_local_delta_xyz,
all_refls_count.as_double(), "%9.3f"
]
]:
r3.append("")
if data is None and weights is None:
r1.append("")
r2.append("")
continue
stats = flex.mean_and_variance(data, weights)
r1.append(fmt % stats.mean())
r2.append(fmt % stats.gsl_stats_wsd())
r1.append("")
r2.append("")
r3.append("%6.1f" % flex.mean(all_refls_count.as_double()))
local_delta_table_data.append(r1)
local_delta_table_data.append(r2)
local_delta_table_data.append(r3)
print "Detector deltas relative to panel group origin"
print table_utils.format(local_delta_table_data,
has_header=3,
justify='center',
delim=" ")
print "PanelG Id: panel group id or panel id, depending on hierarchy_level. For each panel group, weighted means and weighted standard deviations (Sigmas) for the properties listed below are computed using the matching panel groups between the input experiments."
print "Radial dist: distance from center of panel group to the beam center"
print "Local dX, dY and dZ: delta between X, Y and Z coordinates in the local frame of the panel group"
print "N refls: number of reflections summed between both matching panel groups. This number is used as a weight when computing means and standard deviations."
print "All: weighted mean of the values shown"
print
#RMSD table
table_d = {d: row for d, row in zip(pg_bc_dists, rmsds_table_data)}
table_header = ["PanelG"]
table_header2 = ["Id"]
table_header3 = [""]
for i in xrange(len(detectors)):
table_header.extend(["D%d" % i] * 4)
table_header2.extend(["RMSD", "rRMSD", "tRMSD", "N refls"])
table_header3.extend(["(microns)"] * 3)
table_header3.append("")
rmsds_table_data = [table_header, table_header2, table_header3]
rmsds_table_data.extend([table_d[key] for key in sorted(table_d)])
row = ["Overall"]
for refls in reflections:
row.append("%6.1f" % (math.sqrt(
flex.sum_sq(refls['difference_vector_norms']) / len(refls)) *
1000))
row.append("%6.1f" % (math.sqrt(
flex.sum_sq(refls['radial_displacements']) / len(refls)) *
1000))
row.append("%6.1f" % (math.sqrt(
flex.sum_sq(refls['transverse_displacements']) / len(refls)) *
1000))
row.append("%8d" % len(refls))
rmsds_table_data.append(row)
print "RMSDs by detector number"
print table_utils.format(rmsds_table_data,
has_header=3,
justify='center',
delim=" ")
print "PanelG Id: panel group id or panel id, depending on hierarchy_level"
print "RMSD: root mean squared deviation between observed and predicted spot locations"
print "rRMSD: RMSD of radial components of the observed-predicted vectors"
print "tRMSD: RMSD of transverse components of the observed-predicted vectors"
print "N refls: number of reflections"
if params.tag is None:
tag = ""
else:
tag = "%s " % params.tag
if params.show_plots:
# Plot the results
self.detector_plot_dict(detectors[0],
refl_counts,
u"%sN reflections" % tag,
u"%6d",
show=False)
def detector_plot_dict(self,
detector,
data,
title,
units_str,
show=True,
reverse_colormap=False):
"""
Use matplotlib to plot a detector, color coding panels according to data
@param detector detector reference detector object
@param data python dictionary of panel names as keys and numbers as values
@param title title string for plot
@param units_str string with a formatting statment for units on each panel
"""
# initialize the color map
values = flex.double(data.values())
norm = Normalize(vmin=flex.min(values), vmax=flex.max(values))
if reverse_colormap:
cmap = plt.cm.get_cmap(self.params.colormap + "_r")
else:
cmap = plt.cm.get_cmap(self.params.colormap)
sm = cm.ScalarMappable(norm=norm, cmap=cmap)
if len(values) == 0:
print "no values"
return
elif len(values) == 1:
sm.set_array(np.arange(values[0], values[0],
1)) # needed for colorbar
else:
sm.set_array(
np.arange(flex.min(values), flex.max(values),
(flex.max(values) - flex.min(values)) /
20)) # needed for colorbar
fig = plt.figure()
ax = fig.add_subplot(111, aspect='equal')
max_dim = 0
root = detector.hierarchy()
rf = col(root.get_fast_axis())
rs = col(root.get_slow_axis())
for pg_id, pg in enumerate(
iterate_detector_at_level(root, 0,
self.params.hierarchy_level)):
if pg.get_name() not in data:
continue
# get panel coordinates
p0, p1, p2, p3 = get_bounds(root, pg)
v1 = p1 - p0
v2 = p3 - p0
vcen = ((v2 / 2) + (v1 / 2)) + p0
# add the panel to the plot
ax.add_patch(
Polygon((p0[0:2], p1[0:2], p2[0:2], p3[0:2]),
closed=True,
color=sm.to_rgba(data[pg.get_name()]),
fill=True))
ax.annotate("%d %s" % (pg_id, units_str % data[pg.get_name()]),
vcen[0:2],
ha='center')
if self.params.draw_normal_arrows:
pgn = col(pg.get_normal())
v = col((rf.dot(pgn), rs.dot(pgn), 0))
v *= 10000
ax.arrow(vcen[0],
vcen[1],
v[0],
v[1],
head_width=5.0,
head_length=10.0,
fc='k',
ec='k')
# find the plot maximum dimensions
for p in [p0, p1, p2, p3]:
for c in p[0:2]:
if abs(c) > max_dim:
max_dim = abs(c)
# plot the results
ax.set_xlim((-max_dim, max_dim))
ax.set_ylim((-max_dim, max_dim))
ax.set_xlabel("mm")
ax.set_ylabel("mm")
fig.colorbar(sm)
plt.title(title)
if show:
plt.show()
def run(args: List[str] = None, phil: libtbx.phil.scope = phil_scope) -> None:
usage = "dials.missing_reflections [options] scaled.expt scaled.refl"
parser = OptionParser(
usage=usage,
phil=phil,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=__doc__,
)
params, options = parser.parse_args(args=args, show_diff_phil=False)
# Configure the logging.
dials.util.log.config(options.verbose)
logger.info(dials_version())
# Log the difference between the PHIL scope definition and the active PHIL scope,
# which will include the parsed user inputs.
diff_phil = parser.diff_phil.as_str()
if diff_phil:
logger.info("The following parameters have been modified:\n%s",
diff_phil)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if not experiments or not reflections:
parser.print_help()
return
if len(reflections) != 1 and len(experiments) != len(reflections):
sys.exit(
"Number of experiments must equal the number of reflection tables")
from dials.util.multi_dataset_handling import (
assign_unique_identifiers,
parse_multiple_datasets,
)
reflections = parse_multiple_datasets(reflections)
experiments, reflections = assign_unique_identifiers(
experiments, reflections)
if all("inverse_scale_factor" in refl for refl in reflections):
# Assume all arrays have been scaled
miller_array = scaled_data_as_miller_array(reflections,
experiments,
anomalous_flag=False)
else:
# Else get the integrated intensities
miller_arrays = filtered_arrays_from_experiments_reflections(
experiments,
reflections,
)
miller_array = miller_arrays[0]
for ma in miller_arrays[1:]:
miller_array = miller_array.concatenate(ma)
if params.d_min or params.d_max:
miller_array = miller_array.resolution_filter(d_min=params.d_min,
d_max=params.d_max)
# Print overall summary of input miller array
s = io.StringIO()
ma_unique = miller_array.unique_under_symmetry()
ma_unique.show_comprehensive_summary(f=s)
logger.info(f"\n{s.getvalue()}")
# Get the regions of missing reflections
complete_set, unique_ms = missing_reflections.connected_components(
miller_array)
unique_ms = [
ms for ms in unique_ms if ms.size() >= params.min_component_size
]
# Print some output for user
if len(unique_ms):
logger.info(
"The following connected regions of missing reflections have been identified:"
)
n_expected = complete_set.size()
rows = []
for ms in unique_ms:
d_max, d_min = (uctbx.d_star_sq_as_d(ds2)
for ds2 in ms.min_max_d_star_sq())
rows.append([
ms.size(),
f"{100 * ms.size() / n_expected:.1f}",
f"{d_max:.2f}-{d_min:.2f}",
])
logger.info(
tabulate(
rows,
headers=["# reflections", "% missing",
"Resolution range (Å)"]))
else:
logger.info("No connected regions of missing reflections identified")
def run(args=None):
usage = "usage: dxtbx.detector_superpose reference.json moving.json "
parser = OptionParser(
usage=usage,
sort_options=True,
phil=phil_scope,
check_format=False,
epilog=help_message,
)
params, options = parser.parse_args(args, show_diff_phil=True)
reference_experiments = ExperimentListFactory.from_json_file(
params.reference_experiments, check_format=False)
if len(reference_experiments.detectors()) != 1:
sys.exit("Error: Please ensure reference has only 1 detector model")
reference = reference_experiments.detectors()[0]
moving_experiments = ExperimentListFactory.from_json_file(
params.moving_experiments, check_format=False)
if len(moving_experiments.detectors()) != 1:
sys.exit("Error: Please ensure moving has only 1 detector model")
moving = moving_experiments.detectors()[0]
# Get list of panels to compare
if params.panel_list is None or len(params.panel_list) == 0:
assert len(reference) == len(moving), "Detectors not same length"
panel_ids = list(range(len(reference)))
else:
max_p_id = max(params.panel_list)
assert max_p_id < len(reference), (
"Reference detector must be at least %d panels long given the panel list"
% (max_p_id + 1))
assert max_p_id < len(
moving
), "Moving detector must be at least %d panels long given the panel list" % (
max_p_id + 1)
panel_ids = params.panel_list
if params.fit_target == "centers":
assert (
len(panel_ids) >= 3
), "When using centers as target for superpose, detector needs at least 3 panels"
def rmsd_from_centers(a, b):
assert len(a) == len(b)
assert len(a) % 4 == len(b) % 4 == 0
ca = flex.vec3_double()
cb = flex.vec3_double()
for i in range(len(a) // 4):
ca.append(a[i:i + 4].mean())
cb.append(b[i:i + 4].mean())
return 1000 * math.sqrt((ca - cb).sum_sq() / len(ca))
cycles = 0
while True:
cycles += 1
# Treat panels as a list of 4 sites (corners) or 1 site (centers) for use with lsq superpose
reference_sites = flex.vec3_double()
moving_sites = flex.vec3_double()
for panel_id in panel_ids:
for detector, sites in zip([reference, moving],
[reference_sites, moving_sites]):
panel = detector[panel_id]
size = panel.get_image_size()
corners = flex.vec3_double([
panel.get_pixel_lab_coord(point) for point in [
(0, 0),
(0, size[1] - 1),
(size[0] - 1, size[1] - 1),
(size[0] - 1, 0),
]
])
if params.fit_target == "corners":
sites.extend(corners)
elif params.fit_target == "centers":
sites.append(corners.mean())
# Compute super position
rmsd = 1000 * math.sqrt(
(reference_sites - moving_sites).sum_sq() / len(reference_sites))
print("RMSD before fit: %.1f microns" % rmsd)
if params.fit_target == "corners":
rmsd = rmsd_from_centers(reference_sites, moving_sites)
print("RMSD of centers before fit: %.1f microns" % rmsd)
lsq = least_squares_fit(reference_sites, moving_sites)
rmsd = 1000 * math.sqrt(
(reference_sites - lsq.other_sites_best_fit()).sum_sq() /
len(reference_sites))
print("RMSD of fit: %.1f microns" % rmsd)
if params.fit_target == "corners":
rmsd = rmsd_from_centers(reference_sites,
lsq.other_sites_best_fit())
print("RMSD of fit of centers: %.1f microns" % rmsd)
(
angle,
axis,
) = lsq.r.r3_rotation_matrix_as_unit_quaternion(
).unit_quaternion_as_axis_and_angle(deg=True)
print("Axis and angle of rotation: (%.3f, %.3f, %.3f), %.2f degrees" %
(axis[0], axis[1], axis[2], angle))
print("Translation (x, y, z, in microns): (%.3f, %.3f, %.3f)" %
(1000 * lsq.t).elems)
# Apply the shifts
if params.apply_at_hierarchy_level is None:
iterable = moving
else:
iterable = iterate_detector_at_level(
moving.hierarchy(), level=params.apply_at_hierarchy_level)
for group in iterable:
fast = col(group.get_fast_axis())
slow = col(group.get_slow_axis())
ori = col(group.get_origin())
group.set_frame(lsq.r * fast, lsq.r * slow, (lsq.r * ori) + lsq.t)
if not params.repeat_until_converged:
break
if approx_equal(angle, 0.0, out=None) and approx_equal(
(1000 * lsq.t).length(), 0.0, out=None):
print("Converged after", cycles, "cycles")
break
else:
print("Movement not close to zero, repeating fit")
print()
dump.experiment_list(moving_experiments, params.output_experiments)
moved_sites = flex.vec3_double()
for panel_id in panel_ids:
panel = moving[panel_id]
size = panel.get_image_size()
corners = flex.vec3_double([
panel.get_pixel_lab_coord(point) for point in [
(0, 0),
(0, size[1] - 1),
(size[0] - 1, size[1] - 1),
(size[0] - 1, 0),
]
])
if params.fit_target == "corners":
moved_sites.extend(corners)
elif params.fit_target == "centers":
moved_sites.append(corners.mean())
# Re-compute RMSD after moving detector components
rmsd = 1000 * math.sqrt(
(reference_sites - moved_sites).sum_sq() / len(reference_sites))
print("RMSD of fit after movement: %.1f microns" % rmsd)
if params.fit_target == "corners":
rmsd = rmsd_from_centers(reference_sites, moved_sites)
print("RMSD of fit of centers after movement: %.1f microns" % rmsd)
if params.panel_list is not None:
reference_sites = flex.vec3_double()
moved_sites = flex.vec3_double()
for panel_id in range(len(reference)):
for detector, sites in zip([reference, moving],
[reference_sites, moved_sites]):
panel = detector[panel_id]
size = panel.get_image_size()
corners = flex.vec3_double([
panel.get_pixel_lab_coord(point) for point in [
(0, 0),
(0, size[1] - 1),
(size[0] - 1, size[1] - 1),
(size[0] - 1, 0),
]
])
if params.fit_target == "corners":
sites.extend(corners)
elif params.fit_target == "centers":
sites.append(corners.mean())
# Re-compute RMSD for full detector after moving detector components
rmsd = 1000 * math.sqrt(
(reference_sites - moved_sites).sum_sq() / len(reference_sites))
print("RMSD of whole detector fit after movement: %.1f microns" % rmsd)
if params.fit_target == "corners":
rmsd = rmsd_from_centers(reference_sites, moved_sites)
print(
"RMSD of whole detector fit of centers after movement: %.1f microns"
% rmsd)
class Script(object):
'''A class for running the script.'''
def __init__(self):
'''Initialise the script.'''
from dials.util.options import OptionParser
import libtbx.load_env
# The script usage
usage = "usage: %s [options] [param.phil] "\
"datablock.json" \
% libtbx.env.dispatcher_name
# Initialise the base class
self.parser = OptionParser(usage=usage,
phil=phil_scope,
epilog=help_message,
read_datablocks=True,
read_reflections=True)
def run(self):
'''Execute the script.'''
from dials.array_family import flex
from dials.util.options import flatten_datablocks
from dials.util.options import flatten_reflections
from time import time
from dials.util import log
from libtbx.utils import Sorry
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
# Configure the logging
log.config(params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
# Ensure we have a data block
datablocks = flatten_datablocks(params.input.datablock)
reflections = flatten_reflections(params.input.reflections)
if len(datablocks) == 0 and len(reflections) == 0:
self.parser.print_help()
return
elif len(datablocks) != len(reflections):
raise Sorry(
"Must have same number of datablocks and reflection tables")
# Combine the datablocks and reflections
datablock, reflections = combine(datablocks, reflections, params)
# Save the reflections to file
logger.info('\n' + '-' * 80)
reflections.as_pickle(params.output.reflections)
logger.info('Saved {0} reflections to {1}'.format(
len(reflections), params.output.reflections))
# Save the datablock
from dxtbx.datablock import DataBlockDumper
logger.info('Saving datablocks to {0}'.format(params.output.datablock))
dump = DataBlockDumper(datablocks)
dump.as_file(params.output.datablock)
# Print the time
logger.info("Time Taken: %f" % (time() - start_time))
def main():
usage = "dials.generate_distortion_maps [options] image_*.cbf"
parser = OptionParser(
usage=usage,
phil=scope,
read_experiments=True,
read_experiments_from_images=True,
check_format=False,
epilog=help_message,
)
params, options = parser.parse_args()
# Configure the logging
log.config(verbosity=options.verbose, logfile=params.output.log)
logger.info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0:
parser.print_help()
exit()
assert len(experiments) == 1
imagesets = experiments.imagesets()
assert len(imagesets) == 1
imageset = imagesets[0]
if params.mode == "translate":
op = params.translate
logger.info("Generating translation map with dx={0}, dy={1}".format(
op.dx, op.dy))
dx, dy = make_dx_dy_translate(imageset, op.dx, op.dy)
elif params.mode == "ellipse":
op = params.ellipse
logger.info("Generating elliptical map with phi={0}, l1={1}, "
"l2={2}, centre_xy={3},{4}".format(op.phi, op.l1, op.l2,
*op.centre_xy))
dx, dy = make_dx_dy_ellipse(imageset, op.phi, op.l1, op.l2,
op.centre_xy)
else:
raise Sorry("Unrecognised mode")
logger.info("Saving X distortion map to {0}".format(params.output.x_map))
with open(params.output.x_map, "wb") as f:
pickle.dump(dx, f, pickle.HIGHEST_PROTOCOL)
logger.info("Saving Y distortion map to {0}".format(params.output.y_map))
with open(params.output.y_map, "wb") as f:
pickle.dump(dy, f, pickle.HIGHEST_PROTOCOL)
class Script(object):
"""A class for running the script."""
def __init__(self):
"""Initialise the script."""
from libtbx.phil import parse
# The phil scope
phil_scope = parse(
"""
hklin = None
.type = path
.help = "MTZ file (containing observed and calculated structure "
"factors)"
Fo = F
.type = str
.help = "MTZ column name for Fobs"
Fc = FC_ALL
.type = str
.help = "MTZ column name for Fcalc (FC_ALL from Refmac includes the "
"bulk solvent contribution)"
max_Fc = 300
.type = float
.help = "Set plot limits to display data up to this value of Fc"
plot_filename = Fo_vs_Fc.pdf
.type = str
.help = "Filename for plot"
fit_hyperbola = True
.type = bool
.help = "Calculate and show the fit of a hyperbolic function given by "
"|Fo|^2 = |Fc|^2 + |Fe|^2, where |Fe| describes the error term "
"containing information about dynamic scattering and other "
"effects"
show_y_eq_x = True
.type = bool
.help = "Plot y=x as a dashed line"
""",
process_includes=True,
)
# The script usage
usage = "usage: dials.plot_Fo_vs_Fc hklin=refined.mtz"
# Create the parser
self.parser = OptionParser(usage=usage,
phil=phil_scope,
epilog=__doc__)
self.model_fit = None
return
def _extract_data_from_mtz(self):
try:
m = mtz.object(self.params.hklin)
except RuntimeError:
raise Sorry("Could not read {0}".format(self.params.hklin))
mad = m.as_miller_arrays_dict()
mad = {k[-1]: v for (k, v) in mad.items()}
fobs = mad.get(self.params.Fo)
fc = mad.get(self.params.Fc)
if [fobs, fc].count(None) > 0:
raise Sorry(
"Columns {0} not found in available labels: {1}".format(
", ".join([self.params.Fo, self.params.Fc]),
", ".join(m.column_labels()),
))
# Find common reflections (some fobs might be missing)
fobs, fc = fobs.common_sets(fc)
self.fobs = fobs.data()
self.fc = fc.amplitudes().data()
return
def _plot(self):
fig = plt.figure()
ax = fig.add_subplot(111)
minor_loc = MultipleLocator(10)
ax.yaxis.set_minor_locator(minor_loc)
ax.xaxis.set_minor_locator(minor_loc)
ax.grid(True, which="minor")
ax.set_axisbelow(True)
ax.set_aspect("equal")
ax.set_xlabel(r"$F_c$")
ax.set_ylabel(r"$F_o$")
ax.scatter(self.fc, self.fobs, s=1, c="indianred")
if self.params.max_Fc:
ax.set_xlim((0, self.params.max_Fc))
ax.set_ylim((0, self.params.max_Fc))
if self.params.show_y_eq_x:
ax.plot(ax.get_xlim(),
ax.get_ylim(),
ls="--",
c="0.0",
linewidth=0.8)
if self.model_fit:
x = flex.double_range(0, int(ax.get_xlim()[1]))
y = self.model_fit(x)
ax.plot(x, y, c="0.0", linewidth=0.8)
print("Saving plot to {0}".format(self.params.plot_filename))
plt.savefig(self.params.plot_filename)
def run(self):
"""Execute the script."""
# Parse the command line
self.params, _ = self.parser.parse_args(show_diff_phil=True)
if self.params.hklin is None:
self.parser.print_help()
sys.exit()
self._extract_data_from_mtz()
if self.params.fit_hyperbola:
# fit by NLLS Levenberg Marquardt algorithm
hyperbola_fit = HyperbolaFit(self.fc, self.fobs)
hyperbola_fit.restart()
normal_eqns_solving.levenberg_marquardt_iterations(
hyperbola_fit,
track_all=True,
gradient_threshold=1e-8,
step_threshold=1e-8,
tau=1e-4,
n_max_iterations=200,
)
intercept = hyperbola_fit.param[0]
print(
"Model fit described by the formula: |Fo|^2 = sqrt(|Fc|^2 + |Fe|^2)"
)
print("where |Fe| = {:.5f}\n".format(sqrt(intercept)))
print("Goodness of fit:")
gof = hyperbola_fit.goodness_of_fit()
print("SSE: {:.5g}".format(gof["SSE"]))
print("R-square: {:.5f}".format(gof["R-square"]))
print("RMSE: {:.2f}".format(gof["RMSE"]))
print()
# Set the model_fit function using the determined intercept
def hyperbola(x, c):
return flex.sqrt(flex.pow2(x) + c)
from functools import partial
self.model_fit = partial(hyperbola, c=intercept)
if self.params.plot_filename:
self._plot()
return
class Script(object):
""" The integration program. """
def __init__(self, phil=phil_scope):
"""Initialise the script."""
from dials.util.options import OptionParser
# The script usage
usage = "usage: dials.integrate [options] models.expt"
# Create the parser
self.parser = OptionParser(
usage=usage,
phil=phil,
epilog=help_message,
read_experiments=True,
read_reflections=True,
)
def run(self, args=None):
""" Perform the integration. """
from dials.util.command_line import heading
from dials.util.options import flatten_reflections, flatten_experiments
from dials.util import log
from time import time
from dials.util import Sorry
# Check the number of arguments is correct
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(args=args,
show_diff_phil=False)
reference = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reference) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reference) == 0:
reference = None
elif len(reference) != 1:
raise Sorry("more than 1 reflection file was given")
else:
reference = reference[0]
if len(experiments) == 0:
raise Sorry("no experiment list was specified")
# Save phil parameters
if params.output.phil is not None:
with open(params.output.phil, "w") as outfile:
outfile.write(self.parser.diff_phil.as_str())
if __name__ == "__main__":
# Configure logging
log.config(params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
for abs_params in params.absorption_correction:
if abs_params.apply:
if not (params.integration.debug.output
and not params.integration.debug.separate_files):
raise Sorry(
"Shoeboxes must be saved to integration intermediates to apply an absorption correction. "
+
"Set integration.debug.output=True, integration.debug.separate_files=False and "
+
"integration.debug.delete_shoeboxes=True to temporarily store shoeboxes."
)
# Print if we're using a mask
for i, exp in enumerate(experiments):
mask = exp.imageset.external_lookup.mask
if mask.filename is not None:
if mask.data:
logger.info("Using external mask: %s" % mask.filename)
for tile in mask.data:
logger.info(" Mask has %d pixels masked" %
tile.data().count(False))
# Print the experimental models
for i, exp in enumerate(experiments):
logger.info("=" * 80)
logger.info("")
logger.info("Experiments")
logger.info("")
logger.info("Models for experiment %d" % i)
logger.info("")
logger.info(str(exp.beam))
logger.info(str(exp.detector))
if exp.goniometer:
logger.info(str(exp.goniometer))
if exp.scan:
logger.info(str(exp.scan))
logger.info(str(exp.crystal))
logger.info("=" * 80)
logger.info("")
logger.info(heading("Initialising"))
logger.info("")
# Load the data
reference, rubbish = self.process_reference(reference)
# Check pixels don't belong to neighbours
if reference is not None:
if exp.goniometer is not None and exp.scan is not None:
self.filter_reference_pixels(reference, experiments)
logger.info("")
# Initialise the integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
# Modify experiment list if scan range is set.
experiments, reference = self.split_for_scan_range(
experiments, reference, params.scan_range)
# Modify experiment list if exclude images is set
experiments = self.exclude_images(experiments, params.exclude_images)
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info(heading("Predicting reflections"))
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
padding=params.prediction.padding,
)
# Match reference with predicted
if reference:
matched, reference, unmatched = predicted.match_with_reference(
reference)
assert len(matched) == len(predicted)
assert matched.count(True) <= len(reference)
if matched.count(True) == 0:
raise Sorry("""
Invalid input for reference reflections.
Zero reference spots were matched to predictions
""")
elif len(unmatched) != 0:
logger.info("")
logger.info("*" * 80)
logger.info(
"Warning: %d reference spots were not matched to predictions"
% (len(unmatched)))
logger.info("*" * 80)
logger.info("")
rubbish.extend(unmatched)
if len(experiments) > 1:
# filter out any experiments without matched reference reflections
# f_: filtered
from dxtbx.model.experiment_list import ExperimentList
f_reference = flex.reflection_table()
f_predicted = flex.reflection_table()
f_rubbish = flex.reflection_table()
f_experiments = ExperimentList()
good_expt_count = 0
def refl_extend(src, dest, eid):
tmp = src.select(src["id"] == eid)
tmp["id"] = flex.int(len(tmp), good_expt_count)
dest.extend(tmp)
for expt_id, experiment in enumerate(experiments):
if len(reference.select(reference["id"] == expt_id)) != 0:
refl_extend(reference, f_reference, expt_id)
refl_extend(predicted, f_predicted, expt_id)
refl_extend(rubbish, f_rubbish, expt_id)
f_experiments.append(experiment)
good_expt_count += 1
else:
logger.info(
"Removing experiment %d: no reference reflections matched to predictions"
% expt_id)
reference = f_reference
predicted = f_predicted
experiments = f_experiments
rubbish = f_rubbish
# Select a random sample of the predicted reflections
if not params.sampling.integrate_all_reflections:
predicted = self.sample_predictions(experiments, predicted, params)
# Compute the profile model
if (params.create_profile_model and reference is not None
and "shoebox" in reference):
experiments = ProfileModelFactory.create(params, experiments,
reference)
else:
experiments = ProfileModelFactory.create(params, experiments)
for expr in experiments:
if expr.profile is None:
raise Sorry("No profile information in experiment list")
del reference
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
logger.info("")
integrator = IntegratorFactory.create(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
# Append rubbish data onto the end
if rubbish is not None and params.output.include_bad_reference:
mask = flex.bool(len(rubbish), True)
rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
rubbish.set_flags(mask, rubbish.flags.bad_reference)
reflections.extend(rubbish)
# Correct integrated intensities for absorption correction, if necessary
for abs_params in params.absorption_correction:
if abs_params.apply and abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import (
multi_kapton_correction, )
experiments, reflections = multi_kapton_correction(
experiments,
reflections,
abs_params.fuller_kapton,
logger=logger)()
if params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import (
SignificanceFilter, )
from dxtbx.model.experiment_list import ExperimentList
sig_filter = SignificanceFilter(params)
filtered_refls = sig_filter(experiments, reflections)
accepted_expts = ExperimentList()
accepted_refls = flex.reflection_table()
logger.info(
"Removed %d reflections out of %d when applying significance filter"
% (len(reflections) - len(filtered_refls), len(reflections)))
for expt_id, expt in enumerate(experiments):
refls = filtered_refls.select(filtered_refls["id"] == expt_id)
if len(refls) > 0:
accepted_expts.append(expt)
refls["id"] = flex.int(len(refls), len(accepted_expts) - 1)
accepted_refls.extend(refls)
else:
logger.info(
"Removed experiment %d which has no reflections left after applying significance filter"
% expt_id)
if len(accepted_refls) == 0:
raise Sorry(
"No reflections left after applying significance filter")
experiments = accepted_expts
reflections = accepted_refls
# Delete the shoeboxes used for intermediate calculations, if requested
if params.integration.debug.delete_shoeboxes and "shoebox" in reflections:
del reflections["shoebox"]
# Save the reflections
self.save_reflections(reflections, params.output.reflections)
self.save_experiments(experiments, params.output.experiments)
# Write a report if requested
if params.output.report is not None:
integrator.report().as_file(params.output.report)
# Print the total time taken
logger.info("\nTotal time taken: %f" % (time() - start_time))
return experiments, reflections
def process_reference(self, reference):
""" Load the reference spots. """
from time import time
from dials.util import Sorry
if reference is None:
return None, None
st = time()
assert "miller_index" in reference
assert "id" in reference
logger.info("Processing reference reflections")
logger.info(" read %d strong spots" % len(reference))
mask = reference.get_flags(reference.flags.indexed)
rubbish = reference.select(~mask)
if mask.count(False) > 0:
reference.del_selected(~mask)
logger.info(" removing %d unindexed reflections" %
mask.count(False))
if len(reference) == 0:
raise Sorry("""
Invalid input for reference reflections.
Expected > %d indexed spots, got %d
""" % (0, len(reference)))
mask = reference.get_flags(reference.flags.bad_for_refinement,
all=False)
if mask.count(True) > 0:
rubbish.extend(reference.select(mask))
reference.del_selected(mask)
logger.info(
" removing %d reflections marked as bad for refinement" %
mask.count(True))
mask = reference["miller_index"] == (0, 0, 0)
if mask.count(True) > 0:
rubbish.extend(reference.select(mask))
reference.del_selected(mask)
logger.info(" removing %d reflections with hkl (0,0,0)" %
mask.count(True))
mask = reference["id"] < 0
if mask.count(True) > 0:
raise Sorry("""
Invalid input for reference reflections.
%d reference spots have an invalid experiment id
""" % mask.count(True))
if (reference["panel"]
== reference["shoebox"].panels()).count(False) > 0:
raise RuntimeError(
'reflection table "panel" column does not match "shoebox" panel'
)
logger.info(" using %d indexed reflections" % len(reference))
logger.info(" found %d junk reflections" % len(rubbish))
logger.info(" time taken: %g" % (time() - st))
return reference, rubbish
def filter_reference_pixels(self, reference, experiments):
"""
Set any pixel closer to other reflections to background
"""
modified_count = 0
for experiment, indices in reference.iterate_experiments_and_indices(
experiments):
subset = reference.select(indices)
modified = subset["shoebox"].mask_neighbouring(
subset["miller_index"],
experiment.beam,
experiment.detector,
experiment.goniometer,
experiment.scan,
experiment.crystal,
)
modified_count += modified.count(True)
reference.set_selected(indices, subset)
logger.info(" masked neighbouring pixels in %d shoeboxes" %
modified_count)
return reference
def save_reflections(self, reflections, filename):
""" Save the reflections to file. """
from time import time
st = time()
logger.info("Saving %d reflections to %s" %
(len(reflections), filename))
reflections.as_file(filename)
logger.info(" time taken: %g" % (time() - st))
def save_experiments(self, experiments, filename):
""" Save the profile model parameters. """
from time import time
st = time()
logger.info("Saving the experiments to %s" % filename)
experiments.as_file(filename)
logger.info(" time taken: %g" % (time() - st))
def sample_predictions(self, experiments, predicted, params):
""" Select a random sample of the predicted reflections to integrate. """
nref_per_degree = params.sampling.reflections_per_degree
min_sample_size = params.sampling.minimum_sample_size
max_sample_size = params.sampling.maximum_sample_size
# this code is very similar to David's code in algorithms/refinement/reflection_manager.py!
# constants
from math import pi
RAD2DEG = 180.0 / pi
working_isel = flex.size_t()
for iexp, exp in enumerate(experiments):
sel = predicted["id"] == iexp
isel = sel.iselection()
# refs = self._reflections.select(sel)
nrefs = sample_size = len(isel)
# set sample size according to nref_per_degree (per experiment)
if exp.scan and nref_per_degree:
sweep_range_rad = exp.scan.get_oscillation_range(deg=False)
width = abs(sweep_range_rad[1] - sweep_range_rad[0]) * RAD2DEG
sample_size = int(nref_per_degree * width)
else:
sweep_range_rad = None
# adjust sample size if below the chosen limit
sample_size = max(sample_size, min_sample_size)
# set maximum sample size if requested
if max_sample_size:
sample_size = min(sample_size, max_sample_size)
# determine subset and collect indices
if sample_size < nrefs:
isel = isel.select(flex.random_selection(nrefs, sample_size))
working_isel.extend(isel)
# create subset
return predicted.select(working_isel)
def exclude_images(self, experiments, exclude_images):
if exclude_images is not None and len(exclude_images) > 0:
for experiment in experiments:
imageset = experiment.imageset
for index in exclude_images:
imageset.mark_for_rejection(index, True)
return experiments
def split_for_scan_range(self, experiments, reference, scan_range):
""" Update experiments when scan range is set. """
from dxtbx.model.experiment_list import ExperimentList
from dxtbx.model.experiment_list import Experiment
# Only do anything is the scan range is set
if scan_range is not None and len(scan_range) > 0:
# Ensure that all experiments have the same imageset and scan
iset = [e.imageset for e in experiments]
scan = [e.scan for e in experiments]
assert all(x == iset[0] for x in iset)
assert all(x == scan[0] for x in scan)
# Get the imageset and scan
iset = experiments[0].imageset
scan = experiments[0].scan
# Get the array range
if scan is not None:
frames_start, frames_end = scan.get_array_range()
assert scan.get_num_images() == len(iset)
else:
frames_start, frames_end = (0, len(iset))
# Create the new lists
new_experiments = ExperimentList()
new_reference_all = reference.split_by_experiment_id()
new_reference = flex.reflection_table()
for i in range(len(new_reference_all) - len(experiments)):
new_reference_all.append(flex.reflection_table())
assert len(new_reference_all) == len(experiments)
# Loop through all the scan ranges and create a new experiment list with
# the requested scan ranges.
for scan_start, scan_end in scan_range:
# Validate the requested scan range
if scan_end == scan_start:
raise Sorry(
"Scan range end must be higher than start; pass {},{} for single image"
.format(scan_start, scan_start + 1))
if scan_end < scan_start:
raise Sorry("Scan range must be in ascending order")
elif scan_start < frames_start or scan_end > frames_end:
raise Sorry(
"Scan range must be within image range {}..{}".format(
frames_start, frames_end))
assert scan_end > scan_start
assert scan_start >= frames_start
assert scan_end <= frames_end
index_start = scan_start - frames_start
index_end = index_start + (scan_end - scan_start)
assert index_start < index_end
assert index_start >= 0
assert index_end <= len(iset)
new_iset = iset[index_start:index_end]
if scan is None:
new_scan = None
else:
new_scan = scan[index_start:index_end]
for i, e1 in enumerate(experiments):
e2 = Experiment()
e2.beam = e1.beam
e2.detector = e1.detector
e2.goniometer = e1.goniometer
e2.crystal = e1.crystal
e2.profile = e1.profile
e2.imageset = new_iset
e2.scan = new_scan
new_reference_all[i]["id"] = flex.int(
len(new_reference_all[i]), len(new_experiments))
new_reference.extend(new_reference_all[i])
new_experiments.append(e2)
experiments = new_experiments
reference = new_reference
# Print some information
logger.info(
"Modified experiment list to integrate over requested scan range"
)
for scan_start, scan_end in scan_range:
logger.info(" scan_range = %d -> %d" % (scan_start, scan_end))
logger.info("")
# Return the experiments
return experiments, reference
def run(args):
import libtbx.load_env
usage = """\
%s datablock.json reflections.pickle [options]""" % libtbx.env.dispatcher_name
from dials.util.options import OptionParser
from dials.util.options import flatten_datablocks
from dials.util.options import flatten_experiments
from dials.util.options import flatten_reflections
from scitbx.array_family import flex
from scitbx import matrix
from libtbx.utils import Sorry
parser = OptionParser(usage=usage,
phil=master_phil_scope,
read_datablocks=True,
read_experiments=True,
read_reflections=True,
check_format=False)
params, options = parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(datablocks) == 1:
imageset = datablocks[0].extract_imagesets()[0]
elif len(datablocks) > 1:
raise Sorry("Only one DataBlock can be processed at a time")
elif len(experiments.imagesets()) > 0:
imageset = experiments.imagesets()[0]
imageset.set_detector(experiments[0].detector)
imageset.set_beam(experiments[0].beam)
imageset.set_goniometer(experiments[0].goniometer)
else:
parser.print_help()
return
detector = imageset.get_detector()
scan = imageset.get_scan()
panel_origin_shifts = {0: (0, 0, 0)}
try:
hierarchy = detector.hierarchy()
except AttributeError:
hierarchy = None
for i_panel in range(1, len(detector)):
origin_shift = matrix.col(detector[0].get_origin()) \
- matrix.col(detector[i_panel].get_origin())
panel_origin_shifts[i_panel] = origin_shift
observed_xyz = flex.vec3_double()
predicted_xyz = flex.vec3_double()
for reflection_list in reflections:
if len(params.scan_range):
sel = flex.bool(len(reflection_list), False)
xyzcal_px = None
xyzcal_px = None
if 'xyzcal.px' in reflection_list:
xyzcal_px = reflection_list['xyzcal.px']
if 'xyzobs.px.value' in reflection_list:
xyzobs_px = reflection_list['xyzobs.px.value']
if xyzcal_px is not None and not xyzcal_px.norms().all_eq(0):
centroids_frame = xyzcal_px.parts()[2]
elif xyzobs_px is not None and not xyzobs_px.norms().all_eq(0):
centroids_frame = xyzobs_px.parts()[2]
else:
raise Sorry("No pixel coordinates given in input reflections.")
reflections_in_range = False
for scan_range in params.scan_range:
if scan_range is None: continue
range_start, range_end = scan_range
sel |= ((centroids_frame >= range_start) &
(centroids_frame < range_end))
reflection_list = reflection_list.select(sel)
if params.first_n_reflections is not None:
centroid_positions = reflection_list.centroid_position()
centroids_frame = centroid_positions.parts()[2]
perm = flex.sort_permutation(centroids_frame)
perm = perm[:min(reflection_list.size(), params.first_n_reflections
)]
reflection_list = reflection_list.select(perm)
if params.crystal_id is not None:
reflection_list = reflection_list.select(
reflection_list['id'] == params.crystal_id)
xyzcal_px = None
xyzcal_px = None
xyzobs_mm = None
xyzcal_mm = None
if 'xyzcal.px' in reflection_list:
xyzcal_px = reflection_list['xyzcal.px']
if 'xyzobs.px.value' in reflection_list:
xyzobs_px = reflection_list['xyzobs.px.value']
if 'xyzcal.mm' in reflection_list:
xyzcal_mm = reflection_list['xyzcal.mm']
if 'xyzobs.mm.value' in reflection_list:
xyzobs_mm = reflection_list['xyzobs.mm.value']
panel_ids = reflection_list['panel']
if xyzobs_mm is None and xyzobs_px is not None:
xyzobs_mm = flex.vec3_double()
for i_panel in range(len(detector)):
xyzobs_px_panel = xyzobs_px.select(panel_ids == i_panel)
from dials.algorithms.centroid import centroid_px_to_mm_panel
xyzobs_mm_panel, _, _ = centroid_px_to_mm_panel(
detector[i_panel], scan, xyzobs_px_panel,
flex.vec3_double(xyzobs_px_panel.size()),
flex.vec3_double(xyzobs_px_panel.size()))
xyzobs_mm.extend(xyzobs_mm_panel)
if xyzobs_mm is not None:
observed_xyz.extend(xyzobs_mm)
if xyzcal_mm is not None:
predicted_xyz.extend(xyzcal_mm)
obs_x, obs_y, _ = observed_xyz.parts()
pred_x, pred_y, _ = predicted_xyz.parts()
try:
import matplotlib
if not params.output.show_plot:
# http://matplotlib.org/faq/howto_faq.html#generate-images-without-having-a-window-appear
matplotlib.use('Agg') # use a non-interactive backend
from matplotlib import pyplot
except ImportError:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
fig.set_size_inches(params.output.size_inches)
fig.set_dpi(params.output.dpi)
pyplot.axes().set_aspect('equal')
marker_size = params.output.marker_size
if obs_x.size():
pyplot.scatter(obs_x,
obs_y,
marker='o',
c='white',
s=marker_size,
alpha=1)
if pred_x.size():
pyplot.scatter(pred_x, pred_y, marker='+', s=marker_size, c='blue')
#assert len(detector) == 1
panel = detector[0]
#if len(detector) > 1:
xmin = max([
detector[i_panel].get_image_size_mm()[0] +
panel_origin_shifts[i_panel][0] for i_panel in range(len(detector))
])
xmax = max([
detector[i_panel].get_image_size_mm()[0] +
panel_origin_shifts[i_panel][0] for i_panel in range(len(detector))
])
ymax = max([
detector[i_panel].get_image_size_mm()[1] +
panel_origin_shifts[i_panel][1] for i_panel in range(len(detector))
])
ymax = max([
detector[i_panel].get_image_size_mm()[1] +
panel_origin_shifts[i_panel][1] for i_panel in range(len(detector))
])
try:
beam_centre = hierarchy.get_beam_centre(imageset.get_beam().get_s0())
except Exception:
beam_centre = detector[0].get_beam_centre(imageset.get_beam().get_s0())
pyplot.scatter([beam_centre[0]], [beam_centre[1]],
marker='+',
c='blue',
s=100)
pyplot.xlim(0, xmax)
pyplot.ylim(0, ymax)
pyplot.gca().invert_yaxis()
pyplot.title('Centroid x,y-coordinates')
pyplot.xlabel('x-coordinate (mm)')
pyplot.ylabel('y-coordinate (mm)')
if params.output.file_name is not None:
pyplot.savefig(params.output.file_name,
size_inches=params.output.size_inches,
dpi=params.output.dpi,
bbox_inches='tight')
if params.output.show_plot:
pyplot.show()
def run(args=None):
from dials.util.options import OptionParser, flatten_experiments
usage = "dials.shadow_plot [options] models.expt"
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=True,
epilog=help_message,
)
params, options = parser.parse_args(args, show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0:
parser.print_help()
sys.exit(0)
assert len(experiments) == 1
imagesets = experiments.imagesets()
imageset = imagesets[0]
goniometer = imageset.get_goniometer()
detector = imageset.get_detector()
scan = imageset.get_scan()
masker = imageset.masker()
if masker is None:
raise Sorry("Goniometer model does not support shadowing.")
angles = goniometer.get_angles()
names = goniometer.get_names()
scan_axis = goniometer.get_scan_axis()
phi = angles[0]
if params.step_size is libtbx.Auto:
if params.mode == "1d":
step = scan.get_oscillation()[1]
else:
step = 10
else:
step = params.step_size
if params.mode == "1d":
if params.oscillation_range is not None:
start, end = params.oscillation_range
else:
start, end = scan.get_oscillation_range()
scan_points = flex.double(libtbx.utils.frange(start, end, step=step))
n_px_shadowed = flex.double(scan_points.size(), 0)
n_px_tot = flex.double(scan_points.size(), 0)
assert len(angles) == 3
for i, scan_angle in enumerate(scan_points):
shadow = masker.project_extrema(detector, scan_angle)
for p_id in range(len(detector)):
px_x, px_y = detector[p_id].get_image_size()
n_px_tot[i] += px_x * px_y
if shadow[p_id].size() < 4:
continue
n_px_shadowed[i] += polygon_area(shadow[p_id])
else:
kappa_values = flex.double(libtbx.utils.frange(0, 360, step=step))
omega_values = flex.double(libtbx.utils.frange(0, 360, step=step))
grid = flex.grid(kappa_values.size(), omega_values.size())
n_px_shadowed = flex.double(grid, 0)
n_px_tot = flex.double(grid, 0)
assert len(angles) == 3
for i, kappa in enumerate(kappa_values):
for j, omega in enumerate(omega_values):
masker.set_goniometer_angles((phi, kappa, omega))
masker.extrema_at_scan_angle(omega)
shadow = masker.project_extrema(detector, omega)
for p_id in range(len(detector)):
px_x, px_y = detector[p_id].get_image_size()
n_px_tot[i, j] += px_x * px_y
if shadow[p_id].size() < 4:
continue
n_px_shadowed[i, j] += polygon_area(shadow[p_id])
fraction_shadowed = n_px_shadowed / n_px_tot
if params.output.json is not None:
if params.mode == "2d":
raise Sorry("json output not supported for mode=2d")
print("Writing json output to %s" % params.output.json)
d = {
"scan_points": list(scan_points),
"fraction_shadowed": list(fraction_shadowed),
}
with open(params.output.json, "w") as f:
json.dump(d, f)
if params.output.plot is not None:
import matplotlib
matplotlib.use("Agg")
from matplotlib import pyplot as plt
plt.style.use("ggplot")
if params.mode == "1d":
plt.plot(
scan_points.as_numpy_array(), fraction_shadowed.as_numpy_array() * 100
)
plt.xlabel("%s angle (degrees)" % names[scan_axis])
plt.ylabel("Shadowed area (%)")
if params.y_max is not None:
plt.ylim(0, params.y_max)
else:
plt.ylim(0, plt.ylim()[1])
else:
fig = plt.imshow(
fraction_shadowed.as_numpy_array() * 100, interpolation="bicubic"
)
plt.xlabel("%s angle (degrees)" % names[2])
plt.ylabel("%s angle (degrees)" % names[1])
plt.xlim(0, 360 / step - 0.5)
plt.ylim(0, 360 / step - 0.5)
ticks = (0, 50, 100, 150, 200, 250, 300, 350)
fig.axes.xaxis.set_major_locator(
matplotlib.ticker.FixedLocator([k / step for k in ticks])
)
fig.axes.yaxis.set_major_locator(
matplotlib.ticker.FixedLocator([k / step for k in ticks])
)
fig.axes.set_xticklabels(["%.0f" % k for k in ticks])
fig.axes.set_yticklabels(["%.0f" % k for k in ticks])
cbar = plt.colorbar()
cbar.set_label("Shadowed area (%)")
if params.output.size_inches is not None:
fig = plt.gcf()
fig.set_size_inches(params.output.size_inches)
plt.tight_layout()
print("Saving plot to %s" % params.output.plot)
plt.savefig(params.output.plot)
class Script(object):
"""A class for running the script."""
def __init__(self):
"""Initialise the script."""
from libtbx.phil import parse
# The phil scope
phil_scope = parse(
"""
scale = unit *max_cell ewald_sphere_radius
.type = choice
.help = "Choose the scale for the direction vector in orthogonal"
"coordinates prior to transformation into fractional"
"coordinates [uvw]"
plot_filename = None
.type = str
.help = "Filename for a plot of angle between neighbouring frames"
"(set to None for no plot)"
""",
process_includes=True,
)
usage = "dials.frame_orientations refined.expt refined.refl"
# Create the parser
self.parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=False,
epilog=__doc__,
)
def run(self):
"""Execute the script."""
# Parse the command line
self.params, _ = self.parser.parse_args(show_diff_phil=True)
if not self.params.input.experiments:
self.parser.print_help()
sys.exit()
# Try to load the models
experiments = flatten_experiments(self.params.input.experiments)
nexp = len(experiments)
if nexp == 0:
self.parser.print_help()
sys.exit("No Experiments found in the input")
# Set up a plot if requested
if self.params.plot_filename:
plt.figure()
header = [
"Image",
"Beam direction (xyz)",
"Zone axis [uvw]",
"Angles between beam\nand axes a, b, c (deg)",
"Angle from\nprevious image (deg)",
]
for iexp, exp in enumerate(experiments):
print("For Experiment id = {}".format(iexp))
print(exp.beam)
print(exp.crystal)
print(exp.scan)
if self.params.scale == "ewald_sphere_radius":
scale = 1.0 / exp.beam.get_wavelength()
elif self.params.scale == "max_cell":
uc = exp.crystal.get_unit_cell()
scale = max(uc.parameters()[0:3])
else:
scale = 1.0
print("Beam direction scaled by {0} = {1:.3f} to "
"calculate zone axis\n".format(self.params.scale, scale))
dat = extract_experiment_data(exp, scale)
images = dat["images"]
directions = dat["directions"]
zone_axes = dat["zone_axes"]
real_space_axes = dat["real_space_axes"]
# calculate the angle between the beam and each crystal axis
axis_angles = []
for d, rsa in zip(directions, real_space_axes):
angles = [d.angle(a, deg=True) for a in rsa]
axis_angles.append("{:.2f} {:.2f} {:.2f}".format(*angles))
# calculate the orientation offset between each image
offset = [
e1.angle(e2, deg=True)
for e1, e2 in zip(zone_axes[:-1], zone_axes[1:])
]
str_off = ["---"] + ["{:.8f}".format(e) for e in offset]
rows = []
for i, d, z, a, o in zip(
images,
directions,
zone_axes,
axis_angles,
str_off,
):
row = [
str(i),
"{:.8f} {:.8f} {:.8f}".format(*d.elems),
"{:.8f} {:.8f} {:.8f}".format(*z.elems),
a,
o,
]
rows.append(row)
# Print the table
print(tabulate(rows, header))
# Add to the plot, if requested
if self.params.plot_filename:
plt.scatter(images[1:], offset, s=1)
# Finish and save plot, if requested
if self.params.plot_filename:
plt.xlabel("Image number")
plt.ylabel(r"Angle from previous image $\left(^\circ\right)$")
plt.title(r"Angle between neighbouring images")
print("Saving plot to {}".format(self.params.plot_filename))
plt.savefig(self.params.plot_filename)
print()
from dials.util.options import OptionParser
from dials.util.options import flatten_reflections
from dials.util.options import flatten_experiments
dials.util.log.print_banner()
usage_message = "dials.image_viewer models.expt [observations.refl]"
parser = OptionParser(
usage=usage_message,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
read_experiments_from_images=True,
epilog=help_message,
)
params, options = parser.parse_args(show_diff_phil=True)
experiments = [x.data for x in params.input.experiments]
reflections = flatten_reflections(params.input.reflections)
if len(experiments) == 0:
parser.print_help()
exit(0)
flat_expts = flatten_experiments(params.input.experiments)
if params.load_models:
if any(e.detector is None for e in flat_expts):
sys.exit("Error: experiment has no detector")
if any(e.beam is None for e in flat_expts):
sys.exit("Error: experiment has no beam")
if params.mask is not None:
class Script(object):
'''A class for running the script.'''
def __init__(self):
self.mpi_helper = mpi_helper()
self.mpi_logger = mpi_logger()
def __del__(self):
self.mpi_helper.finalize()
def parse_input(self):
'''Parse input at rank 0 and broadcast the input parameters and options to all ranks'''
if self.mpi_helper.rank == 0:
from xfel.merging.application.phil.phil import phil_scope
help_message = '''Merge xfel data.'''
# The script usage
import libtbx.load_env
self.usage = "usage: %s [options] [param.phil] " % libtbx.env.dispatcher_name
self.parser = None
'''Initialize the script.'''
from dials.util.options import OptionParser
# Create the parser
self.parser = OptionParser(usage=self.usage,
phil=phil_scope,
epilog=help_message)
# Parse the command line. quick_parse is required for MPI compatibility
params, options = self.parser.parse_args(show_diff_phil=True,
quick_parse=True)
# Log the modified phil parameters
diff_phil_str = self.parser.diff_phil.as_str()
if diff_phil_str is not "":
self.mpi_logger.main_log(
"The following parameters have been modified:\n%s" %
diff_phil_str)
# prepare for transmitting input parameters to all ranks
transmitted = dict(params=params, options=options)
else:
transmitted = None
# broadcast parameters and options to all ranks
self.mpi_logger.log("Broadcasting input parameters...")
self.mpi_logger.log_step_time("BROADCAST_INPUT_PARAMS")
transmitted = self.mpi_helper.comm.bcast(transmitted, root=0)
self.params = transmitted['params']
self.options = transmitted['options']
self.mpi_logger.set_log_file_paths(self.params)
self.mpi_logger.log("Received input parameters and options")
self.mpi_logger.log_step_time("BROADCAST_INPUT_PARAMS", True)
def run(self):
import datetime
time_now = datetime.datetime.now()
self.mpi_logger.log(str(time_now))
if self.mpi_helper.rank == 0:
self.mpi_logger.main_log(str(time_now))
self.mpi_logger.log_step_time("TOTAL")
self.mpi_logger.log_step_time("PARSE_INPUT_PARAMS")
self.parse_input()
self.mpi_logger.log_step_time("PARSE_INPUT_PARAMS", True)
# Create the workers using the factories
self.mpi_logger.log_step_time("CREATE_WORKERS")
from xfel.merging import application
import importlib
workers = []
steps = default_steps if self.params.dispatch.step_list is None else self.params.dispatch.step_list
for step in steps:
step_factory_name = step
step_additional_info = []
step_info = step.split(' ')
assert len(step_info) > 0
if len(step_info) > 1:
step_factory_name = step_info[0]
step_additional_info = step_info[1:]
factory = importlib.import_module('xfel.merging.application.' +
step_factory_name + '.factory')
workers.extend(
factory.factory.from_parameters(self.params,
step_additional_info,
mpi_helper=self.mpi_helper,
mpi_logger=self.mpi_logger))
# Perform phil validation up front
for worker in workers:
worker.validate()
self.mpi_logger.log_step_time("CREATE_WORKERS", True)
# Do the work
experiments = reflections = None
step = 0
while (workers):
worker = workers.pop(0)
self.mpi_logger.log_step_time("STEP_" + worker.__repr__())
# Log worker name, i.e. execution step name
step += 1
if step > 1:
self.mpi_logger.log('')
step_desc = "STEP %d: %s" % (step, worker)
self.mpi_logger.log(step_desc)
if self.mpi_helper.rank == 0:
if step > 1:
self.mpi_logger.main_log('')
self.mpi_logger.main_log(step_desc)
# Execute worker
experiments, reflections = worker.run(experiments, reflections)
self.mpi_logger.log_step_time("STEP_" + worker.__repr__(), True)
if self.params.output.save_experiments_and_reflections:
from dxtbx.model.experiment_list import ExperimentListDumper
import os
if 'id' not in reflections:
from dials.array_family import flex
id_ = flex.int(len(reflections), -1)
for expt_number, expt in enumerate(experiments):
sel = reflections['exp_id'] == expt.identifier
id_.set_selected(sel, expt_number)
reflections['id'] = id_
reflections.as_pickle(
os.path.join(
self.params.output.output_dir, self.params.output.prefix +
"_%06d.refl" % self.mpi_helper.rank))
dump = ExperimentListDumper(experiments)
dump.as_file(
os.path.join(
self.params.output.output_dir, self.params.output.prefix +
"_%06d.expt" % self.mpi_helper.rank))
self.mpi_logger.log_step_time("TOTAL", True)
class Script(object):
''' Encapsulate the script in a class. '''
def __init__(self):
''' Initialise the script. '''
from dials.util.options import OptionParser
import libtbx.load_env
# The script usage
usage = "usage: %s [options] experiments.json spots.pickle" \
% libtbx.env.dispatcher_name
self.parser = OptionParser(usage=usage,
epilog=help_message,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False)
def run(self):
''' Run the script. '''
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.util.command_line import Command
from dials.array_family import flex
from dials.util.options import flatten_reflections, flatten_experiments
from dxtbx.model.experiment_list import ExperimentListDumper
from libtbx.utils import Sorry
from dials.util import log
log.config()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reflections) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reflections) != 1:
raise Sorry('exactly 1 reflection table must be specified')
if len(experiments) == 0:
raise Sorry('no experiments were specified')
if (not 'background.mean'
in reflections[0]) and params.subtract_background:
raise Sorry(
'for subtract_background need background.mean in reflections')
reflections, _ = self.process_reference(reflections[0], params)
# Check pixels don't belong to neighbours
self.filter_reference_pixels(reflections, experiments)
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
padding=params.prediction.padding)
# Match with predicted
matched, reflections, unmatched = predicted.match_with_reference(
reflections)
assert (len(matched) == len(predicted))
assert (matched.count(True) <= len(reflections))
if matched.count(True) == 0:
raise Sorry('''
Invalid input for reference reflections.
Zero reference spots were matched to predictions
''')
elif len(unmatched) != 0:
logger.info('')
logger.info('*' * 80)
logger.info(
'Warning: %d reference spots were not matched to predictions' %
(len(unmatched)))
logger.info('*' * 80)
logger.info('')
# Create the profile model
experiments = ProfileModelFactory.create(params, experiments,
reflections)
for model in experiments:
sigma_b = model.profile.sigma_b(deg=True)
sigma_m = model.profile.sigma_m(deg=True)
if isinstance(sigma_b, type(1.0)):
logger.info('Sigma B: %f' % sigma_b)
logger.info('Sigma M: %f' % sigma_m)
else: # scan varying
mean_sigma_b = sum(sigma_b) / len(sigma_b)
mean_sigma_m = sum(sigma_m) / len(sigma_m)
logger.info('Sigma B: %f' % mean_sigma_b)
logger.info('Sigma M: %f' % mean_sigma_m)
# Wrtie the parameters
Command.start("Writing experiments to %s" % params.output)
dump = ExperimentListDumper(experiments)
with open(params.output, "w") as outfile:
outfile.write(dump.as_json())
Command.end("Wrote experiments to %s" % params.output)
def process_reference(self, reference, params):
''' Load the reference spots. '''
from dials.array_family import flex
from time import time
from libtbx.utils import Sorry
if reference is None:
return None, None
st = time()
assert ("miller_index" in reference)
assert ("id" in reference)
logger.info('Processing reference reflections')
logger.info(' read %d strong spots' % len(reference))
mask = reference.get_flags(reference.flags.indexed)
rubbish = reference.select(mask == False)
if mask.count(False) > 0:
reference.del_selected(mask == False)
logger.info(' removing %d unindexed reflections' %
mask.count(False))
if len(reference) == 0:
raise Sorry('''
Invalid input for reference reflections.
Expected > %d indexed spots, got %d
''' % (0, len(reference)))
mask = reference.get_flags(reference.flags.centroid_outlier)
if mask.count(True) > 0:
rubbish.extend(reference.select(mask))
reference.del_selected(mask)
logger.info(
' removing %d reflections marked as centroid outliers' %
mask.count(True))
mask = reference['miller_index'] == (0, 0, 0)
if mask.count(True) > 0:
rubbish.extend(reference.select(mask))
reference.del_selected(mask)
logger.info(' removing %d reflections with hkl (0,0,0)' %
mask.count(True))
mask = reference['id'] < 0
if mask.count(True) > 0:
raise Sorry('''
Invalid input for reference reflections.
%d reference spots have an invalid experiment id
''' % mask.count(True))
logger.info(' using %d indexed reflections' % len(reference))
logger.info(' found %d junk reflections' % len(rubbish))
from dials.array_family import flex
if 'background.mean' in reference and params.subtract_background:
logger.info(
' subtracting background from %d reference reflections' %
len(reference))
for spot in reference:
spot['shoebox'].data -= spot['background.mean']
logger.info(' time taken: %g' % (time() - st))
return reference, rubbish
def filter_reference_pixels(self, reference, experiments):
'''
Set any pixel closer to other reflections to background
'''
modified_count = 0
for experiment, indices in reference.iterate_experiments_and_indices(
experiments):
subset = reference.select(indices)
modified = subset['shoebox'].mask_neighbouring(
subset['miller_index'], experiment.beam, experiment.detector,
experiment.goniometer, experiment.scan, experiment.crystal)
modified_count += modified.count(True)
reference.set_selected(indices, subset)
logger.info(" masked neighbouring pixels in %d shoeboxes" %
modified_count)
return reference
def run(args):
import libtbx.load_env
from scitbx import matrix
from cctbx.sgtbx import lattice_symmetry_group
from scitbx.math import r3_rotation_axis_and_angle_from_matrix
usage = "%s [options] experiment_0.json ..." % \
libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = params.input.experiments
# check input
space_group = None
for experiment in experiments:
assert(len(experiment.data.goniometers()) == 1)
assert(len(experiment.data.crystals()) == 1)
crystal = experiment.data.crystals()[0]
if space_group is None:
space_group = crystal.get_space_group()
else:
assert(crystal.get_space_group() == space_group)
reference_U = None
reference_space_group = None
for j, experiment in enumerate(experiments):
goniometer = experiment.data.goniometers()[0]
F = matrix.sqr(goniometer.get_fixed_rotation())
crystal = experiment.data.crystals()[0]
U = matrix.sqr(crystal.get_U())
B = matrix.sqr(crystal.get_B())
UB = F * U * B
UBt = UB.transpose().elems
a, b, c = matrix.col(UBt[0:3]), matrix.col(UBt[3:6]), matrix.col(UBt[6:9])
axis = matrix.col(goniometer.get_rotation_axis())
from math import pi
r2d = 180 / pi
abc = [a, b, c]
abc_names = 'abc'
distances = [(r2d * (min(axis.angle(_a), pi - axis.angle(_a))), k)
for k, _a in enumerate(abc)]
close = sorted(distances)[0]
if reference_U is None:
reference_U = U
reference_space_group = lattice_symmetry_group(crystal.get_unit_cell(),
max_delta=0.0)
print '%s possible lattice ops' % len(reference_space_group.all_ops())
print 'Experiment %d' % j
print 'Closest (original) axis: %s* %.2f' % \
(abc_names[close[1]], close[0])
results = []
for op in reference_space_group.all_ops():
R = B * matrix.sqr(op.r().as_double()).transpose() * B.inverse()
relative = (U * R).inverse() * reference_U
rot = r3_rotation_axis_and_angle_from_matrix(relative)
results.append((abs(rot.angle()), op.r().as_hkl(), rot))
results.sort()
print 'Best reindex op for experiment %d: %12s (%.3f)' % \
(j, results[0][1], 180.0 * results[0][2].angle() / pi)
if results[0][0] > (5 * pi / 180.0):
print 'Rotation: axis: %.4f %.4f %.4f' % results[0][2].axis
print ' angle: %.4f degrees' % \
(180.0 * results[0][2].angle() / pi)
class Script:
"""A class for running the script."""
def __init__(self):
"""Initialise the script."""
# The script usage
usage = ("usage: dials.two_theta_refine [options] [param.phil] "
"models.expt observations.refl")
# Create the parser
self.parser = OptionParser(
usage=usage,
phil=working_phil,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message,
)
@staticmethod
def check_input(reflections):
"""Check the input is suitable for refinement. So far just check keys in
the reflection table. Maybe later check experiments have overlapping models
etc."""
msg = (
"The supplied reflection table does not have the required data " +
"column: {0}")
for key in ["xyzobs.mm.value", "xyzobs.mm.variance"]:
if key not in reflections:
msg = msg.format(key)
sys.exit(msg)
# FIXME add other things to be checked here
return
@staticmethod
def combine_crystals(experiments):
"""Replace all crystals in the experiments list with the first crystal"""
new_experiments = ExperimentList()
ref_crystal = experiments[0].crystal
for exp in experiments:
new_experiments.append(
Experiment(
beam=exp.beam,
detector=exp.detector,
scan=exp.scan,
goniometer=exp.goniometer,
crystal=ref_crystal,
imageset=exp.imageset,
identifier=exp.identifier,
))
return new_experiments
@staticmethod
def filter_integrated_centroids(reflections):
"""Filter reflections to include only those with the integrated and the
strong flag set, but only if there are apparently some integrated
reflections"""
orig_len = len(reflections)
mask = reflections.get_flags(reflections.flags.integrated)
if mask.count(True) == 0:
return reflections
reflections = reflections.select(mask)
mask = reflections.get_flags(reflections.flags.strong)
reflections = reflections.select(mask)
logger.info(
"{} out of {} reflections remain after filtering to keep only strong"
" and integrated centroids".format(len(reflections), orig_len))
return reflections
@staticmethod
def convert_to_P1(reflections, experiments):
"""Convert the input crystals to P 1 and reindex the reflections"""
for iexp, exp in enumerate(experiments):
sel = reflections["id"] == iexp
xl = exp.crystal
sg = xl.get_space_group()
op = sg.info().change_of_basis_op_to_primitive_setting()
exp.crystal = xl.change_basis(op)
exp.crystal.set_space_group(sgtbx.space_group("P 1"))
hkl_reindexed = op.apply(reflections["miller_index"].select(sel))
reflections["miller_index"].set_selected(sel, hkl_reindexed)
return reflections, experiments
@staticmethod
def create_refiner(params, reflections, experiments):
# Only parameterise the crystal unit cell
det_params = None
beam_params = None
xlo_params = None
xluc_params = []
for crystal in experiments.crystals():
exp_ids = experiments.indices(crystal)
xluc_params.append(
CrystalUnitCellParameterisation(crystal,
experiment_ids=exp_ids))
# Two theta prediction equation parameterisation
pred_param = TwoThetaPredictionParameterisation(
experiments, det_params, beam_params, xlo_params, xluc_params)
param_reporter = ParameterReporter(det_params, beam_params, xlo_params,
xluc_params)
# ReflectionManager, currently without outlier rejection
# Note: If not all reflections are used, then the filtering must be
# communicated to generate_cif/mmcif() to be included in the CIF file!
refman = TwoThetaReflectionManager(reflections,
experiments,
outlier_detector=None)
# Reflection predictor
ref_predictor = TwoThetaExperimentsPredictor(experiments)
# Two theta target
target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param)
# Switch on correlation matrix tracking if a correlation plot is requested
journal = None
if params.output.correlation_plot.filename is not None:
journal = refinery_phil_scope.extract().refinery.journal
journal.track_parameter_correlation = True
# Minimisation engine - hardcoded to LevMar for now.
refinery = Refinery(
target=target,
prediction_parameterisation=pred_param,
log=None,
tracking=journal,
max_iterations=20,
)
# Refiner
refiner = Refiner(
experiments=experiments,
pred_param=pred_param,
param_reporter=param_reporter,
refman=refman,
target=target,
refinery=refinery,
)
return refiner
@staticmethod
def cell_param_table(crystal):
"""Construct a table of cell parameters and their ESDs"""
cell = crystal.get_unit_cell().parameters()
esd = crystal.get_cell_parameter_sd()
vol = crystal.get_unit_cell().volume()
vol_esd = crystal.get_cell_volume_sd()
header = ["Parameter", "Value", "Estimated sd"]
rows = []
names = ["a", "b", "c", "alpha", "beta", "gamma"]
for n, p, e in zip(names, cell, esd):
rows.append([n, f"{p:9.5f}", f"{e:9.5f}"])
rows.append(["\nvolume", f"\n{vol:9.5f}", f"\n{vol_esd:9.5f}"])
return tabulate(rows, header)
@staticmethod
def generate_p4p(crystal, beam, filename):
logger.info("Saving P4P info to %s", filename)
cell = crystal.get_unit_cell().parameters()
esd = crystal.get_cell_parameter_sd()
vol = crystal.get_unit_cell().volume()
vol_esd = crystal.get_cell_volume_sd()
open(filename, "w").write("\n".join([
"TITLE Auto-generated .p4p file from dials.two_theta_refine",
"CELL %.4f %.4f %.4f %.4f %.4f %.4f %.4f" % tuple(cell +
(vol, )),
"CELLSD %.4f %.4f %.4f %.4f %.4f %.4f %.4f" % tuple(esd +
(vol_esd, )),
"SOURCE SYNCH %.6f" % beam.get_wavelength(),
"",
]))
return
@staticmethod
def generate_cif(crystal, refiner, filename):
logger.info("Saving CIF information to %s", filename)
block = iotbx.cif.model.block()
block["_audit_creation_method"] = dials_version()
block["_audit_creation_date"] = datetime.date.today().isoformat()
# block["_publ_section_references"] = '' # once there is a reference...
for cell, esd, cifname in zip(
crystal.get_unit_cell().parameters(),
crystal.get_cell_parameter_sd(),
[
"length_a",
"length_b",
"length_c",
"angle_alpha",
"angle_beta",
"angle_gamma",
],
):
block[f"_cell_{cifname}"] = format_float_with_standard_uncertainty(
cell, esd)
block["_cell_volume"] = format_float_with_standard_uncertainty(
crystal.get_unit_cell().volume(), crystal.get_cell_volume_sd())
used_reflections = refiner.get_matches()
block["_cell_measurement_reflns_used"] = len(used_reflections)
block["_cell_measurement_theta_min"] = (
flex.min(used_reflections["2theta_obs.rad"]) * 180 / math.pi / 2)
block["_cell_measurement_theta_max"] = (
flex.max(used_reflections["2theta_obs.rad"]) * 180 / math.pi / 2)
block["_diffrn_reflns_number"] = len(used_reflections)
miller_span = miller.index_span(used_reflections["miller_index"])
min_h, min_k, min_l = miller_span.min()
max_h, max_k, max_l = miller_span.max()
block["_diffrn_reflns_limit_h_min"] = min_h
block["_diffrn_reflns_limit_h_max"] = max_h
block["_diffrn_reflns_limit_k_min"] = min_k
block["_diffrn_reflns_limit_k_max"] = max_k
block["_diffrn_reflns_limit_l_min"] = min_l
block["_diffrn_reflns_limit_l_max"] = max_l
block["_diffrn_reflns_theta_min"] = (
flex.min(used_reflections["2theta_obs.rad"]) * 180 / math.pi / 2)
block["_diffrn_reflns_theta_max"] = (
flex.max(used_reflections["2theta_obs.rad"]) * 180 / math.pi / 2)
cif = iotbx.cif.model.cif()
cif["two_theta_refine"] = block
with open(filename, "w") as fh:
cif.show(out=fh)
@staticmethod
def generate_mmcif(crystal, refiner, filename):
logger.info("Saving mmCIF information to %s", filename)
block = iotbx.cif.model.block()
block["_audit.revision_id"] = 1
block["_audit.creation_method"] = dials_version()
block["_audit.creation_date"] = datetime.date.today().isoformat()
block["_entry.id"] = "two_theta_refine"
# block["_publ.section_references"] = '' # once there is a reference...
block["_cell.entry_id"] = "two_theta_refine"
for cell, esd, cifname in zip(
crystal.get_unit_cell().parameters(),
crystal.get_cell_parameter_sd(),
[
"length_a",
"length_b",
"length_c",
"angle_alpha",
"angle_beta",
"angle_gamma",
],
):
block[f"_cell.{cifname}"] = f"{cell:.8f}"
block[f"_cell.{cifname}_esd"] = f"{esd:.8f}"
block["_cell.volume"] = f"{crystal.get_unit_cell().volume():f}"
block["_cell.volume_esd"] = f"{crystal.get_cell_volume_sd():f}"
used_reflections = refiner.get_matches()
block["_cell_measurement.entry_id"] = "two_theta_refine"
block["_cell_measurement.reflns_used"] = len(used_reflections)
block["_cell_measurement.theta_min"] = (
flex.min(used_reflections["2theta_obs.rad"]) * 180 / math.pi / 2)
block["_cell_measurement.theta_max"] = (
flex.max(used_reflections["2theta_obs.rad"]) * 180 / math.pi / 2)
block["_exptl_crystal.id"] = 1
block["_diffrn.id"] = "two_theta_refine"
block["_diffrn.crystal_id"] = 1
block["_diffrn_reflns.diffrn_id"] = "two_theta_refine"
block["_diffrn_reflns.number"] = len(used_reflections)
miller_span = miller.index_span(used_reflections["miller_index"])
min_h, min_k, min_l = miller_span.min()
max_h, max_k, max_l = miller_span.max()
block["_diffrn_reflns.limit_h_min"] = min_h
block["_diffrn_reflns.limit_h_max"] = max_h
block["_diffrn_reflns.limit_k_min"] = min_k
block["_diffrn_reflns.limit_k_max"] = max_k
block["_diffrn_reflns.limit_l_min"] = min_l
block["_diffrn_reflns.limit_l_max"] = max_l
block["_diffrn_reflns.theta_min"] = (
flex.min(used_reflections["2theta_obs.rad"]) * 180 / math.pi / 2)
block["_diffrn_reflns.theta_max"] = (
flex.max(used_reflections["2theta_obs.rad"]) * 180 / math.pi / 2)
cif = iotbx.cif.model.cif()
cif["two_theta_refine"] = block
with open(filename, "w") as fh:
cif.show(out=fh)
def run(self, args=None):
"""Execute the script."""
# Parse the command line
params, _ = self.parser.parse_args(args, show_diff_phil=False)
# set up global reflections list
reflections = flex.reflection_table()
# loop through the input, building up the global lists
reflections_list, input_experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments)
experiments = copy.deepcopy(input_experiments)
reflections_list = parse_multiple_datasets(reflections_list)
for refs in reflections_list:
reflections.extend(refs)
# Try to load the models and data
nexp = len(experiments)
if nexp == 0:
print("No Experiments found in the input")
self.parser.print_help()
return
if not reflections:
print("No reflection data found in the input")
self.parser.print_help()
return
self.check_input(reflections)
# Configure the logging
log.config(logfile=params.output.log)
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
# Convert to P 1?
if params.refinement.triclinic:
reflections, experiments = self.convert_to_P1(
reflections, experiments)
# Combine crystals?
if params.refinement.combine_crystal_models and len(experiments) > 1:
logger.info("Combining %s crystal models", len(experiments))
experiments = self.combine_crystals(experiments)
# Filter integrated centroids?
if params.refinement.filter_integrated_centroids:
reflections = self.filter_integrated_centroids(reflections)
# Filter data if scaled to remove outliers
if "inverse_scale_factor" in reflections:
try:
reflections = filter_reflection_table(
reflections,
["scale"],
partiality_threshold=params.refinement.
partiality_threshold,
)
except ValueError as e:
logger.warn(e)
logger.info(
"Filtering on scaled data failed, proceeding with integrated data."
)
# Get the refiner
logger.info("Configuring refiner")
refiner = self.create_refiner(params, reflections, experiments)
# Refine the geometry
if nexp == 1:
logger.info("Performing refinement of a single Experiment...")
else:
logger.info(f"Performing refinement of {nexp} Experiments...")
refiner.run()
# get the refined experiments
experiments = copy.deepcopy(input_experiments)
for expt, refined_expt in zip(experiments, refiner.get_experiments()):
expt.crystal.set_recalculated_unit_cell(
refined_expt.crystal.get_unit_cell())
expt.crystal.set_recalculated_cell_parameter_sd(
refined_expt.crystal.get_cell_parameter_sd())
expt.crystal.set_recalculated_cell_volume_sd(
refined_expt.crystal.get_cell_volume_sd())
crystals = refiner.get_experiments().crystals()
if len(crystals) == 1:
# output the refined model for information
logger.info("")
logger.info("Final refined crystal model:")
logger.info(crystals[0])
logger.info(self.cell_param_table(crystals[0]))
# Save the refined experiments to file
output_experiments_filename = params.output.experiments
logger.info(
f"Saving refined experiments to {output_experiments_filename}")
experiments.as_file(output_experiments_filename)
# Create correlation plots
if params.output.correlation_plot.filename is not None:
create_correlation_plots(refiner, params.output)
if params.output.cif is not None:
self.generate_cif(crystals[0], refiner, filename=params.output.cif)
if params.output.p4p is not None:
self.generate_p4p(crystals[0],
experiments[0].beam,
filename=params.output.p4p)
if params.output.mmcif is not None:
self.generate_mmcif(crystals[0],
refiner,
filename=params.output.mmcif)
class Script(object):
'''A class for running the script.'''
def __init__(self):
'''Initialise the script.'''
from dials.util.options import OptionParser
import libtbx.load_env
# The script usage
usage = "usage: %s [options] [param.phil] filenames" % libtbx.env.dispatcher_name
# Create the parser
self.parser = OptionParser(
usage=usage,
phil=phil_scope,
check_format=False,
read_experiments=True,
read_reflections=True,
epilog=help_message
)
def run(self):
'''Execute the script.'''
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
n_bins = 10
arbitrary_padding = 1
legend = []
# local container for axes
figures = {}
for plot in ['deltaXY','isigi']:
fig, ax1 = plt.subplots()
fig.suptitle(plot)
ax2 = ax1.twinx()
figures[plot] = {'fig':fig, 'ax1': ax1, 'ax2': ax2}
def plotit(reflections, experiments):
"""
Make the plots for a set of reflections and experiments.
"""
detector = experiments.detectors()[0]
beam = experiments.beams()[0] # only used to compute resolution of 2theta
reflections = reflections.select(reflections['intensity.sum.variance'] > 0)
# Setup up deltaXY and two theta bins
reflections['difference_vector_norms'] = (reflections['xyzcal.mm']-reflections['xyzobs.mm.value']).norms()
reflections = setup_stats(detector, experiments, reflections, two_theta_only=True) # add two theta to reflection table
sorted_two_theta = flex.sorted(reflections['two_theta_obs'])
bin_low = [sorted_two_theta[int((len(sorted_two_theta)/n_bins) * i)] for i in range(n_bins)]
bin_high = [bin_low[i+1] for i in range(n_bins-1)]
bin_high.append(sorted_two_theta[-1]+arbitrary_padding)
x_centers = flex.double()
n_refls = flex.int()
rmsds = flex.double()
p25r = flex.double()
p50r = flex.double()
p75r = flex.double()
p25i = flex.double()
p50i = flex.double()
p75i = flex.double()
print("# 2theta Res N dXY IsigI")
# Compute stats for each bin
for i in range(n_bins):
refls = reflections.select((reflections['two_theta_obs'] >= bin_low[i]) & (reflections['two_theta_obs'] < bin_high[i]))
# Only compute deltaXY stats on reflections with I/sigI at least 5
i_sigi = refls['intensity.sum.value']/flex.sqrt(refls['intensity.sum.variance'])
refls = refls.select(i_sigi >= 5)
n = len(refls)
if n < 10: continue
min_r, q1_r, med_r, q3_r, max_r = five_number_summary(1000*refls['difference_vector_norms'])
n_refls.append(n)
rmsds_ = 1000*math.sqrt(flex.sum_sq(refls['difference_vector_norms'])/n)
min_i, q1_i, med_i, q3_i, max_i = five_number_summary(i_sigi)
p25i.append(q1_i)
p50i.append(med_i)
p75i.append(q3_i)
# x_center
c = ((bin_high[i]-bin_low[i])/2) + bin_low[i]
# resolution
d = beam.get_wavelength()/(2*math.sin(math.pi*c/(2*180)))
x_centers.append(c)
rmsds.append(rmsds_)
print("%d % 5.1f % 5.1f % 8d %.1f %.1f"%(i, c, d, n, med_r, med_i))
p25r.append(q1_r)
p50r.append(med_r)
p75r.append(q3_r)
# After binning, plot the results
for plot in figures:
ax1 = figures[plot]['ax1']
ax2 = figures[plot]['ax2']
if plot == 'isigi':
line, = ax1.plot(x_centers.as_numpy_array(), p50i.as_numpy_array(), '-')
line.set_label('Median')
ax1.fill_between(x_centers.as_numpy_array(), p25i.as_numpy_array(), p75i.as_numpy_array(),
interpolate = True, alpha = 0.50, color = line.get_color())
line, = ax2.plot(x_centers.as_numpy_array(), n_refls.as_numpy_array(), '-', color = line.get_color())
line.set_label('Median')
elif plot == 'deltaXY':
line, = ax1.plot(x_centers.as_numpy_array(), p50r.as_numpy_array(), '-')
line.set_label('Median')
ax1.fill_between(x_centers.as_numpy_array(), p25r.as_numpy_array(), p75r.as_numpy_array(),
interpolate = True, alpha = 0.50, color = line.get_color())
line, = ax2.plot(x_centers.as_numpy_array(), n_refls.as_numpy_array(), '-', color = line.get_color())
line.set_label('Median')
ax1.legend()
ax2.legend()
assert len(params.input.experiments) == len(params.input.reflections)
# Plotit!
for i in range(len(params.input.experiments)):
plotit(params.input.reflections[i].data, params.input.experiments[i].data)
# Set up labels
for plot in figures:
fig = figures[plot]['fig']
ax1 = figures[plot]['ax1']
ax2 = figures[plot]['ax2']
if plot == 'isigi':
ax1.set_ylabel("I/sigI")
ax2.set_ylabel("N reflections")
ax1.set_xlabel("Two theta (degrees)")
elif plot == 'deltaXY':
ax1.set_ylabel(r"$\Delta$XY")
ax2.set_ylabel("N reflections")
ax1.set_xlabel("Two theta (degrees)")
if params.tag is not None:
pass
if params.save_pdf:
from matplotlib.backends.backend_pdf import PdfPages
pp = PdfPages('reflection_stats.pdf')
for i in plt.get_fignums():
pp.savefig(plt.figure(i), dpi=300)
pp.close()
else:
plt.show()
class Script(object):
def __init__(self):
"""Initialise the script."""
# The script usage
usage = (
"usage: dials.rs_mapper map_file=output.ccp4 [max_resolution=6] [grid_size=192] "
"[reverse_phi=False] [param.phil] "
"{image1.file [image2.file ...]} | imported.expt")
# Initialise the base class
self.parser = OptionParser(usage=usage,
phil=phil_scope,
epilog=help_message,
read_experiments=True)
def run(self):
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
if not params.rs_mapper.map_file:
raise RuntimeError("Please specify output map file (map_file=)")
else:
self.map_file = params.rs_mapper.map_file
# Ensure we have either a data block or an experiment list
self.experiments = flatten_experiments(params.input.experiments)
if len(self.experiments) != 1:
self.parser.print_help()
print("Please pass either an experiment list\n")
return
self.reverse_phi = params.rs_mapper.reverse_phi
self.grid_size = params.rs_mapper.grid_size
self.max_resolution = params.rs_mapper.max_resolution
self.ignore_mask = params.rs_mapper.ignore_mask
self.grid = flex.double(
flex.grid(self.grid_size, self.grid_size, self.grid_size), 0)
self.counts = flex.int(
flex.grid(self.grid_size, self.grid_size, self.grid_size), 0)
for experiment in self.experiments:
self.process_imageset(experiment.imageset)
recviewer.normalize_voxels(self.grid, self.counts)
# Let's use 1/(100A) as the unit so that the absolute numbers in the
# "cell dimensions" field of the ccp4 map are typical for normal
# MX maps. The values in 1/A would give the "cell dimensions" around
# or below 1 and some MX programs would not handle it well.
box_size = 100 * 2.0 / self.max_resolution
uc = uctbx.unit_cell((box_size, box_size, box_size, 90, 90, 90))
ccp4_map.write_ccp4_map(
self.map_file,
uc,
sgtbx.space_group("P1"),
(0, 0, 0),
self.grid.all(),
self.grid,
flex.std_string(["cctbx.miller.fft_map"]),
)
def process_imageset(self, imageset):
rec_range = 1 / self.max_resolution
if len(imageset.get_detector()) != 1:
raise Sorry("This program does not support multi-panel detectors.")
panel = imageset.get_detector()[0]
beam = imageset.get_beam()
s0 = beam.get_s0()
pixel_size = panel.get_pixel_size()
xlim, ylim = imageset.get_raw_data(0)[0].all()
if pixel_size[0] != pixel_size[1]:
raise Sorry("This program does not support non-square pixels.")
# cache transformation
xy = recviewer.get_target_pixels(panel, s0, xlim, ylim,
self.max_resolution)
s1 = panel.get_lab_coord(xy * pixel_size[0])
s1 = s1 / s1.norms() * (1 / beam.get_wavelength())
S = s1 - s0
for i in range(len(imageset)):
axis = imageset.get_goniometer().get_rotation_axis()
osc_range = imageset.get_scan(i).get_oscillation_range()
print("Oscillation range: %.2f - %.2f" %
(osc_range[0], osc_range[1]))
angle = (osc_range[0] + osc_range[1]) / 2 / 180 * math.pi
if not self.reverse_phi:
# the pixel is in S AFTER rotation. Thus we have to rotate BACK.
angle *= -1
rotated_S = S.rotate_around_origin(axis, angle)
data = imageset.get_raw_data(i)[0]
if not self.ignore_mask:
mask = imageset.get_mask(i)[0]
data.set_selected(~mask, 0)
recviewer.fill_voxels(data, self.grid, self.counts, rotated_S, xy,
rec_range)
read_reflections=True,
check_format=False,
phil=phil_scope,
epilog=help_message,
)
else:
parser = OptionParser(
usage=usage,
read_experiments=True,
read_reflections=True,
phil=phil_scope,
epilog=help_message,
)
# Get the parameters
params, options = parser.parse_args(show_diff_phil=False)
# Configure the logging
log.config(info=params.output.log, debug=params.output.debug_log)
# Print the version number
logger.info(dials_version())
if os.getenv("DIALS_EXPORT_DO_NOT_CHECK_FORMAT"):
logger.info("(format checks disabled due to environment variable)")
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
class Script(object):
'''A class for running the script.'''
def __init__(self):
self.mpi_helper = mpi_helper()
self.mpi_logger = mpi_logger()
def __del__(self):
self.mpi_helper.finalize()
def parse_input(self):
'''Parse input at rank 0 and broadcast the input parameters and options to all ranks'''
if self.mpi_helper.rank == 0:
from xfel.merging.application.phil.phil import phil_scope
help_message = '''Merge xfel data.'''
# The script usage
import libtbx.load_env
self.usage = "usage: %s [options] [param.phil] " % libtbx.env.dispatcher_name
self.parser = None
'''Initialize the script.'''
from dials.util.options import OptionParser
# Create the parser
self.parser = OptionParser(
usage=self.usage,
phil=phil_scope,
epilog=help_message)
# Parse the command line. quick_parse is required for MPI compatibility
params, options = self.parser.parse_args(show_diff_phil=True,quick_parse=True)
# Log the modified phil parameters
diff_phil_str = self.parser.diff_phil.as_str()
if diff_phil_str != "":
self.mpi_logger.main_log("The following parameters have been modified:\n%s"%diff_phil_str)
# prepare for transmitting input parameters to all ranks
transmitted = dict(params = params, options = options)
# make the output folders
try:
os.mkdir(params.output.output_dir)
except FileExistsError:
pass
else:
transmitted = None
# broadcast parameters and options to all ranks
self.mpi_logger.log("Broadcasting input parameters...")
self.mpi_logger.log_step_time("BROADCAST_INPUT_PARAMS")
transmitted = self.mpi_helper.comm.bcast(transmitted, root = 0)
self.params = transmitted['params']
self.options = transmitted['options']
self.mpi_logger.set_log_file_paths(self.params)
self.mpi_logger.log("Received input parameters and options")
self.mpi_logger.log_step_time("BROADCAST_INPUT_PARAMS", True)
def run(self):
import datetime
time_now = datetime.datetime.now()
self.mpi_logger.log(str(time_now))
if self.mpi_helper.rank == 0:
self.mpi_logger.main_log(str(time_now))
self.mpi_logger.log_step_time("TOTAL")
self.mpi_logger.log_step_time("PARSE_INPUT_PARAMS")
self.parse_input()
self.mpi_logger.log_step_time("PARSE_INPUT_PARAMS", True)
if self.params.mp.debug.cProfile:
import cProfile
pr = cProfile.Profile()
pr.enable()
# Create the workers using the factories
self.mpi_logger.log_step_time("CREATE_WORKERS")
from xfel.merging import application
import importlib, copy
self._resolve_persistent_columns()
workers = []
steps = self.params.dispatch.step_list if self.params.dispatch.step_list else default_steps
for step in steps:
step_factory_name = step
step_additional_info = []
step_info = step.split('_')
assert len(step_info) > 0
if len(step_info) > 1:
step_factory_name = step_info[0]
step_additional_info = step_info[1:]
try:
factory = importlib.import_module('xfel.merging.application.' + step_factory_name + '.factory')
except ModuleNotFoundError:
# remember the system path so the custom worker can temporarily modify it
sys_path = copy.deepcopy(sys.path)
pathstr = os.path.join(
'~', '.cctbx.xfel', 'merging', 'application', step_factory_name,
'factory.py'
)
pathstr = os.path.expanduser(pathstr)
modulename = 'xfel.merging.application.' + step_factory_name + '.factory'
spec = importlib.util.spec_from_file_location(modulename, pathstr)
factory = importlib.util.module_from_spec(spec)
spec.loader.exec_module(factory)
# reset the path
sys.path = sys_path
workers.extend(factory.factory.from_parameters(self.params, step_additional_info, mpi_helper=self.mpi_helper, mpi_logger=self.mpi_logger))
# Perform phil validation up front
for worker in workers:
worker.validate()
self.mpi_logger.log_step_time("CREATE_WORKERS", True)
# Do the work
experiments = reflections = None
step = 0
while(workers):
worker = workers.pop(0)
self.mpi_logger.log_step_time("STEP_" + worker.__repr__())
# Log worker name, i.e. execution step name
step += 1
if step > 1:
self.mpi_logger.log('')
step_desc = "STEP %d: %s"%(step, worker)
self.mpi_logger.log(step_desc)
if self.mpi_helper.rank == 0:
if step > 1:
self.mpi_logger.main_log('')
self.mpi_logger.main_log(step_desc)
# Execute worker
experiments, reflections = worker.run(experiments, reflections)
self.mpi_logger.log_step_time("STEP_" + worker.__repr__(), True)
if experiments:
self.mpi_logger.log("Ending step with %d experiments"%len(experiments))
if self.params.output.save_experiments_and_reflections:
if len(reflections) and 'id' not in reflections:
from dials.array_family import flex
id_ = flex.int(len(reflections), -1)
if experiments:
for expt_number, expt in enumerate(experiments):
sel = reflections['exp_id'] == expt.identifier
id_.set_selected(sel, expt_number)
else:
for expt_number, exp_id in enumerate(set(reflections['exp_id'])):
sel = reflections['exp_id'] == exp_id
id_.set_selected(sel, expt_number)
reflections['id'] = id_
assert (reflections['id'] == -1).count(True) == 0, ((reflections['id'] == -1).count(True), len(reflections))
if self.mpi_helper.size == 1:
filename_suffix = ""
else:
filename_suffix = "_%06d"%self.mpi_helper.rank
if len(reflections):
reflections.as_pickle(os.path.join(self.params.output.output_dir, "%s%s.refl"%(self.params.output.prefix, filename_suffix)))
if experiments:
experiments.as_file(os.path.join(self.params.output.output_dir, "%s%s.expt"%(self.params.output.prefix, filename_suffix)))
self.mpi_logger.log_step_time("TOTAL", True)
if self.params.mp.debug.cProfile:
pr.disable()
pr.dump_stats(os.path.join(self.params.output.output_dir, "cpu_%s_%d.prof"%(self.params.output.prefix, self.mpi_helper.rank)))
def _resolve_persistent_columns(self):
if self.params.output.expanded_bookkeeping:
if self.params.input.persistent_refl_cols is None:
self.params.input.persistent_refl_cols = []
keysCreatedByMerge = ["input_refl_index", "orig_exp_id", "file_list_mapping", "is_odd_experiment"]
for key in keysCreatedByMerge:
if key not in self.params.input.persistent_refl_cols:
self.params.input.persistent_refl_cols.append(key)
class Script:
def __init__(self):
"""Initialise the script."""
# The phil scope
phil_scope = parse(
"""
by_detector = False
.type = bool
.help = "If True, instead of producing separate files for each"
"experiment, experiments are grouped by unique detector"
"model in the input set of experiments. For example, if"
"there are five detector models in the input data, five"
"sets of files will be produced, each containing"
"experiments that reference a single detector model."
by_wavelength = False
.type = bool
.help = "If True, group experiments by wavelength, from low to high"
"(using a relative tolerance of 1e-4 to match wavelengths)."
output {
experiments_prefix = split
.type = str
.help = "Filename prefix for the split experimental models"
reflections_prefix = split
.type = str
.help = "Filename prefix for the split reflections"
template = "{prefix}_{index:0{maxindexlength:d}d}.{extension}"
.type = str
.expert_level = 2
.help = "Template python format string for output filenames."
"Replaced variables are prefix (with"
"output.{experiments_prefix, reflections_prefix}),"
"index (number of split experiment), maxindexlength"
"(number of digits of total number of split experiments)"
"and extension (default file extension for model and"
"reflection files)"
chunk_size = None
.type = int
.expert_level = 2
.help = "If not None, instead of creating many individual"
"files, create composite files with no more than"
"chunk_size experiments per file."
chunk_sizes = None
.type = ints
.expert_level = 2
.help = "If not None, instead of creating many individual"
"files, create composite files with the number of"
"datasets given in the chunk_sizes list."
}
""",
process_includes=True,
)
# The script usage
usage = ("usage: dials.split_experiments [options] [param.phil] "
"experiments1.expt experiments2.expt reflections1.refl "
"reflections2.refl...")
# Create the parser
self.parser = OptionParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message,
)
def run(self, args=None):
"""Execute the script."""
# Parse the command line
params, _ = self.parser.parse_args(args, show_diff_phil=True)
# Try to load the models and data
if not params.input.experiments:
print("No Experiments found in the input")
self.parser.print_help()
return
if params.input.reflections:
if len(params.input.reflections) != len(params.input.experiments):
raise Sorry(
"The number of input reflections files does not match the "
"number of input experiments")
reflections, experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments)
if reflections:
reflections = reflections[0]
else:
reflections = None
experiments_template = functools.partial(
params.output.template.format,
prefix=params.output.experiments_prefix,
maxindexlength=len(str(len(experiments) - 1)),
extension="expt",
)
reflections_template = functools.partial(
params.output.template.format,
prefix=params.output.reflections_prefix,
maxindexlength=len(str(len(experiments) - 1)),
extension="refl",
)
if params.output.chunk_sizes:
if not sum(params.output.chunk_sizes) == len(experiments):
raise Sorry(
"Sum of chunk sizes list (%s) not equal to number of experiments (%s)"
% (sum(params.output.chunk_sizes), len(experiments)))
if params.by_wavelength:
if reflections:
if not reflections.experiment_identifiers():
raise Sorry(
"Unable to split by wavelength as no experiment "
"identifiers are set in the reflection table.")
if all(experiments.identifiers() == ""):
raise Sorry("Unable to split by wavelength as no experiment "
"identifiers are set in the experiment list.")
wavelengths = match_wavelengths(experiments)
for i, wl in enumerate(sorted(wavelengths.keys())):
expids = []
new_exps = ExperimentList()
exp_nos = wavelengths[wl]
imageset_ids = [] # record imageset ids to set in refl table
imagesets_found = OrderedSet()
for j in exp_nos:
expids.append(experiments[j].identifier) # string
new_exps.append(experiments[j])
imagesets_found.add(experiments[j].imageset)
imageset_ids.append(
imagesets_found.index(experiments[j].imageset))
experiment_filename = experiments_template(index=i)
print(
f"Saving experiments with wavelength {wl} to {experiment_filename}"
)
new_exps.as_json(experiment_filename)
if reflections:
refls = reflections.select_on_experiment_identifiers(
expids)
refls["imageset_id"] = flex.int(refls.size(), 0)
# now set the imageset ids
for k, iset_id in enumerate(imageset_ids):
# select the experiment based on id (unique per sweep),
# and set the imageset_id (not necessarily unique per sweep
# if imageset is shared)
sel = refls["id"] == k
refls["imageset_id"].set_selected(sel, iset_id)
reflections_filename = reflections_template(index=i)
print("Saving reflections with wavelength %s to %s" %
(wl, reflections_filename))
refls.as_file(reflections_filename)
elif params.by_detector:
assert (not params.output.chunk_size
), "chunk_size + by_detector is not implemented"
if reflections is None:
split_data = {
detector: {
"experiments": ExperimentList()
}
for detector in experiments.detectors()
}
else:
split_data = {
detector: {
"experiments": ExperimentList(),
"reflections": flex.reflection_table(),
"imagesets_found": OrderedSet(),
}
for detector in experiments.detectors()
}
for i, experiment in enumerate(experiments):
split_expt_id = experiments.detectors().index(
experiment.detector)
experiment_filename = experiments_template(index=split_expt_id)
print("Adding experiment %d to %s" % (i, experiment_filename))
split_data[experiment.detector]["experiments"].append(
experiment)
if reflections is not None:
reflections_filename = reflections_template(
index=split_expt_id)
split_data[experiment.detector]["imagesets_found"].add(
experiment.imageset)
print("Adding reflections for experiment %d to %s" %
(i, reflections_filename))
if reflections.experiment_identifiers().keys():
# first find which id value corresponds to experiment in question
identifier = experiment.identifier
id_ = None
for k in reflections.experiment_identifiers().keys():
if reflections.experiment_identifiers(
)[k] == identifier:
id_ = k
break
if id_ is None:
raise Sorry(
"Unable to find id matching experiment identifier in reflection table."
)
ref_sel = reflections.select(reflections["id"] == id_)
# now reset ids and reset/update identifiers map
for k in ref_sel.experiment_identifiers().keys():
del ref_sel.experiment_identifiers()[k]
new_id = len(
split_data[experiment.detector]["experiments"]) - 1
ref_sel["id"] = flex.int(len(ref_sel), new_id)
ref_sel.experiment_identifiers()[new_id] = identifier
else:
ref_sel = reflections.select(reflections["id"] == i)
ref_sel["id"] = flex.int(
len(ref_sel),
len(split_data[experiment.detector]["experiments"])
- 1,
)
iset_id = split_data[
experiment.detector]["imagesets_found"].index(
experiment.imageset)
ref_sel["imageset_id"] = flex.int(ref_sel.size(), iset_id)
split_data[experiment.detector]["reflections"].extend(
ref_sel)
for i, detector in enumerate(experiments.detectors()):
experiment_filename = experiments_template(index=i)
print("Saving experiment %d to %s" % (i, experiment_filename))
split_data[detector]["experiments"].as_json(
experiment_filename)
if reflections is not None:
reflections_filename = reflections_template(index=i)
print("Saving reflections for experiment %d to %s" %
(i, reflections_filename))
split_data[detector]["reflections"].as_file(
reflections_filename)
elif params.output.chunk_size or params.output.chunk_sizes:
def save_chunk(chunk_id, expts, refls):
experiment_filename = experiments_template(index=chunk_id)
print("Saving chunk %d to %s" %
(chunk_id, experiment_filename))
expts.as_json(experiment_filename)
if refls is not None:
reflections_filename = reflections_template(index=chunk_id)
print("Saving reflections for chunk %d to %s" %
(chunk_id, reflections_filename))
refls.as_file(reflections_filename)
chunk_counter = 0
chunk_expts = ExperimentList()
if reflections:
chunk_refls = flex.reflection_table()
else:
chunk_refls = None
next_iset_id = 0
imagesets_found = OrderedSet()
for i, experiment in enumerate(experiments):
chunk_expts.append(experiment)
if reflections:
if reflections.experiment_identifiers().keys():
# first find which id value corresponds to experiment in question
identifier = experiment.identifier
id_ = None
for k in reflections.experiment_identifiers().keys():
if reflections.experiment_identifiers(
)[k] == identifier:
id_ = k
break
if id_ is None:
raise Sorry(
"Unable to find id matching experiment identifier in reflection table."
)
ref_sel = reflections.select(reflections["id"] == id_)
# now reset ids and reset/update identifiers map
for k in ref_sel.experiment_identifiers().keys():
del ref_sel.experiment_identifiers()[k]
new_id = len(chunk_expts) - 1
ref_sel["id"] = flex.int(len(ref_sel), new_id)
ref_sel.experiment_identifiers()[new_id] = identifier
else:
ref_sel = reflections.select(reflections["id"] == i)
ref_sel["id"] = flex.int(len(ref_sel),
len(chunk_expts) - 1)
if experiment.imageset not in imagesets_found:
imagesets_found.add(experiment.imageset)
ref_sel["imageset_id"] = flex.int(
ref_sel.size(), next_iset_id)
next_iset_id += 1
else:
iset_id = imagesets_found.index(experiment.imageset)
ref_sel["imageset_id"] = flex.int(
ref_sel.size(), iset_id)
chunk_refls.extend(ref_sel)
if params.output.chunk_sizes:
chunk_limit = params.output.chunk_sizes[chunk_counter]
else:
chunk_limit = params.output.chunk_size
if len(chunk_expts) == chunk_limit:
save_chunk(chunk_counter, chunk_expts, chunk_refls)
chunk_counter += 1
chunk_expts = ExperimentList()
if reflections:
chunk_refls = flex.reflection_table()
else:
chunk_refls = None
if len(chunk_expts) > 0:
save_chunk(chunk_counter, chunk_expts, chunk_refls)
else:
for i, experiment in enumerate(experiments):
experiment_filename = experiments_template(index=i)
print("Saving experiment %d to %s" % (i, experiment_filename))
ExperimentList([experiment]).as_json(experiment_filename)
if reflections is not None:
reflections_filename = reflections_template(index=i)
print("Saving reflections for experiment %d to %s" %
(i, reflections_filename))
ref_sel = reflections.select(reflections["id"] == i)
if ref_sel.experiment_identifiers().keys():
identifier = ref_sel.experiment_identifiers()[i]
for k in ref_sel.experiment_identifiers().keys():
del ref_sel.experiment_identifiers()[k]
ref_sel["id"] = flex.int(ref_sel.size(), 0)
ref_sel.experiment_identifiers()[0] = identifier
else:
ref_sel["id"] = flex.int(len(ref_sel), 0)
ref_sel["imageset_id"] = flex.int(len(ref_sel), 0)
ref_sel.as_file(reflections_filename)
return
def run(args):
from dials.util.options import OptionParser
import libtbx.load_env
usage = "%s [options] find_spots.json" % (libtbx.env.dispatcher_name)
parser = OptionParser(usage=usage, phil=phil_scope, epilog=help_message)
params, options, args = parser.parse_args(show_diff_phil=True,
return_unhandled=True)
positions = None
if params.positions is not None:
with open(params.positions, 'rb') as f:
positions = flex.vec2_double()
for line in f.readlines():
line = line.replace('(', ' ').replace(')', '').replace(
',', ' ').strip().split()
assert len(line) == 3
i, x, y = [float(l) for l in line]
positions.append((x, y))
assert len(args) == 1
json_file = args[0]
import json
with open(json_file, 'rb') as f:
results = json.load(f)
n_indexed = flex.double()
fraction_indexed = flex.double()
n_spots = flex.double()
n_lattices = flex.double()
crystals = []
image_names = flex.std_string()
for r in results:
n_spots.append(r['n_spots_total'])
image_names.append(str(r['image']))
if 'n_indexed' in r:
n_indexed.append(r['n_indexed'])
n_lattices.append(len(r['lattices']))
for d in r['lattices']:
from dxtbx.serialize.crystal import from_dict
crystals.append(from_dict(d['crystal']))
else:
n_indexed.append(0)
n_lattices.append(0)
if n_indexed.size():
sel = n_spots > 0
fraction_indexed = flex.double(n_indexed.size(), 0)
fraction_indexed.set_selected(
sel,
n_indexed.select(sel) / n_spots.select(sel))
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot
blue = '#3498db'
red = '#e74c3c'
marker = 'o'
alpha = 0.5
lw = 0
plot = True
table = True
grid = params.grid
from libtbx import group_args
from dials.algorithms.spot_finding.per_image_analysis \
import plot_stats, print_table
estimated_d_min = flex.double()
d_min_distl_method_1 = flex.double()
d_min_distl_method_2 = flex.double()
n_spots_total = flex.int()
n_spots_no_ice = flex.int()
total_intensity = flex.double()
for d in results:
estimated_d_min.append(d['estimated_d_min'])
d_min_distl_method_1.append(d['d_min_distl_method_1'])
d_min_distl_method_2.append(d['d_min_distl_method_2'])
n_spots_total.append(d['n_spots_total'])
n_spots_no_ice.append(d['n_spots_no_ice'])
total_intensity.append(d['total_intensity'])
stats = group_args(image=image_names,
n_spots_total=n_spots_total,
n_spots_no_ice=n_spots_no_ice,
n_spots_4A=None,
n_indexed=n_indexed,
fraction_indexed=fraction_indexed,
total_intensity=total_intensity,
estimated_d_min=estimated_d_min,
d_min_distl_method_1=d_min_distl_method_1,
d_min_distl_method_2=d_min_distl_method_2,
noisiness_method_1=None,
noisiness_method_2=None)
if plot:
plot_stats(stats)
pyplot.clf()
if table:
print_table(stats)
n_rows = 10
n_rows = min(n_rows, len(n_spots_total))
perm_n_spots_total = flex.sort_permutation(n_spots_total, reverse=True)
print 'Top %i images sorted by number of spots:' % n_rows
print_table(stats, perm=perm_n_spots_total, n_rows=n_rows)
if flex.max(n_indexed) > 0:
perm_n_indexed = flex.sort_permutation(n_indexed, reverse=True)
print 'Top %i images sorted by number of indexed reflections:' % n_rows
print_table(stats, perm=perm_n_indexed, n_rows=n_rows)
print "Number of indexed lattices: ", (n_indexed > 0).count(True)
print "Number with valid d_min but failed indexing: ", (
(d_min_distl_method_1 > 0) & (d_min_distl_method_2 > 0) &
(estimated_d_min > 0) & (n_indexed == 0)).count(True)
n_bins = 20
spot_count_histogram(n_spots_total,
n_bins=n_bins,
filename='hist_n_spots_total.png',
log=True)
spot_count_histogram(n_spots_no_ice,
n_bins=n_bins,
filename='hist_n_spots_no_ice.png',
log=True)
spot_count_histogram(n_indexed.select(n_indexed > 0),
n_bins=n_bins,
filename='hist_n_indexed.png',
log=False)
if len(crystals):
plot_unit_cell_histograms(crystals)
if params.stereographic_projections and len(crystals):
from dxtbx.datablock import DataBlockFactory
datablocks = DataBlockFactory.from_filenames([image_names[0]],
verbose=False)
assert len(datablocks) == 1
imageset = datablocks[0].extract_imagesets()[0]
s0 = imageset.get_beam().get_s0()
# XXX what if no goniometer?
rotation_axis = imageset.get_goniometer().get_rotation_axis()
indices = ((1, 0, 0), (0, 1, 0), (0, 0, 1))
for i, index in enumerate(indices):
from cctbx import crystal, miller
from scitbx import matrix
miller_indices = flex.miller_index([index])
symmetry = crystal.symmetry(
unit_cell=crystals[0].get_unit_cell(),
space_group=crystals[0].get_space_group())
miller_set = miller.set(symmetry, miller_indices)
d_spacings = miller_set.d_spacings()
d_spacings = d_spacings.as_non_anomalous_array().expand_to_p1()
d_spacings = d_spacings.generate_bijvoet_mates()
miller_indices = d_spacings.indices()
# plane normal
d0 = matrix.col(s0).normalize()
d1 = d0.cross(matrix.col(rotation_axis)).normalize()
d2 = d1.cross(d0).normalize()
reference_poles = (d0, d1, d2)
from dials.command_line.stereographic_projection import stereographic_projection
projections = []
for cryst in crystals:
reciprocal_space_points = list(
cryst.get_U() *
cryst.get_B()) * miller_indices.as_vec3_double()
projections.append(
stereographic_projection(reciprocal_space_points,
reference_poles))
#from dials.algorithms.indexing.compare_orientation_matrices import \
# difference_rotation_matrix_and_euler_angles
#R_ij, euler_angles, cb_op = difference_rotation_matrix_and_euler_angles(
# crystals[0], cryst)
#print max(euler_angles)
from dials.command_line.stereographic_projection import plot_projections
plot_projections(projections,
filename='projections_%s.png' % ('hkl'[i]))
pyplot.clf()
def plot_grid(values,
grid,
file_name,
cmap=pyplot.cm.Reds,
vmin=None,
vmax=None,
invalid='white'):
values = values.as_double()
# At DLS, fast direction appears to be largest direction
if grid[0] > grid[1]:
values.reshape(flex.grid(reversed(grid)))
values = values.matrix_transpose()
else:
values.reshape(flex.grid(grid))
Z = values.as_numpy_array()
#f, (ax1, ax2) = pyplot.subplots(2)
f, ax1 = pyplot.subplots(1)
mesh1 = ax1.pcolormesh(values.as_numpy_array(),
cmap=cmap,
vmin=vmin,
vmax=vmax)
mesh1.cmap.set_under(color=invalid, alpha=None)
mesh1.cmap.set_over(color=invalid, alpha=None)
#mesh2 = ax2.contour(Z, cmap=cmap, vmin=vmin, vmax=vmax)
#mesh2 = ax2.contourf(Z, cmap=cmap, vmin=vmin, vmax=vmax)
ax1.set_aspect('equal')
ax1.invert_yaxis()
#ax2.set_aspect('equal')
#ax2.invert_yaxis()
pyplot.colorbar(mesh1, ax=ax1)
#pyplot.colorbar(mesh2, ax=ax2)
pyplot.savefig(file_name, dpi=600)
pyplot.clf()
def plot_positions(values,
positions,
file_name,
cmap=pyplot.cm.Reds,
vmin=None,
vmax=None,
invalid='white'):
values = values.as_double()
assert positions.size() >= values.size()
positions = positions[:values.size()]
if vmin is None:
vmin = flex.min(values)
if vmax is None:
vmax = flex.max(values)
x, y = positions.parts()
dx = flex.abs(x[1:] - x[:-1])
dy = flex.abs(y[1:] - y[:-1])
dx = dx.select(dx > 0)
dy = dy.select(dy > 0)
scale = 1 / flex.min(dx)
#print scale
x = (x * scale).iround()
y = (y * scale).iround()
from libtbx.math_utils import iceil
z = flex.double(
flex.grid(iceil(flex.max(y)) + 1,
iceil(flex.max(x)) + 1), -2)
#print z.all()
for x_, y_, z_ in zip(x, y, values):
z[y_, x_] = z_
plot_grid(z.as_1d(),
z.all(),
file_name,
cmap=cmap,
vmin=vmin,
vmax=vmax,
invalid=invalid)
return
if grid is not None or positions is not None:
if grid is not None:
positions = tuple(reversed(grid))
plotter = plot_grid
else:
plotter = plot_positions
cmap = pyplot.get_cmap(params.cmap)
plotter(n_spots_total,
positions,
'grid_spot_count_total.png',
cmap=cmap,
invalid=params.invalid)
plotter(n_spots_no_ice,
positions,
'grid_spot_count_no_ice.png',
cmap=cmap,
invalid=params.invalid)
plotter(total_intensity,
positions,
'grid_total_intensity.png',
cmap=cmap,
invalid=params.invalid)
if flex.max(n_indexed) > 0:
plotter(n_indexed,
positions,
'grid_n_indexed.png',
cmap=cmap,
invalid=params.invalid)
plotter(fraction_indexed,
positions,
'grid_fraction_indexed.png',
cmap=cmap,
vmin=0,
vmax=1,
invalid=params.invalid)
for i, d_min in enumerate(
(estimated_d_min, d_min_distl_method_1, d_min_distl_method_2)):
from cctbx import uctbx
d_star_sq = uctbx.d_as_d_star_sq(d_min)
d_star_sq.set_selected(d_star_sq == 1, 0)
vmin = flex.min(d_star_sq.select(d_star_sq > 0))
vmax = flex.max(d_star_sq)
vmin = flex.min(d_min.select(d_min > 0))
vmax = flex.max(d_min)
cmap = pyplot.get_cmap('%s_r' % params.cmap)
d_min.set_selected(d_min <= 0, vmax)
if i == 0:
plotter(d_min,
positions,
'grid_d_min.png',
cmap=cmap,
vmin=vmin,
vmax=vmax,
invalid=params.invalid)
else:
plotter(d_min,
positions,
'grid_d_min_method_%i.png' % i,
cmap=cmap,
vmin=vmin,
vmax=vmax,
invalid=params.invalid)
if flex.max(n_indexed) > 0:
pyplot.hexbin(n_spots,
n_indexed,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.scatter(n_spots, n_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
xlim = pyplot.xlim()
ylim = pyplot.ylim()
pyplot.plot([0, max(n_spots)], [0, max(n_spots)], c=red)
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.xlabel('# spots')
pyplot.ylabel('# indexed')
pyplot.savefig('n_spots_vs_n_indexed.png')
pyplot.clf()
pyplot.hexbin(n_spots,
fraction_indexed,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_spots, fraction_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# spots')
pyplot.ylabel('Fraction indexed')
pyplot.savefig('n_spots_vs_fraction_indexed.png')
pyplot.clf()
pyplot.hexbin(n_indexed,
fraction_indexed,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_indexed, fraction_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# indexed')
pyplot.ylabel('Fraction indexed')
pyplot.savefig('n_indexed_vs_fraction_indexed.png')
pyplot.clf()
pyplot.hexbin(n_spots,
n_lattices,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_spots, n_lattices, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# spots')
pyplot.ylabel('# lattices')
pyplot.savefig('n_spots_vs_n_lattices.png')
pyplot.clf()
#pyplot.scatter(
# estimated_d_min, d_min_distl_method_1, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.hexbin(estimated_d_min,
d_min_distl_method_1,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.gca().set_aspect('equal')
xlim = pyplot.xlim()
ylim = pyplot.ylim()
m = max(max(estimated_d_min), max(d_min_distl_method_1))
pyplot.plot([0, m], [0, m], c=red)
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.xlabel('estimated_d_min')
pyplot.ylabel('d_min_distl_method_1')
pyplot.savefig('d_min_vs_distl_method_1.png')
pyplot.clf()
#pyplot.scatter(
# estimated_d_min, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.hexbin(estimated_d_min,
d_min_distl_method_2,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.gca().set_aspect('equal')
xlim = pyplot.xlim()
ylim = pyplot.ylim()
m = max(max(estimated_d_min), max(d_min_distl_method_2))
pyplot.plot([0, m], [0, m], c=red)
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.xlabel('estimated_d_min')
pyplot.ylabel('d_min_distl_method_2')
pyplot.savefig('d_min_vs_distl_method_2.png')
pyplot.clf()
#pyplot.scatter(
# d_min_distl_method_1, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.hexbin(d_min_distl_method_1,
d_min_distl_method_2,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.gca().set_aspect('equal')
xlim = pyplot.xlim()
ylim = pyplot.ylim()
m = max(max(d_min_distl_method_1), max(d_min_distl_method_2))
pyplot.plot([0, m], [0, m], c=red)
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.xlabel('d_min_distl_method_1')
pyplot.ylabel('d_min_distl_method_2')
pyplot.savefig('distl_method_1_vs_distl_method_2.png')
pyplot.clf()
pyplot.hexbin(n_spots,
estimated_d_min,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_spots, estimated_d_min, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# spots')
pyplot.ylabel('estimated_d_min')
pyplot.savefig('n_spots_vs_d_min.png')
pyplot.clf()
pyplot.hexbin(n_spots,
d_min_distl_method_1,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_spots, d_min_distl_method_1, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# spots')
pyplot.ylabel('d_min_distl_method_1')
pyplot.savefig('n_spots_vs_distl_method_1.png')
pyplot.clf()
pyplot.hexbin(n_spots,
d_min_distl_method_2,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_spots, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# spots')
pyplot.ylabel('d_min_distl_method_2')
pyplot.savefig('n_spots_vs_distl_method_2.png')
pyplot.clf()
class Script(ParentScript):
''' Class to parse the command line options. '''
def __init__(self):
''' Set the expected options. '''
from dials.util.options import OptionParser
import libtbx.load_env
# Create the option parser
usage = "usage: %s experiment1.json experiment2.json reflections1.pickle reflections2.pickle" % libtbx.env.dispatcher_name
self.parser = OptionParser(
usage=usage,
sort_options=True,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message)
def run(self):
''' Parse the options. '''
from dials.util.options import flatten_experiments, flatten_reflections
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
self.params = params
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
# Find all detector objects
detectors = []
detectors.extend(experiments.detectors())
# Verify inputs
if len(detectors) != 2:
raise Sorry("Please provide a reference and a moving set of experiments")
reflections = reflections[1]
detector = detectors[1]
if not hasattr(detector, 'hierarchy'):
raise Sorry("Script intended for hierarchical detectors")
if params.max_hierarchy_level is None or str(params.max_hierarchy_level).lower() == 'auto':
params.max_hierarchy_level = 0
root = detector.hierarchy()
while root.is_group():
root = root[0]
params.max_hierarchy_level += 1
print "Found", params.max_hierarchy_level+1, "hierarchy levels"
reference_root = detectors[0].hierarchy()
moving_root = detector.hierarchy()
rori = get_center(reference_root)
rf = col(reference_root.get_fast_axis())
rs = col(reference_root.get_slow_axis())
r_norm = col(reference_root.get_normal())
s0 = col(flex.vec3_double([col(b.get_s0()) for b in experiments.beams()]).mean())
summary_table_header = ["Hierarchy","Delta XY","Delta XY","R Offsets","R Offsets","T Offsets","T Offsets","Z Offsets","Z Offsets","dR Norm","dR Norm","dT Norm","dT Norm","Local dNorm", "Local dNorm", "Rot Z","Rot Z"]
summary_table_header2 = ["Level","","Sigma","","Sigma","","Sigma","","Sigma","","Sigma","","Sigma","","Sigma","","Sigma"]
summary_table_header3 = ["","(microns)","(microns)","(microns)","(microns)","(microns)","(microns)","(microns)","(microns)","(deg)","(deg)","(deg)","(deg)","(deg)","(deg)","(deg)","(deg)"]
summary_table_data = []
summary_table_data.append(summary_table_header)
summary_table_data.append(summary_table_header2)
summary_table_data.append(summary_table_header3)
table_header = ["PanelG","BC dist","Delta XY","R Offsets","T Offsets","Z Offsets","dR Norm","dT Norm","Local dNorm","Rot Z","N Refls"]
table_header2 = ["ID","(mm)","(microns)","(microns)","(microns)","(microns)","(deg)","(deg)","(deg)","(deg)",""]
from xfel.cftbx.detector.cspad_cbf_tbx import basis
def get_full_basis_shift(pg):
"""Compute basis shift from pg to lab space"""
shift = basis(panelgroup=pg)
while True:
parent = pg.parent()
if parent is None:
break
shift = basis(panelgroup=parent) * shift
pg = parent
return shift
# Iterate through the hierarchy levels
for level in range(params.max_hierarchy_level+1):
delta_xy = flex.double()
r_offsets = flex.double()
t_offsets = flex.double()
z_offsets = flex.double()
rot_z = flex.double()
delta_r_norm = flex.double()
delta_t_norm = flex.double()
local_dnorm = flex.double()
bc_dists = flex.double()
weights = flex.double()
rows = []
for pg_id, (pg1, pg2) in enumerate(zip(iterate_detector_at_level(reference_root, 0, level),
iterate_detector_at_level(moving_root, 0, level))):
weight = 0
for panel_id, p in enumerate(iterate_panels(pg2)):
weight += len(reflections.select(reflections['panel'] == id_from_name(detector, p.get_name())))
weights.append(weight)
bc = col(pg1.get_beam_centre_lab(s0))
ori = get_center(pg1)
bc_dist = (ori-bc).length()
bc_dists.append(bc_dist)
z_dists = []
ori_xy = []
for pg in [pg1,pg2]:
ori = pg.get_local_origin()
ori_xy.append(col((ori[0], ori[1])))
z_dists.append(ori[2]*1000)
dxy = (ori_xy[1]-ori_xy[0]).length()*1000
delta_xy.append(dxy)
z_off = z_dists[1]-z_dists[0]
z_offsets.append(z_off)
pgo1 = col(pg1.get_origin())
ro_pgo = pgo1 - rori # vector from the detector origin to the panel group origin
if ro_pgo.length() == 0:
radial = col((0,0,0))
transverse = col((0,0,0))
else:
radial = ((rf.dot(ro_pgo) * rf) + (rs.dot(ro_pgo) * rs)).normalize() # component of ro_pgo in rf rs plane
transverse = r_norm.cross(radial).normalize()
# now radial and transverse are vectors othogonal to each other and the detector normal, such that
# radial points at the panel group origin
# compute shift in local frame, then convert that shift to lab space, then make it relative to the reference's origin, in lab space
lpgo1 = col(pg1.get_local_origin())
lpgo2 = col(pg2.get_local_origin())
delta_pgo = (get_full_basis_shift(pg1) * (lpgo2-lpgo1)) - pgo1
# v is the component of delta_pgo along the radial vector
v = (radial.dot(delta_pgo) * radial)
r_offset = v.length() * 1000
angle = r_norm.angle(v, deg=True)
if r_norm.cross(v).dot(transverse) < 0:
r_offset = -r_offset
r_offsets.append(r_offset)
# v is the component of delta_pgo along the transverse vector
v = (transverse.dot(delta_pgo) * transverse)
t_offset = v.length() * 1000
angle = r_norm.angle(v, deg=True)
if r_norm.cross(v).dot(radial) < 0:
t_offset = -t_offset
t_offsets.append(t_offset)
pgn1 = col(pg1.get_normal())
pgf1 = col(pg1.get_fast_axis())
pgs1 = col(pg1.get_slow_axis())
pgn2 = col(pg2.get_normal())
pgf2 = col(pg2.get_fast_axis())
# v1 and v2 are the component of pgf1 and pgf2 in the rf rs plane
v1 = (rf.dot(pgf1) * rf) + (rs.dot(pgf1) * rs)
v2 = (rf.dot(pgf2) * rf) + (rs.dot(pgf2) * rs)
rz = v1.angle(v2, deg=True)
rot_z.append(rz)
# v1 and v2 are the components of pgn1 and pgn2 in the r_norm radial plane
v1 = (r_norm.dot(pgn1) * r_norm) + (radial.dot(pgn1) * radial)
v2 = (r_norm.dot(pgn2) * r_norm) + (radial.dot(pgn2) * radial)
drn = v1.angle(v2, deg=True)
if v2.cross(v1).dot(transverse) < 0:
drn = -drn
delta_r_norm.append(drn)
# v1 and v2 are the components of pgn1 and pgn2 in the r_norm transverse plane
v1 = (r_norm.dot(pgn1) * r_norm) + (transverse.dot(pgn1) * transverse)
v2 = (r_norm.dot(pgn2) * r_norm) + (transverse.dot(pgn2) * transverse)
dtn = v1.angle(v2, deg=True)
if v2.cross(v1).dot(radial) < 0:
dtn = -dtn
delta_t_norm.append(dtn)
# Determine angle between normals in local space
lpgf1 = col(pg1.get_local_fast_axis())
lpgs1 = col(pg1.get_local_slow_axis())
lpgn1 = lpgf1.cross(lpgs1)
lpgf2 = col(pg2.get_local_fast_axis())
lpgs2 = col(pg2.get_local_slow_axis())
lpgn2 = lpgf2.cross(lpgs2)
ldn = lpgn1.angle(lpgn2, deg=True)
local_dnorm.append(ldn)
row = ["%3d"%pg_id, "%6.1f"%bc_dist, "%6.1f"%dxy,
"%6.1f"%r_offset, "%6.1f"%t_offset, "%6.1f"%z_off,
"%.4f"%drn, "%.4f"%dtn, "%.4f"%ldn, "%.4f"%rz, "%8d"%weight]
rows.append(row)
wm_row = ["Weighted mean", ""]
ws_row = ["Weighted stddev", ""]
s_row = ["%d"%level]
iterable = zip([delta_xy, r_offsets, t_offsets, z_offsets, delta_r_norm, delta_t_norm, local_dnorm, rot_z],
["%6.1f","%6.1f","%6.1f","%6.1f","%.4f","%.4f","%.4f","%.4f"])
if len(z_offsets) == 0:
wm_row.extend(["%6.1f"%0]*8)
ws_row.extend(["%6.1f"%0]*8)
s_row.extend(["%6.1f"%0]*8)
elif len(z_offsets) == 1:
for data, fmt in iterable:
wm_row.append(fmt%data[0])
ws_row.append(fmt%0)
s_row.append(fmt%data[0])
s_row.append(fmt%0)
else:
for data, fmt in iterable:
stats = flex.mean_and_variance(data, weights)
wm_row.append(fmt%stats.mean())
ws_row.append(fmt%stats.gsl_stats_wsd())
s_row.append(fmt%stats.mean())
s_row.append(fmt%stats.gsl_stats_wsd())
wm_row.append("")
ws_row.append("")
summary_table_data.append(s_row)
table_data = [table_header, table_header2]
table_d = {d:row for d, row in zip(bc_dists, rows)}
table_data.extend([table_d[key] for key in sorted(table_d)])
table_data.append(wm_row)
table_data.append(ws_row)
from libtbx import table_utils
print "Hierarchy level %d Detector shifts"%level
print table_utils.format(table_data,has_header=2,justify='center',delim=" ")
print "Detector shifts summary"
print table_utils.format(summary_table_data,has_header=3,justify='center',delim=" ")
print
print """
class Script(object):
'''A class for running the script.'''
def __init__(self):
from dials.util.options import OptionParser
import libtbx.load_env
usage = 'usage: %s [options] experiments.json indexed.pickle' \
% libtbx.env.dispatcher_name
self.parser = OptionParser(usage=usage,
phil=phil_scope,
epilog=help_message,
check_format=False,
read_experiments=True,
read_reflections=True)
def run(self):
from dials.array_family import flex # import dependency
from scitbx import matrix
from dials.util.options import flatten_experiments
from dials.util.options import flatten_reflections
params, options = self.parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(experiments) == 0:
self.parser.print_help()
return
if len(reflections) != 1:
self.parser.print_help()
return
reflections = reflections[0]
print 'Read %d reflections' % len(reflections)
indexed = reflections.select(
reflections.get_flags(reflections.flags.indexed))
print 'Kept %d indexed reflections' % len(indexed)
for name in sorted(indexed.keys()):
print 'Found column %s' % name
for reflection in indexed[:3]:
print reflection
# verify that these experiments correspond to exactly one imageset, one
# detector, one beam (obviously)
for experiment in experiments[1:]:
assert experiment.imageset == experiments[0].imageset
assert experiment.beam == experiments[0].beam
assert experiment.detector == experiments[0].detector
# now perform some calculations - the only things different from one
# experiment to the next will be crystal models
crystals = [experiment.crystal for experiment in experiments]
detector = experiments[0].detector
beam = experiments[0].beam
imageset = experiments[0].imageset
# derived quantities
wavelength = beam.get_wavelength()
s0 = matrix.col(beam.get_s0())
# in here do some jiggery-pokery to cope with this being interpreted as
# a rotation image in here i.e. if scan is not None; derive goniometer
# matrix else set this to identity
scan = experiments[0].scan
goniometer = experiments[0].goniometer
if scan and goniometer:
angle = scan.get_angle_from_array_index(
0.5 * sum(imageset.get_array_range()))
axis = matrix.col(goniometer.get_rotation_axis_datum())
F = matrix.sqr(goniometer.get_fixed_rotation())
S = matrix.sqr(goniometer.get_setting_rotation())
R = S * axis.axis_and_angle_as_r3_rotation_matrix(angle,
deg=True) * F
else:
R = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))
assert (len(detector) == 1)
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
# The script usage
usage = "dials.stereographic_projection [options] [param.phil] indexed.expt"
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=False,
epilog=help_message,
)
params, options = parser.parse_args(args=args, show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if not experiments:
parser.print_help()
return
if not params.hkl and params.hkl_limit is None:
sys.exit("Please provide hkl or hkl_limit parameters.")
if params.plot.labels and len(params.plot.labels) != len(experiments):
sys.exit(
"Number of labels (%i) must equal number of experiments (%i)" %
(len(params.plot.labels), len(experiments)))
if params.hkl is not None and len(params.hkl):
miller_indices = flex.miller_index(params.hkl)
elif params.hkl_limit is not None:
limit = params.hkl_limit
miller_indices = flex.miller_index()
for h in range(-limit, limit + 1):
for k in range(-limit, limit + 1):
for l in range(-limit, limit + 1):
if (h, k, l) == (0, 0, 0):
continue
miller_indices.append((h, k, l))
crystals = experiments.crystals()
symmetry = crystal.symmetry(unit_cell=crystals[0].get_unit_cell(),
space_group=crystals[0].get_space_group())
miller_set = miller.set(symmetry, miller_indices)
d_spacings = miller_set.d_spacings()
if params.eliminate_sys_absent:
d_spacings = d_spacings.eliminate_sys_absent()
if params.expand_to_p1:
d_spacings = d_spacings.as_non_anomalous_array().expand_to_p1()
miller_indices = d_spacings.indices()
# find the greatest common factor (divisor) between miller indices
miller_indices_unique = flex.miller_index()
for hkl in miller_indices:
gcd = gcd_list(hkl)
if gcd > 1:
miller_indices_unique.append(tuple(int(h / gcd) for h in hkl))
elif gcd < 1:
pass
else:
miller_indices_unique.append(hkl)
miller_indices = miller_indices_unique
miller_indices = flex.miller_index(list(set(miller_indices)))
ref_crystal = crystals[0]
U = matrix.sqr(ref_crystal.get_U())
B = matrix.sqr(ref_crystal.get_B())
R = matrix.identity(3)
if params.frame == "laboratory":
reference_poles = reference_poles_perpendicular_to_beam(
experiments[0].beam, experiments[0].goniometer)
if params.use_starting_angle:
rotation_axis = matrix.col(
experiments[0].goniometer.get_rotation_axis())
R = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
experiments[0].scan.get_oscillation()[0], deg=True)
elif params.phi_angle != 0:
rotation_axis = matrix.col(
experiments[0].goniometer.get_rotation_axis())
R = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
params.phi_angle, deg=True)
else:
if params.plane_normal is not None:
plane_normal = params.plane_normal
else:
plane_normal = (0, 0, 1)
reference_poles = reference_poles_crystal(ref_crystal,
plane_normal=plane_normal)
if params.frame == "crystal":
U = matrix.identity(3)
reciprocal_space_points = list(R * U * B) * miller_indices.as_vec3_double()
projections_ref = stereographic_projection(reciprocal_space_points,
reference_poles)
projections_all = [projections_ref]
if experiments:
from dials.algorithms.indexing.compare_orientation_matrices import (
difference_rotation_matrix_axis_angle, )
for expt in experiments[1:]:
cryst = expt.crystal
if params.frame == "crystal":
R_ij, axis, angle, cb_op = difference_rotation_matrix_axis_angle(
ref_crystal, cryst)
U = R_ij
elif params.use_starting_angle:
if params.use_starting_angle:
rotation_axis = matrix.col(
expt.goniometer.get_rotation_axis())
R = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
expt.scan.get_oscillation()[0], deg=True)
else:
U = matrix.sqr(cryst.get_U())
reciprocal_space_points = (
list(R * U * matrix.sqr(cryst.get_B())) *
miller_indices.as_vec3_double())
projections = stereographic_projection(reciprocal_space_points,
reference_poles)
projections_all.append(projections)
if params.save_coordinates:
with open("projections.txt", "w") as f:
f.write("crystal h k l x y" + os.linesep)
for i_cryst, projections in enumerate(projections_all):
for hkl, proj in zip(miller_indices, projections):
f.write("%i " % (i_cryst + 1))
f.write("%i %i %i " % hkl)
f.write(("%f %f" + os.linesep) % proj)
if params.plot.filename:
epochs = None
if params.plot.colour_map is not None:
if experiments[0].scan is not None:
epochs = [expt.scan.get_epochs()[0] for expt in experiments]
else:
epochs = [i for i, expt in enumerate(experiments)]
plot_projections(
projections_all,
filename=params.plot.filename,
colours=params.plot.colours,
marker_size=params.plot.marker_size,
font_size=params.plot.font_size,
gridsize=params.plot.gridsize,
label_indices=miller_indices
if params.plot.label_indices else False,
epochs=epochs,
colour_map=params.plot.colour_map,
)
if params.json.filename:
projections_as_json(projections_all,
filename=params.json.filename,
labels=params.plot.labels)
def run(args):
usage = "dials.plot_reflections models.expt observations.refl [options]"
from dials.util.options import OptionParser, reflections_and_experiments_from_files
from scitbx.array_family import flex
from scitbx import matrix
parser = OptionParser(
usage=usage,
phil=master_phil_scope,
read_experiments=True,
read_reflections=True,
check_format=False,
)
params, options = parser.parse_args(show_diff_phil=True)
reflections, experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments
)
if len(experiments.imagesets()) > 0:
imageset = experiments.imagesets()[0]
imageset.set_detector(experiments[0].detector)
imageset.set_beam(experiments[0].beam)
imageset.set_goniometer(experiments[0].goniometer)
else:
parser.print_help()
return
detector = imageset.get_detector()
scan = imageset.get_scan()
panel_origin_shifts = {0: (0, 0, 0)}
try:
hierarchy = detector.hierarchy()
except AttributeError:
hierarchy = None
for i_panel in range(1, len(detector)):
origin_shift = matrix.col(detector[0].get_origin()) - matrix.col(
detector[i_panel].get_origin()
)
panel_origin_shifts[i_panel] = origin_shift
observed_xyz = flex.vec3_double()
predicted_xyz = flex.vec3_double()
for reflection_list in reflections:
if len(params.scan_range):
sel = flex.bool(len(reflection_list), False)
xyzcal_px = None
xyzobs_px = None
if "xyzcal.px" in reflection_list:
xyzcal_px = reflection_list["xyzcal.px"]
if "xyzobs.px.value" in reflection_list:
xyzobs_px = reflection_list["xyzobs.px.value"]
if xyzcal_px is not None and not xyzcal_px.norms().all_eq(0):
centroids_frame = xyzcal_px.parts()[2]
elif xyzobs_px is not None and not xyzobs_px.norms().all_eq(0):
centroids_frame = xyzobs_px.parts()[2]
else:
raise Sorry("No pixel coordinates given in input reflections.")
for scan_range in params.scan_range:
if scan_range is None:
continue
range_start, range_end = scan_range
sel |= (centroids_frame >= range_start) & (centroids_frame < range_end)
reflection_list = reflection_list.select(sel)
if params.first_n_reflections is not None:
centroid_positions = reflection_list.centroid_position()
centroids_frame = centroid_positions.parts()[2]
perm = flex.sort_permutation(centroids_frame)
perm = perm[: min(reflection_list.size(), params.first_n_reflections)]
reflection_list = reflection_list.select(perm)
if params.crystal_id is not None:
reflection_list = reflection_list.select(
reflection_list["id"] == params.crystal_id
)
xyzobs_px = None
xyzobs_mm = None
xyzcal_mm = None
if "xyzobs.px.value" in reflection_list:
xyzobs_px = reflection_list["xyzobs.px.value"]
if "xyzcal.mm" in reflection_list:
xyzcal_mm = reflection_list["xyzcal.mm"]
if "xyzobs.mm.value" in reflection_list:
xyzobs_mm = reflection_list["xyzobs.mm.value"]
panel_ids = reflection_list["panel"]
if xyzobs_mm is None and xyzobs_px is not None:
xyzobs_mm = flex.vec3_double()
for i_panel in range(len(detector)):
xyzobs_px_panel = xyzobs_px.select(panel_ids == i_panel)
from dials.algorithms.centroid import centroid_px_to_mm_panel
xyzobs_mm_panel, _, _ = centroid_px_to_mm_panel(
detector[i_panel],
scan,
xyzobs_px_panel,
flex.vec3_double(xyzobs_px_panel.size()),
flex.vec3_double(xyzobs_px_panel.size()),
)
xyzobs_mm.extend(xyzobs_mm_panel)
if xyzobs_mm is not None:
observed_xyz.extend(xyzobs_mm)
if xyzcal_mm is not None:
predicted_xyz.extend(xyzcal_mm)
obs_x, obs_y, _ = observed_xyz.parts()
pred_x, pred_y, _ = predicted_xyz.parts()
try:
import matplotlib
if not params.output.show_plot:
# http://matplotlib.org/faq/howto_faq.html#generate-images-without-having-a-window-appear
matplotlib.use("Agg") # use a non-interactive backend
import matplotlib.pyplot as plt
except ImportError:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = plt.figure()
fig.set_size_inches(params.output.size_inches)
fig.set_dpi(params.output.dpi)
plt.axes().set_aspect("equal")
marker_size = params.output.marker_size
if obs_x.size():
plt.scatter(
obs_x,
obs_y,
marker="o",
edgecolors="black",
c="white",
s=marker_size,
alpha=1,
)
if pred_x.size():
plt.scatter(pred_x, pred_y, marker="+", s=marker_size, c="blue")
xmax = max(
[
detector[i_panel].get_image_size_mm()[0] + panel_origin_shifts[i_panel][0]
for i_panel in range(len(detector))
]
)
ymax = max(
[
detector[i_panel].get_image_size_mm()[1] + panel_origin_shifts[i_panel][1]
for i_panel in range(len(detector))
]
)
try:
beam_centre = hierarchy.get_beam_centre(imageset.get_beam().get_s0())
except Exception:
beam_centre = detector[0].get_beam_centre(imageset.get_beam().get_s0())
plt.scatter([beam_centre[0]], [beam_centre[1]], marker="+", c="blue", s=100)
plt.xlim(0, xmax)
plt.ylim(0, ymax)
plt.gca().invert_yaxis()
plt.title("Centroid x,y-coordinates")
plt.xlabel("x-coordinate (mm)")
plt.ylabel("y-coordinate (mm)")
if params.output.file_name is not None:
plt.savefig(
params.output.file_name,
dpi=params.output.dpi,
bbox_inches="tight",
)
if params.output.show_plot:
plt.show()
def run(args=None):
from dials.util.options import OptionParser, reflections_and_experiments_from_files
from dials.util.version import dials_version
usage = "dials.export models.expt reflections.refl [options]"
# Create the option parser
parser = OptionParser(
usage=usage,
read_experiments=True,
read_reflections=True,
check_format=False,
phil=phil_scope,
epilog=help_message,
)
# Get the parameters
params, options = parser.parse_args(args, show_diff_phil=False)
# Configure the logging
log.config(logfile=params.output.log)
# Print the version number
logger.info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
if not params.input.experiments and not params.input.reflections:
parser.print_help()
sys.exit()
# Get the experiments and reflections
reflections, experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments
)
# do auto interpreting of intensity choice:
# note that this may still fail certain checks further down the processing,
# but these are the defaults to try
if params.intensity in ([None], [Auto], ["auto"], Auto) and reflections:
if ("intensity.scale.value" in reflections[0]) and (
"intensity.scale.variance" in reflections[0]
):
params.intensity = ["scale"]
logger.info("Data appears to be scaled, setting intensity = scale")
else:
params.intensity = []
if "intensity.sum.value" in reflections[0]:
params.intensity.append("sum")
if "intensity.prf.value" in reflections[0]:
params.intensity.append("profile")
logger.info(
"Data appears to be unscaled, setting intensity = "
+ "+".join(params.intensity)
)
# Choose the exporter
exporter = {
"mtz": export_mtz,
"sadabs": export_sadabs,
"xds_ascii": export_xdsascii,
"nxs": export_nexus,
"mmcif": export_mmcif,
"mosflm": export_mosflm,
"xds": export_xds,
"json": export_json,
}.get(params.format)
if not exporter:
sys.exit(f"Unknown format: {params.format}")
# Export the data
try:
exporter(params, experiments, reflections)
except Exception as e:
logger.error(f"Error: {e}")
sys.exit(1)
def run(args):
import libtbx.load_env
from libtbx.utils import Sorry
usage = "%s [options] experiments.json indexed.pickle" % libtbx.env.dispatcher_name
parser = OptionParser(usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0 and len(reflections) == 0:
parser.print_help()
return
elif len(experiments.crystals()) > 1:
raise Sorry("Only one crystal can be processed at a time")
if params.change_of_basis_op is None:
raise Sorry("Please provide a change_of_basis_op.")
reference_crystal = None
if params.reference is not None:
from dxtbx.serialize import load
reference_experiments = load.experiment_list(params.reference,
check_format=False)
assert len(reference_experiments.crystals()) == 1
reference_crystal = reference_experiments.crystals()[0]
if len(experiments) and params.change_of_basis_op is libtbx.Auto:
if reference_crystal is not None:
from dials.algorithms.indexing.compare_orientation_matrices \
import difference_rotation_matrix_axis_angle
cryst = experiments.crystals()[0]
R, axis, angle, change_of_basis_op = difference_rotation_matrix_axis_angle(
cryst, reference_crystal)
print "Change of basis op: %s" % change_of_basis_op
print "Rotation matrix to transform input crystal to reference::"
print R.mathematica_form(format="%.3f", one_row_per_line=True)
print "Rotation of %.3f degrees" % angle, "about axis (%.3f, %.3f, %.3f)" % axis
elif len(reflections):
assert len(reflections) == 1
# always re-map reflections to reciprocal space
from dials.algorithms.indexing import indexer
refl_copy = flex.reflection_table()
for i, imageset in enumerate(experiments.imagesets()):
if 'imageset_id' in reflections[0]:
sel = (reflections[0]['imageset_id'] == i)
else:
sel = (reflections[0]['id'] == i)
refl = indexer.indexer_base.map_spots_pixel_to_mm_rad(
reflections[0].select(sel), imageset.get_detector(),
imageset.get_scan())
indexer.indexer_base.map_centroids_to_reciprocal_space(
refl, imageset.get_detector(), imageset.get_beam(),
imageset.get_goniometer())
refl_copy.extend(refl)
# index the reflection list using the input experiments list
refl_copy['id'] = flex.int(len(refl_copy), -1)
from dials.algorithms.indexing import index_reflections
index_reflections(refl_copy, experiments, tolerance=0.2)
hkl_expt = refl_copy['miller_index']
hkl_input = reflections[0]['miller_index']
change_of_basis_op = derive_change_of_basis_op(hkl_input, hkl_expt)
# reset experiments list since we don't want to reindex this
experiments = []
else:
change_of_basis_op = sgtbx.change_of_basis_op(
params.change_of_basis_op)
if len(experiments):
experiment = experiments[0]
cryst_orig = copy.deepcopy(experiment.crystal)
cryst_reindexed = cryst_orig.change_basis(change_of_basis_op)
if params.space_group is not None:
a, b, c = cryst_reindexed.get_real_space_vectors()
cryst_reindexed = crystal_model(
a, b, c, space_group=params.space_group.group())
experiment.crystal.update(cryst_reindexed)
print "Old crystal:"
print cryst_orig
print
print "New crystal:"
print cryst_reindexed
print
print "Saving reindexed experimental models to %s" % params.output.experiments
dump.experiment_list(experiments, params.output.experiments)
if len(reflections):
assert (len(reflections) == 1)
reflections = reflections[0]
miller_indices = reflections['miller_index']
if params.hkl_offset is not None:
h, k, l = miller_indices.as_vec3_double().parts()
h += params.hkl_offset[0]
k += params.hkl_offset[1]
l += params.hkl_offset[2]
miller_indices = flex.miller_index(h.iround(), k.iround(),
l.iround())
non_integral_indices = change_of_basis_op.apply_results_in_non_integral_indices(
miller_indices)
if non_integral_indices.size() > 0:
print "Removing %i/%i reflections (change of basis results in non-integral indices)" % (
non_integral_indices.size(), miller_indices.size())
sel = flex.bool(miller_indices.size(), True)
sel.set_selected(non_integral_indices, False)
miller_indices_reindexed = change_of_basis_op.apply(
miller_indices.select(sel))
reflections['miller_index'].set_selected(sel, miller_indices_reindexed)
reflections['miller_index'].set_selected(~sel, (0, 0, 0))
print "Saving reindexed reflections to %s" % params.output.reflections
easy_pickle.dump(params.output.reflections, reflections)
class Script(object):
''' The integration program. '''
def __init__(self):
'''Initialise the script.'''
from dials.util.options import OptionParser
import libtbx.load_env
# The script usage
usage = "usage: %s [options] experiment.json" % libtbx.env.dispatcher_name
# Create the parser
self.parser = OptionParser(usage=usage,
phil=phil_scope,
epilog=help_message,
read_experiments=True,
read_reflections=True)
def run(self):
''' Perform the integration. '''
from dials.util.command_line import heading
from dials.util.options import flatten_reflections, flatten_experiments
from dials.util import log
from time import time
from libtbx.utils import Sorry
# Check the number of arguments is correct
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
reference = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reference) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reference) == 0:
reference = None
elif len(reference) != 1:
raise Sorry('more than 1 reflection file was given')
else:
reference = reference[0]
if len(experiments) == 0:
raise Sorry('no experiment list was specified')
# Save phil parameters
if params.output.phil is not None:
with open(params.output.phil, "w") as outfile:
outfile.write(self.parser.diff_phil.as_str())
# Configure logging
log.config(params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
# Print if we're using a mask
for i, exp in enumerate(experiments):
mask = exp.imageset.external_lookup.mask
if mask.filename is not None:
if mask.data:
logger.info('Using external mask: %s' % mask.filename)
logger.info(' Mask has %d pixels masked' %
mask.data.count(False))
# Print the experimental models
for i, exp in enumerate(experiments):
logger.debug("Models for experiment %d" % i)
logger.debug("")
logger.debug(str(exp.beam))
logger.debug(str(exp.detector))
if exp.goniometer:
logger.debug(str(exp.goniometer))
if exp.scan:
logger.debug(str(exp.scan))
logger.debug(str(exp.crystal))
logger.info("=" * 80)
logger.info("")
logger.info(heading("Initialising"))
logger.info("")
# Load the data
reference, rubbish = self.process_reference(reference)
logger.info("")
# Initialise the integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Modify experiment list if scan range is set.
experiments, reference = self.split_for_scan_range(
experiments, reference, params.scan_range)
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info(heading("Predicting reflections"))
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static)
# Match reference with predicted
if reference:
matched, reference, unmatched = predicted.match_with_reference(
reference)
assert (len(matched) == len(predicted))
assert (matched.count(True) <= len(reference))
if matched.count(True) == 0:
raise Sorry('''
Invalid input for reference reflections.
Zero reference spots were matched to predictions
''')
elif len(unmatched) != 0:
logger.info('')
logger.info('*' * 80)
logger.info(
'Warning: %d reference spots were not matched to predictions'
% (len(unmatched)))
logger.info('*' * 80)
logger.info('')
rubbish.extend(unmatched)
if len(experiments) > 1:
# filter out any experiments without matched reference reflections
# f_: filtered
from dxtbx.model.experiment.experiment_list import ExperimentList
f_reference = flex.reflection_table()
f_predicted = flex.reflection_table()
f_rubbish = flex.reflection_table()
f_experiments = ExperimentList()
good_expt_count = 0
def refl_extend(src, dest, eid):
tmp = src.select(src['id'] == eid)
tmp['id'] = flex.int(len(tmp), good_expt_count)
dest.extend(tmp)
for expt_id, experiment in enumerate(experiments):
if len(reference.select(reference['id'] == expt_id)) != 0:
refl_extend(reference, f_reference, expt_id)
refl_extend(predicted, f_predicted, expt_id)
refl_extend(rubbish, f_rubbish, expt_id)
f_experiments.append(experiment)
good_expt_count += 1
else:
logger.info(
"Removing experiment %d: no reference reflections matched to predictions"
% expt_id)
reference = f_reference
predicted = f_predicted
experiments = f_experiments
rubbish = f_rubbish
# Select a random sample of the predicted reflections
if not params.sampling.integrate_all_reflections:
predicted = self.sample_predictions(experiments, predicted, params)
# Compute the profile model
if (params.create_profile_model and reference is not None
and "shoebox" in reference):
experiments = ProfileModelFactory.create(params, experiments,
reference)
else:
for expr in experiments:
if expr.profile is None:
raise Sorry('No profile information in experiment list')
expr.profile.params = params.profile
del reference
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
logger.info("")
integrator = IntegratorFactory.create(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
# Append rubbish data onto the end
if rubbish is not None and params.output.include_bad_reference:
mask = flex.bool(len(rubbish), True)
rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
rubbish.set_flags(mask, rubbish.flags.bad_reference)
reflections.extend(rubbish)
# Save the reflections
self.save_reflections(reflections, params.output.reflections)
self.save_experiments(experiments, params.output.experiments)
# Write a report if requested
if params.output.report is not None:
integrator.report().as_file(params.output.report)
# Print the total time taken
logger.info("\nTotal time taken: %f" % (time() - start_time))
def process_reference(self, reference):
''' Load the reference spots. '''
from dials.array_family import flex
from time import time
from libtbx.utils import Sorry
if reference is None:
return None, None
st = time()
assert ("miller_index" in reference)
assert ("id" in reference)
logger.info('Processing reference reflections')
logger.info(' read %d strong spots' % len(reference))
mask = reference.get_flags(reference.flags.indexed)
rubbish = reference.select(mask == False)
if mask.count(False) > 0:
reference.del_selected(mask == False)
logger.info(' removing %d unindexed reflections' %
mask.count(True))
if len(reference) == 0:
raise Sorry('''
Invalid input for reference reflections.
Expected > %d indexed spots, got %d
''' % (0, len(reference)))
mask = reference.get_flags(reference.flags.centroid_outlier)
if mask.count(True) > 0:
rubbish.extend(reference.select(mask))
reference.del_selected(mask)
logger.info(
' removing %d reflections marked as centroid outliers' %
mask.count(True))
mask = reference['miller_index'] == (0, 0, 0)
if mask.count(True) > 0:
rubbish.extend(reference.select(mask))
reference.del_selected(mask)
logger.info(' removing %d reflections with hkl (0,0,0)' %
mask.count(True))
mask = reference['id'] < 0
if mask.count(True) > 0:
raise Sorry('''
Invalid input for reference reflections.
%d reference spots have an invalid experiment id
''' % mask.count(True))
logger.info(' using %d indexed reflections' % len(reference))
logger.info(' found %d junk reflections' % len(rubbish))
logger.info(' time taken: %g' % (time() - st))
return reference, rubbish
def save_reflections(self, reflections, filename):
''' Save the reflections to file. '''
from time import time
st = time()
logger.info('Saving %d reflections to %s' %
(len(reflections), filename))
reflections.as_pickle(filename)
logger.info(' time taken: %g' % (time() - st))
def save_experiments(self, experiments, filename):
''' Save the profile model parameters. '''
from time import time
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
st = time()
logger.info('Saving the experiments to %s' % filename)
dump = ExperimentListDumper(experiments)
with open(filename, "w") as outfile:
outfile.write(dump.as_json())
logger.info(' time taken: %g' % (time() - st))
def sample_predictions(self, experiments, predicted, params):
''' Select a random sample of the predicted reflections to integrate. '''
from dials.array_family import flex
nref_per_degree = params.sampling.reflections_per_degree
min_sample_size = params.sampling.minimum_sample_size
max_sample_size = params.sampling.maximum_sample_size
# this code is very similar to David's code in algorithms/refinement/reflection_manager.py!
# constants
from math import pi
RAD2DEG = 180. / pi
DEG2RAD = pi / 180.
working_isel = flex.size_t()
for iexp, exp in enumerate(experiments):
sel = predicted['id'] == iexp
isel = sel.iselection()
#refs = self._reflections.select(sel)
nrefs = sample_size = len(isel)
# set sample size according to nref_per_degree (per experiment)
if exp.scan and nref_per_degree:
sweep_range_rad = exp.scan.get_oscillation_range(deg=False)
width = abs(sweep_range_rad[1] - sweep_range_rad[0]) * RAD2DEG
sample_size = int(nref_per_degree * width)
else:
sweep_range_rad = None
# adjust sample size if below the chosen limit
sample_size = max(sample_size, min_sample_size)
# set maximum sample size if requested
if max_sample_size:
sample_size = min(sample_size, max_sample_size)
# determine subset and collect indices
if sample_size < nrefs:
isel = isel.select(flex.random_selection(nrefs, sample_size))
working_isel.extend(isel)
# create subset
return predicted.select(working_isel)
def split_for_scan_range(self, experiments, reference, scan_range):
''' Update experiments when scan range is set. '''
from dxtbx.model.experiment.experiment_list import ExperimentList
from dxtbx.model.experiment.experiment_list import Experiment
from dials.array_family import flex
# Only do anything is the scan range is set
if scan_range is not None and len(scan_range) > 0:
# Ensure that all experiments have the same imageset and scan
iset = [e.imageset for e in experiments]
scan = [e.scan for e in experiments]
assert (all(x == iset[0] for x in iset))
assert (all(x == scan[0] for x in scan))
# Get the imageset and scan
iset = experiments[0].imageset
scan = experiments[0].scan
# Get the array range
if scan is not None:
frame10, frame11 = scan.get_array_range()
assert (scan.get_num_images() == len(iset))
else:
frame10, frame11 = (0, len(iset))
# Create the new lists
new_experiments = ExperimentList()
new_reference_all = reference.split_by_experiment_id()
new_reference = flex.reflection_table()
for i in range(len(new_reference_all) - len(experiments)):
new_reference_all.append(flex.reflection_table())
assert (len(new_reference_all) == len(experiments))
# Loop through all the scan ranges and create a new experiment list with
# the requested scan ranges.
for frame00, frame01 in scan_range:
assert (frame01 > frame00)
assert (frame00 >= frame10)
assert (frame01 <= frame11)
index0 = frame00 - frame10
index1 = index0 + (frame01 - frame00)
assert (index0 < index1)
assert (index0 >= 0)
assert (index1 <= len(iset))
new_iset = iset[index0:index1]
if scan is None:
new_scan = None
else:
new_scan = scan[index0:index1]
for i, e1 in enumerate(experiments):
e2 = Experiment()
e2.beam = e1.beam
e2.detector = e1.detector
e2.goniometer = e1.goniometer
e2.crystal = e1.crystal
e2.imageset = new_iset
e2.scan = new_scan
new_reference_all[i]['id'] = flex.int(
len(new_reference_all[i]), len(new_experiments))
new_reference.extend(new_reference_all[i])
new_experiments.append(e2)
experiments = new_experiments
reference = new_reference
# Print some information
logger.info(
'Modified experiment list to integrate over requested scan range'
)
for frame00, frame01 in scan_range:
logger.info(' scan_range = %d -> %d' % (frame00, frame01))
logger.info('')
# Return the experiments
return experiments, reference
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
from dials.util.options import flatten_datablocks
from dials.util.options import flatten_reflections
import libtbx.load_env
usage = '%s [options] experiments.json integrated.pickle' % (
libtbx.env.dispatcher_name)
parser = OptionParser(usage=usage,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(experiments) == 0 or len(reflections) == 0:
parser.print_help()
exit()
experiment = experiments[0]
reflections = reflections[0]
from dials.algorithms.profile_model.gaussian_rs import CoordinateSystem
sel = reflections.get_flags(reflections.flags.integrated)
reflections = reflections.select(sel)
goniometer = experiment.goniometer
beam = experiment.beam
m2 = goniometer.get_rotation_axis()
s0 = beam.get_s0()
from dials.array_family import flex
phi1 = flex.double()
phi2 = flex.double()
phi_range = reflections.compute_phi_range(
goniometer.get_rotation_axis(), beam.get_s0(),
experiment.profile.sigma_m(deg=False), experiment.profile.n_sigma())
phi1, phi2 = phi_range.parts()
scan = experiment.scan
exposure_time = scan.get_exposure_times()[0]
assert scan.get_exposure_times().all_eq(exposure_time)
phi_start, phi_width = scan.get_oscillation(deg=False)
phi_range_dead = phi_width * (params.dead_time / 1000) / exposure_time
sel_good = flex.bool(len(reflections), True)
start, end = scan.get_array_range()
for i in range(start, end):
phi_dead_start = phi_start + (i + 1) * phi_width - phi_range_dead
phi_dead_end = phi_dead_start + phi_range_dead
left = phi1.deep_copy()
left.set_selected(left < phi_dead_start, phi_dead_start)
right = phi2.deep_copy()
right.set_selected(right > phi_dead_end, phi_dead_end)
overlap = (right - left) / (phi2 - phi1)
sel = overlap > params.reject_fraction
sel_good.set_selected(sel, False)
print 'Rejecting %i reflections from image %i' % (sel.count(True), i)
print 'Keeping %i reflections (rejected %i)' % (sel_good.count(True),
sel_good.count(False))
from libtbx import easy_pickle
easy_pickle.dump(params.output.reflections, reflections.select(sel_good))