def __call__(self, rlist):
''' Analyse the strong spots. '''
from dials.util.command_line import Command
# Check we have the required fields
print "Analysing strong spots"
if not ensure_required(rlist, self.required):
return
# Remove I_sigma <= 0
selection = rlist['intensity.sum.variance'] <= 0
if selection.count(True) > 0:
rlist.del_selected(selection)
print ' Removing %d reflections with variance <= 0' % \
selection.count(True)
if 'flags' in rlist:
# Select only strong reflections
Command.start(" Selecting only strong reflections")
mask = rlist.get_flags(rlist.flags.strong)
if mask.count(True) > 0:
threshold = 10
rlist = rlist.select(mask)
Command.end(" Selected %d strong reflections" % len(rlist))
# Look at distribution of spot counts
self.spot_count_per_image(rlist)
self.spot_count_per_panel(rlist)
def run(self, args=None):
"""Run the script."""
# Parse the command line arguments
params, options = self.parser.parse_args(args, show_diff_phil=True)
if len(params.input.reflections) == 0:
self.parser.print_help()
return
if len(params.input.reflections) <= 1:
sys.exit("more than 1 reflection table must be specified")
tables = [p.data for p in params.input.reflections]
# Get the number of rows and columns
nrows = [t.nrows() for t in tables]
ncols = [t.ncols() for t in tables]
# Merge the reflection lists
if params.method == "update":
assert all(n == nrows[0] for n in nrows[1:])
table = tables[0]
for t in tables[1:]:
table.update(t)
elif params.method == "extend":
assert all(n == ncols[0] for n in ncols[1:])
table = tables[0]
for t in tables[1:]:
table.extend(t)
else:
raise RuntimeError(f"unknown method, {params.method}")
# Write the reflections to the file
Command.start(f"Writing {len(table)} reflections to {params.output}")
table.as_file(params.output)
Command.end(f"Wrote {len(table)} reflections to {params.output}")
def __call__(self, rlist):
""" Analyse the strong spots. """
# Check we have the required fields
print("Analysing strong spots")
if not ensure_required(rlist, self.required):
return
# Remove I_sigma <= 0
selection = rlist["intensity.sum.variance"] <= 0
if selection.count(True) > 0:
rlist.del_selected(selection)
print(" Removing %d reflections with variance <= 0" %
selection.count(True))
if "flags" in rlist:
# Select only strong reflections
Command.start(" Selecting only strong reflections")
mask = rlist.get_flags(rlist.flags.strong)
if mask.count(True) > 0:
rlist = rlist.select(mask)
Command.end(" Selected %d strong reflections" % len(rlist))
# Look at distribution of spot counts
self.spot_count_per_image(rlist)
self.spot_count_per_panel(rlist)
def __call__(self, rlist):
""" Analyse the reference profiles. """
# Check we have the required fields
print("Analysing reference profiles")
if not ensure_required(rlist, self.required):
return
# Select only integrated reflections
Command.start(" Selecting only integated reflections")
mask = rlist.get_flags(rlist.flags.integrated)
if mask.count(True) == 0:
return
rlist = rlist.select(mask)
Command.end(" Selected %d integrated reflections" % len(rlist))
# Analyse distribution of reference spots
print(" Analysing reference profile distribution vs x/y")
self.reference_xy(rlist)
print(" Analysing reference profile distribution vs z")
self.reference_z(rlist)
# Look at correlations between profiles
def ideal_correlations(filename, rlist):
""" Call for reference spots and all reflections. """
print(" Analysing reflection profile correlations")
self.ideal_reflection_corr_hist(rlist, filename)
print(" Analysing reflection profile correlations vs x/y")
self.ideal_reflection_corr_vs_xy(rlist, filename)
print(" Analysing reflection profile correlations vs z")
self.ideal_reflection_corr_vs_z(rlist, filename)
print(" Analysing reflection profile correlations vs I/Sigma")
self.ideal_reflection_corr_vs_ios(rlist, filename)
# Look at correlations between profiles
def correlations(filename, rlist):
""" Call for reference spots and all reflections. """
print(" Analysing reflection profile correlations")
self.reflection_corr_hist(rlist, filename)
print(" Analysing reflection profile correlations vs x/y")
self.reflection_corr_vs_xy(rlist, filename)
print(" Analysing reflection profile correlations vs z")
self.reflection_corr_vs_z(rlist, filename)
print(" Analysing reflection profile correlations vs I/Sigma")
self.reflection_corr_vs_ios(rlist, filename)
mask = rlist.get_flags(rlist.flags.reference_spot)
correlations("reference", rlist.select(mask))
correlations("reflection", rlist)
ideal_correlations("reference", rlist.select(mask))
ideal_correlations("reflection", rlist)
def run(self):
'''Execute the script.'''
from dials.util.command_line import Command
from dials.array_family import flex
from dials.util.options import flatten_experiments
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
# Check the number of experiments
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0:
self.parser.print_help()
return
predicted_all = flex.reflection_table()
for i_expt, expt in enumerate(experiments):
if params.buffer_size > 0:
# Hack to make the predicter predict reflections outside of the range
# of the scan
scan = expt.scan
image_range = scan.get_image_range()
oscillation = scan.get_oscillation()
scan.set_image_range((image_range[0]-params.buffer_size,
image_range[1]+params.buffer_size))
scan.set_oscillation((oscillation[0]-params.buffer_size*oscillation[1],
oscillation[1]))
# Populate the reflection table with predictions
predicted = flex.reflection_table.from_predictions(
expt,
force_static=params.force_static,
dmin=params.d_min)
predicted['id'] = flex.int(len(predicted), i_expt)
predicted_all.extend(predicted)
# if we are not ignoring shadows, look for reflections in the masked
# region, see https://github.com/dials/dials/issues/349
if not params.ignore_shadows:
from dials.algorithms.shadowing.filter import filter_shadowed_reflections
shadowed = filter_shadowed_reflections(experiments, predicted_all,
experiment_goniometer=True)
predicted_all = predicted_all.select(~shadowed)
try:
predicted_all.compute_bbox(experiments)
except Exception:
pass
# Save the reflections to file
Command.start('Saving {0} reflections to {1}'.format(
len(predicted_all), params.output))
predicted_all.as_pickle(params.output)
Command.end('Saved {0} reflections to {1}'.format(
len(predicted_all), params.output))
def compute_background(self, reflections):
''' Compute the backgrond. '''
from dials.util.command_line import Command
# Do the background subtraction
Command.start('Calculating reflection background')
mask = self._subtractor(reflections['shoebox'])
reflections.del_selected(mask != True)
Command.end('Calculated {0} background values'.format(len(reflections)))
def compute_background(self, reflections):
"""Compute the background."""
from dials.util.command_line import Command
# Do the background subtraction
Command.start("Calculating reflection background")
mask = self._subtractor(reflections["shoebox"])
reflections.del_selected(mask != True)
Command.end("Calculated {0} background values".format(
len(reflections)))
def run(self, args=None):
"""Execute the script."""
# Parse the command line
params, options = self.parser.parse_args(args, show_diff_phil=True)
# Check the number of experiments
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0:
self.parser.print_help()
return
predicted_all = flex.reflection_table()
for i_expt, expt in enumerate(experiments):
if params.buffer_size > 0:
# Hack to make the predicter predict reflections outside of the range
# of the scan
scan = expt.scan
image_range = scan.get_image_range()
oscillation = scan.get_oscillation()
scan.set_image_range((
image_range[0] - params.buffer_size,
image_range[1] + params.buffer_size,
))
scan.set_oscillation((
oscillation[0] - params.buffer_size * oscillation[1],
oscillation[1],
))
# Populate the reflection table with predictions
predicted = flex.reflection_table.from_predictions(
expt, force_static=params.force_static, dmin=params.d_min)
predicted["id"] = flex.int(len(predicted), i_expt)
predicted_all.extend(predicted)
# if we are not ignoring shadows, look for reflections in the masked
# region, see https://github.com/dials/dials/issues/349
if not params.ignore_shadows:
shadowed = filter_shadowed_reflections(experiments,
predicted_all,
experiment_goniometer=True)
predicted_all = predicted_all.select(~shadowed)
try:
predicted_all.compute_bbox(experiments)
except Exception:
pass
# Save the reflections to file
Command.start(
f"Saving {len(predicted_all)} reflections to {params.output}")
predicted_all.as_file(params.output)
Command.end(
f"Saved {len(predicted_all)} reflections to {params.output}")
def __call__(self, rlist):
''' Analyse the relfection background. '''
from dials.util.command_line import Command
# Check we have the required fields
print "Analysing reflection backgrounds"
if not ensure_required(rlist, self.required):
return
selection = rlist['intensity.sum.variance'] <= 0
if selection.count(True) > 0:
rlist.del_selected(selection)
print ' Removing %d reflections with variance <= 0' % \
selection.count(True)
selection = rlist['background.mse'] < 0
if selection.count(True) > 0:
rlist.del_selected(selection)
print ' Removing %d reflections with negative background model RMSD' % \
selection.count(True)
selection = rlist['background.mean'] <= 0
if selection.count(True) > 0:
rlist.del_selected(selection)
print ' Removing %d reflections with mean background <= 0' % \
selection.count(True)
# Select only integrated reflections
Command.start(" Selecting only integated reflections")
mask = rlist.get_flags(rlist.flags.integrated)
if mask.count(True) == 0:
return
rlist = rlist.select(mask)
Command.end(" Selected %d integrated reflections" % len(rlist))
# Look at distribution of I/Sigma
print " Analysing distribution of background mean"
self.mean_hist(rlist)
print " Analysing distribution of background mean vs XY"
self.mean_vs_xy(rlist)
print " Analysing distribution of background mean vs z"
self.mean_vs_z(rlist)
print " Analysing distribution of background mean vs I/Sigma"
self.mean_vs_ios(rlist)
print " Analysing distribution of background CVRMSD"
self.rmsd_hist(rlist)
print " Analysing distribution of background CVRMSD vs XY"
self.rmsd_vs_xy(rlist)
print " Analysing distribution of background CVRMSD vs z"
self.rmsd_vs_z(rlist)
print " Analysing distribution of background CVRMSD vs I/Sigma"
self.rmsd_vs_ios(rlist)
def __call__(self, rlist):
""" Analyse the relfection background. """
# Check we have the required fields
print("Analysing reflection backgrounds")
if not ensure_required(rlist, self.required):
return
selection = rlist["intensity.sum.variance"] <= 0
if selection.count(True) > 0:
rlist.del_selected(selection)
print(" Removing %d reflections with variance <= 0" %
selection.count(True))
selection = rlist["background.mse"] < 0
if selection.count(True) > 0:
rlist.del_selected(selection)
print(
" Removing %d reflections with negative background model RMSD"
% selection.count(True))
selection = rlist["background.mean"] <= 0
if selection.count(True) > 0:
rlist.del_selected(selection)
print(" Removing %d reflections with mean background <= 0" %
selection.count(True))
# Select only integrated reflections
Command.start(" Selecting only integated reflections")
mask = rlist.get_flags(rlist.flags.integrated)
if mask.count(True) == 0:
return
rlist = rlist.select(mask)
Command.end(" Selected %d integrated reflections" % len(rlist))
# Look at distribution of I/Sigma
print(" Analysing distribution of background mean")
self.mean_hist(rlist)
print(" Analysing distribution of background mean vs XY")
self.mean_vs_xy(rlist)
print(" Analysing distribution of background mean vs z")
self.mean_vs_z(rlist)
print(" Analysing distribution of background mean vs I/Sigma")
self.mean_vs_ios(rlist)
print(" Analysing distribution of background CVRMSD")
self.rmsd_hist(rlist)
print(" Analysing distribution of background CVRMSD vs XY")
self.rmsd_vs_xy(rlist)
print(" Analysing distribution of background CVRMSD vs z")
self.rmsd_vs_z(rlist)
print(" Analysing distribution of background CVRMSD vs I/Sigma")
self.rmsd_vs_ios(rlist)
def _filter_reflections(self, params, experiments, reflections):
''' Filter the reflections to integrate. '''
from dials.util.command_line import Command
from dials.algorithms import filtering
from dials.array_family import flex
# Set all reflections which overlap bad pixels to zero
Command.start('Filtering reflections by detector mask')
if experiments[0].scan == None:
array_range = 1
else:
array_range = experiments[0].scan.get_array_range()
mask = filtering.by_detector_mask(
reflections['bbox'],
experiments[0].imageset.get_raw_data(0)[0] >= 0, array_range)
reflections.del_selected(not mask)
Command.end('Filtered %d reflections by detector mask' %
len(reflections))
# Filter the reflections by zeta
min_zeta = params.integration.filter.by_zeta
if min_zeta > 0:
Command.start('Filtering reflections by zeta >= %f' % min_zeta)
zeta = reflections.compute_zeta(experiments[0])
reflections.del_selected(flex.abs(zeta) < min_zeta)
n = len(reflections)
Command.end('Filtered %d reflections by zeta >= %f' %
(n, min_zeta))
return reflections
def _filter_reflections(self, params, experiments, reflections):
''' Filter the reflections to integrate. '''
from dials.util.command_line import Command
from dials.algorithms import filtering
from dials.array_family import flex
# Set all reflections which overlap bad pixels to zero
Command.start('Filtering reflections by detector mask')
if experiments[0].scan == None:
array_range = 1
else:
array_range = experiments[0].scan.get_array_range()
mask = filtering.by_detector_mask(
reflections['bbox'],
experiments[0].imageset.get_raw_data(0)[0] >= 0,
array_range)
reflections.del_selected(not mask)
Command.end('Filtered %d reflections by detector mask' % len(reflections))
# Filter the reflections by zeta
min_zeta = params.integration.filter.by_zeta
if min_zeta > 0:
Command.start('Filtering reflections by zeta >= %f' % min_zeta)
zeta = reflections.compute_zeta(experiments[0])
reflections.del_selected(flex.abs(zeta) < min_zeta)
n = len(reflections)
Command.end('Filtered %d reflections by zeta >= %f' % (n, min_zeta))
return reflections
def _match_with_reference(self, predicted, reference):
''' Match predictions with reference spots. '''
from dials.algorithms.spot_finding.spot_matcher import SpotMatcher
from dials.util.command_line import Command
Command.start("Matching reference spots with predicted reflections")
match = SpotMatcher(max_separation=1)
rind, pind = match(reference, predicted)
h1 = predicted.select(pind)['miller_index']
h2 = reference.select(rind)['miller_index']
mask = (h1 == h2)
predicted.set_flags(pind.select(mask), predicted.flags.reference_spot)
Command.end("Matched %d reference spots with predicted reflections" %
mask.count(True))
return predicted
def __call__(self):
"""Run the script."""
import cPickle as pickle
from dials.model.data import ReflectionList # import dependency
from dials.util.command_line import Command
# Read the pickle file
Command.start('Reading reflection file.')
with open(self.reflections_filename, 'rb') as f:
self.reflections = pickle.load(f)
Command.end('Read {0} spots from reflection file.'.format(
len(self.reflections)))
self.view()
def process_reference(reference):
''' Load the reference spots. '''
from dials.util.command_line import Command
from dials.array_family import flex
if reference is None:
return None
assert("miller_index" in reference)
Command.start('Removing reference spots with invalid coordinates')
mask = flex.bool([x == (0, 0, 0) for x in reference['xyzcal.mm']])
reference.del_selected(mask)
mask = flex.bool([h == (0, 0, 0) for h in reference['miller_index']])
reference.del_selected(mask)
Command.end('Removed reference spots with invalid coordinates, %d remaining' %
len(reference))
return reference
def _match_with_reference(self, predicted, reference):
''' Match predictions with reference spots. '''
from dials.algorithms.spot_finding.spot_matcher import SpotMatcher
from dials.util.command_line import Command
Command.start("Matching reference spots with predicted reflections")
match = SpotMatcher(max_separation=1)
rind, pind = match(reference, predicted)
h1 = predicted.select(pind)['miller_index']
h2 = reference.select(rind)['miller_index']
mask = (h1 == h2)
predicted.set_flags(pind.select(mask), predicted.flags.reference_spot)
Command.end("Matched %d reference spots with predicted reflections" %
mask.count(True))
return predicted
def run(self):
'''Execute the script.'''
from dials.util.command_line import Command
from dials.array_family import flex
from dials.util.options import flatten_experiments
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
# Check the number of experiments
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0:
self.parser.print_help()
return
predicted_all = flex.reflection_table()
for i_expt, expt in enumerate(experiments):
if params.buffer_size > 0:
# Hack to make the predicter predict reflections outside of the range
# of the scan
scan = expt.scan
image_range = scan.get_image_range()
oscillation = scan.get_oscillation()
scan.set_image_range((image_range[0]-params.buffer_size,
image_range[1]+params.buffer_size))
scan.set_oscillation((oscillation[0]-params.buffer_size*oscillation[1],
oscillation[1]))
# Populate the reflection table with predictions
predicted = flex.reflection_table.from_predictions(
expt,
force_static=params.force_static,
dmin=params.d_min)
predicted['id'] = flex.int(len(predicted), i_expt)
predicted_all.extend(predicted)
try:
predicted_all.compute_bbox(experiments)
except Exception:
pass
# Save the reflections to file
Command.start('Saving {0} reflections to {1}'.format(
len(predicted_all), params.output))
predicted_all.as_pickle(params.output)
Command.end('Saved {0} reflections to {1}'.format(
len(predicted_all), params.output))
def process_reference(reference):
"""Load the reference spots."""
from dials.util.command_line import Command
from dials.array_family import flex
if reference is None:
return None
assert "miller_index" in reference
Command.start("Removing reference spots with invalid coordinates")
mask = flex.bool([x == (0, 0, 0) for x in reference["xyzcal.mm"]])
reference.del_selected(mask)
mask = flex.bool([h == (0, 0, 0) for h in reference["miller_index"]])
reference.del_selected(mask)
Command.end(
"Removed reference spots with invalid coordinates, %d remaining" %
len(reference))
return reference
def __call__(self, rlist):
''' Analyse the reflection centroids. '''
from dials.util.command_line import Command
# FIXME Do the same and a comparison for intensity.prf
# Check we have the required fields
print "Analysing reflection intensities"
if not ensure_required(rlist, self.required):
return
selection = rlist['intensity.sum.variance'] <= 0
if selection.count(True) > 0:
rlist.del_selected(selection)
print ' Removing %d reflections with variance <= 0' % \
selection.count(True)
selection = rlist['intensity.sum.value'] <= 0
if selection.count(True) > 0:
rlist.del_selected(selection)
print ' Removing %d reflections with intensity <= 0' % \
selection.count(True)
# Select only integrated reflections
Command.start(" Selecting only integated reflections")
mask = rlist.get_flags(rlist.flags.integrated)
if mask.count(True) == 0:
return
rlist = rlist.select(mask)
Command.end(" Selected %d integrated reflections" % len(rlist))
# Look at distribution of I/Sigma
print " Analysing distribution of I/Sigma"
self.i_over_s_hist(rlist)
print " Analysing distribution of I/Sigma vs xy"
self.i_over_s_vs_xy(rlist, "sum")
print " Analysing distribution of I/Sigma vs xy"
self.i_over_s_vs_xy(rlist, "prf")
print " Analysing distribution of I/Sigma vs z"
self.i_over_s_vs_z(rlist)
print " Analysing number of background pixels used"
self.num_background_hist(rlist)
print " Analysing number of foreground pixels used"
self.num_foreground_hist(rlist)
def __call__(self, rlist):
""" Analyse the reflection centroids. """
# FIXME Do the same and a comparison for intensity.prf
# Check we have the required fields
print("Analysing reflection intensities")
if not ensure_required(rlist, self.required):
return
selection = rlist["intensity.sum.variance"] <= 0
if selection.count(True) > 0:
rlist.del_selected(selection)
print(" Removing %d reflections with variance <= 0" %
selection.count(True))
selection = rlist["intensity.sum.value"] <= 0
if selection.count(True) > 0:
rlist.del_selected(selection)
print(" Removing %d reflections with intensity <= 0" %
selection.count(True))
# Select only integrated reflections
Command.start(" Selecting only integated reflections")
mask = rlist.get_flags(rlist.flags.integrated)
if mask.count(True) == 0:
return
rlist = rlist.select(mask)
Command.end(" Selected %d integrated reflections" % len(rlist))
# Look at distribution of I/Sigma
print(" Analysing distribution of I/Sigma")
self.i_over_s_hist(rlist)
print(" Analysing distribution of I/Sigma vs xy")
self.i_over_s_vs_xy(rlist, "sum")
print(" Analysing distribution of I/Sigma vs xy")
self.i_over_s_vs_xy(rlist, "prf")
print(" Analysing distribution of I/Sigma vs z")
self.i_over_s_vs_z(rlist)
print(" Analysing number of background pixels used")
self.num_background_hist(rlist)
print(" Analysing number of foreground pixels used")
self.num_foreground_hist(rlist)
def run(self):
''' Run the script. '''
from dials.array_family import flex
from dials.util.command_line import Command
from libtbx.utils import Sorry
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
if len(params.input.reflections) == 0:
self.parser.print_help()
return
if len(params.input.reflections) <= 1:
raise Sorry('more than 1 reflection table must be specified')
tables = [p.data for p in params.input.reflections]
# Get the number of rows and columns
nrows = [t.nrows() for t in tables]
ncols = [t.ncols() for t in tables]
# Merge the reflection lists
if params.method == "update":
assert (all(n == nrows[0] for n in nrows[1:]))
table = tables[0]
for t in tables[1:]:
table.update(t)
elif params.method == "extend":
assert (all(n == ncols[0] for n in ncols[1:]))
table = tables[0]
for t in tables[1:]:
table.extend(t)
else:
raise RuntimeError('unknown method, %s' % params.method)
# Write the reflections to the file
Command.start('Writing %d reflections to %s' %
(len(table), params.output))
table.as_pickle(params.output)
Command.end('Wrote %d reflections to %s' % (len(table), params.output))
def run(self):
''' Run the script. '''
from dials.array_family import flex
from dials.util.command_line import Command
from libtbx.utils import Sorry
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
if len(params.input.reflections) == 0:
self.parser.print_help()
return
if len(params.input.reflections) <= 1:
raise Sorry('more than 1 reflection table must be specified')
tables = [p.data for p in params.input.reflections]
# Get the number of rows and columns
nrows = [t.nrows() for t in tables]
ncols = [t.ncols() for t in tables]
# Merge the reflection lists
if params.method == "update":
assert(all(n == nrows[0] for n in nrows[1:]))
table = tables[0]
for t in tables[1:]:
table.update(t)
elif params.method == "extend":
assert(all(n == ncols[0] for n in ncols[1:]))
table = tables[0]
for t in tables[1:]:
table.extend(t)
else:
raise RuntimeError('unknown method, %s' % params.method)
# Write the reflections to the file
Command.start('Writing %d reflections to %s' % (len(table), params.output))
table.as_pickle(params.output)
Command.end('Wrote %d reflections to %s' % (len(table), params.output))
def __call__(self, rlist):
""" Analyse the reflection centroids. """
# Check we have the required fields
print("Analysing reflection centroids")
if not ensure_required(rlist, self.required):
return
# Remove I_sigma <= 0
selection = rlist["intensity.sum.variance"] <= 0
if selection.count(True) > 0:
rlist.del_selected(selection)
print(" Removing %d reflections with variance <= 0" %
selection.count(True))
# Remove partial reflections as their observed centroids won't be accurate
if "partiality" in rlist:
selection = rlist["partiality"] < 0.99
if selection.count(True) > 0 and selection.count(
True) < selection.size():
rlist.del_selected(selection)
print(" Removing %d partial reflections" %
selection.count(True))
# Select only integrated reflections
Command.start(" Selecting only summation-integated reflections")
mask = rlist.get_flags(rlist.flags.integrated_sum)
if mask.count(True) > 0:
threshold = 10
rlist = rlist.select(mask)
Command.end(" Selected %d summation-integrated reflections" %
len(rlist))
else:
# Select only those reflections used in refinement
threshold = 0
mask = rlist.get_flags(rlist.flags.used_in_refinement)
rlist = rlist.select(mask)
Command.end(" Selected %d refined reflections" % len(rlist))
# Look at differences in calculated/observed position
print(" Analysing centroid differences with I/Sigma > %s" % threshold)
self.centroid_diff_hist(rlist, threshold)
print(" Analysing centroid differences in x/y with I/Sigma > %s" %
threshold)
self.centroid_diff_xy(rlist, threshold)
self.centroid_xy_xz_zy_residuals(rlist, threshold)
print(" Analysing centroid differences in z with I/Sigma > %s" %
threshold)
self.centroid_diff_z(rlist, threshold)
print(" Analysing centroid differences vs phi with I/Sigma > %s" %
threshold)
self.centroid_mean_diff_vs_phi(rlist, threshold)
def __call__(self, rlist):
''' Analyse the reflection centroids. '''
from dials.util.command_line import Command
# Check we have the required fields
print "Analysing reflection centroids"
if not ensure_required(rlist, self.required):
return
# Remove I_sigma <= 0
selection = rlist['intensity.sum.variance'] <= 0
if selection.count(True) > 0:
rlist.del_selected(selection)
print ' Removing %d reflections with variance <= 0' % \
selection.count(True)
# Remove partial reflections as their observed centroids won't be accurate
if 'partiality' in rlist:
selection = rlist['partiality'] < 0.99
if selection.count(True) > 0 and selection.count(True) < selection.size():
rlist.del_selected(selection)
print ' Removing %d partial reflections' % \
selection.count(True)
# Select only integrated reflections
Command.start(" Selecting only summation-integated reflections")
mask = rlist.get_flags(rlist.flags.integrated_sum)
if mask.count(True) > 0:
threshold = 10
rlist = rlist.select(mask)
Command.end(" Selected %d summation-integrated reflections" % len(rlist))
else:
# Select only those reflections used in refinement
threshold = 0
mask = rlist.get_flags(rlist.flags.used_in_refinement)
rlist = rlist.select(mask)
Command.end(" Selected %d refined reflections" % len(rlist))
# Look at differences in calculated/observed position
print " Analysing centroid differences with I/Sigma > %s" %threshold
self.centroid_diff_hist(rlist, threshold)
print " Analysing centroid differences in x/y with I/Sigma > %s" %threshold
self.centroid_diff_xy(rlist, threshold)
self.centroid_xy_xz_zy_residuals(rlist, threshold)
print " Analysing centroid differences in z with I/Sigma > %s" %threshold
self.centroid_diff_z(rlist, threshold)
print " Analysing centroid differences vs phi with I/Sigma > %s" %threshold
self.centroid_mean_diff_vs_phi(rlist, threshold)
import libtbx.load_env
usage = ("usage: %s [options] reflections1.refl reflections2.refl" %
libtbx.env.dispatcher_name)
parser = OptionParser(usage)
# Parse the command line arguments
(options, args) = parser.parse_args()
# Ensure there are only two reflection lists
if len(args) != 2:
parser.print_help()
exit(0)
# Read the first batch of reflections
Command.start("Reading reflections from %s" % args[0])
refl1 = flex.reflection_table.from_file(args[0])
mask = flex.bool(xyz == (0, 0, 0) for xyz in refl1["xyzobs.px.value"])
refl1.del_selected(mask)
Command.end("Read %d reflections from %s" % (len(refl1), args[0]))
# Read the second batch of reflections
Command.start("Reading reflections from %s" % args[1])
refl2 = flex.reflection_table.from_file(args[1])
mask = refl2["intensity.sum.value"] <= 0.0
refl2.del_selected(mask)
mask = refl2["intensity.sum.value"]**2 < refl2["intensity.sum.variance"]
refl2.del_selected(mask)
Command.end("Read %d reflections from %s" % (len(refl2), args[1]))
# perform the match
def run(self, args=None):
"""Run the script."""
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.array_family import flex
from dials.util import Sorry, log
from dials.util.command_line import Command
from dials.util.options import reflections_and_experiments_from_files
log.config()
# Parse the command line
params, options = self.parser.parse_args(args, show_diff_phil=True)
reflections, experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments)
if len(reflections) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reflections) != 1:
raise Sorry("exactly 1 reflection table must be specified")
if len(experiments) == 0:
raise Sorry("no experiments were specified")
if ("background.mean"
not in reflections[0]) and params.subtract_background:
raise Sorry(
"for subtract_background need background.mean in reflections")
reflections, _ = self.process_reference(reflections[0], params)
# Check pixels don't belong to neighbours
self.filter_reference_pixels(reflections, experiments)
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
padding=params.prediction.padding,
)
# Match with predicted
matched, reflections, unmatched = predicted.match_with_reference(
reflections)
assert len(matched) == len(predicted)
assert matched.count(True) <= len(reflections)
if matched.count(True) == 0:
raise Sorry("""
Invalid input for reference reflections.
Zero reference spots were matched to predictions
""")
elif len(unmatched) != 0:
logger.info("")
logger.info("*" * 80)
logger.info(
"Warning: %d reference spots were not matched to predictions",
len(unmatched),
)
logger.info("*" * 80)
logger.info("")
# Create the profile model
experiments = ProfileModelFactory.create(params, experiments,
reflections)
for model in experiments:
sigma_b = model.profile.sigma_b(deg=True)
sigma_m = model.profile.sigma_m(deg=True)
if model.profile.is_scan_varying(): # scan varying
mean_sigma_b = sum(sigma_b) / len(sigma_b)
mean_sigma_m = sum(sigma_m) / len(sigma_m)
logger.info("Sigma B: %f", mean_sigma_b)
logger.info("Sigma M: %f", mean_sigma_m)
else:
logger.info("Sigma B: %f", sigma_b)
logger.info("Sigma M: %f", sigma_m)
# Write the parameters
Command.start(f"Writing experiments to {params.output}")
experiments.as_file(params.output)
Command.end(f"Wrote experiments to {params.output}")
# save cbf file
if self.params.dispatch.dump_strong:
self.save_image(cspad_img, self.params, os.path.join(self.params.output.output_dir, "hit-" + s))
# save strong reflections. self.find_spots() would have done this, but we only
# want to save data if it is enough to try and index it
if self.strong_filename_template:
if "%s" in self.strong_filename_template:
strong_filename = self.strong_filename_template%("hit-" + s)
else:
strong_filename = self.strong_filename_template
strong_filename = os.path.join(self.params.output.output_dir, strong_filename)
from dials.util.command_line import Command
Command.start('Saving {0} reflections to {1}'.format(
len(observed), os.path.basename(strong_filename)))
observed.as_pickle(strong_filename)
Command.end('Saved {0} observed to {1}'.format(
len(observed), os.path.basename(strong_filename)))
if not self.params.dispatch.index:
self.debug_write("strong_shot_%d"%len(observed), "done")
self.log_frame(None, None, run.run(), len(observed), timestamp)
return
# index and refine
self.debug_write("index_start")
try:
experiments, indexed = self.index(datablock, observed)
except Exception, e:
import traceback; traceback.print_exc()
def __call__(self, sweep):
"""The main function of the spot finder. Select the pixels from
the sweep and then group the pixels into spots. Return the data
in the form of a reflection list.
Params:
sweep The sweep object
Returns:
The reflection list
"""
from dials.util.command_line import Command
# Set a command indent to 4
Command.indent = 4
print("\nFinding spot in {0} images...".format(len(sweep)))
# Extract the image pixels from the sweep
Command.start("Extracting pixels from sweep")
coords, intensity = self._extract_pixels(sweep)
Command.end("Extracted {0} strong pixels".format(len(coords)))
# Label the pixels and group into spots
Command.start("Labelling connected components")
labels = self._label_pixels(coords, sweep)
Command.end("Found {0} connected components".format(max(labels) + 1))
# Filter spots that are too small
Command.start("Filtering spots by size")
spots = self._filter_spots(labels)
Command.end("Filtered {0} spots by size".format(len(spots)))
# Calculate the bounding box for each spot
Command.start("Calculating bounding boxes")
bbox = self._calculate_bbox(coords, spots, sweep)
Command.end("Calculated {0} bounding boxes".format(len(bbox)))
# Calculate the spot centroids
Command.start("Calculating centroids")
cpos, cvar = self._calculate_centroids(coords, intensity, spots)
Command.end("Calculated {0} centroids".format(len(cpos)))
# Filter the spots by centroid-maxmimum distance
Command.start("Filtering spots by distance")
index = self._filter_maximum_centroid(coords, intensity, spots, cpos)
Command.end("Filtered {0} spots by distance".format(len(index)))
# Create a reflection list and return
return self._create_reflection_list(coords, intensity, spots, bbox,
cpos, cvar, index)
sweep = load.sweep(os.path.join(path, 'sweep.json'))
crystal = load.crystal(os.path.join(path, 'crystal.json'))
extract = ReflectionExtractor(3)
refl = extract(sweep, crystal)
import copy
refl2 = copy.deepcopy(refl)
from dials.algorithms.reflection_basis import transform
from dials.util.command_line import Command
Command.start('Init transform')
trans = transform.Forward(sweep, crystal, 3, 4)
Command.end('Init transform')
Command.start('Transform')
for i in range(1):
trans(refl)
Command.end('Transform')
Command.start('Init Transform')
spec = transform.TransformSpec(sweep, crystal, 3, 4)
Command.end('Init Transform')
Command.start('Transform')
for i in range(1):
transform.forward_batch(spec, refl2)
def __call__(self, params, options):
"""Import the spot.xds file."""
# Read the SPOT.XDS file
Command.start("Reading SPOT.XDS")
handle = spot_xds.reader()
handle.read_file(self._spot_xds)
centroid = handle.centroid
intensity = handle.intensity
try:
miller_index = handle.miller_index
except AttributeError:
miller_index = None
Command.end(f"Read {len(centroid)} spots from SPOT.XDS file.")
# Create the reflection list
Command.start("Creating reflection list")
table = flex.reflection_table()
table["id"] = flex.int(len(centroid), 0)
table["panel"] = flex.size_t(len(centroid), 0)
if miller_index:
table["miller_index"] = flex.miller_index(miller_index)
table["xyzobs.px.value"] = flex.vec3_double(centroid)
table["intensity.sum.value"] = flex.double(intensity)
Command.end("Created reflection list")
# Remove invalid reflections
Command.start("Removing invalid reflections")
if miller_index and params.remove_invalid:
flags = flex.bool([h != (0, 0, 0) for h in table["miller_index"]])
table = table.select(flags)
Command.end(f"Removed invalid reflections, {len(table)} remaining")
# Fill empty standard columns
if params.add_standard_columns:
Command.start("Adding standard columns")
rt = flex.reflection_table.empty_standard(len(table))
rt.update(table)
table = rt
# set variances to unity
table["xyzobs.mm.variance"] = flex.vec3_double(len(table), (1, 1, 1))
table["xyzobs.px.variance"] = flex.vec3_double(len(table), (1, 1, 1))
Command.end("Standard columns added")
# Output the table to pickle file
if params.output.filename is None:
params.output.filename = "spot_xds.refl"
Command.start(f"Saving reflection table to {params.output.filename}")
table.as_file(params.output.filename)
Command.end(f"Saved reflection table to {params.output.filename}")
def __call__(self, params, options):
''' Import the spot.xds file. '''
from iotbx.xds import spot_xds
from dials.util.command_line import Command
from dials.array_family import flex
# Read the SPOT.XDS file
Command.start('Reading SPOT.XDS')
handle = spot_xds.reader()
handle.read_file(self._spot_xds)
centroid = handle.centroid
intensity = handle.intensity
try:
miller_index = handle.miller_index
except AttributeError:
miller_index = None
Command.end('Read {0} spots from SPOT.XDS file.'.format(len(centroid)))
# Create the reflection list
Command.start('Creating reflection list')
table = flex.reflection_table()
table['id'] = flex.int(len(centroid), 0)
table['panel'] = flex.size_t(len(centroid), 0)
if miller_index:
table['miller_index'] = flex.miller_index(miller_index)
table['xyzobs.px.value'] = flex.vec3_double(centroid)
table['intensity.sum.value'] = flex.double(intensity)
Command.end('Created reflection list')
# Remove invalid reflections
Command.start('Removing invalid reflections')
if miller_index and params.remove_invalid:
flags = flex.bool([h != (0, 0, 0) for h in table['miller_index']])
table = table.select(flags)
Command.end('Removed invalid reflections, %d remaining' % len(table))
# Fill empty standard columns
if params.add_standard_columns:
Command.start('Adding standard columns')
rt = flex.reflection_table.empty_standard(len(table))
rt.update(table)
table = rt
# set variances to unity
table['xyzobs.mm.variance'] = flex.vec3_double(
len(table), (1, 1, 1))
table['xyzobs.px.variance'] = flex.vec3_double(
len(table), (1, 1, 1))
Command.end('Standard columns added')
# Output the table to pickle file
if params.output.filename is None:
params.output.filename = 'spot_xds.pickle'
Command.start('Saving reflection table to %s' % params.output.filename)
table.as_pickle(params.output.filename)
Command.end('Saved reflection table to %s' % params.output.filename)
def __call__(self, observed, predicted):
'''Calculate the divegence/mosaicity parameters.
First match the observed reflections to the predicted reflections
by finding the nearest neighbour based on the observed centroid
and predicted bragg maximum. Then calculate the standard deviation
of the beam divergence followed by the standard deviation of the
mosaicity.
The updated list of reflections can be accessed via the
self.reflections member function.
Params:
observed The observed list of reflections
predicted The predicted list of reflections
Returns:
sigma_d, sigma_m
'''
from math import pi
from dials.util.command_line import Command
# Setup the output
Command.indent = 4
print '\nCalculating e.s.d of beam divergence and mosaicity...'
# Map observed to predicted reflections
Command.start('Matching observed and predicted reflections.')
self._data = self._match_observed_w_predicted(observed, predicted)
Command.end('Matched {0} observed reflections.'.format(len(self._data)))
# Calculate the standard deviation of the beam divergence
Command.start('Calculating e.s.d of the beam divergence')
sigma_d = self._calculate_esd_beam_divergence(self._data)
Command.end('Calculated e.s.d of the beam divergence = {0:.3f} deg' \
.format(sigma_d * 180 / pi))
# Calculate the standard deviation of the reflecting range (mosaicity)
Command.start('Calculating the mosaicity')
sigma_m = self._calculate_esd_reflecting_range(self._data)
Command.end('Calculated mosaicity = {0:.3f} deg'\
.format(sigma_m * 180 / pi))
# Return the parameters
return sigma_d, sigma_m
def __call__(self, reflections):
"""Process the reflections.
Params:
reflections The reflections to process
Returns:
The list of integrated reflections
"""
from dlstbx.algorithms.integration.fit_image import ImageSpaceProfileFitting
from dlstbx.algorithms.integration.fit_image import Spec
from dials.algorithms.integration.integrator import job_id
from dials.array_family import flex
from dials.util.command_line import Command
# Get the flags
flags = flex.reflection_table.flags
# Create the algorithm
algorithm = ImageSpaceProfileFitting(self._grid_size)
# Add the specs
for experiment, model in zip(self._experiments, self._profile_model):
algorithm.add(
Spec(
experiment.beam,
experiment.detector,
experiment.goniometer,
experiment.scan,
model.delta_b(deg=False),
model.delta_m(deg=False),
)
)
# Perform the integration
num = reflections.get_flags(flags.dont_integrate).count(False)
Command.start("Integrating %d reflections with profile fitting" % num)
profiles = algorithm.execute(reflections)
# Print the number integrated
num = reflections.get_flags(flags.integrated_prf).count(True)
Command.end("Integrated %d reflections with profile fitting" % num)
# Output the reference profiles
if self._debug:
import cPickle as pickle
filename = "debug_%d.pickle" % job_id()
print("Saving debugging information to %s" % filename)
reference = [profiles.data(i) for i in range(len(profiles))]
rprofiles = []
for r in reflections:
rprofiles.append(
profiles.get(
r["id"], r["panel"], r["s1"], r["xyzcal.mm"][2], r["bbox"]
)
)
output = {
"reflections": reflections,
"experiments": self._experiments,
"profile_model": self._profile_model,
"reference": reference,
"profiles": rprofiles,
}
with open(filename, "wb") as outfile:
pickle.dump(output, outfile, protocol=pickle.HIGHEST_PROTOCOL)
# Return the reflections
return reflections
import libtbx.load_env
usage = "usage: %s [options] reflections1.pickle reflections2.pickle" \
% libtbx.env.dispatcher_name
parser = OptionParser(usage)
# Parse the command line arguments
(options, args) = parser.parse_args()
# Ensure there are only two reflection lists
if len(args) != 2:
parser.print_help()
exit(0)
# Read the first batch of reflections
Command.start('Reading reflections from %s' % args[0])
refl1 = flex.reflection_table.from_pickle(args[0])
mask = flex.bool(xyz == (0, 0, 0) for xyz in refl1['xyzobs.px.value'])
refl1.del_selected(mask)
Command.end('Read %d reflections from %s' % (len(refl1), args[0]))
# Read the second batch of reflections
Command.start('Reading reflections from %s' % args[1])
refl2 = flex.reflection_table.from_pickle(args[1])
mask = refl2['intensity.sum.value'] <= 0.0
refl2.del_selected(mask)
mask = refl2['intensity.sum.value']**2 < refl2['intensity.sum.variance']
refl2.del_selected(mask)
Command.end('Read %d reflections from %s' % (len(refl2), args[1]))
# perform the match
def __call__(self, rlist):
''' Analyse the reference profiles. '''
from dials.util.command_line import Command
# Check we have the required fields
print "Analysing reference profiles"
if not ensure_required(rlist, self.required):
return
# Select only integrated reflections
Command.start(" Selecting only integated reflections")
mask = rlist.get_flags(rlist.flags.integrated)
if mask.count(True) == 0:
return
rlist = rlist.select(mask)
Command.end(" Selected %d integrated reflections" % len(rlist))
# Analyse distribution of reference spots
print " Analysing reference profile distribution vs x/y"
self.reference_xy(rlist)
print " Analysing reference profile distribution vs z"
self.reference_z(rlist)
# Look at correlations between profiles
def ideal_correlations(filename, rlist):
''' Call for reference spots and all reflections. '''
print " Analysing reflection profile correlations"
self.ideal_reflection_corr_hist(rlist, filename)
print " Analysing reflection profile correlations vs x/y"
self.ideal_reflection_corr_vs_xy(rlist, filename)
print " Analysing reflection profile correlations vs z"
self.ideal_reflection_corr_vs_z(rlist, filename)
print " Analysing reflection profile correlations vs I/Sigma"
self.ideal_reflection_corr_vs_ios(rlist, filename)
# Look at correlations between profiles
def correlations(filename, rlist):
''' Call for reference spots and all reflections. '''
print " Analysing reflection profile correlations"
self.reflection_corr_hist(rlist, filename)
print " Analysing reflection profile correlations vs x/y"
self.reflection_corr_vs_xy(rlist, filename)
print " Analysing reflection profile correlations vs z"
self.reflection_corr_vs_z(rlist, filename)
print " Analysing reflection profile correlations vs I/Sigma"
self.reflection_corr_vs_ios(rlist, filename)
mask = rlist.get_flags(rlist.flags.reference_spot)
correlations("reference", rlist.select(mask))
correlations("reflection", rlist)
ideal_correlations("reference", rlist.select(mask))
ideal_correlations("reflection", rlist)
sweep = load.sweep(os.path.join(path, "sweep.json"))
crystal = load.crystal(os.path.join(path, "crystal.json"))
extract = ReflectionExtractor(3)
refl = extract(sweep, crystal)
import copy
refl2 = copy.deepcopy(refl)
from dials.algorithms.reflection_basis import transform
from dials.util.command_line import Command
Command.start("Init transform")
trans = transform.Forward(sweep, crystal, 3, 4)
Command.end("Init transform")
Command.start("Transform")
for i in range(1):
trans(refl)
Command.end("Transform")
Command.start("Init Transform")
spec = transform.TransformSpec(sweep, crystal, 3, 4)
Command.end("Init Transform")
Command.start("Transform")
for i in range(1):
transform.forward_batch(spec, refl2)
import libtbx.load_env
usage = "usage: %s [options] reflections1.pickle reflections2.pickle" \
% libtbx.env.dispatcher_name
parser = OptionParser(usage)
# Parse the command line arguments
(options, args) = parser.parse_args()
# Ensure there are only two reflection lists
if len(args) != 2:
parser.print_help()
exit(0)
# Read the first batch of reflections
Command.start('Reading reflections from %s' % args[0])
refl1 = flex.reflection_table.from_pickle(args[0])
mask = flex.bool(xyz == (0, 0, 0) for xyz in refl1['xyzobs.px.value'])
refl1.del_selected(mask)
Command.end('Read %d reflections from %s' % (len(refl1), args[0]))
# Read the second batch of reflections
Command.start('Reading reflections from %s' % args[1])
refl2 = flex.reflection_table.from_pickle(args[1])
mask = refl2['intensity.sum.value'] <= 0.0
refl2.del_selected(mask)
mask = refl2['intensity.sum.value']**2 < refl2['intensity.sum.variance']
refl2.del_selected(mask)
Command.end('Read %d reflections from %s' % (len(refl2), args[1]))
# perform the match
def __call__(self, params, options):
''' Import the spot.xds file. '''
from iotbx.xds import spot_xds
from dials.util.command_line import Command
from dials.array_family import flex
# Read the SPOT.XDS file
Command.start('Reading SPOT.XDS')
handle = spot_xds.reader()
handle.read_file(self._spot_xds)
centroid = handle.centroid
intensity = handle.intensity
try:
miller_index = handle.miller_index
except AttributeError:
miller_index = None
Command.end('Read {0} spots from SPOT.XDS file.'.format(len(centroid)))
# Create the reflection list
Command.start('Creating reflection list')
table = flex.reflection_table()
table['id'] = flex.int(len(centroid), 0)
table['panel'] = flex.size_t(len(centroid), 0)
if miller_index:
table['miller_index'] = flex.miller_index(miller_index)
table['xyzobs.px.value'] = flex.vec3_double(centroid)
table['intensity.sum.value'] = flex.double(intensity)
Command.end('Created reflection list')
# Remove invalid reflections
Command.start('Removing invalid reflections')
if miller_index and params.remove_invalid:
flags = flex.bool([h != (0, 0, 0) for h in table['miller_index']])
table = table.select(flags)
Command.end('Removed invalid reflections, %d remaining' % len(table))
# Fill empty standard columns
if params.add_standard_columns:
Command.start('Adding standard columns')
rt = flex.reflection_table.empty_standard(len(table))
rt.update(table)
table = rt
# set variances to unity
table['xyzobs.mm.variance'] = flex.vec3_double(len(table), (1,1,1))
table['xyzobs.px.variance'] = flex.vec3_double(len(table), (1,1,1))
Command.end('Standard columns added')
# Output the table to pickle file
if params.output.filename is None:
params.output.filename = 'spot_xds.pickle'
Command.start('Saving reflection table to %s' % params.output.filename)
table.as_pickle(params.output.filename)
Command.end('Saved reflection table to %s' % params.output.filename)
def generate_predictions(self, N):
""" Generate some reflections. """
from dials.algorithms.profile_model.gaussian_rs import MaskCalculator3D
from dials.array_family import flex
from dials.util.command_line import Command
from dials.algorithms import filtering
from dials.algorithms.shoebox import MaskCode
from dials.algorithms.profile_model.gaussian_rs import Model as ProfileModel
import random
# Set the profile model
self.experiment.profile = ProfileModel(None, self.n_sigma, self.sigma_b, self.sigma_m)
# Generate a list of reflections
refl = flex.reflection_table.from_predictions(self.experiment)
refl["id"] = flex.int(len(refl), 0)
# Filter by zeta
zeta = 0.05
Command.start("Filtering by zeta >= %f" % zeta)
mask = filtering.by_zeta(self.experiment.goniometer, self.experiment.beam, refl["s1"], zeta)
refl.del_selected(mask != True)
Command.end("Filtered %d reflections by zeta >= %f" % (len(refl), zeta))
# Compute the bounding box
refl.compute_bbox([self.experiment])
index = []
image_size = self.experiment.detector[0].get_image_size()
array_range = self.experiment.scan.get_array_range()
bbox = refl["bbox"]
for i in range(len(refl)):
x0, x1, y0, y1, z0, z1 = bbox[i]
if (
x0 < 0
or x1 > image_size[0]
or y0 < 0
or y1 > image_size[1]
or z0 < array_range[0]
or z1 > array_range[1]
):
index.append(i)
refl.del_selected(flex.size_t(index))
# Sample if specified
index = random.sample(range(len(refl)), N)
refl = refl.select(flex.size_t(index))
# Compute the bounding box
# Create a load of shoeboxes
Command.start("Creating shoeboxes for %d reflections" % len(refl))
refl["shoebox"] = flex.shoebox(refl["panel"], refl["bbox"])
refl["shoebox"].allocate_with_value(MaskCode.Valid)
Command.end("Created shoeboxes for %d reflections" % len(refl))
# Get the function object to mask the foreground
Command.start("Masking Foreground for %d reflections" % len(refl))
mask_foreground = MaskCalculator3D(
self.experiment.beam,
self.experiment.detector,
self.experiment.goniometer,
self.experiment.scan,
self.n_sigma * self.sigma_b,
self.n_sigma * self.sigma_m,
)
# Mask the foreground
mask_foreground(refl["shoebox"], refl["s1"], refl["xyzcal.px"].parts()[2], refl["panel"])
Command.end("Masked foreground for %d reflections" % len(refl))
# Return the reflections
return refl
cspad_img, self.params,
os.path.join(self.params.output.output_dir, "hit-" + s))
# save strong reflections. self.find_spots() would have done this, but we only
# want to save data if it is enough to try and index it
if self.strong_filename_template:
if "%s" in self.strong_filename_template:
strong_filename = self.strong_filename_template % ("hit-" +
s)
else:
strong_filename = self.strong_filename_template
strong_filename = os.path.join(self.params.output.output_dir,
strong_filename)
from dials.util.command_line import Command
Command.start('Saving {0} reflections to {1}'.format(
len(observed), os.path.basename(strong_filename)))
observed.as_pickle(strong_filename)
Command.end('Saved {0} observed to {1}'.format(
len(observed), os.path.basename(strong_filename)))
if not self.params.dispatch.index:
self.debug_write("strong_shot_%d" % len(observed), "done")
self.log_frame(None, None, run.run(), len(observed), timestamp)
return
# index and refine
self.debug_write("index_start")
try:
experiments, indexed = self.index(datablock, observed)
except Exception, e:
import traceback
def __call__(self, params, options):
''' Import the integrate.hkl file. '''
from iotbx.xds import integrate_hkl
from dials.array_family import flex
from dials.util.command_line import Command
from cctbx import sgtbx
# Get the unit cell to calculate the resolution
uc = self._experiment.crystal.get_unit_cell()
# Read the INTEGRATE.HKL file
Command.start('Reading INTEGRATE.HKL')
handle = integrate_hkl.reader()
handle.read_file(self._integrate_hkl)
hkl = flex.miller_index(handle.hkl)
xyzcal = flex.vec3_double(handle.xyzcal)
xyzobs = flex.vec3_double(handle.xyzobs)
iobs = flex.double(handle.iobs)
sigma = flex.double(handle.sigma)
rlp = flex.double(handle.rlp)
peak = flex.double(handle.peak) * 0.01
if len(handle.iseg):
panel = flex.size_t(handle.iseg) - 1
else:
panel = flex.size_t(len(hkl), 0)
Command.end('Read %d reflections from INTEGRATE.HKL file.' % len(hkl))
if len(self._experiment.detector) > 1:
for p_id, p in enumerate(self._experiment.detector):
sel = (panel == p_id)
offset = p.get_raw_image_offset()
xyzcal.set_selected(
sel,
xyzcal.select(sel) - (offset[0], offset[1], 0))
xyzobs.set_selected(
sel,
xyzobs.select(sel) - (offset[0], offset[1], 0))
# Derive the reindex matrix
rdx = self.derive_reindex_matrix(handle)
print('Reindex matrix:\n%d %d %d\n%d %d %d\n%d %d %d' % (rdx.elems))
# Reindex the reflections
Command.start('Reindexing reflections')
cb_op = sgtbx.change_of_basis_op(sgtbx.rt_mx(sgtbx.rot_mx(rdx.elems)))
hkl = cb_op.apply(hkl)
Command.end('Reindexed %d reflections' % len(hkl))
# Create the reflection list
Command.start('Creating reflection table')
table = flex.reflection_table()
table['id'] = flex.int(len(hkl), 0)
table['panel'] = panel
table['miller_index'] = hkl
table['xyzcal.px'] = xyzcal
table['xyzobs.px.value'] = xyzobs
table['intensity.cor.value'] = iobs
table['intensity.cor.variance'] = sigma**2
table['intensity.prf.value'] = iobs * peak / rlp
table['intensity.prf.variance'] = (sigma * peak / rlp)**2
table['lp'] = 1.0 / rlp
table['d'] = flex.double(uc.d(h) for h in hkl)
Command.end('Created table with {0} reflections'.format(len(table)))
# Output the table to pickle file
if params.output.filename is None:
params.output.filename = 'integrate_hkl.pickle'
Command.start('Saving reflection table to %s' % params.output.filename)
table.as_pickle(params.output.filename)
Command.end('Saved reflection table to %s' % params.output.filename)
def run(self):
''' Run the script. '''
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.util.command_line import Command
from dials.array_family import flex
from dials.util.options import flatten_reflections, flatten_experiments
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
from libtbx.utils import Sorry
from dials.util import log
log.config()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reflections) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reflections) != 1:
raise Sorry('exactly 1 reflection table must be specified')
if len(experiments) == 0:
raise Sorry('no experiments were specified')
if (not 'background.mean' in reflections[0]) and params.subtract_background:
raise Sorry('for subtract_background need background.mean in reflections')
reflections, _ = self.process_reference(reflections[0], params)
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static)
# Match with predicted
matched, reflections, unmatched = predicted.match_with_reference(reflections)
assert(len(matched) == len(predicted))
assert(matched.count(True) <= len(reflections))
if matched.count(True) == 0:
raise Sorry('''
Invalid input for reference reflections.
Zero reference spots were matched to predictions
''')
elif len(unmatched) != 0:
logger.info('')
logger.info('*' * 80)
logger.info('Warning: %d reference spots were not matched to predictions' % (
len(unmatched)))
logger.info('*' * 80)
logger.info('')
# Create the profile model
experiments = ProfileModelFactory.create(params, experiments, reflections)
for model in experiments:
sigma_b = model.profile.sigma_b(deg=True)
sigma_m = model.profile.sigma_m(deg=True)
if type(sigma_b) == type(1.0):
logger.info('Sigma B: %f' % sigma_b)
logger.info('Sigma M: %f' % sigma_m)
else: # scan varying
mean_sigma_b = sum(sigma_b) / len(sigma_b)
mean_sigma_m = sum(sigma_m) / len(sigma_m)
logger.info('Sigma B: %f' % mean_sigma_b)
logger.info('Sigma M: %f' % mean_sigma_m)
# Wrtie the parameters
Command.start("Writing experiments to %s" % params.output)
dump = ExperimentListDumper(experiments)
with open(params.output, "w") as outfile:
outfile.write(dump.as_json())
Command.end("Wrote experiments to %s" % params.output)
def __call__(self, params, options):
"""Import the integrate.hkl file."""
# Get the unit cell to calculate the resolution
uc = self._experiment.crystal.get_unit_cell()
# Read the INTEGRATE.HKL file
Command.start("Reading INTEGRATE.HKL")
handle = integrate_hkl.reader()
handle.read_file(self._integrate_hkl)
hkl = flex.miller_index(handle.hkl)
xyzcal = flex.vec3_double(handle.xyzcal)
xyzobs = flex.vec3_double(handle.xyzobs)
iobs = flex.double(handle.iobs)
sigma = flex.double(handle.sigma)
rlp = flex.double(handle.rlp)
peak = flex.double(handle.peak) * 0.01
if len(handle.iseg):
panel = flex.size_t(handle.iseg) - 1
else:
panel = flex.size_t(len(hkl), 0)
Command.end(f"Read {len(hkl)} reflections from INTEGRATE.HKL file.")
if len(self._experiment.detector) > 1:
for p_id, p in enumerate(self._experiment.detector):
sel = panel == p_id
offset = p.get_raw_image_offset()
xyzcal.set_selected(sel, xyzcal.select(sel) - (offset[0], offset[1], 0))
xyzobs.set_selected(sel, xyzobs.select(sel) - (offset[0], offset[1], 0))
# Derive the reindex matrix
rdx = self.derive_reindex_matrix(handle)
print("Reindex matrix:\n%d %d %d\n%d %d %d\n%d %d %d" % (rdx.elems))
# Reindex the reflections
Command.start("Reindexing reflections")
cb_op = sgtbx.change_of_basis_op(sgtbx.rt_mx(sgtbx.rot_mx(rdx.elems)))
hkl = cb_op.apply(hkl)
Command.end(f"Reindexed {len(hkl)} reflections")
# Create the reflection list
Command.start("Creating reflection table")
table = flex.reflection_table()
table["id"] = flex.int(len(hkl), 0)
table["panel"] = panel
table["miller_index"] = hkl
table["xyzcal.px"] = xyzcal
table["xyzobs.px.value"] = xyzobs
table["intensity.cor.value"] = iobs
table["intensity.cor.variance"] = flex.pow2(sigma)
table["intensity.prf.value"] = iobs * peak / rlp
table["intensity.prf.variance"] = flex.pow2(sigma * peak / rlp)
table["lp"] = 1.0 / rlp
table["d"] = flex.double(uc.d(h) for h in hkl)
Command.end(f"Created table with {len(table)} reflections")
# Output the table to pickle file
if params.output.filename is None:
params.output.filename = "integrate_hkl.refl"
Command.start(f"Saving reflection table to {params.output.filename}")
table.as_file(params.output.filename)
Command.end(f"Saved reflection table to {params.output.filename}")
def __call__(self, params, options):
''' Import the integrate.hkl file. '''
from iotbx.xds import integrate_hkl
from dials.array_family import flex
from dials.util.command_line import Command
from cctbx import sgtbx
# Get the unit cell to calculate the resolution
uc = self._experiment.crystal.get_unit_cell()
# Read the INTEGRATE.HKL file
Command.start('Reading INTEGRATE.HKL')
handle = integrate_hkl.reader()
handle.read_file(self._integrate_hkl)
hkl = flex.miller_index(handle.hkl)
xyzcal = flex.vec3_double(handle.xyzcal)
xyzobs = flex.vec3_double(handle.xyzobs)
iobs = flex.double(handle.iobs)
sigma = flex.double(handle.sigma)
rlp = flex.double(handle.rlp)
peak = flex.double(handle.peak) * 0.01
if len(handle.iseg):
panel = flex.size_t(handle.iseg) - 1
else:
panel = flex.size_t(len(hkl), 0)
Command.end('Read %d reflections from INTEGRATE.HKL file.' % len(hkl))
if len(self._experiment.detector) > 1:
for p_id, p in enumerate(self._experiment.detector):
sel = (panel == p_id)
offset = p.get_raw_image_offset()
xyzcal.set_selected(sel, xyzcal.select(sel) - (offset[0], offset[1], 0))
xyzobs.set_selected(sel, xyzobs.select(sel) - (offset[0], offset[1], 0))
# Derive the reindex matrix
rdx = self.derive_reindex_matrix(handle)
print 'Reindex matrix:\n%d %d %d\n%d %d %d\n%d %d %d' % (rdx.elems)
# Reindex the reflections
Command.start('Reindexing reflections')
cb_op = sgtbx.change_of_basis_op(sgtbx.rt_mx(sgtbx.rot_mx(rdx.elems)))
hkl = cb_op.apply(hkl)
Command.end('Reindexed %d reflections' % len(hkl))
# Create the reflection list
Command.start('Creating reflection table')
table = flex.reflection_table()
table['id'] = flex.int(len(hkl), 0)
table['panel'] = panel
table['miller_index'] = hkl
table['xyzcal.px'] = xyzcal
table['xyzobs.px.value'] = xyzobs
table['intensity.cor.value'] = iobs
table['intensity.cor.variance'] = sigma**2
table['intensity.prf.value'] = iobs * peak / rlp
table['intensity.prf.variance'] = (sigma * peak / rlp)**2
table['lp'] = 1.0 / rlp
table['d'] = flex.double(uc.d(h) for h in hkl)
Command.end('Created table with {0} reflections'.format(len(table)))
# Output the table to pickle file
if params.output.filename is None:
params.output.filename = 'integrate_hkl.pickle'
Command.start('Saving reflection table to %s' % params.output.filename)
table.as_pickle(params.output.filename)
Command.end('Saved reflection table to %s' % params.output.filename)
def generate_predictions(self, N):
"""Generate some reflections."""
from dials.algorithms import filtering
from dials.algorithms.profile_model.gaussian_rs import MaskCalculator3D
from dials.algorithms.profile_model.gaussian_rs import Model as ProfileModel
from dials.algorithms.shoebox import MaskCode
from dials.util.command_line import Command
# Set the profile model
self.experiment.profile = ProfileModel(None, self.n_sigma,
self.sigma_b, self.sigma_m)
# Generate a list of reflections
refl = flex.reflection_table.from_predictions(self.experiment)
refl["id"] = flex.int(len(refl), 0)
# Filter by zeta
zeta = 0.05
Command.start(f"Filtering by zeta >= {zeta:f}")
mask = filtering.by_zeta(self.experiment.goniometer,
self.experiment.beam, refl["s1"], zeta)
refl.del_selected(~mask)
Command.end(f"Filtered {len(refl)} reflections by zeta >= {zeta:f}")
# Compute the bounding box
refl.compute_bbox([self.experiment])
index = []
image_size = self.experiment.detector[0].get_image_size()
array_range = self.experiment.scan.get_array_range()
bbox = refl["bbox"]
for i in range(len(refl)):
x0, x1, y0, y1, z0, z1 = bbox[i]
if (x0 < 0 or x1 > image_size[0] or y0 < 0 or y1 > image_size[1]
or z0 < array_range[0] or z1 > array_range[1]):
index.append(i)
refl.del_selected(flex.size_t(index))
# Sample if specified
index = random.sample(range(len(refl)), N)
refl = refl.select(flex.size_t(index))
# Compute the bounding box
# Create a load of shoeboxes
Command.start(f"Creating shoeboxes for {len(refl)} reflections")
refl["shoebox"] = flex.shoebox(refl["panel"], refl["bbox"])
refl["shoebox"].allocate_with_value(MaskCode.Valid)
Command.end(f"Created shoeboxes for {len(refl)} reflections")
# Get the function object to mask the foreground
Command.start(f"Masking Foreground for {len(refl)} reflections")
mask_foreground = MaskCalculator3D(
self.experiment.beam,
self.experiment.detector,
self.experiment.goniometer,
self.experiment.scan,
self.n_sigma * self.sigma_b,
self.n_sigma * self.sigma_m,
)
# Mask the foreground
mask_foreground(refl["shoebox"], refl["s1"],
refl["xyzcal.px"].parts()[2], refl["panel"])
Command.end(f"Masked foreground for {len(refl)} reflections")
# Return the reflections
return refl
def __call__(self, sweep):
'''The main function of the spot finder. Select the pixels from
the sweep and then group the pixels into spots. Return the data
in the form of a reflection list.
Params:
sweep The sweep object
Returns:
The reflection list
'''
from dials.util.command_line import Command
# Set a command indent to 4
Command.indent = 4
print '\nFinding spot in {0} images...'.format(len(sweep))
# Extract the image pixels from the sweep
Command.start('Extracting pixels from sweep')
coords, intensity = self._extract_pixels(sweep)
Command.end('Extracted {0} strong pixels'.format(len(coords)))
# Label the pixels and group into spots
Command.start('Labelling connected components')
labels = self._label_pixels(coords, sweep)
Command.end('Found {0} connected components'.format(max(labels) + 1))
# Filter spots that are too small
Command.start('Filtering spots by size')
spots = self._filter_spots(labels)
Command.end('Filtered {0} spots by size'.format(len(spots)))
# Calculate the bounding box for each spot
Command.start('Calculating bounding boxes')
bbox = self._calculate_bbox(coords, spots, sweep)
Command.end('Calculated {0} bounding boxes'.format(len(bbox)))
# Calculate the spot centroids
Command.start('Calculating centroids')
cpos, cvar = self._calculate_centroids(coords, intensity, spots)
Command.end('Calculated {0} centroids'.format(len(cpos)))
# Filter the spots by centroid-maxmimum distance
Command.start('Filtering spots by distance')
index = self._filter_maximum_centroid(coords, intensity, spots, cpos)
Command.end('Filtered {0} spots by distance'.format(len(index)))
# Create a reflection list and return
return self._create_reflection_list(
coords, intensity, spots, bbox, cpos, cvar, index)