def __call__(self, imageset, shoeboxes):
"""
Filter shoeboxes and create reflection table
"""
# Calculate the spot centroids
centroid = shoeboxes.centroid_valid()
logger.info("Calculated {} spot centroids".format(len(shoeboxes)))
# Calculate the spot intensities
intensity = shoeboxes.summed_intensity()
logger.info("Calculated {} spot intensities".format(len(shoeboxes)))
# Create the observations
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
# Filter the reflections and select only the desired spots
flags = self.filter_spots(None,
sequence=imageset,
observations=observed,
shoeboxes=shoeboxes)
observed = observed.select(flags)
shoeboxes = shoeboxes.select(flags)
# Return as a reflection list
return flex.reflection_table(observed, shoeboxes)
def find_spots(self, min_spot_size=2, max_spot_size=100):
from dials.algorithms.spot_finding.threshold import XDSThresholdStrategy
from dials.model.data import PixelList
from dials.model.data import PixelListLabeller
image = self.raw_data
mask = self.imageset.get_mask(0)[0]
threshold_image = XDSThresholdStrategy()
threshold_mask = threshold_image(image, mask)
plist = PixelList(0, image, threshold_mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(plist)
creator = flex.PixelListShoeboxCreator(pixel_labeller, 0, 0, True,
min_spot_size, max_spot_size,
False)
shoeboxes = creator.result()
# turns out we need to manually filter the list to get a sensible answer
size = creator.spot_size()
big = size > max_spot_size
small = size < min_spot_size
bad = big | small
shoeboxes = shoeboxes.select(~bad)
centroid = shoeboxes.centroid_valid()
intensity = shoeboxes.summed_intensity()
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
reflections = flex.reflection_table(observed, shoeboxes)
return reflections
def __call__(self, imageset, shoeboxes):
'''
Filter shoeboxes and create reflection table
'''
from dials.array_family import flex
# Calculate the spot centroids
centroid = shoeboxes.centroid_valid()
logger.info('Calculated {0} spot centroids'.format(len(shoeboxes)))
# Calculate the spot intensities
intensity = shoeboxes.summed_intensity()
logger.info('Calculated {0} spot intensities'.format(len(shoeboxes)))
# Create the observations
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
# Filter the reflections and select only the desired spots
flags = self.filter_spots(None,
sweep=imageset,
observations=observed,
shoeboxes=shoeboxes)
observed = observed.select(flags)
shoeboxes = shoeboxes.select(flags)
# Return as a reflection list
return flex.reflection_table(observed, shoeboxes)
def __call__(self, imageset, shoeboxes):
'''
Filter shoeboxes and create reflection table
'''
from dials.array_family import flex
# Calculate the spot centroids
centroid = shoeboxes.centroid_valid()
logger.info('Calculated {0} spot centroids'.format(len(shoeboxes)))
# Calculate the spot intensities
intensity = shoeboxes.summed_intensity()
logger.info('Calculated {0} spot intensities'.format(len(shoeboxes)))
# Create the observations
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
# Filter the reflections and select only the desired spots
flags = self.filter_spots(None,
sweep=imageset,
observations=observed,
shoeboxes=shoeboxes)
observed = observed.select(flags)
shoeboxes = shoeboxes.select(flags)
# Return as a reflection list
return flex.reflection_table(observed, shoeboxes)
def find_spots(image, mask, min_spot_size=1, max_spot_size=1000):
from dials.algorithms.spot_finding.threshold import XDSThresholdStrategy
from dials.model.data import PixelList
from dials.model.data import PixelListLabeller
threshold_image = XDSThresholdStrategy()
threshold_mask = threshold_image(image, mask)
plist = PixelList(0, image, threshold_mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(plist)
creator = flex.PixelListShoeboxCreator(
pixel_labeller,
0, # panel
0, # zrange
True, # twod
min_spot_size, # min_pixels
max_spot_size, # max_pixels
False,
)
shoeboxes = creator.result()
centroid = shoeboxes.centroid_valid()
intensity = shoeboxes.summed_intensity()
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
return flex.reflection_table(observed, shoeboxes)
def find_spots(self, min_spot_size=2, max_spot_size=100):
"""
Find the strong spots on the image
"""
from dials.algorithms.spot_finding.threshold import DispersionThresholdStrategy
from dials.model.data import PixelList
from dials.model.data import PixelListLabeller
from dials.array_family import flex
print("")
print("-" * 80)
print(" Finding strong spots")
print("-" * 80)
print("")
# Instantiate the threshold function
threshold = DispersionThresholdStrategy()
# Get the raw data and image mask
image = self.experiment.imageset.get_raw_data(0)[0]
mask = self.experiment.imageset.get_mask(0)[0]
# Threshold the image and create the pixel labeller
threshold_mask = threshold(image, mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(PixelList(0, image, threshold_mask))
# Create the shoebox list from the pixel list
creator = flex.PixelListShoeboxCreator(
pixel_labeller,
0, # Panel number
0, # Z start
True, # 2D
self.params.spot_finding.min_spot_size, # Min Pixels
self.params.spot_finding.max_spot_size, # Max Pixels
False,
) # Find hot pixels
shoeboxes = creator.result()
# Filter the list to remove large and small spots
size = creator.spot_size()
large = size > self.params.spot_finding.max_spot_size
small = size < self.params.spot_finding.min_spot_size
bad = large | small
shoeboxes = shoeboxes.select(~bad)
print("Discarding %d spots with < %d pixels" %
(small.count(True), self.params.spot_finding.min_spot_size))
print("Discarding %d spots with > %d pixels" %
(large.count(True), self.params.spot_finding.max_spot_size))
# Extract the strong spot information
centroid = shoeboxes.centroid_valid()
intensity = shoeboxes.summed_intensity()
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
# Create the reflection list
self.reflections = flex.reflection_table(observed, shoeboxes)
print("Using %d strong spots" % len(self.reflections))
def find(greyscale_flex, params, mask=None):
'''Find stars on input greyscale flex image.'''
from dials.algorithms.spot_finding.threshold import \
DispersionThresholdStrategy
from dials.model.data import PixelList
from dials.model.data import PixelListLabeller
from dials.array_family import flex
thresholder = DispersionThresholdStrategy(gain=params.gain)
if not mask:
mask = flex.bool(greyscale_flex.size(), True)
mask.reshape(flex.grid(*greyscale_flex.focus()))
threshold_mask = thresholder(greyscale_flex, mask=mask)
plist = PixelList(0, greyscale_flex, threshold_mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(plist)
creator = flex.PixelListShoeboxCreator(pixel_labeller, 0, 0, True,
params.min_size, params.max_size,
False)
shoeboxes = creator.result()
# remove nonsense
size = creator.spot_size()
big = size > params.max_size
small = size < params.min_size
bad = big | small
shoeboxes = shoeboxes.select(~bad)
centroid = shoeboxes.centroid_valid()
intensity = shoeboxes.summed_intensity()
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
stars = flex.reflection_table(observed, shoeboxes)
return stars
def shoeboxes_to_reflection_table(imageset: ImageSet, shoeboxes: flex.shoebox,
filter_spots) -> flex.reflection_table:
"""Filter shoeboxes and create reflection table"""
# Calculate the spot centroids
centroid = shoeboxes.centroid_valid()
logger.info("Calculated %d spot centroids", len(shoeboxes))
# Calculate the spot intensities
intensity = shoeboxes.summed_intensity()
logger.info("Calculated %d spot intensities", len(shoeboxes))
# Create the observations
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
# Filter the reflections and select only the desired spots
flags = filter_spots(None,
sweep=imageset,
observations=observed,
shoeboxes=shoeboxes)
observed = observed.select(flags)
shoeboxes = shoeboxes.select(flags)
# Return as a reflection list
return flex.reflection_table(observed, shoeboxes)
def combine(datablock_list, reflections_list, params):
'''
Combine the found spots.
'''
from dxtbx.datablock import BeamComparison
from dxtbx.datablock import DetectorComparison
from dxtbx.datablock import GoniometerComparison
from dxtbx.datablock import DataBlock
from dxtbx.imageset import ImageSetFactory
from dials.algorithms.spot_finding import StrongSpotCombiner
from dials.array_family import flex
assert len(datablock_list) == len(reflections_list)
# Get a list of imagesets
imageset_list = []
for db in datablock_list:
iset = db.extract_imagesets()
assert len(iset) == 1
imageset_list.append(iset[0])
compare_beam = BeamComparison(
wavelength_tolerance=params.input.tolerance.beam.wavelength,
direction_tolerance=params.input.tolerance.beam.direction,
polarization_normal_tolerance=params.input.tolerance.beam.polarization_normal,
polarization_fraction_tolerance=params.input.tolerance.beam.polarization_fraction)
compare_detector = DetectorComparison(
fast_axis_tolerance=params.input.tolerance.detector.fast_axis,
slow_axis_tolerance=params.input.tolerance.detector.slow_axis,
origin_tolerance=params.input.tolerance.detector.origin)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=params.input.tolerance.goniometer.rotation_axis,
fixed_rotation_tolerance=params.input.tolerance.goniometer.fixed_rotation,
setting_rotation_tolerance=params.input.tolerance.goniometer.setting_rotation)
scan_tolerance = params.input.tolerance.scan.oscillation
# The initial models
format_class = imageset_list[0].get_format_class()
beam = imageset_list[0].get_beam()
detector = imageset_list[0].get_detector()
goniometer = imageset_list[0].get_goniometer()
scan = imageset_list[0].get_scan()
template = imageset_list[0].get_template()
# Check all the models
for imageset in imageset_list[1:]:
b = imageset.get_beam()
d = imageset.get_detector()
g = imageset.get_goniometer()
s = imageset.get_scan()
if not imageset.get_format_class() == format_class:
raise RuntimeError('Format classes do not match')
if not imageset.get_template() == template:
raise RuntimeError('Templates do not match')
if not compare_beam(beam, b):
raise RuntimeError('Beam models are too dissimilar')
if not compare_detector(detector, d):
raise RuntimeError('Detector models are too dissimilar')
if not compare_goniometer(goniometer, g):
raise RuntimeError('Goniometer models are too dissimilar')
try:
scan.append(s, scan_tolerance=scan_tolerance)
except Exception:
raise RuntimeError('Scans do not match')
# Get the image range
image_range = scan.get_image_range()
image_range = (image_range[0], image_range[1]+1)
# Create the sweep
imageset = ImageSetFactory.make_sweep(
template, range(*image_range),
format_class,
beam, detector,
goniometer, scan)
# Combine spots
combiner = StrongSpotCombiner()
for index, rlist in enumerate(reflections_list, start=1):
assert rlist['id'].all_eq(0)
logger.info("Combining %d reflections from reflection list %d" % (
len(rlist),
index))
combiner.add(rlist['shoebox'])
shoeboxes = combiner.shoeboxes()
# Calculate the spot centroids and intensities
logger.info('Combined into %d reflections' % len(shoeboxes))
centroid = shoeboxes.centroid_valid()
logger.info('Calculated {0} spot centroids'.format(len(shoeboxes)))
intensity = shoeboxes.summed_intensity()
logger.info('Calculated {0} spot intensities'.format(len(shoeboxes)))
# Construct the reflection table
reflections = flex.reflection_table(
flex.observation(
shoeboxes.panels(),
centroid,
intensity),
shoeboxes)
reflections['id'] = flex.int(len(reflections), 0)
reflections.set_flags(
flex.size_t_range(len(reflections)),
reflections.flags.strong)
# Return the datablock and reflections
return DataBlock([imageset]), reflections
def _find_in_imageset(self, imageset):
"""
Do the spot finding.
:param imageset: The imageset to process
:return: The observed spots
"""
from dials.util.masking import MaskGenerator
from dials.array_family import flex
from dials.util.command_line import Command
from dxtbx.imageset import ImageSweep
from logging import info
# The input mask
mask = self.mask_generator.generate(imageset)
if self.mask is not None:
mask = tuple(m1 & m2 for m1, m2 in zip(mask, self.mask))
# Set the spot finding algorithm
extract_spots = ExtractSpots(
threshold_function=self.threshold_function,
mask=mask,
region_of_interest=self.region_of_interest,
max_strong_pixel_fraction=self.max_strong_pixel_fraction,
mp_method=self.mp_method,
nproc=self.nproc,
mp_chunksize=self.mp_chunksize,
min_spot_size=self.min_spot_size,
max_spot_size=self.max_spot_size,
)
# Get the max scan range
if isinstance(imageset, ImageSweep):
max_scan_range = imageset.get_array_range()
else:
max_scan_range = (0, len(imageset))
# Get list of scan ranges
if not self.scan_range or self.scan_range[0] is None:
scan_range = [(max_scan_range[0] + 1, max_scan_range[1])]
else:
scan_range = self.scan_range
# Get spots from bits of scan
spots_all = []
for scan in scan_range:
j0, j1 = scan
assert j1 >= j0 and j0 > max_scan_range[0] and j1 <= max_scan_range[1]
info("\nFinding spots in image {0} to {1}...".format(j0, j1))
j0 -= 1
if isinstance(imageset, ImageSweep):
j0 -= imageset.get_array_range()[0]
j1 -= imageset.get_array_range()[0]
spots_all.extend(extract_spots(imageset[j0:j1]))
# Get the list of shoeboxes
shoeboxes = flex.shoebox(spots_all)
# Calculate the spot centroids
centroid = shoeboxes.centroid_valid()
info("Calculated {0} spot centroids".format(len(shoeboxes)))
# Calculate the spot intensities
intensity = shoeboxes.summed_intensity()
info("Calculated {0} spot intensities".format(len(shoeboxes)))
# Create the observations
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
# Write the hot mask
if self.write_hot_mask:
# Find spots which cover the whole scan range
bbox = flex.int6([sbox.bbox for sbox in shoeboxes])
z0, z1 = bbox.parts()[4:6]
zr = z1 - z0
assert zr.all_gt(0)
possible_hot_spots = zr == len(imageset)
num_possible_hot_spots = possible_hot_spots.count(True)
info("Found %d possible hot spots" % num_possible_hot_spots)
# Create the hot pixel mask
hot_mask = tuple(flex.bool(flex.grid(p.get_image_size()[::-1]), True) for p in imageset.get_detector())
if num_possible_hot_spots > 0:
hot_shoeboxes = shoeboxes.select(possible_hot_spots)
for sbox in hot_shoeboxes:
x0, x1, y0, y1 = sbox.bbox[0:4]
m = sbox.mask
p = sbox.panel
for y in range(m.all()[1]):
for x in range(m.all()[2]):
if m[:, y : y + 1, x : x + 1].all_ne(0):
hot_mask[p][y0 + y, x0 + x] = False
info("Found %d possible hot pixel(s)" % hot_mask.count(False))
else:
hot_mask = None
# Filter the reflections and select only the desired spots
flags = self.filter_spots(None, sweep=imageset, observations=observed, shoeboxes=shoeboxes)
observed = observed.select(flags)
shoeboxes = shoeboxes.select(flags)
# Return as a reflection list
return flex.reflection_table(observed, shoeboxes), hot_mask
def combine(datablock_list, reflections_list, params):
'''
Combine the found spots.
'''
from dxtbx.datablock import BeamComparison
from dxtbx.datablock import DetectorComparison
from dxtbx.datablock import GoniometerComparison
from dxtbx.datablock import DataBlock
from dxtbx.imageset import ImageSetFactory
from dials.algorithms.spot_finding import StrongSpotCombiner
from logging import info
from dials.array_family import flex
assert len(datablock_list) == len(reflections_list)
# Get a list of imagesets
imageset_list = []
for db in datablock_list:
iset = db.extract_imagesets()
assert len(iset) == 1
imageset_list.append(iset[0])
compare_beam = BeamComparison(
wavelength_tolerance=params.input.tolerance.beam.wavelength,
direction_tolerance=params.input.tolerance.beam.direction,
polarization_normal_tolerance=params.input.tolerance.beam.polarization_normal,
polarization_fraction_tolerance=params.input.tolerance.beam.polarization_fraction)
compare_detector = DetectorComparison(
fast_axis_tolerance=params.input.tolerance.detector.fast_axis,
slow_axis_tolerance=params.input.tolerance.detector.slow_axis,
origin_tolerance=params.input.tolerance.detector.origin)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=params.input.tolerance.goniometer.rotation_axis,
fixed_rotation_tolerance=params.input.tolerance.goniometer.fixed_rotation,
setting_rotation_tolerance=params.input.tolerance.goniometer.setting_rotation)
scan_tolerance = params.input.tolerance.scan.oscillation
# The initial models
format_class = imageset_list[0].reader().get_format_class()
beam = imageset_list[0].get_beam()
detector = imageset_list[0].get_detector()
goniometer = imageset_list[0].get_goniometer()
scan = imageset_list[0].get_scan()
template = imageset_list[0].get_template()
# Check all the models
for imageset in imageset_list[1:]:
b = imageset.get_beam()
d = imageset.get_detector()
g = imageset.get_goniometer()
s = imageset.get_scan()
if not imageset.reader().get_format_class() == format_class:
raise RuntimeError('Format classes do not match')
if not imageset.get_template() == template:
raise RuntimeError('Templates do not match')
if not compare_beam(beam, b):
raise RuntimeError('Beam models are too dissimilar')
if not compare_detector(detector, d):
raise RuntimeError('Detector models are too dissimilar')
if not compare_goniometer(goniometer, g):
raise RuntimeError('Goniometer models are too dissimilar')
try:
scan.append(s, scan_tolerance=scan_tolerance)
except Exception:
raise RuntimeError('Scans do not match')
# Get the image range
image_range = scan.get_image_range()
image_range = (image_range[0], image_range[1]+1)
# Create the sweep
imageset = ImageSetFactory.make_sweep(
template, range(*image_range),
format_class,
beam, detector,
goniometer, scan)
# Combine spots
combiner = StrongSpotCombiner()
for index, rlist in enumerate(reflections_list, start=1):
assert rlist['id'].all_eq(0)
info("Combining %d reflections from reflection list %d" % (
len(rlist),
index))
combiner.add(rlist['shoebox'])
shoeboxes = combiner.shoeboxes()
# Calculate the spot centroids and intensities
info('Combined into %d reflections' % len(shoeboxes))
centroid = shoeboxes.centroid_valid()
info('Calculated {0} spot centroids'.format(len(shoeboxes)))
intensity = shoeboxes.summed_intensity()
info('Calculated {0} spot intensities'.format(len(shoeboxes)))
# Construct the reflection table
reflections = flex.reflection_table(
flex.observation(
shoeboxes.panels(),
centroid,
intensity),
shoeboxes)
reflections['id'] = flex.int(len(reflections), 0)
reflections.set_flags(
flex.size_t_range(len(reflections)),
reflections.flags.strong)
# Return the datablock and reflections
return DataBlock([imageset]), reflections
