def find_spots(self, min_spot_size=2, max_spot_size=100):
from dials.algorithms.spot_finding.threshold import XDSThresholdStrategy
from dials.model.data import PixelList
from dials.model.data import PixelListLabeller
image = self.raw_data
mask = self.imageset.get_mask(0)[0]
threshold_image = XDSThresholdStrategy()
threshold_mask = threshold_image(image, mask)
plist = PixelList(0, image, threshold_mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(plist)
creator = flex.PixelListShoeboxCreator(pixel_labeller, 0, 0, True,
min_spot_size, max_spot_size,
False)
shoeboxes = creator.result()
# turns out we need to manually filter the list to get a sensible answer
size = creator.spot_size()
big = size > max_spot_size
small = size < min_spot_size
bad = big | small
shoeboxes = shoeboxes.select(~bad)
centroid = shoeboxes.centroid_valid()
intensity = shoeboxes.summed_intensity()
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
reflections = flex.reflection_table(observed, shoeboxes)
return reflections
def refls_from_sims(panel_imgs, detector, beam, thresh=0, filter=None, **kwargs ):
"""
gets a reflection table from simulated panels
:param panel_imgs: list of numpy arrays , each array is a simulated panel image
:param detector: dxtbx detector, can have multiple nodes
:param beam: dxtbx beam
:param thresh: threshol
:param filter:
:param kwargs:
:return: reflection table, with id coloumn set to 0
"""
pxlst_labs = []
for i in range(len(detector)):
plab = PixelListLabeller()
img = panel_imgs[i]
if filter is not None:
mask = filter(img, **kwargs) > thresh
else:
mask = img > thresh
pl = PixelList(0, flex.double(img), flex.bool(mask))
plab.add(pl)
pxlst_labs.append( plab)
pixlst_to_reftbl = PixelListToReflectionTable(
min_spot_size=1,
max_spot_size=194*184,
filter_spots=FilterRunner(),
write_hot_pixel_mask=False)
dblock = datablock_from_numpyarrays( panel_imgs, detector, beam)
iset = dblock.extract_imagesets()[0]
refls = pixlst_to_reftbl(iset, pxlst_labs)[0]
refls['id'] = flex.int(len(refls),0)
return refls
def find_spots(image, mask, min_spot_size=1, max_spot_size=1000):
from dials.algorithms.spot_finding.threshold import XDSThresholdStrategy
from dials.model.data import PixelList
from dials.model.data import PixelListLabeller
threshold_image = XDSThresholdStrategy()
threshold_mask = threshold_image(image, mask)
plist = PixelList(0, image, threshold_mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(plist)
creator = flex.PixelListShoeboxCreator(
pixel_labeller,
0, # panel
0, # zrange
True, # twod
min_spot_size, # min_pixels
max_spot_size, # max_pixels
False,
)
shoeboxes = creator.result()
centroid = shoeboxes.centroid_valid()
intensity = shoeboxes.summed_intensity()
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
return flex.reflection_table(observed, shoeboxes)
def find_spots(self, min_spot_size=2, max_spot_size=100):
"""
Find the strong spots on the image
"""
from dials.algorithms.spot_finding.threshold import DispersionThresholdStrategy
from dials.model.data import PixelList
from dials.model.data import PixelListLabeller
from dials.array_family import flex
print("")
print("-" * 80)
print(" Finding strong spots")
print("-" * 80)
print("")
# Instantiate the threshold function
threshold = DispersionThresholdStrategy()
# Get the raw data and image mask
image = self.experiment.imageset.get_raw_data(0)[0]
mask = self.experiment.imageset.get_mask(0)[0]
# Threshold the image and create the pixel labeller
threshold_mask = threshold(image, mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(PixelList(0, image, threshold_mask))
# Create the shoebox list from the pixel list
creator = flex.PixelListShoeboxCreator(
pixel_labeller,
0, # Panel number
0, # Z start
True, # 2D
self.params.spot_finding.min_spot_size, # Min Pixels
self.params.spot_finding.max_spot_size, # Max Pixels
False,
) # Find hot pixels
shoeboxes = creator.result()
# Filter the list to remove large and small spots
size = creator.spot_size()
large = size > self.params.spot_finding.max_spot_size
small = size < self.params.spot_finding.min_spot_size
bad = large | small
shoeboxes = shoeboxes.select(~bad)
print("Discarding %d spots with < %d pixels" %
(small.count(True), self.params.spot_finding.min_spot_size))
print("Discarding %d spots with > %d pixels" %
(large.count(True), self.params.spot_finding.max_spot_size))
# Extract the strong spot information
centroid = shoeboxes.centroid_valid()
intensity = shoeboxes.summed_intensity()
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
# Create the reflection list
self.reflections = flex.reflection_table(observed, shoeboxes)
print("Using %d strong spots" % len(self.reflections))
def __call__(self, index):
"""
Extract strong pixels from an image
:param index: The index of the image
"""
from dials.model.data import PixelListLabeller
# Initialise the pixel labeller
num_panels = len(self.imageset.get_detector())
pixel_labeller = [PixelListLabeller() for p in range(num_panels)]
# Call the super function
result = super(ExtractPixelsFromImage2DNoShoeboxes,
self).__call__(index)
# Add pixel lists to the labeller
assert len(pixel_labeller) == len(
result.pixel_list), "Inconsistent size"
for plabeller, plist in zip(pixel_labeller, result.pixel_list):
plabeller.add(plist)
# Create shoeboxes from pixel list
converter = PixelListToReflectionTable(self.min_spot_size,
self.max_spot_size,
self.filter_spots, False)
reflections, _ = converter(self.imageset, pixel_labeller)
# Delete the shoeboxes
del reflections["shoeboxes"]
# Return the reflections
return [reflections]
def __call__(self, index):
"""
Extract strong pixels from an image
:param index: The index of the image
"""
# Initialise the pixel labeller
num_panels = len(self.imageset.get_detector())
pixel_labeller = [PixelListLabeller() for p in range(num_panels)]
# Call the super function
result = super().__call__(index)
# Add pixel lists to the labeller
assert len(pixel_labeller) == len(result), "Inconsistent size"
for plabeller, plist in zip(pixel_labeller, result):
plabeller.add(plist)
# Create shoeboxes from pixel list
reflections, _ = pixel_list_to_reflection_table(
self.imageset,
pixel_labeller,
filter_spots=self.filter_spots,
min_spot_size=self.min_spot_size,
max_spot_size=self.max_spot_size,
write_hot_pixel_mask=False,
)
# Delete the shoeboxes
del reflections["shoeboxes"]
# Return the reflections
return [reflections]
def test_add_image():
from dials.model.data import PixelList, PixelListLabeller
from scitbx.array_family import flex
size = (2000, 2000)
sf = 10
labeller = PixelListLabeller()
count = 0
for i in range(3):
image = flex.random_int_gaussian_distribution(size[0] * size[1], 100,
5)
mask = flex.random_bool(size[0] * size[1], 0.5)
image.reshape(flex.grid(size))
mask.reshape(flex.grid(size))
pl = PixelList(sf + i, image, mask)
count += len(mask.as_1d().select(mask.as_1d()))
labeller.add(pl)
assert len(labeller.values()) == count
def find_spots(image):
from dials.algorithms.spot_finding.threshold import DispersionThresholdStrategy
from dials.model.data import PixelList
from dials.model.data import PixelListLabeller
thresholder = DispersionThresholdStrategy(gain=1)
mask = image.as_1d() >= 0 # flex.bool(image.size(), True)
mask.reshape(flex.grid(*image.focus()))
threshold_mask = thresholder(image, mask=mask)
plist = PixelList(0, image, threshold_mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(plist)
creator = flex.PixelListShoeboxCreator(pixel_labeller, 0, 0, True, 2, 100,
False)
shoeboxes = creator.result()
return shoeboxes
def image_to_shoeboxes(image):
"""For a given image, find spots 2 - 100 pixels im size, assuming a gain
of 1, return the list of spot shoeboxes. Also assumes valid intensities in
range 0...N."""
thresholder = DispersionThresholdStrategy(gain=1)
mask = image.as_1d() >= 0
mask.reshape(flex.grid(*image.focus()))
threshold_mask = thresholder(image, mask=mask)
plist = PixelList(0, image, threshold_mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(plist)
creator = flex.PixelListShoeboxCreator(pixel_labeller, 0, 0, True, 2, 100,
False)
shoeboxes = creator.result()
return shoeboxes
def tst_with_no_points(self):
from dials.model.data import PixelList, PixelListLabeller
from scitbx.array_family import flex
size = (500, 500)
sf = 0
labeller = PixelListLabeller()
count = 0
mask_list = []
for i in range(3):
image = flex.random_int_gaussian_distribution(
size[0] * size[1], 100, 5)
mask = flex.bool(size[0] * size[0], False)
image.reshape(flex.grid(size))
mask.reshape(flex.grid(size))
pl = PixelList(sf + i, image, mask)
count += len(mask.as_1d().select(mask.as_1d()))
labeller.add(pl)
mask_list.append(mask)
coords = labeller.coords()
labels1 = labeller.labels_2d()
labels2 = labeller.labels_2d()
assert len(coords) == 0
assert len(labels1) == 0
assert len(labels2) == 0
print 'OK'
def find(greyscale_flex, params, mask=None):
'''Find stars on input greyscale flex image.'''
from dials.algorithms.spot_finding.threshold import \
DispersionThresholdStrategy
from dials.model.data import PixelList
from dials.model.data import PixelListLabeller
from dials.array_family import flex
thresholder = DispersionThresholdStrategy(gain=params.gain)
if not mask:
mask = flex.bool(greyscale_flex.size(), True)
mask.reshape(flex.grid(*greyscale_flex.focus()))
threshold_mask = thresholder(greyscale_flex, mask=mask)
plist = PixelList(0, greyscale_flex, threshold_mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(plist)
creator = flex.PixelListShoeboxCreator(pixel_labeller, 0, 0, True,
params.min_size, params.max_size,
False)
shoeboxes = creator.result()
# remove nonsense
size = creator.spot_size()
big = size > params.max_size
small = size < params.min_size
bad = big | small
shoeboxes = shoeboxes.select(~bad)
centroid = shoeboxes.centroid_valid()
intensity = shoeboxes.summed_intensity()
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
stars = flex.reflection_table(observed, shoeboxes)
return stars
def tst_labels_3d(self):
from dials.model.data import PixelList, PixelListLabeller
from scitbx.array_family import flex
size = (500, 500)
sf = 0
labeller = PixelListLabeller()
count = 0
mask_list = []
for i in range(3):
image = flex.random_int_gaussian_distribution(
size[0] * size[1], 100, 5)
mask = flex.random_bool(size[0] * size[1], 0.5)
image.reshape(flex.grid(size))
mask.reshape(flex.grid(size))
pl = PixelList(sf + i, image, mask)
count += len(mask.as_1d().select(mask.as_1d()))
labeller.add(pl)
mask_list.append(mask)
coords = labeller.coords()
labels = labeller.labels_3d()
# Create a map of labels
label_map = flex.int(flex.grid(3, size[0], size[1]))
for c, l in zip(coords, labels):
label_map[c] = l
# Ensure all labels are correct
vi = 0
for k in range(3):
for j in range(size[0]):
for i in range(size[1]):
if mask_list[k][j, i]:
l1 = labels[vi]
if k > 0 and mask_list[k - 1][j, i]:
l2 = label_map[k - 1, j, i]
assert (l2 == l1)
if j > 0 and mask_list[k][j - 1, i]:
l2 = label_map[k, j - 1, i]
assert (l2 == l1)
if i > 0 and mask_list[k][j, i - 1]:
l2 = label_map[k, j, i - 1]
assert (l2 == l1)
vi += 1
# Test passed
print 'OK'
def _find_spots(self, imageset):
"""
Find the spots in the imageset
:param imageset: The imageset to process
:return: The list of spot shoeboxes
"""
from dials.model.data import PixelListLabeller
from dials.util.mp import batch_multi_node_parallel_map
# Change the number of processors if necessary
mp_nproc = self.mp_nproc
mp_njobs = self.mp_njobs
if os.name == "nt" and (mp_nproc > 1 or mp_njobs > 1):
logger.warning(_no_multiprocessing_on_windows)
mp_nproc = 1
mp_njobs = 1
if mp_nproc * mp_njobs > len(imageset):
mp_nproc = min(mp_nproc, len(imageset))
mp_njobs = int(math.ceil(len(imageset) / mp_nproc))
mp_method = self.mp_method
mp_chunksize = self.mp_chunksize
if mp_chunksize == libtbx.Auto:
mp_chunksize = self._compute_chunksize(len(imageset),
mp_njobs * mp_nproc,
self.min_chunksize)
logger.info("Setting chunksize=%i" % mp_chunksize)
len_by_nproc = int(math.floor(len(imageset) / (mp_njobs * mp_nproc)))
if mp_chunksize > len_by_nproc:
mp_chunksize = len_by_nproc
if mp_chunksize == 0:
mp_chunksize = 1
assert mp_nproc > 0, "Invalid number of processors"
assert mp_njobs > 0, "Invalid number of jobs"
assert mp_njobs == 1 or mp_method is not None, "Invalid cluster method"
assert mp_chunksize > 0, "Invalid chunk size"
# The extract pixels function
function = ExtractPixelsFromImage(
imageset=imageset,
threshold_function=self.threshold_function,
mask=self.mask,
max_strong_pixel_fraction=self.max_strong_pixel_fraction,
compute_mean_background=self.compute_mean_background,
region_of_interest=self.region_of_interest,
)
# The indices to iterate over
indices = list(range(len(imageset)))
# Initialise the pixel labeller
num_panels = len(imageset.get_detector())
pixel_labeller = [PixelListLabeller() for p in range(num_panels)]
# Do the processing
logger.info("Extracting strong pixels from images")
if mp_njobs > 1:
logger.info(
" Using %s with %d parallel job(s) and %d processes per node\n"
% (mp_method, mp_njobs, mp_nproc))
else:
logger.info(" Using multiprocessing with %d parallel job(s)\n" %
(mp_nproc))
if mp_nproc > 1 or mp_njobs > 1:
def process_output(result):
for message in result[1]:
logger.log(message.levelno, message.msg)
assert len(pixel_labeller) == len(
result[0].pixel_list), "Inconsistent size"
for plabeller, plist in zip(pixel_labeller,
result[0].pixel_list):
plabeller.add(plist)
result[0].pixel_list = None
batch_multi_node_parallel_map(
func=ExtractSpotsParallelTask(function),
iterable=indices,
nproc=mp_nproc,
njobs=mp_njobs,
cluster_method=mp_method,
chunksize=mp_chunksize,
callback=process_output,
)
else:
for task in indices:
result = function(task)
assert len(pixel_labeller) == len(
result.pixel_list), "Inconsistent size"
for plabeller, plist in zip(pixel_labeller, result.pixel_list):
plabeller.add(plist)
result.pixel_list = None
# Create shoeboxes from pixel list
converter = PixelListToReflectionTable(
self.min_spot_size,
self.max_spot_size,
self.filter_spots,
self.write_hot_pixel_mask,
)
return converter(imageset, pixel_labeller)
def refls_from_sims(panel_imgs,
detector,
beam,
thresh=0,
filter=None,
panel_ids=None,
**kwargs):
"""
This class is for converting the centroids in the noiseless simtbx images
to a multi panel reflection table
TODO: bring up poor documentation and consider asking the dials team
to make a push to beter document for the sake of developers
This function took 3 hours to figure out how to do...
:param panel_imgs: list or 3D array of detector panel simulations
currently supports CSPAD only (194x185 shaped panels)
:param detector: dxtbx detector model of a caspad
:param beam: dxtxb beam model
:param thresh: threshol intensity for labeling centroids
:param filter: optional filter to apply to images before
labeling threshold, typically one of scipy.ndimage's filters
:param pids: panel IDS , else assumes panel_imgs is same length as detector
:param kwargs: kwargs to pass along to the optional filter
:return: a reflection table of spot centroids
"""
from dials.algorithms.spot_finding.factory import FilterRunner
from dials.model.data import PixelListLabeller, PixelList
from dials.algorithms.spot_finding.finder import PixelListToReflectionTable
from cxid9114 import utils
if panel_ids is None:
panel_ids = np.arange(len(detector))
pxlst_labs = []
for i, pid in enumerate(panel_ids):
plab = PixelListLabeller()
img = panel_imgs[i]
if filter is not None:
mask = filter(img, **kwargs) > thresh
else:
mask = img > thresh
img_sz = detector[pid].get_image_size()
flex_img = flex.double(img)
flex_img.reshape(flex.grid(img_sz))
flex_mask = flex.bool(mask)
flex_mask.resize(flex.grid(img_sz))
pl = PixelList(0, flex.double(img), flex.bool(mask))
plab.add(pl)
pxlst_labs.append(plab)
pixlst_to_reftbl = PixelListToReflectionTable(
min_spot_size=1,
max_spot_size=194 * 184,
filter_spots=FilterRunner(), # must use a dummie filter runner!
write_hot_pixel_mask=False)
dblock = utils.datablock_from_numpyarrays(panel_imgs, detector, beam)
iset = dblock.extract_imagesets()[0]
refls = pixlst_to_reftbl(iset, pxlst_labs)[0]
return refls
def _find_spots(self, imageset):
"""
Find the spots in the imageset
:param imageset: The imageset to process
:return: The list of spot shoeboxes
"""
# Change the number of processors if necessary
mp_nproc = self.mp_nproc
mp_njobs = self.mp_njobs
if mp_nproc is libtbx.Auto:
mp_nproc = available_cores()
logger.info(f"Setting nproc={mp_nproc}")
if mp_nproc * mp_njobs > len(imageset):
mp_nproc = min(mp_nproc, len(imageset))
mp_njobs = int(math.ceil(len(imageset) / mp_nproc))
mp_method = self.mp_method
mp_chunksize = self.mp_chunksize
if mp_chunksize is libtbx.Auto:
mp_chunksize = self._compute_chunksize(len(imageset),
mp_njobs * mp_nproc,
self.min_chunksize)
logger.info("Setting chunksize=%i", mp_chunksize)
len_by_nproc = int(math.floor(len(imageset) / (mp_njobs * mp_nproc)))
if mp_chunksize > len_by_nproc:
mp_chunksize = len_by_nproc
if mp_chunksize == 0:
mp_chunksize = 1
assert mp_nproc > 0, "Invalid number of processors"
assert mp_njobs > 0, "Invalid number of jobs"
assert mp_njobs == 1 or mp_method is not None, "Invalid cluster method"
assert mp_chunksize > 0, "Invalid chunk size"
# The extract pixels function
function = ExtractPixelsFromImage(
imageset=imageset,
threshold_function=self.threshold_function,
mask=self.mask,
max_strong_pixel_fraction=self.max_strong_pixel_fraction,
compute_mean_background=self.compute_mean_background,
region_of_interest=self.region_of_interest,
)
# The indices to iterate over
indices = list(range(len(imageset)))
# Initialise the pixel labeller
num_panels = len(imageset.get_detector())
pixel_labeller = [PixelListLabeller() for p in range(num_panels)]
# Do the processing
logger.info("Extracting strong pixels from images")
if mp_njobs > 1:
logger.info(
" Using %s with %d parallel job(s) and %d processes per node\n",
mp_method,
mp_njobs,
mp_nproc,
)
else:
logger.info(" Using multiprocessing with %d parallel job(s)\n",
mp_nproc)
if mp_nproc > 1 or mp_njobs > 1:
def process_output(result):
rehandle_cached_records(result[1])
assert len(pixel_labeller) == len(
result[0]), "Inconsistent size"
for plabeller, plist in zip(pixel_labeller, result[0]):
plabeller.add(plist)
batch_multi_node_parallel_map(
func=ExtractSpotsParallelTask(function),
iterable=indices,
nproc=mp_nproc,
njobs=mp_njobs,
cluster_method=mp_method,
chunksize=mp_chunksize,
callback=process_output,
)
else:
for task in indices:
result = function(task)
assert len(pixel_labeller) == len(result), "Inconsistent size"
for plabeller, plist in zip(pixel_labeller, result):
plabeller.add(plist)
result.clear()
# Create shoeboxes from pixel list
return pixel_list_to_reflection_table(
imageset,
pixel_labeller,
filter_spots=self.filter_spots,
min_spot_size=self.min_spot_size,
max_spot_size=self.max_spot_size,
write_hot_pixel_mask=self.write_hot_pixel_mask,
)
def _find_spots(self, imageset):
'''
Find the spots in the imageset
:param imageset: The imageset to process
:return: The list of spot shoeboxes
'''
from dials.array_family import flex
from dxtbx.imageset import ImageSweep
from dials.model.data import PixelListLabeller
from dials.util.mp import batch_multi_node_parallel_map
from math import floor, ceil
import platform
# Change the number of processors if necessary
mp_nproc = self.mp_nproc
mp_njobs = self.mp_njobs
if (mp_nproc > 1 or mp_njobs > 1) and platform.system(
) == "Windows": # platform.system() forks which is bad for MPI, so don't use it unless nproc > 1
logger.warn("")
logger.warn("*" * 80)
logger.warn(
"Multiprocessing is not available on windows. Setting nproc = 1, njobs = 1"
)
logger.warn("*" * 80)
logger.warn("")
mp_nproc = 1
mp_njobs = 1
if mp_nproc * mp_njobs > len(imageset):
mp_nproc = min(mp_nproc, len(imageset))
mp_njobs = int(ceil(len(imageset) / mp_nproc))
mp_method = self.mp_method
mp_chunksize = self.mp_chunksize
import libtbx
if mp_chunksize == libtbx.Auto:
mp_chunksize = self._compute_chunksize(len(imageset),
mp_njobs * mp_nproc,
self.min_chunksize)
logger.info("Setting chunksize=%i" % mp_chunksize)
len_by_nproc = int(floor(len(imageset) / (mp_njobs * mp_nproc)))
if mp_chunksize > len_by_nproc:
mp_chunksize = len_by_nproc
if mp_chunksize == 0:
mp_chunksize = 1
assert mp_nproc > 0, "Invalid number of processors"
assert mp_njobs > 0, "Invalid number of jobs"
assert mp_njobs == 1 or mp_method is not None, "Invalid cluster method"
assert mp_chunksize > 0, "Invalid chunk size"
# The extract pixels function
function = ExtractPixelsFromImage(
imageset=imageset,
threshold_function=self.threshold_function,
mask=self.mask,
max_strong_pixel_fraction=self.max_strong_pixel_fraction,
compute_mean_background=self.compute_mean_background,
region_of_interest=self.region_of_interest)
# The indices to iterate over
indices = list(range(len(imageset)))
# Initialise the pixel labeller
num_panels = len(imageset.get_detector())
pixel_labeller = [PixelListLabeller() for p in range(num_panels)]
# Do the processing
logger.info('Extracting strong pixels from images')
if mp_njobs > 1:
logger.info(
' Using %s with %d parallel job(s) and %d processes per node\n'
% (mp_method, mp_njobs, mp_nproc))
else:
logger.info(' Using multiprocessing with %d parallel job(s)\n' %
(mp_nproc))
if mp_nproc > 1 or mp_njobs > 1:
def process_output(result):
for message in result[1]:
logger.log(message.levelno, message.msg)
assert len(pixel_labeller) == len(
result[0].pixel_list), "Inconsistent size"
for plabeller, plist in zip(pixel_labeller,
result[0].pixel_list):
plabeller.add(plist)
result[0].pixel_list = None
batch_multi_node_parallel_map(
func=ExtractSpotsParallelTask(function),
iterable=indices,
nproc=mp_nproc,
njobs=mp_njobs,
cluster_method=mp_method,
chunksize=mp_chunksize,
callback=process_output)
else:
for task in indices:
result = function(task)
assert len(pixel_labeller) == len(
result.pixel_list), "Inconsistent size"
for plabeller, plist in zip(pixel_labeller, result.pixel_list):
plabeller.add(plist)
result.pixel_list = None
# Create shoeboxes from pixel list
converter = PixelListToReflectionTable(self.min_spot_size,
self.max_spot_size,
self.filter_spots,
self.write_hot_pixel_mask)
return converter(imageset, pixel_labeller)