def tst_with_no_points(self):
from dials.model.data import PixelList, PixelListLabeller
from scitbx.array_family import flex
size = (500, 500)
sf = 0
labeller = PixelListLabeller()
count = 0
mask_list = []
for i in range(3):
image = flex.random_int_gaussian_distribution(
size[0] * size[1], 100, 5)
mask = flex.bool(size[0] * size[0], False)
image.reshape(flex.grid(size))
mask.reshape(flex.grid(size))
pl = PixelList(sf + i, image, mask)
count += len(mask.as_1d().select(mask.as_1d()))
labeller.add(pl)
mask_list.append(mask)
coords = labeller.coords()
labels1 = labeller.labels_2d()
labels2 = labeller.labels_2d()
assert len(coords) == 0
assert len(labels1) == 0
assert len(labels2) == 0
print 'OK'
def refls_from_sims(panel_imgs, detector, beam, thresh=0, filter=None, **kwargs ):
"""
gets a reflection table from simulated panels
:param panel_imgs: list of numpy arrays , each array is a simulated panel image
:param detector: dxtbx detector, can have multiple nodes
:param beam: dxtbx beam
:param thresh: threshol
:param filter:
:param kwargs:
:return: reflection table, with id coloumn set to 0
"""
pxlst_labs = []
for i in range(len(detector)):
plab = PixelListLabeller()
img = panel_imgs[i]
if filter is not None:
mask = filter(img, **kwargs) > thresh
else:
mask = img > thresh
pl = PixelList(0, flex.double(img), flex.bool(mask))
plab.add(pl)
pxlst_labs.append( plab)
pixlst_to_reftbl = PixelListToReflectionTable(
min_spot_size=1,
max_spot_size=194*184,
filter_spots=FilterRunner(),
write_hot_pixel_mask=False)
dblock = datablock_from_numpyarrays( panel_imgs, detector, beam)
iset = dblock.extract_imagesets()[0]
refls = pixlst_to_reftbl(iset, pxlst_labs)[0]
refls['id'] = flex.int(len(refls),0)
return refls
def find_spots(self, min_spot_size=2, max_spot_size=100):
from dials.algorithms.spot_finding.threshold import XDSThresholdStrategy
from dials.model.data import PixelList
from dials.model.data import PixelListLabeller
image = self.raw_data
mask = self.imageset.get_mask(0)[0]
threshold_image = XDSThresholdStrategy()
threshold_mask = threshold_image(image, mask)
plist = PixelList(0, image, threshold_mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(plist)
creator = flex.PixelListShoeboxCreator(pixel_labeller, 0, 0, True,
min_spot_size, max_spot_size,
False)
shoeboxes = creator.result()
# turns out we need to manually filter the list to get a sensible answer
size = creator.spot_size()
big = size > max_spot_size
small = size < min_spot_size
bad = big | small
shoeboxes = shoeboxes.select(~bad)
centroid = shoeboxes.centroid_valid()
intensity = shoeboxes.summed_intensity()
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
reflections = flex.reflection_table(observed, shoeboxes)
return reflections
def find_spots(image, mask, min_spot_size=1, max_spot_size=1000):
from dials.algorithms.spot_finding.threshold import XDSThresholdStrategy
from dials.model.data import PixelList
from dials.model.data import PixelListLabeller
threshold_image = XDSThresholdStrategy()
threshold_mask = threshold_image(image, mask)
plist = PixelList(0, image, threshold_mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(plist)
creator = flex.PixelListShoeboxCreator(
pixel_labeller,
0, # panel
0, # zrange
True, # twod
min_spot_size, # min_pixels
max_spot_size, # max_pixels
False,
)
shoeboxes = creator.result()
centroid = shoeboxes.centroid_valid()
intensity = shoeboxes.summed_intensity()
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
return flex.reflection_table(observed, shoeboxes)
def find_spots(self, min_spot_size=2, max_spot_size=100):
"""
Find the strong spots on the image
"""
from dials.algorithms.spot_finding.threshold import DispersionThresholdStrategy
from dials.model.data import PixelList
from dials.model.data import PixelListLabeller
from dials.array_family import flex
print("")
print("-" * 80)
print(" Finding strong spots")
print("-" * 80)
print("")
# Instantiate the threshold function
threshold = DispersionThresholdStrategy()
# Get the raw data and image mask
image = self.experiment.imageset.get_raw_data(0)[0]
mask = self.experiment.imageset.get_mask(0)[0]
# Threshold the image and create the pixel labeller
threshold_mask = threshold(image, mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(PixelList(0, image, threshold_mask))
# Create the shoebox list from the pixel list
creator = flex.PixelListShoeboxCreator(
pixel_labeller,
0, # Panel number
0, # Z start
True, # 2D
self.params.spot_finding.min_spot_size, # Min Pixels
self.params.spot_finding.max_spot_size, # Max Pixels
False,
) # Find hot pixels
shoeboxes = creator.result()
# Filter the list to remove large and small spots
size = creator.spot_size()
large = size > self.params.spot_finding.max_spot_size
small = size < self.params.spot_finding.min_spot_size
bad = large | small
shoeboxes = shoeboxes.select(~bad)
print("Discarding %d spots with < %d pixels" %
(small.count(True), self.params.spot_finding.min_spot_size))
print("Discarding %d spots with > %d pixels" %
(large.count(True), self.params.spot_finding.max_spot_size))
# Extract the strong spot information
centroid = shoeboxes.centroid_valid()
intensity = shoeboxes.summed_intensity()
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
# Create the reflection list
self.reflections = flex.reflection_table(observed, shoeboxes)
print("Using %d strong spots" % len(self.reflections))
def tst_pickle(self):
from dials.model.data import PixelList
from scitbx.array_family import flex
from random import randint
size = (100, 100)
sf = 10
image = flex.double(flex.grid(size))
mask = flex.bool(flex.grid(size))
for i in range(len(image)):
image[i] = randint(0, 100)
mask[i] = bool(randint(0, 1))
pl = PixelList(sf, image, mask)
assert (pl.size() == size)
assert (pl.frame() == sf)
import cPickle as pickle
obj = pickle.dumps(pl)
pl2 = pickle.loads(obj)
assert (pl2.size() == size)
assert (pl2.frame() == sf)
assert (len(pl2) == len(pl))
assert (pl2.index().all_eq(pl.index()))
assert (pl2.value().all_eq(pl.value()))
print 'OK'
def tst_labels_3d(self):
from dials.model.data import PixelList, PixelListLabeller
from scitbx.array_family import flex
size = (500, 500)
sf = 0
labeller = PixelListLabeller()
count = 0
mask_list = []
for i in range(3):
image = flex.random_int_gaussian_distribution(
size[0] * size[1], 100, 5)
mask = flex.random_bool(size[0] * size[1], 0.5)
image.reshape(flex.grid(size))
mask.reshape(flex.grid(size))
pl = PixelList(sf + i, image, mask)
count += len(mask.as_1d().select(mask.as_1d()))
labeller.add(pl)
mask_list.append(mask)
coords = labeller.coords()
labels = labeller.labels_3d()
# Create a map of labels
label_map = flex.int(flex.grid(3, size[0], size[1]))
for c, l in zip(coords, labels):
label_map[c] = l
# Ensure all labels are correct
vi = 0
for k in range(3):
for j in range(size[0]):
for i in range(size[1]):
if mask_list[k][j, i]:
l1 = labels[vi]
if k > 0 and mask_list[k - 1][j, i]:
l2 = label_map[k - 1, j, i]
assert (l2 == l1)
if j > 0 and mask_list[k][j - 1, i]:
l2 = label_map[k, j - 1, i]
assert (l2 == l1)
if i > 0 and mask_list[k][j, i - 1]:
l2 = label_map[k, j, i - 1]
assert (l2 == l1)
vi += 1
# Test passed
print 'OK'
def test_add_image():
from dials.model.data import PixelList, PixelListLabeller
from scitbx.array_family import flex
size = (2000, 2000)
sf = 10
labeller = PixelListLabeller()
count = 0
for i in range(3):
image = flex.random_int_gaussian_distribution(size[0] * size[1], 100,
5)
mask = flex.random_bool(size[0] * size[1], 0.5)
image.reshape(flex.grid(size))
mask.reshape(flex.grid(size))
pl = PixelList(sf + i, image, mask)
count += len(mask.as_1d().select(mask.as_1d()))
labeller.add(pl)
assert len(labeller.values()) == count
def find_spots(image):
from dials.algorithms.spot_finding.threshold import DispersionThresholdStrategy
from dials.model.data import PixelList
from dials.model.data import PixelListLabeller
thresholder = DispersionThresholdStrategy(gain=1)
mask = image.as_1d() >= 0 # flex.bool(image.size(), True)
mask.reshape(flex.grid(*image.focus()))
threshold_mask = thresholder(image, mask=mask)
plist = PixelList(0, image, threshold_mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(plist)
creator = flex.PixelListShoeboxCreator(pixel_labeller, 0, 0, True, 2, 100,
False)
shoeboxes = creator.result()
return shoeboxes
def image_to_shoeboxes(image):
"""For a given image, find spots 2 - 100 pixels im size, assuming a gain
of 1, return the list of spot shoeboxes. Also assumes valid intensities in
range 0...N."""
thresholder = DispersionThresholdStrategy(gain=1)
mask = image.as_1d() >= 0
mask.reshape(flex.grid(*image.focus()))
threshold_mask = thresholder(image, mask=mask)
plist = PixelList(0, image, threshold_mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(plist)
creator = flex.PixelListShoeboxCreator(pixel_labeller, 0, 0, True, 2, 100,
False)
shoeboxes = creator.result()
return shoeboxes
def find(greyscale_flex, params, mask=None):
'''Find stars on input greyscale flex image.'''
from dials.algorithms.spot_finding.threshold import \
DispersionThresholdStrategy
from dials.model.data import PixelList
from dials.model.data import PixelListLabeller
from dials.array_family import flex
thresholder = DispersionThresholdStrategy(gain=params.gain)
if not mask:
mask = flex.bool(greyscale_flex.size(), True)
mask.reshape(flex.grid(*greyscale_flex.focus()))
threshold_mask = thresholder(greyscale_flex, mask=mask)
plist = PixelList(0, greyscale_flex, threshold_mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(plist)
creator = flex.PixelListShoeboxCreator(pixel_labeller, 0, 0, True,
params.min_size, params.max_size,
False)
shoeboxes = creator.result()
# remove nonsense
size = creator.spot_size()
big = size > params.max_size
small = size < params.min_size
bad = big | small
shoeboxes = shoeboxes.select(~bad)
centroid = shoeboxes.centroid_valid()
intensity = shoeboxes.summed_intensity()
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
stars = flex.reflection_table(observed, shoeboxes)
return stars
def __call__(self, index):
"""
Extract strong pixels from an image
:param index: The index of the image
"""
from dials.model.data import PixelList
from dxtbx.imageset import ImageSequence
# Parallel reading of HDF5 from the same handle is not allowed. Python
# multiprocessing is a bit messed up and used fork on linux so need to
# close and reopen file.
if self.first:
if self.imageset.reader().is_single_file_reader():
self.imageset.reader().nullify_format_instance()
self.first = False
# Get the frame number
if isinstance(self.imageset, ImageSequence):
frame = self.imageset.get_array_range()[0] + index
else:
ind = self.imageset.indices()
if len(ind) > 1:
assert all(i1 + 1 == i2
for i1, i2 in zip(ind[0:-1], ind[1:-1]))
frame = ind[index]
# Create the list of pixel lists
pixel_list = []
# Get the image and mask
image = self.imageset.get_corrected_data(index)
mask = self.imageset.get_mask(index)
# Set the mask
if self.mask is not None:
assert len(self.mask) == len(mask)
mask = tuple(m1 & m2 for m1, m2 in zip(mask, self.mask))
logger.debug("Number of masked pixels for image %i: %i" %
(index, sum(m.count(False) for m in mask)))
# Add the images to the pixel lists
num_strong = 0
average_background = 0
for im, mk in zip(image, mask):
if self.region_of_interest is not None:
x0, x1, y0, y1 = self.region_of_interest
height, width = im.all()
assert x0 < x1, "x0 < x1"
assert y0 < y1, "y0 < y1"
assert x0 >= 0, "x0 >= 0"
assert y0 >= 0, "y0 >= 0"
assert x1 <= width, "x1 <= width"
assert y1 <= height, "y1 <= height"
im_roi = im[y0:y1, x0:x1]
mk_roi = mk[y0:y1, x0:x1]
tm_roi = self.threshold_function.compute_threshold(
im_roi, mk_roi)
threshold_mask = flex.bool(im.accessor(), False)
threshold_mask[y0:y1, x0:x1] = tm_roi
else:
threshold_mask = self.threshold_function.compute_threshold(
im, mk)
# Add the pixel list
plist = PixelList(frame, im, threshold_mask)
pixel_list.append(plist)
# Get average background
if self.compute_mean_background:
background = im.as_1d().select((mk & ~threshold_mask).as_1d())
average_background += flex.mean(background)
# Add to the spot count
num_strong += len(plist)
# Make average background
average_background /= len(image)
# Check total number of strong pixels
if self.max_strong_pixel_fraction < 1:
num_image = 0
for im in image:
num_image += len(im)
max_strong = int(
math.ceil(self.max_strong_pixel_fraction * num_image))
if num_strong > max_strong:
raise RuntimeError("""
The number of strong pixels found (%d) is greater than the
maximum allowed (%d). Try changing spot finding parameters
""" % (num_strong, max_strong))
# Print some info
if self.compute_mean_background:
logger.info(
"Found %d strong pixels on image %d with average background %f"
% (num_strong, frame + 1, average_background))
else:
logger.info("Found %d strong pixels on image %d" %
(num_strong, frame + 1))
# Return the result
return Result(pixel_list)
def refls_from_sims(panel_imgs,
detector,
beam,
thresh=0,
filter=None,
panel_ids=None,
**kwargs):
"""
This class is for converting the centroids in the noiseless simtbx images
to a multi panel reflection table
TODO: bring up poor documentation and consider asking the dials team
to make a push to beter document for the sake of developers
This function took 3 hours to figure out how to do...
:param panel_imgs: list or 3D array of detector panel simulations
currently supports CSPAD only (194x185 shaped panels)
:param detector: dxtbx detector model of a caspad
:param beam: dxtxb beam model
:param thresh: threshol intensity for labeling centroids
:param filter: optional filter to apply to images before
labeling threshold, typically one of scipy.ndimage's filters
:param pids: panel IDS , else assumes panel_imgs is same length as detector
:param kwargs: kwargs to pass along to the optional filter
:return: a reflection table of spot centroids
"""
from dials.algorithms.spot_finding.factory import FilterRunner
from dials.model.data import PixelListLabeller, PixelList
from dials.algorithms.spot_finding.finder import PixelListToReflectionTable
from cxid9114 import utils
if panel_ids is None:
panel_ids = np.arange(len(detector))
pxlst_labs = []
for i, pid in enumerate(panel_ids):
plab = PixelListLabeller()
img = panel_imgs[i]
if filter is not None:
mask = filter(img, **kwargs) > thresh
else:
mask = img > thresh
img_sz = detector[pid].get_image_size()
flex_img = flex.double(img)
flex_img.reshape(flex.grid(img_sz))
flex_mask = flex.bool(mask)
flex_mask.resize(flex.grid(img_sz))
pl = PixelList(0, flex.double(img), flex.bool(mask))
plab.add(pl)
pxlst_labs.append(plab)
pixlst_to_reftbl = PixelListToReflectionTable(
min_spot_size=1,
max_spot_size=194 * 184,
filter_spots=FilterRunner(), # must use a dummie filter runner!
write_hot_pixel_mask=False)
dblock = utils.datablock_from_numpyarrays(panel_imgs, detector, beam)
iset = dblock.extract_imagesets()[0]
refls = pixlst_to_reftbl(iset, pxlst_labs)[0]
return refls
def __call__(self, index):
"""
Extract strong pixels from an image
:param index: The index of the image
"""
# Get the frame number
if isinstance(self.imageset, ImageSequence):
frame = self.imageset.get_array_range()[0] + index
else:
ind = self.imageset.indices()
if len(ind) > 1:
assert all(i1 + 1 == i2
for i1, i2 in zip(ind[0:-1], ind[1:-1]))
frame = ind[index]
# Create the list of pixel lists
pixel_list = []
# Get the image and mask
image = self.imageset.get_corrected_data(index)
mask = self.imageset.get_mask(index)
# Set the mask
if self.mask is not None:
assert len(self.mask) == len(mask)
mask = tuple(m1 & m2 for m1, m2 in zip(mask, self.mask))
logger.debug(
"Number of masked pixels for image %i: %i",
index,
sum(m.count(False) for m in mask),
)
# Add the images to the pixel lists
num_strong = 0
average_background = 0
for im, mk in zip(image, mask):
if self.region_of_interest is not None:
x0, x1, y0, y1 = self.region_of_interest
height, width = im.all()
assert x0 < x1, "x0 < x1"
assert y0 < y1, "y0 < y1"
assert x0 >= 0, "x0 >= 0"
assert y0 >= 0, "y0 >= 0"
assert x1 <= width, "x1 <= width"
assert y1 <= height, "y1 <= height"
im_roi = im[y0:y1, x0:x1]
mk_roi = mk[y0:y1, x0:x1]
tm_roi = self.threshold_function.compute_threshold(
im_roi, mk_roi)
threshold_mask = flex.bool(im.accessor(), False)
threshold_mask[y0:y1, x0:x1] = tm_roi
else:
threshold_mask = self.threshold_function.compute_threshold(
im, mk)
# Add the pixel list
plist = PixelList(frame, im, threshold_mask)
pixel_list.append(plist)
# Get average background
if self.compute_mean_background:
background = im.as_1d().select((mk & ~threshold_mask).as_1d())
average_background += flex.mean(background)
# Add to the spot count
num_strong += len(plist)
# Make average background
average_background /= len(image)
# Check total number of strong pixels
if self.max_strong_pixel_fraction < 1:
num_image = 0
for im in image:
num_image += len(im)
max_strong = int(
math.ceil(self.max_strong_pixel_fraction * num_image))
if num_strong > max_strong:
raise RuntimeError(f"""
The number of strong pixels found ({num_strong}) is greater than the
maximum allowed ({max_strong}). Try changing spot finding parameters
""")
# Print some info
if self.compute_mean_background:
logger.info(
"Found %d strong pixels on image %d with average background %f",
num_strong,
frame + 1,
average_background,
)
else:
logger.info("Found %d strong pixels on image %d", num_strong,
frame + 1)
# Return the result
return pixel_list
#size2 = (maxz2 - minz2 + 1, maxy2 - miny2 + 1, maxx2 - minx2 + 1) #from scitbx.array_family import flex #a1 = flex.int(flex.grid(size1), 0) #a2 = flex.int(flex.grid(size2), 0) #for c in c3: # z = c[0] - minz1 # y = c[1] - miny1 # x = c[2] - minx1 # a1[z, y, x] = 1 # #for c in c4: # z = c[0] - minz2 # y = c[1] - miny2 # x = c[2] - minx2 # a2[z, y, x] = 1 # if z == 10: # print c # #a1 = a1.as_numpy_array() #a2 = a2.as_numpy_array() #print a1 #print a2 #print a1.shape, a2.shape from dials.model.data import PixelList pl = PixelList((2527, 2463), (0, 500), v2, c2) print max(pl.labels_3d())
# from scitbx.array_family import flex # a1 = flex.int(flex.grid(size1), 0) # a2 = flex.int(flex.grid(size2), 0) # for c in c3: # z = c[0] - minz1 # y = c[1] - miny1 # x = c[2] - minx1 # a1[z, y, x] = 1 # # for c in c4: # z = c[0] - minz2 # y = c[1] - miny2 # x = c[2] - minx2 # a2[z, y, x] = 1 # if z == 10: # print c # # a1 = a1.as_numpy_array() # a2 = a2.as_numpy_array() # print a1 # print a2 # print a1.shape, a2.shape from dials.model.data import PixelList pl = PixelList((2527, 2463), (0, 500), v2, c2) print(max(pl.labels_3d()))
