def tst_count_mask_values(self):
from dials.model.data import Shoebox
from random import randint, sample
from dials.array_family import flex
shoebox = flex.shoebox(10)
num = flex.int(10)
value = (1 << 2)
for i in range(10):
x0 = randint(0, 90)
y0 = randint(0, 90)
z0 = randint(0, 90)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))
shoebox[i].allocate()
maxnum = len(shoebox[i].mask)
num[i] = randint(1, maxnum)
indices = sample(list(range(maxnum)), num[i])
for j in indices:
shoebox[i].mask[j] = value
assert(shoebox.count_mask_values(value) == num)
# Test passed
print 'OK'
def __init__(self, data, jobs, npanels, callback):
from dials.array_family import flex
# Allocate shoeboxes
data["shoebox"] = flex.shoebox(data["panel"], data["bbox"])
data["shoebox"].allocate()
# Set the callback
self._callback = callback
# Initialise the base class
super(IntegrationTask3DMultiExecutor, self).__init__(data, jobs, npanels)
def run(self):
from dials.model.serialize import load
from dials.algorithms import shoebox
from dxtbx.serialize.load import crystal as load_crystal
from dials.array_family import flex
# Load the sweep and crystal
self.sweep = load.sweep(self.sweep_filename)
self.crystal = load_crystal(self.crystal_filename)
# Get the reflections and overlaps
reflections, adjacency_list = self.predict_reflections()
reflections['shoebox'] = flex.shoebox(
reflections['panel'],
reflections['bbox'])
reflections['shoebox'].allocate_with_value(shoebox.MaskCode.Valid)
# If the adjacency list is given, then create the reflection mask
assert(len(self.detector) == 1)
image_size = self.detector[0].get_image_size()
shoeboxes = reflections['shoebox']
coords = reflections['xyzcal.px']
shoebox_masker = shoebox.MaskOverlapping()
shoebox_masker(shoeboxes, coords, adjacency_list)
# Loop through all edges
overlapping = []
for e in adjacency_list.edges():
v1, v2 = adjacency_list.source(e), adjacency_list.target(e)
overlapping.append(v1)
overlapping.append(v2)
# Ensure elements are unique
overlapping = set(overlapping)
# Ensure we have some overlaps
assert(len(overlapping) > 0)
# Get all non-overlapping reflections
all_r = set(range(len(reflections)))
non_overlapping = all_r.difference(overlapping)
# Run the tests
self.tst_non_overlapping(reflections, non_overlapping,
self.detector[0].get_image_size())
self.tst_overlapping(reflections, overlapping, adjacency_list,
image_size)
def __call__(self, imageset, pixel_labeller):
'''
Convert the pixel list to shoeboxes
'''
from dxtbx.imageset import ImageSweep
from dials.array_family import flex
# Extract the pixel lists into a list of reflections
shoeboxes = flex.shoebox()
spotsizes = flex.size_t()
hotpixels = tuple(flex.size_t() for i in range(len(imageset.get_detector())))
if isinstance(imageset, ImageSweep):
twod = False
else:
twod = True
for i, (p, hp) in enumerate(zip(pixel_labeller, hotpixels)):
if p.num_pixels() > 0:
creator = flex.PixelListShoeboxCreator(
p,
i, # panel
0, # zrange
twod, # twod
self.min_spot_size, # min_pixels
self.max_spot_size, # max_pixels
self.write_hot_pixel_mask)
shoeboxes.extend(creator.result())
spotsizes.extend(creator.spot_size())
hp.extend(creator.hot_pixels())
logger.info('')
logger.info('Extracted {0} spots'.format(len(shoeboxes)))
# Get the unallocated spots and print some info
selection = shoeboxes.is_allocated()
shoeboxes = shoeboxes.select(selection)
ntoosmall = (spotsizes < self.min_spot_size).count(True)
ntoolarge = (spotsizes > self.max_spot_size).count(True)
assert ntoosmall + ntoolarge == selection.count(False)
logger.info('Removed %d spots with size < %d pixels' % (
ntoosmall,
self.min_spot_size))
logger.info('Removed %d spots with size > %d pixels' % (
ntoolarge,
self.max_spot_size))
# Return the shoeboxes
return shoeboxes, hotpixels
def tst_is_bbox_within_image_volume(self):
from dials.model.data import Shoebox
from dials.array_family import flex
isize = (1000, 1000)
srange = (0, 100)
shoebox = flex.shoebox(7)
shoebox[0] = Shoebox((10, 20, 10, 20, 10, 20))
shoebox[1] = Shoebox((-10, 20, 10, 20, 10, 20))
shoebox[2] = Shoebox((10, 20, -10, 20, 10, 20))
shoebox[3] = Shoebox((10, 20, 10, 20, -10, 20))
shoebox[4] = Shoebox((10, 1020, 10, 20, 10, 20))
shoebox[5] = Shoebox((10, 20, 10, 1020, 10, 20))
shoebox[6] = Shoebox((10, 20, 10, 20, 10, 1020))
assert(shoebox.is_bbox_within_image_volume(isize, srange) ==
flex.bool([True, False, False, False, False, False, False]))
# Test passed
print 'OK'
def tst_does_bbox_contain_bad_pixels(self):
from dials.array_family import flex
from dials.model.data import Shoebox
from random import randint
mask = flex.bool(flex.grid(100, 100), True)
for j in range(100):
for i in range(40, 60):
mask[j,i] = False
mask[i,j] = False
shoebox = flex.shoebox(1000)
res = flex.bool(1000)
for i in range(1000):
x0 = randint(0, 90)
y0 = randint(0, 90)
z0 = randint(0, 90)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))
res2 = False
if x0 >= 40 and x0 < 60:
res2 = True
if x1 > 40 and x1 <= 60:
res2 = True
if y0 >= 40 and y0 < 60:
res2 = True
if y1 > 40 and y1 <= 60:
res2 = True
res[i] = res2
assert(shoebox.does_bbox_contain_bad_pixels(mask) == res)
# Test passed
print 'OK'
def __init__(self):
from dials.algorithms.integration.profile import GridSampler2D
from dials.array_family import flex
from random import randint, uniform
# Number of reflections
nrefl = 1000
# Size of the images
width = 1000
height = 1000
# Create the grid
self.grid = GridSampler2D((width, height), (5,5))
# Create the list of xyz and bboxes
self.xyz = flex.vec3_double(nrefl)
self.bbox = flex.int6(nrefl)
self.panel = flex.size_t(nrefl, 0)
for i in range(nrefl):
x0 = randint(0,width-10)
x1 = x0 + randint(3,10)
y0 = randint(0,height-10)
y1 = y0 + randint(3,10)
z0 = randint(0,10)
z1 = z0 + randint(1,10)
b = x0, x1, y0, y1, z0, z1
c = (x1 + x0) / 2, (y1 + y0) / 2, (z1 + z0) / 2
self.xyz[i] = c
self.bbox[i] = b
# Create the array of shoeboxes
self.sbox = flex.shoebox(self.panel, self.bbox)
self.sbox.allocate()
for i in range(len(self.sbox)):
data = self.sbox[i].data
for j in range(len(data)):
data[j] = uniform(0, 100)
def tst_bounding_boxes(self):
from dials.model.data import Shoebox
from random import randint
from dials.array_family import flex
shoebox = flex.shoebox(10)
bbox = flex.int6(10)
for i in range(10):
x0 = randint(0, 90)
y0 = randint(0, 90)
z0 = randint(0, 90)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
bbox[i] = (x0, x1, y0, y1, z0, z1)
shoebox[i] = Shoebox(bbox[i])
bbox2 = shoebox.bounding_boxes()
for i in range(10):
assert(bbox2[i] == bbox[i])
# Test passed
print 'OK'
def tst_consistent(self):
from random import randint
from dials.model.data import Shoebox
from dials.array_family import flex
shoebox = flex.shoebox(10)
for i in range(10):
x0 = randint(0, 1000)
y0 = randint(0, 1000)
z0 = randint(0, 1000)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))
assert(shoebox.is_consistent() == flex.bool(10, False))
for i in range(10):
shoebox[i].allocate()
assert(shoebox.is_consistent() == flex.bool(10, True))
for i in [0, 2, 4, 6, 8]:
shoebox[i].data.resize(flex.grid(10, 10, 10))
assert(shoebox.is_consistent() == flex.bool([False, True] * 5))
for i in range(10):
shoebox[i].deallocate()
assert(shoebox.is_consistent() == flex.bool(10, False))
# Test passed
print 'OK'
def __call__(self):
'''
Do the processing.
:return: The processed data
'''
from dials.array_family import flex
from time import time
from dials.model.data import make_image
from libtbx.introspection import machine_memory_info
# Get the start time
start_time = time()
# Set the global process ID
job.index = self.index
# Check all reflections have same imageset and get it
exp_id = list(set(self.reflections['id']))
imageset = self.experiments[exp_id[0]].imageset
for i in exp_id[1:]:
assert self.experiments[i].imageset == imageset, "Task can only handle 1 imageset"
# Get the sub imageset
frame00, frame01 = self.job
try:
frame10, frame11 = imageset.get_array_range()
except Exception:
frame10, frame11 = (0, len(imageset))
try:
assert frame00 < frame01
assert frame10 < frame11
assert frame00 >= frame10
assert frame01 <= frame11
index0 = frame00 - frame10
index1 = index0 + (frame01 - frame00)
assert index0 < index1
assert index0 >= 0
assert index1 <= len(imageset)
imageset = imageset[index0:index1]
except Exception:
raise RuntimeError('Programmer Error: bad array range')
try:
frame0, frame1 = imageset.get_array_range()
except Exception:
frame0, frame1 = (0, len(imageset))
# Initlize the executor
self.executor.initialize(frame0, frame1, self.reflections)
# Set the shoeboxes (dont't allocate)
self.reflections['shoebox'] = flex.shoebox(
self.reflections['panel'],
self.reflections['bbox'],
allocate=False,
flatten=self.params.shoebox.flatten)
# Create the processor
processor = ShoeboxProcessor(
self.reflections,
len(imageset.get_detector()),
frame0,
frame1,
self.params.debug.output)
# Compute percentage of max available. The function is not portable to
# windows so need to add a check if the function fails. On windows no
# warning will be printed
memory_info = machine_memory_info()
total_memory = memory_info.memory_total()
sbox_memory = processor.compute_max_memory_usage()
if total_memory is not None:
assert total_memory > 0, "Your system appears to have no memory!"
assert self.params.block.max_memory_usage > 0.0, "maximum memory usage must be > 0"
assert self.params.block.max_memory_usage <= 1.0, "maximum memory usage must be <= 1"
limit_memory = total_memory * self.params.block.max_memory_usage
if sbox_memory > limit_memory:
raise RuntimeError('''
There was a problem allocating memory for shoeboxes. Possible solutions
include increasing the percentage of memory allowed for shoeboxes or
decreasing the block size. This could also be caused by a highly mosaic
crystal model - is your crystal really this mosaic?
Total system memory: %g GB
Limit shoebox memory: %g GB
Required shoebox memory: %g GB
''' % (total_memory/1e9, limit_memory/1e9, sbox_memory/1e9))
else:
logger.info(' Memory usage:')
logger.info(' Total system memory: %g GB' % (total_memory/1e9))
logger.info(' Limit shoebox memory: %g GB' % (limit_memory/1e9))
logger.info(' Required shoebox memory: %g GB' % (sbox_memory/1e9))
logger.info('')
# Loop through the imageset, extract pixels and process reflections
read_time = 0.0
for i in range(len(imageset)):
st = time()
image = imageset.get_corrected_data(i)
mask = imageset.get_mask(i)
if self.params.lookup.mask is not None:
assert len(mask) == len(self.params.lookup.mask), \
"Mask/Image are incorrect size %d %d" % (
len(mask),
len(self.params.lookup.mask))
mask = tuple(m1 & m2 for m1, m2 in zip(self.params.lookup.mask, mask))
read_time += time() - st
processor.next(make_image(image, mask), self.executor)
del image
del mask
assert processor.finished(), "Data processor is not finished"
# Optionally save the shoeboxes
if self.params.debug.output and self.params.debug.separate_files:
output = self.reflections
if self.params.debug.select is not None:
output = output.select(self.params.debug.select(output))
if self.params.debug.split_experiments:
output = output.split_by_experiment_id()
for table in output:
i = table['id'][0]
table.as_pickle('shoeboxes_%d_%d.pickle' % (self.index, i))
else:
output.as_pickle('shoeboxes_%d.pickle' % self.index)
# Delete the shoeboxes
if self.params.debug.separate_files or not self.params.debug.output:
del self.reflections['shoebox']
# Finalize the executor
self.executor.finalize()
# Return the result
result = Result(self.index, self.reflections, self.executor.data())
result.read_time = read_time
result.extract_time = processor.extract_time()
result.process_time = processor.process_time()
result.total_time = time() - start_time
return result
reflection_lookup[j].append(i)
width, height = experiments[0].detector[0].get_image_size()
sum_background = flex.double(flex.grid(height, width), 0)
sum_sq_background = flex.double(flex.grid(height, width), 0)
count = flex.int(flex.grid(height, width), 0)
# Loop through all images
print "START"
for frame in range(len(imageset)):
# Get the subset of reflections on this image and compute the mask
subset = reflections.select(flex.size_t(reflection_lookup[frame]))
subset['shoebox'] = flex.shoebox(
subset['panel'],
subset['bbox'],
allocate=True)
subset.compute_mask(experiments)
# Get the mask and data
mask = imageset.get_mask(frame)[0]
data = imageset.get_raw_data(frame)[0]
sbox_mask = subset['shoebox'].apply_background_mask(frame, 1, (height, width))
mask = mask & sbox_mask
#from dials.algorithms.image.threshold import DispersionThreshold
#threshold = DispersionThreshold(
# data.all(),
def tst_extract_shoeboxes(self):
from dials.array_family import flex
from random import randint, seed
from dials.algorithms.shoebox import MaskCode
import sys
seed(0)
reflections = flex.reflection_table()
reflections['panel'] = flex.size_t()
reflections['bbox'] = flex.int6()
npanels = 2
width = 1000
height = 1000
nrefl = 10000
frame0 = 10
frame1 = 100
nrefl = 1000
for i in range(nrefl):
xs = randint(5, 10)
ys = randint(5, 10)
x0 = randint(-xs+1, width-1)
y0 = randint(-ys+1, height-1)
z0 = randint(frame0, frame1-1)
x1 = x0 + xs
y1 = y0 + ys
z1 = min([z0 + randint(1, 10), frame1])
assert(x1 > x0)
assert(y1 > y0)
assert(z1 > z0)
assert(z0 >= frame0 and z1 <= frame1)
bbox = (x0, x1, y0, y1, z0, z1)
reflections.append({
"panel" : randint(0,1),
"bbox" : bbox,
})
reflections['shoebox'] = flex.shoebox(
reflections['panel'],
reflections['bbox'])
reflections['shoebox'].allocate()
class FakeImageSet(object):
def __init__(self):
from dials.array_family import flex
self.data = flex.int(range(height*width))
self.data.reshape(flex.grid(height,width))
def get_array_range(self):
return (frame0, frame1)
def get_detector(self):
class FakeDetector(object):
def __len__(self):
return npanels
def __getitem__(self, index):
class FakePanel(object):
def get_trusted_range(self):
return (-1, 1000000)
return FakePanel()
return FakeDetector()
def __len__(self):
return frame1 - frame0
def __getitem__(self, index):
f = frame0+index
return (self.data + f*1, self.data + f*2)
def get_corrected_data(self, index):
f = frame0+index
return (self.data + f*1, self.data + f*2)
def get_mask(self, index):
image = self.get_corrected_data(index)
return tuple(im >= 0 for im in image)
imageset = FakeImageSet()
stdout = sys.stdout
class DevNull(object):
def write(self, *args):
pass
def flush(self):
pass
sys.stdout = DevNull()
reflections.extract_shoeboxes(imageset)
sys.stdout = stdout
for i in range(len(reflections)):
sbox = reflections[i]["shoebox"]
assert(sbox.is_consistent())
mask = sbox.mask
data = sbox.data
bbox = sbox.bbox
panel = sbox.panel
x0, x1, y0, y1, z0, z1 = bbox
for z in range(z1 - z0):
for y in range(y1 - y0):
for x in range(x1 - x0):
v1 = data[z,y,x]
m1 = mask[z,y,x]
if (x0 + x >= 0 and y0 + y >= 0 and
x0 + x < width and y0 + y < height):
v2 = imageset.data[y+y0,x+x0] + (z+z0)*(panel+1)
m2 = MaskCode.Valid
assert(v1 == v2)
assert(m1 == m2)
else:
assert(v1 == 0)
assert(m1 == 0)
print 'OK'
def reconstruct_rogues(params):
assert os.path.exists('xia2.json')
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
import cPickle as pickle
import dials # because WARNING:root:No profile class gaussian_rs registered
crystals = xinfo.get_crystals()
assert len(crystals) == 1
for xname in crystals:
crystal = crystals[xname]
scaler = crystal._get_scaler()
epochs = scaler._sweep_handler.get_epochs()
rogues = os.path.join(scaler.get_working_directory(),
xname, 'scale', 'ROGUES')
rogue_reflections = munch_rogues(rogues)
batched_reflections = { }
for epoch in epochs:
si = scaler._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
experiments = ExperimentListFactory.from_json_file(
intgr.get_integrated_experiments())
reflections = pickle.load(open(intgr.get_integrated_reflections()))
batched_reflections[si.get_batch_range()] = (experiments, reflections,
si.get_sweep_name())
# - look up reflection in reflection list, get bounding box
# - pull pixels given from image set, flatten these, write out
from dials.array_family import flex
from annlib_ext import AnnAdaptor as ann_adaptor
reflections_run = { }
for run in batched_reflections:
reflections_run[run] = []
for rogue in rogue_reflections:
b = rogue[0]
for run in batched_reflections:
if b >= run[0] and b <= run[1]:
reflections_run[run].append(rogue)
break
for run_no, run in enumerate(reflections_run):
experiment = batched_reflections[run][0]
reflections = batched_reflections[run][1]
name = batched_reflections[run][2]
rogues = reflections_run[run]
reference = flex.double()
scan = experiment.scans()[0]
images = experiment.imagesets()[0]
for xyz in reflections['xyzcal.px']:
reference.append(xyz[0])
reference.append(xyz[1])
reference.append(xyz[2])
search = flex.double()
for rogue in rogues:
search.append(rogue[1])
search.append(rogue[2])
search.append(scan.get_array_index_from_angle(rogue[3]))
ann = ann_adaptor(data=reference, dim=3, k=1)
ann.query(search)
keep = flex.bool(len(reflections), False)
for j, rogue in enumerate(rogues):
keep[ann.nn[j]] = True
reflections = reflections.select(keep==True)
if params.extract:
reflections["shoebox"] = flex.shoebox(
reflections["panel"],
reflections["bbox"],
allocate=True)
reflections.extract_shoeboxes(images, verbose=False)
if len(reflections_run) > 1:
output = params.output.reflections.replace(
'.pickle', '-%s.pickle' % name)
print 'Extracted %d rogue reflections for %s to %s' % \
(len(reflections), name, output)
reflections.as_pickle(output)
else:
output = params.output.reflections
print 'Extracted %d rogue reflections to %s' % \
(len(reflections), output)
reflections.as_pickle(output)
def _find_in_imageset(self, imageset):
"""
Do the spot finding.
:param imageset: The imageset to process
:return: The observed spots
"""
from dials.util.masking import MaskGenerator
from dials.array_family import flex
from dials.util.command_line import Command
from dxtbx.imageset import ImageSweep
from logging import info
# The input mask
mask = self.mask_generator.generate(imageset)
if self.mask is not None:
mask = tuple(m1 & m2 for m1, m2 in zip(mask, self.mask))
# Set the spot finding algorithm
extract_spots = ExtractSpots(
threshold_function=self.threshold_function,
mask=mask,
region_of_interest=self.region_of_interest,
max_strong_pixel_fraction=self.max_strong_pixel_fraction,
mp_method=self.mp_method,
nproc=self.nproc,
mp_chunksize=self.mp_chunksize,
min_spot_size=self.min_spot_size,
max_spot_size=self.max_spot_size,
)
# Get the max scan range
if isinstance(imageset, ImageSweep):
max_scan_range = imageset.get_array_range()
else:
max_scan_range = (0, len(imageset))
# Get list of scan ranges
if not self.scan_range or self.scan_range[0] is None:
scan_range = [(max_scan_range[0] + 1, max_scan_range[1])]
else:
scan_range = self.scan_range
# Get spots from bits of scan
spots_all = []
for scan in scan_range:
j0, j1 = scan
assert j1 >= j0 and j0 > max_scan_range[0] and j1 <= max_scan_range[1]
info("\nFinding spots in image {0} to {1}...".format(j0, j1))
j0 -= 1
if isinstance(imageset, ImageSweep):
j0 -= imageset.get_array_range()[0]
j1 -= imageset.get_array_range()[0]
spots_all.extend(extract_spots(imageset[j0:j1]))
# Get the list of shoeboxes
shoeboxes = flex.shoebox(spots_all)
# Calculate the spot centroids
centroid = shoeboxes.centroid_valid()
info("Calculated {0} spot centroids".format(len(shoeboxes)))
# Calculate the spot intensities
intensity = shoeboxes.summed_intensity()
info("Calculated {0} spot intensities".format(len(shoeboxes)))
# Create the observations
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
# Write the hot mask
if self.write_hot_mask:
# Find spots which cover the whole scan range
bbox = flex.int6([sbox.bbox for sbox in shoeboxes])
z0, z1 = bbox.parts()[4:6]
zr = z1 - z0
assert zr.all_gt(0)
possible_hot_spots = zr == len(imageset)
num_possible_hot_spots = possible_hot_spots.count(True)
info("Found %d possible hot spots" % num_possible_hot_spots)
# Create the hot pixel mask
hot_mask = tuple(flex.bool(flex.grid(p.get_image_size()[::-1]), True) for p in imageset.get_detector())
if num_possible_hot_spots > 0:
hot_shoeboxes = shoeboxes.select(possible_hot_spots)
for sbox in hot_shoeboxes:
x0, x1, y0, y1 = sbox.bbox[0:4]
m = sbox.mask
p = sbox.panel
for y in range(m.all()[1]):
for x in range(m.all()[2]):
if m[:, y : y + 1, x : x + 1].all_ne(0):
hot_mask[p][y0 + y, x0 + x] = False
info("Found %d possible hot pixel(s)" % hot_mask.count(False))
else:
hot_mask = None
# Filter the reflections and select only the desired spots
flags = self.filter_spots(None, sweep=imageset, observations=observed, shoeboxes=shoeboxes)
observed = observed.select(flags)
shoeboxes = shoeboxes.select(flags)
# Return as a reflection list
return flex.reflection_table(observed, shoeboxes), hot_mask
from dials.array_family import flex
num_ref = 3
ref_table = flex.reflection_table()
shoebox = flex.shoebox(num_ref)
ref_table['shoebox'] = shoebox
intensity = flex.double(num_ref)
ref_table['intensity.sum.value'] = intensity
intensity_var = flex.double(num_ref)
ref_table['intensity.sum.variance'] = intensity_var
iterate = ref_table['shoebox']
n = 0
for arr in iterate:
img = flex.double(flex.grid(3, 3, 3))
bkg = flex.double(flex.grid(3, 3, 3))
msk = flex.int(flex.grid(3, 3, 3))
for row in range(3):
for col in range(3):
for fra in range(3):
img[row, col, fra] = row + col + fra + n * 9
bkg[row, col, fra] = 0.0
msk[row, col, fra] = 3
n += 1
msk[1, 1, 1] = 5
tmp_i = n * n * n * 3
img[1, 1, 1] += tmp_i
print "intensity must be =", tmp_i
arr.data = img[:, :, :]
def generate_predictions(self, N):
""" Generate some reflections. """
from dials.algorithms.profile_model.gaussian_rs import MaskCalculator3D
from dials.array_family import flex
from dials.util.command_line import Command
from dials.algorithms import filtering
from dials.algorithms.shoebox import MaskCode
from dials.algorithms.profile_model.gaussian_rs import Model as ProfileModel
import random
# Set the profile model
self.experiment.profile = ProfileModel(None, self.n_sigma, self.sigma_b, self.sigma_m)
# Generate a list of reflections
refl = flex.reflection_table.from_predictions(self.experiment)
refl["id"] = flex.int(len(refl), 0)
# Filter by zeta
zeta = 0.05
Command.start("Filtering by zeta >= %f" % zeta)
mask = filtering.by_zeta(self.experiment.goniometer, self.experiment.beam, refl["s1"], zeta)
refl.del_selected(mask != True)
Command.end("Filtered %d reflections by zeta >= %f" % (len(refl), zeta))
# Compute the bounding box
refl.compute_bbox([self.experiment])
index = []
image_size = self.experiment.detector[0].get_image_size()
array_range = self.experiment.scan.get_array_range()
bbox = refl["bbox"]
for i in range(len(refl)):
x0, x1, y0, y1, z0, z1 = bbox[i]
if (
x0 < 0
or x1 > image_size[0]
or y0 < 0
or y1 > image_size[1]
or z0 < array_range[0]
or z1 > array_range[1]
):
index.append(i)
refl.del_selected(flex.size_t(index))
# Sample if specified
index = random.sample(range(len(refl)), N)
refl = refl.select(flex.size_t(index))
# Compute the bounding box
# Create a load of shoeboxes
Command.start("Creating shoeboxes for %d reflections" % len(refl))
refl["shoebox"] = flex.shoebox(refl["panel"], refl["bbox"])
refl["shoebox"].allocate_with_value(MaskCode.Valid)
Command.end("Created shoeboxes for %d reflections" % len(refl))
# Get the function object to mask the foreground
Command.start("Masking Foreground for %d reflections" % len(refl))
mask_foreground = MaskCalculator3D(
self.experiment.beam,
self.experiment.detector,
self.experiment.goniometer,
self.experiment.scan,
self.n_sigma * self.sigma_b,
self.n_sigma * self.sigma_m,
)
# Mask the foreground
mask_foreground(refl["shoebox"], refl["s1"], refl["xyzcal.px"].parts()[2], refl["panel"])
Command.end("Masked foreground for %d reflections" % len(refl))
# Return the reflections
return refl
def __call__(self, imageset):
"""
Find the spots in the imageset
:param imageset: The imageset to process
:return: The list of spot shoeboxes
"""
from dials.util.command_line import Command
from dials.array_family import flex
from dxtbx.imageset import ImageSweep
from dials.model.data import PixelListLabeller
from logging import info, warn
from math import floor
import platform
from logging import warn
# Change the number of processors if necessary
mp_nproc = self.nproc
if platform.system() == "Windows" and mp_nproc > 1:
warn("")
warn("*" * 80)
warn("Multiprocessing is not available on windows. Setting nproc = 1")
warn("*" * 80)
warn("")
mp_nproc = 1
if mp_nproc > len(imageset):
mp_nproc = len(imageset)
mp_method = self.mp_method
mp_chunksize = self.mp_chunksize
len_by_nproc = int(floor(len(imageset) / mp_nproc))
if mp_chunksize > len_by_nproc:
mp_chunksize = len_by_nproc
assert mp_nproc > 0, "Invalid number of processors"
assert mp_chunksize > 0, "Invalid chunk size"
# The extract pixels function
function = ExtractPixelsFromImage(
imageset=imageset,
threshold_function=self.threshold_function,
mask=self.mask,
max_strong_pixel_fraction=self.max_strong_pixel_fraction,
region_of_interest=self.region_of_interest,
)
# The indices to iterate over
indices = list(range(len(imageset)))
# Initialise the pixel labeller
num_panels = len(imageset.get_detector())
pixel_labeller = [PixelListLabeller() for p in range(num_panels)]
# Do the processing
info("Extracting strong pixels from images")
info(" Using %s with %d parallel job(s)\n" % (mp_method, mp_nproc))
if mp_nproc > 1:
def process_output(result):
import logging
for message in result[1]:
logging.log(message.levelno, message.msg)
assert len(pixel_labeller) == len(result[0].pixel_list), "Inconsistent size"
for plabeller, plist in zip(pixel_labeller, result[0].pixel_list):
plabeller.add(plist)
result[0].pixel_list = None
# Parallel reading of HDF5 from the same handle is not allowed. Python
# multiprocessing is a bit messed up and used fork on linux so need to
# close and reopen file.
from dxtbx.imageset import SingleFileReader
if isinstance(imageset.reader(), SingleFileReader):
imageset.reader().nullify_format_instance()
batch_parallel_map(
func=ExtractSpotsParallelTask(function),
iterable=indices,
processes=mp_nproc,
callback=process_output,
method=mp_method,
chunksize=mp_chunksize,
)
else:
for task in indices:
result = function(task)
assert len(pixel_labeller) == len(result.pixel_list), "Inconsistent size"
for plabeller, plist in zip(pixel_labeller, result.pixel_list):
plabeller.add(plist)
result.pixel_list = None
# Extract the pixel lists into a list of reflections
shoeboxes = flex.shoebox()
if isinstance(imageset, ImageSweep):
twod = False
else:
twod = True
for i, p in enumerate(pixel_labeller):
if p.num_pixels() > 0:
shoeboxes.extend(
flex.shoebox(
p, i, 0, twod, self.min_spot_size, self.max_spot_size # panel # zrange # twod # min_pixels
)
) # max_pixels
info("")
info("Extracted {0} spots".format(len(shoeboxes)))
# Get the unallocated spots and print some info
selection = shoeboxes.is_allocated()
shoeboxes = shoeboxes.select(selection)
info(
"Removed %d spots with size < %d or > %d pixels"
% (selection.count(False), self.min_spot_size, self.max_spot_size)
)
# Return the shoeboxes
return shoeboxes
imageset = experiments[0].imageset
print list(reflections.keys())
reflections.unset_flags(flex.size_t(range(len(reflections))), reflections.flags.integrated_sum)
reflections.unset_flags(flex.size_t(range(len(reflections))), reflections.flags.integrated_prf)
del reflections['intensity.sum.value']
del reflections['intensity.sum.variance']
del reflections['background.mean']
del reflections['intensity.prf.value']
del reflections['intensity.prf.variance']
del reflections['profile.correlation']
del reflections['profile.rmsd']
# Create the reflection lookup
reflections['shoebox'] = flex.shoebox(
reflections['panel'],
reflections['bbox'])
bbox = reflections['bbox']
reflection_lookup = defaultdict(list)
for i in range(len(bbox)):
for j in range(bbox[i][4],bbox[i][5]):
reflection_lookup[j].append(i)
width, height = experiments[0].detector[0].get_image_size()
# Loop through all images
print "START"
for frame in range(0, len(imageset)):
# Get the subset of reflections on this image and compute the mask
indices = flex.size_t(reflection_lookup[frame])
def tst_split_partials_with_shoebox(self):
from dials.array_family import flex
from random import randint, uniform
from dials.model.data import Shoebox
r = flex.reflection_table()
r['value1'] = flex.double()
r['value2'] = flex.int()
r['value3'] = flex.double()
r['bbox'] = flex.int6()
r['panel'] = flex.size_t()
r['shoebox'] = flex.shoebox()
expected = []
for i in range(100):
x0 = randint(0, 100)
x1 = x0 + randint(1, 10)
y0 = randint(0, 100)
y1 = y0 + randint(1, 10)
z0 = randint(0, 100)
z1 = z0 + randint(1, 10)
v1 = uniform(0, 100)
v2 = randint(0, 100)
v3 = uniform(0, 100)
sbox = Shoebox(0, (x0, x1, y0, y1, z0, z1))
sbox.allocate()
assert(sbox.is_consistent())
w = x1 - x0
h = y1 - y0
for z in range(z0, z1):
for y in range(y0, y1):
for x in range(x0, x1):
sbox.data[z-z0,y-y0,x-x0] = x+y*w+z*w*h
r.append({
'value1' : v1,
'value2' : v2,
'value3' : v3,
'bbox' : (x0, x1, y0, y1, z0, z1),
'panel' : 0,
'shoebox' : sbox
})
for z in range(z0, z1):
sbox = Shoebox(0, (x0, x1, y0, y1, z, z+1))
sbox.allocate()
assert(sbox.is_consistent())
w = x1 - x0
h = y1 - y0
for y in range(y0, y1):
for x in range(x0, x1):
sbox.data[0,y-y0,x-x0] = x+y*w+z*w*h
expected.append({
'value1' : v1,
'value2' : v2,
'value3' : v3,
'bbox' : (x0, x1, y0, y1, z, z+1),
'partial_id' : i,
'panel' : 0,
'shoebox' : sbox
})
r.split_partials_with_shoebox()
assert(len(r) == len(expected))
EPS = 1e-7
for r1, r2 in zip(r, expected):
assert(abs(r1['value1'] - r2['value1']) < EPS)
assert(r1['value2'] == r2['value2'])
assert(abs(r1['value3'] - r2['value3']) < EPS)
assert(r1['bbox'] == r2['bbox'])
assert(r1['partial_id'] == r2['partial_id'])
assert(r1['panel'] == r2['panel'])
assert(r1['shoebox'].data.as_double().as_1d().all_approx_equal(
r2['shoebox'].data.as_double().as_1d()))
print 'OK'
def run(self):
''' Extract the shoeboxes. '''
from dials.util.options import flatten_reflections
from dials.util.options import flatten_experiments
from dials.util.options import flatten_datablocks
from dials.util import log
from dials.array_family import flex
from libtbx.utils import Sorry
from logging import info
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
# Configure logging
log.config()
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
info('The following parameters have been modified:\n')
info(diff_phil)
# Get the data
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
datablocks = flatten_datablocks(params.input.datablock)
if len(experiments) == 0 and len(datablocks) == 0 and len(reflections) == 0:
self.parser.print_help()
exit(0)
elif (len(experiments) != 0 and len(datablocks) != 0):
raise Sorry('Both experiment list and datablocks set')
elif len(experiments) > 1:
raise Sorry('More than 1 experiment set')
elif len(datablocks) > 1:
raise Sorry('More than 1 datablock set')
elif len(experiments) == 1:
imageset = experiments[0].imageset
elif len(datablocks) == 1:
imagesets = datablocks[0].extract_imagesets()
if len(imagesets) != 1:
raise Sorry('Need 1 imageset, got %d' % len(imagesets))
imageset = imagesets[0]
if len(reflections) != 1:
raise Sorry('Need 1 reflection table, got %d' % len(reflections))
else:
reflections = reflections[0]
# Check the reflections contain the necessary stuff
assert("bbox" in reflections)
assert("panel" in reflections)
# Get some models
detector = imageset.get_detector()
scan = imageset.get_scan()
frame0, frame1 = scan.get_array_range()
# Add some padding but limit to image volume
if params.padding > 0:
info('Adding %d pixels as padding' % params.padding)
x0, x1, y0, y1, z0, z1 = reflections['bbox'].parts()
x0 -= params.padding
x1 += params.padding
y0 -= params.padding
y1 += params.padding
z0 -= params.padding
z1 += params.padding
panel = reflections['panel']
for i in range(len(reflections)):
width, height = detector[panel[i]].get_image_size()
if z0[i] < frame0: z0[i] = frame0
if z1[i] > frame1: z1[i] = frame1
reflections['bbox'] = flex.int6(x0, x1, y0, y1, z0, z1)
# Save the old shoeboxes
if "shoebox" in reflections:
old_shoebox = reflections['shoebox']
else:
old_shoebox = None
# Allocate the shoeboxes
reflections["shoebox"] = flex.shoebox(
reflections["panel"],
reflections["bbox"],
allocate=True)
# Extract the shoeboxes
reflections.extract_shoeboxes(imageset, verbose=True)
# Preserve masking
if old_shoebox is not None:
info("Applying old shoebox mask")
new_shoebox = reflections['shoebox']
for i in range(len(reflections)):
bbox0 = old_shoebox[i].bbox
bbox1 = new_shoebox[i].bbox
mask0 = old_shoebox[i].mask
mask1 = new_shoebox[i].mask
mask2 = flex.int(mask1.accessor(), 0)
x0 = bbox0[0] - bbox1[0]
x1 = bbox0[1] - bbox0[0] + x0
y0 = bbox0[2] - bbox1[2]
y1 = bbox0[3] - bbox0[2] + y0
z0 = bbox0[4] - bbox1[4]
z1 = bbox0[5] - bbox0[4] + z0
mask2[z0:z1,y0:y1,x0:x1] = mask0
mask1 = mask1.as_1d() | mask2.as_1d()
mask1.reshape(new_shoebox[i].mask.accessor())
new_shoebox[i].mask = mask1
# Saving the reflections to disk
filename = params.output.reflections
info('Saving %d reflections to %s' % (len(reflections), filename))
reflections.as_pickle(filename)
import numpy
data2d = numpy.zeros((3, 3), dtype = numpy.float64)
data2d[:, :] = 15
data2d[1:2, 1:2] = 50
for row in range(3):
for col in range(3):
data2d[row, col] += row * 2
data2d[row, col] += col * 2
ref_table = flex.reflection_table()
'''
shoebox = flex.shoebox(5)
#panel = flex.size_t(1)
ref_table = flex.reflection_table()
shoebox.data = flex.double(flex.grid(6, 6, 6))
ref_table['shoebox'] = shoebox
#ref_table['panel'] = panel
#efisient_way = '''
its = ref_table['shoebox']
for arr in its:
print arr.data.as_numpy_array()
#'''
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'FAIL: dials_regression not configured'
exit(0)
path = join(dials_regression, "centroid_test_data")
import sys
assert(len(sys.argv) == 1)
sys.argv.append(join(path, "experiments.json"))
sys.argv.append(join(path, "profile.phil"))
parser = OptionParser(phil=phil_scope)
params, options, args = parser.parse_args()
assert(len(args) == 1)
exlist = ExperimentListFactory.from_json_file(args[0])
assert(len(exlist) == 1)
profile_model = ProfileModelList.load(params)
rlist = flex.reflection_table.from_predictions_multi(exlist)
rlist.compute_bbox(exlist, profile_model)
rlist['shoebox'] = flex.shoebox(rlist['panel'], rlist['bbox'])
rlist['shoebox'].allocate()
rlist.extract_shoeboxes(exlist[0].imageset)
show_reflection(rlist[len(rlist)//2])
#show_reflection(rlist[len(rlist)//2], orient = "porTrait")
#show_reflection(rlist[len(rlist)//2], orient = "lanDscape")
'''Set the reflection data.'''
self.set_data(reflections, ReflectionListEncoder())
def get_reflections(self):
'''Get the reflection data.'''
return self.get_data(ReflectionListDecoder())
if __name__ == '__main__':
from dials.array_family import flex
reflections = flex.reflection_table([
('hkl', flex.miller_index(10)),
('s1', flex.vec3_double(10)),
('bbox', flex.int6(10)),
('id', flex.int(10)),
('shoebox', flex.shoebox(10))
])
for i in range(10):
reflections['shoebox'][i].data = flex.double(flex.grid(10,10,10))
reflections['shoebox'][i].mask = flex.int(flex.grid(10,10,10))
reflections['shoebox'][i].background = flex.double(flex.grid(10,10,10))
for i in range(10):
print reflections['shoebox'][i].data.all()
writer = NexusFile('temp.h5', 'w')
writer.set_reflections(reflections)
writer.close()
reader = NexusFile('temp.h5', 'r')
if __name__ == '__main__': from dials.array_family import flex from dxtbx.model.experiment.experiment_list import ExperimentListFactory import os.path path = "/home/upc86896/Projects/cctbx/sources/dials_regression/centroid_test_data" rlist_filename = os.path.join(path, "integrated.pickle") exlist_filename = os.path.join(path, "experiments.json") rlist = flex.reflection_table.from_pickle(rlist_filename) exlist = ExperimentListFactory.from_json_file(exlist_filename) panel = rlist['panel'] bbox = rlist['bbox'] rlist['shoebox'] = flex.shoebox(panel, bbox) rlist['shoebox'].allocate() rlist.fill_shoeboxes(exlist[0].imageset)
def run(i, imp):
from random import randint
from dials.array_family import flex
#building a reflection table
num_ref = 5
ref_table = flex.reflection_table()
shoebox = flex.shoebox(num_ref)
ref_table['shoebox'] = shoebox
intensity = flex.double(num_ref)
ref_table['intensity.sum.value'] = intensity
intensity_var = flex.double(num_ref)
ref_table['intensity.sum.variance'] = intensity_var
iterate = ref_table['shoebox']
i_to_compare = []
# bulding the shoebox with a desired content
# which is a reflection with noise included
n = 0
for arr in iterate:
img = flex.double(flex.grid(3, 3, 3))
bkg = flex.double(flex.grid(3, 3, 3))
msk = flex.int(flex.grid(3, 3, 3))
for row in range(3):
for col in range(3):
for fra in range(3):
img[row, col, fra] = row + col + fra + n * 9 + randint(0, i)
bkg[row, col, fra] = 0.0
msk[row, col, fra] = 3
n += 1
msk[1, 1, 1] = 5
tmp_i = n * n * n * 3
i_to_compare.append(tmp_i)
img[1, 1, 1] += tmp_i
arr.data = img[:, :, :]
arr.background = bkg[:, :, :]
arr.mask = msk[:, :, :]
# calling the functions that we need to test
# first select the algorithm for background calculation
if imp == "inclined":
print "testing inclined_background_subtractor"
from dials.algorithms.background.inclined_background_subtractor \
import layering_and_background_plane
layering_and_background_plane(ref_table)
elif imp == "flat":
print "testing flat_background_subtractor"
from dials.algorithms.background.flat_background_subtractor \
import layering_and_background_avg
layering_and_background_avg(ref_table)
elif imp == "curved":
print "testing curved_background_subtractor"
from dials.algorithms.background.curved_background_subtractor \
import layering_and_background_modl
layering_and_background_modl(ref_table)
# no matter which algorithm was used for background calculation
# the integration summation must remain compatible
from dials.algorithms.integration.summation2d \
import flex_2d_layering_n_integrating
flex_2d_layering_n_integrating(ref_table)
# comparing results
result = "OK"
resl_its = ref_table['intensity.sum.value']
resl_var = ref_table['intensity.sum.variance']
for n_its in range(len(resl_its)):
if resl_its[n_its] <= i_to_compare[n_its] + i and \
resl_its[n_its] >= i_to_compare[n_its] - i and \
resl_var[n_its] > resl_its[n_its]:
print "Ok ", n_its
else:
print "Wrong num", n_its
print "i =", i
print "resl_its[n_its] =", resl_its[n_its]
print "i_to_compare[n_its] =", i_to_compare[n_its]
print "resl_var[n_its] =", resl_var[n_its]
result = "wrong"
raise RuntimeError('wrong result')
return result
def simple_gaussian_spots(params):
from dials.array_family import flex
from dials.algorithms import shoebox
import random
import math
from scitbx import matrix
r = params.rotation
axis = matrix.col((r.axis.x, r.axis.y, r.axis.z))
if axis.length() > 0:
rotation = axis.axis_and_angle_as_r3_rotation_matrix(r.angle, deg=True)
else:
rotation = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))
# generate mask and peak values
from dials.algorithms.shoebox import MaskCode
mask_peak = MaskCode.Valid|MaskCode.Foreground
mask_back = MaskCode.Valid|MaskCode.Background
from dials.util.command_line import ProgressBar
p = ProgressBar(title = 'Generating reflections')
rlist = flex.reflection_table(params.nrefl)
hkl = flex.miller_index(params.nrefl)
s1 = flex.vec3_double(params.nrefl)
xyzmm = flex.vec3_double(params.nrefl)
xyzpx = flex.vec3_double(params.nrefl)
panel = flex.size_t(params.nrefl)
bbox = flex.int6(params.nrefl)
for j in range(params.nrefl):
p.update(j * 100.0 / params.nrefl)
hkl[j] = (random.randint(0, 20),
random.randint(0, 20),
random.randint(0, 20))
phi = 2 * math.pi * random.random()
s1[j] = (0, 0, 0)
xyzpx[j] = (0, 0, 0)
xyzmm[j] = (0, 0, phi)
panel[j] = 0
bbox[j] = (0, params.shoebox_size.x,
0, params.shoebox_size.y,
0, params.shoebox_size.z)
p.finished('Generating %d reflections' % params.nrefl)
intensity = flex.double(params.nrefl)
shoebox = flex.shoebox(panel, bbox)
shoebox.allocate_with_value(MaskCode.Valid)
p = ProgressBar(title = 'Generating shoeboxes')
for i in range(len(rlist)):
p.update(i * 100.0 / params.nrefl)
mask = shoebox[i].mask
if params.pixel_mask == 'precise':
# flag everything as background: peak will me assigned later
for j in range(len(mask)):
mask[j] = mask_back
elif params.pixel_mask == 'all':
# flag we have no idea what anything is
mask_none = MaskCode.Valid|MaskCode.Foreground|MaskCode.Background
for j in range(len(mask)):
mask[j] = mask_none
elif params.pixel_mask == 'static':
import itertools
from scitbx.array_family import flex
x0 = params.spot_offset.x + params.shoebox_size.x / 2
y0 = params.spot_offset.x + params.shoebox_size.y / 2
z0 = params.spot_offset.x + params.shoebox_size.z / 2
sx = params.mask_nsigma * params.spot_size.x
sy = params.mask_nsigma * params.spot_size.y
sz = params.mask_nsigma * params.spot_size.z
# The x, y, z indices
z, y, x = zip(*itertools.product(*(range(n) for n in mask.all())))
xyz = flex.vec3_double(flex.double(x), flex.double(y), flex.double(z))
# Calculate SUM(((xj - xj0) / sxj)**2) for each element
xyz0 = (x0, y0, z0)
isxyz = (1.0/sx, 1.0/sy, 1.0/sz)
dxyz = sum([(x * isx)**2 for x, isx in
zip(((xyz - xyz0) * rotation).parts(), isxyz)])
# Set the mask values
index = dxyz <= 1.0
index.reshape(mask.accessor())
mask.set_selected(index, MaskCode.Valid | MaskCode.Foreground)
mask.set_selected(index != True, MaskCode.Valid | MaskCode.Background)
sbox = shoebox[i].data
# reflection itself, including setting the peak region if we're doing that
# FIXME use flex arrays to make the rotation bit more efficient as this is
# now rather slow...
counts_true = 0
for j in range(params.counts):
_x = random.gauss(0, params.spot_size.x)
_y = random.gauss(0, params.spot_size.y)
_z = random.gauss(0, params.spot_size.z)
Rxyz = rotation * matrix.col((_x, _y, _z)).elems
x = int(Rxyz[0] + params.spot_offset.x + params.shoebox_size.x / 2)
y = int(Rxyz[1] + params.spot_offset.y + params.shoebox_size.y / 2)
z = int(Rxyz[2] + params.spot_offset.z + params.shoebox_size.z / 2)
if x < 0 or x >= params.shoebox_size.x:
continue
if y < 0 or y >= params.shoebox_size.y:
continue
if z < 0 or z >= params.shoebox_size.z:
continue
sbox[z, y, x] += 1
counts_true += 1
if params.pixel_mask == 'precise':
mask[z, y, x] = mask_peak
intensity[i] = counts_true
if params.background:
# background:flat;
for j in range(params.background * len(sbox)):
x = random.randint(0, params.shoebox_size.x - 1)
y = random.randint(0, params.shoebox_size.y - 1)
z = random.randint(0, params.shoebox_size.z - 1)
sbox[z, y, x] += 1
else:
# or inclined
random_background_plane(sbox, params.background_a, params.background_b,
params.background_c, params.background_d)
rlist['miller_index'] = hkl
rlist['s1'] = s1
rlist['xyzcal.px'] = xyzpx
rlist['xyzcal.mm'] = xyzmm
rlist['bbox'] = bbox
rlist['panel'] = panel
rlist['shoebox'] = shoebox
rlist['intensity.sum.value'] = intensity
p.finished('Generating %d shoeboxes' % params.nrefl)
return rlist
