def tst_count_mask_values(self):
from dials.model.data import Shoebox
from random import randint, sample
from dials.array_family import flex
shoebox = flex.shoebox(10)
num = flex.int(10)
value = (1 << 2)
for i in range(10):
x0 = randint(0, 90)
y0 = randint(0, 90)
z0 = randint(0, 90)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))
shoebox[i].allocate()
maxnum = len(shoebox[i].mask)
num[i] = randint(1, maxnum)
indices = sample(list(range(maxnum)), num[i])
for j in indices:
shoebox[i].mask[j] = value
assert(shoebox.count_mask_values(value) == num)
# Test passed
print 'OK'
def test_Simple6ProfileModel_compute_mask(simple6_profile_model,
test_experiment):
experiments = [test_experiment]
# Create the index generator
index_generator = IndexGenerator(
experiments[0].crystal.get_unit_cell(),
experiments[0].crystal.get_space_group().type(),
2.0,
)
# Get an array of miller indices
miller_indices = index_generator.to_array()
reflections = simple6_profile_model.predict_reflections(experiments,
miller_indices,
probability=0.9973)
s2 = reflections["s2"]
s0 = matrix.col(experiments[0].beam.get_s0())
quantile = chisq_quantile(3, 0.9973)
sigma_inv = matrix.sqr(flumpy.from_numpy(
simple6_profile_model.sigma())).inverse()
for s2 in map(matrix.col, reflections["s2"]):
x = s2.normalize() * s0.length() - s2
d = (x.transpose() * sigma_inv * x)[0]
assert d < quantile
simple6_profile_model.compute_bbox(experiments, reflections)
reflections["shoebox"] = flex.shoebox(reflections["panel"],
reflections["bbox"],
allocate=True)
simple6_profile_model.compute_mask(experiments, reflections)
def test_does_bbox_contain_bad_pixels():
from dials.array_family import flex
from dials.model.data import Shoebox
mask = flex.bool(flex.grid(100, 100), True)
for j in range(100):
for i in range(40, 60):
mask[j, i] = False
mask[i, j] = False
shoebox = flex.shoebox(1000)
res = flex.bool(1000)
for i in range(1000):
x0 = random.randint(0, 90)
y0 = random.randint(0, 90)
z0 = random.randint(0, 90)
x1 = random.randint(1, 10) + x0
y1 = random.randint(1, 10) + y0
z1 = random.randint(1, 10) + z0
shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))
res2 = False
if x0 >= 40 and x0 < 60:
res2 = True
if x1 > 40 and x1 <= 60:
res2 = True
if y0 >= 40 and y0 < 60:
res2 = True
if y1 > 40 and y1 <= 60:
res2 = True
res[i] = res2
assert shoebox.does_bbox_contain_bad_pixels(mask) == res
def test_consistent():
from dials.array_family import flex
from dials.model.data import Shoebox
shoebox = flex.shoebox(10)
for i in range(10):
x0 = random.randint(0, 1000)
y0 = random.randint(0, 1000)
z0 = random.randint(0, 1000)
x1 = random.randint(1, 10) + x0
y1 = random.randint(1, 10) + y0
z1 = random.randint(1, 10) + z0
shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))
assert shoebox.is_consistent() == flex.bool(10, False)
for i in range(10):
shoebox[i].allocate()
assert shoebox.is_consistent() == flex.bool(10, True)
for i in [0, 2, 4, 6, 8]:
shoebox[i].data.resize(flex.grid(10, 10, 10))
assert shoebox.is_consistent() == flex.bool([False, True] * 5)
for i in range(10):
shoebox[i].deallocate()
assert shoebox.is_consistent() == flex.bool(10, False)
def tst_count_mask_values(self):
from dials.model.data import Shoebox
from random import randint, sample
from dials.array_family import flex
shoebox = flex.shoebox(10)
num = flex.int(10)
value = (1 << 2)
for i in range(10):
x0 = randint(0, 90)
y0 = randint(0, 90)
z0 = randint(0, 90)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))
shoebox[i].allocate()
maxnum = len(shoebox[i].mask)
num[i] = randint(1, maxnum)
indices = sample(list(range(maxnum)), num[i])
for j in indices:
shoebox[i].mask[j] = value
assert (shoebox.count_mask_values(value) == num)
# Test passed
print 'OK'
def tst_is_bbox_within_image_volume(self):
from dials.model.data import Shoebox
from dials.array_family import flex
isize = (1000, 1000)
srange = (0, 100)
shoebox = flex.shoebox(7)
shoebox[0] = Shoebox((10, 20, 10, 20, 10, 20))
shoebox[1] = Shoebox((-10, 20, 10, 20, 10, 20))
shoebox[2] = Shoebox((10, 20, -10, 20, 10, 20))
shoebox[3] = Shoebox((10, 20, 10, 20, -10, 20))
shoebox[4] = Shoebox((10, 1020, 10, 20, 10, 20))
shoebox[5] = Shoebox((10, 20, 10, 1020, 10, 20))
shoebox[6] = Shoebox((10, 20, 10, 20, 10, 1020))
assert (shoebox.is_bbox_within_image_volume(isize,
srange) == flex.bool([
True, False, False,
False, False, False,
False
]))
# Test passed
print 'OK'
def _allocate_shoebox(self):
"""
Allocate the shoebox
"""
self.reflections["shoebox"] = flex.shoebox(self.reflections["panel"],
self.reflections["bbox"],
allocate=True)
def test(dials_regression, tmpdir):
tmpdir.chdir()
from dxtbx.serialize import load
from dials.algorithms import shoebox
from dials.array_family import flex
# Load the sweep and crystal
sweep_filename = os.path.join(dials_regression, 'centroid_test_data', 'sweep.json')
crystal_filename = os.path.join(dials_regression, 'centroid_test_data', 'crystal.json')
sweep = load.imageset(sweep_filename)
crystal = load.crystal(crystal_filename)
# Get models from the sweep
beam = sweep.get_beam()
detector = sweep.get_detector()
gonio = sweep.get_goniometer()
scan = sweep.get_scan()
# Get the reflections and overlaps
reflections, adjacency_list = predict_reflections(sweep, crystal)
reflections['shoebox'] = flex.shoebox(
reflections['panel'],
reflections['bbox'])
reflections['shoebox'].allocate_with_value(shoebox.MaskCode.Valid)
# If the adjacency list is given, then create the reflection mask
assert(len(detector) == 1)
image_size = detector[0].get_image_size()
shoeboxes = reflections['shoebox']
coords = reflections['xyzcal.px']
shoebox_masker = shoebox.MaskOverlapping()
shoebox_masker(shoeboxes, coords, adjacency_list)
# Loop through all edges
overlapping = []
for e in adjacency_list.edges():
v1, v2 = adjacency_list.source(e), adjacency_list.target(e)
overlapping.append(v1)
overlapping.append(v2)
# Ensure elements are unique
overlapping = set(overlapping)
# Ensure we have some overlaps
assert len(overlapping) > 0
# Get all non-overlapping reflections
all_r = set(range(len(reflections)))
non_overlapping = all_r.difference(overlapping)
# Run the tests
tst_non_overlapping(reflections, non_overlapping,
detector[0].get_image_size())
tst_overlapping(reflections, overlapping, adjacency_list,
image_size)
def __init__(self, data, jobs, npanels, callback):
from dials.array_family import flex
# Allocate shoeboxes
data["shoebox"] = flex.shoebox(data["panel"], data["bbox"])
data["shoebox"].allocate()
# Set the callback
self._callback = callback
# Initialise the base class
super(IntegrationTask3DMultiExecutor, self).__init__(data, jobs, npanels)
def __init__(self, data, jobs, npanels, callback):
from dials.array_family import flex
# Allocate shoeboxes
data["shoebox"] = flex.shoebox(data["panel"], data["bbox"])
data["shoebox"].allocate()
# Set the callback
self._callback = callback
# Initialise the base class
super(IntegrationTask3DMultiExecutor, self).__init__(data, jobs, npanels)
def test_RefinerData(testdata):
experiment = testdata.experiment
reflections = testdata.reflections
panel = experiment.detector[0]
s0_length = matrix.col(experiment.beam.get_s0()).length()
reflections["bbox"] = flex.int6(len(reflections))
reflections["xyzobs.px.value"] = flex.vec3_double(len(reflections))
reflections["s2"] = reflections["s1"].each_normalize() * s0_length
reflections["sp"] = flex.vec3_double(len(reflections))
for i, (x, y, z) in enumerate(reflections["xyzcal.px"]):
x0 = int(x) - 5
x1 = int(x) + 5 + 1
y0 = int(y) - 5
y1 = int(y) + 5 + 1
z0 = int(z)
z1 = z0 + 1
reflections["bbox"][i] = x0, x1, y0, y1, z0, z1
reflections["xyzobs.px.value"][i] = (int(x) + 0.5, int(y) + 0.5,
int(z) + 0.5)
reflections["sp"][i] = (matrix.col(
panel.get_pixel_lab_coord(
reflections["xyzobs.px.value"][i][0:2])).normalize() *
s0_length)
reflections["shoebox"] = flex.shoebox(reflections["panel"],
reflections["bbox"],
allocate=True)
shoebox_data = flex.float(flex.grid(1, 11, 11))
shoebox_mask = flex.int(flex.grid(1, 11, 11))
for j in range(11):
for i in range(11):
shoebox_data[0, j, i] = (100 * exp(-0.5 * (j - 5)**2 / 1**2) *
exp(-0.5 * (i - 5)**2 / 1**2))
shoebox_mask[0, j, i] = 5
for sbox in reflections["shoebox"]:
sbox.data = shoebox_data
sbox.mask = shoebox_mask
data = RefinerData.from_reflections(experiment, reflections)
assert tuple(data.s0) == pytest.approx(experiment.beam.get_s0())
assert data.h_list == reflections["miller_index"]
for i, sp in enumerate(reflections["sp"]):
assert data.sp_list[:, i] == pytest.approx(sp)
assert data.ctot_list[0] == sum(shoebox_data)
mobs1 = np.abs(data.mobs_list[0, :])
mobs2 = np.abs(data.mobs_list[1, :])
assert np.max(mobs1) < 1e-6
assert np.max(mobs2) < 1e-6
def final_integrator(
experiments,
reflection_table,
sigma_d,
use_crude_shoebox_mask=False,
shoebox_probability=0.9973,
):
"""Performs an initial integration of all predicted spots"""
experiment = experiments[0]
logger.info("\n" + "=" * 80 + "\nIntegrating reflections")
# first compute the bbox
if use_crude_shoebox_mask:
reflection_table["bbox"] = _compute_bbox(experiment, reflection_table,
sigma_d, "s1")
else:
# compute bbox from model
profile = experiment.crystal.mosaicity
profile.parameterisation.compute_bbox(experiments, reflection_table,
shoebox_probability)
# Select reflections within detector
x0, x1, y0, y1, _, _ = reflection_table["bbox"].parts()
xsize, ysize = experiment.detector[0].get_image_size()
selection = (x1 > 0) & (y1 > 0) & (x0 < xsize) & (y0 < ysize)
reflection_table = reflection_table.select(selection)
reflection_table["shoebox"] = flex.shoebox(reflection_table["panel"],
reflection_table["bbox"],
allocate=True)
reflection_table.extract_shoeboxes(experiment.imageset)
if use_crude_shoebox_mask:
_compute_mask(experiment, reflection_table, sigma_d, "s1")
else:
# compute the mask from the model.
profile = experiment.crystal.mosaicity
profile.parameterisation.compute_mask(experiments, reflection_table,
shoebox_probability)
logger.info(
f"Computing background, intensity, corrections for {len(reflection_table)} reflections"
)
reflection_table.compute_background(experiments)
reflection_table.compute_summed_intensity()
reflection_table.compute_corrections(experiments)
reflection_table.compute_centroid(experiments)
profile = experiment.crystal.mosaicity
profile.parameterisation.compute_partiality(experiments, reflection_table)
return reflection_table
def __call__(self, imageset, pixel_labeller):
"""
Convert the pixel list to shoeboxes
"""
from dxtbx.imageset import ImageSequence
# Extract the pixel lists into a list of reflections
shoeboxes = flex.shoebox()
spotsizes = flex.size_t()
hotpixels = tuple(flex.size_t() for i in range(len(imageset.get_detector())))
if isinstance(imageset, ImageSequence):
scan = imageset.get_scan()
if scan.is_still():
twod = True
else:
twod = False
else:
twod = True
for i, (p, hp) in enumerate(zip(pixel_labeller, hotpixels)):
if p.num_pixels() > 0:
creator = flex.PixelListShoeboxCreator(
p,
i, # panel
0, # zrange
twod, # twod
self.min_spot_size, # min_pixels
self.max_spot_size, # max_pixels
self.write_hot_pixel_mask,
)
shoeboxes.extend(creator.result())
spotsizes.extend(creator.spot_size())
hp.extend(creator.hot_pixels())
logger.info("")
logger.info("Extracted {} spots".format(len(shoeboxes)))
# Get the unallocated spots and print some info
selection = shoeboxes.is_allocated()
shoeboxes = shoeboxes.select(selection)
ntoosmall = (spotsizes < self.min_spot_size).count(True)
ntoolarge = (spotsizes > self.max_spot_size).count(True)
assert ntoosmall + ntoolarge == selection.count(False)
logger.info(
"Removed %d spots with size < %d pixels" % (ntoosmall, self.min_spot_size)
)
logger.info(
"Removed %d spots with size > %d pixels" % (ntoolarge, self.max_spot_size)
)
# Return the shoeboxes
return shoeboxes, hotpixels
def test(dials_data):
from dxtbx.serialize import load
from dials.algorithms import shoebox
from dials.array_family import flex
# Load the sequence and crystal
sequence = load.imageset(
dials_data("centroid_test_data").join("sweep.json").strpath
)
crystal = load.crystal(
dials_data("centroid_test_data").join("crystal.json").strpath
)
# Get models from the sequence
detector = sequence.get_detector()
# Get the reflections and overlaps
reflections, adjacency_list = predict_reflections(sequence, crystal)
reflections["shoebox"] = flex.shoebox(reflections["panel"], reflections["bbox"])
reflections["shoebox"].allocate_with_value(shoebox.MaskCode.Valid)
# If the adjacency list is given, then create the reflection mask
assert len(detector) == 1
image_size = detector[0].get_image_size()
shoeboxes = reflections["shoebox"]
coords = reflections["xyzcal.px"]
shoebox_masker = shoebox.MaskOverlapping()
shoebox_masker(shoeboxes, coords, adjacency_list)
# Loop through all edges
overlapping = []
for e in adjacency_list.edges():
v1, v2 = adjacency_list.source(e), adjacency_list.target(e)
overlapping.append(v1)
overlapping.append(v2)
# Ensure elements are unique
overlapping = set(overlapping)
# Ensure we have some overlaps
assert len(overlapping) > 0
# Get all non-overlapping reflections
all_r = set(range(len(reflections)))
non_overlapping = all_r.difference(overlapping)
# Run the tests
tst_non_overlapping(reflections, non_overlapping, detector[0].get_image_size())
tst_overlapping(reflections, overlapping, adjacency_list, image_size)
def integrate(experiments, table, params):
_params = Parameters.from_phil(params.integration)
experiments[0].scan = None
BackgroundAlgorithm = dials.extensions.Background.load(
params.integration.background.algorithm
)
flex.reflection_table.background_algorithm = functools.partial(
BackgroundAlgorithm, params
)
CentroidAlgorithm = dials.extensions.Centroid.load(
params.integration.centroid.algorithm
)
flex.reflection_table.centroid_algorithm = functools.partial(
CentroidAlgorithm, params
)
_initialize_stills(experiments, _params, table)
table["shoebox"] = flex.shoebox(table["panel"], table["bbox"], allocate=True)
table.extract_shoeboxes(experiments[0].imageset)
# From integratorexecutor
table.is_overloaded(experiments)
table.compute_mask(experiments)
# Check for invalid pixels in foreground/background
table.contains_invalid_pixels()
sbox = table["shoebox"]
nvalfg = sbox.count_mask_values(MaskCode.Valid | MaskCode.Foreground)
nforeg = sbox.count_mask_values(MaskCode.Foreground)
fraction_valid = nvalfg.as_double() / nforeg.as_double()
selection = (
fraction_valid < params.integration.profile.valid_foreground_threshold
)
table.set_flags(selection, table.flags.dont_integrate)
# Process the data
table.compute_background(experiments)
table.compute_centroid(experiments)
table.compute_summed_intensity()
table["num_pixels.valid"] = sbox.count_mask_values(MaskCode.Valid)
table["num_pixels.background"] = sbox.count_mask_values(
MaskCode.Valid | MaskCode.Background
)
table["num_pixels.background_used"] = sbox.count_mask_values(
MaskCode.Valid | MaskCode.Background | MaskCode.BackgroundUsed
)
table["num_pixels.foreground"] = nvalfg
table, experiments = _finalize_stills(table, experiments, _params)
return table, experiments
def run(self):
from dials.model.serialize import load
from dials.algorithms import shoebox
from dxtbx.serialize.load import crystal as load_crystal
from dials.array_family import flex
# Load the sweep and crystal
self.sweep = load.sweep(self.sweep_filename)
self.crystal = load_crystal(self.crystal_filename)
# Get the reflections and overlaps
reflections, adjacency_list = self.predict_reflections()
reflections['shoebox'] = flex.shoebox(
reflections['panel'],
reflections['bbox'])
reflections['shoebox'].allocate_with_value(shoebox.MaskCode.Valid)
# If the adjacency list is given, then create the reflection mask
assert(len(self.detector) == 1)
image_size = self.detector[0].get_image_size()
shoeboxes = reflections['shoebox']
coords = reflections['xyzcal.px']
shoebox_masker = shoebox.MaskOverlapping()
shoebox_masker(shoeboxes, coords, adjacency_list)
# Loop through all edges
overlapping = []
for e in adjacency_list.edges():
v1, v2 = adjacency_list.source(e), adjacency_list.target(e)
overlapping.append(v1)
overlapping.append(v2)
# Ensure elements are unique
overlapping = set(overlapping)
# Ensure we have some overlaps
assert(len(overlapping) > 0)
# Get all non-overlapping reflections
all_r = set(range(len(reflections)))
non_overlapping = all_r.difference(overlapping)
# Run the tests
self.tst_non_overlapping(reflections, non_overlapping,
self.detector[0].get_image_size())
self.tst_overlapping(reflections, overlapping, adjacency_list,
image_size)
def __call__(self, imageset, pixel_labeller):
'''
Convert the pixel list to shoeboxes
'''
from dxtbx.imageset import ImageSweep
from dials.array_family import flex
# Extract the pixel lists into a list of reflections
shoeboxes = flex.shoebox()
spotsizes = flex.size_t()
hotpixels = tuple(flex.size_t() for i in range(len(imageset.get_detector())))
if isinstance(imageset, ImageSweep):
twod = False
else:
twod = True
for i, (p, hp) in enumerate(zip(pixel_labeller, hotpixels)):
if p.num_pixels() > 0:
creator = flex.PixelListShoeboxCreator(
p,
i, # panel
0, # zrange
twod, # twod
self.min_spot_size, # min_pixels
self.max_spot_size, # max_pixels
self.write_hot_pixel_mask)
shoeboxes.extend(creator.result())
spotsizes.extend(creator.spot_size())
hp.extend(creator.hot_pixels())
logger.info('')
logger.info('Extracted {0} spots'.format(len(shoeboxes)))
# Get the unallocated spots and print some info
selection = shoeboxes.is_allocated()
shoeboxes = shoeboxes.select(selection)
ntoosmall = (spotsizes < self.min_spot_size).count(True)
ntoolarge = (spotsizes > self.max_spot_size).count(True)
assert ntoosmall + ntoolarge == selection.count(False)
logger.info('Removed %d spots with size < %d pixels' % (
ntoosmall,
self.min_spot_size))
logger.info('Removed %d spots with size > %d pixels' % (
ntoolarge,
self.max_spot_size))
# Return the shoeboxes
return shoeboxes, hotpixels
def pixel_list_to_shoeboxes(
imageset: ImageSet,
pixel_labeller: Iterable[PixelListLabeller],
min_spot_size: int,
max_spot_size: int,
write_hot_pixel_mask: bool,
) -> Tuple[flex.shoebox, Tuple[flex.size_t, ...]]:
"""Convert a pixel list to shoeboxes"""
# Extract the pixel lists into a list of reflections
shoeboxes = flex.shoebox()
spotsizes = flex.size_t()
hotpixels = tuple(flex.size_t()
for i in range(len(imageset.get_detector())))
if isinstance(imageset, ImageSequence):
twod = imageset.get_scan().is_still()
else:
twod = True
for i, (p, hp) in enumerate(zip(pixel_labeller, hotpixels)):
if p.num_pixels() > 0:
creator = flex.PixelListShoeboxCreator(
p,
i, # panel
0, # zrange
twod, # twod
min_spot_size, # min_pixels
max_spot_size, # max_pixels
write_hot_pixel_mask,
)
shoeboxes.extend(creator.result())
spotsizes.extend(creator.spot_size())
hp.extend(creator.hot_pixels())
logger.info("\nExtracted %d spots", len(shoeboxes))
# Get the unallocated spots and print some info
selection = shoeboxes.is_allocated()
shoeboxes = shoeboxes.select(selection)
ntoosmall = (spotsizes < min_spot_size).count(True)
ntoolarge = (spotsizes > max_spot_size).count(True)
assert ntoosmall + ntoolarge == selection.count(False)
logger.info("Removed %d spots with size < %d pixels", ntoosmall,
min_spot_size)
logger.info("Removed %d spots with size > %d pixels", ntoolarge,
max_spot_size)
# Return the shoeboxes
return shoeboxes, hotpixels
def run(self):
from dials.model.serialize import load
from dials.algorithms import shoebox
from dxtbx.serialize.load import crystal as load_crystal
from dials.array_family import flex
# Load the sweep and crystal
self.sweep = load.sweep(self.sweep_filename)
self.crystal = load_crystal(self.crystal_filename)
# Get the reflections and overlaps
reflections, adjacency_list = self.predict_reflections()
reflections['shoebox'] = flex.shoebox(reflections['panel'],
reflections['bbox'])
reflections['shoebox'].allocate_with_value(shoebox.MaskCode.Valid)
# If the adjacency list is given, then create the reflection mask
assert (len(self.detector) == 1)
image_size = self.detector[0].get_image_size()
shoeboxes = reflections['shoebox']
coords = reflections['xyzcal.px']
shoebox_masker = shoebox.MaskOverlapping()
shoebox_masker(shoeboxes, coords, adjacency_list)
# Loop through all edges
overlapping = []
for e in adjacency_list.edges():
v1, v2 = adjacency_list.source(e), adjacency_list.target(e)
overlapping.append(v1)
overlapping.append(v2)
# Ensure elements are unique
overlapping = set(overlapping)
# Ensure we have some overlaps
assert (len(overlapping) > 0)
# Get all non-overlapping reflections
all_r = set(range(len(reflections)))
non_overlapping = all_r.difference(overlapping)
# Run the tests
self.tst_non_overlapping(reflections, non_overlapping,
self.detector[0].get_image_size())
self.tst_overlapping(reflections, overlapping, adjacency_list,
image_size)
def initial_integrator(experiments, reflection_table):
"""Performs an initial integration of strong spots"""
# some functions require an experimentlist, others just the experiment
experiment = experiments[0]
sel = reflection_table.get_flags(reflection_table.flags.strong)
strong_refls = reflection_table.select(sel)
strong_shoeboxes = copy.deepcopy(
strong_refls["shoebox"]) # Save the strong shoeboxes
# Compute and initial spot size estimate and beam vector
sigma_d = ComputeEsdBeamDivergence(experiment.detector,
strong_refls).sigma()
sigma_degrees = sigma_d * 180 / pi
logger.info(
f"Initial sigma d estimate for {len(strong_refls)} reflections\n" +
f"Sigma D: {sigma_degrees:.5f} degrees\n", )
strong_refls["s1_obs"] = _compute_beam_vector(experiment, strong_refls)
strong_refls["bbox"] = _compute_bbox(experiment, strong_refls, sigma_d,
"s1_obs")
# allocate and extract shoebox
strong_refls["shoebox"] = flex.shoebox(strong_refls["panel"],
strong_refls["bbox"],
allocate=True)
strong_refls.extract_shoeboxes(experiment.imageset)
_compute_mask(experiment, strong_refls, sigma_d, "s1_obs",
strong_shoeboxes)
logger.info(
f"Computing background, intensity and centroids for {len(strong_refls)} reflections"
)
strong_refls.compute_background(experiments)
strong_refls.compute_summed_intensity()
n_sum = strong_refls.get_flags(
strong_refls.flags.integrated_sum).count(True)
logger.info(f"{n_sum} reflections integrated")
strong_refls.compute_centroid(experiments)
strong_refls.compute_d(experiments)
return strong_refls, sigma_d
def test_bounding_boxes():
from dials.array_family import flex
from dials.model.data import Shoebox
shoebox = flex.shoebox(10)
bbox = flex.int6(10)
for i in range(10):
x0 = random.randint(0, 90)
y0 = random.randint(0, 90)
z0 = random.randint(0, 90)
x1 = random.randint(1, 10) + x0
y1 = random.randint(1, 10) + y0
z1 = random.randint(1, 10) + z0
bbox[i] = (x0, x1, y0, y1, z0, z1)
shoebox[i] = Shoebox(bbox[i])
bbox2 = shoebox.bounding_boxes()
for i in range(10):
assert bbox2[i] == bbox[i]
def refinerdata_testdata(testdata):
experiment = testdata.experiment
reflections = testdata.reflections
panel = experiment.detector[0]
s0_length = matrix.col(experiment.beam.get_s0()).length()
reflections["bbox"] = flex.int6(len(reflections))
reflections["xyzobs.px.value"] = flex.vec3_double(len(reflections))
reflections["s2"] = reflections["s1"].each_normalize() * s0_length
reflections["sp"] = flex.vec3_double(len(reflections))
for i, (x, y, z) in enumerate(reflections["xyzcal.px"]):
x0 = int(x) - 5
x1 = int(x) + 5 + 1
y0 = int(y) - 5
y1 = int(y) + 5 + 1
z0 = int(z)
z1 = z0 + 1
reflections["bbox"][i] = x0, x1, y0, y1, z0, z1
reflections["xyzobs.px.value"][i] = (int(x) + 0.5, int(y) + 0.5,
int(z) + 0.5)
reflections["sp"][i] = (matrix.col(
panel.get_pixel_lab_coord(
reflections["xyzobs.px.value"][i][0:2])).normalize() *
s0_length)
reflections["shoebox"] = flex.shoebox(reflections["panel"],
reflections["bbox"],
allocate=True)
shoebox_data = flex.float(flex.grid(1, 11, 11))
shoebox_mask = flex.int(flex.grid(1, 11, 11))
for j in range(11):
for i in range(11):
shoebox_data[0, j, i] = (100 * exp(-0.5 * (j - 5)**2 / 1**2) *
exp(-0.5 * (i - 5)**2 / 1**2))
shoebox_mask[0, j, i] = 5
for sbox in reflections["shoebox"]:
sbox.data = shoebox_data
sbox.mask = shoebox_mask
return RefinerData.from_reflections(experiment, reflections)
def generate_reflections(detector, beam, scan, experiment, num):
seed(0)
assert len(detector) == 1
beam_vector = flex.vec3_double(num)
xyzcal_px = flex.vec3_double(num)
xyzcal_mm = flex.vec3_double(num)
panel = flex.size_t(num)
s0_length = matrix.col(beam.get_s0()).length()
for i in range(num):
x = randint(0, 2000)
y = randint(0, 2000)
z = randint(0, 8)
s1 = detector[0].get_pixel_lab_coord((x, y))
s1 = matrix.col(s1).normalize() * s0_length
phi = scan.get_angle_from_array_index(z, deg=False)
beam_vector[i] = s1
xyzcal_px[i] = (x, y, z)
(x, y) = detector[0].pixel_to_millimeter((x, y))
xyzcal_mm[i] = (x, y, phi)
panel[i] = 0
rlist = flex.reflection_table()
rlist["id"] = flex.int(len(beam_vector), 0)
rlist["s1"] = beam_vector
rlist["panel"] = panel
rlist["xyzcal.px"] = xyzcal_px
rlist["xyzcal.mm"] = xyzcal_mm
rlist["bbox"] = rlist.compute_bbox(experiment)
index = []
image_size = experiment[0].detector[0].get_image_size()
array_range = experiment[0].scan.get_array_range()
bbox = rlist["bbox"]
for i in range(len(rlist)):
x0, x1, y0, y1, z0, z1 = bbox[i]
if (x0 < 0 or x1 > image_size[0] or y0 < 0 or y1 > image_size[1]
or z0 < array_range[0] or z1 > array_range[1]):
index.append(i)
rlist.del_selected(flex.size_t(index))
rlist["shoebox"] = flex.shoebox(rlist["panel"], rlist["bbox"])
rlist["shoebox"].allocate_with_value(MaskCode.Valid)
return rlist
def tst_is_bbox_within_image_volume(self):
from dials.model.data import Shoebox
from dials.array_family import flex
isize = (1000, 1000)
srange = (0, 100)
shoebox = flex.shoebox(7)
shoebox[0] = Shoebox((10, 20, 10, 20, 10, 20))
shoebox[1] = Shoebox((-10, 20, 10, 20, 10, 20))
shoebox[2] = Shoebox((10, 20, -10, 20, 10, 20))
shoebox[3] = Shoebox((10, 20, 10, 20, -10, 20))
shoebox[4] = Shoebox((10, 1020, 10, 20, 10, 20))
shoebox[5] = Shoebox((10, 20, 10, 1020, 10, 20))
shoebox[6] = Shoebox((10, 20, 10, 20, 10, 1020))
assert(shoebox.is_bbox_within_image_volume(isize, srange) ==
flex.bool([True, False, False, False, False, False, False]))
# Test passed
print 'OK'
def tst_does_bbox_contain_bad_pixels(self):
from dials.array_family import flex
from dials.model.data import Shoebox
from random import randint
mask = flex.bool(flex.grid(100, 100), True)
for j in range(100):
for i in range(40, 60):
mask[j,i] = False
mask[i,j] = False
shoebox = flex.shoebox(1000)
res = flex.bool(1000)
for i in range(1000):
x0 = randint(0, 90)
y0 = randint(0, 90)
z0 = randint(0, 90)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))
res2 = False
if x0 >= 40 and x0 < 60:
res2 = True
if x1 > 40 and x1 <= 60:
res2 = True
if y0 >= 40 and y0 < 60:
res2 = True
if y1 > 40 and y1 <= 60:
res2 = True
res[i] = res2
assert(shoebox.does_bbox_contain_bad_pixels(mask) == res)
# Test passed
print 'OK'
def __init__(self):
from dials.algorithms.integration.profile import GridSampler2D
from dials.array_family import flex
from random import randint, uniform
# Number of reflections
nrefl = 1000
# Size of the images
width = 1000
height = 1000
# Create the grid
self.grid = GridSampler2D((width, height), (5, 5))
# Create the list of xyz and bboxes
self.xyz = flex.vec3_double(nrefl)
self.bbox = flex.int6(nrefl)
self.panel = flex.size_t(nrefl, 0)
for i in range(nrefl):
x0 = randint(0, width - 10)
x1 = x0 + randint(3, 10)
y0 = randint(0, height - 10)
y1 = y0 + randint(3, 10)
z0 = randint(0, 10)
z1 = z0 + randint(1, 10)
b = x0, x1, y0, y1, z0, z1
c = (x1 + x0) / 2, (y1 + y0) / 2, (z1 + z0) / 2
self.xyz[i] = c
self.bbox[i] = b
# Create the array of shoeboxes
self.sbox = flex.shoebox(self.panel, self.bbox)
self.sbox.allocate()
for i in range(len(self.sbox)):
data = self.sbox[i].data
for j in range(len(data)):
data[j] = uniform(0, 100)
def __init__(self):
from dials.algorithms.integration.profile import GridSampler2D
from dials.array_family import flex
from random import randint, uniform
# Number of reflections
nrefl = 1000
# Size of the images
width = 1000
height = 1000
# Create the grid
self.grid = GridSampler2D((width, height), (5,5))
# Create the list of xyz and bboxes
self.xyz = flex.vec3_double(nrefl)
self.bbox = flex.int6(nrefl)
self.panel = flex.size_t(nrefl, 0)
for i in range(nrefl):
x0 = randint(0,width-10)
x1 = x0 + randint(3,10)
y0 = randint(0,height-10)
y1 = y0 + randint(3,10)
z0 = randint(0,10)
z1 = z0 + randint(1,10)
b = x0, x1, y0, y1, z0, z1
c = (x1 + x0) / 2, (y1 + y0) / 2, (z1 + z0) / 2
self.xyz[i] = c
self.bbox[i] = b
# Create the array of shoeboxes
self.sbox = flex.shoebox(self.panel, self.bbox)
self.sbox.allocate()
for i in range(len(self.sbox)):
data = self.sbox[i].data
for j in range(len(data)):
data[j] = uniform(0, 100)
def tst_bounding_boxes(self):
from dials.model.data import Shoebox
from random import randint
from dials.array_family import flex
shoebox = flex.shoebox(10)
bbox = flex.int6(10)
for i in range(10):
x0 = randint(0, 90)
y0 = randint(0, 90)
z0 = randint(0, 90)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
bbox[i] = (x0, x1, y0, y1, z0, z1)
shoebox[i] = Shoebox(bbox[i])
bbox2 = shoebox.bounding_boxes()
for i in range(10):
assert(bbox2[i] == bbox[i])
# Test passed
print 'OK'
def tst_consistent(self):
from random import randint
from dials.model.data import Shoebox
from dials.array_family import flex
shoebox = flex.shoebox(10)
for i in range(10):
x0 = randint(0, 1000)
y0 = randint(0, 1000)
z0 = randint(0, 1000)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))
assert(shoebox.is_consistent() == flex.bool(10, False))
for i in range(10):
shoebox[i].allocate()
assert(shoebox.is_consistent() == flex.bool(10, True))
for i in [0, 2, 4, 6, 8]:
shoebox[i].data.resize(flex.grid(10, 10, 10))
assert(shoebox.is_consistent() == flex.bool([False, True] * 5))
for i in range(10):
shoebox[i].deallocate()
assert(shoebox.is_consistent() == flex.bool(10, False))
# Test passed
print 'OK'
def test_Simple1ProfileModel_compute_mask(simple1_profile_model,
test_experiment):
experiments = [test_experiment]
# Create the index generator
index_generator = IndexGenerator(
experiments[0].crystal.get_unit_cell(),
experiments[0].crystal.get_space_group().type(),
2.0,
)
# Get an array of miller indices
miller_indices = index_generator.to_array()
reflections = simple1_profile_model.predict_reflections(experiments,
miller_indices,
probability=0.9973)
simple1_profile_model.compute_bbox(experiments, reflections)
reflections["shoebox"] = flex.shoebox(reflections["panel"],
reflections["bbox"],
allocate=True)
simple1_profile_model.compute_mask(experiments, reflections)
def run(i, imp):
from random import randint
from dials.array_family import flex
#building a reflection table
num_ref = 5
ref_table = flex.reflection_table()
shoebox = flex.shoebox(num_ref)
ref_table['shoebox'] = shoebox
intensity = flex.double(num_ref)
ref_table['intensity.sum.value'] = intensity
intensity_var = flex.double(num_ref)
ref_table['intensity.sum.variance'] = intensity_var
iterate = ref_table['shoebox']
i_to_compare = []
# bulding the shoebox with a desired content
# which is a reflection with noise included
n = 0
for arr in iterate:
img = flex.double(flex.grid(3, 3, 3))
bkg = flex.double(flex.grid(3, 3, 3))
msk = flex.int(flex.grid(3, 3, 3))
for row in range(3):
for col in range(3):
for fra in range(3):
img[row, col, fra] = row + col + fra + n * 9 + randint(0, i)
bkg[row, col, fra] = 0.0
msk[row, col, fra] = 3
n += 1
msk[1, 1, 1] = 5
tmp_i = n * n * n * 3
i_to_compare.append(tmp_i)
img[1, 1, 1] += tmp_i
arr.data = img[:, :, :]
arr.background = bkg[:, :, :]
arr.mask = msk[:, :, :]
# calling the functions that we need to test
# first select the algorithm for background calculation
if imp == "inclined":
print "testing inclined_background_subtractor"
from dials.algorithms.background.inclined_background_subtractor \
import layering_and_background_plane
layering_and_background_plane(ref_table)
elif imp == "flat":
print "testing flat_background_subtractor"
from dials.algorithms.background.flat_background_subtractor \
import layering_and_background_avg
layering_and_background_avg(ref_table)
elif imp == "curved":
print "testing curved_background_subtractor"
from dials.algorithms.background.curved_background_subtractor \
import layering_and_background_modl
layering_and_background_modl(ref_table)
# no matter which algorithm was used for background calculation
# the integration summation must remain compatible
from dials.algorithms.integration.summation2d \
import flex_2d_layering_n_integrating
flex_2d_layering_n_integrating(ref_table)
# comparing results
result = "OK"
resl_its = ref_table['intensity.sum.value']
resl_var = ref_table['intensity.sum.variance']
for n_its in range(len(resl_its)):
if resl_its[n_its] <= i_to_compare[n_its] + i and \
resl_its[n_its] >= i_to_compare[n_its] - i and \
resl_var[n_its] > resl_its[n_its]:
print "Ok ", n_its
else:
print "Wrong num", n_its
print "i =", i
print "resl_its[n_its] =", resl_its[n_its]
print "i_to_compare[n_its] =", i_to_compare[n_its]
print "resl_var[n_its] =", resl_var[n_its]
result = "wrong"
raise RuntimeError('wrong result')
return result
def simple_gaussian_spots(params):
from dials.array_family import flex
from dials.algorithms import shoebox
import random
import math
from scitbx import matrix
r = params.rotation
axis = matrix.col((r.axis.x, r.axis.y, r.axis.z))
if axis.length() > 0:
rotation = axis.axis_and_angle_as_r3_rotation_matrix(r.angle, deg=True)
else:
rotation = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))
# generate mask and peak values
from dials.algorithms.shoebox import MaskCode
mask_peak = MaskCode.Valid|MaskCode.Foreground
mask_back = MaskCode.Valid|MaskCode.Background
from dials.util.command_line import ProgressBar
p = ProgressBar(title = 'Generating reflections')
rlist = flex.reflection_table(params.nrefl)
hkl = flex.miller_index(params.nrefl)
s1 = flex.vec3_double(params.nrefl)
xyzmm = flex.vec3_double(params.nrefl)
xyzpx = flex.vec3_double(params.nrefl)
panel = flex.size_t(params.nrefl)
bbox = flex.int6(params.nrefl)
for j in range(params.nrefl):
p.update(j * 100.0 / params.nrefl)
hkl[j] = (random.randint(0, 20),
random.randint(0, 20),
random.randint(0, 20))
phi = 2 * math.pi * random.random()
s1[j] = (0, 0, 0)
xyzpx[j] = (0, 0, 0)
xyzmm[j] = (0, 0, phi)
panel[j] = 0
bbox[j] = (0, params.shoebox_size.x,
0, params.shoebox_size.y,
0, params.shoebox_size.z)
p.finished('Generating %d reflections' % params.nrefl)
intensity = flex.double(params.nrefl)
shoebox = flex.shoebox(panel, bbox)
shoebox.allocate_with_value(MaskCode.Valid)
p = ProgressBar(title = 'Generating shoeboxes')
for i in range(len(rlist)):
p.update(i * 100.0 / params.nrefl)
mask = shoebox[i].mask
if params.pixel_mask == 'precise':
# flag everything as background: peak will me assigned later
for j in range(len(mask)):
mask[j] = mask_back
elif params.pixel_mask == 'all':
# flag we have no idea what anything is
mask_none = MaskCode.Valid|MaskCode.Foreground|MaskCode.Background
for j in range(len(mask)):
mask[j] = mask_none
elif params.pixel_mask == 'static':
import itertools
from scitbx.array_family import flex
x0 = params.spot_offset.x + params.shoebox_size.x / 2
y0 = params.spot_offset.x + params.shoebox_size.y / 2
z0 = params.spot_offset.x + params.shoebox_size.z / 2
sx = params.mask_nsigma * params.spot_size.x
sy = params.mask_nsigma * params.spot_size.y
sz = params.mask_nsigma * params.spot_size.z
# The x, y, z indices
z, y, x = zip(*itertools.product(*(range(n) for n in mask.all())))
xyz = flex.vec3_double(flex.double(x), flex.double(y), flex.double(z))
# Calculate SUM(((xj - xj0) / sxj)**2) for each element
xyz0 = (x0, y0, z0)
isxyz = (1.0/sx, 1.0/sy, 1.0/sz)
dxyz = sum([(x * isx)**2 for x, isx in
zip(((xyz - xyz0) * rotation).parts(), isxyz)])
# Set the mask values
index = dxyz <= 1.0
index.reshape(mask.accessor())
mask.set_selected(index, MaskCode.Valid | MaskCode.Foreground)
mask.set_selected(index != True, MaskCode.Valid | MaskCode.Background)
sbox = shoebox[i].data
# reflection itself, including setting the peak region if we're doing that
# FIXME use flex arrays to make the rotation bit more efficient as this is
# now rather slow...
counts_true = 0
for j in range(params.counts):
_x = random.gauss(0, params.spot_size.x)
_y = random.gauss(0, params.spot_size.y)
_z = random.gauss(0, params.spot_size.z)
Rxyz = rotation * matrix.col((_x, _y, _z)).elems
x = int(Rxyz[0] + params.spot_offset.x + params.shoebox_size.x / 2)
y = int(Rxyz[1] + params.spot_offset.y + params.shoebox_size.y / 2)
z = int(Rxyz[2] + params.spot_offset.z + params.shoebox_size.z / 2)
if x < 0 or x >= params.shoebox_size.x:
continue
if y < 0 or y >= params.shoebox_size.y:
continue
if z < 0 or z >= params.shoebox_size.z:
continue
sbox[z, y, x] += 1
counts_true += 1
if params.pixel_mask == 'precise':
mask[z, y, x] = mask_peak
intensity[i] = counts_true
if params.background:
# background:flat;
for j in range(params.background * len(sbox)):
x = random.randint(0, params.shoebox_size.x - 1)
y = random.randint(0, params.shoebox_size.y - 1)
z = random.randint(0, params.shoebox_size.z - 1)
sbox[z, y, x] += 1
else:
# or inclined
random_background_plane(sbox, params.background_a, params.background_b,
params.background_c, params.background_d)
rlist['miller_index'] = hkl
rlist['s1'] = s1
rlist['xyzcal.px'] = xyzpx
rlist['xyzcal.mm'] = xyzmm
rlist['bbox'] = bbox
rlist['panel'] = panel
rlist['shoebox'] = shoebox
rlist['intensity.sum.value'] = intensity
p.finished('Generating %d shoeboxes' % params.nrefl)
return rlist
def tst_extract_shoeboxes(self):
from dials.array_family import flex
from random import randint, seed
from dials.algorithms.shoebox import MaskCode
import sys
seed(0)
reflections = flex.reflection_table()
reflections['panel'] = flex.size_t()
reflections['bbox'] = flex.int6()
npanels = 2
width = 1000
height = 1000
nrefl = 10000
frame0 = 10
frame1 = 100
nrefl = 1000
for i in range(nrefl):
xs = randint(5, 10)
ys = randint(5, 10)
x0 = randint(-xs+1, width-1)
y0 = randint(-ys+1, height-1)
z0 = randint(frame0, frame1-1)
x1 = x0 + xs
y1 = y0 + ys
z1 = min([z0 + randint(1, 10), frame1])
assert(x1 > x0)
assert(y1 > y0)
assert(z1 > z0)
assert(z0 >= frame0 and z1 <= frame1)
bbox = (x0, x1, y0, y1, z0, z1)
reflections.append({
"panel" : randint(0,1),
"bbox" : bbox,
})
reflections['shoebox'] = flex.shoebox(
reflections['panel'],
reflections['bbox'])
reflections['shoebox'].allocate()
class FakeImageSet(object):
def __init__(self):
from dials.array_family import flex
self.data = flex.int(range(height*width))
self.data.reshape(flex.grid(height,width))
def get_array_range(self):
return (frame0, frame1)
def get_detector(self):
class FakeDetector(object):
def __len__(self):
return npanels
def __getitem__(self, index):
class FakePanel(object):
def get_trusted_range(self):
return (-1, 1000000)
return FakePanel()
return FakeDetector()
def __len__(self):
return frame1 - frame0
def __getitem__(self, index):
f = frame0+index
return (self.data + f*1, self.data + f*2)
def get_corrected_data(self, index):
f = frame0+index
return (self.data + f*1, self.data + f*2)
def get_mask(self, index):
image = self.get_corrected_data(index)
return tuple(im >= 0 for im in image)
imageset = FakeImageSet()
stdout = sys.stdout
class DevNull(object):
def write(self, *args):
pass
def flush(self):
pass
sys.stdout = DevNull()
reflections.extract_shoeboxes(imageset)
sys.stdout = stdout
for i in range(len(reflections)):
sbox = reflections[i]["shoebox"]
assert(sbox.is_consistent())
mask = sbox.mask
data = sbox.data
bbox = sbox.bbox
panel = sbox.panel
x0, x1, y0, y1, z0, z1 = bbox
for z in range(z1 - z0):
for y in range(y1 - y0):
for x in range(x1 - x0):
v1 = data[z,y,x]
m1 = mask[z,y,x]
if (x0 + x >= 0 and y0 + y >= 0 and
x0 + x < width and y0 + y < height):
v2 = imageset.data[y+y0,x+x0] + (z+z0)*(panel+1)
m2 = MaskCode.Valid
assert(v1 == v2)
assert(m1 == m2)
else:
assert(v1 == 0)
assert(m1 == 0)
print 'OK'
def test_extract_shoeboxes():
from dials.algorithms.shoebox import MaskCode
random.seed(0)
reflections = flex.reflection_table()
reflections["panel"] = flex.size_t()
reflections["bbox"] = flex.int6()
npanels = 2
width = 1000
height = 1000
frame0 = 10
frame1 = 100
nrefl = 1000
for i in range(nrefl):
xs = random.randint(5, 10)
ys = random.randint(5, 10)
x0 = random.randint(-xs + 1, width - 1)
y0 = random.randint(-ys + 1, height - 1)
z0 = random.randint(frame0, frame1 - 1)
x1 = x0 + xs
y1 = y0 + ys
z1 = min([z0 + random.randint(1, 10), frame1])
assert x1 > x0
assert y1 > y0
assert z1 > z0
assert z0 >= frame0 and z1 <= frame1
bbox = (x0, x1, y0, y1, z0, z1)
reflections.append({"panel": random.randint(0, 1), "bbox": bbox})
reflections["shoebox"] = flex.shoebox(reflections["panel"],
reflections["bbox"])
reflections["shoebox"].allocate()
class FakeImageSet(object):
def __init__(self):
self.data = flex.int(range(height * width))
self.data.reshape(flex.grid(height, width))
def get_array_range(self):
return (frame0, frame1)
def get_detector(self):
class FakeDetector(object):
def __len__(self):
return npanels
def __getitem__(self, index):
class FakePanel(object):
def get_trusted_range(self):
return (-1, 1000000)
return FakePanel()
return FakeDetector()
def __len__(self):
return frame1 - frame0
def __getitem__(self, index):
f = frame0 + index
return (self.data + f * 1, self.data + f * 2)
def get_corrected_data(self, index):
f = frame0 + index
return (self.data + f * 1, self.data + f * 2)
def get_mask(self, index):
image = self.get_corrected_data(index)
return tuple(im >= 0 for im in image)
imageset = FakeImageSet()
reflections.extract_shoeboxes(imageset)
for i in range(len(reflections)):
sbox = reflections[i]["shoebox"]
assert sbox.is_consistent()
mask = sbox.mask
data = sbox.data
bbox = sbox.bbox
panel = sbox.panel
x0, x1, y0, y1, z0, z1 = bbox
for z in range(z1 - z0):
for y in range(y1 - y0):
for x in range(x1 - x0):
v1 = data[z, y, x]
m1 = mask[z, y, x]
if (x0 + x >= 0 and y0 + y >= 0 and x0 + x < width
and y0 + y < height):
v2 = imageset.data[y + y0,
x + x0] + (z + z0) * (panel + 1)
m2 = MaskCode.Valid
assert v1 == v2
assert m1 == m2
else:
assert v1 == 0
assert m1 == 0
def set_reflections(self, reflections):
'''Set the reflection data.'''
self.set_data(reflections, ReflectionListEncoder())
def get_reflections(self):
'''Get the reflection data.'''
return self.get_data(ReflectionListDecoder())
if __name__ == '__main__':
from dials.array_family import flex
reflections = flex.reflection_table([('hkl', flex.miller_index(10)),
('s1', flex.vec3_double(10)),
('bbox', flex.int6(10)),
('id', flex.int(10)),
('shoebox', flex.shoebox(10))])
for i in range(10):
reflections['shoebox'][i].data = flex.double(flex.grid(10, 10, 10))
reflections['shoebox'][i].mask = flex.int(flex.grid(10, 10, 10))
reflections['shoebox'][i].background = flex.double(
flex.grid(10, 10, 10))
for i in range(10):
print reflections['shoebox'][i].data.all()
writer = NexusFile('temp.h5', 'w')
writer.set_reflections(reflections)
writer.close()
reader = NexusFile('temp.h5', 'r')
def reconstruct_peabox(params):
assert os.path.exists('xia2.json')
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
from dxtbx.model.experiment_list import ExperimentListFactory
import cPickle as pickle
import dials # because WARNING:root:No profile class gaussian_rs registered
from dials.array_family import flex
crystals = xinfo.get_crystals()
assert len(crystals) == 1
for xname in crystals:
crystal = crystals[xname]
scaler = crystal._get_scaler()
epochs = scaler._sweep_handler.get_epochs()
from xia2.command_line.rogues_gallery import munch_rogues
from pprint import pprint
batched_reflections = {}
for epoch in epochs:
si = scaler._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
experiments = ExperimentListFactory.from_json_file(
intgr.get_integrated_experiments())
reflections = pickle.load(open(intgr.get_integrated_reflections()))
batched_reflections[si.get_batch_range()] = (experiments, reflections,
si.get_sweep_name())
from dials.util import debug_console
# debug_console()
good = reflections.get_flags(reflections.flags.integrated)
# bad = reflections.get_flags(reflections.flags.bad_spot)
reflections = reflections.select(good)
for r in reflections:
flags = r['flags']
r['flags'] = []
for v, f in reflections.flags.values.iteritems():
if flags & f:
r['flags'].append(str(f))
r['flags'] = ', '.join(r['flags'])
# pprint(r)
print "Collecting shoeboxes for %d reflections" % len(reflections)
reflections["shoebox"] = flex.shoebox(reflections["panel"],
reflections["bbox"],
allocate=True)
reflections.extract_shoeboxes(experiments.imagesets()[0], verbose=True)
print "Consolidating..."
sizes = {}
for r in reflections:
s = r['shoebox']
a = s.size()
if a not in sizes:
sizes[a] = {
'sum': flex.float([0] * a[0] * a[1] * a[2]),
'weights': flex.int([0] * a[0] * a[1] * a[2]),
'count': 0
}
s.mask.set_selected(s.data < 0, 0)
s.data.set_selected(s.mask == 0, 0)
s.background.set_selected(s.mask == 0, 0)
sizes[a]['sum'] += s.data - s.background
sizes[a]['weights'] += s.mask
sizes[a]['count'] += 1
print len(sizes), "shoebox sizes extracted"
for s, c in sizes.iteritems():
print "%dx %s" % (c['count'], str(s))
sdat = iter(c['sum'])
wdat = iter(c['weights'])
for z in range(s[0]):
for y in range(s[1]):
count = [next(sdat) for x in range(s[2])]
weight = [next(wdat) for x in range(s[2])]
truecount = (0 if w == 0 else 10 * c / w
for c, w in zip(count, weight))
visualize = ("X" if c < 0 else
("." if c < 10 else str(int(math.log10(c))))
for c in truecount)
print "".join(visualize)
print ""
print ""
def simple_gaussian_spots(params):
from scitbx import matrix
from dials.array_family import flex
r = params.rotation
axis = matrix.col((r.axis.x, r.axis.y, r.axis.z))
if axis.length() > 0:
rotation = axis.axis_and_angle_as_r3_rotation_matrix(r.angle, deg=True)
else:
rotation = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))
# generate mask and peak values
from dials.algorithms.shoebox import MaskCode
mask_peak = MaskCode.Valid | MaskCode.Foreground
mask_back = MaskCode.Valid | MaskCode.Background
from dials.util.command_line import ProgressBar
p = ProgressBar(title="Generating reflections")
rlist = flex.reflection_table(params.nrefl)
hkl = flex.miller_index(params.nrefl)
s1 = flex.vec3_double(params.nrefl)
xyzmm = flex.vec3_double(params.nrefl)
xyzpx = flex.vec3_double(params.nrefl)
panel = flex.size_t(params.nrefl)
bbox = flex.int6(params.nrefl)
for j in range(params.nrefl):
p.update(j * 100.0 / params.nrefl)
hkl[j] = (random.randint(0, 20), random.randint(0, 20), random.randint(0, 20))
phi = 2 * math.pi * random.random()
s1[j] = (0, 0, 0)
xyzpx[j] = (0, 0, 0)
xyzmm[j] = (0, 0, phi)
panel[j] = 0
bbox[j] = (
0,
params.shoebox_size.x,
0,
params.shoebox_size.y,
0,
params.shoebox_size.z,
)
p.finished("Generating %d reflections" % params.nrefl)
intensity = flex.double(params.nrefl)
shoebox = flex.shoebox(panel, bbox)
shoebox.allocate_with_value(MaskCode.Valid)
p = ProgressBar(title="Generating shoeboxes")
for i in range(len(rlist)):
p.update(i * 100.0 / params.nrefl)
mask = shoebox[i].mask
if params.pixel_mask == "precise":
# flag everything as background: peak will me assigned later
for j in range(len(mask)):
mask[j] = mask_back
elif params.pixel_mask == "all":
# flag we have no idea what anything is
mask_none = MaskCode.Valid | MaskCode.Foreground | MaskCode.Background
for j in range(len(mask)):
mask[j] = mask_none
elif params.pixel_mask == "static":
from scitbx.array_family import flex
x0 = params.spot_offset.x + params.shoebox_size.x / 2
y0 = params.spot_offset.x + params.shoebox_size.y / 2
z0 = params.spot_offset.x + params.shoebox_size.z / 2
sx = params.mask_nsigma * params.spot_size.x
sy = params.mask_nsigma * params.spot_size.y
sz = params.mask_nsigma * params.spot_size.z
# The x, y, z indices
z, y, x = zip(*itertools.product(*(range(n) for n in mask.all())))
xyz = flex.vec3_double(flex.double(x), flex.double(y), flex.double(z))
# Calculate SUM(((xj - xj0) / sxj)**2) for each element
xyz0 = (x0, y0, z0)
isxyz = (1.0 / sx, 1.0 / sy, 1.0 / sz)
dxyz = sum(
(x * isx) ** 2
for x, isx in zip(((xyz - xyz0) * rotation).parts(), isxyz)
)
# Set the mask values
index = dxyz <= 1.0
index.reshape(mask.accessor())
mask.set_selected(index, MaskCode.Valid | MaskCode.Foreground)
mask.set_selected(not index, MaskCode.Valid | MaskCode.Background)
sbox = shoebox[i].data
# reflection itself, including setting the peak region if we're doing that
# FIXME use flex arrays to make the rotation bit more efficient as this is
# now rather slow...
counts_true = 0
for j in range(params.counts):
_x = random.gauss(0, params.spot_size.x)
_y = random.gauss(0, params.spot_size.y)
_z = random.gauss(0, params.spot_size.z)
Rxyz = rotation * matrix.col((_x, _y, _z)).elems
x = int(Rxyz[0] + params.spot_offset.x + params.shoebox_size.x / 2)
y = int(Rxyz[1] + params.spot_offset.y + params.shoebox_size.y / 2)
z = int(Rxyz[2] + params.spot_offset.z + params.shoebox_size.z / 2)
if x < 0 or x >= params.shoebox_size.x:
continue
if y < 0 or y >= params.shoebox_size.y:
continue
if z < 0 or z >= params.shoebox_size.z:
continue
sbox[z, y, x] += 1
counts_true += 1
if params.pixel_mask == "precise":
mask[z, y, x] = mask_peak
intensity[i] = counts_true
if params.background:
# background:flat;
for j in range(params.background * len(sbox)):
x = random.randint(0, params.shoebox_size.x - 1)
y = random.randint(0, params.shoebox_size.y - 1)
z = random.randint(0, params.shoebox_size.z - 1)
sbox[z, y, x] += 1
else:
# or inclined
random_background_plane(
sbox,
params.background_a,
params.background_b,
params.background_c,
params.background_d,
)
rlist["miller_index"] = hkl
rlist["s1"] = s1
rlist["xyzcal.px"] = xyzpx
rlist["xyzcal.mm"] = xyzmm
rlist["bbox"] = bbox
rlist["panel"] = panel
rlist["shoebox"] = shoebox
rlist["intensity.sum.value"] = intensity
p.finished("Generating %d shoeboxes" % params.nrefl)
return rlist
if __name__ == '__main__': from dials.array_family import flex from dxtbx.model.experiment.experiment_list import ExperimentListFactory import os.path path = "/home/upc86896/Projects/cctbx/sources/dials_regression/centroid_test_data" rlist_filename = os.path.join(path, "integrated.pickle") exlist_filename = os.path.join(path, "experiments.json") rlist = flex.reflection_table.from_pickle(rlist_filename) exlist = ExperimentListFactory.from_json_file(exlist_filename) panel = rlist['panel'] bbox = rlist['bbox'] rlist['shoebox'] = flex.shoebox(panel, bbox) rlist['shoebox'].allocate() rlist.fill_shoeboxes(exlist[0].imageset)
from dials.array_family import flex
num_ref = 3
ref_table = flex.reflection_table()
shoebox = flex.shoebox(num_ref)
ref_table['shoebox'] = shoebox
intensity = flex.double(num_ref)
ref_table['intensity.sum.value'] = intensity
intensity_var = flex.double(num_ref)
ref_table['intensity.sum.variance'] = intensity_var
iterate = ref_table['shoebox']
n = 0
for arr in iterate:
img = flex.double(flex.grid(3, 3, 3))
bkg = flex.double(flex.grid(3, 3, 3))
msk = flex.int(flex.grid(3, 3, 3))
for row in range(3):
for col in range(3):
for fra in range(3):
img[row, col, fra] = row + col + fra + n * 9
bkg[row, col, fra] = 0.0
msk[row, col, fra] = 3
n += 1
msk[1, 1, 1] = 5
tmp_i = n * n * n * 3
img[1, 1, 1] += tmp_i
print "intensity must be =", tmp_i
arr.data = img[:, :, :]
import numpy
data2d = numpy.zeros((3, 3), dtype = numpy.float64)
data2d[:, :] = 15
data2d[1:2, 1:2] = 50
for row in range(3):
for col in range(3):
data2d[row, col] += row * 2
data2d[row, col] += col * 2
ref_table = flex.reflection_table()
'''
shoebox = flex.shoebox(5)
#panel = flex.size_t(1)
ref_table = flex.reflection_table()
shoebox.data = flex.double(flex.grid(6, 6, 6))
ref_table['shoebox'] = shoebox
#ref_table['panel'] = panel
#efisient_way = '''
its = ref_table['shoebox']
for arr in its:
print arr.data.as_numpy_array()
#'''
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'FAIL: dials_regression not configured'
exit(0)
path = join(dials_regression, "centroid_test_data")
import sys
assert(len(sys.argv) == 1)
sys.argv.append(join(path, "experiments.json"))
sys.argv.append(join(path, "profile.phil"))
parser = OptionParser(phil=phil_scope)
params, options, args = parser.parse_args()
assert(len(args) == 1)
exlist = ExperimentListFactory.from_json_file(args[0])
assert(len(exlist) == 1)
profile_model = ProfileModelList.load(params)
rlist = flex.reflection_table.from_predictions_multi(exlist)
rlist.compute_bbox(exlist, profile_model)
rlist['shoebox'] = flex.shoebox(rlist['panel'], rlist['bbox'])
rlist['shoebox'].allocate()
rlist.extract_shoeboxes(exlist[0].imageset)
show_reflection(rlist[len(rlist)//2])
#show_reflection(rlist[len(rlist)//2], orient = "porTrait")
#show_reflection(rlist[len(rlist)//2], orient = "lanDscape")
def __call__(self, imageset):
"""
Find the spots in the imageset
:param imageset: The imageset to process
:return: The list of spot shoeboxes
"""
from dials.util.command_line import Command
from dials.array_family import flex
from dxtbx.imageset import ImageSweep
from dials.model.data import PixelListLabeller
from logging import info, warn
from math import floor
import platform
from logging import warn
# Change the number of processors if necessary
mp_nproc = self.nproc
if platform.system() == "Windows" and mp_nproc > 1:
warn("")
warn("*" * 80)
warn("Multiprocessing is not available on windows. Setting nproc = 1")
warn("*" * 80)
warn("")
mp_nproc = 1
if mp_nproc > len(imageset):
mp_nproc = len(imageset)
mp_method = self.mp_method
mp_chunksize = self.mp_chunksize
len_by_nproc = int(floor(len(imageset) / mp_nproc))
if mp_chunksize > len_by_nproc:
mp_chunksize = len_by_nproc
assert mp_nproc > 0, "Invalid number of processors"
assert mp_chunksize > 0, "Invalid chunk size"
# The extract pixels function
function = ExtractPixelsFromImage(
imageset=imageset,
threshold_function=self.threshold_function,
mask=self.mask,
max_strong_pixel_fraction=self.max_strong_pixel_fraction,
region_of_interest=self.region_of_interest,
)
# The indices to iterate over
indices = list(range(len(imageset)))
# Initialise the pixel labeller
num_panels = len(imageset.get_detector())
pixel_labeller = [PixelListLabeller() for p in range(num_panels)]
# Do the processing
info("Extracting strong pixels from images")
info(" Using %s with %d parallel job(s)\n" % (mp_method, mp_nproc))
if mp_nproc > 1:
def process_output(result):
import logging
for message in result[1]:
logging.log(message.levelno, message.msg)
assert len(pixel_labeller) == len(result[0].pixel_list), "Inconsistent size"
for plabeller, plist in zip(pixel_labeller, result[0].pixel_list):
plabeller.add(plist)
result[0].pixel_list = None
# Parallel reading of HDF5 from the same handle is not allowed. Python
# multiprocessing is a bit messed up and used fork on linux so need to
# close and reopen file.
from dxtbx.imageset import SingleFileReader
if isinstance(imageset.reader(), SingleFileReader):
imageset.reader().nullify_format_instance()
batch_parallel_map(
func=ExtractSpotsParallelTask(function),
iterable=indices,
processes=mp_nproc,
callback=process_output,
method=mp_method,
chunksize=mp_chunksize,
)
else:
for task in indices:
result = function(task)
assert len(pixel_labeller) == len(result.pixel_list), "Inconsistent size"
for plabeller, plist in zip(pixel_labeller, result.pixel_list):
plabeller.add(plist)
result.pixel_list = None
# Extract the pixel lists into a list of reflections
shoeboxes = flex.shoebox()
if isinstance(imageset, ImageSweep):
twod = False
else:
twod = True
for i, p in enumerate(pixel_labeller):
if p.num_pixels() > 0:
shoeboxes.extend(
flex.shoebox(
p, i, 0, twod, self.min_spot_size, self.max_spot_size # panel # zrange # twod # min_pixels
)
) # max_pixels
info("")
info("Extracted {0} spots".format(len(shoeboxes)))
# Get the unallocated spots and print some info
selection = shoeboxes.is_allocated()
shoeboxes = shoeboxes.select(selection)
info(
"Removed %d spots with size < %d or > %d pixels"
% (selection.count(False), self.min_spot_size, self.max_spot_size)
)
# Return the shoeboxes
return shoeboxes
def run(self):
''' Extract the shoeboxes. '''
from dials.util.options import flatten_reflections
from dials.util.options import flatten_experiments
from dials.util.options import flatten_datablocks
from dials.util import log
from dials.array_family import flex
from libtbx.utils import Sorry
from logging import info
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
# Configure logging
log.config()
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
info('The following parameters have been modified:\n')
info(diff_phil)
# Get the data
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
datablocks = flatten_datablocks(params.input.datablock)
if len(experiments) == 0 and len(datablocks) == 0 and len(reflections) == 0:
self.parser.print_help()
exit(0)
elif (len(experiments) != 0 and len(datablocks) != 0):
raise Sorry('Both experiment list and datablocks set')
elif len(experiments) > 1:
raise Sorry('More than 1 experiment set')
elif len(datablocks) > 1:
raise Sorry('More than 1 datablock set')
elif len(experiments) == 1:
imageset = experiments[0].imageset
elif len(datablocks) == 1:
imagesets = datablocks[0].extract_imagesets()
if len(imagesets) != 1:
raise Sorry('Need 1 imageset, got %d' % len(imagesets))
imageset = imagesets[0]
if len(reflections) != 1:
raise Sorry('Need 1 reflection table, got %d' % len(reflections))
else:
reflections = reflections[0]
# Check the reflections contain the necessary stuff
assert("bbox" in reflections)
assert("panel" in reflections)
# Get some models
detector = imageset.get_detector()
scan = imageset.get_scan()
frame0, frame1 = scan.get_array_range()
# Add some padding but limit to image volume
if params.padding > 0:
info('Adding %d pixels as padding' % params.padding)
x0, x1, y0, y1, z0, z1 = reflections['bbox'].parts()
x0 -= params.padding
x1 += params.padding
y0 -= params.padding
y1 += params.padding
z0 -= params.padding
z1 += params.padding
panel = reflections['panel']
for i in range(len(reflections)):
width, height = detector[panel[i]].get_image_size()
if z0[i] < frame0: z0[i] = frame0
if z1[i] > frame1: z1[i] = frame1
reflections['bbox'] = flex.int6(x0, x1, y0, y1, z0, z1)
# Save the old shoeboxes
if "shoebox" in reflections:
old_shoebox = reflections['shoebox']
else:
old_shoebox = None
# Allocate the shoeboxes
reflections["shoebox"] = flex.shoebox(
reflections["panel"],
reflections["bbox"],
allocate=True)
# Extract the shoeboxes
reflections.extract_shoeboxes(imageset, verbose=True)
# Preserve masking
if old_shoebox is not None:
info("Applying old shoebox mask")
new_shoebox = reflections['shoebox']
for i in range(len(reflections)):
bbox0 = old_shoebox[i].bbox
bbox1 = new_shoebox[i].bbox
mask0 = old_shoebox[i].mask
mask1 = new_shoebox[i].mask
mask2 = flex.int(mask1.accessor(), 0)
x0 = bbox0[0] - bbox1[0]
x1 = bbox0[1] - bbox0[0] + x0
y0 = bbox0[2] - bbox1[2]
y1 = bbox0[3] - bbox0[2] + y0
z0 = bbox0[4] - bbox1[4]
z1 = bbox0[5] - bbox0[4] + z0
mask2[z0:z1,y0:y1,x0:x1] = mask0
mask1 = mask1.as_1d() | mask2.as_1d()
mask1.reshape(new_shoebox[i].mask.accessor())
new_shoebox[i].mask = mask1
# Saving the reflections to disk
filename = params.output.reflections
info('Saving %d reflections to %s' % (len(reflections), filename))
reflections.as_pickle(filename)
def reconstruct_rogues(params):
assert os.path.exists('xia2.json')
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
import cPickle as pickle
import dials # because WARNING:root:No profile class gaussian_rs registered
crystals = xinfo.get_crystals()
assert len(crystals) == 1
for xname in crystals:
crystal = crystals[xname]
scaler = crystal._get_scaler()
epochs = scaler._sweep_handler.get_epochs()
rogues = os.path.join(scaler.get_working_directory(),
xname, 'scale', 'ROGUES')
rogue_reflections = munch_rogues(rogues)
batched_reflections = { }
for epoch in epochs:
si = scaler._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
experiments = ExperimentListFactory.from_json_file(
intgr.get_integrated_experiments())
reflections = pickle.load(open(intgr.get_integrated_reflections()))
batched_reflections[si.get_batch_range()] = (experiments, reflections,
si.get_sweep_name())
# - look up reflection in reflection list, get bounding box
# - pull pixels given from image set, flatten these, write out
from dials.array_family import flex
from annlib_ext import AnnAdaptor as ann_adaptor
reflections_run = { }
for run in batched_reflections:
reflections_run[run] = []
for rogue in rogue_reflections:
b = rogue[0]
for run in batched_reflections:
if b >= run[0] and b <= run[1]:
reflections_run[run].append(rogue)
break
for run_no, run in enumerate(reflections_run):
experiment = batched_reflections[run][0]
reflections = batched_reflections[run][1]
name = batched_reflections[run][2]
rogues = reflections_run[run]
reference = flex.double()
scan = experiment.scans()[0]
images = experiment.imagesets()[0]
for xyz in reflections['xyzcal.px']:
reference.append(xyz[0])
reference.append(xyz[1])
reference.append(xyz[2])
search = flex.double()
for rogue in rogues:
search.append(rogue[1])
search.append(rogue[2])
search.append(scan.get_array_index_from_angle(rogue[3]))
ann = ann_adaptor(data=reference, dim=3, k=1)
ann.query(search)
keep = flex.bool(len(reflections), False)
for j, rogue in enumerate(rogues):
keep[ann.nn[j]] = True
reflections = reflections.select(keep==True)
if params.extract:
reflections["shoebox"] = flex.shoebox(
reflections["panel"],
reflections["bbox"],
allocate=True)
reflections.extract_shoeboxes(images, verbose=False)
if len(reflections_run) > 1:
output = params.output.reflections.replace(
'.pickle', '-%s.pickle' % name)
print 'Extracted %d rogue reflections for %s to %s' % \
(len(reflections), name, output)
reflections.as_pickle(output)
else:
output = params.output.reflections
print 'Extracted %d rogue reflections to %s' % \
(len(reflections), output)
reflections.as_pickle(output)
def run(self):
''' Extract the shoeboxes. '''
from dials.util.options import flatten_reflections
from dials.util.options import flatten_experiments
from dials.util.options import flatten_datablocks
from dials.util import log
from dials.array_family import flex
from libtbx.utils import Sorry
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
# Configure logging
log.config()
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
# Get the data
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
datablocks = flatten_datablocks(params.input.datablock)
if not any([experiments, datablocks, reflections]):
self.parser.print_help()
exit(0)
elif experiments and datablocks:
raise Sorry('Both experiment list and datablocks set')
elif len(experiments) > 1:
raise Sorry('More than 1 experiment set')
elif len(datablocks) > 1:
raise Sorry('More than 1 datablock set')
elif len(experiments) == 1:
imageset = experiments[0].imageset
elif len(datablocks) == 1:
imagesets = datablocks[0].extract_imagesets()
if len(imagesets) != 1:
raise Sorry('Need 1 imageset, got %d' % len(imagesets))
imageset = imagesets[0]
if len(reflections) != 1:
raise Sorry('Need 1 reflection table, got %d' % len(reflections))
else:
reflections = reflections[0]
# Check the reflections contain the necessary stuff
assert("bbox" in reflections)
assert("panel" in reflections)
# Get some models
detector = imageset.get_detector()
scan = imageset.get_scan()
frame0, frame1 = scan.get_array_range()
# Add some padding but limit to image volume
if params.padding > 0:
logger.info('Adding %d pixels as padding' % params.padding)
x0, x1, y0, y1, z0, z1 = reflections['bbox'].parts()
x0 -= params.padding
x1 += params.padding
y0 -= params.padding
y1 += params.padding
# z0 -= params.padding
# z1 += params.padding
panel = reflections['panel']
for i in range(len(reflections)):
width, height = detector[panel[i]].get_image_size()
if x0[i] < 0: x0[i] = 0
if x1[i] > width: x1[i] = width
if y0[i] < 0: y0[i] = 0
if y1[i] > height: y1[i] = height
if z0[i] < frame0: z0[i] = frame0
if z1[i] > frame1: z1[i] = frame1
reflections['bbox'] = flex.int6(x0, x1, y0, y1, z0, z1)
# Save the old shoeboxes
if "shoebox" in reflections:
old_shoebox = reflections['shoebox']
else:
old_shoebox = None
# Allocate the shoeboxes
reflections["shoebox"] = flex.shoebox(
reflections["panel"],
reflections["bbox"],
allocate=True)
# Extract the shoeboxes
reflections.extract_shoeboxes(imageset, verbose=True)
# Preserve masking
if old_shoebox is not None:
from dials.algorithms.shoebox import MaskCode
logger.info("Applying old shoebox mask")
new_shoebox = reflections['shoebox']
for i in range(len(reflections)):
bbox0 = old_shoebox[i].bbox
bbox1 = new_shoebox[i].bbox
mask0 = old_shoebox[i].mask
mask1 = new_shoebox[i].mask
mask2 = flex.int(mask1.accessor(), 0)
x0 = bbox0[0] - bbox1[0]
x1 = bbox0[1] - bbox0[0] + x0
y0 = bbox0[2] - bbox1[2]
y1 = bbox0[3] - bbox0[2] + y0
z0 = bbox0[4] - bbox1[4]
z1 = bbox0[5] - bbox0[4] + z0
mask2[z0:z1,y0:y1,x0:x1] = mask0
mask1 = mask1.as_1d() | mask2.as_1d()
if params.padding_is_background:
selection = flex.size_t(range(len(mask1))).select(mask1 == MaskCode.Valid)
values = flex.int(len(selection), MaskCode.Valid | MaskCode.Background)
mask1.set_selected(selection, values)
mask1.reshape(new_shoebox[i].mask.accessor())
new_shoebox[i].mask = mask1
# Saving the reflections to disk
filename = params.output.reflections
logger.info('Saving %d reflections to %s' % (len(reflections), filename))
reflections.as_pickle(filename)
def __call__(self):
'''
Do the processing.
:return: The processed data
'''
from dials.array_family import flex
from time import time
from dials.model.data import make_image
from libtbx.introspection import machine_memory_info
# Get the start time
start_time = time()
# Set the global process ID
job.index = self.index
# Check all reflections have same imageset and get it
exp_id = list(set(self.reflections['id']))
imageset = self.experiments[exp_id[0]].imageset
for i in exp_id[1:]:
assert self.experiments[i].imageset == imageset, "Task can only handle 1 imageset"
# Get the sub imageset
frame00, frame01 = self.job
try:
frame10, frame11 = imageset.get_array_range()
except Exception:
frame10, frame11 = (0, len(imageset))
try:
assert frame00 < frame01
assert frame10 < frame11
assert frame00 >= frame10
assert frame01 <= frame11
index0 = frame00 - frame10
index1 = index0 + (frame01 - frame00)
assert index0 < index1
assert index0 >= 0
assert index1 <= len(imageset)
imageset = imageset[index0:index1]
except Exception:
raise RuntimeError('Programmer Error: bad array range')
try:
frame0, frame1 = imageset.get_array_range()
except Exception:
frame0, frame1 = (0, len(imageset))
# Initlize the executor
self.executor.initialize(frame0, frame1, self.reflections)
# Set the shoeboxes (dont't allocate)
self.reflections['shoebox'] = flex.shoebox(
self.reflections['panel'],
self.reflections['bbox'],
allocate=False,
flatten=self.params.shoebox.flatten)
# Create the processor
processor = ShoeboxProcessor(
self.reflections,
len(imageset.get_detector()),
frame0,
frame1,
self.params.debug.output)
# Compute percentage of max available. The function is not portable to
# windows so need to add a check if the function fails. On windows no
# warning will be printed
memory_info = machine_memory_info()
total_memory = memory_info.memory_total()
sbox_memory = processor.compute_max_memory_usage()
if total_memory is not None:
assert total_memory > 0, "Your system appears to have no memory!"
assert self.params.block.max_memory_usage > 0.0, "maximum memory usage must be > 0"
assert self.params.block.max_memory_usage <= 1.0, "maximum memory usage must be <= 1"
limit_memory = total_memory * self.params.block.max_memory_usage
if sbox_memory > limit_memory:
raise RuntimeError('''
There was a problem allocating memory for shoeboxes. Possible solutions
include increasing the percentage of memory allowed for shoeboxes or
decreasing the block size. This could also be caused by a highly mosaic
crystal model - is your crystal really this mosaic?
Total system memory: %g GB
Limit shoebox memory: %g GB
Required shoebox memory: %g GB
''' % (total_memory/1e9, limit_memory/1e9, sbox_memory/1e9))
else:
logger.info(' Memory usage:')
logger.info(' Total system memory: %g GB' % (total_memory/1e9))
logger.info(' Limit shoebox memory: %g GB' % (limit_memory/1e9))
logger.info(' Required shoebox memory: %g GB' % (sbox_memory/1e9))
logger.info('')
# Loop through the imageset, extract pixels and process reflections
read_time = 0.0
for i in range(len(imageset)):
st = time()
image = imageset.get_corrected_data(i)
mask = imageset.get_mask(i)
if self.params.lookup.mask is not None:
assert len(mask) == len(self.params.lookup.mask), \
"Mask/Image are incorrect size %d %d" % (
len(mask),
len(self.params.lookup.mask))
mask = tuple(m1 & m2 for m1, m2 in zip(self.params.lookup.mask, mask))
read_time += time() - st
processor.next(make_image(image, mask), self.executor)
del image
del mask
assert processor.finished(), "Data processor is not finished"
# Optionally save the shoeboxes
if self.params.debug.output and self.params.debug.separate_files:
output = self.reflections
if self.params.debug.select is not None:
output = output.select(self.params.debug.select(output))
if self.params.debug.split_experiments:
output = output.split_by_experiment_id()
for table in output:
i = table['id'][0]
table.as_pickle('shoeboxes_%d_%d.pickle' % (self.index, i))
else:
output.as_pickle('shoeboxes_%d.pickle' % self.index)
# Delete the shoeboxes
if self.params.debug.separate_files or not self.params.debug.output:
del self.reflections['shoebox']
# Finalize the executor
self.executor.finalize()
# Return the result
result = Result(self.index, self.reflections, self.executor.data())
result.read_time = read_time
result.extract_time = processor.extract_time()
result.process_time = processor.process_time()
result.total_time = time() - start_time
return result
def __call__(self):
"""
Do the processing.
:return: The processed data
"""
# Get the start time
start_time = time()
# Set the global process ID
job.index = self.index
# Check all reflections have same imageset and get it
exp_id = list(set(self.reflections["id"]))
imageset = self.experiments[exp_id[0]].imageset
for i in exp_id[1:]:
assert (self.experiments[i].imageset == imageset
), "Task can only handle 1 imageset"
# Get the sub imageset
frame0, frame1 = self.job
try:
allowed_range = imageset.get_array_range()
except Exception:
allowed_range = 0, len(imageset)
try:
# range increasing
assert frame0 < frame1
# within an increasing range
assert allowed_range[1] > allowed_range[0]
# we are processing data which is within range
assert frame0 >= allowed_range[0]
assert frame1 <= allowed_range[1]
# I am 99% sure this is implied by all the code above
assert (frame1 - frame0) <= len(imageset)
if len(imageset) > 1:
imageset = imageset[frame0:frame1]
except Exception as e:
raise RuntimeError(f"Programmer Error: bad array range: {e}")
try:
frame0, frame1 = imageset.get_array_range()
except Exception:
frame0, frame1 = (0, len(imageset))
self.executor.initialize(frame0, frame1, self.reflections)
# Set the shoeboxes (don't allocate)
self.reflections["shoebox"] = flex.shoebox(
self.reflections["panel"],
self.reflections["bbox"],
allocate=False,
flatten=self.params.shoebox.flatten,
)
# Create the processor
processor = ShoeboxProcessor(
self.reflections,
len(imageset.get_detector()),
frame0,
frame1,
self.params.debug.output,
)
# Loop through the imageset, extract pixels and process reflections
read_time = 0.0
for i in range(len(imageset)):
st = time()
image = imageset.get_corrected_data(i)
if imageset.is_marked_for_rejection(i):
mask = tuple(flex.bool(im.accessor(), False) for im in image)
else:
mask = imageset.get_mask(i)
if self.params.lookup.mask is not None:
assert len(mask) == len(
self.params.lookup.mask
), "Mask/Image are incorrect size %d %d" % (
len(mask),
len(self.params.lookup.mask),
)
mask = tuple(
m1 & m2
for m1, m2 in zip(self.params.lookup.mask, mask))
read_time += time() - st
processor.next(make_image(image, mask), self.executor)
del image
del mask
assert processor.finished(), "Data processor is not finished"
# Optionally save the shoeboxes
if self.params.debug.output and self.params.debug.separate_files:
output = self.reflections
if self.params.debug.select is not None:
output = output.select(self.params.debug.select(output))
if self.params.debug.split_experiments:
output = output.split_by_experiment_id()
for table in output:
i = table["id"][0]
table.as_file("shoeboxes_%d_%d.refl" % (self.index, i))
else:
output.as_file("shoeboxes_%d.refl" % self.index)
# Delete the shoeboxes
if self.params.debug.separate_files or not self.params.debug.output:
del self.reflections["shoebox"]
# Finalize the executor
self.executor.finalize()
# Return the result
return dials.algorithms.integration.Result(
index=self.index,
reflections=self.reflections,
data=self.executor.data(),
read_time=read_time,
extract_time=processor.extract_time(),
process_time=processor.process_time(),
total_time=time() - start_time,
)
reflection_lookup[j].append(i)
width, height = experiments[0].detector[0].get_image_size()
sum_background = flex.double(flex.grid(height, width), 0)
sum_sq_background = flex.double(flex.grid(height, width), 0)
count = flex.int(flex.grid(height, width), 0)
# Loop through all images
print "START"
for frame in range(len(imageset)):
# Get the subset of reflections on this image and compute the mask
subset = reflections.select(flex.size_t(reflection_lookup[frame]))
subset['shoebox'] = flex.shoebox(
subset['panel'],
subset['bbox'],
allocate=True)
subset.compute_mask(experiments)
# Get the mask and data
mask = imageset.get_mask(frame)[0]
data = imageset.get_raw_data(frame)[0]
sbox_mask = subset['shoebox'].apply_background_mask(frame, 1, (height, width))
mask = mask & sbox_mask
#from dials.algorithms.image.threshold import DispersionThreshold
#threshold = DispersionThreshold(
# data.all(),
def generate_predictions(self, N):
"""Generate some reflections."""
from dials.algorithms import filtering
from dials.algorithms.profile_model.gaussian_rs import MaskCalculator3D
from dials.algorithms.profile_model.gaussian_rs import Model as ProfileModel
from dials.algorithms.shoebox import MaskCode
from dials.util.command_line import Command
# Set the profile model
self.experiment.profile = ProfileModel(None, self.n_sigma,
self.sigma_b, self.sigma_m)
# Generate a list of reflections
refl = flex.reflection_table.from_predictions(self.experiment)
refl["id"] = flex.int(len(refl), 0)
# Filter by zeta
zeta = 0.05
Command.start(f"Filtering by zeta >= {zeta:f}")
mask = filtering.by_zeta(self.experiment.goniometer,
self.experiment.beam, refl["s1"], zeta)
refl.del_selected(~mask)
Command.end(f"Filtered {len(refl)} reflections by zeta >= {zeta:f}")
# Compute the bounding box
refl.compute_bbox([self.experiment])
index = []
image_size = self.experiment.detector[0].get_image_size()
array_range = self.experiment.scan.get_array_range()
bbox = refl["bbox"]
for i in range(len(refl)):
x0, x1, y0, y1, z0, z1 = bbox[i]
if (x0 < 0 or x1 > image_size[0] or y0 < 0 or y1 > image_size[1]
or z0 < array_range[0] or z1 > array_range[1]):
index.append(i)
refl.del_selected(flex.size_t(index))
# Sample if specified
index = random.sample(range(len(refl)), N)
refl = refl.select(flex.size_t(index))
# Compute the bounding box
# Create a load of shoeboxes
Command.start(f"Creating shoeboxes for {len(refl)} reflections")
refl["shoebox"] = flex.shoebox(refl["panel"], refl["bbox"])
refl["shoebox"].allocate_with_value(MaskCode.Valid)
Command.end(f"Created shoeboxes for {len(refl)} reflections")
# Get the function object to mask the foreground
Command.start(f"Masking Foreground for {len(refl)} reflections")
mask_foreground = MaskCalculator3D(
self.experiment.beam,
self.experiment.detector,
self.experiment.goniometer,
self.experiment.scan,
self.n_sigma * self.sigma_b,
self.n_sigma * self.sigma_m,
)
# Mask the foreground
mask_foreground(refl["shoebox"], refl["s1"],
refl["xyzcal.px"].parts()[2], refl["panel"])
Command.end(f"Masked foreground for {len(refl)} reflections")
# Return the reflections
return refl
imageset = experiments[0].imageset
print list(reflections.keys())
reflections.unset_flags(flex.size_t(range(len(reflections))), reflections.flags.integrated_sum)
reflections.unset_flags(flex.size_t(range(len(reflections))), reflections.flags.integrated_prf)
del reflections['intensity.sum.value']
del reflections['intensity.sum.variance']
del reflections['background.mean']
del reflections['intensity.prf.value']
del reflections['intensity.prf.variance']
del reflections['profile.correlation']
del reflections['profile.rmsd']
# Create the reflection lookup
reflections['shoebox'] = flex.shoebox(
reflections['panel'],
reflections['bbox'])
bbox = reflections['bbox']
reflection_lookup = defaultdict(list)
for i in range(len(bbox)):
for j in range(bbox[i][4],bbox[i][5]):
reflection_lookup[j].append(i)
width, height = experiments[0].detector[0].get_image_size()
# Loop through all images
print "START"
for frame in range(0, len(imageset)):
# Get the subset of reflections on this image and compute the mask
indices = flex.size_t(reflection_lookup[frame])
def test_to_from_msgpack(tmpdir):
from dials.model.data import Shoebox
def gen_shoebox():
shoebox = Shoebox(0, (0, 4, 0, 3, 0, 1))
shoebox.allocate()
for k in range(1):
for j in range(3):
for i in range(4):
shoebox.data[k, j, i] = i + j + k + 0.1
shoebox.mask[k, j, i] = i % 2
shoebox.background[k, j, i] = i * j + 0.2
return shoebox
def compare(a, b):
assert a.is_consistent()
assert b.is_consistent()
assert a.panel == b.panel
assert a.bbox == b.bbox
for aa, bb in zip(a.data, b.data):
if abs(aa - bb) > 1e-9:
return False
for aa, bb in zip(a.background, b.background):
if abs(aa - bb) > 1e-9:
return False
for aa, bb in zip(a.mask, b.mask):
if aa != bb:
return False
return True
# The columns as lists
c1 = list(range(10))
c2 = list(range(10))
c3 = ["a", "b", "c", "d", "e", "f", "g", "i", "j", "k"]
c4 = [True, False, True, False, True] * 2
c5 = list(range(10))
c6 = [(i + 1, i + 2) for i in range(10)]
c7 = [(i + 1, i + 2, i + 3) for i in range(10)]
c8 = [tuple(i + j for j in range(9)) for i in range(10)]
c9 = [tuple(i + j for j in range(6)) for i in range(10)]
c10 = [(i + 1, i + 2, i + 3) for i in range(10)]
c11 = [gen_shoebox() for i in range(10)]
# Create a table with some elements
table = flex.reflection_table()
table["col1"] = flex.int(c1)
table["col2"] = flex.double(c2)
table["col3"] = flex.std_string(c3)
table["col4"] = flex.bool(c4)
table["col5"] = flex.size_t(c5)
table["col6"] = flex.vec2_double(c6)
table["col7"] = flex.vec3_double(c7)
table["col8"] = flex.mat3_double(c8)
table["col9"] = flex.int6(c9)
table["col10"] = flex.miller_index(c10)
table["col11"] = flex.shoebox(c11)
obj = table.as_msgpack()
new_table = flex.reflection_table.from_msgpack(obj)
assert new_table.is_consistent()
assert new_table.nrows() == 10
assert new_table.ncols() == 11
assert all(tuple(a == b for a, b in zip(new_table["col1"], c1)))
assert all(tuple(a == b for a, b in zip(new_table["col2"], c2)))
assert all(tuple(a == b for a, b in zip(new_table["col3"], c3)))
assert all(tuple(a == b for a, b in zip(new_table["col4"], c4)))
assert all(tuple(a == b for a, b in zip(new_table["col5"], c5)))
assert all(tuple(a == b for a, b in zip(new_table["col6"], c6)))
assert all(tuple(a == b for a, b in zip(new_table["col7"], c7)))
assert all(tuple(a == b for a, b in zip(new_table["col8"], c8)))
assert all(tuple(a == b for a, b in zip(new_table["col9"], c9)))
assert all(tuple(a == b for a, b in zip(new_table["col10"], c10)))
assert all(tuple(compare(a, b) for a, b in zip(new_table["col11"], c11)))
table.as_msgpack_file(tmpdir.join("reflections.mpack").strpath)
new_table = flex.reflection_table.from_msgpack_file(
tmpdir.join("reflections.mpack").strpath)
assert new_table.is_consistent()
assert new_table.nrows() == 10
assert new_table.ncols() == 11
assert all(tuple(a == b for a, b in zip(new_table["col1"], c1)))
assert all(tuple(a == b for a, b in zip(new_table["col2"], c2)))
assert all(tuple(a == b for a, b in zip(new_table["col3"], c3)))
assert all(tuple(a == b for a, b in zip(new_table["col4"], c4)))
assert all(tuple(a == b for a, b in zip(new_table["col5"], c5)))
assert all(tuple(a == b for a, b in zip(new_table["col6"], c6)))
assert all(tuple(a == b for a, b in zip(new_table["col7"], c7)))
assert all(tuple(a == b for a, b in zip(new_table["col8"], c8)))
assert all(tuple(a == b for a, b in zip(new_table["col9"], c9)))
assert all(tuple(a == b for a, b in zip(new_table["col10"], c10)))
assert all(tuple(compare(a, b) for a, b in zip(new_table["col11"], c11)))
def tst_split_partials_with_shoebox(self):
from dials.array_family import flex
from random import randint, uniform
from dials.model.data import Shoebox
r = flex.reflection_table()
r['value1'] = flex.double()
r['value2'] = flex.int()
r['value3'] = flex.double()
r['bbox'] = flex.int6()
r['panel'] = flex.size_t()
r['shoebox'] = flex.shoebox()
expected = []
for i in range(100):
x0 = randint(0, 100)
x1 = x0 + randint(1, 10)
y0 = randint(0, 100)
y1 = y0 + randint(1, 10)
z0 = randint(0, 100)
z1 = z0 + randint(1, 10)
v1 = uniform(0, 100)
v2 = randint(0, 100)
v3 = uniform(0, 100)
sbox = Shoebox(0, (x0, x1, y0, y1, z0, z1))
sbox.allocate()
assert(sbox.is_consistent())
w = x1 - x0
h = y1 - y0
for z in range(z0, z1):
for y in range(y0, y1):
for x in range(x0, x1):
sbox.data[z-z0,y-y0,x-x0] = x+y*w+z*w*h
r.append({
'value1' : v1,
'value2' : v2,
'value3' : v3,
'bbox' : (x0, x1, y0, y1, z0, z1),
'panel' : 0,
'shoebox' : sbox
})
for z in range(z0, z1):
sbox = Shoebox(0, (x0, x1, y0, y1, z, z+1))
sbox.allocate()
assert(sbox.is_consistent())
w = x1 - x0
h = y1 - y0
for y in range(y0, y1):
for x in range(x0, x1):
sbox.data[0,y-y0,x-x0] = x+y*w+z*w*h
expected.append({
'value1' : v1,
'value2' : v2,
'value3' : v3,
'bbox' : (x0, x1, y0, y1, z, z+1),
'partial_id' : i,
'panel' : 0,
'shoebox' : sbox
})
r.split_partials_with_shoebox()
assert(len(r) == len(expected))
EPS = 1e-7
for r1, r2 in zip(r, expected):
assert(abs(r1['value1'] - r2['value1']) < EPS)
assert(r1['value2'] == r2['value2'])
assert(abs(r1['value3'] - r2['value3']) < EPS)
assert(r1['bbox'] == r2['bbox'])
assert(r1['partial_id'] == r2['partial_id'])
assert(r1['panel'] == r2['panel'])
assert(r1['shoebox'].data.as_double().as_1d().all_approx_equal(
r2['shoebox'].data.as_double().as_1d()))
print 'OK'
def test_split_partials_with_shoebox():
from dials.model.data import Shoebox
r = flex.reflection_table()
r["value1"] = flex.double()
r["value2"] = flex.int()
r["value3"] = flex.double()
r["bbox"] = flex.int6()
r["panel"] = flex.size_t()
r["shoebox"] = flex.shoebox()
expected = []
for i in range(100):
x0 = random.randint(0, 100)
x1 = x0 + random.randint(1, 10)
y0 = random.randint(0, 100)
y1 = y0 + random.randint(1, 10)
z0 = random.randint(0, 100)
z1 = z0 + random.randint(1, 10)
v1 = random.uniform(0, 100)
v2 = random.randint(0, 100)
v3 = random.uniform(0, 100)
sbox = Shoebox(0, (x0, x1, y0, y1, z0, z1))
sbox.allocate()
assert sbox.is_consistent()
w = x1 - x0
h = y1 - y0
for z in range(z0, z1):
for y in range(y0, y1):
for x in range(x0, x1):
sbox.data[z - z0, y - y0, x - x0] = x + y * w + z * w * h
r.append({
"value1": v1,
"value2": v2,
"value3": v3,
"bbox": (x0, x1, y0, y1, z0, z1),
"panel": 0,
"shoebox": sbox,
})
for z in range(z0, z1):
sbox = Shoebox(0, (x0, x1, y0, y1, z, z + 1))
sbox.allocate()
assert sbox.is_consistent()
w = x1 - x0
h = y1 - y0
for y in range(y0, y1):
for x in range(x0, x1):
sbox.data[0, y - y0, x - x0] = x + y * w + z * w * h
expected.append({
"value1": v1,
"value2": v2,
"value3": v3,
"bbox": (x0, x1, y0, y1, z, z + 1),
"partial_id": i,
"panel": 0,
"shoebox": sbox,
})
r.split_partials_with_shoebox()
assert len(r) == len(expected)
EPS = 1e-7
for r1, r2 in zip(r.rows(), expected):
assert abs(r1["value1"] - r2["value1"]) < EPS
assert r1["value2"] == r2["value2"]
assert abs(r1["value3"] - r2["value3"]) < EPS
assert r1["bbox"] == r2["bbox"]
assert r1["partial_id"] == r2["partial_id"]
assert r1["panel"] == r2["panel"]
assert (r1["shoebox"].data.as_double().as_1d().all_approx_equal(
r2["shoebox"].data.as_double().as_1d()))
'''Set the reflection data.'''
self.set_data(reflections, ReflectionListEncoder())
def get_reflections(self):
'''Get the reflection data.'''
return self.get_data(ReflectionListDecoder())
if __name__ == '__main__':
from dials.array_family import flex
reflections = flex.reflection_table([
('hkl', flex.miller_index(10)),
('s1', flex.vec3_double(10)),
('bbox', flex.int6(10)),
('id', flex.int(10)),
('shoebox', flex.shoebox(10))
])
for i in range(10):
reflections['shoebox'][i].data = flex.double(flex.grid(10,10,10))
reflections['shoebox'][i].mask = flex.int(flex.grid(10,10,10))
reflections['shoebox'][i].background = flex.double(flex.grid(10,10,10))
for i in range(10):
print reflections['shoebox'][i].data.all()
writer = NexusFile('temp.h5', 'w')
writer.set_reflections(reflections)
writer.close()
reader = NexusFile('temp.h5', 'r')
def _find_in_imageset(self, imageset):
"""
Do the spot finding.
:param imageset: The imageset to process
:return: The observed spots
"""
from dials.util.masking import MaskGenerator
from dials.array_family import flex
from dials.util.command_line import Command
from dxtbx.imageset import ImageSweep
from logging import info
# The input mask
mask = self.mask_generator.generate(imageset)
if self.mask is not None:
mask = tuple(m1 & m2 for m1, m2 in zip(mask, self.mask))
# Set the spot finding algorithm
extract_spots = ExtractSpots(
threshold_function=self.threshold_function,
mask=mask,
region_of_interest=self.region_of_interest,
max_strong_pixel_fraction=self.max_strong_pixel_fraction,
mp_method=self.mp_method,
nproc=self.nproc,
mp_chunksize=self.mp_chunksize,
min_spot_size=self.min_spot_size,
max_spot_size=self.max_spot_size,
)
# Get the max scan range
if isinstance(imageset, ImageSweep):
max_scan_range = imageset.get_array_range()
else:
max_scan_range = (0, len(imageset))
# Get list of scan ranges
if not self.scan_range or self.scan_range[0] is None:
scan_range = [(max_scan_range[0] + 1, max_scan_range[1])]
else:
scan_range = self.scan_range
# Get spots from bits of scan
spots_all = []
for scan in scan_range:
j0, j1 = scan
assert j1 >= j0 and j0 > max_scan_range[0] and j1 <= max_scan_range[1]
info("\nFinding spots in image {0} to {1}...".format(j0, j1))
j0 -= 1
if isinstance(imageset, ImageSweep):
j0 -= imageset.get_array_range()[0]
j1 -= imageset.get_array_range()[0]
spots_all.extend(extract_spots(imageset[j0:j1]))
# Get the list of shoeboxes
shoeboxes = flex.shoebox(spots_all)
# Calculate the spot centroids
centroid = shoeboxes.centroid_valid()
info("Calculated {0} spot centroids".format(len(shoeboxes)))
# Calculate the spot intensities
intensity = shoeboxes.summed_intensity()
info("Calculated {0} spot intensities".format(len(shoeboxes)))
# Create the observations
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
# Write the hot mask
if self.write_hot_mask:
# Find spots which cover the whole scan range
bbox = flex.int6([sbox.bbox for sbox in shoeboxes])
z0, z1 = bbox.parts()[4:6]
zr = z1 - z0
assert zr.all_gt(0)
possible_hot_spots = zr == len(imageset)
num_possible_hot_spots = possible_hot_spots.count(True)
info("Found %d possible hot spots" % num_possible_hot_spots)
# Create the hot pixel mask
hot_mask = tuple(flex.bool(flex.grid(p.get_image_size()[::-1]), True) for p in imageset.get_detector())
if num_possible_hot_spots > 0:
hot_shoeboxes = shoeboxes.select(possible_hot_spots)
for sbox in hot_shoeboxes:
x0, x1, y0, y1 = sbox.bbox[0:4]
m = sbox.mask
p = sbox.panel
for y in range(m.all()[1]):
for x in range(m.all()[2]):
if m[:, y : y + 1, x : x + 1].all_ne(0):
hot_mask[p][y0 + y, x0 + x] = False
info("Found %d possible hot pixel(s)" % hot_mask.count(False))
else:
hot_mask = None
# Filter the reflections and select only the desired spots
flags = self.filter_spots(None, sweep=imageset, observations=observed, shoeboxes=shoeboxes)
observed = observed.select(flags)
shoeboxes = shoeboxes.select(flags)
# Return as a reflection list
return flex.reflection_table(observed, shoeboxes), hot_mask
def generate_predictions(self, N):
""" Generate some reflections. """
from dials.algorithms.profile_model.gaussian_rs import MaskCalculator3D
from dials.array_family import flex
from dials.util.command_line import Command
from dials.algorithms import filtering
from dials.algorithms.shoebox import MaskCode
from dials.algorithms.profile_model.gaussian_rs import Model as ProfileModel
import random
# Set the profile model
self.experiment.profile = ProfileModel(None, self.n_sigma, self.sigma_b, self.sigma_m)
# Generate a list of reflections
refl = flex.reflection_table.from_predictions(self.experiment)
refl["id"] = flex.int(len(refl), 0)
# Filter by zeta
zeta = 0.05
Command.start("Filtering by zeta >= %f" % zeta)
mask = filtering.by_zeta(self.experiment.goniometer, self.experiment.beam, refl["s1"], zeta)
refl.del_selected(mask != True)
Command.end("Filtered %d reflections by zeta >= %f" % (len(refl), zeta))
# Compute the bounding box
refl.compute_bbox([self.experiment])
index = []
image_size = self.experiment.detector[0].get_image_size()
array_range = self.experiment.scan.get_array_range()
bbox = refl["bbox"]
for i in range(len(refl)):
x0, x1, y0, y1, z0, z1 = bbox[i]
if (
x0 < 0
or x1 > image_size[0]
or y0 < 0
or y1 > image_size[1]
or z0 < array_range[0]
or z1 > array_range[1]
):
index.append(i)
refl.del_selected(flex.size_t(index))
# Sample if specified
index = random.sample(range(len(refl)), N)
refl = refl.select(flex.size_t(index))
# Compute the bounding box
# Create a load of shoeboxes
Command.start("Creating shoeboxes for %d reflections" % len(refl))
refl["shoebox"] = flex.shoebox(refl["panel"], refl["bbox"])
refl["shoebox"].allocate_with_value(MaskCode.Valid)
Command.end("Created shoeboxes for %d reflections" % len(refl))
# Get the function object to mask the foreground
Command.start("Masking Foreground for %d reflections" % len(refl))
mask_foreground = MaskCalculator3D(
self.experiment.beam,
self.experiment.detector,
self.experiment.goniometer,
self.experiment.scan,
self.n_sigma * self.sigma_b,
self.n_sigma * self.sigma_m,
)
# Mask the foreground
mask_foreground(refl["shoebox"], refl["s1"], refl["xyzcal.px"].parts()[2], refl["panel"])
Command.end("Masked foreground for %d reflections" % len(refl))
# Return the reflections
return refl
