def select_strong_pixels(sweep, trusted_range):
from dials.util.command_line import ProgressBar
import numpy
# Calculate the threshold
coordinate = []
intensity = []
progress = ProgressBar()
for i, flex_image in enumerate(sweep):
image = flex_image.as_numpy_array()
height, width = image.shape
threshold = calculate_threshold(image, trusted_range)
image.shape = -1
mask = image >= threshold
ind = numpy.where(mask != 0)[0]
z = [i] * len(ind)
y = list(ind / width)
x = list(ind % width)
coords = zip(x, y, z)
coordinate.extend(coords)
intensity.extend(list(image[ind]))
progress.update(100.0 * float(i) / len(sweep))
progress.finished()
return coordinate, intensity
def write(self, predictions, imageset):
from dials.util.command_line import ProgressBar
zrange = imageset.get_array_range()
p = ProgressBar(title="Extracting shoeboxes")
for z, image in enumerate(imageset, start=zrange[0]):
batch = self._add_image(z, image)
self._write_pickle(batch)
p.update(100.0 * (z - zrange[0]) / len(imageset))
p.finish("Extracted shoeboxes")
self._write_predictions(predictions)
def generate_background(self, size, N, A, B, C, D):
from dials.algorithms.simulation.generate_test_reflections \
import random_background_plane2
from dials.array_family import flex
from dials.util.command_line import ProgressBar
sboxes = []
masks = []
progress = ProgressBar(title="Generating Background")
for i in range(N):
mask = flex.bool(flex.grid(size), True)
sbox = flex.double(flex.grid(size), 0)
random_background_plane2(sbox, A, B, C, D)
sboxes.append(sbox)
masks.append(mask)
progress.update(100.0 * i / N)
progress.finished("Generated Background")
return sboxes, masks
def _extract_pixels(self, sweep):
"""Extract the pixels from the sweep
Params:
sweep The sweep object
Returns:
The list of selected pixels
"""
from dials.util.command_line import ProgressBar
from scitbx.array_family import flex
from dials.algorithms.peak_finding import flex_vec3_int
# Initialise the pixel arrays
coords = flex_vec3_int()
intensity = flex.int()
# Get the start index and trusted range from the sweep
start = sweep.get_array_range()[0]
trusted_range = sweep.get_detector().get_trusted_range()
# Loop through all the images in the sweep and extract the pixels
# from each of the images
progress = ProgressBar()
for frame, image in enumerate(sweep):
c, i = self._extract_image_pixels(image, frame + start,
trusted_range)
coords.extend(c)
intensity.extend(i)
progress.update(100.0 * float(frame + 1) / len(sweep))
progress.finished()
# Reuturn the pixel values
return coords, intensity
def with_individual_given_intensity(self, N, In, Ba, Bb, Bc, Bd):
"""Generate reflections with given intensity and background."""
from dials.algorithms.simulation import simulate_reciprocal_space_gaussian
from dials.algorithms.simulation.generate_test_reflections import (
random_background_plane2, )
from dials.util.command_line import ProgressBar
# Check the lengths
assert N == len(In)
assert N == len(Ba)
assert N == len(Bb)
assert N == len(Bc)
assert N == len(Bd)
# Generate some predictions
refl = self.generate_predictions(N)
# Calculate the signal
progress = ProgressBar(
title=f"Calculating signal for {len(refl)} reflections")
s1 = refl["s1"]
phi = refl["xyzcal.mm"].parts()[2]
bbox = refl["bbox"]
shoebox = refl["shoebox"]
m = int(len(refl) / 100)
I_exp = flex.double(len(refl), 0)
for i in range(len(refl)):
if In[i] > 0:
data = shoebox[i].data.as_double()
I_exp[i] = simulate_reciprocal_space_gaussian(
self.experiment.beam,
self.experiment.detector,
self.experiment.goniometer,
self.experiment.scan,
self.sigma_b,
self.sigma_m,
s1[i],
phi[i],
bbox[i],
In[i],
data,
shoebox[i].mask,
)
shoebox[i].data = data.as_float()
if i % m == 0:
progress.update(100.0 * float(i) / len(refl))
progress.finished(
f"Calculated signal impacts for {len(refl)} reflections")
# Calculate the background
progress = ProgressBar(
title=f"Calculating background for {len(refl)} reflections")
for l in range(len(refl)):
background = flex.float(flex.grid(shoebox[l].size()), 0.0)
random_background_plane2(background, Ba[l], Bb[l], Bc[l], Bd[l])
shoebox[l].data += background
shoebox[l].background = background
if l % m == 0:
progress.update(100.0 * float(l) / len(refl))
progress.update(100.0 * float(l) / len(refl))
progress.finished(f"Calculated background for {len(refl)} reflections")
## Calculate the expected intensity by monte-carlo integration
# progress = ProgressBar(title='Integrating expected signal for %d reflections' % len(refl))
# s1 = refl['s1']
# phi = refl['xyzcal.mm'].parts()[2]
# bbox = refl['bbox']
# shoebox = refl['shoebox']
# I_exp = flex.double(len(refl), 0)
# m = int(len(refl) / 100)
# for i in range(len(refl)):
# if In[i] > 0:
# I_exp[i] = integrate_reciprocal_space_gaussian(
# self.experiment.beam,
# self.experiment.detector,
# self.experiment.goniometer,
# self.experiment.scan,
# self.sigma_b,
# self.sigma_m,
# s1[i],
# phi[i],
# bbox[i],
# 10000,
# shoebox[i].mask) / 10000.0
# if i % m == 0:
# progress.update(100.0 * float(i) / len(refl))
# progress.finished('Integrated expected signal impacts for %d reflections' % len(refl))
# Save the expected intensity and background
refl["intensity.sim"] = In
refl["background.sim.a"] = Ba
refl["background.sim.b"] = Bb
refl["background.sim.c"] = Bc
refl["background.sim.d"] = Bd
refl["intensity.exp"] = I_exp
# Return the reflections
return refl
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
from dials.util.command_line import ProgressBar
usage = "%s [options] data_master.h5" % (libtbx.env.dispatcher_name)
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
read_experiments_from_images=True,
epilog=help_message,
)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) != 1:
parser.print_help()
print("Please pass an experiment list\n")
return
imagesets = experiments.imagesets()
if len(imagesets) != 1:
raise Sorry("Please pass an experiment list that contains one imageset")
imageset = imagesets[0]
first, last = imageset.get_scan().get_image_range()
images = range(first, last + 1)
if params.images is None and params.image_range is not None:
start, end = params.image_range
params.images = list(range(start, end + 1))
if params.images:
if min(params.images) < first or max(params.images) > last:
raise Sorry("image outside of scan range")
images = params.images
detectors = imageset.get_detector()
assert len(detectors) == 1
detector = detectors[0]
trusted = detector.get_trusted_range()
# construct an integer array same shape as image; accumulate number of
# "signal" pixels in each pixel across data
total = None
p = ProgressBar(title="Finding hot pixels")
for idx in images:
p.update(idx * 100.0 / len(images))
pixels = imageset.get_raw_data(idx - 1)
assert len(pixels) == 1
data = pixels[0]
negative = data < int(round(trusted[0]))
hot = data > int(round(trusted[1]))
bad = negative | hot
data = data.as_double()
spot_params = spot_phil.fetch(
source=iotbx.phil.parse("min_spot_size=1")
).extract()
threshold_function = SpotFinderFactory.configure_threshold(spot_params)
peak_pixels = threshold_function.compute_threshold(data, ~bad)
if total is None:
total = peak_pixels.as_1d().as_int()
else:
total += peak_pixels.as_1d().as_int()
p.finished("Finished finding hot pixels on %d images" % len(images))
hot_mask = total >= (len(images) // 2)
hot_pixels = hot_mask.iselection()
p = ProgressBar(title="Finding twinkies")
twinkies = {}
for h in hot_pixels:
twinkies[h] = []
for idx in images:
p.update(idx * 100.0 / len(images))
pixels = imageset.get_raw_data(idx - 1)
data = pixels[0]
for h in hot_pixels:
twinkies[h].append(data[h])
p.finished("Finished hunting for twinkies on %d images" % len(images))
nslow, nfast = data.focus()
ffff = 0
for h in hot_pixels:
if total[h] == len(images) and data[h] >= trusted[1]:
ffff += 1
continue
print("Pixel %d at %d %d" % (total[h], h // nfast, h % nfast))
if len(set(twinkies[h])) >= len(twinkies[h]) // 2:
print(" ... many possible values")
continue
values = set(twinkies[h])
result = [(twinkies[h].count(value), value) for value in values]
for count, value in reversed(sorted(result)):
print(" %08x %d" % (value, count))
print("Also found %d very hot pixels" % ffff)
hot_mask.reshape(flex.grid(data.focus()))
easy_pickle.dump(params.output.mask, (~hot_mask,))
z.append(dxyz[2] + _z)
hkl.bounding_box = (int(math.floor(min(x))), int(math.floor(max(x)) + 1),
int(math.floor(min(y))), int(math.floor(max(y)) + 1),
int(math.floor(min(z))), int(math.floor(max(z)) + 1))
try:
counts = counts_database[hkl.miller_index]
useful.append(hkl)
except KeyError, e:
continue
from dials.algorithms import shoebox
shoebox.allocate(useful)
from dials.util.command_line import ProgressBar
p = ProgressBar(title = 'Generating shoeboxes')
# now for each reflection perform the simulation
for j, refl in enumerate(useful):
p.update(j * 100.0 / len(useful))
d = d_matrices[j]
from scitbx.random import variate, normal_distribution
g = variate(normal_distribution(mean = 0, sigma = node_size))
counts = counts_database[refl.miller_index]
dhs = g(counts)
dks = g(counts)
dls = g(counts)
self.map_to_image_space(refl, d, dhs, dks, dls)
p.finished('Generated %d shoeboxes' % len(useful))
def simple_gaussian_spots(params):
from scitbx import matrix
from dials.array_family import flex
r = params.rotation
axis = matrix.col((r.axis.x, r.axis.y, r.axis.z))
if axis.length() > 0:
rotation = axis.axis_and_angle_as_r3_rotation_matrix(r.angle, deg=True)
else:
rotation = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))
# generate mask and peak values
from dials.algorithms.shoebox import MaskCode
mask_peak = MaskCode.Valid | MaskCode.Foreground
mask_back = MaskCode.Valid | MaskCode.Background
from dials.util.command_line import ProgressBar
p = ProgressBar(title="Generating reflections")
rlist = flex.reflection_table(params.nrefl)
hkl = flex.miller_index(params.nrefl)
s1 = flex.vec3_double(params.nrefl)
xyzmm = flex.vec3_double(params.nrefl)
xyzpx = flex.vec3_double(params.nrefl)
panel = flex.size_t(params.nrefl)
bbox = flex.int6(params.nrefl)
for j in range(params.nrefl):
p.update(j * 100.0 / params.nrefl)
hkl[j] = (random.randint(0, 20), random.randint(0, 20), random.randint(0, 20))
phi = 2 * math.pi * random.random()
s1[j] = (0, 0, 0)
xyzpx[j] = (0, 0, 0)
xyzmm[j] = (0, 0, phi)
panel[j] = 0
bbox[j] = (
0,
params.shoebox_size.x,
0,
params.shoebox_size.y,
0,
params.shoebox_size.z,
)
p.finished("Generating %d reflections" % params.nrefl)
intensity = flex.double(params.nrefl)
shoebox = flex.shoebox(panel, bbox)
shoebox.allocate_with_value(MaskCode.Valid)
p = ProgressBar(title="Generating shoeboxes")
for i in range(len(rlist)):
p.update(i * 100.0 / params.nrefl)
mask = shoebox[i].mask
if params.pixel_mask == "precise":
# flag everything as background: peak will me assigned later
for j in range(len(mask)):
mask[j] = mask_back
elif params.pixel_mask == "all":
# flag we have no idea what anything is
mask_none = MaskCode.Valid | MaskCode.Foreground | MaskCode.Background
for j in range(len(mask)):
mask[j] = mask_none
elif params.pixel_mask == "static":
from scitbx.array_family import flex
x0 = params.spot_offset.x + params.shoebox_size.x / 2
y0 = params.spot_offset.x + params.shoebox_size.y / 2
z0 = params.spot_offset.x + params.shoebox_size.z / 2
sx = params.mask_nsigma * params.spot_size.x
sy = params.mask_nsigma * params.spot_size.y
sz = params.mask_nsigma * params.spot_size.z
# The x, y, z indices
z, y, x = zip(*itertools.product(*(range(n) for n in mask.all())))
xyz = flex.vec3_double(flex.double(x), flex.double(y), flex.double(z))
# Calculate SUM(((xj - xj0) / sxj)**2) for each element
xyz0 = (x0, y0, z0)
isxyz = (1.0 / sx, 1.0 / sy, 1.0 / sz)
dxyz = sum(
(x * isx) ** 2
for x, isx in zip(((xyz - xyz0) * rotation).parts(), isxyz)
)
# Set the mask values
index = dxyz <= 1.0
index.reshape(mask.accessor())
mask.set_selected(index, MaskCode.Valid | MaskCode.Foreground)
mask.set_selected(not index, MaskCode.Valid | MaskCode.Background)
sbox = shoebox[i].data
# reflection itself, including setting the peak region if we're doing that
# FIXME use flex arrays to make the rotation bit more efficient as this is
# now rather slow...
counts_true = 0
for j in range(params.counts):
_x = random.gauss(0, params.spot_size.x)
_y = random.gauss(0, params.spot_size.y)
_z = random.gauss(0, params.spot_size.z)
Rxyz = rotation * matrix.col((_x, _y, _z)).elems
x = int(Rxyz[0] + params.spot_offset.x + params.shoebox_size.x / 2)
y = int(Rxyz[1] + params.spot_offset.y + params.shoebox_size.y / 2)
z = int(Rxyz[2] + params.spot_offset.z + params.shoebox_size.z / 2)
if x < 0 or x >= params.shoebox_size.x:
continue
if y < 0 or y >= params.shoebox_size.y:
continue
if z < 0 or z >= params.shoebox_size.z:
continue
sbox[z, y, x] += 1
counts_true += 1
if params.pixel_mask == "precise":
mask[z, y, x] = mask_peak
intensity[i] = counts_true
if params.background:
# background:flat;
for j in range(params.background * len(sbox)):
x = random.randint(0, params.shoebox_size.x - 1)
y = random.randint(0, params.shoebox_size.y - 1)
z = random.randint(0, params.shoebox_size.z - 1)
sbox[z, y, x] += 1
else:
# or inclined
random_background_plane(
sbox,
params.background_a,
params.background_b,
params.background_c,
params.background_d,
)
rlist["miller_index"] = hkl
rlist["s1"] = s1
rlist["xyzcal.px"] = xyzpx
rlist["xyzcal.mm"] = xyzmm
rlist["bbox"] = bbox
rlist["panel"] = panel
rlist["shoebox"] = shoebox
rlist["intensity.sum.value"] = intensity
p.finished("Generating %d shoeboxes" % params.nrefl)
return rlist
def main(self):
# FIXME import simulation code
import six.moves.cPickle as pickle
import math
from dials.util.command_line import Importer
from dials.algorithms.integration import ReflectionPredictor
from libtbx.utils import Sorry
# Parse the command line
params, options, args = self.parser.parse_args()
importer = Importer(args)
if len(importer.imagesets) == 0 and len(importer.crystals) == 0:
self.config().print_help()
return
if len(importer.imagesets) != 1:
raise Sorry('need 1 sweep: %d given' % len(importer.imagesets))
if len(importer.crystals) != 1:
raise Sorry('need 1 crystal: %d given' % len(importer.crystals))
sweep = importer.imagesets[0]
crystal = importer.crystals[0]
# generate predictions for possible reflections => generate a
# reflection list
predict = ReflectionPredictor()
predicted = predict(sweep, crystal)
# sort with James's reflection table: should this not go somewhere central?
from dials.scratch.jmp.container.reflection_table import ReflectionTable
# calculate shoebox sizes: take parameters from params & transform
# from reciprocal space to image space to decide how big a shoe box to use
table = ReflectionTable()
table['miller_index'] = predicted.miller_index()
indexer = table.index_map('miller_index')
candidates = []
unique = sorted(indexer)
for h, k, l in unique:
try:
for _h in h - 1, h + 1:
if not indexer[(_h, k, l)]:
raise ValueError('missing')
for _k in k - 1, k + 1:
if not indexer[(h, _k, l)]:
raise ValueError('missing')
for _l in l - 1, l + 1:
if not indexer[(h, k, _l)]:
raise ValueError('missing')
candidates.append((h, k, l))
except ValueError:
continue
from dials.algorithms.simulation.utils import build_prediction_matrix
from dials.algorithms.simulation.generate_test_reflections import \
master_phil
from libtbx.phil import command_line
cmd = command_line.argument_interpreter(master_params=master_phil)
working_phil = cmd.process_and_fetch(args=args[2:])
params = working_phil.extract()
node_size = params.rs_node_size
window_size = params.rs_window_size
reference = params.integrated_data_file
scale = params.integrated_data_file_scale
if reference:
counts_database = {}
from iotbx import mtz
m = mtz.object(reference)
mi = m.extract_miller_indices()
i = m.extract_reals('IMEAN').data
s = m.space_group().build_derived_point_group()
for j in range(len(mi)):
for op in s.all_ops():
hkl = tuple(map(int, op * mi[j]))
counts = max(0, int(math.floor(i[j] * scale)))
counts_database[hkl] = counts
counts_database[(-hkl[0], -hkl[1], -hkl[2])] = counts
else:
def constant_factory(value):
import itertools
return itertools.repeat(value).next
from collections import defaultdict
counts_database = defaultdict(constant_factory(params.counts))
from dials.model.data import ReflectionList
useful = ReflectionList()
d_matrices = []
for h, k, l in candidates:
hkl = predicted[indexer[(h, k, l)][0]]
_x = hkl.image_coord_px[0]
_y = hkl.image_coord_px[1]
_z = hkl.frame_number
# build prediction matrix
mhkl = predicted[indexer[(h - 1, k, l)][0]]
phkl = predicted[indexer[(h + 1, k, l)][0]]
hmkl = predicted[indexer[(h, k - 1, l)][0]]
hpkl = predicted[indexer[(h, k + 1, l)][0]]
hkml = predicted[indexer[(h, k, l - 1)][0]]
hkpl = predicted[indexer[(h, k, l + 1)][0]]
d = build_prediction_matrix(hkl, mhkl, phkl, hmkl, hpkl, hkml,
hkpl)
d_matrices.append(d)
# construct the shoebox parameters: outline the ellipsoid
x, y, z = [], [], []
for dh in (1, 0, 0), (0, 1, 0), (0, 0, 1):
dxyz = -1 * window_size * d * dh
x.append(dxyz[0] + _x)
y.append(dxyz[1] + _y)
z.append(dxyz[2] + _z)
dxyz = window_size * d * dh
x.append(dxyz[0] + _x)
y.append(dxyz[1] + _y)
z.append(dxyz[2] + _z)
hkl.bounding_box = (int(math.floor(min(x))),
int(math.floor(max(x)) + 1),
int(math.floor(min(y))),
int(math.floor(max(y)) + 1),
int(math.floor(min(z))),
int(math.floor(max(z)) + 1))
try:
counts = counts_database[hkl.miller_index]
useful.append(hkl)
except KeyError:
continue
from dials.algorithms import shoebox
shoebox.allocate(useful)
from dials.util.command_line import ProgressBar
p = ProgressBar(title='Generating shoeboxes')
# now for each reflection perform the simulation
for j, refl in enumerate(useful):
p.update(j * 100.0 / len(useful))
d = d_matrices[j]
from scitbx.random import variate, normal_distribution
g = variate(normal_distribution(mean=0, sigma=node_size))
counts = counts_database[refl.miller_index]
dhs = g(counts)
dks = g(counts)
dls = g(counts)
self.map_to_image_space(refl, d, dhs, dks, dls)
p.finished('Generated %d shoeboxes' % len(useful))
# now for each reflection add background
from dials.algorithms.simulation.generate_test_reflections import \
random_background_plane
p = ProgressBar(title='Generating background')
for j, refl in enumerate(useful):
p.update(j * 100.0 / len(useful))
if params.background:
random_background_plane(refl.shoebox, params.background, 0.0,
0.0, 0.0)
else:
random_background_plane(refl.shoebox, params.background_a,
params.background_b,
params.background_c,
params.background_d)
p.finished('Generated %d backgrounds' % len(useful))
if params.output.all:
with open(params.output.all, 'wb') as fh:
pickle.dump(useful, fh, pickle.HIGHEST_PROTOCOL)
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
usage = "%s [options] data_master.h5" % (libtbx.env.dispatcher_name)
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
read_experiments_from_images=True,
epilog=help_message,
)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) != 1:
parser.print_help()
print("Please pass an experiment list\n")
return
imagesets = experiments.imagesets()
if len(imagesets) != 1:
raise Sorry(
"Please pass an experiment list that contains one imageset")
imageset = imagesets[0]
first, last = imageset.get_scan().get_image_range()
images = range(first, last + 1)
if not params.images and params.image_range:
params.images = list(
range(params.image_range[0], params.image_range[1] + 1))
if params.images:
if min(params.images) < first or max(params.images) > last:
raise Sorry("image outside of scan range")
images = params.images
detectors = imageset.get_detector()
assert len(detectors) == 1
detector = detectors[0]
trusted = detector.get_trusted_range()
from dials.util.command_line import ProgressBar
p = ProgressBar(title="Counting events")
events_per_image = {}
for idx in images:
p.update(idx * 100.0 / len(images))
pixels = imageset.get_raw_data(idx - 1)
assert len(pixels) == 1
mask = ~imageset.get_mask(idx)[0].as_1d()
data = pixels[0].as_1d()
data.set_selected(mask, 0)
hot = data >= int(round(trusted[1]))
data.set_selected(hot, 0)
events_per_image[idx] = flex.sum(data)
for idx in images:
print(idx, events_per_image[idx])