class Test(object):
def __init__(self):
from dials.algorithms.background.simple import Creator
from dials.algorithms.background.simple import MosflmOutlierRejector
from dials.algorithms.background.simple import Linear2dModeller
from os.path import join
import libtbx.load_env
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'SKIP: dials_regression not configured'
exit(0)
# The directory path
path = join(
dials_regression,
"integration_test_data",
"i04-weak-data",
"jmp_mosflm_test")
# The input files
self.reflection_filename = join(path, "mosflm_reflections.pickle")
self.shoebox_filename = join(path, "shoeboxes.pickle")
self.fraction = 1.0
self.n_sigma = 4
self.outlier_rejector = MosflmOutlierRejector(self.fraction, self.n_sigma)
self.linear_modeller = Linear2dModeller()
self.background_creator = Creator(
self.linear_modeller,
self.outlier_rejector)
def test_linear2d_modeller(self):
from dials.algorithms.background.simple import Linear2dModeller
from dials.array_family import flex
modeller = Linear2dModeller()
# Generate shoeboxes
ma = 100
mb = 1
mc = 2
sboxes, masks = self.generate_background(self.size, 1000, ma, mb, mc, 0)
pa = []
pv = []
for i in range(1000):
model = modeller.create(sboxes[i], masks[i])
assert len(model.params()) == 9 * 3
assert len(model.variances()) == 9 * 3
p = model.params()
v = model.variances()
for j in range(9):
pa.append(tuple(p[3 * j : 3 * (j + 1)]))
pv.append(tuple(v[3 * j : 3 * (j + 1)]))
a, b, c = zip(*pa)
va, vb, vc = zip(*pv)
# Compute Z for each parameter
za = (flex.double(a) - ma) / flex.sqrt(flex.double(va))
zb = (flex.double(b) - mb) / flex.sqrt(flex.double(vb))
zc = (flex.double(c) - mc) / flex.sqrt(flex.double(vc))
# Check it looks standard normal
self.assert_std_norm(za)
self.assert_std_norm(zb)
self.assert_std_norm(zc)
def select_modeller():
if model == 'constant2d':
return Constant2dModeller()
elif model == 'constant3d':
return Constant3dModeller()
elif model == 'linear2d':
return Linear2dModeller()
elif model == 'linear3d':
return Linear3dModeller()
raise RuntimeError("Unexpected background model: %s" % model)
def select_modeller():
if model == "constant2d":
return Constant2dModeller()
elif model == "constant3d":
return Constant3dModeller()
elif model == "linear2d":
return Linear2dModeller()
elif model == "linear3d":
return Linear3dModeller()
raise RuntimeError(f"Unexpected background model: {model}")
def test_linear2d_modeller(self):
from dials.algorithms.background.simple import Linear2dModeller
modeller = Linear2dModeller()
eps = 1e-7
for i in range(10):
p, data, mask = self.generate_linear_background_2d(self.size, 0, 100)
model = modeller.create(data, mask)
assert(len(model.params()) == 3 * 9)
for j in range(9):
for k in range(3):
assert(abs(model.params()[k+j*3] - p[j][k]) < eps)
def run(self):
from dials.algorithms.background.simple import Creator
from dials.algorithms.background.simple import Linear2dModeller
from dials.algorithms.background.simple import TruncatedOutlierRejector
from dials.algorithms.background.simple import NSigmaOutlierRejector
from dials.algorithms.background.simple import NormalOutlierRejector
modeller = Linear2dModeller()
outlier_rejector = [
None,
TruncatedOutlierRejector(0.01, 0.01),
NSigmaOutlierRejector(3.0, 3.0),
NormalOutlierRejector(10),
]
for rejector in outlier_rejector:
self.tst(Creator(modeller, rejector))
def test_run(dials_regression):
from dials.algorithms.background.simple import (
Linear2dModeller,
MosflmOutlierRejector,
)
# The directory path
path = os.path.join(dials_regression, "integration_test_data",
"i04-weak-data", "jmp_mosflm_test")
# The input files
reflection_filename = os.path.join(path, "mosflm_reflections.pickle")
shoebox_filename = os.path.join(path, "shoeboxes.pickle")
fraction = 1.0
n_sigma = 4
outlier_rejector = MosflmOutlierRejector(fraction, n_sigma)
linear_modeller = Linear2dModeller()
from dials.algorithms.shoebox import MaskCode
from dials.array_family import flex
print(shoebox_filename)
# Read the data
rtable = flex.reflection_table.from_file(reflection_filename)
with open(shoebox_filename, "rb") as fh:
if six.PY3:
shoeboxes, masks = pickle.load(fh, encoding="bytes")
else:
shoeboxes, masks = pickle.load(fh)
assert len(rtable) == len(shoeboxes)
assert len(rtable) == len(masks)
# Compute the background for each reflection and check against the values
# read from the mosflm.lp file. Currently this fails for 1 strange
# reflection whose pixel values in the mosflm file do not match those
# extracted from the images.
errors = 0
VAR1 = []
VAR2 = []
DIFF = []
for i in range(len(rtable)):
I = rtable[i]["intensity.sum.value"]
Ivar = rtable[i]["intensity.sum.variance"]
data = shoeboxes[i].as_double()
mask = masks[i]
assert len(data.all()) == 2
assert len(mask.all()) == 2
data.reshape(flex.grid(1, *data.all()))
mask.reshape(flex.grid(1, *mask.all()))
outlier_rejector(data, mask)
mask2 = (mask.as_1d() & int(MaskCode.BackgroundUsed)) != 0
mask2.reshape(flex.grid(*mask.all()))
model = linear_modeller.create(data, mask2)
assert len(model.params()) == 3
hy = data.all()[1] // 2
hx = data.all()[2] // 2
c1 = model.params()[0]
a1 = model.params()[1]
b1 = model.params()[2]
c3 = c1 + a1 * (0.5 + hx) + b1 * (0.5 + hy)
a2 = rtable[i]["background"][0]
b2 = rtable[i]["background"][1]
c2 = rtable[i]["background"][2]
try:
assert abs(a1 - b2) < 0.01
assert abs(b1 + a2) < 0.01
assert abs(c3 - c2) < 0.1
except Exception:
errors += 1
continue
background = data.as_double()
# hy = background.all()[1] // 2
# hx = background.all()[2] // 2
for jj in range(background.all()[1]):
for ii in range(background.all()[2]):
# x = ii - hx
# y = jj - hy
x = ii + 0.5
y = jj + 0.5
background[0, jj, ii] = a1 * x + b1 * y + c1
# Test the summation results. Edge reflections use profile fitted
# intensity in MOSFLM. Therefore ignore these. There also appears to be a
# some discrepancy with very low <= 0 reflections where an extra 0.5 is
# added. Not sure why this is so ignore these reflections as well.
from dials.algorithms.integration.sum import integrate_by_summation
intensity = integrate_by_summation(data.as_double(), background, mask)
I2 = intensity.intensity()
Ivar2 = intensity.variance()
I1 = I
Ivar1 = Ivar
if mask.count(0) == 0 and mask.count(2) == 0 and I1 > 0:
VAR1.append(math.sqrt(Ivar1))
VAR2.append(math.sqrt(Ivar2))
DIFF.append(math.sqrt(Ivar1) - math.sqrt(Ivar2))
try:
assert abs(I1 - I2) < 1.0
assert abs(math.sqrt(Ivar1) - math.sqrt(Ivar2)) < 1.0
except Exception:
errors += 1
continue
# Only 1 should fail
assert errors == 1
def run(self, flags, sequence=None, shoeboxes=None, **kwargs):
from dials.algorithms.background.simple import Linear2dModeller
modeller = Linear2dModeller()
detector = sequence.get_detector()
# sort shoeboxes by centroid z
frame = shoeboxes.centroid_all().position_frame()
perm = flex.sort_permutation(frame)
shoeboxes = shoeboxes.select(perm)
buffer_size = 1
bg_plus_buffer = self.background_size + buffer_size
t0 = time.time()
for i, shoebox in enumerate(shoeboxes):
if not flags[perm[i]]:
continue
panel = detector[shoebox.panel]
max_x, max_y = panel.get_image_size()
bbox = shoebox.bbox
x1, x2, y1, y2, z1, z2 = bbox
# expand the bbox with a background region around the spotfinder shoebox
# perhaps also should use a buffer zone between the shoebox and the
# background region
expanded_bbox = (
max(0, x1 - bg_plus_buffer),
min(max_x, x2 + bg_plus_buffer),
max(0, y1 - bg_plus_buffer),
min(max_y, y2 + bg_plus_buffer),
z1,
z2,
)
shoebox.bbox = expanded_bbox
t1 = time.time()
logger.info("Time expand_shoebox: %s" % (t1 - t0))
rlist = flex.reflection_table()
rlist["shoebox"] = shoeboxes
rlist["shoebox"].allocate()
rlist["panel"] = shoeboxes.panels()
rlist["bbox"] = shoeboxes.bounding_boxes()
rlist.extract_shoeboxes(sequence)
shoeboxes = rlist["shoebox"]
shoeboxes.flatten()
for i, shoebox in enumerate(shoeboxes):
if not flags[perm[i]]:
continue
panel = detector[shoebox.panel]
trusted_range = panel.get_trusted_range()
ex1, ex2, ey1, ey2, ez1, ez2 = shoebox.bbox
data = shoebox.data
mask = flex.bool(data.accessor(), False)
for i_y, y in enumerate(range(ey1, ey2)):
for i_x, x in enumerate(range(ex1, ex2)):
value = data[0, i_y, i_x]
if (y >= (ey1 + buffer_size) and y < (ey2 - buffer_size)
and x >= (ex1 + buffer_size) and x <
(ex2 - buffer_size)):
mask[0, i_y, i_x] = False # foreground
elif value > trusted_range[0] and value < trusted_range[1]:
mask[0, i_y, i_x] = True # background
model = modeller.create(data.as_double(), mask)
d, a, b = model.params()[:3]
if abs(a) > self.gradient_cutoff or abs(b) > self.gradient_cutoff:
flags[perm[i]] = False
return flags
def run(self, flags, sweep=None, shoeboxes=None, **kwargs):
from dials.array_family import flex
from dials.algorithms.shoebox import MaskCode
from dials.algorithms.background.simple import Linear2dModeller
bg_code = MaskCode.Valid | MaskCode.BackgroundUsed
fg_code = MaskCode.Valid | MaskCode.Foreground
strong_code = MaskCode.Valid | MaskCode.Strong
modeller = Linear2dModeller()
expanded_shoeboxes = flex.shoebox()
detector = sweep.get_detector()
zoffset = 0
if sweep.get_scan() is not None:
zoffset = sweep.get_scan().get_array_range()[0]
from libtbx.containers import OrderedDict
class image_data_cache(object):
def __init__(self, imageset, size=10):
self.imageset = imageset
self.size = size
self._image_data = OrderedDict()
def __getitem__(self, i):
image_data = self._image_data.get(i)
if image_data is None:
image_data = self.imageset.get_raw_data(i)
if len(self._image_data) >= self.size:
# remove the oldest entry in the cache
del self._image_data[self._image_data.keys()[0]]
self._image_data[i] = image_data
return image_data
cache = image_data_cache(sweep)
#cache = sweep
# sort shoeboxes by centroid z
frame = shoeboxes.centroid_all().position_frame()
perm = flex.sort_permutation(frame)
shoeboxes = shoeboxes.select(perm)
buffer_size = 1
bg_plus_buffer = self.background_size + buffer_size
import time
t0 = time.time()
for i, shoebox in enumerate(shoeboxes):
if not flags[perm[i]]:
continue
panel = detector[shoebox.panel]
trusted_range = panel.get_trusted_range()
max_x, max_y = panel.get_image_size()
bbox = shoebox.bbox
x1, x2, y1, y2, z1, z2 = bbox
# expand the bbox with a background region around the spotfinder shoebox
# perhaps also should use a buffer zone between the shoebox and the
# background region
expanded_bbox = (max(0, x1 - bg_plus_buffer),
min(max_x, x2 + bg_plus_buffer),
max(0, y1 - bg_plus_buffer),
min(max_y, y2 + bg_plus_buffer), z1, z2)
shoebox.bbox = expanded_bbox
t1 = time.time()
logger.info("Time expand_shoebox: %s" % (t1 - t0))
rlist = flex.reflection_table()
rlist['shoebox'] = shoeboxes
rlist['shoebox'].allocate()
rlist['panel'] = shoeboxes.panels()
rlist['bbox'] = shoeboxes.bounding_boxes()
t0 = time.time()
rlist.extract_shoeboxes(sweep)
t1 = time.time()
shoeboxes = rlist['shoebox']
shoeboxes.flatten()
t0 = time.time()
for i, shoebox in enumerate(shoeboxes):
if not flags[perm[i]]: continue
panel = detector[shoebox.panel]
trusted_range = panel.get_trusted_range()
max_x, max_y = panel.get_image_size()
ex1, ex2, ey1, ey2, ez1, ez2 = shoebox.bbox
data = shoebox.data
mask = flex.bool(data.accessor(), False)
for i_y, y in enumerate(range(ey1, ey2)):
for i_x, x in enumerate(range(ex1, ex2)):
value = data[0, i_y, i_x]
if (y >= (ey1 + buffer_size) and y < (ey2 - buffer_size)
and x >= (ex1 + buffer_size) and x <
(ex2 - buffer_size)):
mask[0, i_y, i_x] = False # foreground
elif value > trusted_range[0] and value < trusted_range[1]:
mask[0, i_y, i_x] = True # background
model = modeller.create(data.as_double(), mask)
d, a, b = model.params()[:3]
c = -1
if abs(a) > self.gradient_cutoff or abs(b) > self.gradient_cutoff:
flags[perm[i]] = False
t1 = time.time()
return flags
def test_run(dials_regression):
from dials.algorithms.background.simple import Creator
from dials.algorithms.background.simple import MosflmOutlierRejector
from dials.algorithms.background.simple import Linear2dModeller
# The directory path
path = os.path.join(dials_regression, "integration_test_data",
"i04-weak-data", "jmp_mosflm_test")
# The input files
reflection_filename = os.path.join(path, "mosflm_reflections.pickle")
shoebox_filename = os.path.join(path, "shoeboxes.pickle")
fraction = 1.0
n_sigma = 4
outlier_rejector = MosflmOutlierRejector(fraction, n_sigma)
linear_modeller = Linear2dModeller()
background_creator = Creator(linear_modeller, outlier_rejector)
from dials.array_family import flex
from math import sqrt
from dials.algorithms.shoebox import MaskCode
print(shoebox_filename)
# Read the data
rtable = flex.reflection_table.from_pickle(reflection_filename)
with open(shoebox_filename, "rb") as fh:
shoeboxes, masks = pickle.load(fh)
assert len(rtable) == len(shoeboxes)
assert len(rtable) == len(masks)
# Compute the background for each reflection and check against the values
# read from the mosflm.lp file. Currently this fails for 1 strange
# reflection whose pixel values in the mosflm file do not match those
# extracted from the images.
count = 0
VAR1 = []
VAR2 = []
DIFF = []
for i in range(len(rtable)):
xdet, ydet = rtable[i]["xy"]
nx = rtable[i]['nx']
ny = rtable[i]['ny']
nc = rtable[i]['nc']
nrx = rtable[i]['nrx']
nry = rtable[i]['nry']
bbox = rtable[i]['bbox']
I = rtable[i]['intensity.sum.value']
Ivar = rtable[i]['intensity.sum.variance']
lp = rtable[i]['lp']
data = shoeboxes[i].as_double()
mask = masks[i]
fraction = 1.0
nsigma = 4
try:
# model = PlaneModel(data, mask, fraction, nsigma)
assert len(data.all()) == 2
assert len(mask.all()) == 2
data.reshape(flex.grid(1, *data.all()))
mask.reshape(flex.grid(1, *mask.all()))
outlier_rejector(data, mask)
mask2 = (mask.as_1d() & int(MaskCode.BackgroundUsed)) != 0
mask2.reshape(flex.grid(*mask.all()))
model = linear_modeller.create(data, mask2)
except Exception:
count += 1
raise
continue
# n = model.noutlier()
assert (len(model.params()) == 3)
hy = data.all()[1] // 2
hx = data.all()[2] // 2
c1 = model.params()[0]
a1 = model.params()[1]
b1 = model.params()[2]
c3 = c1 + a1 * (0.5 + hx) + b1 * (0.5 + hy)
# a1 = model.a()
# b1 = model.b()
# c1 = model.c()
a2 = rtable[i]['background'][0]
b2 = rtable[i]['background'][1]
c2 = rtable[i]['background'][2]
try:
assert (abs(a1 - b2) < 0.01)
assert (abs(b1 + a2) < 0.01)
assert (abs(c3 - c2) < 0.1)
except Exception:
count += 1
continue
#print "BG %d:(%.2f, %.2f, %.1f), (%.2f, %.2f, %.1f): %d" % \
#(i, a1, b1, c1, a2, b2, c2, n)
#print "X, Y: ", xdet, ydet
#print "NX, NY: ", nx, ny
#print "NRX, NRY, NC", nrx, nry, nc
#print int(floor(xdet + 0.5)) - nx // 2, int(floor(ydet + 0.5)) - ny // 2
#print "BBOX: ", bbox
#print "N Outliers: ", model.noutlier()
#print "N Background: ", model.nbackground()
#print "Max DIff: ", model.maxdiff()
#print data.as_numpy_array().transpose()[::-1,::-1]
#print mask.as_numpy_array().transpose()[::-1,::-1]
#raise
background = data.as_double()
# hy = background.all()[1] // 2
# hx = background.all()[2] // 2
for jj in range(background.all()[1]):
for ii in range(background.all()[2]):
# x = ii - hx
# y = jj - hy
x = ii + 0.5
y = jj + 0.5
background[0, jj, ii] = a1 * x + b1 * y + c1
# Test the summation results. Edge reflections use profile fitted
# intensity in MOSFLM. Therefore ignore these. There also appears to be a
# some discrepancy with very low <= 0 reflections where an extra 0.5 is
# added. Not sure why this is so ignore these reflections as well.
from dials.algorithms.integration.sum import integrate_by_summation
intensity = integrate_by_summation(data.as_double(), background, mask)
I2 = intensity.intensity()
Ivar2 = intensity.variance()
I1 = I
Ivar1 = Ivar
if mask.count(0) == 0 and mask.count(2) == 0 and I1 > 0:
VAR1.append(sqrt(Ivar1))
VAR2.append(sqrt(Ivar2))
DIFF.append(sqrt(Ivar1) - sqrt(Ivar2))
try:
assert (abs(I1 - I2) < 1.0)
assert (abs(sqrt(Ivar1) - sqrt(Ivar2)) < 1.0)
except Exception:
count += 1
#import numpy
#numpy.set_printoptions(precision=4, linewidth=200)
#print "# %d" % i
#print "I: %f, %f, %f" % (I2, I1, lp)
#print "DEBUG: ", c1 * 25
#print "PF: %f" % rtable[i]['intensity.prf.value']
#print "BG (%.4f, %.4f, %.4f), (%.2f, %.2f, %.1f): %d" % \
#(a1, b1, c1, a2, b2, c2, n)
#print "X, Y: ", xdet, ydet
#print "NX, NY: ", nx, ny
#print "NRX, NRY, NC", nrx, nry, nc
#print int(floor(xdet + 0.5)) - nx // 2, int(floor(ydet + 0.5)) - ny // 2
#print "BBOX: ", bbox
#print "N Outliers: ", model.noutlier()
#print "N Background: ", model.nbackground()
#print "Max DIff: ", model.maxdiff()
#temp = (mask == MaskCode.Valid | MaskCode.Foreground).as_1d().as_int()
#temp.resize(flex.grid(*data.all()))
#temp = temp.as_double()
#print data.as_numpy_array().transpose()[::-1,::-1]
#print (background * temp).as_numpy_array().transpose()[::-1,::-1]
#print mask.as_numpy_array().transpose()[::-1,::-1]
#print ((data.as_double() - background) * temp).as_numpy_array().transpose()[::-1,::-1]
#raise
continue
# Only 1 should fail
assert (count == 1)
