class Test(object):
def __init__(self):
from dials.algorithms.background.simple import Creator
from dials.algorithms.background.simple import MosflmOutlierRejector
from dials.algorithms.background.simple import Linear2dModeller
from os.path import join
import libtbx.load_env
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'SKIP: dials_regression not configured'
exit(0)
# The directory path
path = join(
dials_regression,
"integration_test_data",
"i04-weak-data",
"jmp_mosflm_test")
# The input files
self.reflection_filename = join(path, "mosflm_reflections.pickle")
self.shoebox_filename = join(path, "shoeboxes.pickle")
self.fraction = 1.0
self.n_sigma = 4
self.outlier_rejector = MosflmOutlierRejector(self.fraction, self.n_sigma)
self.linear_modeller = Linear2dModeller()
self.background_creator = Creator(
self.linear_modeller,
self.outlier_rejector)
def select_rejector():
if outlier == 'null':
return None
elif outlier == 'truncated':
return TruncatedOutlierRejector(kwargs.get("lower", 0.01),
kwargs.get("upper", 0.01))
elif outlier == 'nsigma':
return NSigmaOutlierRejector(kwargs.get("lower", 3),
kwargs.get("upper", 3))
elif outlier == 'normal':
return NormalOutlierRejector(kwargs.get("min_pixels", 10))
elif outlier == 'plane':
return MosflmOutlierRejector(kwargs.get("fraction", 1.0),
kwargs.get("n_sigma", 4.0))
elif outlier == 'tukey':
return TukeyOutlierRejector(kwargs.get("lower", 1.5),
kwargs.get("upper", 1.5))
raise RuntimeError("Unexpected outlier rejector: %s" % outlier)
def select_rejector():
if outlier == "null":
return None
elif outlier == "truncated":
return TruncatedOutlierRejector(kwargs.get("lower", 0.01),
kwargs.get("upper", 0.01))
elif outlier == "nsigma":
return NSigmaOutlierRejector(kwargs.get("lower", 3),
kwargs.get("upper", 3))
elif outlier == "normal":
return NormalOutlierRejector(kwargs.get("min_pixels", 10))
elif outlier == "plane":
return MosflmOutlierRejector(kwargs.get("fraction", 1.0),
kwargs.get("n_sigma", 4.0))
elif outlier == "tukey":
return TukeyOutlierRejector(kwargs.get("lower", 1.5),
kwargs.get("upper", 1.5))
raise RuntimeError(f"Unexpected outlier rejector: {outlier}")
def test_run(dials_regression):
from dials.algorithms.background.simple import (
Linear2dModeller,
MosflmOutlierRejector,
)
# The directory path
path = os.path.join(dials_regression, "integration_test_data",
"i04-weak-data", "jmp_mosflm_test")
# The input files
reflection_filename = os.path.join(path, "mosflm_reflections.pickle")
shoebox_filename = os.path.join(path, "shoeboxes.pickle")
fraction = 1.0
n_sigma = 4
outlier_rejector = MosflmOutlierRejector(fraction, n_sigma)
linear_modeller = Linear2dModeller()
from dials.algorithms.shoebox import MaskCode
from dials.array_family import flex
print(shoebox_filename)
# Read the data
rtable = flex.reflection_table.from_file(reflection_filename)
with open(shoebox_filename, "rb") as fh:
if six.PY3:
shoeboxes, masks = pickle.load(fh, encoding="bytes")
else:
shoeboxes, masks = pickle.load(fh)
assert len(rtable) == len(shoeboxes)
assert len(rtable) == len(masks)
# Compute the background for each reflection and check against the values
# read from the mosflm.lp file. Currently this fails for 1 strange
# reflection whose pixel values in the mosflm file do not match those
# extracted from the images.
errors = 0
VAR1 = []
VAR2 = []
DIFF = []
for i in range(len(rtable)):
I = rtable[i]["intensity.sum.value"]
Ivar = rtable[i]["intensity.sum.variance"]
data = shoeboxes[i].as_double()
mask = masks[i]
assert len(data.all()) == 2
assert len(mask.all()) == 2
data.reshape(flex.grid(1, *data.all()))
mask.reshape(flex.grid(1, *mask.all()))
outlier_rejector(data, mask)
mask2 = (mask.as_1d() & int(MaskCode.BackgroundUsed)) != 0
mask2.reshape(flex.grid(*mask.all()))
model = linear_modeller.create(data, mask2)
assert len(model.params()) == 3
hy = data.all()[1] // 2
hx = data.all()[2] // 2
c1 = model.params()[0]
a1 = model.params()[1]
b1 = model.params()[2]
c3 = c1 + a1 * (0.5 + hx) + b1 * (0.5 + hy)
a2 = rtable[i]["background"][0]
b2 = rtable[i]["background"][1]
c2 = rtable[i]["background"][2]
try:
assert abs(a1 - b2) < 0.01
assert abs(b1 + a2) < 0.01
assert abs(c3 - c2) < 0.1
except Exception:
errors += 1
continue
background = data.as_double()
# hy = background.all()[1] // 2
# hx = background.all()[2] // 2
for jj in range(background.all()[1]):
for ii in range(background.all()[2]):
# x = ii - hx
# y = jj - hy
x = ii + 0.5
y = jj + 0.5
background[0, jj, ii] = a1 * x + b1 * y + c1
# Test the summation results. Edge reflections use profile fitted
# intensity in MOSFLM. Therefore ignore these. There also appears to be a
# some discrepancy with very low <= 0 reflections where an extra 0.5 is
# added. Not sure why this is so ignore these reflections as well.
from dials.algorithms.integration.sum import integrate_by_summation
intensity = integrate_by_summation(data.as_double(), background, mask)
I2 = intensity.intensity()
Ivar2 = intensity.variance()
I1 = I
Ivar1 = Ivar
if mask.count(0) == 0 and mask.count(2) == 0 and I1 > 0:
VAR1.append(math.sqrt(Ivar1))
VAR2.append(math.sqrt(Ivar2))
DIFF.append(math.sqrt(Ivar1) - math.sqrt(Ivar2))
try:
assert abs(I1 - I2) < 1.0
assert abs(math.sqrt(Ivar1) - math.sqrt(Ivar2)) < 1.0
except Exception:
errors += 1
continue
# Only 1 should fail
assert errors == 1
def test_run(dials_regression):
from dials.algorithms.background.simple import Creator
from dials.algorithms.background.simple import MosflmOutlierRejector
from dials.algorithms.background.simple import Linear2dModeller
# The directory path
path = os.path.join(dials_regression, "integration_test_data",
"i04-weak-data", "jmp_mosflm_test")
# The input files
reflection_filename = os.path.join(path, "mosflm_reflections.pickle")
shoebox_filename = os.path.join(path, "shoeboxes.pickle")
fraction = 1.0
n_sigma = 4
outlier_rejector = MosflmOutlierRejector(fraction, n_sigma)
linear_modeller = Linear2dModeller()
background_creator = Creator(linear_modeller, outlier_rejector)
from dials.array_family import flex
from math import sqrt
from dials.algorithms.shoebox import MaskCode
print(shoebox_filename)
# Read the data
rtable = flex.reflection_table.from_pickle(reflection_filename)
with open(shoebox_filename, "rb") as fh:
shoeboxes, masks = pickle.load(fh)
assert len(rtable) == len(shoeboxes)
assert len(rtable) == len(masks)
# Compute the background for each reflection and check against the values
# read from the mosflm.lp file. Currently this fails for 1 strange
# reflection whose pixel values in the mosflm file do not match those
# extracted from the images.
count = 0
VAR1 = []
VAR2 = []
DIFF = []
for i in range(len(rtable)):
xdet, ydet = rtable[i]["xy"]
nx = rtable[i]['nx']
ny = rtable[i]['ny']
nc = rtable[i]['nc']
nrx = rtable[i]['nrx']
nry = rtable[i]['nry']
bbox = rtable[i]['bbox']
I = rtable[i]['intensity.sum.value']
Ivar = rtable[i]['intensity.sum.variance']
lp = rtable[i]['lp']
data = shoeboxes[i].as_double()
mask = masks[i]
fraction = 1.0
nsigma = 4
try:
# model = PlaneModel(data, mask, fraction, nsigma)
assert len(data.all()) == 2
assert len(mask.all()) == 2
data.reshape(flex.grid(1, *data.all()))
mask.reshape(flex.grid(1, *mask.all()))
outlier_rejector(data, mask)
mask2 = (mask.as_1d() & int(MaskCode.BackgroundUsed)) != 0
mask2.reshape(flex.grid(*mask.all()))
model = linear_modeller.create(data, mask2)
except Exception:
count += 1
raise
continue
# n = model.noutlier()
assert (len(model.params()) == 3)
hy = data.all()[1] // 2
hx = data.all()[2] // 2
c1 = model.params()[0]
a1 = model.params()[1]
b1 = model.params()[2]
c3 = c1 + a1 * (0.5 + hx) + b1 * (0.5 + hy)
# a1 = model.a()
# b1 = model.b()
# c1 = model.c()
a2 = rtable[i]['background'][0]
b2 = rtable[i]['background'][1]
c2 = rtable[i]['background'][2]
try:
assert (abs(a1 - b2) < 0.01)
assert (abs(b1 + a2) < 0.01)
assert (abs(c3 - c2) < 0.1)
except Exception:
count += 1
continue
#print "BG %d:(%.2f, %.2f, %.1f), (%.2f, %.2f, %.1f): %d" % \
#(i, a1, b1, c1, a2, b2, c2, n)
#print "X, Y: ", xdet, ydet
#print "NX, NY: ", nx, ny
#print "NRX, NRY, NC", nrx, nry, nc
#print int(floor(xdet + 0.5)) - nx // 2, int(floor(ydet + 0.5)) - ny // 2
#print "BBOX: ", bbox
#print "N Outliers: ", model.noutlier()
#print "N Background: ", model.nbackground()
#print "Max DIff: ", model.maxdiff()
#print data.as_numpy_array().transpose()[::-1,::-1]
#print mask.as_numpy_array().transpose()[::-1,::-1]
#raise
background = data.as_double()
# hy = background.all()[1] // 2
# hx = background.all()[2] // 2
for jj in range(background.all()[1]):
for ii in range(background.all()[2]):
# x = ii - hx
# y = jj - hy
x = ii + 0.5
y = jj + 0.5
background[0, jj, ii] = a1 * x + b1 * y + c1
# Test the summation results. Edge reflections use profile fitted
# intensity in MOSFLM. Therefore ignore these. There also appears to be a
# some discrepancy with very low <= 0 reflections where an extra 0.5 is
# added. Not sure why this is so ignore these reflections as well.
from dials.algorithms.integration.sum import integrate_by_summation
intensity = integrate_by_summation(data.as_double(), background, mask)
I2 = intensity.intensity()
Ivar2 = intensity.variance()
I1 = I
Ivar1 = Ivar
if mask.count(0) == 0 and mask.count(2) == 0 and I1 > 0:
VAR1.append(sqrt(Ivar1))
VAR2.append(sqrt(Ivar2))
DIFF.append(sqrt(Ivar1) - sqrt(Ivar2))
try:
assert (abs(I1 - I2) < 1.0)
assert (abs(sqrt(Ivar1) - sqrt(Ivar2)) < 1.0)
except Exception:
count += 1
#import numpy
#numpy.set_printoptions(precision=4, linewidth=200)
#print "# %d" % i
#print "I: %f, %f, %f" % (I2, I1, lp)
#print "DEBUG: ", c1 * 25
#print "PF: %f" % rtable[i]['intensity.prf.value']
#print "BG (%.4f, %.4f, %.4f), (%.2f, %.2f, %.1f): %d" % \
#(a1, b1, c1, a2, b2, c2, n)
#print "X, Y: ", xdet, ydet
#print "NX, NY: ", nx, ny
#print "NRX, NRY, NC", nrx, nry, nc
#print int(floor(xdet + 0.5)) - nx // 2, int(floor(ydet + 0.5)) - ny // 2
#print "BBOX: ", bbox
#print "N Outliers: ", model.noutlier()
#print "N Background: ", model.nbackground()
#print "Max DIff: ", model.maxdiff()
#temp = (mask == MaskCode.Valid | MaskCode.Foreground).as_1d().as_int()
#temp.resize(flex.grid(*data.all()))
#temp = temp.as_double()
#print data.as_numpy_array().transpose()[::-1,::-1]
#print (background * temp).as_numpy_array().transpose()[::-1,::-1]
#print mask.as_numpy_array().transpose()[::-1,::-1]
#print ((data.as_double() - background) * temp).as_numpy_array().transpose()[::-1,::-1]
#raise
continue
# Only 1 should fail
assert (count == 1)
