class Test(object):
def __init__(self):
from dials.algorithms.background.simple import Creator
from dials.algorithms.background.simple import MosflmOutlierRejector
from dials.algorithms.background.simple import Linear2dModeller
from os.path import join
import libtbx.load_env
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'SKIP: dials_regression not configured'
exit(0)
# The directory path
path = join(
dials_regression,
"integration_test_data",
"i04-weak-data",
"jmp_mosflm_test")
# The input files
self.reflection_filename = join(path, "mosflm_reflections.pickle")
self.shoebox_filename = join(path, "shoeboxes.pickle")
self.fraction = 1.0
self.n_sigma = 4
self.outlier_rejector = MosflmOutlierRejector(self.fraction, self.n_sigma)
self.linear_modeller = Linear2dModeller()
self.background_creator = Creator(
self.linear_modeller,
self.outlier_rejector)
def __init__(self,
experiments,
outlier='nsigma',
model='constant3d',
**kwargs):
'''
Initialise the algorithm.
:param experiments: The list of experiments
:param outlier: The outlier rejection algorithm
:param model: The background model algorithm
'''
from dials.algorithms.background.simple import Creator
from dials.algorithms.background.simple import TruncatedOutlierRejector
from dials.algorithms.background.simple import NSigmaOutlierRejector
from dials.algorithms.background.simple import NormalOutlierRejector
from dials.algorithms.background.simple import MosflmOutlierRejector
from dials.algorithms.background.simple import TukeyOutlierRejector
from dials.algorithms.background.simple import Constant2dModeller
from dials.algorithms.background.simple import Constant3dModeller
from dials.algorithms.background.simple import Linear2dModeller
from dials.algorithms.background.simple import Linear3dModeller
def select_modeller():
if model == 'constant2d':
return Constant2dModeller()
elif model == 'constant3d':
return Constant3dModeller()
elif model == 'linear2d':
return Linear2dModeller()
elif model == 'linear3d':
return Linear3dModeller()
raise RuntimeError("Unexpected background model: %s" % model)
def select_rejector():
if outlier == 'null':
return None
elif outlier == 'truncated':
return TruncatedOutlierRejector(kwargs.get("lower", 0.01),
kwargs.get("upper", 0.01))
elif outlier == 'nsigma':
return NSigmaOutlierRejector(kwargs.get("lower", 3),
kwargs.get("upper", 3))
elif outlier == 'normal':
return NormalOutlierRejector(kwargs.get("min_pixels", 10))
elif outlier == 'plane':
return MosflmOutlierRejector(kwargs.get("fraction", 1.0),
kwargs.get("n_sigma", 4.0))
elif outlier == 'tukey':
return TukeyOutlierRejector(kwargs.get("lower", 1.5),
kwargs.get("upper", 1.5))
raise RuntimeError("Unexpected outlier rejector: %s" % outlier)
# Get the minimum number of pixels
min_pixels = kwargs.get("min_pixels", 10)
modeller = select_modeller()
rejector = select_rejector()
self._creator = Creator(modeller, rejector, min_pixels=min_pixels)
def run(self):
from dials.algorithms.background.simple import Creator
from dials.algorithms.background.simple import Constant2dModeller
from dials.algorithms.background.simple import TruncatedOutlierRejector
from dials.algorithms.background.simple import NSigmaOutlierRejector
from dials.algorithms.background.simple import NormalOutlierRejector
modeller = Constant2dModeller()
outlier_rejector = [
None,
TruncatedOutlierRejector(0.01, 0.01),
NSigmaOutlierRejector(3.0, 3.0),
NormalOutlierRejector(10),
]
for rejector in outlier_rejector:
self.tst(Creator(modeller, rejector))
def test_run(dials_regression):
from dials.algorithms.background.simple import Creator
from dials.algorithms.background.simple import MosflmOutlierRejector
from dials.algorithms.background.simple import Linear2dModeller
# The directory path
path = os.path.join(dials_regression, "integration_test_data",
"i04-weak-data", "jmp_mosflm_test")
# The input files
reflection_filename = os.path.join(path, "mosflm_reflections.pickle")
shoebox_filename = os.path.join(path, "shoeboxes.pickle")
fraction = 1.0
n_sigma = 4
outlier_rejector = MosflmOutlierRejector(fraction, n_sigma)
linear_modeller = Linear2dModeller()
background_creator = Creator(linear_modeller, outlier_rejector)
from dials.array_family import flex
from math import sqrt
from dials.algorithms.shoebox import MaskCode
print(shoebox_filename)
# Read the data
rtable = flex.reflection_table.from_pickle(reflection_filename)
with open(shoebox_filename, "rb") as fh:
shoeboxes, masks = pickle.load(fh)
assert len(rtable) == len(shoeboxes)
assert len(rtable) == len(masks)
# Compute the background for each reflection and check against the values
# read from the mosflm.lp file. Currently this fails for 1 strange
# reflection whose pixel values in the mosflm file do not match those
# extracted from the images.
count = 0
VAR1 = []
VAR2 = []
DIFF = []
for i in range(len(rtable)):
xdet, ydet = rtable[i]["xy"]
nx = rtable[i]['nx']
ny = rtable[i]['ny']
nc = rtable[i]['nc']
nrx = rtable[i]['nrx']
nry = rtable[i]['nry']
bbox = rtable[i]['bbox']
I = rtable[i]['intensity.sum.value']
Ivar = rtable[i]['intensity.sum.variance']
lp = rtable[i]['lp']
data = shoeboxes[i].as_double()
mask = masks[i]
fraction = 1.0
nsigma = 4
try:
# model = PlaneModel(data, mask, fraction, nsigma)
assert len(data.all()) == 2
assert len(mask.all()) == 2
data.reshape(flex.grid(1, *data.all()))
mask.reshape(flex.grid(1, *mask.all()))
outlier_rejector(data, mask)
mask2 = (mask.as_1d() & int(MaskCode.BackgroundUsed)) != 0
mask2.reshape(flex.grid(*mask.all()))
model = linear_modeller.create(data, mask2)
except Exception:
count += 1
raise
continue
# n = model.noutlier()
assert (len(model.params()) == 3)
hy = data.all()[1] // 2
hx = data.all()[2] // 2
c1 = model.params()[0]
a1 = model.params()[1]
b1 = model.params()[2]
c3 = c1 + a1 * (0.5 + hx) + b1 * (0.5 + hy)
# a1 = model.a()
# b1 = model.b()
# c1 = model.c()
a2 = rtable[i]['background'][0]
b2 = rtable[i]['background'][1]
c2 = rtable[i]['background'][2]
try:
assert (abs(a1 - b2) < 0.01)
assert (abs(b1 + a2) < 0.01)
assert (abs(c3 - c2) < 0.1)
except Exception:
count += 1
continue
#print "BG %d:(%.2f, %.2f, %.1f), (%.2f, %.2f, %.1f): %d" % \
#(i, a1, b1, c1, a2, b2, c2, n)
#print "X, Y: ", xdet, ydet
#print "NX, NY: ", nx, ny
#print "NRX, NRY, NC", nrx, nry, nc
#print int(floor(xdet + 0.5)) - nx // 2, int(floor(ydet + 0.5)) - ny // 2
#print "BBOX: ", bbox
#print "N Outliers: ", model.noutlier()
#print "N Background: ", model.nbackground()
#print "Max DIff: ", model.maxdiff()
#print data.as_numpy_array().transpose()[::-1,::-1]
#print mask.as_numpy_array().transpose()[::-1,::-1]
#raise
background = data.as_double()
# hy = background.all()[1] // 2
# hx = background.all()[2] // 2
for jj in range(background.all()[1]):
for ii in range(background.all()[2]):
# x = ii - hx
# y = jj - hy
x = ii + 0.5
y = jj + 0.5
background[0, jj, ii] = a1 * x + b1 * y + c1
# Test the summation results. Edge reflections use profile fitted
# intensity in MOSFLM. Therefore ignore these. There also appears to be a
# some discrepancy with very low <= 0 reflections where an extra 0.5 is
# added. Not sure why this is so ignore these reflections as well.
from dials.algorithms.integration.sum import integrate_by_summation
intensity = integrate_by_summation(data.as_double(), background, mask)
I2 = intensity.intensity()
Ivar2 = intensity.variance()
I1 = I
Ivar1 = Ivar
if mask.count(0) == 0 and mask.count(2) == 0 and I1 > 0:
VAR1.append(sqrt(Ivar1))
VAR2.append(sqrt(Ivar2))
DIFF.append(sqrt(Ivar1) - sqrt(Ivar2))
try:
assert (abs(I1 - I2) < 1.0)
assert (abs(sqrt(Ivar1) - sqrt(Ivar2)) < 1.0)
except Exception:
count += 1
#import numpy
#numpy.set_printoptions(precision=4, linewidth=200)
#print "# %d" % i
#print "I: %f, %f, %f" % (I2, I1, lp)
#print "DEBUG: ", c1 * 25
#print "PF: %f" % rtable[i]['intensity.prf.value']
#print "BG (%.4f, %.4f, %.4f), (%.2f, %.2f, %.1f): %d" % \
#(a1, b1, c1, a2, b2, c2, n)
#print "X, Y: ", xdet, ydet
#print "NX, NY: ", nx, ny
#print "NRX, NRY, NC", nrx, nry, nc
#print int(floor(xdet + 0.5)) - nx // 2, int(floor(ydet + 0.5)) - ny // 2
#print "BBOX: ", bbox
#print "N Outliers: ", model.noutlier()
#print "N Background: ", model.nbackground()
#print "Max DIff: ", model.maxdiff()
#temp = (mask == MaskCode.Valid | MaskCode.Foreground).as_1d().as_int()
#temp.resize(flex.grid(*data.all()))
#temp = temp.as_double()
#print data.as_numpy_array().transpose()[::-1,::-1]
#print (background * temp).as_numpy_array().transpose()[::-1,::-1]
#print mask.as_numpy_array().transpose()[::-1,::-1]
#print ((data.as_double() - background) * temp).as_numpy_array().transpose()[::-1,::-1]
#raise
continue
# Only 1 should fail
assert (count == 1)
def __init__(self,
experiments,
outlier="nsigma",
model="constant3d",
**kwargs):
"""
Initialise the algorithm.
:param experiments: The list of experiments
:param outlier: The outlier rejection algorithm
:param model: The background model algorithm
"""
from dials.algorithms.background.simple import (
Constant2dModeller,
Constant3dModeller,
Creator,
Linear2dModeller,
Linear3dModeller,
MosflmOutlierRejector,
NormalOutlierRejector,
NSigmaOutlierRejector,
TruncatedOutlierRejector,
TukeyOutlierRejector,
)
def select_modeller():
if model == "constant2d":
return Constant2dModeller()
elif model == "constant3d":
return Constant3dModeller()
elif model == "linear2d":
return Linear2dModeller()
elif model == "linear3d":
return Linear3dModeller()
raise RuntimeError(f"Unexpected background model: {model}")
def select_rejector():
if outlier == "null":
return None
elif outlier == "truncated":
return TruncatedOutlierRejector(kwargs.get("lower", 0.01),
kwargs.get("upper", 0.01))
elif outlier == "nsigma":
return NSigmaOutlierRejector(kwargs.get("lower", 3),
kwargs.get("upper", 3))
elif outlier == "normal":
return NormalOutlierRejector(kwargs.get("min_pixels", 10))
elif outlier == "plane":
return MosflmOutlierRejector(kwargs.get("fraction", 1.0),
kwargs.get("n_sigma", 4.0))
elif outlier == "tukey":
return TukeyOutlierRejector(kwargs.get("lower", 1.5),
kwargs.get("upper", 1.5))
raise RuntimeError(f"Unexpected outlier rejector: {outlier}")
# Get the minimum number of pixels
min_pixels = kwargs.get("min_pixels", 10)
modeller = select_modeller()
rejector = select_rejector()
self._creator = Creator(modeller, rejector, min_pixels=min_pixels)
