def test_with_flat_background_partial():
np.random.seed(0)
# Create profile
p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2))
s = flex.sum(p)
p = p / s
# Copy profile
c0 = add_poisson_noise(100 * p)
b = flex.double(flex.grid(9, 9, 9), 1)
m = flex.bool(flex.grid(9, 9, 9), True)
c = c0 + add_poisson_noise(b)
# Get the partial profiles
pp = p[0:5, :, :]
mp = m[0:5, :, :]
cp = c[0:5, :, :]
bp = b[0:5, :, :]
# Fit
fit = ProfileFitter(cp, bp, mp, pp)
intensity = fit.intensity()
V = fit.variance()
assert fit.niter() < fit.maxiter()
Iknown = 99.06932141277105
Vknown = 504.06932141277105
# Test intensity is the same
eps = 1e-7
assert intensity[0] == pytest.approx(Iknown, abs=eps)
assert V[0] == pytest.approx(Vknown, abs=eps)
def test_with_flat_background():
np.random.seed(0)
# Create profile
p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2))
s = flex.sum(p)
p = p / s
# Copy profile
c0 = add_poisson_noise(100 * p)
b = flex.double(flex.grid(9, 9, 9), 10)
m = flex.bool(flex.grid(9, 9, 9), True)
b0 = add_poisson_noise(b)
c = c0 + b0
# Fit
fit = ProfileFitter(c, b, m, p)
intensity = fit.intensity()
V = fit.variance()
assert fit.niter() < fit.maxiter()
Iknown = 201.67417836585147
Vknown = 7491.6743173001205
# Test intensity is the same
eps = 1e-3
assert intensity[0] == pytest.approx(Iknown, abs=eps)
assert V[0] == pytest.approx(Vknown, abs=eps)
def test_identical_partial():
np.random.seed(0)
# Create profile
p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2))
s = flex.sum(p)
p = p / s
# Copy profile
c = p.deep_copy()
b = flex.double(flex.grid(9, 9, 9), 0)
m = flex.bool(flex.grid(9, 9, 9), True)
# Get the partial profiles
pp = p[0:5, :, :]
mp = m[0:5, :, :]
cp = c[0:5, :, :]
bp = b[0:5, :, :]
# Fit
fit = ProfileFitter(cp, bp, mp, pp)
intensity = fit.intensity()
V = fit.variance()
assert fit.niter() < fit.maxiter()
# Test intensity is the same
eps = 1e-7
assert intensity[0] == pytest.approx(flex.sum(p), abs=eps)
assert V[0] == pytest.approx(flex.sum(p), abs=eps)
def test_identical_partial(): from dials.algorithms.integration.fit import ProfileFitter from scitbx.array_family import flex from numpy.random import seed seed(0) # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c = p.deep_copy() b = flex.double(flex.grid(9, 9, 9), 0) m = flex.bool(flex.grid(9,9,9), True) # Get the partial profiles pp = p[0:5,:,:] mp = m[0:5,:,:] cp = c[0:5,:,:] bp = b[0:5,:,:] # Fit fit = ProfileFitter(cp, bp, mp, pp) I = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() # Test intensity is the same eps = 1e-7 assert(abs(I[0] - flex.sum(p)) < eps) assert(abs(V[0] - flex.sum(p)) < eps)
def test_zero(): from dials.algorithms.integration.fit import ProfileFitter from scitbx.array_family import flex from numpy.random import seed seed(0) # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c = flex.double(flex.grid(9,9,9), 0) b = flex.double(flex.grid(9, 9, 9), 0) m = flex.bool(flex.grid(9,9,9), True) # Fit fit = ProfileFitter(c, b, m, p) I = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() # Test intensity is the same eps = 1e-3 assert(abs(I[0] - flex.sum(c)) < eps) assert(abs(V[0] - I[0]) < eps)
def test_with_flat_background(): from dials.algorithms.integration.fit import ProfileFitter from scitbx.array_family import flex from numpy.random import seed seed(0) # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c0 = add_poisson_noise(100 * p) b = flex.double(flex.grid(9, 9, 9), 10) m = flex.bool(flex.grid(9,9,9), True) b0 = add_poisson_noise(b) c = c0 + b0 # Fit fit = ProfileFitter(c, b, m, p) I = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() Iknown = 201.67417836585147 Vknown = 7491.6743173001205 # Test intensity is the same eps = 1e-3 assert(abs(I[0] - Iknown) < eps) assert(abs(V[0] - Vknown) < eps)
def test_deconvolve_7_with_flat_background():
np.random.seed(0)
I0 = [1000, 1500, 2000, 2500, 3000, 3500, 4000]
# Create profile
p = generate_7_profiles()
Ical = [[], [], [], [], [], [], []]
for it in range(1):
# Copy profile
c = flex.double(flex.grid(40, 40, 40))
for i in range(p.all()[0]):
pp = p[i:i + 1, :, :, :]
pp.reshape(c.accessor())
cc = add_poisson_noise(I0[i] * pp)
c += cc
b = flex.double(flex.grid(40, 40, 40), 1)
m = flex.bool(flex.grid(40, 40, 40), True)
c += add_poisson_noise(b)
# Fit
fit = ProfileFitter(c, b, m, p)
intensity = fit.intensity()
V = fit.variance()
assert fit.niter() < fit.maxiter()
for i in range(7):
Ical[i].append(intensity[i])
for i in range(7):
Ical[i] = sum(Ical[i]) / len(Ical[i])
Iknown = [
1042.4904898451261,
1447.6413897568257,
2053.2524102387893,
2485.2789614429703,
3063.867598583333,
3445.9230910807582,
3977.6744651425543,
]
Vknown = [
65042.49048984377,
65447.6413897533,
66053.25241023625,
66485.27896144328,
67063.86759858252,
67445.92309107889,
67977.67446514373,
]
# Test intensity is the same
eps = 1e-7
for i in range(7):
assert intensity[i] == pytest.approx(Iknown[i], abs=eps)
assert V[i] == pytest.approx(Vknown[i], abs=eps)
def test_deconvolve_7_with_no_background():
from dials.algorithms.integration.fit import ProfileFitter
from scitbx.array_family import flex
from numpy.random import seed
seed(0)
I0 = [1000, 1500, 2000, 2500, 3000, 3500, 4000]
# Create profile
p = generate_7_profiles()
Ical = [[], [], [], [], [], [], []]
for it in range(1):
# Copy profile
c = flex.double(flex.grid(40, 40, 40))
for i in range(p.all()[0]):
pp = p[i : i + 1, :, :, :]
pp.reshape(c.accessor())
cc = add_poisson_noise(I0[i] * pp)
c += cc
b = flex.double(flex.grid(40, 40, 40), 0)
m = flex.bool(flex.grid(40, 40, 40), True)
# Fit
fit = ProfileFitter(c, b, m, p)
I = fit.intensity()
V = fit.variance()
assert fit.niter() < fit.maxiter()
for i in range(7):
Ical[i].append(I[i])
for i in range(7):
Ical[i] = sum(Ical[i]) / len(Ical[i])
Iknown = [
1012.4193633595916,
1472.3322059495797,
2072.136413825826,
2486.4902438469253,
3012.6132119521126,
3409.2053517072773,
3952.803209358826,
]
# Test intensity is the same
eps = 1e-7
for i in range(7):
assert abs(I[i] - Iknown[i]) < eps
assert abs(V[i] - Iknown[i]) < eps
def tst_deconvolve_7_with_flat_background(self):
from dials.algorithms.integration.fit import ProfileFitter
from scitbx.array_family import flex
from numpy.random import seed
seed(0)
I0 = [1000, 1500, 2000, 2500, 3000, 3500, 4000]
# Create profile
p = self.generate_7_profiles()
Ical = [[],[],[],[],[],[],[]]
for it in range(1):
# Copy profile
c = flex.double(flex.grid(40, 40, 40))
for i in range(p.all()[0]):
pp = p[i:i+1,:,:,:]
pp.reshape(c.accessor())
cc = add_poisson_noise(I0[i] * pp)
c += cc
b = flex.double(flex.grid(40, 40, 40), 1)
m = flex.bool(flex.grid(40,40,40), True)
c += add_poisson_noise(b)
# Fit
fit = ProfileFitter(c, b, m, p)
I = fit.intensity()
V = fit.variance()
assert fit.niter() < fit.maxiter()
for i in range(7):
Ical[i].append(I[i])
for i in range(7):
Ical[i] = sum(Ical[i]) / len(Ical[i])
Iknown = [1042.4904898451261, 1447.6413897568257, 2053.2524102387893,
2485.2789614429703, 3063.867598583333, 3445.9230910807582,
3977.6744651425543]
Vknown = [65042.49048984377, 65447.6413897533, 66053.25241023625,
66485.27896144328, 67063.86759858252, 67445.92309107889,
67977.67446514373]
# Test intensity is the same
eps = 1e-7
for i in range(7):
assert(abs(I[i] - Iknown[i]) < eps)
assert(abs(V[i] - Vknown[i]) < eps)
print 'OK'
def tst_deconvolve_3_with_flat_background(self):
from dials.algorithms.integration.fit import ProfileFitter
from scitbx.array_family import flex
from numpy.random import seed
seed(0)
I0 = [1000, 2000, 3000]
# Create profile
p = self.generate_3_profiles()
Ical = [[],[],[]]
for it in range(1):
# Copy profile
c = flex.double(flex.grid(40, 9, 9))
for i in range(p.all()[0]):
pp = p[i:i+1,:,:,:]
pp.reshape(c.accessor())
cc = add_poisson_noise(I0[i] * pp)
c += cc
b = flex.double(flex.grid(40, 9, 9), 1)
m = flex.bool(flex.grid(40,9,9), True)
c += add_poisson_noise(b)
# Fit
fit = ProfileFitter(c, b, m, p)
I = fit.intensity()
V = fit.variance()
assert fit.niter() < fit.maxiter()
for i in range(3):
Ical[i].append(I[i])
for i in range(3):
Ical[i] = sum(Ical[i]) / len(Ical[i])
Iknown = [1030.7018033805357, 1948.7152700952695, 2972.983204218213]
Vknown = [4270.701803380534, 5188.715270095279, 6212.983204218214]
# Test intensity is the same
eps = 1e-7
for i in range(3):
assert(abs(I[i] - Iknown[i]) < eps)
assert(abs(V[i] - Vknown[i]) < eps)
print 'OK'
def tst_deconvolve_3_with_no_background(self):
from dials.algorithms.integration.fit import ProfileFitter
from scitbx.array_family import flex
from numpy.random import seed
seed(0)
I0 = [1000, 2000, 3000]
# Create profile
p = self.generate_3_profiles()
Ical = [[],[],[]]
for it in range(1):
# Copy profile
c = flex.double(flex.grid(40, 9, 9))
for i in range(p.all()[0]):
pp = p[i:i+1,:,:,:]
pp.reshape(c.accessor())
cc = add_poisson_noise(I0[i] * pp)
c += cc
b = flex.double(flex.grid(40, 9, 9), 0)
m = flex.bool(flex.grid(40,9,9), True)
# Fit
fit = ProfileFitter(c, b, m, p)
I = fit.intensity()
V = fit.variance()
assert fit.niter() < fit.maxiter()
for i in range(3):
Ical[i].append(I[i])
for i in range(3):
Ical[i] = sum(Ical[i]) / len(Ical[i])
Iknown = [1048.3221116842406, 1920.9035376774107, 2938.7743506383745]
# Test intensity is the same
eps = 1e-7
for i in range(3):
assert(abs(I[i] - Iknown[i]) < eps)
assert(abs(V[i] - Iknown[i]) < eps)
print 'OK'
def test_deconvolve_3_with_flat_background():
np.random.seed(0)
I0 = [1000, 2000, 3000]
# Create profile
p = generate_3_profiles()
Ical = [[], [], []]
for it in range(1):
# Copy profile
c = flex.double(flex.grid(40, 9, 9))
for i in range(p.all()[0]):
pp = p[i:i + 1, :, :, :]
pp.reshape(c.accessor())
cc = add_poisson_noise(I0[i] * pp)
c += cc
b = flex.double(flex.grid(40, 9, 9), 1)
m = flex.bool(flex.grid(40, 9, 9), True)
c += add_poisson_noise(b)
# Fit
fit = ProfileFitter(c, b, m, p)
intensity = fit.intensity()
V = fit.variance()
assert fit.niter() < fit.maxiter()
for i in range(3):
Ical[i].append(intensity[i])
for i in range(3):
Ical[i] = sum(Ical[i]) / len(Ical[i])
Iknown = [1030.7018033805357, 1948.7152700952695, 2972.983204218213]
Vknown = [4270.701803380534, 5188.715270095279, 6212.983204218214]
# Test intensity is the same
eps = 1e-7
for i in range(3):
assert intensity[i] == pytest.approx(Iknown[i], abs=eps)
assert V[i] == pytest.approx(Vknown[i], abs=eps)
def test_deconvolve_3_with_no_background():
np.random.seed(0)
I0 = [1000, 2000, 3000]
# Create profile
p = generate_3_profiles()
Ical = [[], [], []]
for it in range(1):
# Copy profile
c = flex.double(flex.grid(40, 9, 9))
for i in range(p.all()[0]):
pp = p[i:i + 1, :, :, :]
pp.reshape(c.accessor())
cc = add_poisson_noise(I0[i] * pp)
c += cc
b = flex.double(flex.grid(40, 9, 9), 0)
m = flex.bool(flex.grid(40, 9, 9), True)
# Fit
fit = ProfileFitter(c, b, m, p)
intensity = fit.intensity()
V = fit.variance()
assert fit.niter() < fit.maxiter()
for i in range(3):
Ical[i].append(intensity[i])
for i in range(3):
Ical[i] = sum(Ical[i]) / len(Ical[i])
Iknown = [1048.3221116842406, 1920.9035376774107, 2938.7743506383745]
# Test intensity is the same
eps = 1e-7
for i in range(3):
assert intensity[i] == pytest.approx(Iknown[i], abs=eps)
assert V[i] == pytest.approx(Iknown[i], abs=eps)
def tst_with_flat_background_partial(self):
from dials.algorithms.integration.fit import ProfileFitter
from scitbx.array_family import flex
from numpy.random import seed
seed(0)
# Create profile
p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2))
s = flex.sum(p)
p = p / s
# Copy profile
c0 = add_poisson_noise(100 * p)
b = flex.double(flex.grid(9, 9, 9), 1)
m = flex.bool(flex.grid(9,9,9), True)
c = c0 + add_poisson_noise(b)
# Get the partial profiles
pp = p[0:5,:,:]
mp = m[0:5,:,:]
cp = c[0:5,:,:]
bp = b[0:5,:,:]
c0p = c0[0:5,:,:]
# Fit
fit = ProfileFitter(cp, bp, mp, pp)
I = fit.intensity()
V = fit.variance()
assert fit.niter() < fit.maxiter()
Iknown = 99.06932141277105
Vknown = 504.06932141277105
# Test intensity is the same
eps = 1e-7
assert(abs(I[0] - Iknown) < eps)
assert(abs(V[0] - Vknown) < eps)
print 'OK'
def test_mask():
np.random.seed(0)
# Create profile
p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2))
s = flex.sum(p)
p = p / s
# Copy profile
c = add_poisson_noise(100 * p)
b = flex.double(flex.grid(9, 9, 9), 0)
m = flex.bool(flex.grid(9, 9, 9), True)
m[4, 4, 4] = False
# Fit
fit = ProfileFitter(c, b, m, p)
intensity = fit.intensity()
V = fit.variance()
assert fit.niter() < fit.maxiter()
# Test intensity is approximately the same
assert intensity[0] == pytest.approx(flex.sum(c), rel=0.01)
assert intensity[0] == pytest.approx(V[0], abs=1e-3)
def test_zero():
np.random.seed(0)
# Create profile
p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2))
s = flex.sum(p)
p = p / s
# Copy profile
c = flex.double(flex.grid(9, 9, 9), 0)
b = flex.double(flex.grid(9, 9, 9), 0)
m = flex.bool(flex.grid(9, 9, 9), True)
# Fit
fit = ProfileFitter(c, b, m, p)
intensity = fit.intensity()
V = fit.variance()
assert fit.niter() < fit.maxiter()
# Test intensity is the same
eps = 1e-3
assert intensity[0] == pytest.approx(flex.sum(c), abs=eps)
assert intensity[0] == pytest.approx(V[0], abs=eps)
def test_with_no_background_partial():
from dials.algorithms.integration.fit import ProfileFitter
from scitbx.array_family import flex
from numpy.random import seed
seed(0)
# Create profile
p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2))
s = flex.sum(p)
p = p / s
# Copy profile
c = add_poisson_noise(100 * p)
b = flex.double(flex.grid(9, 9, 9), 0)
m = flex.bool(flex.grid(9, 9, 9), True)
# Get the partial profiles
pp = p[0:5, :, :]
mp = m[0:5, :, :]
cp = c[0:5, :, :]
bp = b[0:5, :, :]
# Fit
fit = ProfileFitter(cp, bp, mp, pp)
I = fit.intensity()
V = fit.variance()
assert fit.niter() < fit.maxiter()
Iknown = 94.67151440319306
Vknown = 94.67151440319306
# Test intensity is the same
eps = 1e-7
assert abs(I[0] - Iknown) < eps
assert abs(V[0] - Vknown) < eps
def integrate_job(block,
experiments,
reflections,
reference,
grid_size=5,
detector_space=False):
from dials.algorithms.profile_model.modeller import CircleSampler
from dials.array_family import flex
from dials.algorithms.profile_model.gaussian_rs.transform import TransformReverse
from dials.algorithms.profile_model.gaussian_rs.transform import TransformForward
from dials.algorithms.profile_model.gaussian_rs.transform import (
TransformReverseNoModel, )
from dials.algorithms.profile_model.gaussian_rs.transform import TransformSpec
from dials.algorithms.profile_model.gaussian_rs import CoordinateSystem
from dials.algorithms.integration.fit import ProfileFitter
from dials.array_family import flex
from dials.model.data import make_image
reflections["shoebox"] = flex.shoebox(reflections["panel"],
reflections["bbox"],
allocate=True)
frame0, frame1 = experiments[0].scan.get_array_range()
frame0 = frame0 + block[0]
frame1 = frame0 + block[1]
reflections["shoebox"] = flex.shoebox(reflections["panel"],
reflections["bbox"],
allocate=True)
extractor = flex.ShoeboxExtractor(reflections, 1, frame0, frame1)
iset = experiments[0].imageset[block[0]:block[1]]
for i in range(len(iset)):
print("Reading image %d" % i)
data = iset.get_raw_data(i)
mask = iset.get_mask(i)
extractor.next(make_image(data, mask))
print("Computing mask")
reflections.compute_mask(experiments)
print("Computing background")
reflections.compute_background(experiments)
print("Computing centroid")
reflections.compute_centroid(experiments)
print("Computing summed intensity")
reflections.compute_summed_intensity()
assert len(reference) % 9 == 0
num_scan_points = len(reference) // 9
sampler = CircleSampler(
experiments[0].detector[0].get_image_size(),
experiments[0].scan.get_array_range(),
num_scan_points,
)
spec = TransformSpec(
experiments[0].beam,
experiments[0].detector,
experiments[0].goniometer,
experiments[0].scan,
experiments[0].profile.sigma_b(deg=False),
experiments[0].profile.sigma_m(deg=False),
experiments[0].profile.n_sigma() * 1.5,
grid_size,
)
m2 = experiments[0].goniometer.get_rotation_axis()
s0 = experiments[0].beam.get_s0()
Iprf = flex.double(len(reflections))
Vprf = flex.double(len(reflections))
Cprf = flex.double(len(reflections))
Fprf = flex.bool(len(reflections))
Part = reflections["partiality"]
reflections["intensity.prf_old.value"] = reflections["intensity.prf.value"]
reflections["intensity.prf_old.variance"] = reflections[
"intensity.prf.variance"]
selection = reflections.get_flags(reflections.flags.integrated_prf)
reflections.unset_flags(~Fprf, reflections.flags.integrated_prf)
for i, r in enumerate(reflections):
if selection[i] == False:
continue
s1 = r["s1"]
phi = r["xyzcal.mm"][2]
xyz = r["xyzcal.px"]
bbox = r["bbox"]
panel = r["panel"]
image = r["shoebox"].data.as_double()
background = r["shoebox"].background.as_double()
mask = r["shoebox"].mask.as_1d(
) == 5 # | (r['shoebox'].mask.as_1d() == 3)
mask.reshape(image.accessor())
cs = CoordinateSystem(m2, s0, s1, phi)
index = sampler.nearest(0, xyz)
profile, profile_mask = reference[index]
# print flex.sum(profile)
# print r['partiality']
if detector_space:
transform = TransformReverseNoModel(spec, cs, bbox, panel, profile)
p = transform.profile()
d = image
m = mask
b = background
# print flex.sum(p)
Part[i] = flex.sum(p)
# ysize, xsize = p.all()[1:3]
# p1 = flex.double(flex.grid(1, ysize , xsize))
# d1 = flex.double(flex.grid(1, ysize , xsize))
# b1 = flex.double(flex.grid(1, ysize , xsize))
# m1 = flex.double(flex.grid(1, ysize , xsize))
# for k in range(p.all()[0]):
# p1 += p[k:k+1,:,:]
# d1 += d[k:k+1,:,:]
# b1 += b[k:k+1,:,:]
# m1 = m[k:k+1,:,:]
try:
fit = ProfileFitter(d, b, m, p, 1e-3, 100)
assert fit.niter() < 100
Iprf[i] = fit.intensity()
Vprf[i] = fit.variance()
Cprf[i] = fit.correlation()
Fprf[i] = True
# if r['intensity.sum.value'] > 10 and abs(fit.intensity()) < 1e-3:
print(
r["miller_index"],
i,
fit.intensity(),
r["intensity.sum.value"],
r["intensity.prf_old.value"],
Part[i],
fit.niter(),
)
# from matplotlib import pylab
# pylab.imshow(p1.as_numpy_array()[0,:,:], interpolation='none')
# pylab.show()
except Exception as e:
print(e)
pass
else:
try:
transform = TransformForward(spec, cs, bbox, panel, image,
background, mask)
p = profile
d = transform.profile()
b = transform.background()
m = transform.mask() & profile_mask
# if r['miller_index'] == (9, -25, 74):
# print list(p)
# print list(m)
# print list(b)
# print list(d)
# exit(0)
fit = ProfileFitter(d, b, m, p, 1e-3, 100)
assert fit.niter() < 100
Iprf[i] = fit.intensity()[0]
Vprf[i] = fit.variance()[0]
Cprf[i] = fit.correlation()
Fprf[i] = True
print(r["miller_index"], i, fit.intensity(),
r["intensity.prf_old.value"])
# from matplotlib import pylab
# pylab.imshow(p1.as_numpy_array()[0,:,:], interpolation='none')
# pylab.show()
except Exception as e:
pass
reflections["intensity.prf.value"] = Iprf
reflections["intensity.prf.variance"] = Vprf
reflections["intensity.prf.correlation"] = Cprf
reflections.set_flags(Fprf, reflections.flags.integrated_prf)
del reflections["shoebox"]
return reflections