def test_zero(): from dials.algorithms.integration.fit import ProfileFitter from scitbx.array_family import flex from numpy.random import seed seed(0) # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c = flex.double(flex.grid(9,9,9), 0) b = flex.double(flex.grid(9, 9, 9), 0) m = flex.bool(flex.grid(9,9,9), True) # Fit fit = ProfileFitter(c, b, m, p) I = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() # Test intensity is the same eps = 1e-3 assert(abs(I[0] - flex.sum(c)) < eps) assert(abs(V[0] - I[0]) < eps)
def tst_deconvolve_zero(self): from dials.algorithms.integration.fit import ProfileFitter from scitbx.array_family import flex from numpy.random import seed seed(0) I0 = [1000, 2000, 3000] # Create profile p = self.generate_3_profiles() # Copy profile c = flex.double(flex.grid(40, 9, 9), 0) b = flex.double(flex.grid(40, 9, 9), 0) m = flex.bool(flex.grid(40,9,9), True) # Fit passed = False try: fit = ProfileFitter(c, b, m, p) I = fit.intensity() V = fit.variance() passed = True except Exception: pass if passed: assert False, "This should fail" print 'OK'
def test_with_flat_background(): np.random.seed(0) # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c0 = add_poisson_noise(100 * p) b = flex.double(flex.grid(9, 9, 9), 10) m = flex.bool(flex.grid(9, 9, 9), True) b0 = add_poisson_noise(b) c = c0 + b0 # Fit fit = ProfileFitter(c, b, m, p) intensity = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() Iknown = 201.67417836585147 Vknown = 7491.6743173001205 # Test intensity is the same eps = 1e-3 assert intensity[0] == pytest.approx(Iknown, abs=eps) assert V[0] == pytest.approx(Vknown, abs=eps)
def test_identical_partial(): np.random.seed(0) # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c = p.deep_copy() b = flex.double(flex.grid(9, 9, 9), 0) m = flex.bool(flex.grid(9, 9, 9), True) # Get the partial profiles pp = p[0:5, :, :] mp = m[0:5, :, :] cp = c[0:5, :, :] bp = b[0:5, :, :] # Fit fit = ProfileFitter(cp, bp, mp, pp) intensity = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() # Test intensity is the same eps = 1e-7 assert intensity[0] == pytest.approx(flex.sum(p), abs=eps) assert V[0] == pytest.approx(flex.sum(p), abs=eps)
def test_with_flat_background_partial(): np.random.seed(0) # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c0 = add_poisson_noise(100 * p) b = flex.double(flex.grid(9, 9, 9), 1) m = flex.bool(flex.grid(9, 9, 9), True) c = c0 + add_poisson_noise(b) # Get the partial profiles pp = p[0:5, :, :] mp = m[0:5, :, :] cp = c[0:5, :, :] bp = b[0:5, :, :] # Fit fit = ProfileFitter(cp, bp, mp, pp) intensity = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() Iknown = 99.06932141277105 Vknown = 504.06932141277105 # Test intensity is the same eps = 1e-7 assert intensity[0] == pytest.approx(Iknown, abs=eps) assert V[0] == pytest.approx(Vknown, abs=eps)
def test_with_flat_background(): from dials.algorithms.integration.fit import ProfileFitter from scitbx.array_family import flex from numpy.random import seed seed(0) # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c0 = add_poisson_noise(100 * p) b = flex.double(flex.grid(9, 9, 9), 10) m = flex.bool(flex.grid(9,9,9), True) b0 = add_poisson_noise(b) c = c0 + b0 # Fit fit = ProfileFitter(c, b, m, p) I = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() Iknown = 201.67417836585147 Vknown = 7491.6743173001205 # Test intensity is the same eps = 1e-3 assert(abs(I[0] - Iknown) < eps) assert(abs(V[0] - Vknown) < eps)
def test_identical_partial(): from dials.algorithms.integration.fit import ProfileFitter from scitbx.array_family import flex from numpy.random import seed seed(0) # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c = p.deep_copy() b = flex.double(flex.grid(9, 9, 9), 0) m = flex.bool(flex.grid(9,9,9), True) # Get the partial profiles pp = p[0:5,:,:] mp = m[0:5,:,:] cp = c[0:5,:,:] bp = b[0:5,:,:] # Fit fit = ProfileFitter(cp, bp, mp, pp) I = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() # Test intensity is the same eps = 1e-7 assert(abs(I[0] - flex.sum(p)) < eps) assert(abs(V[0] - flex.sum(p)) < eps)
def test_deconvolve_7_with_flat_background(): np.random.seed(0) I0 = [1000, 1500, 2000, 2500, 3000, 3500, 4000] # Create profile p = generate_7_profiles() Ical = [[], [], [], [], [], [], []] for it in range(1): # Copy profile c = flex.double(flex.grid(40, 40, 40)) for i in range(p.all()[0]): pp = p[i:i + 1, :, :, :] pp.reshape(c.accessor()) cc = add_poisson_noise(I0[i] * pp) c += cc b = flex.double(flex.grid(40, 40, 40), 1) m = flex.bool(flex.grid(40, 40, 40), True) c += add_poisson_noise(b) # Fit fit = ProfileFitter(c, b, m, p) intensity = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() for i in range(7): Ical[i].append(intensity[i]) for i in range(7): Ical[i] = sum(Ical[i]) / len(Ical[i]) Iknown = [ 1042.4904898451261, 1447.6413897568257, 2053.2524102387893, 2485.2789614429703, 3063.867598583333, 3445.9230910807582, 3977.6744651425543, ] Vknown = [ 65042.49048984377, 65447.6413897533, 66053.25241023625, 66485.27896144328, 67063.86759858252, 67445.92309107889, 67977.67446514373, ] # Test intensity is the same eps = 1e-7 for i in range(7): assert intensity[i] == pytest.approx(Iknown[i], abs=eps) assert V[i] == pytest.approx(Vknown[i], abs=eps)
def test_deconvolve_7_with_no_background(): from dials.algorithms.integration.fit import ProfileFitter from scitbx.array_family import flex from numpy.random import seed seed(0) I0 = [1000, 1500, 2000, 2500, 3000, 3500, 4000] # Create profile p = generate_7_profiles() Ical = [[], [], [], [], [], [], []] for it in range(1): # Copy profile c = flex.double(flex.grid(40, 40, 40)) for i in range(p.all()[0]): pp = p[i : i + 1, :, :, :] pp.reshape(c.accessor()) cc = add_poisson_noise(I0[i] * pp) c += cc b = flex.double(flex.grid(40, 40, 40), 0) m = flex.bool(flex.grid(40, 40, 40), True) # Fit fit = ProfileFitter(c, b, m, p) I = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() for i in range(7): Ical[i].append(I[i]) for i in range(7): Ical[i] = sum(Ical[i]) / len(Ical[i]) Iknown = [ 1012.4193633595916, 1472.3322059495797, 2072.136413825826, 2486.4902438469253, 3012.6132119521126, 3409.2053517072773, 3952.803209358826, ] # Test intensity is the same eps = 1e-7 for i in range(7): assert abs(I[i] - Iknown[i]) < eps assert abs(V[i] - Iknown[i]) < eps
def tst_deconvolve_7_with_flat_background(self): from dials.algorithms.integration.fit import ProfileFitter from scitbx.array_family import flex from numpy.random import seed seed(0) I0 = [1000, 1500, 2000, 2500, 3000, 3500, 4000] # Create profile p = self.generate_7_profiles() Ical = [[],[],[],[],[],[],[]] for it in range(1): # Copy profile c = flex.double(flex.grid(40, 40, 40)) for i in range(p.all()[0]): pp = p[i:i+1,:,:,:] pp.reshape(c.accessor()) cc = add_poisson_noise(I0[i] * pp) c += cc b = flex.double(flex.grid(40, 40, 40), 1) m = flex.bool(flex.grid(40,40,40), True) c += add_poisson_noise(b) # Fit fit = ProfileFitter(c, b, m, p) I = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() for i in range(7): Ical[i].append(I[i]) for i in range(7): Ical[i] = sum(Ical[i]) / len(Ical[i]) Iknown = [1042.4904898451261, 1447.6413897568257, 2053.2524102387893, 2485.2789614429703, 3063.867598583333, 3445.9230910807582, 3977.6744651425543] Vknown = [65042.49048984377, 65447.6413897533, 66053.25241023625, 66485.27896144328, 67063.86759858252, 67445.92309107889, 67977.67446514373] # Test intensity is the same eps = 1e-7 for i in range(7): assert(abs(I[i] - Iknown[i]) < eps) assert(abs(V[i] - Vknown[i]) < eps) print 'OK'
def tst_deconvolve_3_with_flat_background(self): from dials.algorithms.integration.fit import ProfileFitter from scitbx.array_family import flex from numpy.random import seed seed(0) I0 = [1000, 2000, 3000] # Create profile p = self.generate_3_profiles() Ical = [[],[],[]] for it in range(1): # Copy profile c = flex.double(flex.grid(40, 9, 9)) for i in range(p.all()[0]): pp = p[i:i+1,:,:,:] pp.reshape(c.accessor()) cc = add_poisson_noise(I0[i] * pp) c += cc b = flex.double(flex.grid(40, 9, 9), 1) m = flex.bool(flex.grid(40,9,9), True) c += add_poisson_noise(b) # Fit fit = ProfileFitter(c, b, m, p) I = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() for i in range(3): Ical[i].append(I[i]) for i in range(3): Ical[i] = sum(Ical[i]) / len(Ical[i]) Iknown = [1030.7018033805357, 1948.7152700952695, 2972.983204218213] Vknown = [4270.701803380534, 5188.715270095279, 6212.983204218214] # Test intensity is the same eps = 1e-7 for i in range(3): assert(abs(I[i] - Iknown[i]) < eps) assert(abs(V[i] - Vknown[i]) < eps) print 'OK'
def tst_deconvolve_3_with_no_background(self): from dials.algorithms.integration.fit import ProfileFitter from scitbx.array_family import flex from numpy.random import seed seed(0) I0 = [1000, 2000, 3000] # Create profile p = self.generate_3_profiles() Ical = [[],[],[]] for it in range(1): # Copy profile c = flex.double(flex.grid(40, 9, 9)) for i in range(p.all()[0]): pp = p[i:i+1,:,:,:] pp.reshape(c.accessor()) cc = add_poisson_noise(I0[i] * pp) c += cc b = flex.double(flex.grid(40, 9, 9), 0) m = flex.bool(flex.grid(40,9,9), True) # Fit fit = ProfileFitter(c, b, m, p) I = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() for i in range(3): Ical[i].append(I[i]) for i in range(3): Ical[i] = sum(Ical[i]) / len(Ical[i]) Iknown = [1048.3221116842406, 1920.9035376774107, 2938.7743506383745] # Test intensity is the same eps = 1e-7 for i in range(3): assert(abs(I[i] - Iknown[i]) < eps) assert(abs(V[i] - Iknown[i]) < eps) print 'OK'
def test_deconvolve_3_with_flat_background(): np.random.seed(0) I0 = [1000, 2000, 3000] # Create profile p = generate_3_profiles() Ical = [[], [], []] for it in range(1): # Copy profile c = flex.double(flex.grid(40, 9, 9)) for i in range(p.all()[0]): pp = p[i:i + 1, :, :, :] pp.reshape(c.accessor()) cc = add_poisson_noise(I0[i] * pp) c += cc b = flex.double(flex.grid(40, 9, 9), 1) m = flex.bool(flex.grid(40, 9, 9), True) c += add_poisson_noise(b) # Fit fit = ProfileFitter(c, b, m, p) intensity = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() for i in range(3): Ical[i].append(intensity[i]) for i in range(3): Ical[i] = sum(Ical[i]) / len(Ical[i]) Iknown = [1030.7018033805357, 1948.7152700952695, 2972.983204218213] Vknown = [4270.701803380534, 5188.715270095279, 6212.983204218214] # Test intensity is the same eps = 1e-7 for i in range(3): assert intensity[i] == pytest.approx(Iknown[i], abs=eps) assert V[i] == pytest.approx(Vknown[i], abs=eps)
def test_deconvolve_3_with_no_background(): np.random.seed(0) I0 = [1000, 2000, 3000] # Create profile p = generate_3_profiles() Ical = [[], [], []] for it in range(1): # Copy profile c = flex.double(flex.grid(40, 9, 9)) for i in range(p.all()[0]): pp = p[i:i + 1, :, :, :] pp.reshape(c.accessor()) cc = add_poisson_noise(I0[i] * pp) c += cc b = flex.double(flex.grid(40, 9, 9), 0) m = flex.bool(flex.grid(40, 9, 9), True) # Fit fit = ProfileFitter(c, b, m, p) intensity = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() for i in range(3): Ical[i].append(intensity[i]) for i in range(3): Ical[i] = sum(Ical[i]) / len(Ical[i]) Iknown = [1048.3221116842406, 1920.9035376774107, 2938.7743506383745] # Test intensity is the same eps = 1e-7 for i in range(3): assert intensity[i] == pytest.approx(Iknown[i], abs=eps) assert V[i] == pytest.approx(Iknown[i], abs=eps)
def tst_with_flat_background_partial(self): from dials.algorithms.integration.fit import ProfileFitter from scitbx.array_family import flex from numpy.random import seed seed(0) # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c0 = add_poisson_noise(100 * p) b = flex.double(flex.grid(9, 9, 9), 1) m = flex.bool(flex.grid(9,9,9), True) c = c0 + add_poisson_noise(b) # Get the partial profiles pp = p[0:5,:,:] mp = m[0:5,:,:] cp = c[0:5,:,:] bp = b[0:5,:,:] c0p = c0[0:5,:,:] # Fit fit = ProfileFitter(cp, bp, mp, pp) I = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() Iknown = 99.06932141277105 Vknown = 504.06932141277105 # Test intensity is the same eps = 1e-7 assert(abs(I[0] - Iknown) < eps) assert(abs(V[0] - Vknown) < eps) print 'OK'
def test_mask(): np.random.seed(0) # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c = add_poisson_noise(100 * p) b = flex.double(flex.grid(9, 9, 9), 0) m = flex.bool(flex.grid(9, 9, 9), True) m[4, 4, 4] = False # Fit fit = ProfileFitter(c, b, m, p) intensity = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() # Test intensity is approximately the same assert intensity[0] == pytest.approx(flex.sum(c), rel=0.01) assert intensity[0] == pytest.approx(V[0], abs=1e-3)
def test_deconvolve_zero(): from dials.algorithms.integration.fit import ProfileFitter from scitbx.array_family import flex from numpy.random import seed seed(0) I0 = [1000, 2000, 3000] # Create profile p = generate_3_profiles() # Copy profile c = flex.double(flex.grid(40, 9, 9), 0) b = flex.double(flex.grid(40, 9, 9), 0) m = flex.bool(flex.grid(40,9,9), True) # Fit passed = False with pytest.raises(Exception): fit = ProfileFitter(c, b, m, p) I = fit.intensity() V = fit.variance()
def test_zero(): np.random.seed(0) # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c = flex.double(flex.grid(9, 9, 9), 0) b = flex.double(flex.grid(9, 9, 9), 0) m = flex.bool(flex.grid(9, 9, 9), True) # Fit fit = ProfileFitter(c, b, m, p) intensity = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() # Test intensity is the same eps = 1e-3 assert intensity[0] == pytest.approx(flex.sum(c), abs=eps) assert intensity[0] == pytest.approx(V[0], abs=eps)
def test_with_no_background_partial(): from dials.algorithms.integration.fit import ProfileFitter from scitbx.array_family import flex from numpy.random import seed seed(0) # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c = add_poisson_noise(100 * p) b = flex.double(flex.grid(9, 9, 9), 0) m = flex.bool(flex.grid(9, 9, 9), True) # Get the partial profiles pp = p[0:5, :, :] mp = m[0:5, :, :] cp = c[0:5, :, :] bp = b[0:5, :, :] # Fit fit = ProfileFitter(cp, bp, mp, pp) I = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() Iknown = 94.67151440319306 Vknown = 94.67151440319306 # Test intensity is the same eps = 1e-7 assert abs(I[0] - Iknown) < eps assert abs(V[0] - Vknown) < eps
def integrate_job(block, experiments, reflections, reference, grid_size=5, detector_space=False): from dials.algorithms.profile_model.modeller import CircleSampler from dials.array_family import flex from dials.algorithms.profile_model.gaussian_rs.transform import TransformReverse from dials.algorithms.profile_model.gaussian_rs.transform import TransformForward from dials.algorithms.profile_model.gaussian_rs.transform import ( TransformReverseNoModel, ) from dials.algorithms.profile_model.gaussian_rs.transform import TransformSpec from dials.algorithms.profile_model.gaussian_rs import CoordinateSystem from dials.algorithms.integration.fit import ProfileFitter from dials.array_family import flex from dials.model.data import make_image reflections["shoebox"] = flex.shoebox(reflections["panel"], reflections["bbox"], allocate=True) frame0, frame1 = experiments[0].scan.get_array_range() frame0 = frame0 + block[0] frame1 = frame0 + block[1] reflections["shoebox"] = flex.shoebox(reflections["panel"], reflections["bbox"], allocate=True) extractor = flex.ShoeboxExtractor(reflections, 1, frame0, frame1) iset = experiments[0].imageset[block[0]:block[1]] for i in range(len(iset)): print("Reading image %d" % i) data = iset.get_raw_data(i) mask = iset.get_mask(i) extractor.next(make_image(data, mask)) print("Computing mask") reflections.compute_mask(experiments) print("Computing background") reflections.compute_background(experiments) print("Computing centroid") reflections.compute_centroid(experiments) print("Computing summed intensity") reflections.compute_summed_intensity() assert len(reference) % 9 == 0 num_scan_points = len(reference) // 9 sampler = CircleSampler( experiments[0].detector[0].get_image_size(), experiments[0].scan.get_array_range(), num_scan_points, ) spec = TransformSpec( experiments[0].beam, experiments[0].detector, experiments[0].goniometer, experiments[0].scan, experiments[0].profile.sigma_b(deg=False), experiments[0].profile.sigma_m(deg=False), experiments[0].profile.n_sigma() * 1.5, grid_size, ) m2 = experiments[0].goniometer.get_rotation_axis() s0 = experiments[0].beam.get_s0() Iprf = flex.double(len(reflections)) Vprf = flex.double(len(reflections)) Cprf = flex.double(len(reflections)) Fprf = flex.bool(len(reflections)) Part = reflections["partiality"] reflections["intensity.prf_old.value"] = reflections["intensity.prf.value"] reflections["intensity.prf_old.variance"] = reflections[ "intensity.prf.variance"] selection = reflections.get_flags(reflections.flags.integrated_prf) reflections.unset_flags(~Fprf, reflections.flags.integrated_prf) for i, r in enumerate(reflections): if selection[i] == False: continue s1 = r["s1"] phi = r["xyzcal.mm"][2] xyz = r["xyzcal.px"] bbox = r["bbox"] panel = r["panel"] image = r["shoebox"].data.as_double() background = r["shoebox"].background.as_double() mask = r["shoebox"].mask.as_1d( ) == 5 # | (r['shoebox'].mask.as_1d() == 3) mask.reshape(image.accessor()) cs = CoordinateSystem(m2, s0, s1, phi) index = sampler.nearest(0, xyz) profile, profile_mask = reference[index] # print flex.sum(profile) # print r['partiality'] if detector_space: transform = TransformReverseNoModel(spec, cs, bbox, panel, profile) p = transform.profile() d = image m = mask b = background # print flex.sum(p) Part[i] = flex.sum(p) # ysize, xsize = p.all()[1:3] # p1 = flex.double(flex.grid(1, ysize , xsize)) # d1 = flex.double(flex.grid(1, ysize , xsize)) # b1 = flex.double(flex.grid(1, ysize , xsize)) # m1 = flex.double(flex.grid(1, ysize , xsize)) # for k in range(p.all()[0]): # p1 += p[k:k+1,:,:] # d1 += d[k:k+1,:,:] # b1 += b[k:k+1,:,:] # m1 = m[k:k+1,:,:] try: fit = ProfileFitter(d, b, m, p, 1e-3, 100) assert fit.niter() < 100 Iprf[i] = fit.intensity() Vprf[i] = fit.variance() Cprf[i] = fit.correlation() Fprf[i] = True # if r['intensity.sum.value'] > 10 and abs(fit.intensity()) < 1e-3: print( r["miller_index"], i, fit.intensity(), r["intensity.sum.value"], r["intensity.prf_old.value"], Part[i], fit.niter(), ) # from matplotlib import pylab # pylab.imshow(p1.as_numpy_array()[0,:,:], interpolation='none') # pylab.show() except Exception as e: print(e) pass else: try: transform = TransformForward(spec, cs, bbox, panel, image, background, mask) p = profile d = transform.profile() b = transform.background() m = transform.mask() & profile_mask # if r['miller_index'] == (9, -25, 74): # print list(p) # print list(m) # print list(b) # print list(d) # exit(0) fit = ProfileFitter(d, b, m, p, 1e-3, 100) assert fit.niter() < 100 Iprf[i] = fit.intensity()[0] Vprf[i] = fit.variance()[0] Cprf[i] = fit.correlation() Fprf[i] = True print(r["miller_index"], i, fit.intensity(), r["intensity.prf_old.value"]) # from matplotlib import pylab # pylab.imshow(p1.as_numpy_array()[0,:,:], interpolation='none') # pylab.show() except Exception as e: pass reflections["intensity.prf.value"] = Iprf reflections["intensity.prf.variance"] = Vprf reflections["intensity.prf.correlation"] = Cprf reflections.set_flags(Fprf, reflections.flags.integrated_prf) del reflections["shoebox"] return reflections