def test_does_bbox_contain_bad_pixels():
from scitbx.array_family import flex
mask = flex.bool(flex.grid(100, 100), True)
for j in range(100):
for i in range(40, 60):
mask[j, i] = False
mask[i, j] = False
for i in range(1000):
x0 = random.randint(0, 90)
y0 = random.randint(0, 90)
z0 = random.randint(0, 90)
x1 = random.randint(1, 10) + x0
y1 = random.randint(1, 10) + y0
z1 = random.randint(1, 10) + z0
shoebox = Shoebox((x0, x1, y0, y1, z0, z1))
res1 = shoebox.does_bbox_contain_bad_pixels(mask)
res2 = False
if x0 >= 40 and x0 < 60:
res2 = True
if x1 > 40 and x1 <= 60:
res2 = True
if y0 >= 40 and y0 < 60:
res2 = True
if y1 > 40 and y1 <= 60:
res2 = True
assert res1 == res2
def tst_count_mask_values(self):
from dials.model.data import Shoebox
from random import randint, sample
from dials.array_family import flex
shoebox = flex.shoebox(10)
num = flex.int(10)
value = (1 << 2)
for i in range(10):
x0 = randint(0, 90)
y0 = randint(0, 90)
z0 = randint(0, 90)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))
shoebox[i].allocate()
maxnum = len(shoebox[i].mask)
num[i] = randint(1, maxnum)
indices = sample(list(range(maxnum)), num[i])
for j in indices:
shoebox[i].mask[j] = value
assert(shoebox.count_mask_values(value) == num)
# Test passed
print 'OK'
def tst_count_mask_values(self):
from dials.model.data import Shoebox
from random import randint, sample
from dials.array_family import flex
shoebox = flex.shoebox(10)
num = flex.int(10)
value = (1 << 2)
for i in range(10):
x0 = randint(0, 90)
y0 = randint(0, 90)
z0 = randint(0, 90)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))
shoebox[i].allocate()
maxnum = len(shoebox[i].mask)
num[i] = randint(1, maxnum)
indices = sample(list(range(maxnum)), num[i])
for j in indices:
shoebox[i].mask[j] = value
assert (shoebox.count_mask_values(value) == num)
# Test passed
print 'OK'
def test_consistent():
from dials.array_family import flex
from dials.model.data import Shoebox
shoebox = flex.shoebox(10)
for i in range(10):
x0 = random.randint(0, 1000)
y0 = random.randint(0, 1000)
z0 = random.randint(0, 1000)
x1 = random.randint(1, 10) + x0
y1 = random.randint(1, 10) + y0
z1 = random.randint(1, 10) + z0
shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))
assert shoebox.is_consistent() == flex.bool(10, False)
for i in range(10):
shoebox[i].allocate()
assert shoebox.is_consistent() == flex.bool(10, True)
for i in [0, 2, 4, 6, 8]:
shoebox[i].data.resize(flex.grid(10, 10, 10))
assert shoebox.is_consistent() == flex.bool([False, True] * 5)
for i in range(10):
shoebox[i].deallocate()
assert shoebox.is_consistent() == flex.bool(10, False)
def test_does_bbox_contain_bad_pixels():
from dials.array_family import flex
from dials.model.data import Shoebox
mask = flex.bool(flex.grid(100, 100), True)
for j in range(100):
for i in range(40, 60):
mask[j, i] = False
mask[i, j] = False
shoebox = flex.shoebox(1000)
res = flex.bool(1000)
for i in range(1000):
x0 = random.randint(0, 90)
y0 = random.randint(0, 90)
z0 = random.randint(0, 90)
x1 = random.randint(1, 10) + x0
y1 = random.randint(1, 10) + y0
z1 = random.randint(1, 10) + z0
shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))
res2 = False
if x0 >= 40 and x0 < 60:
res2 = True
if x1 > 40 and x1 <= 60:
res2 = True
if y0 >= 40 and y0 < 60:
res2 = True
if y1 > 40 and y1 <= 60:
res2 = True
res[i] = res2
assert shoebox.does_bbox_contain_bad_pixels(mask) == res
def tst_is_bbox_within_image_volume(self):
from dials.model.data import Shoebox
from dials.array_family import flex
isize = (1000, 1000)
srange = (0, 100)
shoebox = flex.shoebox(7)
shoebox[0] = Shoebox((10, 20, 10, 20, 10, 20))
shoebox[1] = Shoebox((-10, 20, 10, 20, 10, 20))
shoebox[2] = Shoebox((10, 20, -10, 20, 10, 20))
shoebox[3] = Shoebox((10, 20, 10, 20, -10, 20))
shoebox[4] = Shoebox((10, 1020, 10, 20, 10, 20))
shoebox[5] = Shoebox((10, 20, 10, 1020, 10, 20))
shoebox[6] = Shoebox((10, 20, 10, 20, 10, 1020))
assert (shoebox.is_bbox_within_image_volume(isize,
srange) == flex.bool([
True, False, False,
False, False, False,
False
]))
# Test passed
print 'OK'
def gen_shoebox():
shoebox = Shoebox(0, (0, 4, 0, 3, 0, 1))
shoebox.allocate()
for k in range(1):
for j in range(3):
for i in range(4):
shoebox.data[k, j, i] = i + j + k + 0.1
shoebox.mask[k, j, i] = i % 2
shoebox.background[k, j, i] = i * j + 0.2
return shoebox
def test_bounding_boxes():
from dials.array_family import flex
from dials.model.data import Shoebox
shoebox = flex.shoebox(10)
bbox = flex.int6(10)
for i in range(10):
x0 = random.randint(0, 90)
y0 = random.randint(0, 90)
z0 = random.randint(0, 90)
x1 = random.randint(1, 10) + x0
y1 = random.randint(1, 10) + y0
z1 = random.randint(1, 10) + z0
bbox[i] = (x0, x1, y0, y1, z0, z1)
shoebox[i] = Shoebox(bbox[i])
bbox2 = shoebox.bounding_boxes()
for i in range(10):
assert bbox2[i] == bbox[i]
def test_offset():
for i in range(10):
x0 = random.randint(0, 1000)
y0 = random.randint(0, 1000)
z0 = random.randint(0, 1000)
x1 = random.randint(1, 10) + x0
y1 = random.randint(1, 10) + y0
z1 = random.randint(1, 10) + z0
shoebox = Shoebox((x0, x1, y0, y1, z0, z1))
assert shoebox.xoffset() == x0
assert shoebox.yoffset() == y0
assert shoebox.zoffset() == z0
assert shoebox.offset() == (z0, y0, x0)
def test_size():
for i in range(10):
x0 = random.randint(0, 1000)
y0 = random.randint(0, 1000)
z0 = random.randint(0, 1000)
x1 = random.randint(1, 10) + x0
y1 = random.randint(1, 10) + y0
z1 = random.randint(1, 10) + z0
shoebox = Shoebox((x0, x1, y0, y1, z0, z1))
assert shoebox.xsize() == x1 - x0
assert shoebox.ysize() == y1 - y0
assert shoebox.zsize() == z1 - z0
assert shoebox.size() == (z1 - z0, y1 - y0, x1 - x0)
def test_allocate():
for i in range(10):
x0 = random.randint(0, 1000)
y0 = random.randint(0, 1000)
z0 = random.randint(0, 1000)
x1 = random.randint(1, 10) + x0
y1 = random.randint(1, 10) + y0
z1 = random.randint(1, 10) + z0
shoebox = Shoebox((x0, x1, y0, y1, z0, z1))
shoebox.allocate()
assert shoebox.data.all() == (z1 - z0, y1 - y0, x1 - x0)
assert shoebox.mask.all() == (z1 - z0, y1 - y0, x1 - x0)
shoebox.deallocate()
assert shoebox.data.all() == (0, 0, 0)
assert shoebox.mask.all() == (0, 0, 0)
def test_count_mask_values():
for i in range(10):
x0 = random.randint(0, 90)
y0 = random.randint(0, 90)
z0 = random.randint(0, 90)
x1 = random.randint(1, 10) + x0
y1 = random.randint(1, 10) + y0
z1 = random.randint(1, 10) + z0
shoebox = Shoebox((x0, x1, y0, y1, z0, z1))
shoebox.allocate()
maxnum = len(shoebox.mask)
num = random.randint(1, maxnum)
indices = random.sample(list(range(maxnum)), num)
value = 1 << 2
for i in indices:
shoebox.mask[i] = value
assert shoebox.count_mask_values(value) == num
def tst_size(self):
from random import randint
from dials.model.data import Shoebox
for i in range(10):
x0 = randint(0, 1000)
y0 = randint(0, 1000)
z0 = randint(0, 1000)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
shoebox = Shoebox((x0, x1, y0, y1, z0, z1))
assert (shoebox.xsize() == x1 - x0)
assert (shoebox.ysize() == y1 - y0)
assert (shoebox.zsize() == z1 - z0)
assert (shoebox.size() == (z1 - z0, y1 - y0, x1 - x0))
# Test passed
print 'OK'
def tst_offset(self):
from random import randint
from dials.model.data import Shoebox
for i in range(10):
x0 = randint(0, 1000)
y0 = randint(0, 1000)
z0 = randint(0, 1000)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
shoebox = Shoebox((x0, x1, y0, y1, z0, z1))
assert (shoebox.xoffset() == x0)
assert (shoebox.yoffset() == y0)
assert (shoebox.zoffset() == z0)
assert (shoebox.offset() == (z0, y0, x0))
# Test passed
print 'OK'
def tst_allocate(self):
from random import randint
from dials.model.data import Shoebox
for i in range(10):
x0 = randint(0, 1000)
y0 = randint(0, 1000)
z0 = randint(0, 1000)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
shoebox = Shoebox((x0, x1, y0, y1, z0, z1))
shoebox.allocate()
assert (shoebox.data.all() == (z1 - z0, y1 - y0, x1 - x0))
assert (shoebox.mask.all() == (z1 - z0, y1 - y0, x1 - x0))
shoebox.deallocate()
assert (shoebox.data.all() == (0, 0, 0))
assert (shoebox.mask.all() == (0, 0, 0))
# Test passed
print 'OK'
def tst_split_partials_with_shoebox(self):
from dials.array_family import flex
from random import randint, uniform
from dials.model.data import Shoebox
r = flex.reflection_table()
r['value1'] = flex.double()
r['value2'] = flex.int()
r['value3'] = flex.double()
r['bbox'] = flex.int6()
r['panel'] = flex.size_t()
r['shoebox'] = flex.shoebox()
expected = []
for i in range(100):
x0 = randint(0, 100)
x1 = x0 + randint(1, 10)
y0 = randint(0, 100)
y1 = y0 + randint(1, 10)
z0 = randint(0, 100)
z1 = z0 + randint(1, 10)
v1 = uniform(0, 100)
v2 = randint(0, 100)
v3 = uniform(0, 100)
sbox = Shoebox(0, (x0, x1, y0, y1, z0, z1))
sbox.allocate()
assert(sbox.is_consistent())
w = x1 - x0
h = y1 - y0
for z in range(z0, z1):
for y in range(y0, y1):
for x in range(x0, x1):
sbox.data[z-z0,y-y0,x-x0] = x+y*w+z*w*h
r.append({
'value1' : v1,
'value2' : v2,
'value3' : v3,
'bbox' : (x0, x1, y0, y1, z0, z1),
'panel' : 0,
'shoebox' : sbox
})
for z in range(z0, z1):
sbox = Shoebox(0, (x0, x1, y0, y1, z, z+1))
sbox.allocate()
assert(sbox.is_consistent())
w = x1 - x0
h = y1 - y0
for y in range(y0, y1):
for x in range(x0, x1):
sbox.data[0,y-y0,x-x0] = x+y*w+z*w*h
expected.append({
'value1' : v1,
'value2' : v2,
'value3' : v3,
'bbox' : (x0, x1, y0, y1, z, z+1),
'partial_id' : i,
'panel' : 0,
'shoebox' : sbox
})
r.split_partials_with_shoebox()
assert(len(r) == len(expected))
EPS = 1e-7
for r1, r2 in zip(r, expected):
assert(abs(r1['value1'] - r2['value1']) < EPS)
assert(r1['value2'] == r2['value2'])
assert(abs(r1['value3'] - r2['value3']) < EPS)
assert(r1['bbox'] == r2['bbox'])
assert(r1['partial_id'] == r2['partial_id'])
assert(r1['panel'] == r2['panel'])
assert(r1['shoebox'].data.as_double().as_1d().all_approx_equal(
r2['shoebox'].data.as_double().as_1d()))
print 'OK'
def tst_consistent(self):
from random import randint
from dials.model.data import Shoebox
from dials.array_family import flex
for i in range(1000):
x0 = randint(0, 1000)
y0 = randint(0, 1000)
z0 = randint(0, 1000)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
try:
shoebox = Shoebox((x0, x1, y0, y1, z0, z1))
assert (not shoebox.is_consistent())
shoebox.allocate()
assert (shoebox.is_consistent())
shoebox.data = flex.real(flex.grid(20, 20, 20))
assert (not shoebox.is_consistent())
shoebox.deallocate()
assert (not shoebox.is_consistent())
except Exception, e:
print x0, y0, z0, x1, y1, z1
raise
def generate_shoebox(self, bbox, centre, intensity, mask=False):
from dials.model.data import Shoebox
from dials.algorithms.shoebox import MaskCode
shoebox = Shoebox()
shoebox.bbox = bbox
shoebox.allocate()
for i in range(len(shoebox.mask)):
shoebox.mask[i] = MaskCode.Valid | MaskCode.Foreground
shoebox.data = self.gaussian(
shoebox.size(), 1.0,
[c - o for c, o in zip(centre[::-1], shoebox.offset())],
[s / 8.0 for s in shoebox.size()])
if mask:
shoebox.mask = self.create_mask(
shoebox.size(),
[c - o for c, o in zip(centre[::-1], shoebox.offset())],
MaskCode.Valid | MaskCode.Foreground)
tot = 0
mask_code = MaskCode.Valid | MaskCode.Foreground
for i in range(len(shoebox.data)):
if shoebox.mask[i] & mask_code == mask_code:
tot += shoebox.data[i]
if tot > 0:
shoebox.data *= intensity / tot
return shoebox
def tst_is_bbox_within_image_volume(self):
from dials.model.data import Shoebox
isize = (1000, 1000)
srange = (0, 100)
shoebox = Shoebox((10, 20, 10, 20, 10, 20))
assert (shoebox.is_bbox_within_image_volume(isize, srange))
shoebox = Shoebox((-10, 20, 10, 20, 10, 20))
assert (not shoebox.is_bbox_within_image_volume(isize, srange))
shoebox = Shoebox((10, 20, -10, 20, 10, 20))
assert (not shoebox.is_bbox_within_image_volume(isize, srange))
shoebox = Shoebox((10, 20, 10, 20, -10, 20))
assert (not shoebox.is_bbox_within_image_volume(isize, srange))
shoebox = Shoebox((10, 1020, 10, 20, 10, 20))
assert (not shoebox.is_bbox_within_image_volume(isize, srange))
shoebox = Shoebox((10, 20, 10, 1020, 10, 20))
assert (not shoebox.is_bbox_within_image_volume(isize, srange))
shoebox = Shoebox((10, 20, 10, 20, 10, 1020))
assert (not shoebox.is_bbox_within_image_volume(isize, srange))
# Test passed
print 'OK'
def tilt_fit(imgs, is_bg_pix, delta_q, photon_gain, sigma_rdout, zinger_zscore,
exper, predicted_refls, sb_pad=0, filter_boundary_spots=False,
minsnr=None, mintilt=None, plot=False, verbose=False, is_BAD_pix=None,
min_strong=None, min_bg=10, min_dist_to_bad_pix=7, **kwargs):
if is_BAD_pix is None:
is_BAD_pix = np.zeros(np.array(is_bg_pix).shape, np.bool)
predicted_refls['id'] = flex.int(len(predicted_refls), -1)
predicted_refls['imageset_id'] = flex.int(len(predicted_refls), 0)
El = ExperimentList()
El.append(exper)
predicted_refls.centroid_px_to_mm(El)
predicted_refls.map_centroids_to_reciprocal_space(El)
ss_dim, fs_dim = imgs[0].shape
n_refl = len(predicted_refls)
integrations = []
variances = []
coeffs = []
new_shoeboxes = []
tilt_error = []
boundary = []
detdist = exper.detector[0].get_distance()
pixsize = exper.detector[0].get_pixel_size()[0]
ave_wave = exper.beam.get_wavelength()
bad_trees = {}
unique_panels = set(predicted_refls["panel"])
for p in unique_panels:
panel_bad_pix = is_BAD_pix[p]
ybad, xbad = np.where(is_BAD_pix[0])
if ybad.size:
bad_pts = zip(ybad, xbad)
bad_trees[p] = cKDTree(bad_pts)
else:
bad_trees[p] = None
sel = []
for i_ref in range(len(predicted_refls)):
ref = predicted_refls[i_ref]
i_com, j_com, _ = ref['xyzobs.px.value']
# which detector panel am I on ?
i_panel = ref['panel']
if bad_trees[i_panel] is not None:
if bad_trees[i_panel].query_ball_point((i_com, j_com), r=min_dist_to_bad_pix):
sel.append(False)
integrations.append(None)
variances.append(None)
coeffs.append(None)
new_shoeboxes.append(None)
tilt_error.append(None)
boundary.append(None)
continue
i1_a, i2_a, j1_a, j2_a, _, _ = ref['bbox'] # bbox of prediction
i1_ = max(i1_a, 0)
i2_ = min(i2_a, fs_dim-1)
j1_ = max(j1_a, 0)
j2_ = min(j2_a, ss_dim-1)
# get the number of pixels spanning the box in pixels
Qmag = 2*np.pi*np.linalg.norm(ref['rlp']) # magnitude momentum transfer of the RLP in physicist convention
rad1 = (detdist/pixsize) * np.tan(2*np.arcsin((Qmag-delta_q*.5)*ave_wave/4/np.pi))
rad2 = (detdist/pixsize) * np.tan(2*np.arcsin((Qmag+delta_q*.5)*ave_wave/4/np.pi))
bbox_extent = (rad2-rad1) / np.sqrt(2) # rad2 - rad1 is the diagonal across the bbox
i_com = i_com - 0.5
j_com = j_com - 0.5
i_low = int(i_com - bbox_extent/2.)
i_high = int(i_com + bbox_extent/2.)
j_low = int(j_com - bbox_extent/2.)
j_high = int(j_com + bbox_extent/2.)
i1_orig = max(i_low, 0)
i2_orig = min(i_high, fs_dim-1)
j1_orig = max(j_low, 0)
j2_orig = min(j_high, ss_dim-1)
i_low = i_low - sb_pad
i_high = i_high + sb_pad
j_low = j_low - sb_pad
j_high = j_high + sb_pad
i1 = max(i_low, 0)
i2 = min(i_high, fs_dim-1)
j1 = max(j_low, 0)
j2 = min(j_high, ss_dim-1)
i1_p = i1_orig - i1
i2_p = i1_p + i2_orig-i1_orig
j1_p = j1_orig - j1
j2_p = j1_p + j2_orig-j1_orig
if i1 == 0 or i2 == fs_dim or j1 == 0 or j2 == ss_dim:
boundary.append(True)
if filter_boundary_spots:
sel.append(False)
integrations.append(None)
variances.append(None)
coeffs.append(None)
new_shoeboxes.append(None)
tilt_error.append(None)
continue
else:
boundary.append(False)
# get the iamge and mask
shoebox_img = imgs[i_panel][j1:j2, i1:i2] / photon_gain # NOTE: gain is imortant here!
dials_mask = np.zeros(shoebox_img.shape).astype(np.int32)
# initially all pixels are valid
dials_mask += MaskCode.Valid
shoebox_mask = is_bg_pix[i_panel][j1:j2, i1:i2]
badpix_mask = is_BAD_pix[i_panel][j1:j2, i1:i2]
dials_mask[shoebox_mask] = dials_mask[shoebox_mask] + MaskCode.Background
new_shoebox = Shoebox((i1_orig, i2_orig, j1_orig, j2_orig, 0, 1))
new_shoebox.allocate()
new_shoebox.data = flex.float(np.ascontiguousarray(shoebox_img[None, j1_p:j2_p, i1_p: i2_p]))
#new_shoebox.data = flex.float(shoebox_img[None,])
# get coordinates arrays of the image
Y, X = np.indices(shoebox_img.shape)
# determine if any more outliers are present in background pixels
img1d = shoebox_img.ravel()
mask1d = shoebox_mask.ravel() # mask specifies which pixels are bg
# out1d specifies which bg pixels are outliers (zingers)
out1d = np.zeros(mask1d.shape, bool)
out1d[mask1d] = is_outlier(img1d[mask1d].ravel(), zinger_zscore)
out2d = out1d.reshape(shoebox_img.shape)
# combine bad2d with badpix mask
out2d = np.logical_or(out2d, badpix_mask)
# these are points we fit to: both zingers and original mask
fit_sel = np.logical_and(~out2d, shoebox_mask) # fit plane to these points, no outliers, no masked
if np.sum(fit_sel) < min_bg:
integrations.append(None)
variances.append(None)
coeffs.append(None)
new_shoeboxes.append(None)
tilt_error.append(None)
sel.append(False)
continue
# update the dials mask...
dials_mask[fit_sel] = dials_mask[fit_sel] + MaskCode.BackgroundUsed
# fast scan pixels, slow scan pixels, pixel values (corrected for gain)
fast, slow, rho_bg = X[fit_sel], Y[fit_sel], shoebox_img[fit_sel]
# do the fit of the background plane
A = np.array([fast, slow, np.ones_like(fast)]).T
# weights matrix:
W = np.diag(1 / (sigma_rdout ** 2 + rho_bg))
AWA = np.dot(A.T, np.dot(W, A))
try:
AWA_inv = np.linalg.inv(AWA)
except np.linalg.LinAlgError:
print ("WARNING: Fit did not work.. investigate reflection")
print (ref)
integrations.append(None)
variances.append(None)
coeffs.append(None)
new_shoeboxes.append(None)
tilt_error.append(None)
sel.append(False)
continue
AtW = np.dot(A.T, W)
a, b, c = np.dot(np.dot(AWA_inv, AtW), rho_bg)
coeffs.append((a, b, c))
# fit of the tilt plane background
X1d = np.ravel(X)
Y1d = np.ravel(Y)
background = (X1d * a + Y1d * b + c).reshape(shoebox_img.shape)
new_shoebox.background = flex.float(np.ascontiguousarray(background[None, j1_p: j2_p, i1_p:i2_p]))
# vector of residuals
r = rho_bg - np.dot(A, (a, b, c))
Nbg = len(rho_bg)
Nparam = 3
r_fact = np.dot(r.T, np.dot(W, r)) / (Nbg - Nparam)
var_covar = AWA_inv * r_fact
abc_var = var_covar[0][0], var_covar[1][1], var_covar[2][2]
# place the strong spot mask in the expanded shoebox
peak_mask = ref['shoebox'].mask.as_numpy_array()[0] == MaskCode.Valid + MaskCode.Foreground
peak_mask_valid = peak_mask[j1_-j1_a:- j1_a + j2_, i1_-i1_a:-i1_a + i2_]
peak_mask_expanded = np.zeros_like(shoebox_mask)
# overlap region
i1_o = max(i1_, i1)
i2_o = min(i2_, i2)
j1_o = max(j1_, j1)
j2_o = min(j2_, j2)
pk_mask_istart = i1_o - i1_
pk_mask_jstart = j1_o - j1_
pk_mask_istop = peak_mask_valid.shape[1] - (i2_ - i2_o)
pk_mask_jstop = peak_mask_valid.shape[0] - (j2_ - j2_o)
peak_mask_overlap = peak_mask_valid[pk_mask_jstart: pk_mask_jstop, pk_mask_istart: pk_mask_istop]
pk_mask_exp_i1 = i1_o - i1
pk_mask_exp_j1 = j1_o - j1
pk_mask_exp_i2 = peak_mask_expanded.shape[1] - (i2 - i2_o)
pk_mask_exp_j2 = peak_mask_expanded.shape[0] - (j2 - j2_o)
peak_mask_expanded[pk_mask_exp_j1: pk_mask_exp_j2, pk_mask_exp_i1: pk_mask_exp_i2] = peak_mask_overlap
# update the dials mask
dials_mask[peak_mask_expanded] = dials_mask[peak_mask_expanded] + MaskCode.Foreground
p = X[peak_mask_expanded] # fast scan coords
q = Y[peak_mask_expanded] # slow scan coords
rho_peak = shoebox_img[peak_mask_expanded] # pixel values
Isum = np.sum(rho_peak - a*p - b*q - c) # summed spot intensity
var_rho_peak = sigma_rdout ** 2 + rho_peak # include readout noise in the variance
Ns = len(rho_peak) # number of integrated peak pixels
# variance propagated from tilt plane constants
var_a_term = abc_var[0] * ((np.sum(p))**2)
var_b_term = abc_var[1] * ((np.sum(q))**2)
var_c_term = abc_var[2] * (Ns**2)
tilt_error.append(var_a_term + var_b_term + var_c_term)
# total variance of the spot
var_Isum = np.sum(var_rho_peak) + var_a_term + var_b_term + var_c_term
integrations.append(Isum)
variances.append(var_Isum)
new_shoebox.mask = flex.int(np.ascontiguousarray(dials_mask[None, j1_p:j2_p, i1_p:i2_p]))
new_shoeboxes.append(new_shoebox)
sel.append(True)
if i_ref % 50 == 0 and verbose:
print("Integrated refls %d / %d" % (i_ref+1, n_refl))
#if filter_boundary_spots:
# sel = flex.bool([I is not None for I in integrations])
boundary = np.array(boundary)[sel].astype(bool)
integrations = np.array([I for I in integrations if I is not None])
variances = np.array([v for v in variances if v is not None])
coeffs = np.array([c for c in coeffs if c is not None])
tilt_error = np.array([te for te in tilt_error if te is not None])
#boundary = np.zeros(tilt_error.shape).astype(np.bool)
predicted_refls = predicted_refls.select(flex.bool(sel))
predicted_refls['resolution'] = flex.double( 1/ np.linalg.norm(predicted_refls['rlp'], axis=1))
predicted_refls['boundary'] = flex.bool(boundary)
predicted_refls["intensity.sum.value.Leslie99"] = flex.double(integrations)
predicted_refls["intensity.sum.variance.Leslie99"] = flex.double(variances)
predicted_refls['shoebox'] = flex.shoebox([sb for sb in new_shoeboxes if sb is not None])
idx_assign = assign_indices.AssignIndicesGlobal(tolerance=0.333)
idx_assign(predicted_refls, El)
return predicted_refls, coeffs, tilt_error, integrations, variances
all_residual.append(np.mean(residual))
ref["snr"] = np.nan_to_num(Isum / varIsum)
ref["tilt_error"] = np.diag(variance_matrix).sum()
ref["shoebox_roi"] = shoebox_roi
panels.append(ref["panel"])
xyzobs.append(list(ref["xyzobs.px.value"]))
snr = np.array(integrations) / np.sqrt(variances)
data = {"panel": panels, "shoebox_roi": bboxes, "integration": integrations, "variance": variances,
"tilt_errors": tilt_error, "xyzobs.px.value": xyzobs, "snr": snr, "dips_below_zero": dips_below_zero}
df = pandas.DataFrame(data)
shoeboxes = []
for i1,i2,j1,j2 in bboxes:
new_shoebox = Shoebox((i1,i2, j1, j2, 0, 1))
new_shoebox.allocate()
shoeboxes.append(new_shoebox)
embed()
if not args.plotassembled:
#from cxid9114.prediction import prediction_utils
import pylab as plt
#refls_predict_bypanel = prediction_utils.refls_by_panelname(Rnew)
pause = args.pause
plt.figure(1)
ax = plt.gca()
refls_bypanel = df.groupby("panel")
for panel_id in df.panel.unique():
if args.pause == -1:
def test_consistent():
from dials.array_family import flex
for i in range(1000):
x0 = random.randint(0, 1000)
y0 = random.randint(0, 1000)
z0 = random.randint(0, 1000)
x1 = random.randint(1, 10) + x0
y1 = random.randint(1, 10) + y0
z1 = random.randint(1, 10) + z0
try:
shoebox = Shoebox((x0, x1, y0, y1, z0, z1))
assert not shoebox.is_consistent()
shoebox.allocate()
assert shoebox.is_consistent()
shoebox.data = flex.real(flex.grid(20, 20, 20))
assert not shoebox.is_consistent()
shoebox.deallocate()
assert not shoebox.is_consistent()
except Exception:
print(x0, y0, z0, x1, y1, z1)
raise
def test_is_bbox_within_image_volume():
isize = (1000, 1000)
srange = (0, 100)
shoebox = Shoebox((10, 20, 10, 20, 10, 20))
assert shoebox.is_bbox_within_image_volume(isize, srange)
shoebox = Shoebox((-10, 20, 10, 20, 10, 20))
assert not shoebox.is_bbox_within_image_volume(isize, srange)
shoebox = Shoebox((10, 20, -10, 20, 10, 20))
assert not shoebox.is_bbox_within_image_volume(isize, srange)
shoebox = Shoebox((10, 20, 10, 20, -10, 20))
assert not shoebox.is_bbox_within_image_volume(isize, srange)
shoebox = Shoebox((10, 1020, 10, 20, 10, 20))
assert not shoebox.is_bbox_within_image_volume(isize, srange)
shoebox = Shoebox((10, 20, 10, 1020, 10, 20))
assert not shoebox.is_bbox_within_image_volume(isize, srange)
shoebox = Shoebox((10, 20, 10, 20, 10, 1020))
assert not shoebox.is_bbox_within_image_volume(isize, srange)
def test_overlap_counter():
from dials.model.data import Shoebox
from dials.array_family import flex
from dxtbx.model.experiment_list import ExperimentListFactory, ExperimentListDumper
# Generate input experiments file
experiments = ExperimentListFactory.from_json_file(
"/dls/mx-scratch/gwx73773/data_files/3SNK/DEFAULT/NATIVE/SWEEP1/integrate/13_integrated_experiments.json"
)
experiments[0].imageset = experiments[0].imageset[0:1]
experiments[0].scan = experiments[0].scan[0:1]
input_file = "one_image_integrated_experiments.json"
experiments.as_json("one_image_integrated_experiments.json")
# Generate 1 example shoebox file
reflections = flex.reflection_table(8)
reflections['bbox'] = flex.int6(8)
reflections['bbox'][0] = (1, 6, 1, 6, 0, 1)
reflections['bbox'][1] = (1, 6, 2, 8, 0, 1)
reflections['bbox'][2] = (5, 10, 4, 9, 0, 1)
reflections['bbox'][3] = (8, 13, 1, 7, 0, 1)
reflections['bbox'][4] = (8, 13, 5, 10, 0, 1)
reflections['bbox'][5] = (1, 5, 12, 16, 0, 1)
reflections['bbox'][6] = (4, 8, 12, 16, 0, 1)
reflections['bbox'][7] = (7, 12, 12, 16, 0, 1)
reflections['shoebox'] = flex.shoebox(8)
for i in range(8):
reflections['shoebox'][i] = Shoebox(reflections['bbox'][i])
reflections['shoebox'][0].mask = flex.int(
np.array([[[3, 3, 3, 3, 3], [3, 5, 5, 5, 3], [3, 5, 5, 5, 3],
[3, 5, 5, 5, 3], [3, 3, 3, 3, 3]]],
dtype=np.int32))
reflections['shoebox'][1].mask = flex.int(
np.array([[[3, 3, 3, 3, 3], [3, 5, 5, 5, 3], [3, 5, 5, 5, 3],
[3, 5, 5, 5, 3], [3, 3, 3, 3, 3], [3, 3, 3, 3, 3]]],
dtype=np.int32))
reflections['shoebox'][2].mask = flex.int(
np.array([[[3, 3, 3, 3, 3], [3, 5, 5, 5, 3], [3, 5, 5, 5, 3],
[3, 5, 5, 5, 3], [3, 3, 3, 3, 3]]],
dtype=np.int32))
reflections['shoebox'][3].mask = flex.int(
np.array([[[3, 3, 3, 3, 3], [3, 3, 3, 3, 3], [3, 5, 5, 5, 3],
[3, 5, 5, 5, 3], [3, 5, 5, 5, 3], [3, 3, 3, 3, 3]]],
dtype=np.int32))
reflections['shoebox'][4].mask = flex.int(
np.array([[[3, 3, 5, 5, 3], [3, 3, 5, 5, 3], [3, 3, 5, 5, 3],
[3, 3, 3, 3, 3], [3, 3, 3, 3, 3]]],
dtype=np.int32))
reflections['shoebox'][5].mask = flex.int(
np.array([[[3, 3, 3, 3], [3, 5, 5, 3], [3, 5, 5, 3], [3, 3, 3, 3]]],
dtype=np.int32))
reflections['shoebox'][6].mask = flex.int(
np.array([[[3, 3, 3, 3], [3, 5, 5, 3], [3, 5, 5, 3], [3, 3, 3, 3]]],
dtype=np.int32))
reflections['shoebox'][7].mask = flex.int(
np.array([[[3, 3, 3, 3, 3], [3, 5, 5, 3, 3], [3, 5, 5, 3, 3],
[3, 3, 3, 3, 3]]],
dtype=np.int32))
reflections['id'] = flex.int(8)
reflections['d'] = flex.double(8)
reflections['panel'] = flex.size_t(8)
reflections['imageset_id'] = flex.int(8)
for i in range(8):
reflections['d'][i] = random.random()
reflections['id'][i] = 0
reflections['imageset_id'][i] = 0
reflections['panel'][i] = 0
reflections.as_file("shoeboxes_0.pickle")
# Run counter
counter = OverlapCounter(input_file, 4, "test_overlap_lists.txt",
"test_overlap_total.txt", True)
total, fg, bg, bg_fg = counter.main()
# Test output
assert total == 3.375
assert bg == 2
assert fg == 0.5
assert bg_fg == 0.875
def sequence_to_stills(experiments, reflections, params):
assert len(reflections) == 1
reflections = reflections[0]
new_experiments = ExperimentList()
new_reflections = flex.reflection_table()
# This is the subset needed to integrate
for key in [
"id",
"imageset_id",
"shoebox",
"bbox",
"intensity.sum.value",
"intensity.sum.variance",
"entering",
"flags",
"miller_index",
"panel",
"xyzobs.px.value",
"xyzobs.px.variance",
]:
if key in reflections:
new_reflections[key] = type(reflections[key])()
elif key == "imageset_id":
assert len(experiments.imagesets()) == 1
reflections["imageset_id"] = flex.int(len(reflections), 0)
new_reflections["imageset_id"] = flex.int()
elif key == "entering":
reflections["entering"] = flex.bool(len(reflections), False)
new_reflections["entering"] = flex.bool()
else:
raise RuntimeError(
"Expected key not found in reflection table: %s" % key)
for expt_id, experiment in enumerate(experiments):
# Get the goniometr setting matrix
goniometer_setting_matrix = matrix.sqr(
experiment.goniometer.get_setting_rotation())
goniometer_axis = matrix.col(experiment.goniometer.get_rotation_axis())
step = experiment.scan.get_oscillation()[1]
refls = reflections.select(reflections["id"] == expt_id)
_, _, _, _, z1, z2 = refls["bbox"].parts()
# Create an experiment for each scanpoint
for i_scan_point in range(*experiment.scan.get_array_range()):
if params.max_scan_points and i_scan_point >= params.max_scan_points:
break
# The A matrix is the goniometer setting matrix for this scan point
# times the scan varying A matrix at this scan point. Note, the
# goniometer setting matrix for scan point zero will be the identity
# matrix and represents the beginning of the oscillation.
# For stills, the A matrix needs to be positioned in the midpoint of an
# oscillation step. Hence, here the goniometer setting matrixis rotated
# by a further half oscillation step.
A = (goniometer_axis.axis_and_angle_as_r3_rotation_matrix(
angle=experiment.scan.get_angle_from_array_index(i_scan_point)
+ (step / 2),
deg=True,
) * goniometer_setting_matrix * matrix.sqr(
experiment.crystal.get_A_at_scan_point(i_scan_point)))
crystal = MosaicCrystalSauter2014(experiment.crystal)
crystal.set_A(A)
# Copy in mosaic parameters if available
if params.output.domain_size_ang is None and hasattr(
experiment.crystal, "get_domain_size_ang"):
crystal.set_domain_size_ang(
experiment.crystal.get_domain_size_ang())
elif params.output.domain_size_ang is not None:
crystal.set_domain_size_ang(params.output.domain_size_ang)
if params.output.half_mosaicity_deg is None and hasattr(
experiment.crystal, "get_half_mosaicity_deg"):
crystal.set_half_mosaicity_deg(
experiment.crystal.get_half_mosaicity_deg())
elif params.output.half_mosaicity_deg is not None:
crystal.set_half_mosaicity_deg(
params.output.half_mosaicity_deg)
new_experiment = Experiment(
detector=experiment.detector,
beam=experiment.beam,
crystal=crystal,
imageset=experiment.imageset.as_imageset()
[i_scan_point:i_scan_point + 1],
)
new_experiments.append(new_experiment)
# Each reflection in a 3D shoebox can be found on multiple images.
# Slice the reflections such that any reflection on this scan point
# is included with this image
new_id = len(new_experiments) - 1
subrefls = refls.select((i_scan_point >= z1) & (i_scan_point < z2))
for refl in subrefls.rows():
assert i_scan_point in range(*refl["bbox"][4:6])
new_sb = Shoebox()
start = i_scan_point - refl["bbox"][4] # z1
new_sb.data = refl["shoebox"].data[start:start + 1, :, :]
new_sb.background = refl["shoebox"].background[start:start +
1, :, :]
new_sb.mask = refl["shoebox"].mask[start:start + 1, :, :]
intensity = new_sb.summed_intensity()
new_sb.bbox = tuple(
list(refl["bbox"])[0:4] +
[0, 1]) # keep the original shoebox but reset the z values
new_sb.panel = refl["panel"]
new_refl = {}
new_refl["id"] = new_refl["imageset_id"] = new_id
new_refl["shoebox"] = new_sb
new_refl["bbox"] = new_sb.bbox
new_refl["intensity.sum.value"] = intensity.observed.value
new_refl[
"intensity.sum.variance"] = intensity.observed.variance
for key in ["entering", "flags", "miller_index", "panel"]:
new_refl[key] = refl[key]
centroid = new_sb.centroid_foreground_minus_background()
new_refl["xyzobs.px.value"] = centroid.px.position
new_refl["xyzobs.px.variance"] = centroid.px.variance
new_reflections.append({})
for key in new_refl:
new_reflections[key][-1] = new_refl[key]
# Re-predict using the reflection slices and the stills predictors
ref_predictor = ExperimentsPredictorFactory.from_experiments(
new_experiments, force_stills=new_experiments.all_stills())
new_reflections = ref_predictor(new_reflections)
return (new_experiments, new_reflections)
def tst_split_partials_with_shoebox(self):
from dials.array_family import flex
from random import randint, uniform
from dials.model.data import Shoebox
r = flex.reflection_table()
r['value1'] = flex.double()
r['value2'] = flex.int()
r['value3'] = flex.double()
r['bbox'] = flex.int6()
r['panel'] = flex.size_t()
r['shoebox'] = flex.shoebox()
expected = []
for i in range(100):
x0 = randint(0, 100)
x1 = x0 + randint(1, 10)
y0 = randint(0, 100)
y1 = y0 + randint(1, 10)
z0 = randint(0, 100)
z1 = z0 + randint(1, 10)
v1 = uniform(0, 100)
v2 = randint(0, 100)
v3 = uniform(0, 100)
sbox = Shoebox(0, (x0, x1, y0, y1, z0, z1))
sbox.allocate()
assert (sbox.is_consistent())
w = x1 - x0
h = y1 - y0
for z in range(z0, z1):
for y in range(y0, y1):
for x in range(x0, x1):
sbox.data[z - z0, y - y0,
x - x0] = x + y * w + z * w * h
r.append({
'value1': v1,
'value2': v2,
'value3': v3,
'bbox': (x0, x1, y0, y1, z0, z1),
'panel': 0,
'shoebox': sbox
})
for z in range(z0, z1):
sbox = Shoebox(0, (x0, x1, y0, y1, z, z + 1))
sbox.allocate()
assert (sbox.is_consistent())
w = x1 - x0
h = y1 - y0
for y in range(y0, y1):
for x in range(x0, x1):
sbox.data[0, y - y0, x - x0] = x + y * w + z * w * h
expected.append({
'value1': v1,
'value2': v2,
'value3': v3,
'bbox': (x0, x1, y0, y1, z, z + 1),
'partial_id': i,
'panel': 0,
'shoebox': sbox
})
r.split_partials_with_shoebox()
assert (len(r) == len(expected))
EPS = 1e-7
for r1, r2 in zip(r, expected):
assert (abs(r1['value1'] - r2['value1']) < EPS)
assert (r1['value2'] == r2['value2'])
assert (abs(r1['value3'] - r2['value3']) < EPS)
assert (r1['bbox'] == r2['bbox'])
assert (r1['partial_id'] == r2['partial_id'])
assert (r1['panel'] == r2['panel'])
assert (r1['shoebox'].data.as_double().as_1d().all_approx_equal(
r2['shoebox'].data.as_double().as_1d()))
print 'OK'
def test_split_partials_with_shoebox():
from dials.model.data import Shoebox
r = flex.reflection_table()
r["value1"] = flex.double()
r["value2"] = flex.int()
r["value3"] = flex.double()
r["bbox"] = flex.int6()
r["panel"] = flex.size_t()
r["shoebox"] = flex.shoebox()
expected = []
for i in range(100):
x0 = random.randint(0, 100)
x1 = x0 + random.randint(1, 10)
y0 = random.randint(0, 100)
y1 = y0 + random.randint(1, 10)
z0 = random.randint(0, 100)
z1 = z0 + random.randint(1, 10)
v1 = random.uniform(0, 100)
v2 = random.randint(0, 100)
v3 = random.uniform(0, 100)
sbox = Shoebox(0, (x0, x1, y0, y1, z0, z1))
sbox.allocate()
assert sbox.is_consistent()
w = x1 - x0
h = y1 - y0
for z in range(z0, z1):
for y in range(y0, y1):
for x in range(x0, x1):
sbox.data[z - z0, y - y0, x - x0] = x + y * w + z * w * h
r.append({
"value1": v1,
"value2": v2,
"value3": v3,
"bbox": (x0, x1, y0, y1, z0, z1),
"panel": 0,
"shoebox": sbox,
})
for z in range(z0, z1):
sbox = Shoebox(0, (x0, x1, y0, y1, z, z + 1))
sbox.allocate()
assert sbox.is_consistent()
w = x1 - x0
h = y1 - y0
for y in range(y0, y1):
for x in range(x0, x1):
sbox.data[0, y - y0, x - x0] = x + y * w + z * w * h
expected.append({
"value1": v1,
"value2": v2,
"value3": v3,
"bbox": (x0, x1, y0, y1, z, z + 1),
"partial_id": i,
"panel": 0,
"shoebox": sbox,
})
r.split_partials_with_shoebox()
assert len(r) == len(expected)
EPS = 1e-7
for r1, r2 in zip(r.rows(), expected):
assert abs(r1["value1"] - r2["value1"]) < EPS
assert r1["value2"] == r2["value2"]
assert abs(r1["value3"] - r2["value3"]) < EPS
assert r1["bbox"] == r2["bbox"]
assert r1["partial_id"] == r2["partial_id"]
assert r1["panel"] == r2["panel"]
assert (r1["shoebox"].data.as_double().as_1d().all_approx_equal(
r2["shoebox"].data.as_double().as_1d()))
def test_is_bbox_within_image_volume():
from dials.model.data import Shoebox
isize = (1000, 1000)
srange = (0, 100)
shoebox = Shoebox((10, 20, 10, 20, 10, 20))
assert shoebox.is_bbox_within_image_volume(isize, srange)
shoebox = Shoebox((-10, 20, 10, 20, 10, 20))
assert not shoebox.is_bbox_within_image_volume(isize, srange)
shoebox = Shoebox((10, 20, -10, 20, 10, 20))
assert not shoebox.is_bbox_within_image_volume(isize, srange)
shoebox = Shoebox((10, 20, 10, 20, -10, 20))
assert not shoebox.is_bbox_within_image_volume(isize, srange)
shoebox = Shoebox((10, 1020, 10, 20, 10, 20))
assert not shoebox.is_bbox_within_image_volume(isize, srange)
shoebox = Shoebox((10, 20, 10, 1020, 10, 20))
assert not shoebox.is_bbox_within_image_volume(isize, srange)
shoebox = Shoebox((10, 20, 10, 20, 10, 1020))
assert not shoebox.is_bbox_within_image_volume(isize, srange)