def generate_shoebox(self, bbox, centre, intensity, mask=False): from dials.model.data import Shoebox from dials.algorithms.shoebox import MaskCode shoebox = Shoebox() shoebox.bbox = bbox shoebox.allocate() for i in range(len(shoebox.mask)): shoebox.mask[i] = MaskCode.Valid | MaskCode.Foreground shoebox.data = self.gaussian( shoebox.size(), 1.0, [c - o for c, o in zip(centre[::-1], shoebox.offset())], [s / 8.0 for s in shoebox.size()]) if mask: shoebox.mask = self.create_mask( shoebox.size(), [c - o for c, o in zip(centre[::-1], shoebox.offset())], MaskCode.Valid | MaskCode.Foreground) tot = 0 mask_code = MaskCode.Valid | MaskCode.Foreground for i in range(len(shoebox.data)): if shoebox.mask[i] & mask_code == mask_code: tot += shoebox.data[i] if tot > 0: shoebox.data *= intensity / tot return shoebox
def tst_consistent(self): from random import randint from dials.model.data import Shoebox from dials.array_family import flex for i in range(1000): x0 = randint(0, 1000) y0 = randint(0, 1000) z0 = randint(0, 1000) x1 = randint(1, 10) + x0 y1 = randint(1, 10) + y0 z1 = randint(1, 10) + z0 try: shoebox = Shoebox((x0, x1, y0, y1, z0, z1)) assert (not shoebox.is_consistent()) shoebox.allocate() assert (shoebox.is_consistent()) shoebox.data = flex.real(flex.grid(20, 20, 20)) assert (not shoebox.is_consistent()) shoebox.deallocate() assert (not shoebox.is_consistent()) except Exception, e: print x0, y0, z0, x1, y1, z1 raise
def tst_count_mask_values(self): from dials.model.data import Shoebox from random import randint, sample for i in range(10): x0 = randint(0, 90) y0 = randint(0, 90) z0 = randint(0, 90) x1 = randint(1, 10) + x0 y1 = randint(1, 10) + y0 z1 = randint(1, 10) + z0 shoebox = Shoebox((x0, x1, y0, y1, z0, z1)) shoebox.allocate() maxnum = len(shoebox.mask) num = randint(1, maxnum) indices = sample(list(range(maxnum)), num) value = (1 << 2) for i in indices: shoebox.mask[i] = value assert (shoebox.count_mask_values(value) == num) # Test passed print 'OK'
def gen_shoebox(): shoebox = Shoebox(0, (0, 4, 0, 3, 0, 1)) shoebox.allocate() for k in range(1): for j in range(3): for i in range(4): shoebox.data[k, j, i] = i + j + k + 0.1 shoebox.mask[k, j, i] = i % 2 shoebox.background[k, j, i] = i * j + 0.2 return shoebox
def test_allocate(): for i in range(10): x0 = random.randint(0, 1000) y0 = random.randint(0, 1000) z0 = random.randint(0, 1000) x1 = random.randint(1, 10) + x0 y1 = random.randint(1, 10) + y0 z1 = random.randint(1, 10) + z0 shoebox = Shoebox((x0, x1, y0, y1, z0, z1)) shoebox.allocate() assert shoebox.data.all() == (z1 - z0, y1 - y0, x1 - x0) assert shoebox.mask.all() == (z1 - z0, y1 - y0, x1 - x0) shoebox.deallocate() assert shoebox.data.all() == (0, 0, 0) assert shoebox.mask.all() == (0, 0, 0)
def test_count_mask_values(): for i in range(10): x0 = random.randint(0, 90) y0 = random.randint(0, 90) z0 = random.randint(0, 90) x1 = random.randint(1, 10) + x0 y1 = random.randint(1, 10) + y0 z1 = random.randint(1, 10) + z0 shoebox = Shoebox((x0, x1, y0, y1, z0, z1)) shoebox.allocate() maxnum = len(shoebox.mask) num = random.randint(1, maxnum) indices = random.sample(list(range(maxnum)), num) value = 1 << 2 for i in indices: shoebox.mask[i] = value assert shoebox.count_mask_values(value) == num
def test_consistent(): from dials.array_family import flex for i in range(1000): x0 = random.randint(0, 1000) y0 = random.randint(0, 1000) z0 = random.randint(0, 1000) x1 = random.randint(1, 10) + x0 y1 = random.randint(1, 10) + y0 z1 = random.randint(1, 10) + z0 try: shoebox = Shoebox((x0, x1, y0, y1, z0, z1)) assert not shoebox.is_consistent() shoebox.allocate() assert shoebox.is_consistent() shoebox.data = flex.real(flex.grid(20, 20, 20)) assert not shoebox.is_consistent() shoebox.deallocate() assert not shoebox.is_consistent() except Exception: print(x0, y0, z0, x1, y1, z1) raise
def tst_allocate(self): from random import randint from dials.model.data import Shoebox for i in range(10): x0 = randint(0, 1000) y0 = randint(0, 1000) z0 = randint(0, 1000) x1 = randint(1, 10) + x0 y1 = randint(1, 10) + y0 z1 = randint(1, 10) + z0 shoebox = Shoebox((x0, x1, y0, y1, z0, z1)) shoebox.allocate() assert (shoebox.data.all() == (z1 - z0, y1 - y0, x1 - x0)) assert (shoebox.mask.all() == (z1 - z0, y1 - y0, x1 - x0)) shoebox.deallocate() assert (shoebox.data.all() == (0, 0, 0)) assert (shoebox.mask.all() == (0, 0, 0)) # Test passed print 'OK'
def test_split_partials_with_shoebox(): from dials.model.data import Shoebox r = flex.reflection_table() r["value1"] = flex.double() r["value2"] = flex.int() r["value3"] = flex.double() r["bbox"] = flex.int6() r["panel"] = flex.size_t() r["shoebox"] = flex.shoebox() expected = [] for i in range(100): x0 = random.randint(0, 100) x1 = x0 + random.randint(1, 10) y0 = random.randint(0, 100) y1 = y0 + random.randint(1, 10) z0 = random.randint(0, 100) z1 = z0 + random.randint(1, 10) v1 = random.uniform(0, 100) v2 = random.randint(0, 100) v3 = random.uniform(0, 100) sbox = Shoebox(0, (x0, x1, y0, y1, z0, z1)) sbox.allocate() assert sbox.is_consistent() w = x1 - x0 h = y1 - y0 for z in range(z0, z1): for y in range(y0, y1): for x in range(x0, x1): sbox.data[z - z0, y - y0, x - x0] = x + y * w + z * w * h r.append({ "value1": v1, "value2": v2, "value3": v3, "bbox": (x0, x1, y0, y1, z0, z1), "panel": 0, "shoebox": sbox, }) for z in range(z0, z1): sbox = Shoebox(0, (x0, x1, y0, y1, z, z + 1)) sbox.allocate() assert sbox.is_consistent() w = x1 - x0 h = y1 - y0 for y in range(y0, y1): for x in range(x0, x1): sbox.data[0, y - y0, x - x0] = x + y * w + z * w * h expected.append({ "value1": v1, "value2": v2, "value3": v3, "bbox": (x0, x1, y0, y1, z, z + 1), "partial_id": i, "panel": 0, "shoebox": sbox, }) r.split_partials_with_shoebox() assert len(r) == len(expected) EPS = 1e-7 for r1, r2 in zip(r.rows(), expected): assert abs(r1["value1"] - r2["value1"]) < EPS assert r1["value2"] == r2["value2"] assert abs(r1["value3"] - r2["value3"]) < EPS assert r1["bbox"] == r2["bbox"] assert r1["partial_id"] == r2["partial_id"] assert r1["panel"] == r2["panel"] assert (r1["shoebox"].data.as_double().as_1d().all_approx_equal( r2["shoebox"].data.as_double().as_1d()))
def tilt_fit(imgs, is_bg_pix, delta_q, photon_gain, sigma_rdout, zinger_zscore, exper, predicted_refls, sb_pad=0, filter_boundary_spots=False, minsnr=None, mintilt=None, plot=False, verbose=False, is_BAD_pix=None, min_strong=None, min_bg=10, min_dist_to_bad_pix=7, **kwargs): if is_BAD_pix is None: is_BAD_pix = np.zeros(np.array(is_bg_pix).shape, np.bool) predicted_refls['id'] = flex.int(len(predicted_refls), -1) predicted_refls['imageset_id'] = flex.int(len(predicted_refls), 0) El = ExperimentList() El.append(exper) predicted_refls.centroid_px_to_mm(El) predicted_refls.map_centroids_to_reciprocal_space(El) ss_dim, fs_dim = imgs[0].shape n_refl = len(predicted_refls) integrations = [] variances = [] coeffs = [] new_shoeboxes = [] tilt_error = [] boundary = [] detdist = exper.detector[0].get_distance() pixsize = exper.detector[0].get_pixel_size()[0] ave_wave = exper.beam.get_wavelength() bad_trees = {} unique_panels = set(predicted_refls["panel"]) for p in unique_panels: panel_bad_pix = is_BAD_pix[p] ybad, xbad = np.where(is_BAD_pix[0]) if ybad.size: bad_pts = zip(ybad, xbad) bad_trees[p] = cKDTree(bad_pts) else: bad_trees[p] = None sel = [] for i_ref in range(len(predicted_refls)): ref = predicted_refls[i_ref] i_com, j_com, _ = ref['xyzobs.px.value'] # which detector panel am I on ? i_panel = ref['panel'] if bad_trees[i_panel] is not None: if bad_trees[i_panel].query_ball_point((i_com, j_com), r=min_dist_to_bad_pix): sel.append(False) integrations.append(None) variances.append(None) coeffs.append(None) new_shoeboxes.append(None) tilt_error.append(None) boundary.append(None) continue i1_a, i2_a, j1_a, j2_a, _, _ = ref['bbox'] # bbox of prediction i1_ = max(i1_a, 0) i2_ = min(i2_a, fs_dim-1) j1_ = max(j1_a, 0) j2_ = min(j2_a, ss_dim-1) # get the number of pixels spanning the box in pixels Qmag = 2*np.pi*np.linalg.norm(ref['rlp']) # magnitude momentum transfer of the RLP in physicist convention rad1 = (detdist/pixsize) * np.tan(2*np.arcsin((Qmag-delta_q*.5)*ave_wave/4/np.pi)) rad2 = (detdist/pixsize) * np.tan(2*np.arcsin((Qmag+delta_q*.5)*ave_wave/4/np.pi)) bbox_extent = (rad2-rad1) / np.sqrt(2) # rad2 - rad1 is the diagonal across the bbox i_com = i_com - 0.5 j_com = j_com - 0.5 i_low = int(i_com - bbox_extent/2.) i_high = int(i_com + bbox_extent/2.) j_low = int(j_com - bbox_extent/2.) j_high = int(j_com + bbox_extent/2.) i1_orig = max(i_low, 0) i2_orig = min(i_high, fs_dim-1) j1_orig = max(j_low, 0) j2_orig = min(j_high, ss_dim-1) i_low = i_low - sb_pad i_high = i_high + sb_pad j_low = j_low - sb_pad j_high = j_high + sb_pad i1 = max(i_low, 0) i2 = min(i_high, fs_dim-1) j1 = max(j_low, 0) j2 = min(j_high, ss_dim-1) i1_p = i1_orig - i1 i2_p = i1_p + i2_orig-i1_orig j1_p = j1_orig - j1 j2_p = j1_p + j2_orig-j1_orig if i1 == 0 or i2 == fs_dim or j1 == 0 or j2 == ss_dim: boundary.append(True) if filter_boundary_spots: sel.append(False) integrations.append(None) variances.append(None) coeffs.append(None) new_shoeboxes.append(None) tilt_error.append(None) continue else: boundary.append(False) # get the iamge and mask shoebox_img = imgs[i_panel][j1:j2, i1:i2] / photon_gain # NOTE: gain is imortant here! dials_mask = np.zeros(shoebox_img.shape).astype(np.int32) # initially all pixels are valid dials_mask += MaskCode.Valid shoebox_mask = is_bg_pix[i_panel][j1:j2, i1:i2] badpix_mask = is_BAD_pix[i_panel][j1:j2, i1:i2] dials_mask[shoebox_mask] = dials_mask[shoebox_mask] + MaskCode.Background new_shoebox = Shoebox((i1_orig, i2_orig, j1_orig, j2_orig, 0, 1)) new_shoebox.allocate() new_shoebox.data = flex.float(np.ascontiguousarray(shoebox_img[None, j1_p:j2_p, i1_p: i2_p])) #new_shoebox.data = flex.float(shoebox_img[None,]) # get coordinates arrays of the image Y, X = np.indices(shoebox_img.shape) # determine if any more outliers are present in background pixels img1d = shoebox_img.ravel() mask1d = shoebox_mask.ravel() # mask specifies which pixels are bg # out1d specifies which bg pixels are outliers (zingers) out1d = np.zeros(mask1d.shape, bool) out1d[mask1d] = is_outlier(img1d[mask1d].ravel(), zinger_zscore) out2d = out1d.reshape(shoebox_img.shape) # combine bad2d with badpix mask out2d = np.logical_or(out2d, badpix_mask) # these are points we fit to: both zingers and original mask fit_sel = np.logical_and(~out2d, shoebox_mask) # fit plane to these points, no outliers, no masked if np.sum(fit_sel) < min_bg: integrations.append(None) variances.append(None) coeffs.append(None) new_shoeboxes.append(None) tilt_error.append(None) sel.append(False) continue # update the dials mask... dials_mask[fit_sel] = dials_mask[fit_sel] + MaskCode.BackgroundUsed # fast scan pixels, slow scan pixels, pixel values (corrected for gain) fast, slow, rho_bg = X[fit_sel], Y[fit_sel], shoebox_img[fit_sel] # do the fit of the background plane A = np.array([fast, slow, np.ones_like(fast)]).T # weights matrix: W = np.diag(1 / (sigma_rdout ** 2 + rho_bg)) AWA = np.dot(A.T, np.dot(W, A)) try: AWA_inv = np.linalg.inv(AWA) except np.linalg.LinAlgError: print ("WARNING: Fit did not work.. investigate reflection") print (ref) integrations.append(None) variances.append(None) coeffs.append(None) new_shoeboxes.append(None) tilt_error.append(None) sel.append(False) continue AtW = np.dot(A.T, W) a, b, c = np.dot(np.dot(AWA_inv, AtW), rho_bg) coeffs.append((a, b, c)) # fit of the tilt plane background X1d = np.ravel(X) Y1d = np.ravel(Y) background = (X1d * a + Y1d * b + c).reshape(shoebox_img.shape) new_shoebox.background = flex.float(np.ascontiguousarray(background[None, j1_p: j2_p, i1_p:i2_p])) # vector of residuals r = rho_bg - np.dot(A, (a, b, c)) Nbg = len(rho_bg) Nparam = 3 r_fact = np.dot(r.T, np.dot(W, r)) / (Nbg - Nparam) var_covar = AWA_inv * r_fact abc_var = var_covar[0][0], var_covar[1][1], var_covar[2][2] # place the strong spot mask in the expanded shoebox peak_mask = ref['shoebox'].mask.as_numpy_array()[0] == MaskCode.Valid + MaskCode.Foreground peak_mask_valid = peak_mask[j1_-j1_a:- j1_a + j2_, i1_-i1_a:-i1_a + i2_] peak_mask_expanded = np.zeros_like(shoebox_mask) # overlap region i1_o = max(i1_, i1) i2_o = min(i2_, i2) j1_o = max(j1_, j1) j2_o = min(j2_, j2) pk_mask_istart = i1_o - i1_ pk_mask_jstart = j1_o - j1_ pk_mask_istop = peak_mask_valid.shape[1] - (i2_ - i2_o) pk_mask_jstop = peak_mask_valid.shape[0] - (j2_ - j2_o) peak_mask_overlap = peak_mask_valid[pk_mask_jstart: pk_mask_jstop, pk_mask_istart: pk_mask_istop] pk_mask_exp_i1 = i1_o - i1 pk_mask_exp_j1 = j1_o - j1 pk_mask_exp_i2 = peak_mask_expanded.shape[1] - (i2 - i2_o) pk_mask_exp_j2 = peak_mask_expanded.shape[0] - (j2 - j2_o) peak_mask_expanded[pk_mask_exp_j1: pk_mask_exp_j2, pk_mask_exp_i1: pk_mask_exp_i2] = peak_mask_overlap # update the dials mask dials_mask[peak_mask_expanded] = dials_mask[peak_mask_expanded] + MaskCode.Foreground p = X[peak_mask_expanded] # fast scan coords q = Y[peak_mask_expanded] # slow scan coords rho_peak = shoebox_img[peak_mask_expanded] # pixel values Isum = np.sum(rho_peak - a*p - b*q - c) # summed spot intensity var_rho_peak = sigma_rdout ** 2 + rho_peak # include readout noise in the variance Ns = len(rho_peak) # number of integrated peak pixels # variance propagated from tilt plane constants var_a_term = abc_var[0] * ((np.sum(p))**2) var_b_term = abc_var[1] * ((np.sum(q))**2) var_c_term = abc_var[2] * (Ns**2) tilt_error.append(var_a_term + var_b_term + var_c_term) # total variance of the spot var_Isum = np.sum(var_rho_peak) + var_a_term + var_b_term + var_c_term integrations.append(Isum) variances.append(var_Isum) new_shoebox.mask = flex.int(np.ascontiguousarray(dials_mask[None, j1_p:j2_p, i1_p:i2_p])) new_shoeboxes.append(new_shoebox) sel.append(True) if i_ref % 50 == 0 and verbose: print("Integrated refls %d / %d" % (i_ref+1, n_refl)) #if filter_boundary_spots: # sel = flex.bool([I is not None for I in integrations]) boundary = np.array(boundary)[sel].astype(bool) integrations = np.array([I for I in integrations if I is not None]) variances = np.array([v for v in variances if v is not None]) coeffs = np.array([c for c in coeffs if c is not None]) tilt_error = np.array([te for te in tilt_error if te is not None]) #boundary = np.zeros(tilt_error.shape).astype(np.bool) predicted_refls = predicted_refls.select(flex.bool(sel)) predicted_refls['resolution'] = flex.double( 1/ np.linalg.norm(predicted_refls['rlp'], axis=1)) predicted_refls['boundary'] = flex.bool(boundary) predicted_refls["intensity.sum.value.Leslie99"] = flex.double(integrations) predicted_refls["intensity.sum.variance.Leslie99"] = flex.double(variances) predicted_refls['shoebox'] = flex.shoebox([sb for sb in new_shoeboxes if sb is not None]) idx_assign = assign_indices.AssignIndicesGlobal(tolerance=0.333) idx_assign(predicted_refls, El) return predicted_refls, coeffs, tilt_error, integrations, variances
all_residual.append(np.mean(residual)) ref["snr"] = np.nan_to_num(Isum / varIsum) ref["tilt_error"] = np.diag(variance_matrix).sum() ref["shoebox_roi"] = shoebox_roi panels.append(ref["panel"]) xyzobs.append(list(ref["xyzobs.px.value"])) snr = np.array(integrations) / np.sqrt(variances) data = {"panel": panels, "shoebox_roi": bboxes, "integration": integrations, "variance": variances, "tilt_errors": tilt_error, "xyzobs.px.value": xyzobs, "snr": snr, "dips_below_zero": dips_below_zero} df = pandas.DataFrame(data) shoeboxes = [] for i1,i2,j1,j2 in bboxes: new_shoebox = Shoebox((i1,i2, j1, j2, 0, 1)) new_shoebox.allocate() shoeboxes.append(new_shoebox) embed() if not args.plotassembled: #from cxid9114.prediction import prediction_utils import pylab as plt #refls_predict_bypanel = prediction_utils.refls_by_panelname(Rnew) pause = args.pause plt.figure(1) ax = plt.gca() refls_bypanel = df.groupby("panel") for panel_id in df.panel.unique(): if args.pause == -1: plt.figure(1)
def tst_split_partials_with_shoebox(self): from dials.array_family import flex from random import randint, uniform from dials.model.data import Shoebox r = flex.reflection_table() r['value1'] = flex.double() r['value2'] = flex.int() r['value3'] = flex.double() r['bbox'] = flex.int6() r['panel'] = flex.size_t() r['shoebox'] = flex.shoebox() expected = [] for i in range(100): x0 = randint(0, 100) x1 = x0 + randint(1, 10) y0 = randint(0, 100) y1 = y0 + randint(1, 10) z0 = randint(0, 100) z1 = z0 + randint(1, 10) v1 = uniform(0, 100) v2 = randint(0, 100) v3 = uniform(0, 100) sbox = Shoebox(0, (x0, x1, y0, y1, z0, z1)) sbox.allocate() assert (sbox.is_consistent()) w = x1 - x0 h = y1 - y0 for z in range(z0, z1): for y in range(y0, y1): for x in range(x0, x1): sbox.data[z - z0, y - y0, x - x0] = x + y * w + z * w * h r.append({ 'value1': v1, 'value2': v2, 'value3': v3, 'bbox': (x0, x1, y0, y1, z0, z1), 'panel': 0, 'shoebox': sbox }) for z in range(z0, z1): sbox = Shoebox(0, (x0, x1, y0, y1, z, z + 1)) sbox.allocate() assert (sbox.is_consistent()) w = x1 - x0 h = y1 - y0 for y in range(y0, y1): for x in range(x0, x1): sbox.data[0, y - y0, x - x0] = x + y * w + z * w * h expected.append({ 'value1': v1, 'value2': v2, 'value3': v3, 'bbox': (x0, x1, y0, y1, z, z + 1), 'partial_id': i, 'panel': 0, 'shoebox': sbox }) r.split_partials_with_shoebox() assert (len(r) == len(expected)) EPS = 1e-7 for r1, r2 in zip(r, expected): assert (abs(r1['value1'] - r2['value1']) < EPS) assert (r1['value2'] == r2['value2']) assert (abs(r1['value3'] - r2['value3']) < EPS) assert (r1['bbox'] == r2['bbox']) assert (r1['partial_id'] == r2['partial_id']) assert (r1['panel'] == r2['panel']) assert (r1['shoebox'].data.as_double().as_1d().all_approx_equal( r2['shoebox'].data.as_double().as_1d())) print 'OK'
def tst_split_partials_with_shoebox(self): from dials.array_family import flex from random import randint, uniform from dials.model.data import Shoebox r = flex.reflection_table() r['value1'] = flex.double() r['value2'] = flex.int() r['value3'] = flex.double() r['bbox'] = flex.int6() r['panel'] = flex.size_t() r['shoebox'] = flex.shoebox() expected = [] for i in range(100): x0 = randint(0, 100) x1 = x0 + randint(1, 10) y0 = randint(0, 100) y1 = y0 + randint(1, 10) z0 = randint(0, 100) z1 = z0 + randint(1, 10) v1 = uniform(0, 100) v2 = randint(0, 100) v3 = uniform(0, 100) sbox = Shoebox(0, (x0, x1, y0, y1, z0, z1)) sbox.allocate() assert(sbox.is_consistent()) w = x1 - x0 h = y1 - y0 for z in range(z0, z1): for y in range(y0, y1): for x in range(x0, x1): sbox.data[z-z0,y-y0,x-x0] = x+y*w+z*w*h r.append({ 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z0, z1), 'panel' : 0, 'shoebox' : sbox }) for z in range(z0, z1): sbox = Shoebox(0, (x0, x1, y0, y1, z, z+1)) sbox.allocate() assert(sbox.is_consistent()) w = x1 - x0 h = y1 - y0 for y in range(y0, y1): for x in range(x0, x1): sbox.data[0,y-y0,x-x0] = x+y*w+z*w*h expected.append({ 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z, z+1), 'partial_id' : i, 'panel' : 0, 'shoebox' : sbox }) r.split_partials_with_shoebox() assert(len(r) == len(expected)) EPS = 1e-7 for r1, r2 in zip(r, expected): assert(abs(r1['value1'] - r2['value1']) < EPS) assert(r1['value2'] == r2['value2']) assert(abs(r1['value3'] - r2['value3']) < EPS) assert(r1['bbox'] == r2['bbox']) assert(r1['partial_id'] == r2['partial_id']) assert(r1['panel'] == r2['panel']) assert(r1['shoebox'].data.as_double().as_1d().all_approx_equal( r2['shoebox'].data.as_double().as_1d())) print 'OK'