class Test(object): def __init__(self): from dials.algorithms.background.simple import Creator from dials.algorithms.background.simple import MosflmOutlierRejector from dials.algorithms.background.simple import Linear2dModeller from os.path import join import libtbx.load_env try: dials_regression = libtbx.env.dist_path('dials_regression') except KeyError, e: print 'SKIP: dials_regression not configured' exit(0) # The directory path path = join( dials_regression, "integration_test_data", "i04-weak-data", "jmp_mosflm_test") # The input files self.reflection_filename = join(path, "mosflm_reflections.pickle") self.shoebox_filename = join(path, "shoeboxes.pickle") self.fraction = 1.0 self.n_sigma = 4 self.outlier_rejector = MosflmOutlierRejector(self.fraction, self.n_sigma) self.linear_modeller = Linear2dModeller() self.background_creator = Creator( self.linear_modeller, self.outlier_rejector)
def test_linear2d_modeller(self): from dials.algorithms.background.simple import Linear2dModeller from dials.array_family import flex modeller = Linear2dModeller() # Generate shoeboxes ma = 100 mb = 1 mc = 2 sboxes, masks = self.generate_background(self.size, 1000, ma, mb, mc, 0) pa = [] pv = [] for i in range(1000): model = modeller.create(sboxes[i], masks[i]) assert len(model.params()) == 9 * 3 assert len(model.variances()) == 9 * 3 p = model.params() v = model.variances() for j in range(9): pa.append(tuple(p[3 * j : 3 * (j + 1)])) pv.append(tuple(v[3 * j : 3 * (j + 1)])) a, b, c = zip(*pa) va, vb, vc = zip(*pv) # Compute Z for each parameter za = (flex.double(a) - ma) / flex.sqrt(flex.double(va)) zb = (flex.double(b) - mb) / flex.sqrt(flex.double(vb)) zc = (flex.double(c) - mc) / flex.sqrt(flex.double(vc)) # Check it looks standard normal self.assert_std_norm(za) self.assert_std_norm(zb) self.assert_std_norm(zc)
def select_modeller(): if model == 'constant2d': return Constant2dModeller() elif model == 'constant3d': return Constant3dModeller() elif model == 'linear2d': return Linear2dModeller() elif model == 'linear3d': return Linear3dModeller() raise RuntimeError("Unexpected background model: %s" % model)
def select_modeller(): if model == "constant2d": return Constant2dModeller() elif model == "constant3d": return Constant3dModeller() elif model == "linear2d": return Linear2dModeller() elif model == "linear3d": return Linear3dModeller() raise RuntimeError(f"Unexpected background model: {model}")
def test_linear2d_modeller(self): from dials.algorithms.background.simple import Linear2dModeller modeller = Linear2dModeller() eps = 1e-7 for i in range(10): p, data, mask = self.generate_linear_background_2d(self.size, 0, 100) model = modeller.create(data, mask) assert(len(model.params()) == 3 * 9) for j in range(9): for k in range(3): assert(abs(model.params()[k+j*3] - p[j][k]) < eps)
def run(self): from dials.algorithms.background.simple import Creator from dials.algorithms.background.simple import Linear2dModeller from dials.algorithms.background.simple import TruncatedOutlierRejector from dials.algorithms.background.simple import NSigmaOutlierRejector from dials.algorithms.background.simple import NormalOutlierRejector modeller = Linear2dModeller() outlier_rejector = [ None, TruncatedOutlierRejector(0.01, 0.01), NSigmaOutlierRejector(3.0, 3.0), NormalOutlierRejector(10), ] for rejector in outlier_rejector: self.tst(Creator(modeller, rejector))
def test_run(dials_regression): from dials.algorithms.background.simple import ( Linear2dModeller, MosflmOutlierRejector, ) # The directory path path = os.path.join(dials_regression, "integration_test_data", "i04-weak-data", "jmp_mosflm_test") # The input files reflection_filename = os.path.join(path, "mosflm_reflections.pickle") shoebox_filename = os.path.join(path, "shoeboxes.pickle") fraction = 1.0 n_sigma = 4 outlier_rejector = MosflmOutlierRejector(fraction, n_sigma) linear_modeller = Linear2dModeller() from dials.algorithms.shoebox import MaskCode from dials.array_family import flex print(shoebox_filename) # Read the data rtable = flex.reflection_table.from_file(reflection_filename) with open(shoebox_filename, "rb") as fh: if six.PY3: shoeboxes, masks = pickle.load(fh, encoding="bytes") else: shoeboxes, masks = pickle.load(fh) assert len(rtable) == len(shoeboxes) assert len(rtable) == len(masks) # Compute the background for each reflection and check against the values # read from the mosflm.lp file. Currently this fails for 1 strange # reflection whose pixel values in the mosflm file do not match those # extracted from the images. errors = 0 VAR1 = [] VAR2 = [] DIFF = [] for i in range(len(rtable)): I = rtable[i]["intensity.sum.value"] Ivar = rtable[i]["intensity.sum.variance"] data = shoeboxes[i].as_double() mask = masks[i] assert len(data.all()) == 2 assert len(mask.all()) == 2 data.reshape(flex.grid(1, *data.all())) mask.reshape(flex.grid(1, *mask.all())) outlier_rejector(data, mask) mask2 = (mask.as_1d() & int(MaskCode.BackgroundUsed)) != 0 mask2.reshape(flex.grid(*mask.all())) model = linear_modeller.create(data, mask2) assert len(model.params()) == 3 hy = data.all()[1] // 2 hx = data.all()[2] // 2 c1 = model.params()[0] a1 = model.params()[1] b1 = model.params()[2] c3 = c1 + a1 * (0.5 + hx) + b1 * (0.5 + hy) a2 = rtable[i]["background"][0] b2 = rtable[i]["background"][1] c2 = rtable[i]["background"][2] try: assert abs(a1 - b2) < 0.01 assert abs(b1 + a2) < 0.01 assert abs(c3 - c2) < 0.1 except Exception: errors += 1 continue background = data.as_double() # hy = background.all()[1] // 2 # hx = background.all()[2] // 2 for jj in range(background.all()[1]): for ii in range(background.all()[2]): # x = ii - hx # y = jj - hy x = ii + 0.5 y = jj + 0.5 background[0, jj, ii] = a1 * x + b1 * y + c1 # Test the summation results. Edge reflections use profile fitted # intensity in MOSFLM. Therefore ignore these. There also appears to be a # some discrepancy with very low <= 0 reflections where an extra 0.5 is # added. Not sure why this is so ignore these reflections as well. from dials.algorithms.integration.sum import integrate_by_summation intensity = integrate_by_summation(data.as_double(), background, mask) I2 = intensity.intensity() Ivar2 = intensity.variance() I1 = I Ivar1 = Ivar if mask.count(0) == 0 and mask.count(2) == 0 and I1 > 0: VAR1.append(math.sqrt(Ivar1)) VAR2.append(math.sqrt(Ivar2)) DIFF.append(math.sqrt(Ivar1) - math.sqrt(Ivar2)) try: assert abs(I1 - I2) < 1.0 assert abs(math.sqrt(Ivar1) - math.sqrt(Ivar2)) < 1.0 except Exception: errors += 1 continue # Only 1 should fail assert errors == 1
def run(self, flags, sequence=None, shoeboxes=None, **kwargs): from dials.algorithms.background.simple import Linear2dModeller modeller = Linear2dModeller() detector = sequence.get_detector() # sort shoeboxes by centroid z frame = shoeboxes.centroid_all().position_frame() perm = flex.sort_permutation(frame) shoeboxes = shoeboxes.select(perm) buffer_size = 1 bg_plus_buffer = self.background_size + buffer_size t0 = time.time() for i, shoebox in enumerate(shoeboxes): if not flags[perm[i]]: continue panel = detector[shoebox.panel] max_x, max_y = panel.get_image_size() bbox = shoebox.bbox x1, x2, y1, y2, z1, z2 = bbox # expand the bbox with a background region around the spotfinder shoebox # perhaps also should use a buffer zone between the shoebox and the # background region expanded_bbox = ( max(0, x1 - bg_plus_buffer), min(max_x, x2 + bg_plus_buffer), max(0, y1 - bg_plus_buffer), min(max_y, y2 + bg_plus_buffer), z1, z2, ) shoebox.bbox = expanded_bbox t1 = time.time() logger.info("Time expand_shoebox: %s" % (t1 - t0)) rlist = flex.reflection_table() rlist["shoebox"] = shoeboxes rlist["shoebox"].allocate() rlist["panel"] = shoeboxes.panels() rlist["bbox"] = shoeboxes.bounding_boxes() rlist.extract_shoeboxes(sequence) shoeboxes = rlist["shoebox"] shoeboxes.flatten() for i, shoebox in enumerate(shoeboxes): if not flags[perm[i]]: continue panel = detector[shoebox.panel] trusted_range = panel.get_trusted_range() ex1, ex2, ey1, ey2, ez1, ez2 = shoebox.bbox data = shoebox.data mask = flex.bool(data.accessor(), False) for i_y, y in enumerate(range(ey1, ey2)): for i_x, x in enumerate(range(ex1, ex2)): value = data[0, i_y, i_x] if (y >= (ey1 + buffer_size) and y < (ey2 - buffer_size) and x >= (ex1 + buffer_size) and x < (ex2 - buffer_size)): mask[0, i_y, i_x] = False # foreground elif value > trusted_range[0] and value < trusted_range[1]: mask[0, i_y, i_x] = True # background model = modeller.create(data.as_double(), mask) d, a, b = model.params()[:3] if abs(a) > self.gradient_cutoff or abs(b) > self.gradient_cutoff: flags[perm[i]] = False return flags
def run(self, flags, sweep=None, shoeboxes=None, **kwargs): from dials.array_family import flex from dials.algorithms.shoebox import MaskCode from dials.algorithms.background.simple import Linear2dModeller bg_code = MaskCode.Valid | MaskCode.BackgroundUsed fg_code = MaskCode.Valid | MaskCode.Foreground strong_code = MaskCode.Valid | MaskCode.Strong modeller = Linear2dModeller() expanded_shoeboxes = flex.shoebox() detector = sweep.get_detector() zoffset = 0 if sweep.get_scan() is not None: zoffset = sweep.get_scan().get_array_range()[0] from libtbx.containers import OrderedDict class image_data_cache(object): def __init__(self, imageset, size=10): self.imageset = imageset self.size = size self._image_data = OrderedDict() def __getitem__(self, i): image_data = self._image_data.get(i) if image_data is None: image_data = self.imageset.get_raw_data(i) if len(self._image_data) >= self.size: # remove the oldest entry in the cache del self._image_data[self._image_data.keys()[0]] self._image_data[i] = image_data return image_data cache = image_data_cache(sweep) #cache = sweep # sort shoeboxes by centroid z frame = shoeboxes.centroid_all().position_frame() perm = flex.sort_permutation(frame) shoeboxes = shoeboxes.select(perm) buffer_size = 1 bg_plus_buffer = self.background_size + buffer_size import time t0 = time.time() for i, shoebox in enumerate(shoeboxes): if not flags[perm[i]]: continue panel = detector[shoebox.panel] trusted_range = panel.get_trusted_range() max_x, max_y = panel.get_image_size() bbox = shoebox.bbox x1, x2, y1, y2, z1, z2 = bbox # expand the bbox with a background region around the spotfinder shoebox # perhaps also should use a buffer zone between the shoebox and the # background region expanded_bbox = (max(0, x1 - bg_plus_buffer), min(max_x, x2 + bg_plus_buffer), max(0, y1 - bg_plus_buffer), min(max_y, y2 + bg_plus_buffer), z1, z2) shoebox.bbox = expanded_bbox t1 = time.time() logger.info("Time expand_shoebox: %s" % (t1 - t0)) rlist = flex.reflection_table() rlist['shoebox'] = shoeboxes rlist['shoebox'].allocate() rlist['panel'] = shoeboxes.panels() rlist['bbox'] = shoeboxes.bounding_boxes() t0 = time.time() rlist.extract_shoeboxes(sweep) t1 = time.time() shoeboxes = rlist['shoebox'] shoeboxes.flatten() t0 = time.time() for i, shoebox in enumerate(shoeboxes): if not flags[perm[i]]: continue panel = detector[shoebox.panel] trusted_range = panel.get_trusted_range() max_x, max_y = panel.get_image_size() ex1, ex2, ey1, ey2, ez1, ez2 = shoebox.bbox data = shoebox.data mask = flex.bool(data.accessor(), False) for i_y, y in enumerate(range(ey1, ey2)): for i_x, x in enumerate(range(ex1, ex2)): value = data[0, i_y, i_x] if (y >= (ey1 + buffer_size) and y < (ey2 - buffer_size) and x >= (ex1 + buffer_size) and x < (ex2 - buffer_size)): mask[0, i_y, i_x] = False # foreground elif value > trusted_range[0] and value < trusted_range[1]: mask[0, i_y, i_x] = True # background model = modeller.create(data.as_double(), mask) d, a, b = model.params()[:3] c = -1 if abs(a) > self.gradient_cutoff or abs(b) > self.gradient_cutoff: flags[perm[i]] = False t1 = time.time() return flags
def test_run(dials_regression): from dials.algorithms.background.simple import Creator from dials.algorithms.background.simple import MosflmOutlierRejector from dials.algorithms.background.simple import Linear2dModeller # The directory path path = os.path.join(dials_regression, "integration_test_data", "i04-weak-data", "jmp_mosflm_test") # The input files reflection_filename = os.path.join(path, "mosflm_reflections.pickle") shoebox_filename = os.path.join(path, "shoeboxes.pickle") fraction = 1.0 n_sigma = 4 outlier_rejector = MosflmOutlierRejector(fraction, n_sigma) linear_modeller = Linear2dModeller() background_creator = Creator(linear_modeller, outlier_rejector) from dials.array_family import flex from math import sqrt from dials.algorithms.shoebox import MaskCode print(shoebox_filename) # Read the data rtable = flex.reflection_table.from_pickle(reflection_filename) with open(shoebox_filename, "rb") as fh: shoeboxes, masks = pickle.load(fh) assert len(rtable) == len(shoeboxes) assert len(rtable) == len(masks) # Compute the background for each reflection and check against the values # read from the mosflm.lp file. Currently this fails for 1 strange # reflection whose pixel values in the mosflm file do not match those # extracted from the images. count = 0 VAR1 = [] VAR2 = [] DIFF = [] for i in range(len(rtable)): xdet, ydet = rtable[i]["xy"] nx = rtable[i]['nx'] ny = rtable[i]['ny'] nc = rtable[i]['nc'] nrx = rtable[i]['nrx'] nry = rtable[i]['nry'] bbox = rtable[i]['bbox'] I = rtable[i]['intensity.sum.value'] Ivar = rtable[i]['intensity.sum.variance'] lp = rtable[i]['lp'] data = shoeboxes[i].as_double() mask = masks[i] fraction = 1.0 nsigma = 4 try: # model = PlaneModel(data, mask, fraction, nsigma) assert len(data.all()) == 2 assert len(mask.all()) == 2 data.reshape(flex.grid(1, *data.all())) mask.reshape(flex.grid(1, *mask.all())) outlier_rejector(data, mask) mask2 = (mask.as_1d() & int(MaskCode.BackgroundUsed)) != 0 mask2.reshape(flex.grid(*mask.all())) model = linear_modeller.create(data, mask2) except Exception: count += 1 raise continue # n = model.noutlier() assert (len(model.params()) == 3) hy = data.all()[1] // 2 hx = data.all()[2] // 2 c1 = model.params()[0] a1 = model.params()[1] b1 = model.params()[2] c3 = c1 + a1 * (0.5 + hx) + b1 * (0.5 + hy) # a1 = model.a() # b1 = model.b() # c1 = model.c() a2 = rtable[i]['background'][0] b2 = rtable[i]['background'][1] c2 = rtable[i]['background'][2] try: assert (abs(a1 - b2) < 0.01) assert (abs(b1 + a2) < 0.01) assert (abs(c3 - c2) < 0.1) except Exception: count += 1 continue #print "BG %d:(%.2f, %.2f, %.1f), (%.2f, %.2f, %.1f): %d" % \ #(i, a1, b1, c1, a2, b2, c2, n) #print "X, Y: ", xdet, ydet #print "NX, NY: ", nx, ny #print "NRX, NRY, NC", nrx, nry, nc #print int(floor(xdet + 0.5)) - nx // 2, int(floor(ydet + 0.5)) - ny // 2 #print "BBOX: ", bbox #print "N Outliers: ", model.noutlier() #print "N Background: ", model.nbackground() #print "Max DIff: ", model.maxdiff() #print data.as_numpy_array().transpose()[::-1,::-1] #print mask.as_numpy_array().transpose()[::-1,::-1] #raise background = data.as_double() # hy = background.all()[1] // 2 # hx = background.all()[2] // 2 for jj in range(background.all()[1]): for ii in range(background.all()[2]): # x = ii - hx # y = jj - hy x = ii + 0.5 y = jj + 0.5 background[0, jj, ii] = a1 * x + b1 * y + c1 # Test the summation results. Edge reflections use profile fitted # intensity in MOSFLM. Therefore ignore these. There also appears to be a # some discrepancy with very low <= 0 reflections where an extra 0.5 is # added. Not sure why this is so ignore these reflections as well. from dials.algorithms.integration.sum import integrate_by_summation intensity = integrate_by_summation(data.as_double(), background, mask) I2 = intensity.intensity() Ivar2 = intensity.variance() I1 = I Ivar1 = Ivar if mask.count(0) == 0 and mask.count(2) == 0 and I1 > 0: VAR1.append(sqrt(Ivar1)) VAR2.append(sqrt(Ivar2)) DIFF.append(sqrt(Ivar1) - sqrt(Ivar2)) try: assert (abs(I1 - I2) < 1.0) assert (abs(sqrt(Ivar1) - sqrt(Ivar2)) < 1.0) except Exception: count += 1 #import numpy #numpy.set_printoptions(precision=4, linewidth=200) #print "# %d" % i #print "I: %f, %f, %f" % (I2, I1, lp) #print "DEBUG: ", c1 * 25 #print "PF: %f" % rtable[i]['intensity.prf.value'] #print "BG (%.4f, %.4f, %.4f), (%.2f, %.2f, %.1f): %d" % \ #(a1, b1, c1, a2, b2, c2, n) #print "X, Y: ", xdet, ydet #print "NX, NY: ", nx, ny #print "NRX, NRY, NC", nrx, nry, nc #print int(floor(xdet + 0.5)) - nx // 2, int(floor(ydet + 0.5)) - ny // 2 #print "BBOX: ", bbox #print "N Outliers: ", model.noutlier() #print "N Background: ", model.nbackground() #print "Max DIff: ", model.maxdiff() #temp = (mask == MaskCode.Valid | MaskCode.Foreground).as_1d().as_int() #temp.resize(flex.grid(*data.all())) #temp = temp.as_double() #print data.as_numpy_array().transpose()[::-1,::-1] #print (background * temp).as_numpy_array().transpose()[::-1,::-1] #print mask.as_numpy_array().transpose()[::-1,::-1] #print ((data.as_double() - background) * temp).as_numpy_array().transpose()[::-1,::-1] #raise continue # Only 1 should fail assert (count == 1)