def select_strong_pixels(sweep, trusted_range): from dials.util.command_line import ProgressBar import numpy # Calculate the threshold coordinate = [] intensity = [] progress = ProgressBar() for i, flex_image in enumerate(sweep): image = flex_image.as_numpy_array() height, width = image.shape threshold = calculate_threshold(image, trusted_range) image.shape = -1 mask = image >= threshold ind = numpy.where(mask != 0)[0] z = [i] * len(ind) y = list(ind / width) x = list(ind % width) coords = zip(x, y, z) coordinate.extend(coords) intensity.extend(list(image[ind])) progress.update(100.0 * float(i) / len(sweep)) progress.finished() return coordinate, intensity
def select_strong_pixels(sweep, trusted_range): from dials.util.command_line import ProgressBar import numpy # Calculate the threshold coordinate = [] intensity = [] progress = ProgressBar() for i, flex_image in enumerate(sweep): image = flex_image.as_numpy_array() height, width = image.shape threshold = calculate_threshold(image, trusted_range) image.shape = -1 mask = image >= threshold ind = numpy.where(mask != 0)[0] z = [i] * len(ind) y = list(ind / width) x = list(ind % width) coords = zip(x, y, z) coordinate.extend(coords) intensity.extend(list(image[ind])) progress.update(100.0 * float(i) / len(sweep)) progress.finished() return coordinate, intensity
def write(self, predictions, imageset): from dials.util.command_line import ProgressBar zrange = imageset.get_array_range() p = ProgressBar(title="Extracting shoeboxes") for z, image in enumerate(imageset, start=zrange[0]): batch = self._add_image(z, image) self._write_pickle(batch) p.update(100.0*(z - zrange[0])/len(imageset)) p.finish("Extracted shoeboxes") self._write_predictions(predictions)
def write(self, predictions, imageset): from dials.util.command_line import ProgressBar zrange = imageset.get_array_range() p = ProgressBar(title="Extracting shoeboxes") for z, image in enumerate(imageset, start=zrange[0]): batch = self._add_image(z, image) self._write_pickle(batch) p.update(100.0 * (z - zrange[0]) / len(imageset)) p.finish("Extracted shoeboxes") self._write_predictions(predictions)
def generate_background(self, size, N, A, B, C, D): from dials.algorithms.simulation.generate_test_reflections import random_background_plane2 from dials.array_family import flex from dials.util.command_line import ProgressBar sboxes = [] masks = [] progress = ProgressBar(title="Generating Background") for i in range(N): mask = flex.bool(flex.grid(size), True) sbox = flex.double(flex.grid(size), 0) random_background_plane2(sbox, A, B, C, D) sboxes.append(sbox) masks.append(mask) progress.update(100.0 * i / N) progress.finished("Generated Background") return sboxes, masks
def generate_background(self, size, N, A, B, C, D): from dials.algorithms.simulation.generate_test_reflections \ import random_background_plane2 from dials.array_family import flex from dials.util.command_line import ProgressBar sboxes = [] masks = [] progress = ProgressBar(title="Generating Background") for i in range(N): mask = flex.bool(flex.grid(size), True) sbox = flex.double(flex.grid(size), 0) random_background_plane2(sbox, A, B, C, D) sboxes.append(sbox) masks.append(mask) progress.update(100.0 * i / N) progress.finished("Generated Background") return sboxes, masks
def _extract_pixels(self, sweep): """Extract the pixels from the sweep Params: sweep The sweep object Returns: The list of selected pixels """ from dials.util.command_line import ProgressBar from scitbx.array_family import flex from dials.algorithms.peak_finding import flex_vec3_int # Initialise the pixel arrays coords = flex_vec3_int() intensity = flex.int() # Get the start index and trusted range from the sweep start = sweep.get_array_range()[0] trusted_range = sweep.get_detector().get_trusted_range() # Loop through all the images in the sweep and extract the pixels # from each of the images progress = ProgressBar() for frame, image in enumerate(sweep): c, i = self._extract_image_pixels(image, frame + start, trusted_range) coords.extend(c) intensity.extend(i) progress.update(100.0 * float(frame + 1) / len(sweep)) progress.finished() # Reuturn the pixel values return coords, intensity
def _extract_pixels(self, sweep): '''Extract the pixels from the sweep Params: sweep The sweep object Returns: The list of selected pixels ''' from dials.util.command_line import ProgressBar from scitbx.array_family import flex from dials.algorithms.peak_finding import flex_vec3_int # Initialise the pixel arrays coords = flex_vec3_int() intensity = flex.int() # Get the start index and trusted range from the sweep start = sweep.get_array_range()[0] trusted_range = sweep.get_detector().get_trusted_range() # Loop through all the images in the sweep and extract the pixels # from each of the images progress = ProgressBar() for frame, image in enumerate(sweep): c, i = self._extract_image_pixels(image, frame + start, trusted_range) coords.extend(c) intensity.extend(i) progress.update(100.0 * float(frame + 1) / len(sweep)) progress.finished() # Reuturn the pixel values return coords, intensity
int(math.floor(min(z))), int(math.floor(max(z)) + 1)) try: counts = counts_database[hkl.miller_index] useful.append(hkl) except KeyError, e: continue from dials.algorithms import shoebox shoebox.allocate(useful) from dials.util.command_line import ProgressBar p = ProgressBar(title = 'Generating shoeboxes') # now for each reflection perform the simulation for j, refl in enumerate(useful): p.update(j * 100.0 / len(useful)) d = d_matrices[j] from scitbx.random import variate, normal_distribution g = variate(normal_distribution(mean = 0, sigma = node_size)) counts = counts_database[refl.miller_index] dhs = g(counts) dks = g(counts) dls = g(counts) self.map_to_image_space(refl, d, dhs, dks, dls) p.finished('Generated %d shoeboxes' % len(useful)) # now for each reflection add background from dials.algorithms.simulation.generate_test_reflections import \ random_background_plane
def simple_gaussian_spots(params): from scitbx import matrix from dials.array_family import flex r = params.rotation axis = matrix.col((r.axis.x, r.axis.y, r.axis.z)) if axis.length() > 0: rotation = axis.axis_and_angle_as_r3_rotation_matrix(r.angle, deg=True) else: rotation = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1)) # generate mask and peak values from dials.algorithms.shoebox import MaskCode mask_peak = MaskCode.Valid | MaskCode.Foreground mask_back = MaskCode.Valid | MaskCode.Background from dials.util.command_line import ProgressBar p = ProgressBar(title="Generating reflections") rlist = flex.reflection_table(params.nrefl) hkl = flex.miller_index(params.nrefl) s1 = flex.vec3_double(params.nrefl) xyzmm = flex.vec3_double(params.nrefl) xyzpx = flex.vec3_double(params.nrefl) panel = flex.size_t(params.nrefl) bbox = flex.int6(params.nrefl) for j in range(params.nrefl): p.update(j * 100.0 / params.nrefl) hkl[j] = (random.randint(0, 20), random.randint(0, 20), random.randint(0, 20)) phi = 2 * math.pi * random.random() s1[j] = (0, 0, 0) xyzpx[j] = (0, 0, 0) xyzmm[j] = (0, 0, phi) panel[j] = 0 bbox[j] = ( 0, params.shoebox_size.x, 0, params.shoebox_size.y, 0, params.shoebox_size.z, ) p.finished("Generating %d reflections" % params.nrefl) intensity = flex.double(params.nrefl) shoebox = flex.shoebox(panel, bbox) shoebox.allocate_with_value(MaskCode.Valid) p = ProgressBar(title="Generating shoeboxes") for i in range(len(rlist)): p.update(i * 100.0 / params.nrefl) mask = shoebox[i].mask if params.pixel_mask == "precise": # flag everything as background: peak will me assigned later for j in range(len(mask)): mask[j] = mask_back elif params.pixel_mask == "all": # flag we have no idea what anything is mask_none = MaskCode.Valid | MaskCode.Foreground | MaskCode.Background for j in range(len(mask)): mask[j] = mask_none elif params.pixel_mask == "static": from scitbx.array_family import flex x0 = params.spot_offset.x + params.shoebox_size.x / 2 y0 = params.spot_offset.x + params.shoebox_size.y / 2 z0 = params.spot_offset.x + params.shoebox_size.z / 2 sx = params.mask_nsigma * params.spot_size.x sy = params.mask_nsigma * params.spot_size.y sz = params.mask_nsigma * params.spot_size.z # The x, y, z indices z, y, x = zip(*itertools.product(*(range(n) for n in mask.all()))) xyz = flex.vec3_double(flex.double(x), flex.double(y), flex.double(z)) # Calculate SUM(((xj - xj0) / sxj)**2) for each element xyz0 = (x0, y0, z0) isxyz = (1.0 / sx, 1.0 / sy, 1.0 / sz) dxyz = sum( (x * isx) ** 2 for x, isx in zip(((xyz - xyz0) * rotation).parts(), isxyz) ) # Set the mask values index = dxyz <= 1.0 index.reshape(mask.accessor()) mask.set_selected(index, MaskCode.Valid | MaskCode.Foreground) mask.set_selected(not index, MaskCode.Valid | MaskCode.Background) sbox = shoebox[i].data # reflection itself, including setting the peak region if we're doing that # FIXME use flex arrays to make the rotation bit more efficient as this is # now rather slow... counts_true = 0 for j in range(params.counts): _x = random.gauss(0, params.spot_size.x) _y = random.gauss(0, params.spot_size.y) _z = random.gauss(0, params.spot_size.z) Rxyz = rotation * matrix.col((_x, _y, _z)).elems x = int(Rxyz[0] + params.spot_offset.x + params.shoebox_size.x / 2) y = int(Rxyz[1] + params.spot_offset.y + params.shoebox_size.y / 2) z = int(Rxyz[2] + params.spot_offset.z + params.shoebox_size.z / 2) if x < 0 or x >= params.shoebox_size.x: continue if y < 0 or y >= params.shoebox_size.y: continue if z < 0 or z >= params.shoebox_size.z: continue sbox[z, y, x] += 1 counts_true += 1 if params.pixel_mask == "precise": mask[z, y, x] = mask_peak intensity[i] = counts_true if params.background: # background:flat; for j in range(params.background * len(sbox)): x = random.randint(0, params.shoebox_size.x - 1) y = random.randint(0, params.shoebox_size.y - 1) z = random.randint(0, params.shoebox_size.z - 1) sbox[z, y, x] += 1 else: # or inclined random_background_plane( sbox, params.background_a, params.background_b, params.background_c, params.background_d, ) rlist["miller_index"] = hkl rlist["s1"] = s1 rlist["xyzcal.px"] = xyzpx rlist["xyzcal.mm"] = xyzmm rlist["bbox"] = bbox rlist["panel"] = panel rlist["shoebox"] = shoebox rlist["intensity.sum.value"] = intensity p.finished("Generating %d shoeboxes" % params.nrefl) return rlist
def with_individual_given_intensity(self, N, I, Ba, Bb, Bc, Bd): """ Generate reflections with given intensity and background. """ from dials.array_family import flex from dials.util.command_line import ProgressBar from dials.algorithms.simulation import simulate_reciprocal_space_gaussian from dials.algorithms.simulation.generate_test_reflections import random_background_plane2 # Check the lengths assert N == len(I) assert N == len(Ba) assert N == len(Bb) assert N == len(Bc) assert N == len(Bd) # Generate some predictions refl = self.generate_predictions(N) # Calculate the signal progress = ProgressBar(title="Calculating signal for %d reflections" % len(refl)) s1 = refl["s1"] phi = refl["xyzcal.mm"].parts()[2] bbox = refl["bbox"] shoebox = refl["shoebox"] m = int(len(refl) / 100) I_exp = flex.double(len(refl), 0) for i in range(len(refl)): if I[i] > 0: data = shoebox[i].data.as_double() I_exp[i] = simulate_reciprocal_space_gaussian( self.experiment.beam, self.experiment.detector, self.experiment.goniometer, self.experiment.scan, self.sigma_b, self.sigma_m, s1[i], phi[i], bbox[i], I[i], data, shoebox[i].mask, ) shoebox[i].data = data.as_float() if i % m == 0: progress.update(100.0 * float(i) / len(refl)) progress.finished("Calculated signal impacts for %d reflections" % len(refl)) # Calculate the background progress = ProgressBar(title="Calculating background for %d reflections" % len(refl)) for l in range(len(refl)): background = flex.float(flex.grid(shoebox[l].size()), 0.0) random_background_plane2(background, Ba[l], Bb[l], Bc[l], Bd[l]) shoebox[l].data += background shoebox[l].background = background if l % m == 0: progress.update(100.0 * float(l) / len(refl)) progress.update(100.0 * float(l) / len(refl)) progress.finished("Calculated background for %d reflections" % len(refl)) ## Calculate the expected intensity by monte-carlo integration # progress = ProgressBar(title='Integrating expected signal for %d reflections' % len(refl)) # s1 = refl['s1'] # phi = refl['xyzcal.mm'].parts()[2] # bbox = refl['bbox'] # shoebox = refl['shoebox'] # I_exp = flex.double(len(refl), 0) # m = int(len(refl) / 100) # for i in range(len(refl)): # if I[i] > 0: # I_exp[i] = integrate_reciprocal_space_gaussian( # self.experiment.beam, # self.experiment.detector, # self.experiment.goniometer, # self.experiment.scan, # self.sigma_b, # self.sigma_m, # s1[i], # phi[i], # bbox[i], # 10000, # shoebox[i].mask) / 10000.0 # if i % m == 0: # progress.update(100.0 * float(i) / len(refl)) # progress.finished('Integrated expected signal impacts for %d reflections' % len(refl)) # Save the expected intensity and background refl["intensity.sim"] = I refl["background.sim.a"] = Ba refl["background.sim.b"] = Bb refl["background.sim.c"] = Bc refl["background.sim.d"] = Bd refl["intensity.exp"] = I_exp # Return the reflections return refl
int(math.floor(min(z))), int(math.floor(max(z)) + 1)) try: counts = counts_database[hkl.miller_index] useful.append(hkl) except KeyError, e: continue from dials.algorithms import shoebox shoebox.allocate(useful) from dials.util.command_line import ProgressBar p = ProgressBar(title = 'Generating shoeboxes') # now for each reflection perform the simulation for j, refl in enumerate(useful): p.update(j * 100.0 / len(useful)) d = d_matrices[j] from scitbx.random import variate, normal_distribution g = variate(normal_distribution(mean = 0, sigma = node_size)) counts = counts_database[refl.miller_index] dhs = g(counts) dks = g(counts) dls = g(counts) self.map_to_image_space(refl, d, dhs, dks, dls) p.finished('Generated %d shoeboxes' % len(useful)) # now for each reflection add background from dials.algorithms.simulation.generate_test_reflections import \ random_background_plane
def simple_gaussian_spots(params): from dials.array_family import flex from dials.algorithms import shoebox import random import math from scitbx import matrix r = params.rotation axis = matrix.col((r.axis.x, r.axis.y, r.axis.z)) if axis.length() > 0: rotation = axis.axis_and_angle_as_r3_rotation_matrix(r.angle, deg=True) else: rotation = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1)) # generate mask and peak values from dials.algorithms.shoebox import MaskCode mask_peak = MaskCode.Valid|MaskCode.Foreground mask_back = MaskCode.Valid|MaskCode.Background from dials.util.command_line import ProgressBar p = ProgressBar(title = 'Generating reflections') rlist = flex.reflection_table(params.nrefl) hkl = flex.miller_index(params.nrefl) s1 = flex.vec3_double(params.nrefl) xyzmm = flex.vec3_double(params.nrefl) xyzpx = flex.vec3_double(params.nrefl) panel = flex.size_t(params.nrefl) bbox = flex.int6(params.nrefl) for j in range(params.nrefl): p.update(j * 100.0 / params.nrefl) hkl[j] = (random.randint(0, 20), random.randint(0, 20), random.randint(0, 20)) phi = 2 * math.pi * random.random() s1[j] = (0, 0, 0) xyzpx[j] = (0, 0, 0) xyzmm[j] = (0, 0, phi) panel[j] = 0 bbox[j] = (0, params.shoebox_size.x, 0, params.shoebox_size.y, 0, params.shoebox_size.z) p.finished('Generating %d reflections' % params.nrefl) intensity = flex.double(params.nrefl) shoebox = flex.shoebox(panel, bbox) shoebox.allocate_with_value(MaskCode.Valid) p = ProgressBar(title = 'Generating shoeboxes') for i in range(len(rlist)): p.update(i * 100.0 / params.nrefl) mask = shoebox[i].mask if params.pixel_mask == 'precise': # flag everything as background: peak will me assigned later for j in range(len(mask)): mask[j] = mask_back elif params.pixel_mask == 'all': # flag we have no idea what anything is mask_none = MaskCode.Valid|MaskCode.Foreground|MaskCode.Background for j in range(len(mask)): mask[j] = mask_none elif params.pixel_mask == 'static': import itertools from scitbx.array_family import flex x0 = params.spot_offset.x + params.shoebox_size.x / 2 y0 = params.spot_offset.x + params.shoebox_size.y / 2 z0 = params.spot_offset.x + params.shoebox_size.z / 2 sx = params.mask_nsigma * params.spot_size.x sy = params.mask_nsigma * params.spot_size.y sz = params.mask_nsigma * params.spot_size.z # The x, y, z indices z, y, x = zip(*itertools.product(*(range(n) for n in mask.all()))) xyz = flex.vec3_double(flex.double(x), flex.double(y), flex.double(z)) # Calculate SUM(((xj - xj0) / sxj)**2) for each element xyz0 = (x0, y0, z0) isxyz = (1.0/sx, 1.0/sy, 1.0/sz) dxyz = sum([(x * isx)**2 for x, isx in zip(((xyz - xyz0) * rotation).parts(), isxyz)]) # Set the mask values index = dxyz <= 1.0 index.reshape(mask.accessor()) mask.set_selected(index, MaskCode.Valid | MaskCode.Foreground) mask.set_selected(index != True, MaskCode.Valid | MaskCode.Background) sbox = shoebox[i].data # reflection itself, including setting the peak region if we're doing that # FIXME use flex arrays to make the rotation bit more efficient as this is # now rather slow... counts_true = 0 for j in range(params.counts): _x = random.gauss(0, params.spot_size.x) _y = random.gauss(0, params.spot_size.y) _z = random.gauss(0, params.spot_size.z) Rxyz = rotation * matrix.col((_x, _y, _z)).elems x = int(Rxyz[0] + params.spot_offset.x + params.shoebox_size.x / 2) y = int(Rxyz[1] + params.spot_offset.y + params.shoebox_size.y / 2) z = int(Rxyz[2] + params.spot_offset.z + params.shoebox_size.z / 2) if x < 0 or x >= params.shoebox_size.x: continue if y < 0 or y >= params.shoebox_size.y: continue if z < 0 or z >= params.shoebox_size.z: continue sbox[z, y, x] += 1 counts_true += 1 if params.pixel_mask == 'precise': mask[z, y, x] = mask_peak intensity[i] = counts_true if params.background: # background:flat; for j in range(params.background * len(sbox)): x = random.randint(0, params.shoebox_size.x - 1) y = random.randint(0, params.shoebox_size.y - 1) z = random.randint(0, params.shoebox_size.z - 1) sbox[z, y, x] += 1 else: # or inclined random_background_plane(sbox, params.background_a, params.background_b, params.background_c, params.background_d) rlist['miller_index'] = hkl rlist['s1'] = s1 rlist['xyzcal.px'] = xyzpx rlist['xyzcal.mm'] = xyzmm rlist['bbox'] = bbox rlist['panel'] = panel rlist['shoebox'] = shoebox rlist['intensity.sum.value'] = intensity p.finished('Generating %d shoeboxes' % params.nrefl) return rlist
def run(args): from dials.util.options import OptionParser from dials.util.options import flatten_experiments usage = "%s [options] data_master.h5" % (libtbx.env.dispatcher_name) parser = OptionParser( usage=usage, phil=phil_scope, read_experiments=True, read_experiments_from_images=True, epilog=help_message, ) params, options = parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) if len(experiments) != 1: parser.print_help() print("Please pass an experiment list\n") return imagesets = experiments.imagesets() if len(imagesets) != 1: raise Sorry( "Please pass an experiment list that contains one imageset") imageset = imagesets[0] first, last = imageset.get_scan().get_image_range() images = range(first, last + 1) if not params.images and params.image_range: params.images = list( range(params.image_range[0], params.image_range[1] + 1)) if params.images: if min(params.images) < first or max(params.images) > last: raise Sorry("image outside of scan range") images = params.images detectors = imageset.get_detector() assert len(detectors) == 1 detector = detectors[0] trusted = detector.get_trusted_range() from dials.util.command_line import ProgressBar p = ProgressBar(title="Counting events") events_per_image = {} for idx in images: p.update(idx * 100.0 / len(images)) pixels = imageset.get_raw_data(idx - 1) assert len(pixels) == 1 mask = ~imageset.get_mask(idx)[0].as_1d() data = pixels[0].as_1d() data.set_selected(mask, 0) hot = data >= int(round(trusted[1])) data.set_selected(hot, 0) events_per_image[idx] = flex.sum(data) for idx in images: print(idx, events_per_image[idx])
def run(args): from dials.util.options import OptionParser from dials.util.options import flatten_experiments from dials.util.command_line import ProgressBar usage = "%s [options] data_master.h5" % (libtbx.env.dispatcher_name) parser = OptionParser( usage=usage, phil=phil_scope, read_experiments=True, read_experiments_from_images=True, epilog=help_message, ) params, options = parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) if len(experiments) != 1: parser.print_help() print("Please pass an experiment list\n") return imagesets = experiments.imagesets() if len(imagesets) != 1: raise Sorry("Please pass an experiment list that contains one imageset") imageset = imagesets[0] first, last = imageset.get_scan().get_image_range() images = range(first, last + 1) if params.images is None and params.image_range is not None: start, end = params.image_range params.images = list(range(start, end + 1)) if params.images: if min(params.images) < first or max(params.images) > last: raise Sorry("image outside of scan range") images = params.images detectors = imageset.get_detector() assert len(detectors) == 1 detector = detectors[0] trusted = detector.get_trusted_range() # construct an integer array same shape as image; accumulate number of # "signal" pixels in each pixel across data total = None p = ProgressBar(title="Finding hot pixels") for idx in images: p.update(idx * 100.0 / len(images)) pixels = imageset.get_raw_data(idx - 1) assert len(pixels) == 1 data = pixels[0] negative = data < int(round(trusted[0])) hot = data > int(round(trusted[1])) bad = negative | hot data = data.as_double() spot_params = spot_phil.fetch( source=iotbx.phil.parse("min_spot_size=1") ).extract() threshold_function = SpotFinderFactory.configure_threshold(spot_params) peak_pixels = threshold_function.compute_threshold(data, ~bad) if total is None: total = peak_pixels.as_1d().as_int() else: total += peak_pixels.as_1d().as_int() p.finished("Finished finding hot pixels on %d images" % len(images)) hot_mask = total >= (len(images) // 2) hot_pixels = hot_mask.iselection() p = ProgressBar(title="Finding twinkies") twinkies = {} for h in hot_pixels: twinkies[h] = [] for idx in images: p.update(idx * 100.0 / len(images)) pixels = imageset.get_raw_data(idx - 1) data = pixels[0] for h in hot_pixels: twinkies[h].append(data[h]) p.finished("Finished hunting for twinkies on %d images" % len(images)) nslow, nfast = data.focus() ffff = 0 for h in hot_pixels: if total[h] == len(images) and data[h] >= trusted[1]: ffff += 1 continue print("Pixel %d at %d %d" % (total[h], h // nfast, h % nfast)) if len(set(twinkies[h])) >= len(twinkies[h]) // 2: print(" ... many possible values") continue values = set(twinkies[h]) result = [(twinkies[h].count(value), value) for value in values] for count, value in reversed(sorted(result)): print(" %08x %d" % (value, count)) print("Also found %d very hot pixels" % ffff) hot_mask.reshape(flex.grid(data.focus())) easy_pickle.dump(params.output.mask, (~hot_mask,))
def with_individual_given_intensity(self, N, In, Ba, Bb, Bc, Bd): """Generate reflections with given intensity and background.""" from dials.algorithms.simulation import simulate_reciprocal_space_gaussian from dials.algorithms.simulation.generate_test_reflections import ( random_background_plane2, ) from dials.util.command_line import ProgressBar # Check the lengths assert N == len(In) assert N == len(Ba) assert N == len(Bb) assert N == len(Bc) assert N == len(Bd) # Generate some predictions refl = self.generate_predictions(N) # Calculate the signal progress = ProgressBar( title=f"Calculating signal for {len(refl)} reflections") s1 = refl["s1"] phi = refl["xyzcal.mm"].parts()[2] bbox = refl["bbox"] shoebox = refl["shoebox"] m = int(len(refl) / 100) I_exp = flex.double(len(refl), 0) for i in range(len(refl)): if In[i] > 0: data = shoebox[i].data.as_double() I_exp[i] = simulate_reciprocal_space_gaussian( self.experiment.beam, self.experiment.detector, self.experiment.goniometer, self.experiment.scan, self.sigma_b, self.sigma_m, s1[i], phi[i], bbox[i], In[i], data, shoebox[i].mask, ) shoebox[i].data = data.as_float() if i % m == 0: progress.update(100.0 * float(i) / len(refl)) progress.finished( f"Calculated signal impacts for {len(refl)} reflections") # Calculate the background progress = ProgressBar( title=f"Calculating background for {len(refl)} reflections") for l in range(len(refl)): background = flex.float(flex.grid(shoebox[l].size()), 0.0) random_background_plane2(background, Ba[l], Bb[l], Bc[l], Bd[l]) shoebox[l].data += background shoebox[l].background = background if l % m == 0: progress.update(100.0 * float(l) / len(refl)) progress.update(100.0 * float(l) / len(refl)) progress.finished(f"Calculated background for {len(refl)} reflections") ## Calculate the expected intensity by monte-carlo integration # progress = ProgressBar(title='Integrating expected signal for %d reflections' % len(refl)) # s1 = refl['s1'] # phi = refl['xyzcal.mm'].parts()[2] # bbox = refl['bbox'] # shoebox = refl['shoebox'] # I_exp = flex.double(len(refl), 0) # m = int(len(refl) / 100) # for i in range(len(refl)): # if In[i] > 0: # I_exp[i] = integrate_reciprocal_space_gaussian( # self.experiment.beam, # self.experiment.detector, # self.experiment.goniometer, # self.experiment.scan, # self.sigma_b, # self.sigma_m, # s1[i], # phi[i], # bbox[i], # 10000, # shoebox[i].mask) / 10000.0 # if i % m == 0: # progress.update(100.0 * float(i) / len(refl)) # progress.finished('Integrated expected signal impacts for %d reflections' % len(refl)) # Save the expected intensity and background refl["intensity.sim"] = In refl["background.sim.a"] = Ba refl["background.sim.b"] = Bb refl["background.sim.c"] = Bc refl["background.sim.d"] = Bd refl["intensity.exp"] = I_exp # Return the reflections return refl
def main(self): # FIXME import simulation code import six.moves.cPickle as pickle import math from dials.util.command_line import Importer from dials.algorithms.integration import ReflectionPredictor from libtbx.utils import Sorry # Parse the command line params, options, args = self.parser.parse_args() importer = Importer(args) if len(importer.imagesets) == 0 and len(importer.crystals) == 0: self.config().print_help() return if len(importer.imagesets) != 1: raise Sorry('need 1 sweep: %d given' % len(importer.imagesets)) if len(importer.crystals) != 1: raise Sorry('need 1 crystal: %d given' % len(importer.crystals)) sweep = importer.imagesets[0] crystal = importer.crystals[0] # generate predictions for possible reflections => generate a # reflection list predict = ReflectionPredictor() predicted = predict(sweep, crystal) # sort with James's reflection table: should this not go somewhere central? from dials.scratch.jmp.container.reflection_table import ReflectionTable # calculate shoebox sizes: take parameters from params & transform # from reciprocal space to image space to decide how big a shoe box to use table = ReflectionTable() table['miller_index'] = predicted.miller_index() indexer = table.index_map('miller_index') candidates = [] unique = sorted(indexer) for h, k, l in unique: try: for _h in h - 1, h + 1: if not indexer[(_h, k, l)]: raise ValueError('missing') for _k in k - 1, k + 1: if not indexer[(h, _k, l)]: raise ValueError('missing') for _l in l - 1, l + 1: if not indexer[(h, k, _l)]: raise ValueError('missing') candidates.append((h, k, l)) except ValueError: continue from dials.algorithms.simulation.utils import build_prediction_matrix from dials.algorithms.simulation.generate_test_reflections import \ master_phil from libtbx.phil import command_line cmd = command_line.argument_interpreter(master_params=master_phil) working_phil = cmd.process_and_fetch(args=args[2:]) params = working_phil.extract() node_size = params.rs_node_size window_size = params.rs_window_size reference = params.integrated_data_file scale = params.integrated_data_file_scale if reference: counts_database = {} from iotbx import mtz m = mtz.object(reference) mi = m.extract_miller_indices() i = m.extract_reals('IMEAN').data s = m.space_group().build_derived_point_group() for j in range(len(mi)): for op in s.all_ops(): hkl = tuple(map(int, op * mi[j])) counts = max(0, int(math.floor(i[j] * scale))) counts_database[hkl] = counts counts_database[(-hkl[0], -hkl[1], -hkl[2])] = counts else: def constant_factory(value): import itertools return itertools.repeat(value).next from collections import defaultdict counts_database = defaultdict(constant_factory(params.counts)) from dials.model.data import ReflectionList useful = ReflectionList() d_matrices = [] for h, k, l in candidates: hkl = predicted[indexer[(h, k, l)][0]] _x = hkl.image_coord_px[0] _y = hkl.image_coord_px[1] _z = hkl.frame_number # build prediction matrix mhkl = predicted[indexer[(h - 1, k, l)][0]] phkl = predicted[indexer[(h + 1, k, l)][0]] hmkl = predicted[indexer[(h, k - 1, l)][0]] hpkl = predicted[indexer[(h, k + 1, l)][0]] hkml = predicted[indexer[(h, k, l - 1)][0]] hkpl = predicted[indexer[(h, k, l + 1)][0]] d = build_prediction_matrix(hkl, mhkl, phkl, hmkl, hpkl, hkml, hkpl) d_matrices.append(d) # construct the shoebox parameters: outline the ellipsoid x, y, z = [], [], [] for dh in (1, 0, 0), (0, 1, 0), (0, 0, 1): dxyz = -1 * window_size * d * dh x.append(dxyz[0] + _x) y.append(dxyz[1] + _y) z.append(dxyz[2] + _z) dxyz = window_size * d * dh x.append(dxyz[0] + _x) y.append(dxyz[1] + _y) z.append(dxyz[2] + _z) hkl.bounding_box = (int(math.floor(min(x))), int(math.floor(max(x)) + 1), int(math.floor(min(y))), int(math.floor(max(y)) + 1), int(math.floor(min(z))), int(math.floor(max(z)) + 1)) try: counts = counts_database[hkl.miller_index] useful.append(hkl) except KeyError: continue from dials.algorithms import shoebox shoebox.allocate(useful) from dials.util.command_line import ProgressBar p = ProgressBar(title='Generating shoeboxes') # now for each reflection perform the simulation for j, refl in enumerate(useful): p.update(j * 100.0 / len(useful)) d = d_matrices[j] from scitbx.random import variate, normal_distribution g = variate(normal_distribution(mean=0, sigma=node_size)) counts = counts_database[refl.miller_index] dhs = g(counts) dks = g(counts) dls = g(counts) self.map_to_image_space(refl, d, dhs, dks, dls) p.finished('Generated %d shoeboxes' % len(useful)) # now for each reflection add background from dials.algorithms.simulation.generate_test_reflections import \ random_background_plane p = ProgressBar(title='Generating background') for j, refl in enumerate(useful): p.update(j * 100.0 / len(useful)) if params.background: random_background_plane(refl.shoebox, params.background, 0.0, 0.0, 0.0) else: random_background_plane(refl.shoebox, params.background_a, params.background_b, params.background_c, params.background_d) p.finished('Generated %d backgrounds' % len(useful)) if params.output.all: with open(params.output.all, 'wb') as fh: pickle.dump(useful, fh, pickle.HIGHEST_PROTOCOL)