Ejemplo n.º 1
0
  def tst_count_mask_values(self):
    from dials.model.data import Shoebox
    from random import randint, sample
    from dials.array_family import flex

    shoebox = flex.shoebox(10)
    num = flex.int(10)
    value = (1 << 2)
    for i in range(10):
      x0 = randint(0, 90)
      y0 = randint(0, 90)
      z0 = randint(0, 90)
      x1 = randint(1, 10) + x0
      y1 = randint(1, 10) + y0
      z1 = randint(1, 10) + z0

      shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))
      shoebox[i].allocate()
      maxnum = len(shoebox[i].mask)
      num[i] = randint(1, maxnum)
      indices = sample(list(range(maxnum)), num[i])
      for j in indices:
        shoebox[i].mask[j] = value

    assert(shoebox.count_mask_values(value) == num)

    # Test passed
    print 'OK'
Ejemplo n.º 2
0
  def __init__(self, data, jobs, npanels, callback):
    from dials.array_family import flex

    # Allocate shoeboxes
    data["shoebox"] = flex.shoebox(data["panel"], data["bbox"])
    data["shoebox"].allocate()

    # Set the callback
    self._callback = callback

    # Initialise the base class
    super(IntegrationTask3DMultiExecutor, self).__init__(data, jobs, npanels)
Ejemplo n.º 3
0
  def run(self):
    from dials.model.serialize import load
    from dials.algorithms import shoebox
    from dxtbx.serialize.load import crystal as load_crystal
    from dials.array_family import flex

    # Load the sweep and crystal
    self.sweep = load.sweep(self.sweep_filename)
    self.crystal = load_crystal(self.crystal_filename)

    # Get the reflections and overlaps
    reflections, adjacency_list = self.predict_reflections()
    reflections['shoebox'] = flex.shoebox(
      reflections['panel'],
      reflections['bbox'])
    reflections['shoebox'].allocate_with_value(shoebox.MaskCode.Valid)

    # If the adjacency list is given, then create the reflection mask
    assert(len(self.detector) == 1)
    image_size = self.detector[0].get_image_size()
    shoeboxes = reflections['shoebox']
    coords = reflections['xyzcal.px']
    shoebox_masker = shoebox.MaskOverlapping()
    shoebox_masker(shoeboxes, coords, adjacency_list)

    # Loop through all edges
    overlapping = []
    for e in adjacency_list.edges():
      v1, v2 = adjacency_list.source(e), adjacency_list.target(e)
      overlapping.append(v1)
      overlapping.append(v2)

    # Ensure elements are unique
    overlapping = set(overlapping)

    # Ensure we have some overlaps
    assert(len(overlapping) > 0)

    # Get all non-overlapping reflections
    all_r = set(range(len(reflections)))
    non_overlapping = all_r.difference(overlapping)

    # Run the tests
    self.tst_non_overlapping(reflections, non_overlapping,
        self.detector[0].get_image_size())
    self.tst_overlapping(reflections, overlapping, adjacency_list,
        image_size)
Ejemplo n.º 4
0
  def __call__(self, imageset, pixel_labeller):
    '''
    Convert the pixel list to shoeboxes

    '''
    from dxtbx.imageset import ImageSweep
    from dials.array_family import flex

    # Extract the pixel lists into a list of reflections
    shoeboxes = flex.shoebox()
    spotsizes = flex.size_t()
    hotpixels = tuple(flex.size_t() for i in range(len(imageset.get_detector())))
    if isinstance(imageset, ImageSweep):
      twod = False
    else:
      twod = True
    for i, (p, hp) in enumerate(zip(pixel_labeller, hotpixels)):
      if p.num_pixels() > 0:
        creator = flex.PixelListShoeboxCreator(
            p,
            i,                   # panel
            0,                   # zrange
            twod,                # twod
            self.min_spot_size,  # min_pixels
            self.max_spot_size,  # max_pixels
            self.write_hot_pixel_mask)
        shoeboxes.extend(creator.result())
        spotsizes.extend(creator.spot_size())
        hp.extend(creator.hot_pixels())
    logger.info('')
    logger.info('Extracted {0} spots'.format(len(shoeboxes)))

    # Get the unallocated spots and print some info
    selection = shoeboxes.is_allocated()
    shoeboxes = shoeboxes.select(selection)
    ntoosmall = (spotsizes < self.min_spot_size).count(True)
    ntoolarge = (spotsizes > self.max_spot_size).count(True)
    assert ntoosmall + ntoolarge == selection.count(False)
    logger.info('Removed %d spots with size < %d pixels' % (
      ntoosmall,
      self.min_spot_size))
    logger.info('Removed %d spots with size > %d pixels' % (
      ntoolarge,
      self.max_spot_size))

    # Return the shoeboxes
    return shoeboxes, hotpixels
Ejemplo n.º 5
0
  def tst_is_bbox_within_image_volume(self):
    from dials.model.data import Shoebox
    from dials.array_family import flex

    isize = (1000, 1000)
    srange = (0, 100)

    shoebox = flex.shoebox(7)
    shoebox[0] = Shoebox((10, 20, 10, 20, 10, 20))
    shoebox[1] = Shoebox((-10, 20, 10, 20, 10, 20))
    shoebox[2] = Shoebox((10, 20, -10, 20, 10, 20))
    shoebox[3] = Shoebox((10, 20, 10, 20, -10, 20))
    shoebox[4] = Shoebox((10, 1020, 10, 20, 10, 20))
    shoebox[5] = Shoebox((10, 20, 10, 1020, 10, 20))
    shoebox[6] = Shoebox((10, 20, 10, 20, 10, 1020))

    assert(shoebox.is_bbox_within_image_volume(isize, srange) ==
      flex.bool([True, False, False, False, False, False, False]))

    # Test passed
    print 'OK'
Ejemplo n.º 6
0
  def tst_does_bbox_contain_bad_pixels(self):
    from dials.array_family import flex
    from dials.model.data import Shoebox
    from random import randint

    mask = flex.bool(flex.grid(100, 100), True)
    for j in range(100):
      for i in range(40, 60):
        mask[j,i] = False
        mask[i,j] = False

    shoebox = flex.shoebox(1000)
    res = flex.bool(1000)
    for i in range(1000):
      x0 = randint(0, 90)
      y0 = randint(0, 90)
      z0 = randint(0, 90)
      x1 = randint(1, 10) + x0
      y1 = randint(1, 10) + y0
      z1 = randint(1, 10) + z0

      shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))

      res2 = False
      if x0 >= 40 and x0 < 60:
        res2 = True
      if x1 > 40 and x1 <= 60:
        res2 = True
      if y0 >= 40 and y0 < 60:
        res2 = True
      if y1 > 40 and y1 <= 60:
        res2 = True

      res[i] = res2

    assert(shoebox.does_bbox_contain_bad_pixels(mask) == res)

    # Test passed
    print 'OK'
Ejemplo n.º 7
0
  def __init__(self):
    from dials.algorithms.integration.profile import GridSampler2D
    from dials.array_family import flex
    from random import randint, uniform

    # Number of reflections
    nrefl = 1000

    # Size of the images
    width = 1000
    height = 1000

    # Create the grid
    self.grid = GridSampler2D((width, height), (5,5))

    # Create the list of xyz and bboxes
    self.xyz = flex.vec3_double(nrefl)
    self.bbox = flex.int6(nrefl)
    self.panel = flex.size_t(nrefl, 0)
    for i in range(nrefl):
      x0 = randint(0,width-10)
      x1 = x0 + randint(3,10)
      y0 = randint(0,height-10)
      y1 = y0 + randint(3,10)
      z0 = randint(0,10)
      z1 = z0 + randint(1,10)
      b = x0, x1, y0, y1, z0, z1
      c = (x1 + x0) / 2, (y1 + y0) / 2, (z1 + z0) / 2
      self.xyz[i] = c
      self.bbox[i] = b

    # Create the array of shoeboxes
    self.sbox = flex.shoebox(self.panel, self.bbox)
    self.sbox.allocate()
    for i in range(len(self.sbox)):
      data = self.sbox[i].data
      for j in range(len(data)):
        data[j] = uniform(0, 100)
Ejemplo n.º 8
0
  def tst_bounding_boxes(self):
    from dials.model.data import Shoebox
    from random import randint
    from dials.array_family import flex

    shoebox = flex.shoebox(10)
    bbox = flex.int6(10)
    for i in range(10):
      x0 = randint(0, 90)
      y0 = randint(0, 90)
      z0 = randint(0, 90)
      x1 = randint(1, 10) + x0
      y1 = randint(1, 10) + y0
      z1 = randint(1, 10) + z0
      bbox[i] = (x0, x1, y0, y1, z0, z1)
      shoebox[i] = Shoebox(bbox[i])

    bbox2 = shoebox.bounding_boxes()
    for i in range(10):
      assert(bbox2[i] == bbox[i])

    # Test passed
    print 'OK'
Ejemplo n.º 9
0
  def tst_consistent(self):

    from random import randint
    from dials.model.data import Shoebox
    from dials.array_family import flex

    shoebox = flex.shoebox(10)

    for i in range(10):
      x0 = randint(0, 1000)
      y0 = randint(0, 1000)
      z0 = randint(0, 1000)
      x1 = randint(1, 10) + x0
      y1 = randint(1, 10) + y0
      z1 = randint(1, 10) + z0
      shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))

    assert(shoebox.is_consistent() == flex.bool(10, False))

    for i in range(10):
      shoebox[i].allocate()

    assert(shoebox.is_consistent() == flex.bool(10, True))

    for i in [0, 2, 4, 6, 8]:
      shoebox[i].data.resize(flex.grid(10, 10, 10))

    assert(shoebox.is_consistent() == flex.bool([False, True] * 5))

    for i in range(10):
      shoebox[i].deallocate()

    assert(shoebox.is_consistent() == flex.bool(10, False))

    # Test passed
    print 'OK'
Ejemplo n.º 10
0
  def __call__(self):
    '''
    Do the processing.

    :return: The processed data

    '''
    from dials.array_family import flex
    from time import time
    from dials.model.data import make_image
    from libtbx.introspection import machine_memory_info

    # Get the start time
    start_time = time()

    # Set the global process ID
    job.index = self.index

    # Check all reflections have same imageset and get it
    exp_id = list(set(self.reflections['id']))
    imageset = self.experiments[exp_id[0]].imageset
    for i in exp_id[1:]:
      assert self.experiments[i].imageset == imageset, "Task can only handle 1 imageset"

    # Get the sub imageset
    frame00, frame01 = self.job
    try:
      frame10, frame11 = imageset.get_array_range()
    except Exception:
      frame10, frame11 = (0, len(imageset))
    try:
      assert frame00 < frame01
      assert frame10 < frame11
      assert frame00 >= frame10
      assert frame01 <= frame11
      index0 = frame00 - frame10
      index1 = index0 + (frame01 - frame00)
      assert index0 < index1
      assert index0 >= 0
      assert index1 <= len(imageset)
      imageset = imageset[index0:index1]
    except Exception:
      raise RuntimeError('Programmer Error: bad array range')
    try:
      frame0, frame1 = imageset.get_array_range()
    except Exception:
      frame0, frame1 = (0, len(imageset))

    # Initlize the executor
    self.executor.initialize(frame0, frame1, self.reflections)

    # Set the shoeboxes (dont't allocate)
    self.reflections['shoebox'] = flex.shoebox(
      self.reflections['panel'],
      self.reflections['bbox'],
      allocate=False,
      flatten=self.params.shoebox.flatten)

    # Create the processor
    processor = ShoeboxProcessor(
      self.reflections,
      len(imageset.get_detector()),
      frame0,
      frame1,
      self.params.debug.output)

    # Compute percentage of max available. The function is not portable to
    # windows so need to add a check if the function fails. On windows no
    # warning will be printed
    memory_info = machine_memory_info()
    total_memory = memory_info.memory_total()
    sbox_memory = processor.compute_max_memory_usage()
    if total_memory is not None:
      assert total_memory > 0, "Your system appears to have no memory!"
      assert self.params.block.max_memory_usage >  0.0, "maximum memory usage must be > 0"
      assert self.params.block.max_memory_usage <= 1.0, "maximum memory usage must be <= 1"
      limit_memory = total_memory * self.params.block.max_memory_usage
      if sbox_memory > limit_memory:
        raise RuntimeError('''
        There was a problem allocating memory for shoeboxes. Possible solutions
        include increasing the percentage of memory allowed for shoeboxes or
        decreasing the block size. This could also be caused by a highly mosaic
        crystal model - is your crystal really this mosaic?
          Total system memory: %g GB
          Limit shoebox memory: %g GB
          Required shoebox memory: %g GB
        ''' % (total_memory/1e9, limit_memory/1e9, sbox_memory/1e9))
      else:
        logger.info(' Memory usage:')
        logger.info('  Total system memory: %g GB' % (total_memory/1e9))
        logger.info('  Limit shoebox memory: %g GB' % (limit_memory/1e9))
        logger.info('  Required shoebox memory: %g GB' % (sbox_memory/1e9))
        logger.info('')

    # Loop through the imageset, extract pixels and process reflections
    read_time = 0.0
    for i in range(len(imageset)):
      st = time()
      image = imageset.get_corrected_data(i)
      mask = imageset.get_mask(i)
      if self.params.lookup.mask is not None:
        assert len(mask) == len(self.params.lookup.mask), \
          "Mask/Image are incorrect size %d %d" % (
            len(mask),
            len(self.params.lookup.mask))
        mask = tuple(m1 & m2 for m1, m2 in zip(self.params.lookup.mask, mask))

      read_time += time() - st
      processor.next(make_image(image, mask), self.executor)
      del image
      del mask
    assert processor.finished(), "Data processor is not finished"

    # Optionally save the shoeboxes
    if self.params.debug.output and self.params.debug.separate_files:
      output = self.reflections
      if self.params.debug.select is not None:
        output = output.select(self.params.debug.select(output))
      if self.params.debug.split_experiments:
        output = output.split_by_experiment_id()
        for table in output:
          i = table['id'][0]
          table.as_pickle('shoeboxes_%d_%d.pickle' % (self.index, i))
      else:
        output.as_pickle('shoeboxes_%d.pickle' % self.index)

    # Delete the shoeboxes
    if self.params.debug.separate_files or not self.params.debug.output:
      del self.reflections['shoebox']

    # Finalize the executor
    self.executor.finalize()

    # Return the result
    result = Result(self.index, self.reflections, self.executor.data())
    result.read_time = read_time
    result.extract_time = processor.extract_time()
    result.process_time = processor.process_time()
    result.total_time = time() - start_time
    return result
Ejemplo n.º 11
0
      reflection_lookup[j].append(i)

  width, height = experiments[0].detector[0].get_image_size()
  sum_background = flex.double(flex.grid(height, width), 0)
  sum_sq_background = flex.double(flex.grid(height, width), 0)
  count = flex.int(flex.grid(height, width), 0)

  # Loop through all images
  print "START"
  for frame in range(len(imageset)):

    # Get the subset of reflections on this image and compute the mask
    subset = reflections.select(flex.size_t(reflection_lookup[frame]))

    subset['shoebox'] = flex.shoebox(
      subset['panel'],
      subset['bbox'],
      allocate=True)

    subset.compute_mask(experiments)

    # Get the mask and data
    mask = imageset.get_mask(frame)[0]
    data = imageset.get_raw_data(frame)[0]

    sbox_mask = subset['shoebox'].apply_background_mask(frame, 1, (height, width))

    mask = mask & sbox_mask

    #from dials.algorithms.image.threshold import DispersionThreshold
    #threshold = DispersionThreshold(
    #  data.all(),
Ejemplo n.º 12
0
  def tst_extract_shoeboxes(self):
    from dials.array_family import flex
    from random import randint, seed
    from dials.algorithms.shoebox import MaskCode
    import sys
    seed(0)

    reflections = flex.reflection_table()
    reflections['panel'] = flex.size_t()
    reflections['bbox'] = flex.int6()

    npanels = 2
    width = 1000
    height = 1000
    nrefl = 10000
    frame0 = 10
    frame1 = 100
    nrefl = 1000

    for i in range(nrefl):
      xs = randint(5, 10)
      ys = randint(5, 10)
      x0 = randint(-xs+1, width-1)
      y0 = randint(-ys+1, height-1)
      z0 = randint(frame0, frame1-1)
      x1 = x0 + xs
      y1 = y0 + ys
      z1 = min([z0 + randint(1, 10), frame1])
      assert(x1 > x0)
      assert(y1 > y0)
      assert(z1 > z0)
      assert(z0 >= frame0 and z1 <= frame1)
      bbox = (x0, x1, y0, y1, z0, z1)
      reflections.append({
        "panel" : randint(0,1),
        "bbox" : bbox,
      })

    reflections['shoebox'] = flex.shoebox(
      reflections['panel'],
      reflections['bbox'])
    reflections['shoebox'].allocate()

    class FakeImageSet(object):
      def __init__(self):
        from dials.array_family import flex
        self.data = flex.int(range(height*width))
        self.data.reshape(flex.grid(height,width))
      def get_array_range(self):
        return (frame0, frame1)
      def get_detector(self):
        class FakeDetector(object):
          def __len__(self):
            return npanels
          def __getitem__(self, index):
            class FakePanel(object):
              def get_trusted_range(self):
                return (-1, 1000000)
            return FakePanel()
        return FakeDetector()
      def __len__(self):
        return frame1 - frame0
      def __getitem__(self, index):
        f = frame0+index
        return (self.data + f*1, self.data + f*2)
      def get_corrected_data(self, index):
        f = frame0+index
        return (self.data + f*1, self.data + f*2)
      def get_mask(self, index):
        image = self.get_corrected_data(index)
        return tuple(im >= 0 for im in image)
    imageset = FakeImageSet()

    stdout = sys.stdout
    class DevNull(object):
      def write(self, *args):
        pass
      def flush(self):
        pass
    sys.stdout = DevNull()
    reflections.extract_shoeboxes(imageset)
    sys.stdout = stdout

    for i in range(len(reflections)):
      sbox = reflections[i]["shoebox"]
      assert(sbox.is_consistent())
      mask = sbox.mask
      data = sbox.data
      bbox = sbox.bbox
      panel = sbox.panel
      x0, x1, y0, y1, z0, z1 = bbox
      for z in range(z1 - z0):
        for y in range(y1 - y0):
          for x in range(x1 - x0):
            v1 = data[z,y,x]
            m1 = mask[z,y,x]
            if (x0 + x >= 0 and y0 + y >= 0 and
                x0 + x < width and y0 + y < height):
              v2 = imageset.data[y+y0,x+x0] + (z+z0)*(panel+1)
              m2 = MaskCode.Valid
              assert(v1 == v2)
              assert(m1 == m2)
            else:
              assert(v1 == 0)
              assert(m1 == 0)

    print 'OK'
Ejemplo n.º 13
0
def reconstruct_rogues(params):
  assert os.path.exists('xia2.json')
  from xia2.Schema.XProject import XProject
  xinfo = XProject.from_json(filename='xia2.json')

  from dxtbx.model.experiment.experiment_list import ExperimentListFactory
  import cPickle as pickle
  import dials # because WARNING:root:No profile class gaussian_rs registered
  crystals = xinfo.get_crystals()
  assert len(crystals) == 1

  for xname in crystals:
    crystal = crystals[xname]

  scaler = crystal._get_scaler()

  epochs = scaler._sweep_handler.get_epochs()

  rogues = os.path.join(scaler.get_working_directory(),
                        xname, 'scale', 'ROGUES')

  rogue_reflections = munch_rogues(rogues)

  batched_reflections = { }

  for epoch in epochs:
    si = scaler._sweep_handler.get_sweep_information(epoch)
    intgr = si.get_integrater()
    experiments = ExperimentListFactory.from_json_file(
      intgr.get_integrated_experiments())
    reflections = pickle.load(open(intgr.get_integrated_reflections()))
    batched_reflections[si.get_batch_range()] = (experiments, reflections,
                                                 si.get_sweep_name())

  # - look up reflection in reflection list, get bounding box
  # - pull pixels given from image set, flatten these, write out

  from dials.array_family import flex
  from annlib_ext import AnnAdaptor as ann_adaptor

  reflections_run = { }
  for run in batched_reflections:
    reflections_run[run] = []

  for rogue in rogue_reflections:
    b = rogue[0]
    for run in batched_reflections:
      if b >= run[0] and b <= run[1]:
        reflections_run[run].append(rogue)
        break

  for run_no, run in enumerate(reflections_run):
    experiment = batched_reflections[run][0]
    reflections = batched_reflections[run][1]
    name = batched_reflections[run][2]
    rogues = reflections_run[run]
    reference = flex.double()
    scan = experiment.scans()[0]
    images = experiment.imagesets()[0]
    for xyz in reflections['xyzcal.px']:
      reference.append(xyz[0])
      reference.append(xyz[1])
      reference.append(xyz[2])

    search = flex.double()
    for rogue in rogues:
      search.append(rogue[1])
      search.append(rogue[2])
      search.append(scan.get_array_index_from_angle(rogue[3]))

    ann = ann_adaptor(data=reference, dim=3, k=1)
    ann.query(search)

    keep = flex.bool(len(reflections), False)

    for j, rogue in enumerate(rogues):
      keep[ann.nn[j]] = True

    reflections = reflections.select(keep==True)

    if params.extract:
      reflections["shoebox"] = flex.shoebox(
        reflections["panel"],
        reflections["bbox"],
        allocate=True)
      reflections.extract_shoeboxes(images, verbose=False)

    if len(reflections_run) > 1:
      output = params.output.reflections.replace(
          '.pickle', '-%s.pickle' % name)
      print 'Extracted %d rogue reflections for %s to %s' % \
        (len(reflections), name, output)
      reflections.as_pickle(output)
    else:
      output = params.output.reflections
      print 'Extracted %d rogue reflections to %s' % \
        (len(reflections), output)
      reflections.as_pickle(output)
Ejemplo n.º 14
0
    def _find_in_imageset(self, imageset):
        """
    Do the spot finding.

    :param imageset: The imageset to process
    :return: The observed spots

    """
        from dials.util.masking import MaskGenerator
        from dials.array_family import flex
        from dials.util.command_line import Command
        from dxtbx.imageset import ImageSweep
        from logging import info

        # The input mask
        mask = self.mask_generator.generate(imageset)
        if self.mask is not None:
            mask = tuple(m1 & m2 for m1, m2 in zip(mask, self.mask))

        # Set the spot finding algorithm
        extract_spots = ExtractSpots(
            threshold_function=self.threshold_function,
            mask=mask,
            region_of_interest=self.region_of_interest,
            max_strong_pixel_fraction=self.max_strong_pixel_fraction,
            mp_method=self.mp_method,
            nproc=self.nproc,
            mp_chunksize=self.mp_chunksize,
            min_spot_size=self.min_spot_size,
            max_spot_size=self.max_spot_size,
        )

        # Get the max scan range
        if isinstance(imageset, ImageSweep):
            max_scan_range = imageset.get_array_range()
        else:
            max_scan_range = (0, len(imageset))

        # Get list of scan ranges
        if not self.scan_range or self.scan_range[0] is None:
            scan_range = [(max_scan_range[0] + 1, max_scan_range[1])]
        else:
            scan_range = self.scan_range

        # Get spots from bits of scan
        spots_all = []
        for scan in scan_range:
            j0, j1 = scan
            assert j1 >= j0 and j0 > max_scan_range[0] and j1 <= max_scan_range[1]
            info("\nFinding spots in image {0} to {1}...".format(j0, j1))
            j0 -= 1
            if isinstance(imageset, ImageSweep):
                j0 -= imageset.get_array_range()[0]
                j1 -= imageset.get_array_range()[0]
            spots_all.extend(extract_spots(imageset[j0:j1]))

        # Get the list of shoeboxes
        shoeboxes = flex.shoebox(spots_all)

        # Calculate the spot centroids
        centroid = shoeboxes.centroid_valid()
        info("Calculated {0} spot centroids".format(len(shoeboxes)))

        # Calculate the spot intensities
        intensity = shoeboxes.summed_intensity()
        info("Calculated {0} spot intensities".format(len(shoeboxes)))

        # Create the observations
        observed = flex.observation(shoeboxes.panels(), centroid, intensity)

        # Write the hot mask
        if self.write_hot_mask:
            # Find spots which cover the whole scan range
            bbox = flex.int6([sbox.bbox for sbox in shoeboxes])
            z0, z1 = bbox.parts()[4:6]
            zr = z1 - z0
            assert zr.all_gt(0)
            possible_hot_spots = zr == len(imageset)
            num_possible_hot_spots = possible_hot_spots.count(True)
            info("Found %d possible hot spots" % num_possible_hot_spots)

            # Create the hot pixel mask
            hot_mask = tuple(flex.bool(flex.grid(p.get_image_size()[::-1]), True) for p in imageset.get_detector())
            if num_possible_hot_spots > 0:
                hot_shoeboxes = shoeboxes.select(possible_hot_spots)
                for sbox in hot_shoeboxes:
                    x0, x1, y0, y1 = sbox.bbox[0:4]
                    m = sbox.mask
                    p = sbox.panel
                    for y in range(m.all()[1]):
                        for x in range(m.all()[2]):
                            if m[:, y : y + 1, x : x + 1].all_ne(0):
                                hot_mask[p][y0 + y, x0 + x] = False
            info("Found %d possible hot pixel(s)" % hot_mask.count(False))

        else:
            hot_mask = None

        # Filter the reflections and select only the desired spots
        flags = self.filter_spots(None, sweep=imageset, observations=observed, shoeboxes=shoeboxes)
        observed = observed.select(flags)
        shoeboxes = shoeboxes.select(flags)

        # Return as a reflection list
        return flex.reflection_table(observed, shoeboxes), hot_mask
Ejemplo n.º 15
0
from dials.array_family import flex
num_ref = 3
ref_table = flex.reflection_table()

shoebox = flex.shoebox(num_ref)
ref_table['shoebox'] = shoebox

intensity = flex.double(num_ref)
ref_table['intensity.sum.value'] = intensity

intensity_var = flex.double(num_ref)
ref_table['intensity.sum.variance'] = intensity_var

iterate = ref_table['shoebox']
n = 0
for arr in iterate:
  img = flex.double(flex.grid(3, 3, 3))
  bkg = flex.double(flex.grid(3, 3, 3))
  msk = flex.int(flex.grid(3, 3, 3))
  for row in range(3):
    for col in range(3):
      for fra in range(3):
        img[row, col, fra] = row + col + fra + n * 9
        bkg[row, col, fra] = 0.0
        msk[row, col, fra] = 3
  n += 1
  msk[1, 1, 1] = 5
  tmp_i = n * n * n * 3
  img[1, 1, 1] += tmp_i
  print "intensity must be =", tmp_i
  arr.data = img[:, :, :]
Ejemplo n.º 16
0
    def generate_predictions(self, N):
        """ Generate some reflections. """
        from dials.algorithms.profile_model.gaussian_rs import MaskCalculator3D
        from dials.array_family import flex
        from dials.util.command_line import Command
        from dials.algorithms import filtering
        from dials.algorithms.shoebox import MaskCode
        from dials.algorithms.profile_model.gaussian_rs import Model as ProfileModel
        import random

        # Set the profile model
        self.experiment.profile = ProfileModel(None, self.n_sigma, self.sigma_b, self.sigma_m)

        # Generate a list of reflections
        refl = flex.reflection_table.from_predictions(self.experiment)
        refl["id"] = flex.int(len(refl), 0)

        # Filter by zeta
        zeta = 0.05
        Command.start("Filtering by zeta >= %f" % zeta)
        mask = filtering.by_zeta(self.experiment.goniometer, self.experiment.beam, refl["s1"], zeta)
        refl.del_selected(mask != True)
        Command.end("Filtered %d reflections by zeta >= %f" % (len(refl), zeta))

        # Compute the bounding box
        refl.compute_bbox([self.experiment])
        index = []
        image_size = self.experiment.detector[0].get_image_size()
        array_range = self.experiment.scan.get_array_range()
        bbox = refl["bbox"]
        for i in range(len(refl)):
            x0, x1, y0, y1, z0, z1 = bbox[i]
            if (
                x0 < 0
                or x1 > image_size[0]
                or y0 < 0
                or y1 > image_size[1]
                or z0 < array_range[0]
                or z1 > array_range[1]
            ):
                index.append(i)
        refl.del_selected(flex.size_t(index))

        # Sample if specified
        index = random.sample(range(len(refl)), N)
        refl = refl.select(flex.size_t(index))

        # Compute the bounding box
        # Create a load of shoeboxes
        Command.start("Creating shoeboxes for %d reflections" % len(refl))
        refl["shoebox"] = flex.shoebox(refl["panel"], refl["bbox"])
        refl["shoebox"].allocate_with_value(MaskCode.Valid)
        Command.end("Created shoeboxes for %d reflections" % len(refl))

        # Get the function object to mask the foreground
        Command.start("Masking Foreground for %d reflections" % len(refl))
        mask_foreground = MaskCalculator3D(
            self.experiment.beam,
            self.experiment.detector,
            self.experiment.goniometer,
            self.experiment.scan,
            self.n_sigma * self.sigma_b,
            self.n_sigma * self.sigma_m,
        )

        # Mask the foreground
        mask_foreground(refl["shoebox"], refl["s1"], refl["xyzcal.px"].parts()[2], refl["panel"])
        Command.end("Masked foreground for %d reflections" % len(refl))

        # Return the reflections
        return refl
Ejemplo n.º 17
0
    def __call__(self, imageset):
        """
    Find the spots in the imageset

    :param imageset: The imageset to process
    :return: The list of spot shoeboxes

    """
        from dials.util.command_line import Command
        from dials.array_family import flex
        from dxtbx.imageset import ImageSweep
        from dials.model.data import PixelListLabeller
        from logging import info, warn
        from math import floor
        import platform
        from logging import warn

        # Change the number of processors if necessary
        mp_nproc = self.nproc
        if platform.system() == "Windows" and mp_nproc > 1:
            warn("")
            warn("*" * 80)
            warn("Multiprocessing is not available on windows. Setting nproc = 1")
            warn("*" * 80)
            warn("")
            mp_nproc = 1
        if mp_nproc > len(imageset):
            mp_nproc = len(imageset)
        mp_method = self.mp_method
        mp_chunksize = self.mp_chunksize
        len_by_nproc = int(floor(len(imageset) / mp_nproc))
        if mp_chunksize > len_by_nproc:
            mp_chunksize = len_by_nproc
        assert mp_nproc > 0, "Invalid number of processors"
        assert mp_chunksize > 0, "Invalid chunk size"

        # The extract pixels function
        function = ExtractPixelsFromImage(
            imageset=imageset,
            threshold_function=self.threshold_function,
            mask=self.mask,
            max_strong_pixel_fraction=self.max_strong_pixel_fraction,
            region_of_interest=self.region_of_interest,
        )

        # The indices to iterate over
        indices = list(range(len(imageset)))

        # Initialise the pixel labeller
        num_panels = len(imageset.get_detector())
        pixel_labeller = [PixelListLabeller() for p in range(num_panels)]

        # Do the processing
        info("Extracting strong pixels from images")
        info(" Using %s with %d parallel job(s)\n" % (mp_method, mp_nproc))
        if mp_nproc > 1:

            def process_output(result):
                import logging

                for message in result[1]:
                    logging.log(message.levelno, message.msg)
                assert len(pixel_labeller) == len(result[0].pixel_list), "Inconsistent size"
                for plabeller, plist in zip(pixel_labeller, result[0].pixel_list):
                    plabeller.add(plist)
                result[0].pixel_list = None

            # Parallel reading of HDF5 from the same handle is not allowed. Python
            # multiprocessing is a bit messed up and used fork on linux so need to
            # close and reopen file.
            from dxtbx.imageset import SingleFileReader

            if isinstance(imageset.reader(), SingleFileReader):
                imageset.reader().nullify_format_instance()

            batch_parallel_map(
                func=ExtractSpotsParallelTask(function),
                iterable=indices,
                processes=mp_nproc,
                callback=process_output,
                method=mp_method,
                chunksize=mp_chunksize,
            )
        else:
            for task in indices:
                result = function(task)
                assert len(pixel_labeller) == len(result.pixel_list), "Inconsistent size"
                for plabeller, plist in zip(pixel_labeller, result.pixel_list):
                    plabeller.add(plist)
                    result.pixel_list = None

        # Extract the pixel lists into a list of reflections
        shoeboxes = flex.shoebox()
        if isinstance(imageset, ImageSweep):
            twod = False
        else:
            twod = True
        for i, p in enumerate(pixel_labeller):
            if p.num_pixels() > 0:
                shoeboxes.extend(
                    flex.shoebox(
                        p, i, 0, twod, self.min_spot_size, self.max_spot_size  # panel  # zrange  # twod  # min_pixels
                    )
                )  # max_pixels
        info("")
        info("Extracted {0} spots".format(len(shoeboxes)))

        # Get the unallocated spots and print some info
        selection = shoeboxes.is_allocated()
        shoeboxes = shoeboxes.select(selection)
        info(
            "Removed %d spots with size < %d or > %d pixels"
            % (selection.count(False), self.min_spot_size, self.max_spot_size)
        )

        # Return the shoeboxes
        return shoeboxes
Ejemplo n.º 18
0
  imageset = experiments[0].imageset

  print list(reflections.keys())
  reflections.unset_flags(flex.size_t(range(len(reflections))), reflections.flags.integrated_sum)
  reflections.unset_flags(flex.size_t(range(len(reflections))), reflections.flags.integrated_prf)
  del reflections['intensity.sum.value']
  del reflections['intensity.sum.variance']
  del reflections['background.mean']
  del reflections['intensity.prf.value']
  del reflections['intensity.prf.variance']
  del reflections['profile.correlation']
  del reflections['profile.rmsd']

  # Create the reflection lookup
  reflections['shoebox'] = flex.shoebox(
    reflections['panel'],
    reflections['bbox'])
  bbox = reflections['bbox']
  reflection_lookup = defaultdict(list)
  for i in range(len(bbox)):
    for j in range(bbox[i][4],bbox[i][5]):
      reflection_lookup[j].append(i)

  width, height = experiments[0].detector[0].get_image_size()

  # Loop through all images
  print "START"
  for frame in range(0, len(imageset)):

    # Get the subset of reflections on this image and compute the mask
    indices = flex.size_t(reflection_lookup[frame])
Ejemplo n.º 19
0
  def tst_split_partials_with_shoebox(self):
    from dials.array_family import flex
    from random import randint, uniform
    from dials.model.data import Shoebox
    r = flex.reflection_table()
    r['value1'] = flex.double()
    r['value2'] = flex.int()
    r['value3'] = flex.double()
    r['bbox'] = flex.int6()
    r['panel'] = flex.size_t()
    r['shoebox'] = flex.shoebox()
    expected = []
    for i in range(100):
      x0 = randint(0, 100)
      x1 = x0 + randint(1, 10)
      y0 = randint(0, 100)
      y1 = y0 + randint(1, 10)
      z0 = randint(0, 100)
      z1 = z0 + randint(1, 10)
      v1 = uniform(0, 100)
      v2 = randint(0, 100)
      v3 = uniform(0, 100)
      sbox = Shoebox(0, (x0, x1, y0, y1, z0, z1))
      sbox.allocate()
      assert(sbox.is_consistent())
      w = x1 - x0
      h = y1 - y0
      for z in range(z0, z1):
        for y in range(y0, y1):
          for x in range(x0, x1):
            sbox.data[z-z0,y-y0,x-x0] = x+y*w+z*w*h
      r.append({
        'value1' : v1,
        'value2' : v2,
        'value3' : v3,
        'bbox' : (x0, x1, y0, y1, z0, z1),
        'panel' : 0,
        'shoebox' : sbox
      })
      for z in range(z0, z1):
        sbox = Shoebox(0, (x0, x1, y0, y1, z, z+1))
        sbox.allocate()
        assert(sbox.is_consistent())
        w = x1 - x0
        h = y1 - y0
        for y in range(y0, y1):
          for x in range(x0, x1):
            sbox.data[0,y-y0,x-x0] = x+y*w+z*w*h
        expected.append({
          'value1' : v1,
          'value2' : v2,
          'value3' : v3,
          'bbox' : (x0, x1, y0, y1, z, z+1),
          'partial_id' : i,
          'panel' : 0,
          'shoebox' : sbox
        })

    r.split_partials_with_shoebox()
    assert(len(r) == len(expected))
    EPS = 1e-7
    for r1, r2 in zip(r, expected):
      assert(abs(r1['value1'] - r2['value1']) < EPS)
      assert(r1['value2'] == r2['value2'])
      assert(abs(r1['value3'] - r2['value3']) < EPS)
      assert(r1['bbox'] == r2['bbox'])
      assert(r1['partial_id'] == r2['partial_id'])
      assert(r1['panel'] == r2['panel'])
      assert(r1['shoebox'].data.as_double().as_1d().all_approx_equal(
        r2['shoebox'].data.as_double().as_1d()))

    print 'OK'
Ejemplo n.º 20
0
  def run(self):
    ''' Extract the shoeboxes. '''
    from dials.util.options import flatten_reflections
    from dials.util.options import flatten_experiments
    from dials.util.options import flatten_datablocks
    from dials.util import log
    from dials.array_family import flex
    from libtbx.utils import Sorry
    from logging import info

    # Parse the command line
    params, options = self.parser.parse_args(show_diff_phil=False)

    # Configure logging
    log.config()

    # Log the diff phil
    diff_phil = self.parser.diff_phil.as_str()
    if diff_phil is not '':
      info('The following parameters have been modified:\n')
      info(diff_phil)

    # Get the data
    reflections = flatten_reflections(params.input.reflections)
    experiments = flatten_experiments(params.input.experiments)
    datablocks = flatten_datablocks(params.input.datablock)
    if len(experiments) == 0 and len(datablocks) == 0 and len(reflections) == 0:
      self.parser.print_help()
      exit(0)
    elif (len(experiments) != 0 and len(datablocks) != 0):
      raise Sorry('Both experiment list and datablocks set')
    elif len(experiments) > 1:
      raise Sorry('More than 1 experiment set')
    elif len(datablocks) > 1:
      raise Sorry('More than 1 datablock set')
    elif len(experiments) == 1:
      imageset = experiments[0].imageset
    elif len(datablocks) == 1:
      imagesets = datablocks[0].extract_imagesets()
      if len(imagesets) != 1:
        raise Sorry('Need 1 imageset, got %d' % len(imagesets))
      imageset = imagesets[0]
    if len(reflections) != 1:
      raise Sorry('Need 1 reflection table, got %d' % len(reflections))
    else:
      reflections = reflections[0]

    # Check the reflections contain the necessary stuff
    assert("bbox" in reflections)
    assert("panel" in reflections)

    # Get some models
    detector = imageset.get_detector()
    scan = imageset.get_scan()
    frame0, frame1 = scan.get_array_range()

    # Add some padding but limit to image volume
    if params.padding > 0:
      info('Adding %d pixels as padding' % params.padding)
      x0, x1, y0, y1, z0, z1 = reflections['bbox'].parts()
      x0 -= params.padding
      x1 += params.padding
      y0 -= params.padding
      y1 += params.padding
      z0 -= params.padding
      z1 += params.padding
      panel = reflections['panel']
      for i in range(len(reflections)):
        width, height = detector[panel[i]].get_image_size()
        if z0[i] < frame0: z0[i] = frame0
        if z1[i] > frame1: z1[i] = frame1
      reflections['bbox'] = flex.int6(x0, x1, y0, y1, z0, z1)

    # Save the old shoeboxes
    if "shoebox" in reflections:
      old_shoebox = reflections['shoebox']
    else:
      old_shoebox = None

    # Allocate the shoeboxes
    reflections["shoebox"] = flex.shoebox(
      reflections["panel"],
      reflections["bbox"],
      allocate=True)

    # Extract the shoeboxes
    reflections.extract_shoeboxes(imageset, verbose=True)

    # Preserve masking
    if old_shoebox is not None:
      info("Applying old shoebox mask")
      new_shoebox = reflections['shoebox']
      for i in range(len(reflections)):
        bbox0 = old_shoebox[i].bbox
        bbox1 = new_shoebox[i].bbox
        mask0 = old_shoebox[i].mask
        mask1 = new_shoebox[i].mask
        mask2 = flex.int(mask1.accessor(), 0)
        x0 = bbox0[0] - bbox1[0]
        x1 = bbox0[1] - bbox0[0] + x0
        y0 = bbox0[2] - bbox1[2]
        y1 = bbox0[3] - bbox0[2] + y0
        z0 = bbox0[4] - bbox1[4]
        z1 = bbox0[5] - bbox0[4] + z0
        mask2[z0:z1,y0:y1,x0:x1] = mask0
        mask1 = mask1.as_1d() | mask2.as_1d()
        mask1.reshape(new_shoebox[i].mask.accessor())
        new_shoebox[i].mask = mask1

    # Saving the reflections to disk
    filename = params.output.reflections
    info('Saving %d reflections to %s' % (len(reflections), filename))
    reflections.as_pickle(filename)
Ejemplo n.º 21
0
import numpy
data2d = numpy.zeros((3, 3), dtype = numpy.float64)

data2d[:, :] = 15
data2d[1:2, 1:2] = 50

for row in range(3):
  for col in range(3):
    data2d[row, col] += row * 2
    data2d[row, col] += col * 2

ref_table = flex.reflection_table()
'''


shoebox = flex.shoebox(5)
#panel = flex.size_t(1)
ref_table = flex.reflection_table()
shoebox.data = flex.double(flex.grid(6, 6, 6))
ref_table['shoebox'] = shoebox
#ref_table['panel'] = panel




#efisient_way = '''
its = ref_table['shoebox']
for arr in its:
  print arr.data.as_numpy_array()
#'''
Ejemplo n.º 22
0
    dials_regression = libtbx.env.dist_path('dials_regression')
  except KeyError, e:
    print 'FAIL: dials_regression not configured'
    exit(0)
  path = join(dials_regression, "centroid_test_data")
  import sys
  assert(len(sys.argv) == 1)
  sys.argv.append(join(path, "experiments.json"))
  sys.argv.append(join(path, "profile.phil"))

  parser = OptionParser(phil=phil_scope)
  params, options, args = parser.parse_args()
  assert(len(args) == 1)

  exlist = ExperimentListFactory.from_json_file(args[0])
  assert(len(exlist) == 1)


  profile_model = ProfileModelList.load(params)

  rlist = flex.reflection_table.from_predictions_multi(exlist)
  rlist.compute_bbox(exlist, profile_model)
  rlist['shoebox'] = flex.shoebox(rlist['panel'], rlist['bbox'])
  rlist['shoebox'].allocate()

  rlist.extract_shoeboxes(exlist[0].imageset)

  show_reflection(rlist[len(rlist)//2])
  #show_reflection(rlist[len(rlist)//2], orient = "porTrait")
  #show_reflection(rlist[len(rlist)//2], orient = "lanDscape")
Ejemplo n.º 23
0
    '''Set the reflection data.'''
    self.set_data(reflections, ReflectionListEncoder())

  def get_reflections(self):
    '''Get the reflection data.'''
    return self.get_data(ReflectionListDecoder())


if __name__ == '__main__':
  from dials.array_family import flex
  reflections = flex.reflection_table([
    ('hkl', flex.miller_index(10)),
    ('s1', flex.vec3_double(10)),
    ('bbox', flex.int6(10)),
    ('id', flex.int(10)),
    ('shoebox', flex.shoebox(10))
  ])

  for i in range(10):
    reflections['shoebox'][i].data = flex.double(flex.grid(10,10,10))
    reflections['shoebox'][i].mask = flex.int(flex.grid(10,10,10))
    reflections['shoebox'][i].background = flex.double(flex.grid(10,10,10))

  for i in range(10):
    print reflections['shoebox'][i].data.all()

  writer = NexusFile('temp.h5', 'w')
  writer.set_reflections(reflections)
  writer.close()

  reader = NexusFile('temp.h5', 'r')
Ejemplo n.º 24
0

if __name__ == '__main__':

  from dials.array_family import flex
  from dxtbx.model.experiment.experiment_list import ExperimentListFactory
  import os.path
  path = "/home/upc86896/Projects/cctbx/sources/dials_regression/centroid_test_data"

  rlist_filename = os.path.join(path, "integrated.pickle")
  exlist_filename = os.path.join(path, "experiments.json")

  rlist = flex.reflection_table.from_pickle(rlist_filename)
  exlist = ExperimentListFactory.from_json_file(exlist_filename)

  panel = rlist['panel']
  bbox = rlist['bbox']

  rlist['shoebox'] = flex.shoebox(panel, bbox)
  rlist['shoebox'].allocate()

  rlist.fill_shoeboxes(exlist[0].imageset)
Ejemplo n.º 25
0
def run(i, imp):
  from random import randint
  from dials.array_family import flex

  #building a reflection table
  num_ref = 5
  ref_table = flex.reflection_table()

  shoebox = flex.shoebox(num_ref)
  ref_table['shoebox'] = shoebox

  intensity = flex.double(num_ref)
  ref_table['intensity.sum.value'] = intensity

  intensity_var = flex.double(num_ref)
  ref_table['intensity.sum.variance'] = intensity_var

  iterate = ref_table['shoebox']
  i_to_compare = []

  # bulding the shoebox with a desired content
  # which is a reflection with noise included

  n = 0
  for arr in iterate:
    img = flex.double(flex.grid(3, 3, 3))
    bkg = flex.double(flex.grid(3, 3, 3))
    msk = flex.int(flex.grid(3, 3, 3))
    for row in range(3):
      for col in range(3):
        for fra in range(3):
          img[row, col, fra] = row + col + fra + n * 9 + randint(0, i)
          bkg[row, col, fra] = 0.0
          msk[row, col, fra] = 3
    n += 1
    msk[1, 1, 1] = 5
    tmp_i = n * n * n * 3
    i_to_compare.append(tmp_i)
    img[1, 1, 1] += tmp_i

    arr.data = img[:, :, :]
    arr.background = bkg[:, :, :]
    arr.mask = msk[:, :, :]

  # calling the functions that we need to test
  # first select the algorithm for background calculation

  if imp == "inclined":
    print "testing inclined_background_subtractor"
    from dials.algorithms.background.inclined_background_subtractor \
     import layering_and_background_plane
    layering_and_background_plane(ref_table)
  elif imp == "flat":
    print "testing flat_background_subtractor"
    from dials.algorithms.background.flat_background_subtractor \
     import layering_and_background_avg
    layering_and_background_avg(ref_table)
  elif imp == "curved":
    print "testing curved_background_subtractor"
    from dials.algorithms.background.curved_background_subtractor \
     import layering_and_background_modl
    layering_and_background_modl(ref_table)

  # no matter which algorithm was used for background calculation
  # the integration summation must remain compatible

  from dials.algorithms.integration.summation2d \
    import  flex_2d_layering_n_integrating
  flex_2d_layering_n_integrating(ref_table)

  # comparing results

  result = "OK"
  resl_its = ref_table['intensity.sum.value']
  resl_var = ref_table['intensity.sum.variance']
  for n_its in range(len(resl_its)):
    if resl_its[n_its] <= i_to_compare[n_its] + i and \
       resl_its[n_its] >= i_to_compare[n_its] - i and \
       resl_var[n_its] > resl_its[n_its]:
      print "Ok ", n_its
    else:
      print "Wrong num", n_its

      print "i =", i
      print "resl_its[n_its] =", resl_its[n_its]
      print "i_to_compare[n_its] =", i_to_compare[n_its]
      print "resl_var[n_its] =", resl_var[n_its]

      result = "wrong"
      raise RuntimeError('wrong result')
  return result
Ejemplo n.º 26
0
def simple_gaussian_spots(params):
  from dials.array_family import flex
  from dials.algorithms import shoebox
  import random
  import math

  from scitbx import matrix
  r = params.rotation
  axis = matrix.col((r.axis.x, r.axis.y, r.axis.z))
  if axis.length() > 0:
    rotation = axis.axis_and_angle_as_r3_rotation_matrix(r.angle, deg=True)
  else:
    rotation = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))

  # generate mask and peak values

  from dials.algorithms.shoebox import MaskCode
  mask_peak = MaskCode.Valid|MaskCode.Foreground
  mask_back = MaskCode.Valid|MaskCode.Background

  from dials.util.command_line import ProgressBar
  p = ProgressBar(title = 'Generating reflections')

  rlist = flex.reflection_table(params.nrefl)
  hkl = flex.miller_index(params.nrefl)
  s1 = flex.vec3_double(params.nrefl)
  xyzmm = flex.vec3_double(params.nrefl)
  xyzpx = flex.vec3_double(params.nrefl)
  panel = flex.size_t(params.nrefl)
  bbox = flex.int6(params.nrefl)

  for j in range(params.nrefl):
    p.update(j * 100.0 / params.nrefl)
    hkl[j] = (random.randint(0, 20),
              random.randint(0, 20),
              random.randint(0, 20))
    phi = 2 * math.pi * random.random()
    s1[j] = (0, 0, 0)
    xyzpx[j] = (0, 0, 0)
    xyzmm[j] = (0, 0, phi)
    panel[j] = 0
    bbox[j] = (0, params.shoebox_size.x,
               0, params.shoebox_size.y,
               0, params.shoebox_size.z)

  p.finished('Generating %d reflections' % params.nrefl)
  intensity = flex.double(params.nrefl)
  shoebox = flex.shoebox(panel, bbox)
  shoebox.allocate_with_value(MaskCode.Valid)

  p = ProgressBar(title = 'Generating shoeboxes')

  for i in range(len(rlist)):

    p.update(i * 100.0 / params.nrefl)
    mask = shoebox[i].mask

    if params.pixel_mask == 'precise':
      # flag everything as background: peak will me assigned later
      for j in range(len(mask)):
        mask[j] = mask_back
    elif params.pixel_mask == 'all':
      # flag we have no idea what anything is
      mask_none = MaskCode.Valid|MaskCode.Foreground|MaskCode.Background
      for j in range(len(mask)):
        mask[j] = mask_none
    elif params.pixel_mask == 'static':
      import itertools
      from scitbx.array_family import flex
      x0 = params.spot_offset.x + params.shoebox_size.x / 2
      y0 = params.spot_offset.x + params.shoebox_size.y / 2
      z0 = params.spot_offset.x + params.shoebox_size.z / 2
      sx = params.mask_nsigma * params.spot_size.x
      sy = params.mask_nsigma * params.spot_size.y
      sz = params.mask_nsigma * params.spot_size.z

      # The x, y, z indices
      z, y, x = zip(*itertools.product(*(range(n) for n in mask.all())))
      xyz = flex.vec3_double(flex.double(x), flex.double(y), flex.double(z))

      # Calculate SUM(((xj - xj0) / sxj)**2) for each element
      xyz0 = (x0, y0, z0)
      isxyz = (1.0/sx, 1.0/sy, 1.0/sz)
      dxyz = sum([(x * isx)**2 for x, isx in
        zip(((xyz - xyz0) * rotation).parts(), isxyz)])

      # Set the mask values
      index = dxyz <= 1.0
      index.reshape(mask.accessor())
      mask.set_selected(index, MaskCode.Valid | MaskCode.Foreground)
      mask.set_selected(index != True, MaskCode.Valid | MaskCode.Background)

    sbox = shoebox[i].data

    # reflection itself, including setting the peak region if we're doing that
    # FIXME use flex arrays to make the rotation bit more efficient as this is
    # now rather slow...

    counts_true = 0
    for j in range(params.counts):
      _x = random.gauss(0, params.spot_size.x)
      _y = random.gauss(0, params.spot_size.y)
      _z = random.gauss(0, params.spot_size.z)

      Rxyz = rotation * matrix.col((_x, _y, _z)).elems

      x = int(Rxyz[0] + params.spot_offset.x + params.shoebox_size.x / 2)
      y = int(Rxyz[1] + params.spot_offset.y + params.shoebox_size.y / 2)
      z = int(Rxyz[2] + params.spot_offset.z + params.shoebox_size.z / 2)

      if x < 0 or x >= params.shoebox_size.x:
        continue
      if y < 0 or y >= params.shoebox_size.y:
        continue
      if z < 0 or z >= params.shoebox_size.z:
        continue
      sbox[z, y, x] += 1
      counts_true += 1
      if params.pixel_mask == 'precise':
        mask[z, y, x] = mask_peak

    intensity[i] = counts_true

    if params.background:
      # background:flat;
      for j in range(params.background * len(sbox)):
        x = random.randint(0, params.shoebox_size.x - 1)
        y = random.randint(0, params.shoebox_size.y - 1)
        z = random.randint(0, params.shoebox_size.z - 1)
        sbox[z, y, x] += 1
    else:
      # or inclined
      random_background_plane(sbox, params.background_a, params.background_b,
                              params.background_c, params.background_d)


  rlist['miller_index'] = hkl
  rlist['s1'] = s1
  rlist['xyzcal.px'] = xyzpx
  rlist['xyzcal.mm'] = xyzmm
  rlist['bbox'] = bbox
  rlist['panel'] = panel
  rlist['shoebox'] = shoebox
  rlist['intensity.sum.value'] = intensity
  p.finished('Generating %d shoeboxes' % params.nrefl)

  return rlist