def select_strong_pixels(sweep, trusted_range):
  from dials.util.command_line import ProgressBar
  import numpy

  # Calculate the threshold
  coordinate = []
  intensity = []
  progress = ProgressBar()
  for i, flex_image in enumerate(sweep):
    image = flex_image.as_numpy_array()
    height, width = image.shape
    threshold = calculate_threshold(image, trusted_range)
    image.shape = -1
    mask = image >= threshold

    ind = numpy.where(mask != 0)[0]
    z = [i] * len(ind)
    y = list(ind / width)
    x = list(ind % width)
    coords = zip(x, y, z)
    coordinate.extend(coords)
    intensity.extend(list(image[ind]))
    progress.update(100.0 * float(i) / len(sweep))
  progress.finished()

  return coordinate, intensity
Beispiel #2
0
 def write(self, predictions, imageset):
   from dials.util.command_line import ProgressBar
   zrange = imageset.get_array_range()
   p = ProgressBar(title="Extracting shoeboxes")
   for z, image in enumerate(imageset, start=zrange[0]):
     batch = self._add_image(z, image)
     self._write_pickle(batch)
     p.update(100.0*(z - zrange[0])/len(imageset))
   p.finish("Extracted shoeboxes")
   self._write_predictions(predictions)
Beispiel #3
0
    def generate_background(self, size, N, A, B, C, D):
        from dials.algorithms.simulation.generate_test_reflections import random_background_plane2
        from dials.array_family import flex
        from dials.util.command_line import ProgressBar

        sboxes = []
        masks = []
        progress = ProgressBar(title="Generating Background")
        for i in range(N):
            mask = flex.bool(flex.grid(size), True)
            sbox = flex.double(flex.grid(size), 0)
            random_background_plane2(sbox, A, B, C, D)
            sboxes.append(sbox)
            masks.append(mask)
            progress.update(100.0 * i / N)
        progress.finished("Generated Background")
        return sboxes, masks
Beispiel #4
0
def select_strong_pixels(sweep, trusted_range):
    from dials.util.command_line import ProgressBar
    import numpy

    # Calculate the threshold
    coordinate = []
    intensity = []
    progress = ProgressBar()
    for i, flex_image in enumerate(sweep):
        image = flex_image.as_numpy_array()
        height, width = image.shape
        threshold = calculate_threshold(image, trusted_range)
        image.shape = -1
        mask = image >= threshold

        ind = numpy.where(mask != 0)[0]
        z = [i] * len(ind)
        y = list(ind / width)
        x = list(ind % width)
        coords = zip(x, y, z)
        coordinate.extend(coords)
        intensity.extend(list(image[ind]))
        progress.update(100.0 * float(i) / len(sweep))
    progress.finished()

    return coordinate, intensity
Beispiel #5
0
  def _extract_pixels(self, sweep):
    '''Extract the pixels from the sweep

    Params:
        sweep The sweep object

    Returns:
        The list of selected pixels

    '''
    from dials.util.command_line import ProgressBar
    from scitbx.array_family import flex
    from dials.algorithms.peak_finding import flex_vec3_int

    # Initialise the pixel arrays
    coords = flex_vec3_int()
    intensity = flex.int()

    # Get the start index and trusted range from the sweep
    start = sweep.get_array_range()[0]
    trusted_range = sweep.get_detector().get_trusted_range()

    # Loop through all the images in the sweep and extract the pixels
    # from each of the images
    progress = ProgressBar()
    for frame, image in enumerate(sweep):
      c, i = self._extract_image_pixels(image, frame + start,
          trusted_range)
      coords.extend(c)
      intensity.extend(i)
      progress.update(100.0 * float(frame + 1) / len(sweep))

    progress.finished()

    # Reuturn the pixel values
    return coords, intensity
Beispiel #6
0
    def write(self, predictions, imageset):
        from dials.util.command_line import ProgressBar

        zrange = imageset.get_array_range()
        p = ProgressBar(title="Extracting shoeboxes")
        for z, image in enumerate(imageset, start=zrange[0]):
            batch = self._add_image(z, image)
            self._write_pickle(batch)
            p.update(100.0 * (z - zrange[0]) / len(imageset))
        p.finish("Extracted shoeboxes")
        self._write_predictions(predictions)
Beispiel #7
0
 def generate_background(self, size, N, A, B, C, D):
     from dials.algorithms.simulation.generate_test_reflections \
       import random_background_plane2
     from dials.array_family import flex
     from dials.util.command_line import ProgressBar
     sboxes = []
     masks = []
     progress = ProgressBar(title="Generating Background")
     for i in range(N):
         mask = flex.bool(flex.grid(size), True)
         sbox = flex.double(flex.grid(size), 0)
         random_background_plane2(sbox, A, B, C, D)
         sboxes.append(sbox)
         masks.append(mask)
         progress.update(100.0 * i / N)
     progress.finished("Generated Background")
     return sboxes, masks
Beispiel #8
0
    def _extract_pixels(self, sweep):
        """Extract the pixels from the sweep

        Params:
            sweep The sweep object

        Returns:
            The list of selected pixels

        """
        from dials.util.command_line import ProgressBar
        from scitbx.array_family import flex
        from dials.algorithms.peak_finding import flex_vec3_int

        # Initialise the pixel arrays
        coords = flex_vec3_int()
        intensity = flex.int()

        # Get the start index and trusted range from the sweep
        start = sweep.get_array_range()[0]
        trusted_range = sweep.get_detector().get_trusted_range()

        # Loop through all the images in the sweep and extract the pixels
        # from each of the images
        progress = ProgressBar()
        for frame, image in enumerate(sweep):
            c, i = self._extract_image_pixels(image, frame + start,
                                              trusted_range)
            coords.extend(c)
            intensity.extend(i)
            progress.update(100.0 * float(frame + 1) / len(sweep))

        progress.finished()

        # Reuturn the pixel values
        return coords, intensity
Beispiel #9
0
        z.append(dxyz[2] + _z)

      hkl.bounding_box = (int(math.floor(min(x))), int(math.floor(max(x)) + 1),
                          int(math.floor(min(y))), int(math.floor(max(y)) + 1),
                          int(math.floor(min(z))), int(math.floor(max(z)) + 1))
      try:
        counts = counts_database[hkl.miller_index]
        useful.append(hkl)
      except KeyError, e:
        continue

    from dials.algorithms import shoebox
    shoebox.allocate(useful)

    from dials.util.command_line import ProgressBar
    p = ProgressBar(title = 'Generating shoeboxes')

    # now for each reflection perform the simulation
    for j, refl in enumerate(useful):
      p.update(j * 100.0 / len(useful))
      d = d_matrices[j]

      from scitbx.random import variate, normal_distribution
      g = variate(normal_distribution(mean = 0, sigma = node_size))
      counts = counts_database[refl.miller_index]
      dhs = g(counts)
      dks = g(counts)
      dls = g(counts)
      self.map_to_image_space(refl, d, dhs, dks, dls)

    p.finished('Generated %d shoeboxes' % len(useful))
def simple_gaussian_spots(params):
    from scitbx import matrix

    from dials.array_family import flex

    r = params.rotation
    axis = matrix.col((r.axis.x, r.axis.y, r.axis.z))
    if axis.length() > 0:
        rotation = axis.axis_and_angle_as_r3_rotation_matrix(r.angle, deg=True)
    else:
        rotation = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))

    # generate mask and peak values

    from dials.algorithms.shoebox import MaskCode

    mask_peak = MaskCode.Valid | MaskCode.Foreground
    mask_back = MaskCode.Valid | MaskCode.Background

    from dials.util.command_line import ProgressBar

    p = ProgressBar(title="Generating reflections")

    rlist = flex.reflection_table(params.nrefl)
    hkl = flex.miller_index(params.nrefl)
    s1 = flex.vec3_double(params.nrefl)
    xyzmm = flex.vec3_double(params.nrefl)
    xyzpx = flex.vec3_double(params.nrefl)
    panel = flex.size_t(params.nrefl)
    bbox = flex.int6(params.nrefl)

    for j in range(params.nrefl):
        p.update(j * 100.0 / params.nrefl)
        hkl[j] = (random.randint(0, 20), random.randint(0, 20), random.randint(0, 20))
        phi = 2 * math.pi * random.random()
        s1[j] = (0, 0, 0)
        xyzpx[j] = (0, 0, 0)
        xyzmm[j] = (0, 0, phi)
        panel[j] = 0
        bbox[j] = (
            0,
            params.shoebox_size.x,
            0,
            params.shoebox_size.y,
            0,
            params.shoebox_size.z,
        )

    p.finished("Generating %d reflections" % params.nrefl)
    intensity = flex.double(params.nrefl)
    shoebox = flex.shoebox(panel, bbox)
    shoebox.allocate_with_value(MaskCode.Valid)

    p = ProgressBar(title="Generating shoeboxes")

    for i in range(len(rlist)):

        p.update(i * 100.0 / params.nrefl)
        mask = shoebox[i].mask

        if params.pixel_mask == "precise":
            # flag everything as background: peak will me assigned later
            for j in range(len(mask)):
                mask[j] = mask_back
        elif params.pixel_mask == "all":
            # flag we have no idea what anything is
            mask_none = MaskCode.Valid | MaskCode.Foreground | MaskCode.Background
            for j in range(len(mask)):
                mask[j] = mask_none
        elif params.pixel_mask == "static":
            from scitbx.array_family import flex

            x0 = params.spot_offset.x + params.shoebox_size.x / 2
            y0 = params.spot_offset.x + params.shoebox_size.y / 2
            z0 = params.spot_offset.x + params.shoebox_size.z / 2
            sx = params.mask_nsigma * params.spot_size.x
            sy = params.mask_nsigma * params.spot_size.y
            sz = params.mask_nsigma * params.spot_size.z

            # The x, y, z indices
            z, y, x = zip(*itertools.product(*(range(n) for n in mask.all())))
            xyz = flex.vec3_double(flex.double(x), flex.double(y), flex.double(z))

            # Calculate SUM(((xj - xj0) / sxj)**2) for each element
            xyz0 = (x0, y0, z0)
            isxyz = (1.0 / sx, 1.0 / sy, 1.0 / sz)
            dxyz = sum(
                (x * isx) ** 2
                for x, isx in zip(((xyz - xyz0) * rotation).parts(), isxyz)
            )

            # Set the mask values
            index = dxyz <= 1.0
            index.reshape(mask.accessor())
            mask.set_selected(index, MaskCode.Valid | MaskCode.Foreground)
            mask.set_selected(not index, MaskCode.Valid | MaskCode.Background)

        sbox = shoebox[i].data

        # reflection itself, including setting the peak region if we're doing that
        # FIXME use flex arrays to make the rotation bit more efficient as this is
        # now rather slow...

        counts_true = 0
        for j in range(params.counts):
            _x = random.gauss(0, params.spot_size.x)
            _y = random.gauss(0, params.spot_size.y)
            _z = random.gauss(0, params.spot_size.z)

            Rxyz = rotation * matrix.col((_x, _y, _z)).elems

            x = int(Rxyz[0] + params.spot_offset.x + params.shoebox_size.x / 2)
            y = int(Rxyz[1] + params.spot_offset.y + params.shoebox_size.y / 2)
            z = int(Rxyz[2] + params.spot_offset.z + params.shoebox_size.z / 2)

            if x < 0 or x >= params.shoebox_size.x:
                continue
            if y < 0 or y >= params.shoebox_size.y:
                continue
            if z < 0 or z >= params.shoebox_size.z:
                continue
            sbox[z, y, x] += 1
            counts_true += 1
            if params.pixel_mask == "precise":
                mask[z, y, x] = mask_peak

        intensity[i] = counts_true

        if params.background:
            # background:flat;
            for j in range(params.background * len(sbox)):
                x = random.randint(0, params.shoebox_size.x - 1)
                y = random.randint(0, params.shoebox_size.y - 1)
                z = random.randint(0, params.shoebox_size.z - 1)
                sbox[z, y, x] += 1
        else:
            # or inclined
            random_background_plane(
                sbox,
                params.background_a,
                params.background_b,
                params.background_c,
                params.background_d,
            )

    rlist["miller_index"] = hkl
    rlist["s1"] = s1
    rlist["xyzcal.px"] = xyzpx
    rlist["xyzcal.mm"] = xyzmm
    rlist["bbox"] = bbox
    rlist["panel"] = panel
    rlist["shoebox"] = shoebox
    rlist["intensity.sum.value"] = intensity
    p.finished("Generating %d shoeboxes" % params.nrefl)

    return rlist
Beispiel #11
0
    def with_individual_given_intensity(self, N, I, Ba, Bb, Bc, Bd):
        """ Generate reflections with given intensity and background. """
        from dials.array_family import flex
        from dials.util.command_line import ProgressBar
        from dials.algorithms.simulation import simulate_reciprocal_space_gaussian
        from dials.algorithms.simulation.generate_test_reflections import random_background_plane2

        # Check the lengths
        assert N == len(I)
        assert N == len(Ba)
        assert N == len(Bb)
        assert N == len(Bc)
        assert N == len(Bd)

        # Generate some predictions
        refl = self.generate_predictions(N)

        # Calculate the signal
        progress = ProgressBar(title="Calculating signal for %d reflections" % len(refl))
        s1 = refl["s1"]
        phi = refl["xyzcal.mm"].parts()[2]
        bbox = refl["bbox"]
        shoebox = refl["shoebox"]
        m = int(len(refl) / 100)
        I_exp = flex.double(len(refl), 0)
        for i in range(len(refl)):
            if I[i] > 0:
                data = shoebox[i].data.as_double()
                I_exp[i] = simulate_reciprocal_space_gaussian(
                    self.experiment.beam,
                    self.experiment.detector,
                    self.experiment.goniometer,
                    self.experiment.scan,
                    self.sigma_b,
                    self.sigma_m,
                    s1[i],
                    phi[i],
                    bbox[i],
                    I[i],
                    data,
                    shoebox[i].mask,
                )
                shoebox[i].data = data.as_float()
            if i % m == 0:
                progress.update(100.0 * float(i) / len(refl))
        progress.finished("Calculated signal impacts for %d reflections" % len(refl))

        # Calculate the background
        progress = ProgressBar(title="Calculating background for %d reflections" % len(refl))
        for l in range(len(refl)):
            background = flex.float(flex.grid(shoebox[l].size()), 0.0)
            random_background_plane2(background, Ba[l], Bb[l], Bc[l], Bd[l])
            shoebox[l].data += background
            shoebox[l].background = background
            if l % m == 0:
                progress.update(100.0 * float(l) / len(refl))
            progress.update(100.0 * float(l) / len(refl))
        progress.finished("Calculated background for %d reflections" % len(refl))

        ## Calculate the expected intensity by monte-carlo integration
        # progress = ProgressBar(title='Integrating expected signal for %d reflections' % len(refl))
        # s1 = refl['s1']
        # phi = refl['xyzcal.mm'].parts()[2]
        # bbox = refl['bbox']
        # shoebox = refl['shoebox']
        # I_exp = flex.double(len(refl), 0)
        # m = int(len(refl) / 100)
        # for i in range(len(refl)):
        # if I[i] > 0:
        # I_exp[i] = integrate_reciprocal_space_gaussian(
        # self.experiment.beam,
        # self.experiment.detector,
        # self.experiment.goniometer,
        # self.experiment.scan,
        # self.sigma_b,
        # self.sigma_m,
        # s1[i],
        # phi[i],
        # bbox[i],
        # 10000,
        # shoebox[i].mask) / 10000.0
        # if i % m == 0:
        # progress.update(100.0 * float(i) / len(refl))
        # progress.finished('Integrated expected signal impacts for %d reflections' % len(refl))

        # Save the expected intensity and background
        refl["intensity.sim"] = I
        refl["background.sim.a"] = Ba
        refl["background.sim.b"] = Bb
        refl["background.sim.c"] = Bc
        refl["background.sim.d"] = Bd
        refl["intensity.exp"] = I_exp

        # Return the reflections
        return refl
Beispiel #12
0
        z.append(dxyz[2] + _z)

      hkl.bounding_box = (int(math.floor(min(x))), int(math.floor(max(x)) + 1),
                          int(math.floor(min(y))), int(math.floor(max(y)) + 1),
                          int(math.floor(min(z))), int(math.floor(max(z)) + 1))
      try:
        counts = counts_database[hkl.miller_index]
        useful.append(hkl)
      except KeyError, e:
        continue

    from dials.algorithms import shoebox
    shoebox.allocate(useful)

    from dials.util.command_line import ProgressBar
    p = ProgressBar(title = 'Generating shoeboxes')

    # now for each reflection perform the simulation
    for j, refl in enumerate(useful):
      p.update(j * 100.0 / len(useful))
      d = d_matrices[j]

      from scitbx.random import variate, normal_distribution
      g = variate(normal_distribution(mean = 0, sigma = node_size))
      counts = counts_database[refl.miller_index]
      dhs = g(counts)
      dks = g(counts)
      dls = g(counts)
      self.map_to_image_space(refl, d, dhs, dks, dls)

    p.finished('Generated %d shoeboxes' % len(useful))
def simple_gaussian_spots(params):
  from dials.array_family import flex
  from dials.algorithms import shoebox
  import random
  import math

  from scitbx import matrix
  r = params.rotation
  axis = matrix.col((r.axis.x, r.axis.y, r.axis.z))
  if axis.length() > 0:
    rotation = axis.axis_and_angle_as_r3_rotation_matrix(r.angle, deg=True)
  else:
    rotation = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))

  # generate mask and peak values

  from dials.algorithms.shoebox import MaskCode
  mask_peak = MaskCode.Valid|MaskCode.Foreground
  mask_back = MaskCode.Valid|MaskCode.Background

  from dials.util.command_line import ProgressBar
  p = ProgressBar(title = 'Generating reflections')

  rlist = flex.reflection_table(params.nrefl)
  hkl = flex.miller_index(params.nrefl)
  s1 = flex.vec3_double(params.nrefl)
  xyzmm = flex.vec3_double(params.nrefl)
  xyzpx = flex.vec3_double(params.nrefl)
  panel = flex.size_t(params.nrefl)
  bbox = flex.int6(params.nrefl)

  for j in range(params.nrefl):
    p.update(j * 100.0 / params.nrefl)
    hkl[j] = (random.randint(0, 20),
              random.randint(0, 20),
              random.randint(0, 20))
    phi = 2 * math.pi * random.random()
    s1[j] = (0, 0, 0)
    xyzpx[j] = (0, 0, 0)
    xyzmm[j] = (0, 0, phi)
    panel[j] = 0
    bbox[j] = (0, params.shoebox_size.x,
               0, params.shoebox_size.y,
               0, params.shoebox_size.z)

  p.finished('Generating %d reflections' % params.nrefl)
  intensity = flex.double(params.nrefl)
  shoebox = flex.shoebox(panel, bbox)
  shoebox.allocate_with_value(MaskCode.Valid)

  p = ProgressBar(title = 'Generating shoeboxes')

  for i in range(len(rlist)):

    p.update(i * 100.0 / params.nrefl)
    mask = shoebox[i].mask

    if params.pixel_mask == 'precise':
      # flag everything as background: peak will me assigned later
      for j in range(len(mask)):
        mask[j] = mask_back
    elif params.pixel_mask == 'all':
      # flag we have no idea what anything is
      mask_none = MaskCode.Valid|MaskCode.Foreground|MaskCode.Background
      for j in range(len(mask)):
        mask[j] = mask_none
    elif params.pixel_mask == 'static':
      import itertools
      from scitbx.array_family import flex
      x0 = params.spot_offset.x + params.shoebox_size.x / 2
      y0 = params.spot_offset.x + params.shoebox_size.y / 2
      z0 = params.spot_offset.x + params.shoebox_size.z / 2
      sx = params.mask_nsigma * params.spot_size.x
      sy = params.mask_nsigma * params.spot_size.y
      sz = params.mask_nsigma * params.spot_size.z

      # The x, y, z indices
      z, y, x = zip(*itertools.product(*(range(n) for n in mask.all())))
      xyz = flex.vec3_double(flex.double(x), flex.double(y), flex.double(z))

      # Calculate SUM(((xj - xj0) / sxj)**2) for each element
      xyz0 = (x0, y0, z0)
      isxyz = (1.0/sx, 1.0/sy, 1.0/sz)
      dxyz = sum([(x * isx)**2 for x, isx in
        zip(((xyz - xyz0) * rotation).parts(), isxyz)])

      # Set the mask values
      index = dxyz <= 1.0
      index.reshape(mask.accessor())
      mask.set_selected(index, MaskCode.Valid | MaskCode.Foreground)
      mask.set_selected(index != True, MaskCode.Valid | MaskCode.Background)

    sbox = shoebox[i].data

    # reflection itself, including setting the peak region if we're doing that
    # FIXME use flex arrays to make the rotation bit more efficient as this is
    # now rather slow...

    counts_true = 0
    for j in range(params.counts):
      _x = random.gauss(0, params.spot_size.x)
      _y = random.gauss(0, params.spot_size.y)
      _z = random.gauss(0, params.spot_size.z)

      Rxyz = rotation * matrix.col((_x, _y, _z)).elems

      x = int(Rxyz[0] + params.spot_offset.x + params.shoebox_size.x / 2)
      y = int(Rxyz[1] + params.spot_offset.y + params.shoebox_size.y / 2)
      z = int(Rxyz[2] + params.spot_offset.z + params.shoebox_size.z / 2)

      if x < 0 or x >= params.shoebox_size.x:
        continue
      if y < 0 or y >= params.shoebox_size.y:
        continue
      if z < 0 or z >= params.shoebox_size.z:
        continue
      sbox[z, y, x] += 1
      counts_true += 1
      if params.pixel_mask == 'precise':
        mask[z, y, x] = mask_peak

    intensity[i] = counts_true

    if params.background:
      # background:flat;
      for j in range(params.background * len(sbox)):
        x = random.randint(0, params.shoebox_size.x - 1)
        y = random.randint(0, params.shoebox_size.y - 1)
        z = random.randint(0, params.shoebox_size.z - 1)
        sbox[z, y, x] += 1
    else:
      # or inclined
      random_background_plane(sbox, params.background_a, params.background_b,
                              params.background_c, params.background_d)


  rlist['miller_index'] = hkl
  rlist['s1'] = s1
  rlist['xyzcal.px'] = xyzpx
  rlist['xyzcal.mm'] = xyzmm
  rlist['bbox'] = bbox
  rlist['panel'] = panel
  rlist['shoebox'] = shoebox
  rlist['intensity.sum.value'] = intensity
  p.finished('Generating %d shoeboxes' % params.nrefl)

  return rlist
Beispiel #14
0
def run(args):

    from dials.util.options import OptionParser
    from dials.util.options import flatten_experiments

    usage = "%s [options] data_master.h5" % (libtbx.env.dispatcher_name)

    parser = OptionParser(
        usage=usage,
        phil=phil_scope,
        read_experiments=True,
        read_experiments_from_images=True,
        epilog=help_message,
    )

    params, options = parser.parse_args(show_diff_phil=True)

    experiments = flatten_experiments(params.input.experiments)
    if len(experiments) != 1:
        parser.print_help()
        print("Please pass an experiment list\n")
        return

    imagesets = experiments.imagesets()

    if len(imagesets) != 1:
        raise Sorry(
            "Please pass an experiment list that contains one imageset")

    imageset = imagesets[0]

    first, last = imageset.get_scan().get_image_range()
    images = range(first, last + 1)

    if not params.images and params.image_range:
        params.images = list(
            range(params.image_range[0], params.image_range[1] + 1))

    if params.images:
        if min(params.images) < first or max(params.images) > last:
            raise Sorry("image outside of scan range")
        images = params.images

    detectors = imageset.get_detector()
    assert len(detectors) == 1
    detector = detectors[0]
    trusted = detector.get_trusted_range()

    from dials.util.command_line import ProgressBar

    p = ProgressBar(title="Counting events")

    events_per_image = {}

    for idx in images:

        p.update(idx * 100.0 / len(images))

        pixels = imageset.get_raw_data(idx - 1)
        assert len(pixels) == 1
        mask = ~imageset.get_mask(idx)[0].as_1d()
        data = pixels[0].as_1d()
        data.set_selected(mask, 0)
        hot = data >= int(round(trusted[1]))
        data.set_selected(hot, 0)
        events_per_image[idx] = flex.sum(data)

    for idx in images:
        print(idx, events_per_image[idx])
Beispiel #15
0
def run(args):

    from dials.util.options import OptionParser
    from dials.util.options import flatten_experiments
    from dials.util.command_line import ProgressBar

    usage = "%s [options] data_master.h5" % (libtbx.env.dispatcher_name)

    parser = OptionParser(
        usage=usage,
        phil=phil_scope,
        read_experiments=True,
        read_experiments_from_images=True,
        epilog=help_message,
    )

    params, options = parser.parse_args(show_diff_phil=True)

    experiments = flatten_experiments(params.input.experiments)
    if len(experiments) != 1:
        parser.print_help()
        print("Please pass an experiment list\n")
        return

    imagesets = experiments.imagesets()

    if len(imagesets) != 1:
        raise Sorry("Please pass an experiment list that contains one imageset")

    imageset = imagesets[0]

    first, last = imageset.get_scan().get_image_range()
    images = range(first, last + 1)

    if params.images is None and params.image_range is not None:
        start, end = params.image_range
        params.images = list(range(start, end + 1))

    if params.images:
        if min(params.images) < first or max(params.images) > last:
            raise Sorry("image outside of scan range")
        images = params.images

    detectors = imageset.get_detector()
    assert len(detectors) == 1
    detector = detectors[0]
    trusted = detector.get_trusted_range()

    # construct an integer array same shape as image; accumulate number of
    # "signal" pixels in each pixel across data

    total = None

    p = ProgressBar(title="Finding hot pixels")

    for idx in images:

        p.update(idx * 100.0 / len(images))

        pixels = imageset.get_raw_data(idx - 1)
        assert len(pixels) == 1
        data = pixels[0]

        negative = data < int(round(trusted[0]))
        hot = data > int(round(trusted[1]))
        bad = negative | hot

        data = data.as_double()

        spot_params = spot_phil.fetch(
            source=iotbx.phil.parse("min_spot_size=1")
        ).extract()
        threshold_function = SpotFinderFactory.configure_threshold(spot_params)
        peak_pixels = threshold_function.compute_threshold(data, ~bad)

        if total is None:
            total = peak_pixels.as_1d().as_int()
        else:
            total += peak_pixels.as_1d().as_int()

    p.finished("Finished finding hot pixels on %d images" % len(images))

    hot_mask = total >= (len(images) // 2)
    hot_pixels = hot_mask.iselection()

    p = ProgressBar(title="Finding twinkies")

    twinkies = {}
    for h in hot_pixels:
        twinkies[h] = []

    for idx in images:

        p.update(idx * 100.0 / len(images))

        pixels = imageset.get_raw_data(idx - 1)
        data = pixels[0]

        for h in hot_pixels:
            twinkies[h].append(data[h])

    p.finished("Finished hunting for twinkies on %d images" % len(images))

    nslow, nfast = data.focus()

    ffff = 0

    for h in hot_pixels:
        if total[h] == len(images) and data[h] >= trusted[1]:
            ffff += 1
            continue
        print("Pixel %d at %d %d" % (total[h], h // nfast, h % nfast))
        if len(set(twinkies[h])) >= len(twinkies[h]) // 2:
            print("  ... many possible values")
            continue
        values = set(twinkies[h])
        result = [(twinkies[h].count(value), value) for value in values]
        for count, value in reversed(sorted(result)):
            print("  %08x %d" % (value, count))

    print("Also found %d very hot pixels" % ffff)
    hot_mask.reshape(flex.grid(data.focus()))

    easy_pickle.dump(params.output.mask, (~hot_mask,))
Beispiel #16
0
    def with_individual_given_intensity(self, N, In, Ba, Bb, Bc, Bd):
        """Generate reflections with given intensity and background."""
        from dials.algorithms.simulation import simulate_reciprocal_space_gaussian
        from dials.algorithms.simulation.generate_test_reflections import (
            random_background_plane2, )
        from dials.util.command_line import ProgressBar

        # Check the lengths
        assert N == len(In)
        assert N == len(Ba)
        assert N == len(Bb)
        assert N == len(Bc)
        assert N == len(Bd)

        # Generate some predictions
        refl = self.generate_predictions(N)

        # Calculate the signal
        progress = ProgressBar(
            title=f"Calculating signal for {len(refl)} reflections")
        s1 = refl["s1"]
        phi = refl["xyzcal.mm"].parts()[2]
        bbox = refl["bbox"]
        shoebox = refl["shoebox"]
        m = int(len(refl) / 100)
        I_exp = flex.double(len(refl), 0)
        for i in range(len(refl)):
            if In[i] > 0:
                data = shoebox[i].data.as_double()
                I_exp[i] = simulate_reciprocal_space_gaussian(
                    self.experiment.beam,
                    self.experiment.detector,
                    self.experiment.goniometer,
                    self.experiment.scan,
                    self.sigma_b,
                    self.sigma_m,
                    s1[i],
                    phi[i],
                    bbox[i],
                    In[i],
                    data,
                    shoebox[i].mask,
                )
                shoebox[i].data = data.as_float()
            if i % m == 0:
                progress.update(100.0 * float(i) / len(refl))
        progress.finished(
            f"Calculated signal impacts for {len(refl)} reflections")

        # Calculate the background
        progress = ProgressBar(
            title=f"Calculating background for {len(refl)} reflections")
        for l in range(len(refl)):
            background = flex.float(flex.grid(shoebox[l].size()), 0.0)
            random_background_plane2(background, Ba[l], Bb[l], Bc[l], Bd[l])
            shoebox[l].data += background
            shoebox[l].background = background
            if l % m == 0:
                progress.update(100.0 * float(l) / len(refl))
            progress.update(100.0 * float(l) / len(refl))
        progress.finished(f"Calculated background for {len(refl)} reflections")

        ## Calculate the expected intensity by monte-carlo integration
        # progress = ProgressBar(title='Integrating expected signal for %d reflections' % len(refl))
        # s1 = refl['s1']
        # phi = refl['xyzcal.mm'].parts()[2]
        # bbox = refl['bbox']
        # shoebox = refl['shoebox']
        # I_exp = flex.double(len(refl), 0)
        # m = int(len(refl) / 100)
        # for i in range(len(refl)):
        # if In[i] > 0:
        # I_exp[i] = integrate_reciprocal_space_gaussian(
        # self.experiment.beam,
        # self.experiment.detector,
        # self.experiment.goniometer,
        # self.experiment.scan,
        # self.sigma_b,
        # self.sigma_m,
        # s1[i],
        # phi[i],
        # bbox[i],
        # 10000,
        # shoebox[i].mask) / 10000.0
        # if i % m == 0:
        # progress.update(100.0 * float(i) / len(refl))
        # progress.finished('Integrated expected signal impacts for %d reflections' % len(refl))

        # Save the expected intensity and background
        refl["intensity.sim"] = In
        refl["background.sim.a"] = Ba
        refl["background.sim.b"] = Bb
        refl["background.sim.c"] = Bc
        refl["background.sim.d"] = Bd
        refl["intensity.exp"] = I_exp

        # Return the reflections
        return refl
Beispiel #17
0
    def main(self):
        # FIXME import simulation code
        import six.moves.cPickle as pickle
        import math
        from dials.util.command_line import Importer
        from dials.algorithms.integration import ReflectionPredictor
        from libtbx.utils import Sorry

        # Parse the command line
        params, options, args = self.parser.parse_args()

        importer = Importer(args)
        if len(importer.imagesets) == 0 and len(importer.crystals) == 0:
            self.config().print_help()
            return
        if len(importer.imagesets) != 1:
            raise Sorry('need 1 sweep: %d given' % len(importer.imagesets))
        if len(importer.crystals) != 1:
            raise Sorry('need 1 crystal: %d given' % len(importer.crystals))
        sweep = importer.imagesets[0]
        crystal = importer.crystals[0]

        # generate predictions for possible reflections => generate a
        # reflection list

        predict = ReflectionPredictor()
        predicted = predict(sweep, crystal)

        # sort with James's reflection table: should this not go somewhere central?
        from dials.scratch.jmp.container.reflection_table import ReflectionTable

        # calculate shoebox sizes: take parameters from params & transform
        # from reciprocal space to image space to decide how big a shoe box to use

        table = ReflectionTable()
        table['miller_index'] = predicted.miller_index()
        indexer = table.index_map('miller_index')

        candidates = []

        unique = sorted(indexer)

        for h, k, l in unique:

            try:
                for _h in h - 1, h + 1:
                    if not indexer[(_h, k, l)]:
                        raise ValueError('missing')
                for _k in k - 1, k + 1:
                    if not indexer[(h, _k, l)]:
                        raise ValueError('missing')
                for _l in l - 1, l + 1:
                    if not indexer[(h, k, _l)]:
                        raise ValueError('missing')
                candidates.append((h, k, l))
            except ValueError:
                continue

        from dials.algorithms.simulation.utils import build_prediction_matrix

        from dials.algorithms.simulation.generate_test_reflections import \
         master_phil
        from libtbx.phil import command_line
        cmd = command_line.argument_interpreter(master_params=master_phil)
        working_phil = cmd.process_and_fetch(args=args[2:])
        params = working_phil.extract()

        node_size = params.rs_node_size
        window_size = params.rs_window_size
        reference = params.integrated_data_file
        scale = params.integrated_data_file_scale

        if reference:
            counts_database = {}
            from iotbx import mtz
            m = mtz.object(reference)
            mi = m.extract_miller_indices()
            i = m.extract_reals('IMEAN').data
            s = m.space_group().build_derived_point_group()
            for j in range(len(mi)):
                for op in s.all_ops():
                    hkl = tuple(map(int, op * mi[j]))
                    counts = max(0, int(math.floor(i[j] * scale)))
                    counts_database[hkl] = counts
                    counts_database[(-hkl[0], -hkl[1], -hkl[2])] = counts
        else:

            def constant_factory(value):
                import itertools
                return itertools.repeat(value).next

            from collections import defaultdict
            counts_database = defaultdict(constant_factory(params.counts))

        from dials.model.data import ReflectionList

        useful = ReflectionList()
        d_matrices = []

        for h, k, l in candidates:
            hkl = predicted[indexer[(h, k, l)][0]]
            _x = hkl.image_coord_px[0]
            _y = hkl.image_coord_px[1]
            _z = hkl.frame_number

            # build prediction matrix
            mhkl = predicted[indexer[(h - 1, k, l)][0]]
            phkl = predicted[indexer[(h + 1, k, l)][0]]
            hmkl = predicted[indexer[(h, k - 1, l)][0]]
            hpkl = predicted[indexer[(h, k + 1, l)][0]]
            hkml = predicted[indexer[(h, k, l - 1)][0]]
            hkpl = predicted[indexer[(h, k, l + 1)][0]]
            d = build_prediction_matrix(hkl, mhkl, phkl, hmkl, hpkl, hkml,
                                        hkpl)
            d_matrices.append(d)

            # construct the shoebox parameters: outline the ellipsoid
            x, y, z = [], [], []

            for dh in (1, 0, 0), (0, 1, 0), (0, 0, 1):
                dxyz = -1 * window_size * d * dh
                x.append(dxyz[0] + _x)
                y.append(dxyz[1] + _y)
                z.append(dxyz[2] + _z)
                dxyz = window_size * d * dh
                x.append(dxyz[0] + _x)
                y.append(dxyz[1] + _y)
                z.append(dxyz[2] + _z)

            hkl.bounding_box = (int(math.floor(min(x))),
                                int(math.floor(max(x)) + 1),
                                int(math.floor(min(y))),
                                int(math.floor(max(y)) + 1),
                                int(math.floor(min(z))),
                                int(math.floor(max(z)) + 1))
            try:
                counts = counts_database[hkl.miller_index]
                useful.append(hkl)
            except KeyError:
                continue

        from dials.algorithms import shoebox
        shoebox.allocate(useful)

        from dials.util.command_line import ProgressBar
        p = ProgressBar(title='Generating shoeboxes')

        # now for each reflection perform the simulation
        for j, refl in enumerate(useful):
            p.update(j * 100.0 / len(useful))
            d = d_matrices[j]

            from scitbx.random import variate, normal_distribution
            g = variate(normal_distribution(mean=0, sigma=node_size))
            counts = counts_database[refl.miller_index]
            dhs = g(counts)
            dks = g(counts)
            dls = g(counts)
            self.map_to_image_space(refl, d, dhs, dks, dls)

        p.finished('Generated %d shoeboxes' % len(useful))

        # now for each reflection add background
        from dials.algorithms.simulation.generate_test_reflections import \
         random_background_plane

        p = ProgressBar(title='Generating background')
        for j, refl in enumerate(useful):
            p.update(j * 100.0 / len(useful))
            if params.background:
                random_background_plane(refl.shoebox, params.background, 0.0,
                                        0.0, 0.0)
            else:
                random_background_plane(refl.shoebox, params.background_a,
                                        params.background_b,
                                        params.background_c,
                                        params.background_d)

        p.finished('Generated %d backgrounds' % len(useful))
        if params.output.all:
            with open(params.output.all, 'wb') as fh:
                pickle.dump(useful, fh, pickle.HIGHEST_PROTOCOL)