Esempio n. 1
0
def signal(imageset, indx):
    from dials.array_family import flex
    from libtbx.phil import parse

    detectors = imageset.get_detector()
    assert len(detectors) == 1
    detector = detectors[0]
    trusted = detector.get_trusted_range()

    data = imageset.get_raw_data(indx)
    assert len(data) == 1
    data = data[0]
    negative = data < 0
    hot = data > int(round(trusted[1]))
    bad = negative | hot

    from dials.algorithms.spot_finding.factory import SpotFinderFactory
    from dials.algorithms.spot_finding.factory import phil_scope

    data = data.as_double()

    spot_params = phil_scope.fetch(source=parse("min_spot_size=1")).extract()
    threshold_function = SpotFinderFactory.configure_threshold(spot_params)
    peak_pixels = threshold_function.compute_threshold(data, ~bad)
    return peak_pixels
Esempio n. 2
0
def find_constant_signal_pixels(imageset, images):
    """Find pixels which are constantly reporting as signal through the
    images in imageset: on every image the pixel dispersion index is computed,
    and signal pixels identified using the default settings. A map is then
    calculated of the number of times a pixel is identified as signal: if this
    is >= 50% of the images (say) that pixel is untrustworthy."""

    panels = imageset.get_detector()

    # only cope with monilithic detectors or the I23 Pilatus 12M
    assert len(panels) in (1, 24)

    # trusted range the same for all panels anyway
    detector = panels[0]
    trusted = detector.get_trusted_range()

    # construct an integer array same shape as image; accumulate number of
    # "signal" pixels in each pixel across data

    total = None

    for idx in images:
        pixels = imageset.get_raw_data(idx - 1)
        known_mask = imageset.get_mask(idx - 1)

        # apply known mask
        for _pixel, _panel, _mask in zip(pixels, panels, known_mask):
            _pixel.set_selected(~_mask, -1)
            for f0, s0, f1, s1 in _panel.get_mask():
                blank = flex.int(flex.grid(s1 - s0, f1 - f0), 0)
                _pixel.matrix_paste_block_in_place(blank, s0, f0)

        if len(pixels) == 1:
            data = pixels[0]
        else:
            ny, nx = pixels[0].focus()
            data = flex.int(flex.grid(24 * ny + 23 * 17, nx), -1)
            for j in range(24):
                data.matrix_paste_block_in_place(pixels[j], j * (ny + 17), 0)

        negative = data < int(round(trusted[0]))
        hot = data > int(round(trusted[1]))
        bad = negative | hot

        data = data.as_double()

        spot_params = spot_phil.fetch(
            source=iotbx.phil.parse("min_spot_size=1")
        ).extract()
        threshold_function = SpotFinderFactory.configure_threshold(spot_params)
        peak_pixels = threshold_function.compute_threshold(data, ~bad)

        if total is None:
            total = peak_pixels.as_1d().as_int()
        else:
            total += peak_pixels.as_1d().as_int()

    return total
Esempio n. 3
0
def extract_signal_mask(data):
    from dials.algorithms.spot_finding.factory import SpotFinderFactory
    from dials.algorithms.spot_finding.factory import phil_scope

    data = data.as_double()

    spot_params = phil_scope.fetch(source=iotbx.phil.parse(
        "spotfinder.threshold.dispersion.gain=1")).extract()
    threshold_function = SpotFinderFactory.configure_threshold(spot_params)
    negative = data < 0
    signal = threshold_function.compute_threshold(data, ~negative)

    return signal
Esempio n. 4
0
def extract_signal_mask(data):
  from dials.algorithms.spot_finding.factory import SpotFinderFactory
  from dials.algorithms.spot_finding.factory import phil_scope

  data = data.as_double()

  from dxtbx import datablock

  spot_params = phil_scope.fetch(source=iotbx.phil.parse(
    "spotfinder.threshold.xds.gain=1")).extract()
  threshold_function = SpotFinderFactory.configure_threshold(
    spot_params, None)
  negative = (data < 0)
  signal = threshold_function.compute_threshold(data, ~negative)

  return signal
Esempio n. 5
0
def find_signal_pixels(mask, image):
    """Mask: 2D array of unsigned char: if 0 -> invalid, if 1 -> valid,
    image: stack of 32 modules, so 3D numpy array of unsigned short. Returns
    3D array of unsigned char."""

    shape = mask.shape
    assert mask.shape == image.shape

    spot_params = spot_phil.fetch(source=iotbx.phil.parse(PHIL_SETTINGS)).extract()

    signal = np.empty(shape, dtype=np.uint8)
    threshold_function = SpotFinderFactory.configure_threshold(spot_params)

    for module in range(shape[0]):
        _mask = flex.int(mask[module, :, :].astype(np.int32)) == 1
        _image = flex.double(image[module, :, :].astype(np.float64))
        _signal = threshold_function.compute_threshold(_image, _mask)
        signal[module, :, :] = _signal.as_numpy_array().astype(np.uint8)

    return signal
Esempio n. 6
0
def background(imageset, indx, n_bins, mask_params=None):
    from dials.array_family import flex
    from libtbx.phil import parse
    from scitbx import matrix

    if mask_params is None:
        # Default mask params for trusted range
        mask_params = phil_scope.fetch(parse("")).extract().masking

    from dials.util.masking import MaskGenerator

    mask_generator = MaskGenerator(mask_params)
    mask = mask_generator.generate(imageset)

    detector = imageset.get_detector()
    beam = imageset.get_beam()
    # Only working with single panel detector for now
    assert len(detector) == 1
    panel = detector[0]
    mask = mask[0]

    n = matrix.col(panel.get_normal()).normalize()
    b = matrix.col(beam.get_s0()).normalize()
    wavelength = beam.get_wavelength()

    if math.fabs(b.dot(n)) < 0.95:
        raise Sorry("Detector not perpendicular to beam")

    data = imageset.get_raw_data(indx)
    assert len(data) == 1
    data = data[0]

    data = data.as_double()

    spot_params = spot_phil.fetch(source=parse("")).extract()
    threshold_function = SpotFinderFactory.configure_threshold(spot_params)
    peak_pixels = threshold_function.compute_threshold(data, mask)
    signal = data.select(peak_pixels.iselection())
    background_pixels = mask & ~peak_pixels
    background = data.select(background_pixels.iselection())

    # print some summary information
    print("Mean background: %.3f" % (flex.sum(background) / background.size()))
    if len(signal) > 0:
        print("Max/total signal pixels: %.0f / %.0f" %
              (flex.max(signal), flex.sum(signal)))
    else:
        print("No signal pixels on this image")
    print("Peak/background/masked pixels: %d / %d / %d" %
          (peak_pixels.count(True), background.size(), mask.count(False)))

    # compute histogram of two-theta values, then same weighted
    # by pixel values, finally divide latter by former to get
    # the radial profile out, need to set the number of bins
    # sensibly; inspired by method in PyFAI

    two_theta_array = panel.get_two_theta_array(beam.get_s0())
    two_theta_array = two_theta_array.as_1d().select(
        background_pixels.iselection())

    # Use flex.weighted_histogram
    h0 = flex.weighted_histogram(two_theta_array, n_slots=n_bins)
    h1 = flex.weighted_histogram(two_theta_array, background, n_slots=n_bins)
    h2 = flex.weighted_histogram(two_theta_array,
                                 background * background,
                                 n_slots=n_bins)

    d0 = h0.slots()
    d1 = h1.slots()
    d2 = h2.slots()

    I = d1 / d0
    I2 = d2 / d0
    sig = flex.sqrt(I2 - flex.pow2(I))

    tt = h0.slot_centers()
    d_spacings = wavelength / (2.0 * flex.sin(0.5 * tt))

    return d_spacings, I, sig
Esempio n. 7
0
def run(args):

    from dials.util.options import OptionParser
    from dials.util.options import flatten_experiments
    from dials.util.command_line import ProgressBar

    usage = "%s [options] data_master.h5" % (libtbx.env.dispatcher_name)

    parser = OptionParser(
        usage=usage,
        phil=phil_scope,
        read_experiments=True,
        read_experiments_from_images=True,
        epilog=help_message,
    )

    params, options = parser.parse_args(show_diff_phil=True)

    experiments = flatten_experiments(params.input.experiments)
    if len(experiments) != 1:
        parser.print_help()
        print("Please pass an experiment list\n")
        return

    imagesets = experiments.imagesets()

    if len(imagesets) != 1:
        raise Sorry("Please pass an experiment list that contains one imageset")

    imageset = imagesets[0]

    first, last = imageset.get_scan().get_image_range()
    images = range(first, last + 1)

    if params.images is None and params.image_range is not None:
        start, end = params.image_range
        params.images = list(range(start, end + 1))

    if params.images:
        if min(params.images) < first or max(params.images) > last:
            raise Sorry("image outside of scan range")
        images = params.images

    detectors = imageset.get_detector()
    assert len(detectors) == 1
    detector = detectors[0]
    trusted = detector.get_trusted_range()

    # construct an integer array same shape as image; accumulate number of
    # "signal" pixels in each pixel across data

    total = None

    p = ProgressBar(title="Finding hot pixels")

    for idx in images:

        p.update(idx * 100.0 / len(images))

        pixels = imageset.get_raw_data(idx - 1)
        assert len(pixels) == 1
        data = pixels[0]

        negative = data < int(round(trusted[0]))
        hot = data > int(round(trusted[1]))
        bad = negative | hot

        data = data.as_double()

        spot_params = spot_phil.fetch(
            source=iotbx.phil.parse("min_spot_size=1")
        ).extract()
        threshold_function = SpotFinderFactory.configure_threshold(spot_params)
        peak_pixels = threshold_function.compute_threshold(data, ~bad)

        if total is None:
            total = peak_pixels.as_1d().as_int()
        else:
            total += peak_pixels.as_1d().as_int()

    p.finished("Finished finding hot pixels on %d images" % len(images))

    hot_mask = total >= (len(images) // 2)
    hot_pixels = hot_mask.iselection()

    p = ProgressBar(title="Finding twinkies")

    twinkies = {}
    for h in hot_pixels:
        twinkies[h] = []

    for idx in images:

        p.update(idx * 100.0 / len(images))

        pixels = imageset.get_raw_data(idx - 1)
        data = pixels[0]

        for h in hot_pixels:
            twinkies[h].append(data[h])

    p.finished("Finished hunting for twinkies on %d images" % len(images))

    nslow, nfast = data.focus()

    ffff = 0

    for h in hot_pixels:
        if total[h] == len(images) and data[h] >= trusted[1]:
            ffff += 1
            continue
        print("Pixel %d at %d %d" % (total[h], h // nfast, h % nfast))
        if len(set(twinkies[h])) >= len(twinkies[h]) // 2:
            print("  ... many possible values")
            continue
        values = set(twinkies[h])
        result = [(twinkies[h].count(value), value) for value in values]
        for count, value in reversed(sorted(result)):
            print("  %08x %d" % (value, count))

    print("Also found %d very hot pixels" % ffff)
    hot_mask.reshape(flex.grid(data.focus()))

    easy_pickle.dump(params.output.mask, (~hot_mask,))
Esempio n. 8
0
def background(imageset, indx, n_bins):
  from dials.array_family import flex
  from libtbx.phil import parse
  from scitbx import matrix
  import math

  detector = imageset.get_detector()
  beam = imageset.get_beam()
  assert(len(detector) == 1)
  detector = detector[0]
  trusted = detector.get_trusted_range()

  n = matrix.col(detector.get_normal()).normalize()
  b = matrix.col(beam.get_s0()).normalize()
  wavelength = beam.get_wavelength()

  if math.fabs(b.dot(n)) < 0.95:
    from libtbx.utils import Sorry
    raise Sorry('Detector not perpendicular to beam')

  data = imageset.get_raw_data(indx)
  assert(len(data) == 1)
  data = data[0]
  negative = (data < 0)
  hot = (data > int(round(trusted[1])))
  bad = negative | hot

  from dials.algorithms.spot_finding.factory import SpotFinderFactory
  from dials.algorithms.spot_finding.factory import phil_scope

  data = data.as_double()

  from dxtbx import datablock

  spot_params = phil_scope.fetch(source=parse("")).extract()
  threshold_function = SpotFinderFactory.configure_threshold(
    spot_params, datablock.DataBlock([imageset]))
  peak_pixels = threshold_function.compute_threshold(data, ~bad)
  signal = data.select(peak_pixels.iselection())
  background = data.select((~bad & ~peak_pixels).iselection())

  # print some summary information
  print 'Mean background: %.3f' % (flex.sum(background) / background.size())
  print 'Max/total signal pixels: %.0f / %.0f' % (flex.max(signal),
                                                 flex.sum(signal))
  print 'Peak/background/hot pixels: %d / %d / %d' % (peak_pixels.count(True),
                                                      background.size(),
                                                      hot.count(True))

  # compute histogram of two-theta values, then same weighted
  # by pixel values, finally divide latter by former to get
  # the radial profile out, need to set the number of bins
  # sensibly; flex.histogram does not allow weights so use
  # numpy.histogram to get the same effect... inspired by
  # method in PyFAI

  data = data.as_1d()
  two_theta_array = detector.get_two_theta_array(beam.get_s0())
  two_theta_array.set_selected((bad | peak_pixels).iselection(), 0.0)
  data.set_selected((bad | peak_pixels).iselection(), 0.0)

  # new fangled flex.weighted_histogram :-)
  h0 = flex.weighted_histogram(two_theta_array, n_slots=n_bins)
  h1 = flex.weighted_histogram(two_theta_array, data, n_slots=n_bins)
  h2 = flex.weighted_histogram(two_theta_array, data * data, n_slots=n_bins)

  d0 = h0.slots()
  d1 = h1.slots()
  d2 = h2.slots()

  I = d1 / d0
  I2 = d2 / d0
  sig = flex.sqrt(I2 - flex.pow2(I))

  tt = h0.slot_centers()
  d_spacings = wavelength / (2.0 * flex.sin(0.5 * tt))

  return d_spacings, I, sig
Esempio n. 9
0
def background(imageset,
               indx,
               n_bins,
               corrected=False,
               mask_params=None,
               show_summary=False):
    if mask_params is None:
        # Default mask params for trusted range
        mask_params = phil_scope.fetch(parse("")).extract().masking

    detector = imageset.get_detector()
    beam = imageset.get_beam()

    # Only working with single panel detector for now
    assert len(detector) == 1
    panel = detector[0]
    imageset_mask = imageset.get_mask(indx)[0]
    mask = dials.util.masking.generate_mask(imageset, mask_params)[0]
    mask = imageset_mask & mask

    n = matrix.col(panel.get_normal()).normalize()
    b = matrix.col(beam.get_s0()).normalize()
    wavelength = beam.get_wavelength()

    if math.fabs(b.dot(n)) < 0.95:
        raise Sorry("Detector not perpendicular to beam")

    # Use corrected data to determine signal and background regions
    corrected_data = imageset.get_corrected_data(indx)
    assert len(corrected_data) == 1
    corrected_data = corrected_data[0].as_double()

    # Use choice of raw or corrected data to evaluate the background values
    if corrected:
        data = corrected_data
    else:
        data = imageset.get_raw_data(indx)[0].as_double()

    spot_params = spot_phil.fetch(source=parse("")).extract()
    threshold_function = SpotFinderFactory.configure_threshold(spot_params)
    peak_pixels = threshold_function.compute_threshold(corrected_data, mask)
    signal = data.select(peak_pixels.iselection())
    background_pixels = mask & ~peak_pixels
    background = data.select(background_pixels.iselection())

    # print some summary information
    if show_summary:
        logger.info(
            f"Mean background: {flex.sum(background) / background.size():.3f}")
        if len(signal) > 0:
            logger.info(
                f"Max/total signal pixels: {flex.max(signal):.0f} / {flex.sum(signal):.0f}"
            )
        else:
            logger.info("No signal pixels on this image")
        logger.info(
            "Peak/background/masked pixels: %d / %d / %d" %
            (peak_pixels.count(True), background.size(), mask.count(False)))

    # compute histogram of two-theta values, then same weighted
    # by pixel values, finally divide latter by former to get
    # the radial profile out, need to set the number of bins
    # sensibly; inspired by method in PyFAI

    two_theta_array = panel.get_two_theta_array(beam.get_s0())
    two_theta_array = two_theta_array.as_1d().select(
        background_pixels.iselection())

    # Use flex.weighted_histogram
    h0 = flex.weighted_histogram(two_theta_array, n_slots=n_bins)
    h1 = flex.weighted_histogram(two_theta_array, background, n_slots=n_bins)
    h2 = flex.weighted_histogram(two_theta_array,
                                 background * background,
                                 n_slots=n_bins)

    d0 = h0.slots()
    d1 = h1.slots()
    d2 = h2.slots()

    I = d1 / d0
    I2 = d2 / d0
    sig = flex.sqrt(I2 - flex.pow2(I))

    tt = h0.slot_centers()
    d_spacings = wavelength / (2.0 * flex.sin(0.5 * tt))

    return d_spacings, I, sig
Esempio n. 10
0
def background(imageset, indx, n_bins):
  from dials.array_family import flex
  from libtbx.phil import parse
  from scitbx import matrix
  import math

  detector = imageset.get_detector()
  beam = imageset.get_beam()
  assert(len(detector) == 1)
  detector = detector[0]
  trusted = detector.get_trusted_range()

  n = matrix.col(detector.get_normal()).normalize()
  b = matrix.col(beam.get_s0()).normalize()
  wavelength = beam.get_wavelength()

  if math.fabs(b.dot(n)) < 0.95:
    from libtbx.utils import Sorry
    raise Sorry('Detector not perpendicular to beam')

  data = imageset.get_raw_data(indx)
  assert(len(data) == 1)
  data = data[0]
  negative = (data < 0)
  hot = (data > int(round(trusted[1])))
  bad = negative | hot

  from dials.algorithms.spot_finding.factory import SpotFinderFactory
  from dials.algorithms.spot_finding.factory import phil_scope

  data = data.as_double()

  from dxtbx import datablock

  spot_params = phil_scope.fetch(source=parse("")).extract()
  threshold_function = SpotFinderFactory.configure_threshold(
    spot_params, datablock.DataBlock([imageset]))
  peak_pixels = threshold_function.compute_threshold(data, ~bad)
  signal = data.select(peak_pixels.iselection())
  background = data.select((~bad & ~peak_pixels).iselection())

  # print some summary information
  print('Mean background: %.3f' % (flex.sum(background) / background.size()))
  print('Max/total signal pixels: %.0f / %.0f' % (flex.max(signal),
                                                  flex.sum(signal)))
  print('Peak/background/hot pixels: %d / %d / %d' % (peak_pixels.count(True),
                                                      background.size(),
                                                      hot.count(True)))

  # compute histogram of two-theta values, then same weighted
  # by pixel values, finally divide latter by former to get
  # the radial profile out, need to set the number of bins
  # sensibly; flex.histogram does not allow weights so use
  # numpy.histogram to get the same effect... inspired by
  # method in PyFAI

  data = data.as_1d()
  two_theta_array = detector.get_two_theta_array(beam.get_s0())
  two_theta_array.set_selected((bad | peak_pixels).iselection(), 0.0)
  data.set_selected((bad | peak_pixels).iselection(), 0.0)

  # new fangled flex.weighted_histogram :-)
  h0 = flex.weighted_histogram(two_theta_array, n_slots=n_bins)
  h1 = flex.weighted_histogram(two_theta_array, data, n_slots=n_bins)
  h2 = flex.weighted_histogram(two_theta_array, data * data, n_slots=n_bins)

  d0 = h0.slots()
  d1 = h1.slots()
  d2 = h2.slots()

  I = d1 / d0
  I2 = d2 / d0
  sig = flex.sqrt(I2 - flex.pow2(I))

  tt = h0.slot_centers()
  d_spacings = wavelength / (2.0 * flex.sin(0.5 * tt))

  return d_spacings, I, sig
Esempio n. 11
0
def stability_fft(imageset, params):
  from i19.util.time_analysis import fft

  scan = imageset.get_scan()
  detector = imageset.get_detector()[0]
  if params.exposure_time:
    exposure = params.exposure_time
  else:
    exposure = scan.get_exposure_times()[0]

  trusted = detector.get_trusted_range()

  indices = imageset.indices()

  if params.scan_range:
    start, end = params.scan_range
    indices = indices[start:end]
  else:
    start = 0
    end = indices[-1] + 1

  counts = flex.double(len(indices))

  t0 = time.time()

  if params.remove_spots:
    from dials.algorithms.spot_finding.factory import SpotFinderFactory
    from dials.algorithms.spot_finding.factory import phil_scope
    from dxtbx import datablock
    spot_params = phil_scope.fetch(source=iotbx.phil.parse("")).extract()
    threshold_function = SpotFinderFactory.configure_threshold(
      spot_params, datablock.DataBlock([imageset]))
  else:
    threshold_function = None

  for i in indices:
    pixels = imageset.get_raw_data(i)[0].as_double()

    negative = (pixels < 0)
    hot = (pixels > int(round(trusted[1])))
    bad = negative | hot

    if threshold_function:
      peak_pixels = threshold_function.compute_threshold(pixels, ~bad)
      good = pixels.select((~bad & ~peak_pixels).iselection())
    else:
      good = pixels.select((~bad).iselection())

    counts[i-start] = flex.sum(good)

  t1 = time.time()

  print 'Read data for %d images in %.1fs' % (len(indices), t1 - t0)

  # scale data to give sensible FFT values

  mean = flex.sum(counts) / counts.size()
  counts /= mean

  power = fft(counts)

  f_hz = 1.0 / exposure
  f_scale = f_hz / counts.size()

  print 'Sample frequency: %.2f Hz' % f_hz
  print 'Writing output to: %s' % params.output_file

  fout = open(params.output_file, 'w')
  for j in range(power.size()):
    fout.write('%f %f\n' % (j * f_scale, power[j]))
  fout.close()