Ejemplo n.º 1
0
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.
    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(mod_radial_average, self).event(evt, env)
    if (evt.get("skip_event")):
      return

    # This module only applies to detectors for which a distance is
    # available.
    distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
    if distance is None:
      self.nfail += 1
      self.logger.warning("event(): no distance, shot skipped")
      evt.put(skip_event_flag(), "skip_event")
      return

    # See r17537 of mod_average.py.
    device = cspad_tbx.address_split(self.address)[2]
    if device == 'Cspad':
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
    elif device == 'marccd':
      pixel_size = 0.079346
      saturated_value = 2**16 - 1

    d = cspad_tbx.dpack(
      active_areas=self.active_areas,
      address=self.address,
      beam_center_x=pixel_size * self.beam_center[0],
      beam_center_y=pixel_size * self.beam_center[1],
      data=self.cspad_img.iround(), # XXX ouch!
      distance=distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=self.timestamp,
      wavelength=self.wavelength,
      xtal_target=self.m_xtal_target)

    from xfel.command_line.radial_average import run
    args = [
      "file_path=XTC stream",
      "xfel_target=%s"%self.m_xtal_target,
      "verbose=False"
    ]

    t = self.timestamp
    s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[20:23]

    if self._dirname is not None:
      dest_path = os.path.join(self._dirname, self._basename + s + ".txt")
      args.append("output_file=%s"%dest_path)

    self.logger.info("Calculating radial average for image %s"%s)
    xvals, results = run(args, d)

    evt.put(xvals, "cctbx.xfel.radial_average.xvals")
    evt.put(results, "cctbx.xfel.radial_average.results")
Ejemplo n.º 2
0
def do_work(img_no):
    n_fails = 0
    while True:
        try:
            raw_data = data.get_raw_data(img_no)
            break
        except (KeyError, ValueError):
            n_fails += 1
            print "Fail to read, attempt number", n_fails
            if n_fails > 100:
                raise Exception("Couldn't read the data")
        import time
        time.sleep(n_fails * 0.1)

    imgdict = cspad_tbx.dpack(data=raw_data,
                              distance=distance,
                              pixel_size=pixel_size,
                              wavelength=wavelength,
                              beam_center_x=beam_x,
                              beam_center_y=beam_y,
                              ccd_image_saturation=overload,
                              saturated_value=overload,
                              address="Sacla.MPCCD.8tile",
                              active_areas=active_areas)
    imgdict = crop_image_pickle(
        imgdict,
        preserve_active_areas_even_though_cropping_would_invalidate_them=True)

    dest_path = os.path.join(dest_dir, dest_base + "_%06d.pickle" % img_no)
    print "Saving image", img_no, "to", dest_path
    easy_pickle.dump(dest_path, imgdict)
Ejemplo n.º 3
0
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.

    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(mod_image_dict, self).event(evt, env)
    if (evt.get("skip_event")):
      return

    # This module only applies to detectors for which a distance is
    # available.
    distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
    if distance is None:
      self.nfail += 1
      self.logger.warning("event(): no distance, shot skipped")
      evt.put(skip_event_flag(), "skip_event")
      return

    device = cspad_tbx.address_split(self.address)[2]

    self.logger.info("Subprocess %02d: process image #%05d @ %s" %
                     (env.subprocess(), self.nshots, self.timestamp))

    # See r17537 of mod_average.py.
    if device == 'Cspad':
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
    elif device == 'Rayonix':
      pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
      saturated_value = rayonix_tbx.rayonix_saturated_value
    elif device == 'marccd':
      pixel_size = evt.get("marccd_pixel_size")
      saturated_value = evt.get("marccd_saturated_value")
      if distance == 0:
        distance = evt.get("marccd_distance")

    d = cspad_tbx.dpack(
      active_areas=self.active_areas,
      address=self.address,
      beam_center_x=pixel_size * self.beam_center[0],
      beam_center_y=pixel_size * self.beam_center[1],
      data=self.cspad_img.iround(), # XXX ouch!
      distance=distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=self.timestamp,
      wavelength=self.wavelength)

    evt.put(d, self.m_out_key)

    # Diagnostic message emitted only when all the processing is done.
    if (env.subprocess() >= 0):
      self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
                       (env.subprocess(), self.nshots, self.timestamp))
    else:
      self.logger.info("Accepted #%05d @ %s" %
                       (self.nshots, self.timestamp))
Ejemplo n.º 4
0
def save_image(
    command_line,
    imgpath,
    scan,
    raw_data,
    distance,
    pixel_size,
    wavelength,
    beam_x,
    beam_y,
    overload,
    timestamp,
    image_number=None,
):
    if image_number is None:
        destpath = os.path.join(
            os.path.dirname(imgpath),
            os.path.splitext(os.path.basename(imgpath))[0] + ".pickle",
        )
    else:
        destpath = os.path.join(
            os.path.dirname(imgpath),
            os.path.splitext(os.path.basename(imgpath))[0] +
            "%05d.pickle" % image_number,
        )
    if command_line.options.skip_converted and os.path.isfile(destpath):
        if command_line.options.verbose:
            print("Skipping %s, file exists" % imgpath)
            return

    data = dpack(
        data=raw_data,
        distance=distance,
        pixel_size=pixel_size,
        wavelength=wavelength,
        beam_center_x=beam_x,
        beam_center_y=beam_y,
        ccd_image_saturation=overload,
        saturated_value=overload,
        timestamp=timestamp,
    )

    if scan is not None:
        osc_start, osc_range = scan.get_oscillation()
        if osc_start != osc_range:
            data["OSC_START"] = osc_start
            data["OSC_RANGE"] = osc_range

            data["TIME"] = scan.get_exposure_times()[0]

    if command_line.options.crop:
        data = crop_image_pickle(data)

    if command_line.options.verbose:
        print("Writing", destpath)

    easy_pickle.dump(destpath, data)
Ejemplo n.º 5
0
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.  It
    outputs the detector image associated with the event @p evt to the
    file system.

    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(mod_dump, self).event(evt, env)
    if (evt.get('skip_event')):
      return

    if self.cspad_img is None:
      print "No image to save for %s"%self.timestamp
      return

    # Where the sample-detector distance is not available, set it to
    # zero.
    distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
    if distance is None:
      distance = 0

    # See r17537 of mod_average.py.
    device = cspad_tbx.address_split(self.address)[2]
    if device == 'Cspad':
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
      output_filename = self._basename
    elif device == 'Rayonix':
      pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
      saturated_value = rayonix_tbx.rayonix_saturated_value
      output_filename = self._basename
    elif device == 'marccd':
      if distance == 0:
        distance = evt.get('marccd_distance')
      pixel_size = 0.079346
      saturated_value = 2**16 - 1
      output_filename = self._basename + evt.get(str, 'mccd_name') + "_"

    d = cspad_tbx.dpack(
      active_areas=self.active_areas,
      address=self.address,
      beam_center_x=pixel_size * self.beam_center[0],
      beam_center_y=pixel_size * self.beam_center[1],
      data=self.cspad_img.iround(), # XXX ouch!
      distance=distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=self.timestamp,
      wavelength=self.wavelength)
    if self._format == "pickle":
      cspad_tbx.dwritef(d, self._dirname, output_filename)
    elif self._format == "tiff":
      cspad_tbx.write_tiff(d, self._dirname, output_filename)
    output_filename = None
Ejemplo n.º 6
0
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.  It
    outputs the detector image associated with the event @p evt to the
    file system.

    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(mod_dump, self).event(evt, env)
    if (evt.get('skip_event')):
      return

    if self.cspad_img is None:
      print("No image to save for %s"%self.timestamp)
      return

    # Where the sample-detector distance is not available, set it to
    # zero.
    distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
    if distance is None:
      distance = 0

    # See r17537 of mod_average.py.
    device = cspad_tbx.address_split(self.address)[2]
    if device == 'Cspad':
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
      output_filename = self._basename
    elif device == 'Rayonix':
      pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
      saturated_value = rayonix_tbx.rayonix_saturated_value
      output_filename = self._basename
    elif device == 'marccd':
      if distance == 0:
        distance = evt.get('marccd_distance')
      pixel_size = 0.079346
      saturated_value = 2**16 - 1
      output_filename = self._basename + evt.get(str, 'mccd_name') + "_"

    d = cspad_tbx.dpack(
      active_areas=self.active_areas,
      address=self.address,
      beam_center_x=pixel_size * self.beam_center[0],
      beam_center_y=pixel_size * self.beam_center[1],
      data=self.cspad_img.iround(), # XXX ouch!
      distance=distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=self.timestamp,
      wavelength=self.wavelength)
    if self._format == "pickle":
      cspad_tbx.dwritef(d, self._dirname, output_filename)
    elif self._format == "tiff":
      cspad_tbx.write_tiff(d, self._dirname, output_filename)
    output_filename = None
Ejemplo n.º 7
0
    def set_up_hitfinder(self):
        # See r17537 of mod_average.py.
        device = cspad_tbx.address_split(self.address)[2]
        if device == 'Cspad':
            img_dim = (1765, 1765)
            pixel_size = cspad_tbx.pixel_size
        elif device == 'marccd':
            img_dim = (4500, 4500)
            pixel_size = 0.079346
        elif device == 'Rayonix':
            img_dim = rayonix_tbx.get_rayonix_detector_dimensions(
                self.bin_size)
            pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
        else:
            raise RuntimeError("Unsupported device %s" % self.address)

        if self.beam_center is None:
            self.beam_center = [0, 0]

        self.hitfinder_d = cspad_tbx.dpack(
            active_areas=self.active_areas,
            beam_center_x=pixel_size * self.beam_center[0],
            beam_center_y=pixel_size * self.beam_center[1],
            data=flex.int(flex.grid(img_dim[0], img_dim[1]), 0),
            xtal_target=self.m_xtal_target)

        if device == 'Cspad':
            # Figure out which ASIC:s are on the central four sensors.  This
            # only applies to the CSPAD.
            assert len(self.active_areas) % 4 == 0
            distances = flex.double()
            for i in range(0, len(self.active_areas), 4):
                cenasic = (
                    (self.active_areas[i + 0] + self.active_areas[i + 2]) / 2,
                    (self.active_areas[i + 1] + self.active_areas[i + 3]) / 2)
                distances.append(
                    math.hypot(cenasic[0] - self.beam_center[0],
                               cenasic[1] - self.beam_center[1]))
            orders = flex.sort_permutation(distances)

            # Use the central 8 ASIC:s (central 4 sensors).
            flags = flex.int(len(self.active_areas) // 4, 0)
            for i in range(8):
                flags[orders[i]] = 1
            self.asic_filter = "distl.tile_flags=" + ",".join(
                ["%1d" % b for b in flags])

        elif device == 'marccd':
            # There is only one active area for the MAR CCD, so use it.
            self.asic_filter = "distl.tile_flags=1"
        elif device == 'Rayonix':
            # There is only one active area for the Rayonix, so use it.
            self.asic_filter = "distl.tile_flags=1"
Ejemplo n.º 8
0
  def set_up_hitfinder(self):
    # See r17537 of mod_average.py.
    device = cspad_tbx.address_split(self.address)[2]
    if device == 'Cspad':
      img_dim = (1765, 1765)
      pixel_size = cspad_tbx.pixel_size
    elif device == 'marccd':
      img_dim = (4500, 4500)
      pixel_size = 0.079346
    elif device == 'Rayonix':
      img_dim = rayonix_tbx.get_rayonix_detector_dimensions(self.bin_size)
      pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
    else:
      raise RuntimeError("Unsupported device %s" % self.address)

    if self.beam_center is None:
      self.beam_center = [0,0]

    self.hitfinder_d = cspad_tbx.dpack(
      active_areas=self.active_areas,
      beam_center_x=pixel_size * self.beam_center[0],
      beam_center_y=pixel_size * self.beam_center[1],
      data=flex.int(flex.grid(img_dim[0], img_dim[1]), 0),
      xtal_target=self.m_xtal_target)

    if device == 'Cspad':
      # Figure out which ASIC:s are on the central four sensors.  This
      # only applies to the CSPAD.
      assert len(self.active_areas) % 4 == 0
      distances = flex.double()
      for i in range(0, len(self.active_areas), 4):
        cenasic = ((self.active_areas[i + 0] + self.active_areas[i + 2]) / 2,
                   (self.active_areas[i + 1] + self.active_areas[i + 3]) / 2)
        distances.append(math.hypot(cenasic[0] - self.beam_center[0],
                                    cenasic[1] - self.beam_center[1]))
      orders = flex.sort_permutation(distances)

      # Use the central 8 ASIC:s (central 4 sensors).
      flags = flex.int(len(self.active_areas) // 4, 0)
      for i in range(8):
        flags[orders[i]] = 1
      self.asic_filter = "distl.tile_flags=" + ",".join(
        ["%1d" % b for b in flags])

    elif device == 'marccd':
      # There is only one active area for the MAR CCD, so use it.
      self.asic_filter = "distl.tile_flags=1"
    elif device == 'Rayonix':
      # There is only one active area for the Rayonix, so use it.
      self.asic_filter = "distl.tile_flags=1"
Ejemplo n.º 9
0
def run(args):
  assert len(args) == 3
  d1 = easy_pickle.load(args[0])
  d2 = easy_pickle.load(args[1])

  image_1 = d1["DATA"]
  image_2 = d2["DATA"]

  assert image_1.all() == image_2.all()
  diff_image = image_1 - image_2
  d = cspad_tbx.dpack(
    data=diff_image,
    timestamp=cspad_tbx.evt_timestamp(),
    distance=1,
  )
  easy_pickle.dump(args[2], d)
Ejemplo n.º 10
0
def run(args):
    assert len(args) == 3
    d1 = easy_pickle.load(args[0])
    d2 = easy_pickle.load(args[1])

    image_1 = d1["DATA"]
    image_2 = d2["DATA"]

    assert image_1.all() == image_2.all()
    diff_image = image_1 - image_2
    d = cspad_tbx.dpack(
        data=diff_image,
        timestamp=cspad_tbx.evt_timestamp(),
        distance=1,
    )
    easy_pickle.dump(args[2], d)
Ejemplo n.º 11
0
def save_image(command_line, imgpath, scan, raw_data, distance, pixel_size, wavelength, beam_x, beam_y, overload, timestamp, image_number = None):
  if image_number is None:
    destpath = os.path.join(os.path.dirname(imgpath), os.path.splitext(os.path.basename(imgpath))[0] + ".pickle")
  else:
    destpath = os.path.join(os.path.dirname(imgpath), os.path.splitext(os.path.basename(imgpath))[0] + "%05d.pickle"%image_number)
  if command_line.options.skip_converted and os.path.isfile(destpath):
    if command_line.options.verbose:
      print "Skipping %s, file exists"%imgpath
      return

  data = dpack(data=raw_data,
               distance=distance,
               pixel_size=pixel_size,
               wavelength=wavelength,
               beam_center_x=beam_x,
               beam_center_y=beam_y,
               ccd_image_saturation=overload,
               saturated_value=overload,
               timestamp=timestamp
               )

  if scan is not None:
    osc_start, osc_range = scan.get_oscillation()
    if osc_start != osc_range:
      data['OSC_START'] = osc_start
      data['OSC_RANGE'] = osc_range

      data['TIME'] = scan.get_exposure_times()[0]

  if command_line.options.crop:
    data = crop_image_pickle(data)

  if command_line.options.verbose:
    print "Writing", destpath

  easy_pickle.dump(destpath, data)
Ejemplo n.º 12
0
  def __init__(self,
               runs,
               output_dirname=".",
               roi=None):
    avg_basename="avg_"
    stddev_basename="stddev"
    self.sum_img = None
    self.sumsq_img = None
    self.nmemb = 0
    self.roi = cspad_tbx.getOptROI(roi)
    self.unbound_pixel_mask = cspad_unbound_pixel_mask()
    for i_run, run in enumerate(runs):
      run_scratch_dir = run
      result = finalise_one_run(run_scratch_dir)
      if result.sum_img is None: continue
      if self.sum_img is None:
        self.sum_img = result.sum_img
        self.sumsq_img = result.sumsq_img
      else:
        self.sum_img += result.sum_img
        self.sumsq_img += result.sumsq_img
      self.nmemb += result.nmemb

    self.avg_img = self.sum_img.as_double() / self.nmemb
    self.stddev_img = flex.sqrt((self.sumsq_img.as_double() - self.sum_img.as_double() * self.avg_img) / (self.nmemb - 1))

    self.mask = flex.int(self.sum_img.accessor(), 0)
    self.mask.set_selected(self.sum_img == 0, 1)
    self.mask.set_selected(self.unbound_pixel_mask > 0, 1)

    if (output_dirname is not None and
        avg_basename is not None):
      if (not os.path.isdir(output_dirname)):
        os.makedirs(output_dirname)
      d = cspad_tbx.dpack(
        address='CxiSc1-0|Cspad2x2-0',
        data=self.avg_img,
        distance=1,
      )
      cspad_tbx.dwritef(d, output_dirname, avg_basename)
      d = cspad_tbx.dpack(
        address='CxiSc1-0|Cspad2x2-0',
        data=self.sum_img,
        distance=1,
      )
      cspad_tbx.dwritef(d, output_dirname, "sum_")
      if 1:
        output_image(self.avg_img, "%s/avg.png" %output_dirname)
        output_image(self.avg_img, "%s/avg_inv.png" %output_dirname, invert=True)

      if 1:
        output_matlab_form(self.sum_img, "%s/sum.m" %output_dirname)
        output_matlab_form(self.avg_img, "%s/avg.m" %output_dirname)
        output_matlab_form(self.stddev_img, "%s/stddev.m" %output_dirname)

    if (stddev_basename is not None):
      d = cspad_tbx.dpack(
        address='CxiSc1-0|Cspad2x2-0',
        data=self.stddev_img,
        distance=1,
      )
      cspad_tbx.dwritef(d, output_dirname, stddev_basename)

      # XXX we should really figure out automatically the area where the spectrum is
      #write an integrated spectrum from lines 186-227
      #spectrum_focus = self.sum_img.as_numpy_array()[186:228,:]
      img = self.sum_img
      if self.roi is None:
        spectrum_focus = img
        mask_focus = self.mask
      else:
        slices = (slice(self.roi[2],self.roi[3]), slice(self.roi[0],self.roi[1]))
        spectrum_focus = img[slices]
        mask_focus = self.mask[slices]
      if False:
        from matplotlib import pylab
        pylab.imshow(spectrum_focus.as_numpy_array())
        pylab.show()

    output_spectrum(spectrum_focus, mask_focus=mask_focus,
                    output_dirname=output_dirname)

    print "Total number of images used from %i runs: %i" %(i_run+1, self.nmemb)
Ejemplo n.º 13
0
    def event(self, evt, env):
        """The event() function is called for every L1Accept transition.
    @param evt Event data object, a configure object
    @param env Environment object
    """

        super(mod_radial_average, self).event(evt, env)
        if (evt.get("skip_event")):
            return

        # This module only applies to detectors for which a distance is
        # available.
        distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
        if distance is None:
            self.nfail += 1
            self.logger.warning("event(): no distance, shot skipped")
            evt.put(skip_event_flag(), "skip_event")
            return

        # See r17537 of mod_average.py.
        device = cspad_tbx.address_split(self.address)[2]
        if device == 'Cspad':
            pixel_size = cspad_tbx.pixel_size
            saturated_value = cspad_tbx.cspad_saturated_value
        elif device == 'marccd':
            pixel_size = 0.079346
            saturated_value = 2**16 - 1
        elif device == 'Rayonix':
            pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
            saturated_value = rayonix_tbx.rayonix_saturated_value

        d = cspad_tbx.dpack(
            active_areas=self.active_areas,
            address=self.address,
            beam_center_x=pixel_size * self.beam_center[0],
            beam_center_y=pixel_size * self.beam_center[1],
            data=self.cspad_img.iround(),  # XXX ouch!
            distance=distance,
            pixel_size=pixel_size,
            saturated_value=saturated_value,
            timestamp=self.timestamp,
            wavelength=self.wavelength,
            xtal_target=self.m_xtal_target)

        from xfel.command_line.radial_average import run
        args = [
            "file_path=XTC stream",
            "xfel_target=%s" % self.m_xtal_target, "verbose=False"
        ]

        t = self.timestamp
        s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[
            20:23]

        if self._dirname is not None:
            dest_path = os.path.join(self._dirname,
                                     self._basename + s + ".txt")
            args.append("output_file=%s" % dest_path)

        self.logger.info("Calculating radial average for image %s" % s)
        xvals, results = run(args, d)

        evt.put(xvals, "cctbx.xfel.radial_average.xvals")
        evt.put(results, "cctbx.xfel.radial_average.results")

        def get_closest_idx(data, val):
            from scitbx.array_family import flex
            deltas = flex.abs(data - val)
            return flex.first_index(deltas, flex.min(deltas))

        if self._two_theta_low is not None:
            i_low = results[get_closest_idx(xvals, self._two_theta_low)]
            evt.put(i_low, "cctbx.xfel.radial_average.two_theta_low")

        if self._two_theta_high is not None:
            i_high = results[get_closest_idx(xvals, self._two_theta_high)]
            evt.put(i_high, "cctbx.xfel.radial_average.two_theta_high")
Ejemplo n.º 14
0
def run(args):
  command_line = (option_parser()
                  .option("-o", "--output_filename",
                          action="store",
                          type="string",
                          help="Filename for the output pickle file",
                          default="gain_map.pickle")
                  .option("-f", "--detector_format_version",
                          action="store",
                          type="string",
                          help="Detector format version to use for generating active areas and laying out tiles",
                          default=None)
                  .option("-m", "--optical_metrology_path",
                          action="store",
                          type="string",
                          help="Path to slac optical metrology file. If not set, use Run 4 metrology",
                          default=None)
                  .option("-d", "--distance",
                          action="store",
                          type="int",
                          help="Detector distance put into the gain pickle file. Not needed for processing.",
                          default="0")
                  .option("-w", "--wavelength",
                          action="store",
                          type="float",
                          help="Incident beam wavelength put into the gain pickle file. Not needed for processing.",
                          default="0")
                     ).process(args=args)
  output_filename = command_line.options.output_filename
  detector_format_version = command_line.options.detector_format_version
  if detector_format_version is None or 'XPP' not in detector_format_version:
    beam_center_x = None
    beam_center_y = None
  else:
    beam_center_x = 1765 // 2 * 0.11
    beam_center_y = 1765 // 2 * 0.11
  address, timestamp = address_and_timestamp_from_detector_format_version(detector_format_version)

  # if no detector format version is provided, make sure to write no address to the image pickle
  # but CsPadDetector (called later), needs an address, so give it a fake one
  save_address = address is not None
  if not save_address:
    address = "CxiDs1-0|Cspad-0" # time stamp will still be None
  timestamp = evt_timestamp((timestamp,0))
  args = command_line.args
  assert len(args) == 1
  if args[0].endswith('.npy'):
    data = numpy.load(args[0])
    det, active_areas = convert_2x2(data, detector_format_version, address)
  elif args[0].endswith('.txt') or args[0].endswith('.gain'):
    raw_data = numpy.loadtxt(args[0])
    assert raw_data.shape in [(5920, 388), (11840, 194)]
    det, active_areas = convert_detector(raw_data, detector_format_version, address, command_line.options.optical_metrology_path)
  img_diff = det
  img_sel = (img_diff > 0).as_1d()
  gain_map = flex.double(img_diff.accessor(), 0)
  gain_map.as_1d().set_selected(img_sel.iselection(), 1/img_diff.as_1d().select(img_sel))
  gain_map /= flex.mean(gain_map.as_1d().select(img_sel))

  if not save_address:
    address = None
  d = cspad_tbx.dpack(data=gain_map, address=address, active_areas=active_areas, timestamp=timestamp,
    distance=command_line.options.distance,wavelength=command_line.options.wavelength,
    beam_center_x = beam_center_x, beam_center_y = beam_center_y)
  easy_pickle.dump(output_filename, d)
Ejemplo n.º 15
0
  def load_image(self):
    """ Reads raw image file and extracts data for conversion into pickle
        format. Also estimates gain if turned on."""
    # Load raw image or image pickle

    try:
      with misc.Capturing() as junk_output:
        loaded_img = dxtbx.load(self.raw_img)
    except Exception as e:
      print 'IOTA IMPORT ERROR:', e
      loaded_img = None
      pass

    # Extract image information
    if loaded_img is not None:
      raw_data   = loaded_img.get_raw_data()
      detector   = loaded_img.get_detector()[0]
      beam       = loaded_img.get_beam()
      scan       = loaded_img.get_scan()
      distance   = detector.get_distance()
      pixel_size = detector.get_pixel_size()[0]
      overload   = detector.get_trusted_range()[1]
      wavelength = beam.get_wavelength()
      beam_x     = detector.get_beam_centre(beam.get_s0())[0]
      beam_y     = detector.get_beam_centre(beam.get_s0())[1]

      if scan is None:
        timestamp = None
        img_type = 'pickle'
      else:
        img_type = 'raw'
        msec, sec = math.modf(scan.get_epochs()[0])
        timestamp = evt_timestamp((sec,msec))

      # Assemble datapack
      data = dpack(data=raw_data,
                   distance=distance,
                   pixel_size=pixel_size,
                   wavelength=wavelength,
                   beam_center_x=beam_x,
                   beam_center_y=beam_y,
                   ccd_image_saturation=overload,
                   saturated_value=overload,
                   timestamp=timestamp
                   )

      if scan is not None:
        osc_start, osc_range = scan.get_oscillation()
        if osc_start != osc_range:
          data['OSC_START'] = 0 #osc_start
          data['OSC_RANGE'] = 0 #osc_start
          data['TIME'] = scan.get_exposure_times()[0]
    else:
      data = None
      img_type = 'not imported'

    # Estimate gain (or set gain to 1.00 if cannot calculate)
    # Cribbed from estimate_gain.py by Richard Gildea
    if self.params.advanced.estimate_gain:
      from dxtbx.datablock import DataBlockFactory
      from dials.command_line.estimate_gain import estimate_gain
      with misc.Capturing() as junk_output:
        try:
          datablock = DataBlockFactory.from_filenames([self.raw_img])[0]
          imageset = datablock.extract_imagesets()[0]
          self.gain = estimate_gain(imageset)
        except Exception as e:
          self.gain = 1.0
    else:
      self.gain = 1.0

    return data, img_type
Ejemplo n.º 16
0
class xes_from_histograms(object):
    def __init__(self,
                 pixel_histograms,
                 output_dirname=".",
                 gain_map_path=None,
                 gain_map=None,
                 estimated_gain=30,
                 roi=None,
                 run=None):

        self.sum_img = flex.double(flex.grid(
            370, 391), 0)  # XXX define the image size some other way?
        gain_img = flex.double(self.sum_img.accessor(), 0)

        assert [gain_map, gain_map_path].count(None) > 0
        if gain_map_path is not None:
            d = easy_pickle.load(gain_map_path)
            gain_map = d["DATA"]

        mask = flex.int(self.sum_img.accessor(), 0)

        start_row = 370
        end_row = 0
        print len(pixel_histograms.histograms)

        pixels = list(pixel_histograms.pixels())
        n_pixels = len(pixels)
        if roi is not None:
            for k, (i, j) in enumerate(reversed(pixels)):
                if (i < roi[2] or i > roi[3] or j < roi[0] or j > roi[1]):
                    del pixels[n_pixels - k - 1]

        if gain_map is None:
            fixed_func = pixel_histograms.fit_one_histogram
        else:

            def fixed_func(pixel):
                return pixel_histograms.fit_one_histogram(pixel, n_gaussians=1)

        chi_squared_list = flex.double()

        for i, pixel in enumerate(pixels):
            #print i,pixel
            LEG = False
            start_row = min(start_row, pixel[0])
            end_row = max(end_row, pixel[0])
            n_photons = 0

            try:
                if LEG:
                    gaussians, two_photon_flag = pixel_histograms.fit_one_histogram(
                        pixel)
                alt_gaussians = pixel_histograms.fit_one_histogram_two_gaussians(
                    pixel)
            except ZeroDivisionError:
                print "HEY DIVIDE BY ZERO"
                #pixel_histograms.plot_combo(pixel, gaussians)
                mask[pixel] = 1
                continue
            except RuntimeError, e:
                print "Error fitting pixel %s" % str(pixel)
                print str(e)
                mask[pixel] = 1
                continue

            hist = pixel_histograms.histograms[pixel]

            if not LEG:
                gs = alt_gaussians[1].params
                fit_photons = gs[0] * gs[2] * math.sqrt(2. * math.pi)
                n_photons = int(round(fit_photons, 0))
                fit_interpretation = pixel_histograms.multiphoton_and_fit_residual(
                    pixel_histograms.histograms[pixel], alt_gaussians)
                multi_photons = fit_interpretation.get_multiphoton_count()
                total_photons = n_photons + multi_photons

                if False and n_photons < 0:  # Generally, do not mask negative values; if fit is still OK
                    print "\n%d pixel %s altrn %d photons from curvefitting" % (
                        i, pixel, n_photons)
                    pixel_histograms.plot_combo(
                        pixel,
                        alt_gaussians,
                        interpretation=fit_interpretation)
                    mask[
                        pixel] = 1  # do not mask out negative pixels if the Gaussian fit is good
                    continue

                chi_squared_list.append(fit_interpretation.chi_squared())
                suspect = False  # don't know the optimal statistical test.  Histograms vary primarily by total count & # photons
                if total_photons <= 3:
                    if fit_interpretation.chi_squared(
                    ) > 2.5 or fit_interpretation.quality_factor < 5:
                        suspect = True
                elif 3 < total_photons <= 10:
                    if fit_interpretation.chi_squared(
                    ) > 5 or fit_interpretation.quality_factor < 10:
                        suspect = True
                elif 10 < total_photons <= 33:
                    if fit_interpretation.chi_squared(
                    ) > 10 or fit_interpretation.quality_factor < 20:
                        suspect = True
                elif 33 < total_photons <= 100:
                    if fit_interpretation.chi_squared(
                    ) > 20 or fit_interpretation.quality_factor < 20:
                        suspect = True
                elif 100 < total_photons <= 330:
                    if fit_interpretation.chi_squared(
                    ) > 30 or fit_interpretation.quality_factor < 25:
                        suspect = True
                elif 330 < total_photons <= 1000:
                    if fit_interpretation.chi_squared(
                    ) > 40 or fit_interpretation.quality_factor < 30:
                        suspect = True
                elif 1000 < total_photons:
                    if fit_interpretation.chi_squared(
                    ) > 50 or fit_interpretation.quality_factor < 30:
                        suspect = True

                if suspect:
                    print "\n%d pixel %s Bad quality 0/1-photon fit" % (
                        i, pixel), fit_interpretation.quality_factor
                    print "   with chi-squared %10.5f" % fit_interpretation.chi_squared(
                    )
                    print "   Suspect", suspect
                    print "%d fit photons, %d total photons" % (n_photons,
                                                                total_photons)
                    #pixel_histograms.plot_combo(pixel, alt_gaussians,
                    #                            interpretation=fit_interpretation)
                    mask[pixel] = 1
                    continue

                self.sum_img[pixel] = n_photons + multi_photons

        mask.set_selected(self.sum_img == 0, 1)
        unbound_pixel_mask = xes_finalise.cspad_unbound_pixel_mask()
        mask.set_selected(unbound_pixel_mask > 0, 1)
        bad_pixel_mask = xes_finalise.cspad2x2_bad_pixel_mask_cxi_run7()
        mask.set_selected(bad_pixel_mask > 0, 1)

        for row in range(self.sum_img.all()[0]):
            self.sum_img[row:row + 1, :].count(0)

        spectrum_focus = self.sum_img[start_row:end_row, :]
        mask_focus = mask[start_row:end_row, :]

        spectrum_focus.set_selected(mask_focus > 0, 0)

        xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)

        print "Number of rows: %i" % spectrum_focus.all()[0]
        print "Estimated no. photons counted: %i" % flex.sum(spectrum_focus)
        print "Number of images used: %i" % flex.sum(
            pixel_histograms.histograms.values()[0].slots())

        d = cspad_tbx.dpack(
            address='CxiSc1-0|Cspad2x2-0',
            data=spectrum_focus,
            distance=1,
            ccd_image_saturation=2e8,  # XXX
        )
        if run is not None: runstr = "_%04d" % run
        else: runstr = ""
        cspad_tbx.dwritef(d, output_dirname, 'sum%s_' % runstr)

        plot_x, plot_y = xes_finalise.output_spectrum(
            spectrum_focus.iround(),
            mask_focus=mask_focus,
            output_dirname=output_dirname,
            run=run)
        self.spectrum = (plot_x, plot_y)
        self.spectrum_focus = spectrum_focus
        xes_finalise.output_matlab_form(
            spectrum_focus, "%s/sum%s.m" % (output_dirname, runstr))
        print output_dirname
        print "Average chi squared is", flex.mean(
            chi_squared_list), "on %d shots" % flex.sum(hist.slots())
Ejemplo n.º 17
0
def run(argv=None):
    import libtbx.option_parser

    if (argv is None):
        argv = sys.argv

    command_line = (libtbx.option_parser.option_parser(
        usage=
        "%s [-v] [-p poly_mask] [-c circle_mask] [-a avg_max] [-s stddev_max] [-m maxproj_min] [-x mask_pix_val] [-o output] avg_path stddev_path max_path"
        % libtbx.env.dispatcher_name
    ).option(
        None,
        "--verbose",
        "-v",
        action="store_true",
        default=False,
        dest="verbose",
        help="Print more information about progress"
    ).option(
        None,
        "--poly_mask",
        "-p",
        type="string",
        default=None,
        dest="poly_mask",
        help="Polygon to mask out.  Comma-seperated string of xy pairs."
    ).option(
        None,
        "--circle_mask",
        "-c",
        type="string",
        default=None,
        dest="circle_mask",
        help="Circle to mask out.  Comma-seperated string of x, y, and radius."
    ).option(
        None,
        "--avg_max",
        "-a",
        type="float",
        default=2000.0,
        dest="avg_max",
        help=
        "Maximum ADU that pixels in the average image are allowed to have before masked out"
    ).option(
        None,
        "--stddev_max",
        "-s",
        type="float",
        default=10.0,
        dest="stddev_max",
        help=
        "Maximum ADU that pixels in the standard deviation image are allowed to have before masked out"
    ).option(
        None,
        "--maxproj_min",
        "-m",
        type="float",
        default=300.0,
        dest="maxproj_min",
        help=
        "Minimum ADU that pixels in the maximum projection image are allowed to have before masked out"
    ).option(None,
             "--mask_pix_val",
             "-x",
             type="int",
             default=-2,
             dest="mask_pix_val",
             help="Value for masked out pixels").option(
                 None,
                 "--detector_format_version",
                 "-d",
                 type="string",
                 default=None,
                 dest="detector_format_version",
                 help="detector format version string").option(
                     None,
                     "--output",
                     "-o",
                     type="string",
                     default="mask_.pickle",
                     dest="destpath",
                     help="output file path, should be *.pickle")).process(
                         args=argv[1:])

    # Must have exactly three remaining arguments.
    paths = command_line.args
    if (len(paths) != 3):
        command_line.parser.print_usage(file=sys.stderr)
        return

    if command_line.options.detector_format_version is None:
        address = timestamp = None
    else:
        from xfel.cxi.cspad_ana.cspad_tbx import evt_timestamp
        from iotbx.detectors.cspad_detector_formats import address_and_timestamp_from_detector_format_version
        address, timestamp = address_and_timestamp_from_detector_format_version(
            command_line.options.detector_format_version)
        timestamp = evt_timestamp((timestamp, 0))

    poly_mask = None
    if not command_line.options.poly_mask == None:
        poly_mask = []
        poly_mask_tmp = command_line.options.poly_mask.split(",")
        if len(poly_mask_tmp) % 2 != 0:
            command_line.parser.print_usage(file=sys.stderr)
            return
        odd = True
        for item in poly_mask_tmp:
            try:
                if odd:
                    poly_mask.append(int(item))
                else:
                    poly_mask[-1] = (poly_mask[-1], int(item))
            except ValueError:
                command_line.parser.print_usage(file=sys.stderr)
                return
            odd = not odd

    circle_mask = None
    if command_line.options.circle_mask is not None:
        circle_mask_tmp = command_line.options.circle_mask.split(",")
        if len(circle_mask_tmp) != 3:
            command_line.parser.print_usage(file=sys.stderr)
            return
        try:
            circle_mask = (int(circle_mask_tmp[0]), int(circle_mask_tmp[1]),
                           int(circle_mask_tmp[2]))
        except ValueError:
            command_line.parser.print_usage(file=sys.stderr)
            return

    avg_path = paths[0]
    stddev_path = paths[1]
    max_path = paths[2]

    # load the three images
    format_class = Registry.find(avg_path)
    avg_f = format_class(avg_path)
    avg_i = avg_f.get_detectorbase()
    avg_d = avg_i.get_raw_data()

    stddev_f = format_class(stddev_path)
    stddev_i = stddev_f.get_detectorbase()
    stddev_d = stddev_i.get_raw_data()

    max_f = format_class(max_path)
    max_i = max_f.get_detectorbase()
    max_d = max_i.get_raw_data()

    # first find all the pixels in the average that are less than zero or greater
    # than a cutoff and set them to the masking value
    avg_d.set_selected((avg_d <= 0) | (avg_d > command_line.options.avg_max),
                       command_line.options.mask_pix_val)

    # set all the rest of the pixels to zero.  They will be accepted
    avg_d.set_selected(avg_d != command_line.options.mask_pix_val, 0)

    # mask out the overly noisy or flat pixels
    avg_d.set_selected(stddev_d <= 0, command_line.options.mask_pix_val)
    avg_d.set_selected(stddev_d >= command_line.options.stddev_max,
                       command_line.options.mask_pix_val)

    # these are the non-bonded pixels
    avg_d.set_selected(max_d < command_line.options.maxproj_min,
                       command_line.options.mask_pix_val)

    # calculate the beam center
    panel = avg_f.get_detector()[0]
    bcx, bcy = panel.get_beam_centre(avg_f.get_beam().get_s0())

    if poly_mask is not None or circle_mask is not None:
        minx = miny = 0
        maxx = avg_d.focus()[0]
        maxy = avg_d.focus()[1]
        if poly_mask is not None:
            minx = min([x[0] for x in poly_mask])
            miny = min([y[1] for y in poly_mask])
            maxx = max([x[0] for x in poly_mask])
            maxy = max([y[1] for y in poly_mask])
        if circle_mask is not None:
            circle_x, circle_y, radius = circle_mask

            if circle_x - radius < minx: minx = circle_x - radius
            if circle_y - radius < miny: miny = circle_y - radius
            if circle_x + radius > maxx: maxx = circle_x + radius
            if circle_y + radius > maxy: maxy = circle_y + radius

        sel = avg_d == command_line.options.mask_pix_val
        for j in range(miny, maxy):
            for i in range(minx, maxx):
                idx = j * avg_d.focus()[0] + i
                if not sel[idx]:
                    if poly_mask is not None and point_in_polygon(
                        (i, j), poly_mask):
                        sel[idx] = True
                    elif circle_mask is not None and point_inside_circle(
                            i, j, circle_x, circle_y, radius):
                        sel[idx] = True
        avg_d.set_selected(sel, command_line.options.mask_pix_val)

    # have to re-layout the data to match how it was stored originally
    shifted_int_data_old = avg_d
    shifted_int_data_new = shifted_int_data_old.__class__(
        flex.grid(shifted_int_data_old.focus()))
    shifted_int_data_new += command_line.options.mask_pix_val

    phil = avg_i.horizons_phil_cache
    manager = avg_i.get_tile_manager(phil)

    for i, shift in enumerate(manager.effective_translations()):
        shift_slow = shift[0]
        shift_fast = shift[1]

        ur_slow = phil.distl.detector_tiling[4 * i + 0] + shift_slow
        ur_fast = phil.distl.detector_tiling[4 * i + 1] + shift_fast
        ll_slow = phil.distl.detector_tiling[4 * i + 2] + shift_slow
        ll_fast = phil.distl.detector_tiling[4 * i + 3] + shift_fast

        #print "Shifting tile at (%d, %d) by (%d, %d)" % (ur_slow-shift_slow, ur_fast-shift_fast, -shift_slow, -shift_fast)

        shifted_int_data_new.matrix_paste_block_in_place(
            block=shifted_int_data_old.matrix_copy_block(
                i_row=ur_slow,
                i_column=ur_fast,
                n_rows=ll_slow - ur_slow,
                n_columns=ll_fast - ur_fast),
            i_row=ur_slow - shift_slow,
            i_column=ur_fast - shift_fast)

    d = dpack(active_areas=avg_i.parameters['ACTIVE_AREAS'],
              address=address,
              beam_center_x=bcx,
              beam_center_y=bcy,
              data=shifted_int_data_new,
              distance=avg_i.distance,
              timestamp=timestamp,
              wavelength=avg_i.wavelength,
              xtal_target=None,
              pixel_size=avg_i.pixel_size,
              saturated_value=avg_i.saturation)

    dwritef2(d, command_line.options.destpath)

    #the minimum number of pixels to mask out cooresponding to the interstitial regions for the CS-PAD
    min_count = 818265  # (1765 * 1765) - (194 * 185 * 64)
    masked_out = len(avg_d.as_1d().select(
        (avg_d == command_line.options.mask_pix_val).as_1d()))
    assert masked_out >= min_count

    print "Masked out %d pixels out of %d (%.2f%%)"% \
      (masked_out-min_count,len(avg_d)-min_count,(masked_out-min_count)*100/(len(avg_d)-min_count))
Ejemplo n.º 18
0
class SingleImage(object):
    def __init__(self, img, init, verbose=True, imported_grid=None):
        """ Constructor for the SingleImage object using a raw image file or pickle
    """

        # Initialize parameters
        self.params = init.params
        self.args = init.args
        self.raw_img = img[2]
        self.conv_img = img[2]
        self.img_index = img[0]
        self.status = None
        self.fail = None
        self.final = None
        self.log_info = []
        self.gs_results = []
        self.main_log = init.logfile
        self.verbose = verbose
        self.hmed = self.params.cctbx.grid_search.height_median
        self.amed = self.params.cctbx.grid_search.area_median

        self.input_base = init.input_base
        self.conv_base = init.conv_base
        self.int_base = init.int_base
        self.obj_base = init.obj_base
        self.fin_base = init.fin_base
        self.viz_base = init.viz_base
        self.tmp_base = init.tmp_base
        self.abort_file = os.path.join(self.int_base, '.abort.tmp')

        self.obj_path = None
        self.obj_file = None
        self.fin_path = None
        self.fin_file = None
        self.viz_path = None

# ============================== SELECTION-ONLY FUNCTIONS ============================== #

    def import_int_file(self, init):
        """ Replaces path settings in imported image object with new settings
        NEED TO RE-DO LATER """

        if os.path.isfile(self.abort_file):
            self.fail = 'aborted'
            return self

        # Generate paths to output files
        self.params = init.params
        self.main_log = init.logfile
        self.input_base = init.input_base
        self.conv_base = init.conv_base
        self.int_base = init.int_base
        self.obj_base = init.obj_base
        self.fin_base = init.fin_base
        self.viz_base = init.viz_base
        self.obj_path = misc.make_image_path(self.conv_img, self.input_base,
                                             self.obj_base)
        self.obj_file = os.path.abspath(
            os.path.join(
                self.obj_path,
                os.path.basename(self.conv_img).split('.')[0] + ".int"))
        self.fin_path = misc.make_image_path(self.conv_img, self.input_base,
                                             self.fin_base)
        self.fin_file = os.path.abspath(
            os.path.join(
                self.fin_path,
                os.path.basename(self.conv_img).split('.')[0] + "_int.pickle"))
        self.final['final'] = self.fin_file
        self.final['img'] = self.conv_img
        self.viz_path = misc.make_image_path(self.conv_img, self.input_base,
                                             self.viz_base)
        self.viz_file = os.path.join(
            self.viz_path,
            os.path.basename(self.conv_img).split('.')[0] + "_int.png")

        # Create actual folders (if necessary)
        try:
            if not os.path.isdir(self.obj_path):
                os.makedirs(self.obj_path)
            if not os.path.isdir(self.fin_path):
                os.makedirs(self.fin_path)
            if not os.path.isdir(self.viz_path):
                os.makedirs(self.viz_path)
        except OSError:
            pass

        # Grid search / integration log file
        self.int_log = os.path.join(
            self.fin_path,
            os.path.basename(self.conv_img).split('.')[0] + '.tmp')

        # Reset status to 'grid search' to pick up at selection (if no fail)
        if self.fail == None:
            self.status = 'bypass grid search'

        return self

    def determine_gs_result_file(self):
        """ For 'selection-only' cctbx.xfel runs, determine where the image objects are """
        if self.params.cctbx.selection.select_only.grid_search_path != None:
            obj_path = os.path.abspath(
                self.params.cctbx.selection.select_only.grid_search_path)
        else:
            run_number = int(os.path.basename(self.int_base)) - 1
            obj_path = "{}/integration/{:03d}/image_objects"\
                      "".format(os.path.abspath(os.curdir), run_number)
        gs_result_file = os.path.join(obj_path,
                                      os.path.basename(self.obj_file))
        return gs_result_file

# =============================== IMAGE IMPORT FUNCTIONS =============================== #

    def load_image(self):
        """ Reads raw image file and extracts data for conversion into pickle
        format. Also estimates gain if turned on."""
        # Load raw image or image pickle

        try:
            with misc.Capturing() as junk_output:
                loaded_img = dxtbx.load(self.raw_img)
        except IOError, e:
            loaded_img = None
            pass

        # Extract image information
        if loaded_img is not None:
            raw_data = loaded_img.get_raw_data()
            detector = loaded_img.get_detector()[0]
            beam = loaded_img.get_beam()
            scan = loaded_img.get_scan()
            distance = detector.get_distance()
            pixel_size = detector.get_pixel_size()[0]
            overload = detector.get_trusted_range()[1]
            wavelength = beam.get_wavelength()
            beam_x = detector.get_beam_centre(beam.get_s0())[0]
            beam_y = detector.get_beam_centre(beam.get_s0())[1]

            if scan is None:
                timestamp = None
                img_type = 'pickle'
            else:
                img_type = 'raw'
                msec, sec = math.modf(scan.get_epochs()[0])
                timestamp = evt_timestamp((sec, msec))

            # Assemble datapack
            data = dpack(data=raw_data,
                         distance=distance,
                         pixel_size=pixel_size,
                         wavelength=wavelength,
                         beam_center_x=beam_x,
                         beam_center_y=beam_y,
                         ccd_image_saturation=overload,
                         saturated_value=overload,
                         timestamp=timestamp)

            if scan is not None:
                osc_start, osc_range = scan.get_oscillation()
                if osc_start != osc_range:
                    data['OSC_START'] = 0  #osc_start
                    data['OSC_RANGE'] = 0  #osc_start
                    data['TIME'] = scan.get_exposure_times()[0]
        else:
            data = None
            img_type = 'not imported'

        # Estimate gain (or set gain to 1.00 if cannot calculate)
        # Cribbed from estimate_gain.py by Richard Gildea
        if self.params.advanced.estimate_gain:
            try:
                from dials.algorithms.image.threshold import KabschDebug
                raw_data = [raw_data]

                gain_value = 1
                kernel_size = (10, 10)
                gain_map = [
                    flex.double(raw_data[i].accessor(), gain_value)
                    for i in range(len(loaded_img.get_detector()))
                ]
                mask = loaded_img.get_mask()
                min_local = 0

                # dummy values, shouldn't affect results: REPLACE WITH SETTINGS!
                nsigma_b = 6
                nsigma_s = 3
                global_threshold = 0

                kabsch_debug_list = []
                for i_panel in range(len(loaded_img.get_detector())):
                    kabsch_debug_list.append(
                        KabschDebug(raw_data[i_panel].as_double(),
                                    mask[i_panel], gain_map[i_panel],
                                    kernel_size, nsigma_b, nsigma_s,
                                    global_threshold, min_local))

                dispersion = flex.double()
                for kabsch in kabsch_debug_list:
                    dispersion.extend(
                        kabsch.coefficient_of_variation().as_1d())

                sorted_dispersion = flex.sorted(dispersion)
                from libtbx.math_utils import nearest_integer as nint

                q1 = sorted_dispersion[nint(len(sorted_dispersion) / 4)]
                q2 = sorted_dispersion[nint(len(sorted_dispersion) / 2)]
                q3 = sorted_dispersion[nint(len(sorted_dispersion) * 3 / 4)]
                iqr = q3 - q1

                inlier_sel = (sorted_dispersion >
                              (q1 - 1.5 * iqr)) & (sorted_dispersion <
                                                   (q3 + 1.5 * iqr))
                sorted_dispersion = sorted_dispersion.select(inlier_sel)
                self.gain = sorted_dispersion[nint(len(sorted_dispersion) / 2)]
            except IndexError:
                self.gain = 1.0
        else:
            self.gain = 1.0

        return data, img_type
Ejemplo n.º 19
0
def run(argv=None):
  import libtbx.option_parser

  if (argv is None):
    argv = sys.argv

  command_line = (libtbx.option_parser.option_parser(
    usage="%s [-v] [-p poly_mask] [-c circle_mask] [-a avg_max] [-s stddev_max] [-m maxproj_min] [-x mask_pix_val] [-o output] avg_path stddev_path max_path" % libtbx.env.dispatcher_name)
                  .option(None, "--verbose", "-v",
                          action="store_true",
                          default=False,
                          dest="verbose",
                          help="Print more information about progress")
                  .option(None, "--poly_mask", "-p",
                          type="string",
                          default=None,
                          dest="poly_mask",
                          help="Polygon to mask out.  Comma-seperated string of xy pairs.")
                  .option(None, "--circle_mask", "-c",
                          type="string",
                          default=None,
                          dest="circle_mask",
                          help="Circle to mask out.  Comma-seperated string of x, y, and radius.")
                  .option(None, "--avg_max", "-a",
                          type="float",
                          default=2000.0,
                          dest="avg_max",
                          help="Maximum ADU that pixels in the average image are allowed to have before masked out")
                  .option(None, "--stddev_max", "-s",
                          type="float",
                          default=10.0,
                          dest="stddev_max",
                          help="Maximum ADU that pixels in the standard deviation image are allowed to have before masked out")
                  .option(None, "--maxproj_min", "-m",
                          type="float",
                          default=300.0,
                          dest="maxproj_min",
                          help="Minimum ADU that pixels in the maximum projection image are allowed to have before masked out")
                  .option(None, "--mask_pix_val", "-x",
                          type="int",
                          default=-2,
                          dest="mask_pix_val",
                          help="Value for masked out pixels")
                  .option(None, "--detector_format_version", "-d",
                          type="string",
                          default=None,
                          dest="detector_format_version",
                          help="detector format version string")
                  .option(None, "--output", "-o",
                          type="string",
                          default="mask_.pickle",
                          dest="destpath",
                          help="output file path, should be *.pickle")
                  ).process(args=argv[1:])

  # Must have exactly three remaining arguments.
  paths = command_line.args
  if (len(paths) != 3):
    command_line.parser.print_usage(file=sys.stderr)
    return

  if command_line.options.detector_format_version is None:
    address = timestamp = None
  else:
    from xfel.cxi.cspad_ana.cspad_tbx import evt_timestamp
    from xfel.detector_formats import address_and_timestamp_from_detector_format_version
    address, timestamp = address_and_timestamp_from_detector_format_version(command_line.options.detector_format_version)
    timestamp = evt_timestamp((timestamp,0))

  poly_mask = None
  if not command_line.options.poly_mask == None:
    poly_mask = []
    poly_mask_tmp = command_line.options.poly_mask.split(",")
    if len(poly_mask_tmp) % 2 != 0:
      command_line.parser.print_usage(file=sys.stderr)
      return
    odd = True
    for item in poly_mask_tmp:
      try:
        if odd:
          poly_mask.append(int(item))
        else:
          poly_mask[-1] = (poly_mask[-1],int(item))
      except ValueError:
        command_line.parser.print_usage(file=sys.stderr)
        return
      odd = not odd

  circle_mask = None
  if command_line.options.circle_mask is not None:
    circle_mask_tmp = command_line.options.circle_mask.split(",")
    if len(circle_mask_tmp) != 3:
      command_line.parser.print_usage(file=sys.stderr)
      return
    try:
      circle_mask = (int(circle_mask_tmp[0]),int(circle_mask_tmp[1]),int(circle_mask_tmp[2]))
    except ValueError:
      command_line.parser.print_usage(file=sys.stderr)
      return

  avg_path    = paths[0]
  stddev_path = paths[1]
  max_path    = paths[2]

  # load the three images
  format_class = Registry.find(avg_path)
  avg_f = format_class(avg_path)
  avg_i = avg_f.get_detectorbase()
  avg_d = avg_i.get_raw_data()

  stddev_f = format_class(stddev_path)
  stddev_i = stddev_f.get_detectorbase()
  stddev_d = stddev_i.get_raw_data()

  max_f = format_class(max_path)
  max_i = max_f.get_detectorbase()
  max_d = max_i.get_raw_data()

  # first find all the pixels in the average that are less than zero or greater
  # than a cutoff and set them to the masking value
  avg_d.set_selected((avg_d <= 0) | (avg_d > command_line.options.avg_max), command_line.options.mask_pix_val)

  # set all the rest of the pixels to zero.  They will be accepted
  avg_d.set_selected(avg_d != command_line.options.mask_pix_val, 0)

  # mask out the overly noisy or flat pixels
  avg_d.set_selected(stddev_d <= 0, command_line.options.mask_pix_val)
  avg_d.set_selected(stddev_d >= command_line.options.stddev_max, command_line.options.mask_pix_val)

  # these are the non-bonded pixels
  avg_d.set_selected(max_d < command_line.options.maxproj_min, command_line.options.mask_pix_val)

  # calculate the beam center
  panel = avg_f.get_detector()[0]
  bcx, bcy = panel.get_beam_centre(avg_f.get_beam().get_s0())

  if poly_mask is not None or circle_mask is not None:
    minx = miny = 0
    maxx = avg_d.focus()[0]
    maxy = avg_d.focus()[1]
    if poly_mask is not None:
      minx = min([x[0] for x in poly_mask])
      miny = min([y[1] for y in poly_mask])
      maxx = max([x[0] for x in poly_mask])
      maxy = max([y[1] for y in poly_mask])
    if circle_mask is not None:
      circle_x, circle_y, radius = circle_mask

      if circle_x - radius < minx: minx = circle_x - radius
      if circle_y - radius < miny: miny = circle_y - radius
      if circle_x + radius > maxx: maxx = circle_x + radius
      if circle_y + radius > maxy: maxy = circle_y + radius

    sel = avg_d == command_line.options.mask_pix_val
    for j in xrange(miny, maxy):
      for i in xrange(minx, maxx):
        idx = j * avg_d.focus()[0] + i
        if not sel[idx]:
          if poly_mask is not None and point_in_polygon((i,j),poly_mask):
            sel[idx] = True
          elif circle_mask is not None and point_inside_circle(i,j,circle_x,circle_y,radius):
            sel[idx] = True
    avg_d.set_selected(sel,command_line.options.mask_pix_val)

  # have to re-layout the data to match how it was stored originally
  shifted_int_data_old = avg_d
  shifted_int_data_new = shifted_int_data_old.__class__(
    flex.grid(shifted_int_data_old.focus()))
  shifted_int_data_new += command_line.options.mask_pix_val

  phil = avg_i.horizons_phil_cache
  manager = avg_i.get_tile_manager(phil)

  for i,shift in enumerate(manager.effective_translations()):
    shift_slow = shift[0]
    shift_fast = shift[1]

    ur_slow = phil.distl.detector_tiling[4 * i + 0] + shift_slow
    ur_fast = phil.distl.detector_tiling[4 * i + 1] + shift_fast
    ll_slow = phil.distl.detector_tiling[4 * i + 2] + shift_slow
    ll_fast = phil.distl.detector_tiling[4 * i + 3] + shift_fast

    #print "Shifting tile at (%d, %d) by (%d, %d)" % (ur_slow-shift_slow, ur_fast-shift_fast, -shift_slow, -shift_fast)

    shifted_int_data_new.matrix_paste_block_in_place(
      block = shifted_int_data_old.matrix_copy_block(
        i_row=ur_slow,i_column=ur_fast,
        n_rows=ll_slow-ur_slow, n_columns=ll_fast-ur_fast),
      i_row = ur_slow - shift_slow,
      i_column = ur_fast - shift_fast
    )

  d = dpack(
    active_areas=avg_i.parameters['ACTIVE_AREAS'],
    address=address,
    beam_center_x=bcx,
    beam_center_y=bcy,
    data=shifted_int_data_new,
    distance=avg_i.distance,
    timestamp=timestamp,
    wavelength=avg_i.wavelength,
    xtal_target=None,
    pixel_size=avg_i.pixel_size,
    saturated_value=avg_i.saturation)

  dwritef2(d, command_line.options.destpath)

  #the minimum number of pixels to mask out cooresponding to the interstitial regions for the CS-PAD
  min_count  = 818265 # (1765 * 1765) - (194 * 185 * 64)
  masked_out = len(avg_d.as_1d().select((avg_d == command_line.options.mask_pix_val).as_1d()))
  assert masked_out >= min_count

  print "Masked out %d pixels out of %d (%.2f%%)"% \
    (masked_out-min_count,len(avg_d)-min_count,(masked_out-min_count)*100/(len(avg_d)-min_count))
Ejemplo n.º 20
0
            break
        except KeyError, ValueError:
            n_fails += 1
            print "Fail to read, attempt number", n_fails
            if n_fails > 100:
                raise Exception("Couldn't read the data")
        import time

        time.sleep(n_fails * 0.1)

    imgdict = cspad_tbx.dpack(
        data=raw_data,
        distance=distance,
        pixel_size=pixel_size,
        wavelength=wavelength,
        beam_center_x=beam_x,
        beam_center_y=beam_y,
        ccd_image_saturation=overload,
        saturated_value=overload,
        address="Sacla.MPCCD.8tile",
        active_areas=active_areas,
    )
    imgdict = crop_image_pickle(imgdict, preserve_active_areas_even_though_cropping_would_invalidate_them=True)

    dest_path = os.path.join(dest_dir, dest_base + "_%06d.pickle" % img_no)
    print "Saving image", img_no, "to", dest_path
    easy_pickle.dump(dest_path, imgdict)


easy_mp.pool_map(args=range(data.get_num_images()), func=do_work, processes=4)
          quad_asics.append(numpy.concatenate((a,b),axis=1))
      quad_data = numpy.dstack(quad_asics)
      quad_data = numpy.rollaxis(quad_data, 2,0)
      data3d.append(fake_cspad_ElementV2(quad_data, i_quad))

    env = fake_env(fake_config())
    evt = fake_evt(data3d)
    beam_center, active_areas = cbcaa(fake_config(),sections)
    data = flex.int(CsPadDetector(address, evt, env, sections).astype(numpy.float64))

  img_dict = dpack(
          active_areas=active_areas,
          address=address,
          beam_center_x=beam_center[0]*pixel_size,
          beam_center_y=beam_center[1]*pixel_size,
          data=data,
          distance=params.distance,
          pixel_size=pixel_size,
          timestamp=timestamp,
          wavelength=params.wavelength)

  img = NpyImage("", source_data=img_dict)

  args = ["distl.detector_format_version=%s"%params.detector_format_version]
  horizons_phil = cxi_phil.cxi_versioned_extract(args)

  img.readHeader(horizons_phil)
  img.translate_tiles(horizons_phil)
  tm = img.get_tile_manager(horizons_phil)
  effective_active_areas = tm.effective_tiling_as_flex_int()
Ejemplo n.º 22
0
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.
    XXX more?

    Previously, common-mode correction was applied only after initial
    threshold filtering.  Since the common_mode class applies the
    (lengthy) common-mode correction immediately after reading the
    image from the stream, this optimisation is currently not
    (elegantly) doable.

    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(mod_hitfind, self).event(evt, env)
    if (evt.get("skip_event")):
      return

    # This module only applies to detectors for which a distance is
    # available.
    distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
    if distance is None:
      self.nfail += 1
      self.logger.warning("event(): no distance, shot skipped")
      evt.put(skip_event_flag(), "skip_event")
      return

    device = cspad_tbx.address_split(self.address)[2]

    # ***** HITFINDING ***** XXX For hitfinding it may be interesting
    # to look at the fraction of subzero pixels in the dark-corrected
    # image.
    if (self.m_threshold is not None):
      # If a threshold value is given it can be applied in one of three ways:
      #    1.  Apply it over the whole image
      if (self.m_roi is None and self.m_distl_min_peaks is None):
        vmax = flex.max(self.cspad_img)
        if (vmax < self.m_threshold):
          if not self.m_negate_hits:
            # Tell downstream modules to skip this event if the threshold was not met.
            evt.put(skip_event_flag(), "skip_event")
            return
        elif self.m_negate_hits:
          evt.put(skip_event_flag(), "skip_event")
          return

      #    2. Apply threshold over a rectangular region of interest.
      elif (self.m_roi is not None):
        vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
                                       self.m_roi[0]:self.m_roi[1]])
        if (vmax < self.m_threshold):
          if not self.m_negate_hits:
            evt.put(skip_event_flag(), "skip_event")
            return
        elif self.m_negate_hits:
          evt.put(skip_event_flag(), "skip_event")
          return

      #    3. Determine the spotfinder spots within the central ASICS, and accept the
      #       image as a hit if there are m_distl_min_peaks exceeding m_threshold.
      #       As a further requirement, the peaks must exceed 2.5 * the 90-percentile
      #       pixel value of the central ASICS.  This filter was added to avoid high-background
      #       false positives.
      elif (self.m_distl_min_peaks is not None):
        if device == 'marccd':
          self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
          self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
        elif device == 'Rayonix':
          self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
          self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]

        peak_heights,outvalue = self.distl_filter(
          self.address,
          self.cspad_img.iround(), # XXX correct?
          distance,
          self.timestamp,
          self.wavelength)
        if ('permissive' in self.m_distl_flags):
          number_of_accepted_peaks = (peak_heights > self.m_threshold).count(True)
        else:
          number_of_accepted_peaks = ((peak_heights > self.m_threshold).__and__(outvalue==0)).count(True)

        sec,ms = cspad_tbx.evt_time(evt)
        evt_time = sec + ms/1000
        self.stats_logger.info("BRAGG %.3f %d" %(evt_time, number_of_accepted_peaks))

        skip_event = False
        if number_of_accepted_peaks < self.m_distl_min_peaks:
          self.logger.info("Subprocess %02d: Spotfinder NO  HIT image #%05d @ %s; %d spots > %d" %(
            env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))

          if not self.m_negate_hits:
            skip_event = True
        else:
          self.logger.info("Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d" %(
            env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))

          if self.m_negate_hits:
            skip_event = True

        if skip_event:
          if self.m_db_logging:
            # log misses to the database
            self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, number_of_accepted_peaks, distance,
                              self.sifoil, self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, self.m_db_tags))
          evt.put(skip_event_flag(), "skip_event")
          return
        # the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
        # hit will not be logged in the db.
        evt.put(number_of_accepted_peaks, 'sfspots')

    self.logger.info("Subprocess %02d: process image #%05d @ %s" %
                     (env.subprocess(), self.nshots, self.timestamp))

    # See r17537 of mod_average.py.
    if device == 'Cspad':
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
    elif device == 'marccd':
      pixel_size = evt.get("marccd_pixel_size")
      saturated_value = evt.get("marccd_saturated_value")
    elif device == 'Rayonix':
      pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
      saturated_value = rayonix_tbx.rayonix_saturated_value

    d = cspad_tbx.dpack(
      active_areas=self.active_areas,
      address=self.address,
      beam_center_x=pixel_size * self.beam_center[0],
      beam_center_y=pixel_size * self.beam_center[1],
      data=self.cspad_img.iround(), # XXX ouch!
      distance=distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=self.timestamp,
      wavelength=self.wavelength,
      xtal_target=self.m_xtal_target)

    if (self.m_dispatch == "index"):
      import sys
      from xfel.cxi.integrate_image_api import integrate_one_image
      info = integrate_one_image(d,
                                 integration_dirname  = self.m_integration_dirname,
                                 integration_basename = self.m_integration_basename)
      sys.stdout = sys.__stdout__
      sys.stderr = sys.__stderr__

      indexed = info is not None
      if indexed and self.m_progress_logging:
        # integration pickle dictionary is available here as info.last_saved_best
        if info.last_saved_best["identified_isoform"] is not None:
          #print info.last_saved_best.keys()
          from cxi_xdr_xes.cftbx.cspad_ana import db
          dbobj = db.dbconnect(self.m_db_host, self.m_db_name, self.m_db_user, self.m_db_password)
          cursor = dbobj.cursor()
          if info.last_saved_best["identified_isoform"] in self.isoforms:
            PM, indices, miller_id = self.isoforms[info.last_saved_best["identified_isoform"]]
          else:
            from xfel.xpp.progress_support import progress_manager
            PM = progress_manager(info.last_saved_best,self.m_db_experiment_tag, self.m_trial_id, self.m_rungroup_id, evt.run())
            indices, miller_id = PM.get_HKL(cursor)
            # cache these as they don't change for a given isoform
            self.isoforms[info.last_saved_best["identified_isoform"]] = PM, indices, miller_id
          if self.m_sql_buffer_size > 1:
            self.queue_progress_entry(PM.scale_frame_detail(self.timestamp,cursor,do_inserts=False))
          else:
            PM.scale_frame_detail(self.timestamp,cursor,do_inserts=True)
            dbobj.commit()
            cursor.close()
            dbobj.close()

      if self.m_db_logging:
        sec,ms = cspad_tbx.evt_time(evt)
        evt_time = sec + ms/1000
        sfspots = evt.get('sfspots')
        if sfspots is None:
          if indexed:
            n_spots = len(info.spotfinder_results.images[info.frames[0]]['spots_total'])
          else:
            n_spots = 0
        else:
          n_spots = sfspots

        if indexed:
          mosaic_bloc_rotation = info.last_saved_best.get('ML_half_mosaicity_deg', [0])[0]
          mosaic_block_size = info.last_saved_best.get('ML_domain_size_ang', [0])[0]
          ewald_proximal_volume = info.last_saved_best.get('ewald_proximal_volume', [0])[0]

          obs = info.last_saved_best['observations'][0]
          cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell().parameters()
          pointgroup = info.last_saved_best['pointgroup']
          resolution = obs.d_min()
        else:
          mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
            cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0

        self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, n_spots, distance,
                          self.sifoil, self.wavelength, indexed, mosaic_bloc_rotation,
                          mosaic_block_size, ewald_proximal_volume, pointgroup, cell_a,
                          cell_b, cell_c, cell_alpha, cell_beta, cell_gamma, resolution,
                          self.m_db_tags))

      if (not indexed):
        evt.put(skip_event_flag(), "skip_event")
        return

    elif (self.m_dispatch == "nop"):
      pass

    elif (self.m_dispatch == "view"): #interactive image viewer

      args = ["indexing.data=dummy"]
      detector_format_version = detector_format_function(
        self.address, evt.GetTime())
      if detector_format_version is not None:
        args += ["distl.detector_format_version=%" % detector_format_version]

      from xfel.phil_preferences import load_cxi_phil
      horizons_phil = load_cxi_phil(self.m_xtal_target, args)
      horizons_phil.indexing.data = d

      from xfel.cxi import display_spots
      display_spots.parameters.horizons_phil = horizons_phil
      display_spots.wrapper_of_callback().display(horizons_phil.indexing.data)

    elif (self.m_dispatch == "spots"): #interactive spotfinder viewer

      args = ["indexing.data=dummy"]
      detector_format_version = detector_format_function(
        self.address, evt.GetTime())
      if detector_format_version is not None:
        args += ["distl.detector_format_version=%s" % detector_format_version]

      from xfel.phil_preferences import load_cxi_phil
      horizons_phil = load_cxi_phil(self.m_xtal_target, args)
      horizons_phil.indexing.data = d

      from xfel.cxi import display_spots
      display_spots.parameters.horizons_phil = horizons_phil

      from rstbx.new_horizons.index import pre_indexing_validation,pack_names
      pre_indexing_validation(horizons_phil)
      imagefile_arguments = pack_names(horizons_phil)
      horizons_phil.persist.show()
      from spotfinder.applications import signal_strength
      info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments)

      work = display_spots.wrapper_of_callback(info)
      work.display_with_callback(horizons_phil.indexing.data)

    elif (self.m_dispatch == "write_dict"):
      self.logger.warning(
        "event(): deprecated dispatch 'write_dict', use mod_dump instead")
      if (self.m_out_dirname  is not None or
          self.m_out_basename is not None):
        cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)

    # Diagnostic message emitted only when all the processing is done.
    if (env.subprocess() >= 0):
      self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
                       (env.subprocess(), self.nshots, self.timestamp))
    else:
      self.logger.info("Accepted #%05d @ %s" %
                       (self.nshots, self.timestamp))
Ejemplo n.º 23
0
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.

    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(average_mixin, self).event(evt, env)
    if evt.get('skip_event'):
      return

    # Get the distance for the detectors that should have it, and set
    # it to NaN for those that should not.
    if self.detector == 'CxiDs1' or \
       self.detector == 'CxiDs2' or \
       self.detector == 'CxiDsd' or \
       self.detector == 'XppGon':
      distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
      if distance is None:
        self._nfail += 1
        self.logger.warning("event(): no distance, shot skipped")
        evt.put(skip_event_flag(), 'skip_event')
        return
    else:
      distance = float('nan')

    if ("skew" in self.flags):
      # Take out inactive pixels
      if self.roi is not None:
        pixels = self.cspad_img[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]]
        dark_mask = self.dark_mask[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]]
        pixels = pixels.as_1d().select(dark_mask.as_1d())
      else:
        pixels = self.cspad_img.as_1d().select(self.dark_mask.as_1d()).as_double()
      stats = scitbx.math.basic_statistics(pixels.as_double())
      #stats.show()
      self.logger.info("skew: %.3f" %stats.skew)
      self.logger.info("kurtosis: %.3f" %stats.kurtosis)
      if 0:
        from matplotlib import pyplot
        hist_min, hist_max = flex.min(flex_cspad_img.as_double()), flex.max(flex_cspad_img.as_double())
        print hist_min, hist_max
        n_slots = 100
        n, bins, patches = pyplot.hist(flex_cspad_img.as_1d().as_numpy_array(), bins=n_slots, range=(hist_min, hist_max))
        pyplot.show()

      # XXX This skew threshold probably needs fine-tuning
      skew_threshold = 0.35
      if stats.skew < skew_threshold:
        self._nfail += 1
        self.logger.warning("event(): skew < %f, shot skipped" % skew_threshold)
        evt.put(skip_event_flag(), 'skip_event')
        return
      #self.cspad_img *= stats.skew

    if ("inactive" in self.flags):
      self.cspad_img.set_selected(self.dark_stddev <= 0, 0)

    if ("noelastic" in self.flags):
      ELASTIC_THRESHOLD = self.elastic_threshold
      self.cspad_img.set_selected(self.cspad_img > ELASTIC_THRESHOLD, 0)

    if self.hot_threshold is not None:
      HOT_THRESHOLD = self.hot_threshold
      self.cspad_img.set_selected(self.dark_img > HOT_THRESHOLD, 0)

    if self.gain_map is not None and self.gain_threshold is not None:
      # XXX comparing each pixel to a moving average would probably be better
      # since the gain should vary approximately smoothly over different areas
      # of the detector
      GAIN_THRESHOLD = self.gain_threshold
      #self.logger.debug(
        #"rejecting: %i" %(self.gain_map > GAIN_THRESHOLD).count(True))
      self.cspad_img.set_selected(self.gain_map > GAIN_THRESHOLD, 0)

    if ("nonoise" in self.flags):
      NOISE_THRESHOLD = self.noise_threshold
      self.cspad_img.set_selected(self.cspad_img < NOISE_THRESHOLD, 0)

    if ("sigma_scaling" in self.flags):
      self.do_sigma_scaling()

    if ("symnoise" in self.flags):
      SYMNOISE_THRESHOLD = self.symnoise_threshold
      self.cspad_img.set_selected((-SYMNOISE_THRESHOLD < self.cspad_img) &
                                  ( self.cspad_img  < SYMNOISE_THRESHOLD), 0)

    if ("output" in self.flags):
      try:
        import cPickle as pickle
      except ImportError:
        import pickle
      import os
      if (not os.path.isdir(self.pickle_dirname)):
        os.makedirs(self.pickle_dirname)
      flexdata = flex.int(self.cspad_img.astype(numpy.int32))
      d = cspad_tbx.dpack(
        address=self.address,
        data=flexdata,
        timestamp=cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt))
      )
      G = open(os.path.join(".",self.pickle_dirname)+"/"+self.pickle_basename,
               "ab")
      pickle.dump(d,G,pickle.HIGHEST_PROTOCOL)
      G.close()

    if self.photon_threshold is not None and self.two_photon_threshold is not None:
      self.do_photon_counting()

    if self.background_path is not None:
      self.cspad_img -= self.background_img


    # t and self._sum_time are a two-long arrays of seconds and
    # milliseconds which hold time with respect to the base time.
    t = [t1 - t2 for (t1, t2) in zip(cspad_tbx.evt_time(evt),
                                     self._metadata['time_base'])]
    if self._nmemb == 0:
      # The peers metadata item is a bit field where a bit is set if
      # the partial sum from the corresponding worker process is
      # pending.  If this is the first frame a worker process sees,
      # set its corresponding bit in the bit field since it will
      # contribute a partial sum.
      if env.subprocess() >= 0:
        self._lock.acquire()
        if 'peers' in self._metadata.keys():
          self._metadata['peers'] |= (1 << env.subprocess())
        else:
          self._metadata['peers'] = (1 << env.subprocess())
        self._lock.release()

      self._sum_distance = distance
      self._sum_time = (t[0], t[1])
      self._sum_wavelength = self.wavelength

      if self._have_max:
        self._max_img = self.cspad_img.deep_copy()
      if self._have_mean:
        self._sum_img = self.cspad_img.deep_copy()
      if self._have_std:
        self._ssq_img = flex.pow2(self.cspad_img)

    else:
      self._sum_distance += distance
      self._sum_time = (self._sum_time[0] + t[0], self._sum_time[1] + t[1])
      self._sum_wavelength += self.wavelength

      if self._have_max:
        sel = (self.cspad_img > self._max_img).as_1d()
        self._max_img.as_1d().set_selected(
          sel, self.cspad_img.as_1d().select(sel))
      if self._have_mean:
        self._sum_img += self.cspad_img
      if self._have_std:
        self._ssq_img += flex.pow2(self.cspad_img)

    self._nmemb += 1
Ejemplo n.º 24
0
        try:
            raw_data = data.get_raw_data(img_no)
            break
        except KeyError, ValueError:
            n_fails += 1
            print "Fail to read, attempt number", n_fails
            if n_fails > 100:
                raise Exception("Couldn't read the data")
        import time
        time.sleep(n_fails * 0.1)

    imgdict = cspad_tbx.dpack(data=raw_data,
                              distance=distance,
                              pixel_size=pixel_size,
                              wavelength=wavelength,
                              beam_center_x=beam_x,
                              beam_center_y=beam_y,
                              ccd_image_saturation=overload,
                              saturated_value=overload,
                              address="Sacla.MPCCD.8tile",
                              active_areas=active_areas)
    imgdict = crop_image_pickle(
        imgdict,
        preserve_active_areas_even_though_cropping_would_invalidate_them=True)

    dest_path = os.path.join(dest_dir, dest_base + "_%06d.pickle" % img_no)
    print "Saving image", img_no, "to", dest_path
    easy_pickle.dump(dest_path, imgdict)


easy_mp.pool_map(args=range(data.get_num_images()), func=do_work, processes=4)
Ejemplo n.º 25
0
  def load_image(self):
    """ Reads raw image file and extracts data for conversion into pickle
        format. Also estimates gain if turned on."""
    # Load raw image or image pickle
    try:
      with misc.Capturing() as junk_output:
        loaded_img = dxtbx.load(self.raw_img)
    except IOError:
      loaded_img = None
      pass

    # Extract image information
    if loaded_img is not None:
      raw_data   = loaded_img.get_raw_data()
      detector   = loaded_img.get_detector()[0]
      beam       = loaded_img.get_beam()
      scan       = loaded_img.get_scan()
      distance   = detector.get_distance()
      pixel_size = detector.get_pixel_size()[0]
      overload   = detector.get_trusted_range()[1]
      wavelength = beam.get_wavelength()
      beam_x     = detector.get_beam_centre(beam.get_s0())[0]
      beam_y     = detector.get_beam_centre(beam.get_s0())[1]

      if scan is None:
        timestamp = None
        if abs(beam_x - beam_y) <= 0.1 or self.params.image_conversion.square_mode == "None":
          img_type = 'converted'
        else:
          img_type = 'unconverted'
      else:
        msec, sec = math.modf(scan.get_epochs()[0])
        timestamp = evt_timestamp((sec,msec))

      if self.params.image_conversion.beamstop != 0 or\
         self.params.image_conversion.beam_center.x != 0 or\
         self.params.image_conversion.beam_center.y != 0 or\
         self.params.image_conversion.rename_pickle_prefix != 'Auto' or\
         self.params.image_conversion.rename_pickle_prefix != None:
        img_type = 'unconverted'

      # Assemble datapack
      data = dpack(data=raw_data,
                   distance=distance,
                   pixel_size=pixel_size,
                   wavelength=wavelength,
                   beam_center_x=beam_x,
                   beam_center_y=beam_y,
                   ccd_image_saturation=overload,
                   saturated_value=overload,
                   timestamp=timestamp
                   )

      #print "data: ", type(raw_data)
      #print "pixel size: ", type(pixel_size)
      #print 'wavelength: ', type(wavelength)
      #print "beamX: ", type(beam_x)
      #print "saturation: ", type(overload)
      #print "timestamp: ", type(timestamp)

      #for i in dir(raw_data): print i

      #exit()

      if scan is not None:
        osc_start, osc_range = scan.get_oscillation()
        img_type = 'unconverted'
        if osc_start != osc_range:
          data['OSC_START'] = osc_start
          data['OSC_RANGE'] = osc_range
          data['TIME'] = scan.get_exposure_times()[0]

      # Estimate gain (or set gain to 1.00 if cannot calculate)
      # Cribbed from estimate_gain.py by Richard Gildea
      if self.params.advanced.estimate_gain:
        try:
          from dials.algorithms.image.threshold import KabschDebug
          raw_data = [raw_data]

          gain_value = 1
          kernel_size=(10,10)
          gain_map = [flex.double(raw_data[i].accessor(), gain_value)
                      for i in range(len(loaded_img.get_detector()))]
          mask = loaded_img.get_mask()
          min_local = 0

          # dummy values, shouldn't affect results
          nsigma_b = 6
          nsigma_s = 3
          global_threshold = 0

          kabsch_debug_list = []
          for i_panel in range(len(loaded_img.get_detector())):
            kabsch_debug_list.append(
              KabschDebug(
                raw_data[i_panel].as_double(), mask[i_panel], gain_map[i_panel],
                kernel_size, nsigma_b, nsigma_s, global_threshold, min_local))

          dispersion = flex.double()
          for kabsch in kabsch_debug_list:
            dispersion.extend(kabsch.coefficient_of_variation().as_1d())

          sorted_dispersion = flex.sorted(dispersion)
          from libtbx.math_utils import nearest_integer as nint

          q1 = sorted_dispersion[nint(len(sorted_dispersion)/4)]
          q2 = sorted_dispersion[nint(len(sorted_dispersion)/2)]
          q3 = sorted_dispersion[nint(len(sorted_dispersion)*3/4)]
          iqr = q3-q1

          inlier_sel = (sorted_dispersion > (q1 - 1.5*iqr)) & (sorted_dispersion < (q3 + 1.5*iqr))
          sorted_dispersion = sorted_dispersion.select(inlier_sel)
          self.gain = sorted_dispersion[nint(len(sorted_dispersion)/2)]
        except IndexError:
          self.gain = 1.0
      else:
        self.gain = 1.0

    else:
      data = None

    return data, img_type
Ejemplo n.º 26
0
def run(argv=None):
  """Compute mean, standard deviation, and maximum projection images
  from a set of images given on the command line.

  @param argv Command line argument list
  @return     @c 0 on successful termination, @c 1 on error, and @c 2
              for command line syntax errors
  """

  import libtbx.load_env

  from libtbx import easy_pickle, option_parser
  from scitbx.array_family import flex
  from xfel.cxi.cspad_ana import cspad_tbx
  from iotbx.detectors.cspad_detector_formats import reverse_timestamp

  if argv is None:
    argv = sys.argv
  command_line = (option_parser.option_parser(
    usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \
    "image1 image2 [image3 ...]" % libtbx.env.dispatcher_name)
                  .option(None, "--average-path", "-a",
                          type="string",
                          default=None,
                          dest="avg_path",
                          metavar="PATH",
                          help="Write average image to PATH")
                  .option(None, "--maximum-path", "-m",
                          type="string",
                          default=None,
                          dest="max_path",
                          metavar="PATH",
                          help="Write maximum projection image to PATH")
                  .option(None, "--stddev-path", "-s",
                          type="string",
                          default=None,
                          dest="stddev_path",
                          metavar="PATH",
                          help="Write standard deviation image to PATH")
                  .option(None, "--verbose", "-v",
                          action="store_true",
                          default=False,
                          dest="verbose",
                          help="Print more information about progress")
                  ).process(args=argv[1:])

  # Note that it is not an error to omit the output paths, because
  # certain statistics could still be printed, e.g. with the verbose
  # option.
  paths = command_line.args
  if len(paths) == 0:
    command_line.parser.print_usage(file=sys.stderr)
    return 2

  # Loop over all images and accumulate statistics.
  nfail = 0
  nmemb = 0
  for path in paths:
    if command_line.options.verbose:
      sys.stdout.write("Processing %s...\n" % path)

    try:
      # Promote the image to double-precision floating point type.
      # All real-valued flex arrays have the as_double() function.
      d = easy_pickle.load(path)
      distance = d['DISTANCE']
      img = d['DATA'].as_1d().as_double()
      wavelength = d['WAVELENGTH']
      time_tuple = reverse_timestamp(d['TIMESTAMP'])

      # Warn if the header items across the set of images do not match
      # up.  Note that discrepancies regarding the image size are
      # fatal.
      if 'active_areas' in locals():
        if (active_areas != d['ACTIVE_AREAS']).count(True) != 0:
          sys.stderr.write("Active areas do not match\n")
      else:
        active_areas = d['ACTIVE_AREAS']

      if 'beam_center' in locals():
        if beam_center != (d['BEAM_CENTER_X'], d['BEAM_CENTER_Y']):
          sys.stderr.write("Beam centers do not match\n")
      else:
        beam_center = (d['BEAM_CENTER_X'], d['BEAM_CENTER_Y'])

      if 'detector_address' in locals():
        if detector_address != d['DETECTOR_ADDRESS']:
          sys.stderr.write("Detector addresses do not match\n")
      else:
        detector_address = d['DETECTOR_ADDRESS']

      if 'saturated_value' in locals():
        if saturated_value != d['SATURATED_VALUE']:
          sys.stderr.write("Saturated values do not match\n")
      else:
        saturated_value = d['SATURATED_VALUE']

      if 'size' in locals():
        if size != (d['SIZE1'], d['SIZE2']):
          sys.stderr.write("Image sizes do not match\n")
          return 1
      else:
        size = (d['SIZE1'], d['SIZE2'])
      if size != d['DATA'].focus():
        sys.stderr.write("Image size does not match pixel array\n")
        return 1

      if 'pixel_size' in locals():
        if pixel_size != d['PIXEL_SIZE']:
          sys.stderr.write("Pixel sizes do not match\n")
          return 1
      else:
        if 'PIXEL_SIZE' in d:
          pixel_size = d['PIXEL_SIZE']
        else:
          pixel_size = None


    except Exception:
      try:
        # Fall back on reading the image with dxtbx, and shoehorn the
        # extracted information into what would have been found in a
        # pickle file.  XXX This code assumes a monolithic detector!

        from dxtbx.format.Registry import Registry

        format_class = Registry.find(path)
        i = format_class(path)

        beam = i.get_beam()
        assert len(i.get_detector()) == 1
        detector = i.get_detector()[0]

        beam_center = detector.get_beam_centre(beam.get_s0())
        detector_address = format_class.__name__
        distance = detector.get_distance()
        img = i.get_raw_data().as_1d().as_double()
        pixel_size = 0.5 * sum(detector.get_pixel_size())
        saturated_value = int(round(detector.get_trusted_range()[1]))
        size = detector.get_image_size()
        time_tuple = (i.get_scan().get_epochs()[0], 0)
        wavelength = beam.get_wavelength()

        active_areas = flex.int((0, 0, size[0], size[1]))


      except Exception:
        nfail += 1
        continue


    # See also event() in xfel.cxi.cspad_ana.average_tbx.  Record the
    # base time as the timestamp of the first image.
    #
    # The sum-of-squares image is accumulated using long integers, as
    # this delays the point where overflow occurs.  But really, this
    # is just a band-aid...
    if nmemb == 0:
      max_img = img.deep_copy()
      sum_distance = distance
      sum_img = img.deep_copy()
      ssq_img = flex.pow2(img)
      sum_wavelength = wavelength
      sum_time = (0, 0)
      time_base = time_tuple

    else:
      sel = (img > max_img).as_1d()
      max_img.set_selected(sel, img.select(sel))

      sum_distance += distance
      sum_img += img
      ssq_img += flex.pow2(img)
      sum_wavelength += wavelength
      sum_time = (sum_time[0] + (time_tuple[0] - time_base[0]),
                  sum_time[1] + (time_tuple[1] - time_base[1]))

    nmemb += 1

  # Early exit if no statistics were accumulated.
  if command_line.options.verbose:
    sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail))
  if nmemb == 0:
    return 0

  # Calculate averages for measures where other statistics do not make
  # sense.  Note that avg_img is required for stddev_img.
  avg_img = sum_img.as_double() / nmemb
  avg_distance = sum_distance / nmemb
  avg_timestamp = cspad_tbx.evt_timestamp(
    (time_base[0] + int(round(sum_time[0] / nmemb)),
     time_base[1] + int(round(sum_time[1] / nmemb))))
  avg_wavelength = sum_wavelength / nmemb

  # Output the average image, maximum projection image, and standard
  # deviation image, if requested.
  if command_line.options.avg_path is not None:
    avg_img.resize(flex.grid(size[0], size[1]))
    d = cspad_tbx.dpack(
      active_areas=active_areas,
      address=detector_address,
      beam_center_x=beam_center[0],
      beam_center_y=beam_center[1],
      data=avg_img,
      distance=avg_distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=avg_timestamp,
      wavelength=avg_wavelength)
    easy_pickle.dump(command_line.options.avg_path, d)

  if command_line.options.max_path is not None:
    max_img.resize(flex.grid(size[0], size[1]))
    d = cspad_tbx.dpack(
      active_areas=active_areas,
      address=detector_address,
      beam_center_x=beam_center[0],
      beam_center_y=beam_center[1],
      data=max_img,
      distance=avg_distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=avg_timestamp,
      wavelength=avg_wavelength)
    easy_pickle.dump(command_line.options.max_path, d)

  if command_line.options.stddev_path is not None:
    stddev_img = ssq_img.as_double() - sum_img.as_double() * avg_img

    # Accumulating floating-point numbers introduces errors, which may
    # cause negative variances.  Since a two-pass approach is
    # unacceptable, the standard deviation is clamped at zero.
    stddev_img.set_selected(stddev_img < 0, 0)
    if nmemb == 1:
      stddev_img = flex.sqrt(stddev_img)
    else:
      stddev_img = flex.sqrt(stddev_img / (nmemb - 1))

    stddev_img.resize(flex.grid(size[0], size[1]))
    d = cspad_tbx.dpack(
      active_areas=active_areas,
      address=detector_address,
      beam_center_x=beam_center[0],
      beam_center_y=beam_center[1],
      data=stddev_img,
      distance=avg_distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=avg_timestamp,
      wavelength=avg_wavelength)
    easy_pickle.dump(command_line.options.stddev_path, d)

  return 0
Ejemplo n.º 27
0
def HitFinder(IO,XSetup,HFParams,Frelon,DataCorr,AI,index):
      
      hit=0
      fname=IO.fname_list[index]
      peaks=[]
      peakslist=[]
      try :
		img = fabio.open(fname)
      except:
		print 'Warning : problem while opening file %s, file skiped '%fname
		return
      if img.data.shape == Frelon.resolution:
		
		#Apply the dark, flatfield and distortion correction (as specified by the user)
		img.data=DataCorr.apply_correction(img.data,HFParams.DoDarkCorr,HFParams.DoFlatCorr,HFParams.DoDist)
			 
		#Remove beam stop area (i.e = 0)
		extend=15
			 
		#BkgCorr with pyFAI Azimuthal Integrator
		working=AI.ai.separate(img.data,npt_rad=1024, npt_azim=512, unit="2th_deg",percentile=50, mask=AI.mask,restore_mask=False)[0]
		
		imgmax,imgmin,imgmed = np.max(working), np.min(working), np.median(working)
		working[XSetup.beam_y-extend:XSetup.beam_y+extend,XSetup.beam_x-extend:XSetup.beam_x+extend]=0
			 
		# Get number of peaks above threshold in the current frame
		cropped=working[20:Frelon.resolution[1]-20,20:Frelon.resolution[1]-20]
		#print cropped.shape
		peaks=cropped[np.where(cropped>float(HFParams.threshold))]
		#peaks=working[np.where(working[20:Frelon.resolution[1]-20][20:Frelon.resolution[0]-20]>float(HFParams.threshold))]
		#peaks=working[np.where(working>float(HFParams.threshold))]# If enough peaks in current frame - This is a hit - Save it !!
			 
		if len(peaks) >= HFParams.npixels:
			hit = 1
			root=os.path.basename(fname)
			root=os.path.splitext(root)[0]
			
			
			if HFParams.DoPeakSearch:
			    
			    local_max=pf.find_local_max(working[0:1023,0:1004].astype(np.float),d_rad=1,threshold=HFParams.threshold)
			    local_max_crop=pf.local_max_crop(working[0:1023,0:1004].astype(np.float), local_max, 3)
			    peakslist=np.array(pf.subpixel_centroid(working[0:1023,0:1004],local_max_crop,3))
	
			    
			if IO.edf:
			    OutputFileName =os.path.join(IO.procdir, IO.EDFDir,"%s.edf" %root)
			    img.data = working
			    img.write(OutputFileName)
			    
			
			#Conversion to H5
			if IO.H5:
			    
			    OutputFileName =os.path.join(IO.procdir, IO.H5Dir,"%s.h5" %root)
			    #OutputFileName =os.path.join(IO.procdir, "HDF5/%s.h5" %root)
			    OutputFile = h5py.File(OutputFileName,'w')
			    working[0:1023,1004:1023]=0
			    OutputFile.create_dataset("data",data=working.astype(np.int32))
			    if HFParams.DoPeakSearch:
			     OutputFile.create_dataset("processing/hitfinder/peakinfo",data=peakslist.astype(np.int32))
			     #g1=OutputFile.create_group("processing")
                             #g2=g1.create_group("hitfinder")
                             #g2.create_dataset("peakinfo",data=peakslist.astype(np.int32))
			    OutputFile.close()
				 
			#if conversion to Pickle
			if IO.pickle:
				   pixels=flex.int(working.astype(np.int32))
				   pixel_size=Frelon.pixel_size
				   data = dpack(data=pixels,
				                distance=XSetup.distance,
					        pixel_size=pixel_size,
					        wavelength=XSetup.wavelength,
					        beam_center_x=XSetup.beam_y*pixel_size,
						beam_center_y=XSetup.beam_x*pixel_size,
						ccd_image_saturation=Frelon.overload,
						saturated_value=Frelon.overload)
				   data=crop_image_pickle(data)
				   OutputFileName =os.path.join(IO.procdir, IO.PicklesDir , "%s.pickle" %root)
				   easy_pickle.dump(OutputFileName,data)
				 
		else: working =0	 

					
      else:
			print 'Warning : data shape problem for file %s, file skiped '%fname
			#continue  
	
      return [hit,imgmax,imgmin,imgmed,index,peakslist,fname,working]
Ejemplo n.º 28
0
        #beamX=602.015+(ds.env().epicsStore().value("robot_x"))
        #beamY=51.682+(ds.env().epicsStore().value("robot_y"))
        #beamDist=-563.083+60.+ (ds.env().epicsStore().value("robot_z"))

        beamX = 87.96
        beamY = 85.57
        beamDist = 150

        print 'beam: ',beamX,beamY,beamDist

        # Assemble data pack
        data = cspad_tbx.dpack(data=image,
                               distance=beamDist,
                               pixel_size=pixelSize,
                               wavelength=lambda_wavelength,
                               beam_center_x=beamX,
                               beam_center_y=beamY,
                               ccd_image_saturation=saturation,
                               saturated_value=saturation,
                               timestamp = evt_timestamp)

        # output image pickle
        fpklname='/reg/d/psdm/xpp/%s/ftc/Run%s/Run_%s_idx_{}.pickle'%(expname,run,run)
        ep.dump(fpklname.format(ievt), data)
        print 'Importing image {}, beamX = {}, beamY = {}, Z-offset = {}, wavelength = {} '.format(fpklname, beamX, beamY, beamDist, lambda_wavelength)

#    if ievt>5:
#        break

print 'is done '
Ejemplo n.º 29
0
  def endjob(self, obj1, obj2=None):
    """The endjob() function writes the mean and standard deviation images
    to disk.

    @param evt Event object (psana only)
    @param env Environment object
    """
    if obj2 is None:
      env = obj1
    else:
      evt = obj1
      env = obj2

    stats = super(mod_average, self).endjob(env)
    if stats is None:
      return

    device = cspad_tbx.address_split(self.address)[2]
    if device == 'Andor':
      beam_center = (0, 0) # XXX Fiction!
      pixel_size = 13.5e-3 # XXX Should not be hardcoded here!
      saturated_value = 10000
    elif device == 'Cspad' or device == 'Cspad2x2':
      beam_center = self.beam_center
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
    elif device == 'marccd':
      beam_center = tuple(t // 2 for t in d['mean_img'].focus())
      pixel_size = 0.079346
      saturated_value = 2**16 - 1

    if stats['nmemb'] > 0:
      if self.avg_dirname  is not None or \
         self.avg_basename is not None or \
         self._mean_out    is not None:
        d = cspad_tbx.dpack(
          active_areas=self.active_areas,
          address=self.address,
          beam_center_x=pixel_size * beam_center[0],
          beam_center_y=pixel_size * beam_center[1],
          data=stats['mean_img'],
          distance=stats['distance'],
          pixel_size=pixel_size,
          saturated_value=saturated_value,
          timestamp=cspad_tbx.evt_timestamp(stats['time']),
          wavelength=stats['wavelength'])
        if self._mean_out is not None:
          p = cspad_tbx.dwritef2(d, self._mean_out)
        else:
          p = cspad_tbx.dwritef(d, self.avg_dirname, self.avg_basename)
        self.logger.info("Average written to %s" % p)

      if self.stddev_dirname  is not None or \
         self.stddev_basename is not None or \
         self._std_out    is not None:
        d = cspad_tbx.dpack(
          active_areas=self.active_areas,
          address=self.address,
          beam_center_x=pixel_size * beam_center[0],
          beam_center_y=pixel_size * beam_center[1],
          data=stats['std_img'],
          distance=stats['distance'],
          pixel_size=pixel_size,
          saturated_value=saturated_value,
          timestamp=cspad_tbx.evt_timestamp(stats['time']),
          wavelength=stats['wavelength'])
        if self._std_out is not None:
          p = cspad_tbx.dwritef2(d, self._std_out)
        else:
          p = cspad_tbx.dwritef(d, self.stddev_dirname, self.stddev_basename)
        self.logger.info("Standard deviation written to %s" % p)

      if self.max_dirname  is not None or \
         self.max_basename is not None or \
         self._max_out    is not None:
        d = cspad_tbx.dpack(
          active_areas=self.active_areas,
          address=self.address,
          beam_center_x=pixel_size * beam_center[0],
          beam_center_y=pixel_size * beam_center[1],
          data=stats['max_img'],
          distance=stats['distance'],
          pixel_size=pixel_size,
          saturated_value=saturated_value,
          timestamp=cspad_tbx.evt_timestamp(stats['time']),
          wavelength=stats['wavelength'])
        if self._max_out is not None:
          p = cspad_tbx.dwritef2(d, self._max_out)
        else:
          p = cspad_tbx.dwritef(d, self.max_dirname, self.max_basename)
        self.logger.info("Max written to %s" % p)

    if stats['nfail'] == 0:
      self.logger.info("%d images processed" % stats['nmemb'])
    else:
      self.logger.warning(
        "%d images processed, %d failed" % (stats['nmemb'], stats['nfail']))
Ejemplo n.º 30
0
        spectrum_focus = self.sum_img[start_row:end_row, :]
        mask_focus = mask[start_row:end_row, :]

        spectrum_focus.set_selected(mask_focus > 0, 0)

        xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)

        print "Number of rows: %i" % spectrum_focus.all()[0]
        print "Estimated no. photons counted: %i" % flex.sum(spectrum_focus)
        print "Number of images used: %i" % flex.sum(
            pixel_histograms.histograms.values()[0].slots())

        d = cspad_tbx.dpack(
            address='CxiSc1-0|Cspad2x2-0',
            data=spectrum_focus,
            distance=1,
            ccd_image_saturation=2e8,  # XXX
        )
        if run is not None: runstr = "_%04d" % run
        else: runstr = ""
        cspad_tbx.dwritef(d, output_dirname, 'sum%s_' % runstr)

        if gain_map is None:
            gain_map = flex.double(gain_img.accessor(), 0)
            img_sel = (gain_img > 0).as_1d()
            d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0',
                                data=gain_img,
                                distance=1)
            cspad_tbx.dwritef(d, output_dirname, 'raw_gain_map_')
            gain_map.as_1d().set_selected(img_sel.iselection(),
                                          1 / gain_img.as_1d().select(img_sel))
Ejemplo n.º 31
0
def run(argv=None):
  import libtbx.option_parser

  if (argv is None):
    argv = sys.argv

  command_line = (libtbx.option_parser.option_parser(
    usage="%s [-v] [-p poly_mask] [-c circle_mask]  [-x mask_pix_val] [-o output] -W mask_width -H mask_height" % libtbx.env.dispatcher_name)
                  .option(None, "--verbose", "-v",
                          action="store_true",
                          default=False,
                          dest="verbose",
                          help="Print more information about progress")
                  .option(None, "--poly_mask", "-p",
                          type="string",
                          default=None,
                          dest="poly_mask",
                          help="Polygon to mask out.  Comma-seperated string of xy pairs.")
                  .option(None, "--circle_mask", "-c",
                          type="string",
                          default=None,
                          dest="circle_mask",
                          help="Circle to mask out.  Comma-seperated string of x, y, and radius.")
                  .option(None, "--mask_pix_val", "-x",
                          type="int",
                          default=-2,
                          dest="mask_pix_val",
                          help="Value for masked out pixels")
                  .option(None, "--mask_width", "-W",
                          type="int",
                          default=None,
                          dest="mask_width",
                          help="Width of output mask")
                   .option(None, "--mask_height", "-H",
                          type="int",
                          default=None,
                          dest="mask_height",
                          help="Height of output mask")
                   .option(None, "--output", "-o",
                          type="string",
                          default="mask.pickle",
                          dest="destpath",
                          help="Output file path, should be *.pickle")
                   .option(None, "--pixel_size", "-s",
                          type="float",
                          default=None,
                          dest="pixel_size",
                          help="Pixel size for detector")
                  ).process(args=argv[1:])

  # Must have width and height set
  if command_line.options.mask_height is None or command_line.options.mask_width is None:
    command_line.parser.print_usage(file=sys.stderr)
    return

  poly_mask = None
  if not command_line.options.poly_mask == None:
    poly_mask = []
    poly_mask_tmp = command_line.options.poly_mask.split(",")
    if len(poly_mask_tmp) % 2 != 0:
      command_line.parser.print_usage(file=sys.stderr)
      return
    odd = True
    for item in poly_mask_tmp:
      try:
        if odd:
          poly_mask.append(int(item))
        else:
          poly_mask[-1] = (poly_mask[-1],int(item))
      except ValueError:
        command_line.parser.print_usage(file=sys.stderr)
        return
      odd = not odd

  circle_mask = None
  if command_line.options.circle_mask is not None:
    circle_mask_tmp = command_line.options.circle_mask.split(",")
    if len(circle_mask_tmp) != 3:
      command_line.parser.print_usage(file=sys.stderr)
      return
    try:
      circle_mask = (int(circle_mask_tmp[0]),int(circle_mask_tmp[1]),int(circle_mask_tmp[2]))
    except ValueError:
      command_line.parser.print_usage(file=sys.stderr)
      return

  mask = flex.int(flex.grid(command_line.options.mask_width,
                             command_line.options.mask_height))

  if poly_mask is not None or circle_mask is not None:
    minx = miny = 0
    maxx = mask.focus()[0]
    maxy = mask.focus()[1]
    if poly_mask is not None:
      minx = min([x[0] for x in poly_mask])
      miny = min([y[1] for y in poly_mask])
      maxx = max([x[0] for x in poly_mask])
      maxy = max([y[1] for y in poly_mask])
    if circle_mask is not None:
      circle_x, circle_y, radius = circle_mask

      if circle_x - radius < minx: minx = circle_x - radius
      if circle_y - radius < miny: miny = circle_y - radius
      if circle_x + radius > maxx: maxx = circle_x + radius
      if circle_y + radius > maxy: maxy = circle_y + radius

    sel = mask == command_line.options.mask_pix_val
    for j in xrange(miny, maxy):
      for i in xrange(minx, maxx):
        idx = j * mask.focus()[0] + i
        if not sel[idx]:
          if poly_mask is not None and point_in_polygon((i,j),poly_mask):
            sel[idx] = True
          elif circle_mask is not None and point_inside_circle(i,j,circle_x,circle_y,radius):
            sel[idx] = True
    mask.set_selected(sel,command_line.options.mask_pix_val)

  masked_out = len(mask.as_1d().select((mask == command_line.options.mask_pix_val).as_1d()))

  print "Masked out %d pixels out of %d (%.2f%%)"% \
    (masked_out,len(mask),(masked_out)*100/(len(mask)))

  easy_pickle.dump(command_line.options.destpath, mask)

  d = dpack(
    active_areas=[0,0,command_line.options.mask_width,command_line.options.mask_height],
    address=None,
    beam_center_x=None,
    beam_center_y=None,
    data=mask,
    distance=None,
    timestamp=None,
    wavelength=1,
    xtal_target=None,
    pixel_size=command_line.options.pixel_size,
    saturated_value=None)

  dwritef2(d, command_line.options.destpath)
Ejemplo n.º 32
0
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.

    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(average_mixin, self).event(evt, env)
    if evt.get('skip_event'):
      return

    # Get the distance for the detectors that should have it, and set
    # it to NaN for those that should not.
    if self.detector == 'CxiDs1' or \
       self.detector == 'CxiDs2' or \
       self.detector == 'CxiDsd' or \
       self.detector == 'XppGon':
      distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
      if distance is None:
        self._nfail += 1
        self.logger.warning("event(): no distance, shot skipped")
        evt.put(skip_event_flag(), 'skip_event')
        return
    else:
      distance = float('nan')

    if ("skew" in self.flags):
      # Take out inactive pixels
      if self.roi is not None:
        pixels = self.cspad_img[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]]
        dark_mask = self.dark_mask[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]]
        pixels = pixels.as_1d().select(dark_mask.as_1d())
      else:
        pixels = self.cspad_img.as_1d().select(self.dark_mask.as_1d()).as_double()
      stats = scitbx.math.basic_statistics(pixels.as_double())
      #stats.show()
      self.logger.info("skew: %.3f" %stats.skew)
      self.logger.info("kurtosis: %.3f" %stats.kurtosis)
      if 0:
        from matplotlib import pyplot
        hist_min, hist_max = flex.min(flex_cspad_img.as_double()), flex.max(flex_cspad_img.as_double())
        print hist_min, hist_max
        n_slots = 100
        n, bins, patches = pyplot.hist(flex_cspad_img.as_1d().as_numpy_array(), bins=n_slots, range=(hist_min, hist_max))
        pyplot.show()

      # XXX This skew threshold probably needs fine-tuning
      skew_threshold = 0.35
      if stats.skew < skew_threshold:
        self._nfail += 1
        self.logger.warning("event(): skew < %f, shot skipped" % skew_threshold)
        evt.put(skip_event_flag(), 'skip_event')
        return
      #self.cspad_img *= stats.skew

    if ("inactive" in self.flags):
      self.cspad_img.set_selected(self.dark_stddev <= 0, 0)

    if ("noelastic" in self.flags):
      ELASTIC_THRESHOLD = self.elastic_threshold
      self.cspad_img.set_selected(self.cspad_img > ELASTIC_THRESHOLD, 0)

    if self.hot_threshold is not None:
      HOT_THRESHOLD = self.hot_threshold
      self.cspad_img.set_selected(self.dark_img > HOT_THRESHOLD, 0)

    if self.gain_map is not None and self.gain_threshold is not None:
      # XXX comparing each pixel to a moving average would probably be better
      # since the gain should vary approximately smoothly over different areas
      # of the detector
      GAIN_THRESHOLD = self.gain_threshold
      #self.logger.debug(
        #"rejecting: %i" %(self.gain_map > GAIN_THRESHOLD).count(True))
      self.cspad_img.set_selected(self.gain_map > GAIN_THRESHOLD, 0)

    if ("nonoise" in self.flags):
      NOISE_THRESHOLD = self.noise_threshold
      self.cspad_img.set_selected(self.cspad_img < NOISE_THRESHOLD, 0)

    if ("sigma_scaling" in self.flags):
      self.do_sigma_scaling()

    if ("symnoise" in self.flags):
      SYMNOISE_THRESHOLD = self.symnoise_threshold
      self.cspad_img.set_selected((-SYMNOISE_THRESHOLD < self.cspad_img) &
                                  ( self.cspad_img  < SYMNOISE_THRESHOLD), 0)

    if ("output" in self.flags):
      import pickle,os
      if (not os.path.isdir(self.pickle_dirname)):
        os.makedirs(self.pickle_dirname)
      flexdata = flex.int(self.cspad_img.astype(numpy.int32))
      d = cspad_tbx.dpack(
        address=self.address,
        data=flexdata,
        timestamp=cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt))
      )
      G = open(os.path.join(".",self.pickle_dirname)+"/"+self.pickle_basename,
               "ab")
      pickle.dump(d,G,pickle.HIGHEST_PROTOCOL)
      G.close()

    if self.photon_threshold is not None and self.two_photon_threshold is not None:
      self.do_photon_counting()

    if self.background_path is not None:
      self.cspad_img -= self.background_img


    # t and self._sum_time are a two-long arrays of seconds and
    # milliseconds which hold time with respect to the base time.
    t = [t1 - t2 for (t1, t2) in zip(cspad_tbx.evt_time(evt),
                                     self._metadata['time_base'])]
    if self._nmemb == 0:
      # The peers metadata item is a bit field where a bit is set if
      # the partial sum from the corresponding worker process is
      # pending.  If this is the first frame a worker process sees,
      # set its corresponding bit in the bit field since it will
      # contribute a partial sum.
      if env.subprocess() >= 0:
        self._lock.acquire()
        if 'peers' in self._metadata.keys():
          self._metadata['peers'] |= (1 << env.subprocess())
        else:
          self._metadata['peers'] = (1 << env.subprocess())
        self._lock.release()

      self._sum_distance = distance
      self._sum_time = (t[0], t[1])
      self._sum_wavelength = self.wavelength

      if self._have_max:
        self._max_img = self.cspad_img.deep_copy()
      if self._have_mean:
        self._sum_img = self.cspad_img.deep_copy()
      if self._have_std:
        self._ssq_img = flex.pow2(self.cspad_img)

    else:
      self._sum_distance += distance
      self._sum_time = (self._sum_time[0] + t[0], self._sum_time[1] + t[1])
      self._sum_wavelength += self.wavelength

      if self._have_max:
        sel = (self.cspad_img > self._max_img).as_1d()
        self._max_img.as_1d().set_selected(
          sel, self.cspad_img.as_1d().select(sel))
      if self._have_mean:
        self._sum_img += self.cspad_img
      if self._have_std:
        self._ssq_img += flex.pow2(self.cspad_img)

    self._nmemb += 1
Ejemplo n.º 33
0
def run(argv=None):
    if argv is None:
        argv = sys.argv[1:]

    command_line = (
        libtbx.option_parser.option_parser(
            usage="%s [-v] [-c] [-s] [-w wavelength] [-d distance] [-p pixel_size] [-x beam_x] [-y beam_y] [-o overload] files"
            % libtbx.env.dispatcher_name
        )
        .option(
            None,
            "--verbose",
            "-v",
            action="store_true",
            default=False,
            dest="verbose",
            help="Print more information about progress",
        )
        .option(
            None,
            "--crop",
            "-c",
            action="store_true",
            default=False,
            dest="crop",
            help="Crop the image such that the beam center is in the middle",
        )
        .option(
            None,
            "--skip_converted",
            "-s",
            action="store_true",
            default=False,
            dest="skip_converted",
            help="Skip converting if an image already exist that matches the destination file name",
        )
        .option(
            None,
            "--wavelength",
            "-w",
            type="float",
            default=None,
            dest="wavelength",
            help="Override the image's wavelength (angstroms)",
        )
        .option(
            None,
            "--distance",
            "-d",
            type="float",
            default=None,
            dest="distance",
            help="Override the detector distance (mm)",
        )
        .option(
            None,
            "--pixel_size",
            "-p",
            type="float",
            default=None,
            dest="pixel_size",
            help="Override the detector pixel size (mm)",
        )
        .option(
            None,
            "--beam_x",
            "-x",
            type="float",
            default=None,
            dest="beam_center_x",
            help="Override the beam x position (pixels)",
        )
        .option(
            None,
            "--beam_y",
            "-y",
            type="float",
            default=None,
            dest="beam_center_y",
            help="Override the beam y position (pixels)",
        )
        .option(
            None,
            "--overload",
            "-o",
            type="float",
            default=None,
            dest="overload",
            help="Override the detector overload value (ADU)",
        )
    ).process(args=argv)

    paths = command_line.args
    if len(paths) <= 0:
        raise Usage("No files specified")

    for imgpath in paths:
        destpath = os.path.join(os.path.dirname(imgpath), os.path.splitext(os.path.basename(imgpath))[0] + ".pickle")
        if command_line.options.skip_converted and os.path.isfile(destpath):
            if command_line.options.verbose:
                print "Skipping %s, file exists" % imgpath
                continue

        if command_line.options.verbose:
            print "Converting %s to %s..." % (imgpath, destpath)

        try:
            img = dxtbx.load(imgpath)
        except IOError:
            img = None
            pass

        if img is None:
            import numpy as np

            try:
                raw_data = np.loadtxt(imgpath)

                from scitbx.array_family import flex

                raw_data = flex.double(raw_data.astype(np.double))
            except ValueError:
                raise Usage("Couldn't load %s, no supported readers" % imgpath)

            detector = None
            beam = None
            scan = None
        else:
            raw_data = img.get_raw_data()
            detector = img.get_detector()
            beam = img.get_beam()
            scan = img.get_scan()

        if detector is None:
            if command_line.options.distance is None:
                raise Usage("Can't get distance from image. Override with -d")
            if command_line.options.pixel_size is None:
                raise Usage("Can't get pixel size from image. Override with -p")
            if command_line.options.overload is None:
                raise Usage("Can't get overload value from image. Override with -o")
            distance = command_line.options.distance
            pixel_size = command_line.options.pixel_size
            overload = command_line.options.overload
        else:
            detector = detector[0]
            if command_line.options.distance is None:
                distance = detector.get_distance()
            else:
                distance = command_line.options.distance

            if command_line.options.pixel_size is None:
                pixel_size = detector.get_pixel_size()[0]
            else:
                pixel_size = command_line.options.pixel_size

            if command_line.options.overload is None:
                overload = detector.get_trusted_range()[1]
            else:
                overload = command_line.options.overload

        if beam is None:
            if command_line.options.wavelength is None:
                raise Usage("Can't get wavelength from image. Override with -w")
            wavelength = command_line.options.wavelength
        else:
            if command_line.options.wavelength is None:
                wavelength = beam.get_wavelength()
            else:
                wavelength = command_line.options.wavelength

        if beam is None and detector is None:
            if command_line.options.beam_center_x is None:
                print "Can't get beam x position from image. Using image center. Override with -x"
                beam_x = raw_data.focus()[0] * pixel_size
            else:
                beam_x = command_line.options.beam_center_x * pixel_size

            if command_line.options.beam_center_y is None:
                print "Can't get beam y position from image. Using image center. Override with -y"
                beam_y = raw_data.focus()[1] * pixel_size
            else:
                beam_y = command_line.options.beam_center_y * pixel_size
        else:
            if command_line.options.beam_center_x is None:
                beam_x = detector.get_beam_centre(beam.get_s0())[0]
            else:
                beam_x = command_line.options.beam_center_x * pixel_size

            if command_line.options.beam_center_y is None:
                beam_y = detector.get_beam_centre(beam.get_s0())[1]
            else:
                beam_y = command_line.options.beam_center_y * pixel_size

        if scan is None:
            timestamp = None
        else:
            msec, sec = math.modf(scan.get_epochs()[0])
            timestamp = evt_timestamp((sec, msec))

        data = dpack(
            data=raw_data,
            distance=distance,
            pixel_size=pixel_size,
            wavelength=wavelength,
            beam_center_x=beam_x,
            beam_center_y=beam_y,
            ccd_image_saturation=overload,
            saturated_value=overload,
            timestamp=timestamp,
        )

        if scan is not None:
            osc_start, osc_range = scan.get_oscillation()
            if osc_start != osc_range:
                data["OSC_START"] = osc_start
                data["OSC_RANGE"] = osc_range

                data["TIME"] = scan.get_exposure_times()[0]

        if command_line.options.crop:
            data = crop_image_pickle(data)
        easy_pickle.dump(destpath, data)
                    quad_asics.append(numpy.concatenate((a, b), axis=1))
            quad_data = numpy.dstack(quad_asics)
            quad_data = numpy.rollaxis(quad_data, 2, 0)
            data3d.append(fake_cspad_ElementV2(quad_data, i_quad))

        env = fake_env(fake_config())
        evt = fake_evt(data3d)
        beam_center, active_areas = cbcaa(fake_config(), sections)
        data = flex.int(
            CsPadDetector(address, evt, env, sections).astype(numpy.float64))

    img_dict = dpack(active_areas=active_areas,
                     address=address,
                     beam_center_x=beam_center[0] * pixel_size,
                     beam_center_y=beam_center[1] * pixel_size,
                     data=data,
                     distance=params.distance,
                     pixel_size=pixel_size,
                     timestamp=timestamp,
                     wavelength=params.wavelength)

    img = NpyImage("", source_data=img_dict)

    args = [
        "distl.detector_format_version=%s" % params.detector_format_version
    ]
    horizons_phil = cxi_phil.cxi_versioned_extract(args)

    img.readHeader(horizons_phil)
    img.translate_tiles(horizons_phil)
    tm = img.get_tile_manager(horizons_phil)
Ejemplo n.º 35
0
def run(argv=None):
  import libtbx.option_parser

  if (argv is None):
    argv = sys.argv

  command_line = (libtbx.option_parser.option_parser(
    usage="%s [-v] [-p poly_mask] [-c circle_mask]  [-x mask_pix_val] [-o output] -W mask_width -H mask_height" % libtbx.env.dispatcher_name)
                  .option(None, "--verbose", "-v",
                          action="store_true",
                          default=False,
                          dest="verbose",
                          help="Print more information about progress")
                  .option(None, "--poly_mask", "-p",
                          type="string",
                          default=None,
                          dest="poly_mask",
                          help="Polygon to mask out.  Comma-seperated string of xy pairs.")
                  .option(None, "--circle_mask", "-c",
                          type="string",
                          default=None,
                          dest="circle_mask",
                          help="Circle to mask out.  Comma-seperated string of x, y, and radius.")
                  .option(None, "--mask_pix_val", "-x",
                          type="int",
                          default=-2,
                          dest="mask_pix_val",
                          help="Value for masked out pixels")
                  .option(None, "--mask_width", "-W",
                          type="int",
                          default=None,
                          dest="mask_width",
                          help="Width of output mask")
                   .option(None, "--mask_height", "-H",
                          type="int",
                          default=None,
                          dest="mask_height",
                          help="Height of output mask")
                   .option(None, "--output", "-o",
                          type="string",
                          default="mask.pickle",
                          dest="destpath",
                          help="Output file path, should be *.pickle")
                   .option(None, "--pixel_size", "-s",
                          type="float",
                          default=None,
                          dest="pixel_size",
                          help="Pixel size for detector")
                  ).process(args=argv[1:])

  # Must have width and height set
  if command_line.options.mask_height is None or command_line.options.mask_width is None:
    command_line.parser.print_usage(file=sys.stderr)
    return

  poly_mask = None
  if not command_line.options.poly_mask == None:
    poly_mask = []
    poly_mask_tmp = command_line.options.poly_mask.split(",")
    if len(poly_mask_tmp) % 2 != 0:
      command_line.parser.print_usage(file=sys.stderr)
      return
    odd = True
    for item in poly_mask_tmp:
      try:
        if odd:
          poly_mask.append(int(item))
        else:
          poly_mask[-1] = (poly_mask[-1],int(item))
      except ValueError:
        command_line.parser.print_usage(file=sys.stderr)
        return
      odd = not odd

  circle_mask = None
  if command_line.options.circle_mask is not None:
    circle_mask_tmp = command_line.options.circle_mask.split(",")
    if len(circle_mask_tmp) != 3:
      command_line.parser.print_usage(file=sys.stderr)
      return
    try:
      circle_mask = (int(circle_mask_tmp[0]),int(circle_mask_tmp[1]),int(circle_mask_tmp[2]))
    except ValueError:
      command_line.parser.print_usage(file=sys.stderr)
      return

  mask = flex.int(flex.grid(command_line.options.mask_width,
                             command_line.options.mask_height))

  if poly_mask is not None or circle_mask is not None:
    minx = miny = 0
    maxx = mask.focus()[0]
    maxy = mask.focus()[1]
    if poly_mask is not None:
      minx = min([x[0] for x in poly_mask])
      miny = min([y[1] for y in poly_mask])
      maxx = max([x[0] for x in poly_mask])
      maxy = max([y[1] for y in poly_mask])
    if circle_mask is not None:
      circle_x, circle_y, radius = circle_mask

      if circle_x - radius < minx: minx = circle_x - radius
      if circle_y - radius < miny: miny = circle_y - radius
      if circle_x + radius > maxx: maxx = circle_x + radius
      if circle_y + radius > maxy: maxy = circle_y + radius

    sel = mask == command_line.options.mask_pix_val
    for j in range(miny, maxy):
      for i in range(minx, maxx):
        idx = j * mask.focus()[0] + i
        if not sel[idx]:
          if poly_mask is not None and point_in_polygon((i,j),poly_mask):
            sel[idx] = True
          elif circle_mask is not None and point_inside_circle(i,j,circle_x,circle_y,radius):
            sel[idx] = True
    mask.set_selected(sel,command_line.options.mask_pix_val)

  masked_out = len(mask.as_1d().select((mask == command_line.options.mask_pix_val).as_1d()))

  print("Masked out %d pixels out of %d (%.2f%%)"% \
    (masked_out,len(mask),(masked_out)*100/(len(mask))))

  easy_pickle.dump(command_line.options.destpath, mask)

  d = dpack(
    active_areas=[0,0,command_line.options.mask_width,command_line.options.mask_height],
    address=None,
    beam_center_x=None,
    beam_center_y=None,
    data=mask,
    distance=None,
    timestamp=None,
    wavelength=1,
    xtal_target=None,
    pixel_size=command_line.options.pixel_size,
    saturated_value=None)

  dwritef2(d, command_line.options.destpath)
Ejemplo n.º 36
0
    def load_image_file(self, filepath):
        """ Reads a single image file (e.g. a CBF or MCCD) and returns raw data,
    experiment information, and an image type (e.g. pickle, raw image, or none)
    :param img: path to image file
    :return: data: raw image data and experiment info
             img_type: which type of image this was, or None if not loaded
    """

        error = None
        try:
            with util.Capturing() as junk_output:
                loaded_img = dxtbx.load(filepath)
        except Exception as e:
            error = 'IOTA IMPORTER ERROR: DXTBX failed! {}'.format(e)
            print(e)
            loaded_img = None

        # Extract image information
        if loaded_img is not None:
            raw_data = loaded_img.get_raw_data()
            detector = loaded_img.get_detector()[0]
            beam = loaded_img.get_beam()
            scan = loaded_img.get_scan()
            distance = detector.get_distance()
            pixel_size = detector.get_pixel_size()[0]
            overload = detector.get_trusted_range()[1]
            wavelength = beam.get_wavelength()
            beam_x = detector.get_beam_centre(beam.get_s0())[0]
            beam_y = detector.get_beam_centre(beam.get_s0())[1]

            if scan is None:
                timestamp = None
            else:
                msec, sec = math.modf(scan.get_epochs()[0])
                timestamp = evt_timestamp((sec, msec))

            # Assemble datapack
            data = dpack(data=raw_data,
                         distance=distance,
                         pixel_size=pixel_size,
                         wavelength=wavelength,
                         beam_center_x=beam_x,
                         beam_center_y=beam_y,
                         ccd_image_saturation=overload,
                         saturated_value=overload,
                         timestamp=timestamp)

            if scan is not None:
                osc_start, osc_range = scan.get_oscillation()
                if osc_start != osc_range:
                    data['OSC_START'] = 0  #osc_start
                    data['OSC_RANGE'] = 0  #osc_start
                    data['TIME'] = scan.get_exposure_times()[0]

            # Populate image object information
            self.img_object.final['pixel_size'] = pixel_size
            self.img_object.final['img_size'] = (data['SIZE1'], data['SIZE2'])
            self.img_object.final['beamX'] = beam_x
            self.img_object.final['beamY'] = beam_y
            self.img_object.final['gain'] = detector.get_gain()
            self.img_object.final['distance'] = distance
            self.img_object.final['wavelength'] = wavelength

        else:
            data = None

        return data, error
Ejemplo n.º 37
0
    spectrum_focus = self.sum_img[start_row:end_row,:]
    mask_focus = mask[start_row:end_row,:]

    spectrum_focus.set_selected(mask_focus > 0, 0)

    xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)

    print "Number of rows: %i" %spectrum_focus.all()[0]
    print "Estimated no. photons counted: %i" %flex.sum(spectrum_focus)
    print "Number of images used: %i" %flex.sum(
      pixel_histograms.histograms.values()[0].slots())

    d = cspad_tbx.dpack(
      address='CxiSc1-0|Cspad2x2-0',
      data=spectrum_focus,
      distance=1,
      ccd_image_saturation=2e8, # XXX
    )
    if run is not None: runstr="_%04d"%run
    else: runstr=""
    cspad_tbx.dwritef(d, output_dirname, 'sum%s_'%runstr)

    if gain_map is None:
      gain_map = flex.double(gain_img.accessor(), 0)
      img_sel = (gain_img > 0).as_1d()
      d = cspad_tbx.dpack(
        address='CxiSc1-0|Cspad2x2-0',
        data=gain_img,
        distance=1
      )
      cspad_tbx.dwritef(d, output_dirname, 'raw_gain_map_')
Ejemplo n.º 38
0
def run(argv=None):
    """Compute mean, standard deviation, and maximum projection images
  from a set of images given on the command line.

  @param argv Command line argument list
  @return     @c 0 on successful termination, @c 1 on error, and @c 2
              for command line syntax errors
  """
    import libtbx.load_env

    from libtbx import easy_pickle, option_parser
    from scitbx.array_family import flex
    from xfel.cxi.cspad_ana import cspad_tbx

    if argv is None:
        argv = sys.argv
    command_line = (option_parser.option_parser(
      usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \
      "image1 image2 [image3 ...]" % libtbx.env.dispatcher_name)
                    .option(None, "--average-path", "-a",
                            type="string",
                            default=None,
                            dest="avg_path",
                            metavar="PATH",
                            help="Write average image to PATH")
                    .option(None, "--maximum-path", "-m",
                            type="string",
                            default=None,
                            dest="max_path",
                            metavar="PATH",
                            help="Write maximum projection image to PATH")
                    .option(None, "--stddev-path", "-s",
                            type="string",
                            default=None,
                            dest="stddev_path",
                            metavar="PATH",
                            help="Write standard deviation image to PATH")
                    .option(None, "--verbose", "-v",
                            action="store_true",
                            default=False,
                            dest="verbose",
                            help="Print more information about progress")
                    ).process(args=argv[1:])

    # Note that it is not an error to omit the output paths, because
    # certain statistics could still be printed, e.g. with the verbose
    # option.
    paths = command_line.args
    if len(paths) == 0:
        command_line.parser.print_usage(file=sys.stderr)
        return 2

    # Loop over all images and accumulate statistics.
    nfail = 0
    nmemb = 0

    if len(paths) == 1:
        # test if the iamge is a multi-image
        from dxtbx.format.Registry import Registry
        from dxtbx.format.FormatMultiImage import FormatMultiImage
        format_class = Registry.find(paths[0])
        if not issubclass(format_class, FormatMultiImage):
            from libtbx.utils import Usage
            raise Usage("Supply more than one image")

        print "Loading image..."
        i = format_class(paths[0])
        print "Loaded"

        def read_single_image(n):
            if command_line.options.verbose:
                sys.stdout.write("Processing %s: %d...\n" % (paths[0], n))

            beam = i.get_beam(n)
            assert len(i.get_detector(n)) == 1
            detector = i.get_detector(n)[0]

            beam_center = detector.get_beam_centre(beam.get_s0())
            detector_address = format_class.__name__
            distance = detector.get_distance()
            img = i.get_raw_data(n).as_1d().as_double()
            pixel_size = 0.5 * sum(detector.get_pixel_size())
            saturated_value = int(round(detector.get_trusted_range()[1]))
            size = detector.get_image_size()
            scan = i.get_scan(n)
            if scan is None:
                time_tuple = (0, 0)
            else:
                time_tuple = (scan.get_epochs()[0], 0)
            wavelength = beam.get_wavelength()

            active_areas = flex.int((0, 0, size[0], size[1]))

            return beam_center, detector_address, distance, img, pixel_size, saturated_value, size, time_tuple, wavelength, active_areas

        iterable = xrange(i.get_num_images())
    else:

        def read_single_image(path):
            if command_line.options.verbose:
                sys.stdout.write("Processing %s...\n" % path)

            from dxtbx.format.Registry import Registry
            format_class = Registry.find(path)
            i = format_class(path)

            beam = i.get_beam()
            assert len(i.get_detector()) == 1
            detector = i.get_detector()[0]

            beam_center = detector.get_beam_centre(beam.get_s0())
            detector_address = format_class.__name__
            distance = detector.get_distance()
            img = i.get_raw_data().as_1d().as_double()
            pixel_size = 0.5 * sum(detector.get_pixel_size())
            saturated_value = int(round(detector.get_trusted_range()[1]))
            size = detector.get_image_size()
            scan = i.get_scan()
            if scan is None:
                time_tuple = (0, 0)
            else:
                time_tuple = (scan.get_epochs()[0], 0)
            wavelength = beam.get_wavelength()

            active_areas = flex.int((0, 0, size[0], size[1]))
            return beam_center, detector_address, distance, img, pixel_size, saturated_value, size, time_tuple, wavelength, active_areas

        iterable = paths

    for item in iterable:
        try:
            #XXX This code assumes a monolithic detector!
            beam_center, detector_address, distance, img, pixel_size, saturated_value, size, time_tuple, wavelength, active_areas = \
              read_single_image(item)
        except Exception:
            nfail += 1
            continue

        # See also event() in xfel.cxi.cspad_ana.average_tbx.  Record the
        # base time as the timestamp of the first image.
        #
        # The sum-of-squares image is accumulated using long integers, as
        # this delays the point where overflow occurs.  But really, this
        # is just a band-aid...
        if nmemb == 0:
            max_img = img.deep_copy()
            sum_distance = distance
            sum_img = img.deep_copy()
            ssq_img = flex.pow2(img)
            sum_wavelength = wavelength
            sum_time = (0, 0)
            time_base = time_tuple

        else:
            sel = (img > max_img).as_1d()
            max_img.set_selected(sel, img.select(sel))

            sum_distance += distance
            sum_img += img
            ssq_img += flex.pow2(img)
            sum_wavelength += wavelength
            sum_time = (sum_time[0] + (time_tuple[0] - time_base[0]),
                        sum_time[1] + (time_tuple[1] - time_base[1]))

        nmemb += 1

    # Early exit if no statistics were accumulated.
    if command_line.options.verbose:
        sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail))
    if nmemb == 0:
        return 0

    # Calculate averages for measures where other statistics do not make
    # sense.  Note that avg_img is required for stddev_img.
    avg_img = sum_img.as_double() / nmemb
    avg_distance = sum_distance / nmemb
    avg_timestamp = cspad_tbx.evt_timestamp(
        (time_base[0] + int(round(sum_time[0] / nmemb)),
         time_base[1] + int(round(sum_time[1] / nmemb))))
    avg_wavelength = sum_wavelength / nmemb

    # Output the average image, maximum projection image, and standard
    # deviation image, if requested.
    if command_line.options.avg_path is not None:
        avg_img.resize(flex.grid(size[1], size[0]))
        d = cspad_tbx.dpack(active_areas=active_areas,
                            address=detector_address,
                            beam_center_x=beam_center[0],
                            beam_center_y=beam_center[1],
                            data=avg_img,
                            distance=avg_distance,
                            pixel_size=pixel_size,
                            saturated_value=saturated_value,
                            timestamp=avg_timestamp,
                            wavelength=avg_wavelength)
        easy_pickle.dump(command_line.options.avg_path, d)

    if command_line.options.max_path is not None:
        max_img.resize(flex.grid(size[1], size[0]))
        d = cspad_tbx.dpack(active_areas=active_areas,
                            address=detector_address,
                            beam_center_x=beam_center[0],
                            beam_center_y=beam_center[1],
                            data=max_img,
                            distance=avg_distance,
                            pixel_size=pixel_size,
                            saturated_value=saturated_value,
                            timestamp=avg_timestamp,
                            wavelength=avg_wavelength)
        easy_pickle.dump(command_line.options.max_path, d)

    if command_line.options.stddev_path is not None:
        stddev_img = ssq_img.as_double() - sum_img.as_double() * avg_img

        # Accumulating floating-point numbers introduces errors, which may
        # cause negative variances.  Since a two-pass approach is
        # unacceptable, the standard deviation is clamped at zero.
        stddev_img.set_selected(stddev_img < 0, 0)
        if nmemb == 1:
            stddev_img = flex.sqrt(stddev_img)
        else:
            stddev_img = flex.sqrt(stddev_img / (nmemb - 1))

        stddev_img.resize(flex.grid(size[1], size[0]))
        d = cspad_tbx.dpack(active_areas=active_areas,
                            address=detector_address,
                            beam_center_x=beam_center[0],
                            beam_center_y=beam_center[1],
                            data=stddev_img,
                            distance=avg_distance,
                            pixel_size=pixel_size,
                            saturated_value=saturated_value,
                            timestamp=avg_timestamp,
                            wavelength=avg_wavelength)
        easy_pickle.dump(command_line.options.stddev_path, d)

    return 0
Ejemplo n.º 39
0
def average(argv=None):
  if argv == None:
    argv = sys.argv[1:]

  try:
    from mpi4py import MPI
  except ImportError:
    raise Sorry("MPI not found")

  command_line = (libtbx.option_parser.option_parser(
    usage="""
%s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f]

To write image pickles use -p, otherwise the program writes CSPAD CBFs.
Writing CBFs requires the geometry to be already deployed.

Examples:
cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571

Use one process on the current node to process all the events from run 25 of
experiment cxi49812, using a detz_offset of 571.

mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571

As above, using 16 cores on the current node.

bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812

As above, using the psanaq and 100 cores, putting the log in average.out and
the output images in the folder cxi49812.
""" % libtbx.env.dispatcher_name)
                .option(None, "--as_pickle", "-p",
                        action="store_true",
                        default=False,
                        dest="as_pickle",
                        help="Write results as image pickle files instead of cbf files")
                .option(None, "--config", "-c",
                        type="string",
                        default=None,
                        dest="config",
                        metavar="PATH",
                        help="psana config file")
                .option(None, "--experiment", "-x",
                        type="string",
                        default=None,
                        dest="experiment",
                        help="experiment name (eg cxi84914)")
                .option(None, "--run", "-r",
                        type="int",
                        default=None,
                        dest="run",
                        help="run number")
                .option(None, "--address", "-a",
                        type="string",
                        default="CxiDs2.0:Cspad.0",
                        dest="address",
                        help="detector address name (eg CxiDs2.0:Cspad.0)")
                .option(None, "--detz_offset", "-d",
                        type="float",
                        default=None,
                        dest="detz_offset",
                        help="offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)")
                .option(None, "--outputdir", "-o",
                        type="string",
                        default=".",
                        dest="outputdir",
                        metavar="PATH",
                        help="Optional path to output directory for output files")
                .option(None, "--averagebase", "-A",
                        type="string",
                        default="{experiment!l}_avg-r{run:04d}",
                        dest="averagepath",
                        metavar="PATH",
                        help="Path to output average image without extension. String substitution allowed")
                .option(None, "--stddevbase", "-S",
                        type="string",
                        default="{experiment!l}_stddev-r{run:04d}",
                        dest="stddevpath",
                        metavar="PATH",
                        help="Path to output standard deviation image without extension. String substitution allowed")
                .option(None, "--maxbase", "-M",
                        type="string",
                        default="{experiment!l}_max-r{run:04d}",
                        dest="maxpath",
                        metavar="PATH",
                        help="Path to output maximum projection image without extension. String substitution allowed")
                .option(None, "--numevents", "-n",
                        type="int",
                        default=None,
                        dest="numevents",
                        help="Maximum number of events to process. Default: all")
                .option(None, "--skipevents", "-s",
                        type="int",
                        default=0,
                        dest="skipevents",
                        help="Number of events in the beginning of the run to skip. Default: 0")
                .option(None, "--verbose", "-v",
                        action="store_true",
                        default=False,
                        dest="verbose",
                        help="Print more information about progress")
                .option(None, "--pickle-optical-metrology", "-m",
                        action="store_true",
                        default=False,
                        dest="pickle_optical_metrology",
                        help="If writing pickle files, use the optical metrology in the experiment's calib directory")
                .option(None, "--bin_size", "-b",
                        type="int",
                        default=None,
                        dest="bin_size",
                        help="Rayonix detector bin size")
                .option(None, "--override_beam_x", "-X",
                        type="float",
                        default=None,
                        dest="override_beam_x",
                        help="Rayonix detector beam center x coordinate")
                .option(None, "--override_beam_y", "-Y",
                        type="float",
                        default=None,
                        dest="override_beam_y",
                        help="Rayonix detector beam center y coordinate")
                .option(None, "--calib_dir", "-C",
                        type="string",
                        default=None,
                        dest="calib_dir",
                        metavar="PATH",
                        help="calibration directory")
                .option(None, "--xtc_dir", "-D",
                        type="string",
                        default=None,
                        dest="xtc_dir",
                        metavar="PATH",
                        help="xtc stream directory")
                .option(None, "--use_ffb", "-f",
                        action="store_true",
                        default=False,
                        dest="use_ffb",
                        help="Use the fast feedback filesystem at LCLS. Only for the active experiment!")
                ).process(args=argv)


  if len(command_line.args) > 0 or \
      command_line.options.as_pickle is None or \
      command_line.options.experiment is None or \
      command_line.options.run is None or \
      command_line.options.address is None or \
      command_line.options.detz_offset is None or \
      command_line.options.averagepath is None or \
      command_line.options.stddevpath is None or \
      command_line.options.maxpath is None or \
      command_line.options.pickle_optical_metrology is None:
    command_line.parser.show_help()
    return

  # set this to sys.maxint to analyze all events
  if command_line.options.numevents is None:
    maxevents = sys.maxint
  else:
    maxevents = command_line.options.numevents

  comm = MPI.COMM_WORLD
  rank = comm.Get_rank()
  size = comm.Get_size()

  if command_line.options.config is not None:
    psana.setConfigFile(command_line.options.config)
  dataset_name = "exp=%s:run=%d:idx"%(command_line.options.experiment, command_line.options.run)
  if command_line.options.xtc_dir is not None:
    if command_line.options.use_ffb:
      raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system")
    dataset_name += ":dir=%s"%command_line.options.xtc_dir
  elif command_line.options.use_ffb:
    # as ffb is only at SLAC, ok to hardcode /reg/d here
    dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc"%(command_line.options.experiment[0:3],command_line.options.experiment)
  ds = psana.DataSource(dataset_name)
  address = command_line.options.address
  src = psana.Source('DetInfo(%s)'%address)
  if not command_line.options.as_pickle:
    psana_det = psana.Detector(address, ds.env())

  nevent = np.array([0.])

  for run in ds.runs():
    runnumber = run.run()
    # list of all events
    if command_line.options.skipevents > 0:
      print "Skipping first %d events"%command_line.options.skipevents

    times = run.times()[command_line.options.skipevents:]
    nevents = min(len(times),maxevents)
    # chop the list into pieces, depending on rank.  This assigns each process
    # events such that the get every Nth event where N is the number of processes
    mytimes = [times[i] for i in xrange(nevents) if (i+rank)%size == 0]
    for i in xrange(len(mytimes)):
      if i%10==0: print 'Rank',rank,'processing event',rank*len(mytimes)+i,', ',i,'of',len(mytimes)
      evt = run.event(mytimes[i])
      #print "Event #",rank*mylength+i," has id:",evt.get(EventId)
      if 'Rayonix' in command_line.options.address:
        data = evt.get(Camera.FrameV1,src)
        if data is None:
          print "No data"
          continue
        data=data.data16().astype(np.float64)
      elif command_line.options.as_pickle:
        data = evt.get(psana.ndarray_float64_3, src, 'image0')
      else:
        # get numpy array, 32x185x388
        data = psana_det.calib(evt) # applies psana's complex run-dependent calibrations
      if data is None:
        print "No data"
        continue

      d = cspad_tbx.env_distance(address, run.env(), command_line.options.detz_offset)
      if d is None:
        print "No distance, skipping shot"
        continue
      if 'distance' in locals():
        distance += d
      else:
        distance = np.array([float(d)])

      w = cspad_tbx.evt_wavelength(evt)
      if w is None:
        print "No wavelength, skipping shot"
        continue
      if 'wavelength' in locals():
        wavelength += w
      else:
        wavelength = np.array([w])

      t = cspad_tbx.evt_time(evt)
      if t is None:
        print "No timestamp, skipping shot"
        continue
      if 'timestamp' in locals():
        timestamp += t[0] + (t[1]/1000)
      else:
        timestamp = np.array([t[0] + (t[1]/1000)])

      if 'sum' in locals():
        sum+=data
      else:
        sum=np.array(data, copy=True)
      if 'sumsq' in locals():
        sumsq+=data*data
      else:
        sumsq=data*data
      if 'maximum' in locals():
        maximum=np.maximum(maximum,data)
      else:
        maximum=np.array(data, copy=True)

      nevent += 1

  #sum the images across mpi cores
  if size > 1:
    print "Synchronizing rank", rank
  totevent = np.zeros(nevent.shape)
  comm.Reduce(nevent,totevent)

  if rank == 0 and totevent[0] == 0:
    raise Sorry("No events found in the run")

  sumall = np.zeros(sum.shape).astype(sum.dtype)
  comm.Reduce(sum,sumall)

  sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype)
  comm.Reduce(sumsq,sumsqall)

  maxall = np.zeros(maximum.shape).astype(maximum.dtype)
  comm.Reduce(maximum,maxall, op=MPI.MAX)

  waveall = np.zeros(wavelength.shape).astype(wavelength.dtype)
  comm.Reduce(wavelength,waveall)

  distall = np.zeros(distance.shape).astype(distance.dtype)
  comm.Reduce(distance,distall)

  timeall = np.zeros(timestamp.shape).astype(timestamp.dtype)
  comm.Reduce(timestamp,timeall)

  if rank==0:
    if size > 1:
      print "Synchronized"

    # Accumulating floating-point numbers introduces errors,
    # which may cause negative variances.  Since a two-pass
    # approach is unacceptable, the standard deviation is
    # clamped at zero.
    mean = sumall / float(totevent[0])
    variance = (sumsqall / float(totevent[0])) - (mean**2)
    variance[variance < 0] = 0
    stddev = np.sqrt(variance)

    wavelength = waveall[0] / totevent[0]
    distance = distall[0] / totevent[0]
    pixel_size = cspad_tbx.pixel_size
    saturated_value = cspad_tbx.cspad_saturated_value
    timestamp = timeall[0] / totevent[0]
    timestamp = (int(timestamp), timestamp % int(timestamp) * 1000)
    timestamp = cspad_tbx.evt_timestamp(timestamp)


    if command_line.options.as_pickle:
      extension = ".pickle"
    else:
      extension = ".cbf"

    dest_paths = [cspad_tbx.pathsubst(command_line.options.averagepath + extension, evt, ds.env()),
                  cspad_tbx.pathsubst(command_line.options.stddevpath  + extension, evt, ds.env()),
                  cspad_tbx.pathsubst(command_line.options.maxpath     + extension, evt, ds.env())]
    dest_paths = [os.path.join(command_line.options.outputdir, path) for path in dest_paths]
    if 'Rayonix' in command_line.options.address:
      from xfel.cxi.cspad_ana import rayonix_tbx
      pixel_size = rayonix_tbx.get_rayonix_pixel_size(command_line.options.bin_size)
      beam_center = [command_line.options.override_beam_x,command_line.options.override_beam_y]
      detector_dimensions = rayonix_tbx.get_rayonix_detector_dimensions(command_line.options.bin_size)
      active_areas = flex.int([0,0,detector_dimensions[0],detector_dimensions[1]])
      split_address = cspad_tbx.address_split(address)
      old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3]
      for data, path in zip([mean, stddev, maxall], dest_paths):
        print "Saving", path
        d = cspad_tbx.dpack(
            active_areas=active_areas,
            address=old_style_address,
            beam_center_x=pixel_size * beam_center[0],
            beam_center_y=pixel_size * beam_center[1],
            data=flex.double(data),
            distance=distance,
            pixel_size=pixel_size,
            saturated_value=rayonix_tbx.rayonix_saturated_value,
            timestamp=timestamp,
            wavelength=wavelength)
        easy_pickle.dump(path, d)
    elif command_line.options.as_pickle:
      split_address = cspad_tbx.address_split(address)
      old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3]

      xpp = 'xpp' in address.lower()
      if xpp:
        evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
        timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format
        from xfel.detector_formats import detector_format_version, reverse_timestamp
        from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
        version_lookup = detector_format_version(old_style_address, reverse_timestamp(timestamp)[0])
        assert version_lookup is not None
        active_areas = xpp_active_areas[version_lookup]['active_areas']
        beam_center = [1765 // 2, 1765 // 2]
      else:
        if command_line.options.calib_dir is not None:
          metro_path = command_line.options.calib_dir
        elif command_line.options.pickle_optical_metrology:
          from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path
          metro_path = get_calib_file_path(run.env(), address, run)
        else:
          metro_path = libtbx.env.find_in_repositories("xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
        sections = parse_calib.calib2sections(metro_path)
        beam_center, active_areas = cspad_tbx.cbcaa(
          cspad_tbx.getConfig(address, ds.env()), sections)

      class fake_quad(object):
        def __init__(self, q, d):
          self.q = q
          self.d = d

        def quad(self):
          return self.q

        def data(self):
          return self.d

      if xpp:
        quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)]
        mean = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
        mean = flex.double(mean.astype(np.float64))

        quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)]
        stddev = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
        stddev = flex.double(stddev.astype(np.float64))

        quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)]
        maxall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
        maxall = flex.double(maxall.astype(np.float64))
      else:
        quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)]
        mean = cspad_tbx.CsPadDetector(
          address, evt, ds.env(), sections, quads=quads)
        mean = flex.double(mean.astype(np.float64))

        quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)]
        stddev = cspad_tbx.CsPadDetector(
          address, evt, ds.env(), sections, quads=quads)
        stddev = flex.double(stddev.astype(np.float64))

        quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)]
        maxall = cspad_tbx.CsPadDetector(
          address, evt, ds.env(), sections, quads=quads)
        maxall = flex.double(maxall.astype(np.float64))

      for data, path in zip([mean, stddev, maxall], dest_paths):
        print "Saving", path

        d = cspad_tbx.dpack(
          active_areas=active_areas,
          address=old_style_address,
          beam_center_x=pixel_size * beam_center[0],
          beam_center_y=pixel_size * beam_center[1],
          data=data,
          distance=distance,
          pixel_size=pixel_size,
          saturated_value=saturated_value,
          timestamp=timestamp,
          wavelength=wavelength)

        easy_pickle.dump(path, d)
    else:
      # load a header only cspad cbf from the slac metrology
      from xfel.cftbx.detector import cspad_cbf_tbx
      import pycbf
      base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(run, address)
      if base_dxtbx is None:
        raise Sorry("Couldn't load calibration file for run %d"%run.run())

      for data, path in zip([mean, stddev, maxall], dest_paths):
        print "Saving", path

        cspad_img = cspad_cbf_tbx.format_object_from_data(base_dxtbx, data, distance, wavelength, timestamp, address)
        cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
          pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
Ejemplo n.º 40
0
def run(argv=None):
    """Compute mean, standard deviation, and maximum projection images
  from a set of images given on the command line.

  @param argv Command line argument list
  @return     @c 0 on successful termination, @c 1 on error, and @c 2
              for command line syntax errors
  """
    import libtbx.load_env

    from libtbx import easy_pickle, option_parser
    from xfel.cxi.cspad_ana import cspad_tbx

    if argv is None:
        argv = sys.argv
    command_line = (option_parser.option_parser(
      usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \
      "image1 image2 [image3 ...]" % libtbx.env.dispatcher_name)
                    .option(None, "--average-path", "-a",
                            type="string",
                            default=None,
                            dest="avg_path",
                            metavar="PATH",
                            help="Write average image to PATH")
                    .option(None, "--maximum-path", "-m",
                            type="string",
                            default=None,
                            dest="max_path",
                            metavar="PATH",
                            help="Write maximum projection image to PATH")
                    .option(None, "--stddev-path", "-s",
                            type="string",
                            default=None,
                            dest="stddev_path",
                            metavar="PATH",
                            help="Write standard deviation image to PATH")
                    .option(None, "--verbose", "-v",
                            action="store_true",
                            default=False,
                            dest="verbose",
                            help="Print more information about progress")
                    .option(None, "--nproc", "-n",
                            type="int",
                            default=1,
                            dest="nproc",
                            help="Number of processors")
                    .option(None, "--num-images-max", "-N",
                            type="int",
                            default=None,
                            dest="num_images_max",
                            help="Maximum number of frames to average")
                    ).process(args=argv[1:])

    # Note that it is not an error to omit the output paths, because
    # certain statistics could still be printed, e.g. with the verbose
    # option.
    paths = command_line.args
    if len(paths) == 0:
        command_line.parser.print_usage(file=sys.stderr)
        return 2

    if len(paths) == 1:
        # test if the iamge is a multi-image
        from dxtbx.datablock import DataBlockFactory
        datablocks = DataBlockFactory.from_filenames([paths[0]])
        assert len(datablocks) == 1
        datablock = datablocks[0]
        imagesets = datablock.extract_imagesets()
        assert len(imagesets) == 1
        imageset = imagesets[0]
        if not imageset.reader().is_single_file_reader():
            from libtbx.utils import Usage
            raise Usage("Supply more than one image")

        worker = multi_image_worker(command_line, paths[0], imageset)
        if command_line.options.num_images_max is not None and command_line.options.num_images_max < len(
                imageset):
            iterable = range(command_line.options.num_images_max)
        else:
            iterable = range(len(imageset))
    else:
        # Multiple images provided
        worker = single_image_worker(command_line)
        if command_line.options.num_images_max is not None and command_line.options.num_images_max < len(
                paths):
            iterable = paths[:command_line.options.num_images_max]
        else:
            iterable = paths
    if command_line.options.nproc > 1:
        iterable = splitit(iterable, command_line.options.nproc)

    from libtbx import easy_mp
    if command_line.options.nproc == 1:
        results = [worker(iterable)]
    else:
        results = easy_mp.parallel_map(func=worker,
                                       iterable=iterable,
                                       processes=command_line.options.nproc)

    nfail = 0
    nmemb = 0
    for i, (r_nfail, r_nmemb, r_max_img, r_sum_distance, r_sum_img, r_ssq_img,
            r_sum_wavelength, size, active_areas, detector_address,
            beam_center, pixel_size, saturated_value) in enumerate(results):
        nfail += r_nfail
        nmemb += r_nmemb
        if i == 0:
            max_img = r_max_img
            sum_distance = r_sum_distance
            sum_img = r_sum_img
            ssq_img = r_ssq_img
            sum_wavelength = r_sum_wavelength
        else:
            sel = (r_max_img > max_img).as_1d()
            max_img.set_selected(sel, r_max_img.select(sel))

            sum_distance += r_sum_distance
            sum_img += r_sum_img
            ssq_img += r_ssq_img
            sum_wavelength += r_sum_wavelength

    # Early exit if no statistics were accumulated.
    if command_line.options.verbose:
        sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail))
    if nmemb == 0:
        return 0

    # Calculate averages for measures where other statistics do not make
    # sense.  Note that avg_img is required for stddev_img.
    avg_img = sum_img.as_double() / nmemb
    avg_distance = sum_distance / nmemb
    avg_wavelength = sum_wavelength / nmemb

    # Output the average image, maximum projection image, and standard
    # deviation image, if requested.
    if command_line.options.avg_path is not None:
        avg_img.resize(flex.grid(size[1], size[0]))
        d = cspad_tbx.dpack(active_areas=active_areas,
                            address=detector_address,
                            beam_center_x=beam_center[0],
                            beam_center_y=beam_center[1],
                            data=avg_img,
                            distance=avg_distance,
                            pixel_size=pixel_size,
                            saturated_value=saturated_value,
                            wavelength=avg_wavelength)
        easy_pickle.dump(command_line.options.avg_path, d)

    if command_line.options.max_path is not None:
        max_img.resize(flex.grid(size[1], size[0]))
        d = cspad_tbx.dpack(active_areas=active_areas,
                            address=detector_address,
                            beam_center_x=beam_center[0],
                            beam_center_y=beam_center[1],
                            data=max_img,
                            distance=avg_distance,
                            pixel_size=pixel_size,
                            saturated_value=saturated_value,
                            wavelength=avg_wavelength)
        easy_pickle.dump(command_line.options.max_path, d)

    if command_line.options.stddev_path is not None:
        stddev_img = ssq_img.as_double() - sum_img.as_double() * avg_img

        # Accumulating floating-point numbers introduces errors, which may
        # cause negative variances.  Since a two-pass approach is
        # unacceptable, the standard deviation is clamped at zero.
        stddev_img.set_selected(stddev_img < 0, 0)
        if nmemb == 1:
            stddev_img = flex.sqrt(stddev_img)
        else:
            stddev_img = flex.sqrt(stddev_img / (nmemb - 1))

        stddev_img.resize(flex.grid(size[1], size[0]))
        d = cspad_tbx.dpack(active_areas=active_areas,
                            address=detector_address,
                            beam_center_x=beam_center[0],
                            beam_center_y=beam_center[1],
                            data=stddev_img,
                            distance=avg_distance,
                            pixel_size=pixel_size,
                            saturated_value=saturated_value,
                            wavelength=avg_wavelength)
        easy_pickle.dump(command_line.options.stddev_path, d)

    return 0
Ejemplo n.º 41
0
    def event(self, evt, env):
        """The event() function is called for every L1Accept transition.
    XXX more?

    Previously, common-mode correction was applied only after initial
    threshold filtering.  Since the common_mode class applies the
    (lengthy) common-mode correction immediately after reading the
    image from the stream, this optimisation is currently not
    (elegantly) doable.

    @param evt Event data object, a configure object
    @param env Environment object
    """

        super(mod_hitfind, self).event(evt, env)
        if (evt.get("skip_event")):
            return

        # This module only applies to detectors for which a distance is
        # available.
        distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
        if distance is None:
            self.nfail += 1
            self.logger.warning("event(): no distance, shot skipped")
            evt.put(skip_event_flag(), "skip_event")
            return

        device = cspad_tbx.address_split(self.address)[2]

        # ***** HITFINDING ***** XXX For hitfinding it may be interesting
        # to look at the fraction of subzero pixels in the dark-corrected
        # image.
        if (self.m_threshold is not None):
            # If a threshold value is given it can be applied in one of three ways:
            #    1.  Apply it over the whole image
            if (self.m_roi is None and self.m_distl_min_peaks is None):
                vmax = flex.max(self.cspad_img)
                if (vmax < self.m_threshold):
                    if not self.m_negate_hits:
                        # Tell downstream modules to skip this event if the threshold was not met.
                        evt.put(skip_event_flag(), "skip_event")
                        return
                elif self.m_negate_hits:
                    evt.put(skip_event_flag(), "skip_event")
                    return

            #    2. Apply threshold over a rectangular region of interest.
            elif (self.m_roi is not None):
                vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
                                               self.m_roi[0]:self.m_roi[1]])
                if (vmax < self.m_threshold):
                    if not self.m_negate_hits:
                        evt.put(skip_event_flag(), "skip_event")
                        return
                elif self.m_negate_hits:
                    evt.put(skip_event_flag(), "skip_event")
                    return

            #    3. Determine the spotfinder spots within the central ASICS, and accept the
            #       image as a hit if there are m_distl_min_peaks exceeding m_threshold.
            #       As a further requirement, the peaks must exceed 2.5 * the 90-percentile
            #       pixel value of the central ASICS.  This filter was added to avoid high-background
            #       false positives.
            elif (self.m_distl_min_peaks is not None):
                if device == 'marccd':
                    self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
                    self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
                elif device == 'Rayonix':
                    self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
                    self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]

                peak_heights, outvalue = self.distl_filter(
                    self.address,
                    self.cspad_img.iround(),  # XXX correct?
                    distance,
                    self.timestamp,
                    self.wavelength)
                if ('permissive' in self.m_distl_flags):
                    number_of_accepted_peaks = (peak_heights >
                                                self.m_threshold).count(True)
                else:
                    number_of_accepted_peaks = ((
                        peak_heights > self.m_threshold).__and__(
                            outvalue == 0)).count(True)

                sec, ms = cspad_tbx.evt_time(evt)
                evt_time = sec + ms / 1000
                self.stats_logger.info("BRAGG %.3f %d" %
                                       (evt_time, number_of_accepted_peaks))

                skip_event = False
                if number_of_accepted_peaks < self.m_distl_min_peaks:
                    self.logger.info(
                        "Subprocess %02d: Spotfinder NO  HIT image #%05d @ %s; %d spots > %d"
                        % (env.subprocess(), self.nshots, self.timestamp,
                           number_of_accepted_peaks, self.m_threshold))

                    if not self.m_negate_hits:
                        skip_event = True
                else:
                    self.logger.info(
                        "Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d"
                        % (env.subprocess(), self.nshots, self.timestamp,
                           number_of_accepted_peaks, self.m_threshold))

                    if self.m_negate_hits:
                        skip_event = True

                if skip_event:
                    if self.m_db_logging:
                        # log misses to the database
                        self.queue_entry(
                            (self.trial, evt.run(), "%.3f" % evt_time,
                             number_of_accepted_peaks, distance, self.sifoil,
                             self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                             0, 0, self.m_db_tags))
                    evt.put(skip_event_flag(), "skip_event")
                    return
                # the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
                # hit will not be logged in the db.
                evt.put(number_of_accepted_peaks, 'sfspots')

        self.logger.info("Subprocess %02d: process image #%05d @ %s" %
                         (env.subprocess(), self.nshots, self.timestamp))

        # See r17537 of mod_average.py.
        if device == 'Cspad':
            pixel_size = cspad_tbx.pixel_size
            saturated_value = cspad_tbx.cspad_saturated_value
        elif device == 'marccd':
            pixel_size = evt.get("marccd_pixel_size")
            saturated_value = evt.get("marccd_saturated_value")
        elif device == 'Rayonix':
            pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
            saturated_value = rayonix_tbx.rayonix_saturated_value

        d = cspad_tbx.dpack(
            active_areas=self.active_areas,
            address=self.address,
            beam_center_x=pixel_size * self.beam_center[0],
            beam_center_y=pixel_size * self.beam_center[1],
            data=self.cspad_img.iround(),  # XXX ouch!
            distance=distance,
            pixel_size=pixel_size,
            saturated_value=saturated_value,
            timestamp=self.timestamp,
            wavelength=self.wavelength,
            xtal_target=self.m_xtal_target)

        if (self.m_dispatch == "index"):
            import sys
            from xfel.cxi.integrate_image_api import integrate_one_image
            info = integrate_one_image(
                d,
                integration_dirname=self.m_integration_dirname,
                integration_basename=self.m_integration_basename)
            sys.stdout = sys.__stdout__
            sys.stderr = sys.__stderr__

            indexed = info is not None and hasattr(info, 'spotfinder_results')
            if self.m_progress_logging:
                if self.m_db_version == 'v1':
                    if indexed:
                        # integration pickle dictionary is available here as info.last_saved_best
                        if info.last_saved_best[
                                "identified_isoform"] is not None:
                            #print info.last_saved_best.keys()
                            from cxi_xdr_xes.cftbx.cspad_ana import db
                            dbobj = db.dbconnect(self.m_db_host,
                                                 self.m_db_name,
                                                 self.m_db_user,
                                                 self.m_db_password)
                            cursor = dbobj.cursor()
                            if info.last_saved_best[
                                    "identified_isoform"] in self.isoforms:
                                PM, indices, miller_id = self.isoforms[
                                    info.last_saved_best["identified_isoform"]]
                            else:
                                from xfel.xpp.progress_support import progress_manager
                                PM = progress_manager(info.last_saved_best,
                                                      self.m_db_experiment_tag,
                                                      self.m_trial_id,
                                                      self.m_rungroup_id,
                                                      evt.run())
                                indices, miller_id = PM.get_HKL(cursor)
                                # cache these as they don't change for a given isoform
                                self.isoforms[info.last_saved_best[
                                    "identified_isoform"]] = PM, indices, miller_id
                            if self.m_sql_buffer_size > 1:
                                self.queue_progress_entry(
                                    PM.scale_frame_detail(self.timestamp,
                                                          cursor,
                                                          do_inserts=False))
                            else:
                                PM.scale_frame_detail(self.timestamp,
                                                      cursor,
                                                      do_inserts=True)
                                dbobj.commit()
                                cursor.close()
                                dbobj.close()
                elif self.m_db_version == 'v2':
                    key_low = 'cctbx.xfel.radial_average.two_theta_low'
                    key_high = 'cctbx.xfel.radial_average.two_theta_high'
                    tt_low = evt.get(key_low)
                    tt_high = evt.get(key_high)

                    from xfel.ui.db.dxtbx_db import log_frame
                    if indexed:
                        n_spots = len(info.spotfinder_results.images[
                            info.frames[0]]['spots_total'])
                    else:
                        sfspots = evt.get('sfspots')
                        if sfspots is None:
                            if info is None or not isinstance(info, int):
                                n_spots = 0
                            else:
                                n_spots = info
                        else:
                            n_spots = sfspots

                    if indexed:
                        known_setting = info.horizons_phil.known_setting
                        indexed_setting = info.organizer.info[
                            'best_integration']['counter']
                        if known_setting is None or known_setting == indexed_setting:
                            from xfel.command_line.frame_unpickler import construct_reflection_table_and_experiment_list
                            c = construct_reflection_table_and_experiment_list(
                                info.last_saved_best,
                                None,
                                pixel_size,
                                proceed_without_image=True)
                            c.assemble_experiments()
                            c.assemble_reflections()
                            log_frame(c.experiment_list, c.reflections,
                                      self.db_params, evt.run(), n_spots,
                                      self.timestamp, tt_low, tt_high)
                        else:
                            print(
                                "Not logging %s, wrong bravais setting (expecting %d, got %d)"
                                % (self.timestamp, known_setting,
                                   indexed_setting))
                    else:
                        log_frame(None, None, self.db_params, evt.run(),
                                  n_spots, self.timestamp, tt_low, tt_high)

            if self.m_db_logging:
                sec, ms = cspad_tbx.evt_time(evt)
                evt_time = sec + ms / 1000
                sfspots = evt.get('sfspots')
                if sfspots is None:
                    if indexed:
                        n_spots = len(info.spotfinder_results.images[
                            info.frames[0]]['spots_total'])
                    else:
                        n_spots = 0
                else:
                    n_spots = sfspots

                if indexed:
                    mosaic_bloc_rotation = info.last_saved_best.get(
                        'ML_half_mosaicity_deg', [0])[0]
                    mosaic_block_size = info.last_saved_best.get(
                        'ML_domain_size_ang', [0])[0]
                    ewald_proximal_volume = info.last_saved_best.get(
                        'ewald_proximal_volume', [0])[0]

                    obs = info.last_saved_best['observations'][0]
                    cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell(
                    ).parameters()
                    pointgroup = info.last_saved_best['pointgroup']
                    resolution = obs.d_min()
                else:
                    mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
                      cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0

                self.queue_entry(
                    (self.trial, evt.run(), "%.3f" % evt_time, n_spots,
                     distance, self.sifoil, self.wavelength, indexed,
                     mosaic_bloc_rotation, mosaic_block_size,
                     ewald_proximal_volume, pointgroup, cell_a, cell_b, cell_c,
                     cell_alpha, cell_beta, cell_gamma, resolution,
                     self.m_db_tags))

            if (not indexed):
                evt.put(skip_event_flag(), "skip_event")
                return

        elif (self.m_dispatch == "nop"):
            pass

        elif (self.m_dispatch == "view"):  #interactive image viewer

            args = ["indexing.data=dummy"]
            detector_format_version = detector_format_function(
                self.address, evt.GetTime())
            if detector_format_version is not None:
                args += [
                    "distl.detector_format_version=%" % detector_format_version
                ]

            from xfel.phil_preferences import load_cxi_phil
            horizons_phil = load_cxi_phil(self.m_xtal_target, args)
            horizons_phil.indexing.data = d

            from xfel.cxi import display_spots
            display_spots.parameters.horizons_phil = horizons_phil
            display_spots.wrapper_of_callback().display(
                horizons_phil.indexing.data)

        elif (self.m_dispatch == "spots"):  #interactive spotfinder viewer

            args = ["indexing.data=dummy"]
            detector_format_version = detector_format_function(
                self.address, evt.GetTime())
            if detector_format_version is not None:
                args += [
                    "distl.detector_format_version=%s" %
                    detector_format_version
                ]

            from xfel.phil_preferences import load_cxi_phil
            horizons_phil = load_cxi_phil(self.m_xtal_target, args)
            horizons_phil.indexing.data = d

            from xfel.cxi import display_spots
            display_spots.parameters.horizons_phil = horizons_phil

            from rstbx.new_horizons.index import pre_indexing_validation, pack_names
            pre_indexing_validation(horizons_phil)
            imagefile_arguments = pack_names(horizons_phil)
            horizons_phil.persist.show()
            from spotfinder.applications import signal_strength
            info = signal_strength.run_signal_strength_core(
                horizons_phil, imagefile_arguments)

            work = display_spots.wrapper_of_callback(info)
            work.display_with_callback(horizons_phil.indexing.data)

        elif (self.m_dispatch == "write_dict"):
            self.logger.warning(
                "event(): deprecated dispatch 'write_dict', use mod_dump instead"
            )
            if (self.m_out_dirname is not None
                    or self.m_out_basename is not None):
                cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)

        # Diagnostic message emitted only when all the processing is done.
        if (env.subprocess() >= 0):
            self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
                             (env.subprocess(), self.nshots, self.timestamp))
        else:
            self.logger.info("Accepted #%05d @ %s" %
                             (self.nshots, self.timestamp))
Ejemplo n.º 42
0
def average(argv=None):
    if argv == None:
        argv = sys.argv[1:]

    try:
        from mpi4py import MPI
    except ImportError:
        raise Sorry("MPI not found")

    command_line = (libtbx.option_parser.option_parser(usage="""
%s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] [-g gain_mask_value] [--min] [--minpath minpath]

To write image pickles use -p, otherwise the program writes CSPAD CBFs.
Writing CBFs requires the geometry to be already deployed.

Examples:
cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571

Use one process on the current node to process all the events from run 25 of
experiment cxi49812, using a detz_offset of 571.

mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571

As above, using 16 cores on the current node.

bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812

As above, using the psanaq and 100 cores, putting the log in average.out and
the output images in the folder cxi49812.
""" % libtbx.env.dispatcher_name).option(
        None,
        "--as_pickle",
        "-p",
        action="store_true",
        default=False,
        dest="as_pickle",
        help="Write results as image pickle files instead of cbf files"
    ).option(
        None,
        "--raw_data",
        "-R",
        action="store_true",
        default=False,
        dest="raw_data",
        help=
        "Disable psana corrections such as dark pedestal subtraction or common mode (cbf only)"
    ).option(
        None,
        "--background_pickle",
        "-B",
        default=None,
        dest="background_pickle",
        help=""
    ).option(
        None,
        "--config",
        "-c",
        type="string",
        default=None,
        dest="config",
        metavar="PATH",
        help="psana config file"
    ).option(
        None,
        "--experiment",
        "-x",
        type="string",
        default=None,
        dest="experiment",
        help="experiment name (eg cxi84914)"
    ).option(
        None,
        "--run",
        "-r",
        type="int",
        default=None,
        dest="run",
        help="run number"
    ).option(
        None,
        "--address",
        "-a",
        type="string",
        default="CxiDs2.0:Cspad.0",
        dest="address",
        help="detector address name (eg CxiDs2.0:Cspad.0)"
    ).option(
        None,
        "--detz_offset",
        "-d",
        type="float",
        default=None,
        dest="detz_offset",
        help=
        "offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)"
    ).option(
        None,
        "--outputdir",
        "-o",
        type="string",
        default=".",
        dest="outputdir",
        metavar="PATH",
        help="Optional path to output directory for output files"
    ).option(
        None,
        "--averagebase",
        "-A",
        type="string",
        default="{experiment!l}_avg-r{run:04d}",
        dest="averagepath",
        metavar="PATH",
        help=
        "Path to output average image without extension. String substitution allowed"
    ).option(
        None,
        "--stddevbase",
        "-S",
        type="string",
        default="{experiment!l}_stddev-r{run:04d}",
        dest="stddevpath",
        metavar="PATH",
        help=
        "Path to output standard deviation image without extension. String substitution allowed"
    ).option(
        None,
        "--maxbase",
        "-M",
        type="string",
        default="{experiment!l}_max-r{run:04d}",
        dest="maxpath",
        metavar="PATH",
        help=
        "Path to output maximum projection image without extension. String substitution allowed"
    ).option(
        None,
        "--numevents",
        "-n",
        type="int",
        default=None,
        dest="numevents",
        help="Maximum number of events to process. Default: all"
    ).option(
        None,
        "--skipevents",
        "-s",
        type="int",
        default=0,
        dest="skipevents",
        help="Number of events in the beginning of the run to skip. Default: 0"
    ).option(
        None,
        "--verbose",
        "-v",
        action="store_true",
        default=False,
        dest="verbose",
        help="Print more information about progress"
    ).option(
        None,
        "--pickle-optical-metrology",
        "-m",
        action="store_true",
        default=False,
        dest="pickle_optical_metrology",
        help=
        "If writing pickle files, use the optical metrology in the experiment's calib directory"
    ).option(
        None,
        "--bin_size",
        "-b",
        type="int",
        default=None,
        dest="bin_size",
        help="Rayonix detector bin size"
    ).option(
        None,
        "--override_beam_x",
        "-X",
        type="float",
        default=None,
        dest="override_beam_x",
        help="Rayonix detector beam center x coordinate"
    ).option(
        None,
        "--override_beam_y",
        "-Y",
        type="float",
        default=None,
        dest="override_beam_y",
        help="Rayonix detector beam center y coordinate"
    ).option(
        None,
        "--calib_dir",
        "-C",
        type="string",
        default=None,
        dest="calib_dir",
        metavar="PATH",
        help="calibration directory"
    ).option(
        None,
        "--pickle_calib_dir",
        "-P",
        type="string",
        default=None,
        dest="pickle_calib_dir",
        metavar="PATH",
        help=
        "pickle calibration directory specification. Replaces --calib_dir functionality."
    ).option(
        None,
        "--xtc_dir",
        "-D",
        type="string",
        default=None,
        dest="xtc_dir",
        metavar="PATH",
        help="xtc stream directory"
    ).option(
        None,
        "--use_ffb",
        "-f",
        action="store_true",
        default=False,
        dest="use_ffb",
        help=
        "Use the fast feedback filesystem at LCLS. Only for the active experiment!"
    ).option(
        None,
        "--gain_mask_value",
        "-g",
        type="float",
        default=None,
        dest="gain_mask_value",
        help=
        "Ratio between low and high gain pixels, if CSPAD in mixed-gain mode. Only used in CBF averaging mode."
    ).option(
        None,
        "--min",
        None,
        action="store_true",
        default=False,
        dest="do_minimum_projection",
        help="Output a minimum projection"
    ).option(
        None,
        "--minpath",
        None,
        type="string",
        default="{experiment!l}_min-r{run:04d}",
        dest="minpath",
        metavar="PATH",
        help=
        "Path to output minimum image without extension. String substitution allowed"
    )).process(args=argv)


    if len(command_line.args) > 0 or \
        command_line.options.as_pickle is None or \
        command_line.options.experiment is None or \
        command_line.options.run is None or \
        command_line.options.address is None or \
        command_line.options.detz_offset is None or \
        command_line.options.averagepath is None or \
        command_line.options.stddevpath is None or \
        command_line.options.maxpath is None or \
        command_line.options.pickle_optical_metrology is None:
        command_line.parser.show_help()
        return

    # set this to sys.maxint to analyze all events
    if command_line.options.numevents is None:
        maxevents = sys.maxsize
    else:
        maxevents = command_line.options.numevents

    comm = MPI.COMM_WORLD
    rank = comm.Get_rank()
    size = comm.Get_size()

    if command_line.options.config is not None:
        psana.setConfigFile(command_line.options.config)
    dataset_name = "exp=%s:run=%d:smd" % (command_line.options.experiment,
                                          command_line.options.run)
    if command_line.options.xtc_dir is not None:
        if command_line.options.use_ffb:
            raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system")
        dataset_name += ":dir=%s" % command_line.options.xtc_dir
    elif command_line.options.use_ffb:
        # as ffb is only at SLAC, ok to hardcode /reg/d here
        dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc" % (
            command_line.options.experiment[0:3],
            command_line.options.experiment)
    if command_line.options.calib_dir is not None:
        psana.setOption('psana.calib-dir', command_line.options.calib_dir)
    ds = psana.DataSource(dataset_name)
    address = command_line.options.address
    src = psana.Source('DetInfo(%s)' % address)
    nevent = np.array([0.])

    if command_line.options.background_pickle is not None:
        background = easy_pickle.load(
            command_line.options.background_pickle)['DATA'].as_numpy_array()

    for run in ds.runs():
        runnumber = run.run()

        if not command_line.options.as_pickle:
            psana_det = psana.Detector(address, ds.env())

        # list of all events
        if command_line.options.skipevents > 0:
            print("Skipping first %d events" % command_line.options.skipevents)
        elif "Rayonix" in command_line.options.address:
            print("Skipping first image in the Rayonix detector"
                  )  # Shuttering issue
            command_line.options.skipevents = 1

        for i, evt in enumerate(run.events()):
            if i % size != rank: continue
            if i < command_line.options.skipevents: continue
            if i >= maxevents: break
            if i % 10 == 0: print('Rank', rank, 'processing event', i)
            #print "Event #",rank*mylength+i," has id:",evt.get(EventId)
            if 'Rayonix' in command_line.options.address or 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
                data = evt.get(psana.Camera.FrameV1, src)
                if data is None:
                    print("No data")
                    continue
                data = data.data16().astype(np.float64)
            elif command_line.options.as_pickle:
                data = evt.get(psana.ndarray_float64_3, src, 'image0')
            else:
                # get numpy array, 32x185x388
                from xfel.cftbx.detector.cspad_cbf_tbx import get_psana_corrected_data
                if command_line.options.raw_data:
                    data = get_psana_corrected_data(psana_det,
                                                    evt,
                                                    use_default=False,
                                                    dark=False,
                                                    common_mode=None,
                                                    apply_gain_mask=False,
                                                    per_pixel_gain=False)
                else:
                    if command_line.options.gain_mask_value is None:
                        data = get_psana_corrected_data(psana_det,
                                                        evt,
                                                        use_default=True)
                    else:
                        data = get_psana_corrected_data(
                            psana_det,
                            evt,
                            use_default=False,
                            dark=True,
                            common_mode=None,
                            apply_gain_mask=True,
                            gain_mask_value=command_line.options.
                            gain_mask_value,
                            per_pixel_gain=False)

            if data is None:
                print("No data")
                continue

            if command_line.options.background_pickle is not None:
                data -= background

            if 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
                distance = np.array([0.0])
                wavelength = np.array([1.0])
            else:
                d = cspad_tbx.env_distance(address, run.env(),
                                           command_line.options.detz_offset)
                if d is None:
                    print("No distance, using distance",
                          command_line.options.detz_offset)
                    assert command_line.options.detz_offset is not None
                    if 'distance' not in locals():
                        distance = np.array([command_line.options.detz_offset])
                    else:
                        distance += command_line.options.detz_offset
                else:
                    if 'distance' in locals():
                        distance += d
                    else:
                        distance = np.array([float(d)])

                w = cspad_tbx.evt_wavelength(evt)
                if w is None:
                    print("No wavelength")
                    if 'wavelength' not in locals():
                        wavelength = np.array([1.0])
                else:
                    if 'wavelength' in locals():
                        wavelength += w
                    else:
                        wavelength = np.array([w])

            t = cspad_tbx.evt_time(evt)
            if t is None:
                print("No timestamp, skipping shot")
                continue
            if 'timestamp' in locals():
                timestamp += t[0] + (t[1] / 1000)
            else:
                timestamp = np.array([t[0] + (t[1] / 1000)])

            if 'sum' in locals():
                sum += data
            else:
                sum = np.array(data, copy=True)
            if 'sumsq' in locals():
                sumsq += data * data
            else:
                sumsq = data * data
            if 'maximum' in locals():
                maximum = np.maximum(maximum, data)
            else:
                maximum = np.array(data, copy=True)

            if command_line.options.do_minimum_projection:
                if 'minimum' in locals():
                    minimum = np.minimum(minimum, data)
                else:
                    minimum = np.array(data, copy=True)

            nevent += 1

    #sum the images across mpi cores
    if size > 1:
        print("Synchronizing rank", rank)
    totevent = np.zeros(nevent.shape)
    comm.Reduce(nevent, totevent)

    if rank == 0 and totevent[0] == 0:
        raise Sorry("No events found in the run")

    sumall = np.zeros(sum.shape).astype(sum.dtype)
    comm.Reduce(sum, sumall)

    sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype)
    comm.Reduce(sumsq, sumsqall)

    maxall = np.zeros(maximum.shape).astype(maximum.dtype)
    comm.Reduce(maximum, maxall, op=MPI.MAX)

    if command_line.options.do_minimum_projection:
        minall = np.zeros(maximum.shape).astype(minimum.dtype)
        comm.Reduce(minimum, minall, op=MPI.MIN)

    waveall = np.zeros(wavelength.shape).astype(wavelength.dtype)
    comm.Reduce(wavelength, waveall)

    distall = np.zeros(distance.shape).astype(distance.dtype)
    comm.Reduce(distance, distall)

    timeall = np.zeros(timestamp.shape).astype(timestamp.dtype)
    comm.Reduce(timestamp, timeall)

    if rank == 0:
        if size > 1:
            print("Synchronized")

        # Accumulating floating-point numbers introduces errors,
        # which may cause negative variances.  Since a two-pass
        # approach is unacceptable, the standard deviation is
        # clamped at zero.
        mean = sumall / float(totevent[0])
        variance = (sumsqall / float(totevent[0])) - (mean**2)
        variance[variance < 0] = 0
        stddev = np.sqrt(variance)

        wavelength = waveall[0] / totevent[0]
        distance = distall[0] / totevent[0]
        pixel_size = cspad_tbx.pixel_size
        saturated_value = cspad_tbx.cspad_saturated_value
        timestamp = timeall[0] / totevent[0]
        timestamp = (int(timestamp), timestamp % int(timestamp) * 1000)
        timestamp = cspad_tbx.evt_timestamp(timestamp)

        if command_line.options.as_pickle:
            extension = ".pickle"
        else:
            extension = ".cbf"

        dest_paths = [
            cspad_tbx.pathsubst(command_line.options.averagepath + extension,
                                evt, ds.env()),
            cspad_tbx.pathsubst(command_line.options.stddevpath + extension,
                                evt, ds.env()),
            cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt,
                                ds.env())
        ]
        if command_line.options.do_minimum_projection:
            dest_paths.append(
                cspad_tbx.pathsubst(command_line.options.minpath + extension,
                                    evt, ds.env()))

        dest_paths = [
            os.path.join(command_line.options.outputdir, path)
            for path in dest_paths
        ]
        if 'Rayonix' in command_line.options.address:
            all_data = [mean, stddev, maxall]
            if command_line.options.do_minimum_projection:
                all_data.append(minall)
            from xfel.cxi.cspad_ana import rayonix_tbx
            pixel_size = rayonix_tbx.get_rayonix_pixel_size(
                command_line.options.bin_size)
            beam_center = [
                command_line.options.override_beam_x,
                command_line.options.override_beam_y
            ]
            active_areas = flex.int([0, 0, mean.shape[1], mean.shape[0]])
            split_address = cspad_tbx.address_split(address)
            old_style_address = split_address[0] + "-" + split_address[
                1] + "|" + split_address[2] + "-" + split_address[3]
            for data, path in zip(all_data, dest_paths):
                print("Saving", path)
                d = cspad_tbx.dpack(
                    active_areas=active_areas,
                    address=old_style_address,
                    beam_center_x=pixel_size * beam_center[0],
                    beam_center_y=pixel_size * beam_center[1],
                    data=flex.double(data),
                    distance=distance,
                    pixel_size=pixel_size,
                    saturated_value=rayonix_tbx.rayonix_saturated_value,
                    timestamp=timestamp,
                    wavelength=wavelength)
                easy_pickle.dump(path, d)
        elif 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
            all_data = [mean, stddev, maxall]
            split_address = cspad_tbx.address_split(address)
            old_style_address = split_address[0] + "-" + split_address[
                1] + "|" + split_address[2] + "-" + split_address[3]
            if command_line.options.do_minimum_projection:
                all_data.append(minall)
            for data, path in zip(all_data, dest_paths):
                d = cspad_tbx.dpack(address=old_style_address,
                                    data=flex.double(data),
                                    distance=distance,
                                    pixel_size=0.1,
                                    timestamp=timestamp,
                                    wavelength=wavelength)
                print("Saving", path)
                easy_pickle.dump(path, d)
        elif command_line.options.as_pickle:
            split_address = cspad_tbx.address_split(address)
            old_style_address = split_address[0] + "-" + split_address[
                1] + "|" + split_address[2] + "-" + split_address[3]

            xpp = 'xpp' in address.lower()
            if xpp:
                evt_time = cspad_tbx.evt_time(
                    evt)  # tuple of seconds, milliseconds
                timestamp = cspad_tbx.evt_timestamp(
                    evt_time)  # human readable format
                from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
                from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
                version_lookup = detector_format_version(
                    old_style_address,
                    reverse_timestamp(timestamp)[0])
                assert version_lookup is not None
                active_areas = xpp_active_areas[version_lookup]['active_areas']
                beam_center = [1765 // 2, 1765 // 2]
            else:
                if command_line.options.pickle_calib_dir is not None:
                    metro_path = command_line.options.pickle_calib_dir
                elif command_line.options.pickle_optical_metrology:
                    from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path
                    metro_path = get_calib_file_path(run.env(), address, run)
                else:
                    metro_path = libtbx.env.find_in_repositories(
                        "xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
                sections = parse_calib.calib2sections(metro_path)
                beam_center, active_areas = cspad_tbx.cbcaa(
                    cspad_tbx.getConfig(address, ds.env()), sections)

            class fake_quad(object):
                def __init__(self, q, d):
                    self.q = q
                    self.d = d

                def quad(self):
                    return self.q

                def data(self):
                    return self.d

            if xpp:
                quads = [
                    fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
                    for i in range(4)
                ]
                mean = cspad_tbx.image_xpp(old_style_address,
                                           None,
                                           ds.env(),
                                           active_areas,
                                           quads=quads)
                mean = flex.double(mean.astype(np.float64))

                quads = [
                    fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
                    for i in range(4)
                ]
                stddev = cspad_tbx.image_xpp(old_style_address,
                                             None,
                                             ds.env(),
                                             active_areas,
                                             quads=quads)
                stddev = flex.double(stddev.astype(np.float64))

                quads = [
                    fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
                    for i in range(4)
                ]
                maxall = cspad_tbx.image_xpp(old_style_address,
                                             None,
                                             ds.env(),
                                             active_areas,
                                             quads=quads)
                maxall = flex.double(maxall.astype(np.float64))

                if command_line.options.do_minimum_projection:
                    quads = [
                        fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
                        for i in range(4)
                    ]
                    minall = cspad_tbx.image_xpp(old_style_address,
                                                 None,
                                                 ds.env(),
                                                 active_areas,
                                                 quads=quads)
                    minall = flex.double(minall.astype(np.float64))
            else:
                quads = [
                    fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
                    for i in range(4)
                ]
                mean = cspad_tbx.CsPadDetector(address,
                                               evt,
                                               ds.env(),
                                               sections,
                                               quads=quads)
                mean = flex.double(mean.astype(np.float64))

                quads = [
                    fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
                    for i in range(4)
                ]
                stddev = cspad_tbx.CsPadDetector(address,
                                                 evt,
                                                 ds.env(),
                                                 sections,
                                                 quads=quads)
                stddev = flex.double(stddev.astype(np.float64))

                quads = [
                    fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
                    for i in range(4)
                ]
                maxall = cspad_tbx.CsPadDetector(address,
                                                 evt,
                                                 ds.env(),
                                                 sections,
                                                 quads=quads)
                maxall = flex.double(maxall.astype(np.float64))

                if command_line.options.do_minimum_projection:
                    quads = [
                        fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
                        for i in range(4)
                    ]
                    minall = cspad_tbx.CsPadDetector(address,
                                                     evt,
                                                     ds.env(),
                                                     sections,
                                                     quads=quads)
                    minall = flex.double(minall.astype(np.float64))

            all_data = [mean, stddev, maxall]
            if command_line.options.do_minimum_projection:
                all_data.append(minall)

            for data, path in zip(all_data, dest_paths):
                print("Saving", path)

                d = cspad_tbx.dpack(active_areas=active_areas,
                                    address=old_style_address,
                                    beam_center_x=pixel_size * beam_center[0],
                                    beam_center_y=pixel_size * beam_center[1],
                                    data=data,
                                    distance=distance,
                                    pixel_size=pixel_size,
                                    saturated_value=saturated_value,
                                    timestamp=timestamp,
                                    wavelength=wavelength)

                easy_pickle.dump(path, d)
        else:
            # load a header only cspad cbf from the slac metrology
            from xfel.cftbx.detector import cspad_cbf_tbx
            import pycbf
            base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(
                run, address)
            if base_dxtbx is None:
                raise Sorry("Couldn't load calibration file for run %d" %
                            run.run())

            all_data = [mean, stddev, maxall]
            if command_line.options.do_minimum_projection:
                all_data.append(minall)

            for data, path in zip(all_data, dest_paths):
                print("Saving", path)
                cspad_img = cspad_cbf_tbx.format_object_from_data(
                    base_dxtbx,
                    data,
                    distance,
                    wavelength,
                    timestamp,
                    address,
                    round_to_int=False)
                cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
                  pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
Ejemplo n.º 43
0
    def __init__(self,
                 pixel_histograms,
                 output_dirname=".",
                 gain_map_path=None,
                 gain_map=None,
                 method="photon_counting",
                 estimated_gain=30,
                 nproc=None,
                 photon_threshold=2 / 3,
                 roi=None,
                 run=None):
        assert method in ("sum_adu", "photon_counting")
        self.sum_img = flex.double(flex.grid(
            370, 391), 0)  # XXX define the image size some other way?
        gain_img = flex.double(self.sum_img.accessor(), 0)

        assert [gain_map, gain_map_path].count(None) > 0
        if gain_map_path is not None:
            d = easy_pickle.load(gain_map_path)
            gain_map = d["DATA"]

        two_photon_threshold = photon_threshold + 1

        mask = flex.int(self.sum_img.accessor(), 0)

        start_row = 370
        end_row = 0
        print(len(pixel_histograms.histograms))

        pixels = list(pixel_histograms.pixels())
        n_pixels = len(pixels)
        if roi is not None:
            for k, (i, j) in enumerate(reversed(pixels)):
                if (i < roi[2] or i > roi[3] or j < roi[0] or j > roi[1]):
                    del pixels[n_pixels - k - 1]

        if gain_map is None:
            fixed_func = pixel_histograms.fit_one_histogram
        else:

            def fixed_func(pixel):
                return pixel_histograms.fit_one_histogram(pixel, n_gaussians=1)

        results = None
        if nproc is None: nproc = easy_mp.Auto
        nproc = easy_mp.get_processes(nproc)
        print("nproc: ", nproc)

        stdout_and_results = easy_mp.pool_map(
            processes=nproc,
            fixed_func=fixed_func,
            args=pixels,
            func_wrapper="buffer_stdout_stderr")
        results = [r for so, r in stdout_and_results]

        gains = flex.double()

        for i, pixel in enumerate(pixels):
            start_row = min(start_row, pixel[0])
            end_row = max(end_row, pixel[0])
            n_photons = 0
            if results is None:
                # i.e. not multiprocessing
                try:
                    gaussians = pixel_histograms.fit_one_histogram(pixel)
                except RuntimeError as e:
                    print("Error fitting pixel %s" % str(pixel))
                    print(str(e))
                    mask[pixel] = 1
                    continue
            else:
                gaussians = results[i]
            hist = pixel_histograms.histograms[pixel]
            if gaussians is None:
                # Presumably the peak fitting failed in some way
                print("Skipping pixel %s" % str(pixel))
                continue
            zero_peak_diff = gaussians[0].params[1]
            if gain_map is None:
                try:
                    view_pixel_histograms.check_pixel_histogram_fit(
                        hist, gaussians)
                except view_pixel_histograms.PixelFitError as e:
                    print("PixelFitError:", str(pixel), str(e))
                    mask[pixel] = 1
                    continue
                gain = gaussians[1].params[1] - gaussians[0].params[1]
                gain_img[pixel] = gain
                gain_ratio = gain / estimated_gain
            else:
                gain = gain_map[pixel]
                if gain == 0:
                    print("bad gain!!!!!", pixel)
                    continue
                gain = 30 / gain
                gain_ratio = 1 / gain
            gains.append(gain)

            #for g in gaussians:
            #sigma = abs(g.params[2])
            #if sigma < 1 or sigma > 10:
            #print "bad sigma!!!!!", pixel, sigma
            #mask[pixel] = 1
            #continue
            if method == "sum_adu":
                sum_adu = 0
                one_photon_cutoff, two_photon_cutoff = [
                    (threshold * gain + zero_peak_diff)
                    for threshold in (photon_threshold, two_photon_threshold)
                ]
                i_one_photon_cutoff = hist.get_i_slot(one_photon_cutoff)
                slots = hist.slots().as_double()
                slot_centers = hist.slot_centers()
                slots -= gaussians[0](slot_centers)
                for j in range(i_one_photon_cutoff, len(slots)):
                    center = slot_centers[j]
                    sum_adu += slots[j] * (center - zero_peak_diff) * 30 / gain

                self.sum_img[pixel] = sum_adu
            elif method == "photon_counting":
                one_photon_cutoff, two_photon_cutoff = [
                    (threshold * gain + zero_peak_diff)
                    for threshold in (photon_threshold, two_photon_threshold)
                ]
                i_one_photon_cutoff = hist.get_i_slot(one_photon_cutoff)
                i_two_photon_cutoff = hist.get_i_slot(two_photon_cutoff)
                slots = hist.slots()
                for j in range(i_one_photon_cutoff, len(slots)):
                    if j == i_one_photon_cutoff:
                        center = hist.slot_centers()[j]
                        upper = center + 0.5 * hist.slot_width()
                        n_photons += int(
                            round((upper - one_photon_cutoff) /
                                  hist.slot_width() * slots[j]))
                    elif j == i_two_photon_cutoff:
                        center = hist.slot_centers()[j]
                        upper = center + 0.5 * hist.slot_width()
                        n_photons += 2 * int(
                            round((upper - two_photon_cutoff) /
                                  hist.slot_width() * slots[j]))
                    elif j < i_two_photon_cutoff:
                        n_photons += int(round(slots[j]))
                    else:
                        n_photons += 2 * int(round(slots[j]))
                self.sum_img[pixel] = n_photons

        stats = scitbx.math.basic_statistics(gains)
        print("gain statistics:")
        stats.show()

        mask.set_selected(self.sum_img == 0, 1)
        unbound_pixel_mask = xes_finalise.cspad_unbound_pixel_mask()
        mask.set_selected(unbound_pixel_mask > 0, 1)
        bad_pixel_mask = xes_finalise.cspad2x2_bad_pixel_mask_cxi_run7()
        mask.set_selected(bad_pixel_mask > 0, 1)

        for row in range(self.sum_img.all()[0]):
            self.sum_img[row:row + 1, :].count(0)

        spectrum_focus = self.sum_img[start_row:end_row, :]
        mask_focus = mask[start_row:end_row, :]

        spectrum_focus.set_selected(mask_focus > 0, 0)

        xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)

        print("Number of rows: %i" % spectrum_focus.all()[0])
        print("Estimated no. photons counted: %i" % flex.sum(spectrum_focus))
        print("Number of images used: %i" %
              flex.sum(pixel_histograms.histograms.values()[0].slots()))

        d = cspad_tbx.dpack(
            address='CxiSc1-0|Cspad2x2-0',
            data=spectrum_focus,
            distance=1,
            ccd_image_saturation=2e8,  # XXX
        )
        if run is not None: runstr = "_%04d" % run
        else: runstr = ""
        cspad_tbx.dwritef(d, output_dirname, 'sum%s_' % runstr)

        if gain_map is None:
            gain_map = flex.double(gain_img.accessor(), 0)
            img_sel = (gain_img > 0).as_1d()
            d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0',
                                data=gain_img,
                                distance=1)
            cspad_tbx.dwritef(d, output_dirname, 'raw_gain_map_')
            gain_map.as_1d().set_selected(img_sel.iselection(),
                                          1 / gain_img.as_1d().select(img_sel))
            gain_map /= flex.mean(gain_map.as_1d().select(img_sel))
            d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0',
                                data=gain_map,
                                distance=1)
            cspad_tbx.dwritef(d, output_dirname, 'gain_map_')

        plot_x, plot_y = xes_finalise.output_spectrum(
            spectrum_focus.iround(),
            mask_focus=mask_focus,
            output_dirname=output_dirname,
            run=run)
        self.spectrum = (plot_x, plot_y)
        self.spectrum_focus = spectrum_focus

        xes_finalise.output_matlab_form(
            spectrum_focus, "%s/sum%s.m" % (output_dirname, runstr))
        print(output_dirname)
Ejemplo n.º 44
0
    def __init__(self, runs, output_dirname=".", roi=None):
        avg_basename = "avg_"
        stddev_basename = "stddev"
        self.sum_img = None
        self.sumsq_img = None
        self.nmemb = 0
        self.roi = cspad_tbx.getOptROI(roi)
        self.unbound_pixel_mask = cspad_unbound_pixel_mask()
        for i_run, run in enumerate(runs):
            run_scratch_dir = run
            result = finalise_one_run(run_scratch_dir)
            if result.sum_img is None: continue
            if self.sum_img is None:
                self.sum_img = result.sum_img
                self.sumsq_img = result.sumsq_img
            else:
                self.sum_img += result.sum_img
                self.sumsq_img += result.sumsq_img
            self.nmemb += result.nmemb

        self.avg_img = self.sum_img.as_double() / self.nmemb
        self.stddev_img = flex.sqrt(
            (self.sumsq_img.as_double() -
             self.sum_img.as_double() * self.avg_img) / (self.nmemb - 1))

        self.mask = flex.int(self.sum_img.accessor(), 0)
        self.mask.set_selected(self.sum_img == 0, 1)
        self.mask.set_selected(self.unbound_pixel_mask > 0, 1)

        if (output_dirname is not None and avg_basename is not None):
            if (not os.path.isdir(output_dirname)):
                os.makedirs(output_dirname)
            d = cspad_tbx.dpack(
                address='CxiSc1-0|Cspad2x2-0',
                data=self.avg_img,
                distance=1,
            )
            cspad_tbx.dwritef(d, output_dirname, avg_basename)
            d = cspad_tbx.dpack(
                address='CxiSc1-0|Cspad2x2-0',
                data=self.sum_img,
                distance=1,
            )
            cspad_tbx.dwritef(d, output_dirname, "sum_")
            if 1:
                output_image(self.avg_img, "%s/avg.png" % output_dirname)
                output_image(self.avg_img,
                             "%s/avg_inv.png" % output_dirname,
                             invert=True)

            if 1:
                output_matlab_form(self.sum_img, "%s/sum.m" % output_dirname)
                output_matlab_form(self.avg_img, "%s/avg.m" % output_dirname)
                output_matlab_form(self.stddev_img,
                                   "%s/stddev.m" % output_dirname)

        if (stddev_basename is not None):
            d = cspad_tbx.dpack(
                address='CxiSc1-0|Cspad2x2-0',
                data=self.stddev_img,
                distance=1,
            )
            cspad_tbx.dwritef(d, output_dirname, stddev_basename)

            # XXX we should really figure out automatically the area where the spectrum is
            #write an integrated spectrum from lines 186-227
            #spectrum_focus = self.sum_img.as_numpy_array()[186:228,:]
            img = self.sum_img
            if self.roi is None:
                spectrum_focus = img
                mask_focus = self.mask
            else:
                slices = (slice(self.roi[2],
                                self.roi[3]), slice(self.roi[0], self.roi[1]))
                spectrum_focus = img[slices]
                mask_focus = self.mask[slices]
            if False:
                from matplotlib import pylab
                pylab.imshow(spectrum_focus.as_numpy_array())
                pylab.show()

        output_spectrum(spectrum_focus,
                        mask_focus=mask_focus,
                        output_dirname=output_dirname)

        print "Total number of images used from %i runs: %i" % (i_run + 1,
                                                                self.nmemb)
Ejemplo n.º 45
0
def run(args):
    command_line = (option_parser().option(
        "-o",
        "--output_filename",
        action="store",
        type="string",
        help="Filename for the output pickle file",
        default="gain_map.pickle"
    ).option(
        "-f",
        "--detector_format_version",
        action="store",
        type="string",
        help=
        "Detector format version to use for generating active areas and laying out tiles",
        default=None
    ).option(
        "-m",
        "--optical_metrology_path",
        action="store",
        type="string",
        help=
        "Path to slac optical metrology file. If not set, use Run 4 metrology",
        default=None
    ).option(
        "-d",
        "--distance",
        action="store",
        type="int",
        help=
        "Detector distance put into the gain pickle file. Not needed for processing.",
        default="0"
    ).option(
        "-w",
        "--wavelength",
        action="store",
        type="float",
        help=
        "Incident beam wavelength put into the gain pickle file. Not needed for processing.",
        default="0")).process(args=args)
    output_filename = command_line.options.output_filename
    detector_format_version = command_line.options.detector_format_version
    if detector_format_version is None or 'XPP' not in detector_format_version:
        beam_center_x = None
        beam_center_y = None
    else:
        beam_center_x = 1765 // 2 * 0.11
        beam_center_y = 1765 // 2 * 0.11
    address, timestamp = address_and_timestamp_from_detector_format_version(
        detector_format_version)

    # if no detector format version is provided, make sure to write no address to the image pickle
    # but CsPadDetector (called later), needs an address, so give it a fake one
    save_address = address is not None
    if not save_address:
        address = "CxiDs1-0|Cspad-0"  # time stamp will still be None
    timestamp = evt_timestamp((timestamp, 0))
    args = command_line.args
    assert len(args) == 1
    if args[0].endswith('.npy'):
        data = numpy.load(args[0])
        det, active_areas = convert_2x2(data, detector_format_version, address)
    elif args[0].endswith('.txt') or args[0].endswith('.gain'):
        raw_data = numpy.loadtxt(args[0])
        assert raw_data.shape in [(5920, 388), (11840, 194)]
        det, active_areas = convert_detector(
            raw_data, detector_format_version, address,
            command_line.options.optical_metrology_path)
    img_diff = det
    img_sel = (img_diff > 0).as_1d()
    gain_map = flex.double(img_diff.accessor(), 0)
    gain_map.as_1d().set_selected(img_sel.iselection(),
                                  1 / img_diff.as_1d().select(img_sel))
    gain_map /= flex.mean(gain_map.as_1d().select(img_sel))

    if not save_address:
        address = None
    d = cspad_tbx.dpack(data=gain_map,
                        address=address,
                        active_areas=active_areas,
                        timestamp=timestamp,
                        distance=command_line.options.distance,
                        wavelength=command_line.options.wavelength,
                        beam_center_x=beam_center_x,
                        beam_center_y=beam_center_y)
    easy_pickle.dump(output_filename, d)
Ejemplo n.º 46
0
class SingleImage(object):
    def __init__(self, img, init, verbose=True, imported_grid=None):
        """ Constructor for the SingleImage object using a raw image file or pickle
    """

        # Initialize parameters
        self.params = init.params
        self.args = init.args
        self.user_id = init.user_id
        self.raw_img = img[2]
        self.conv_img = img[2]
        self.img_index = img[0]
        self.center_int = 0
        self.status = None
        self.fail = None
        self.final = None
        self.log_info = []
        self.gs_results = []
        self.main_log = init.logfile
        self.verbose = verbose
        self.hmed = self.params.cctbx.grid_search.height_median
        self.amed = self.params.cctbx.grid_search.area_median

        self.input_base = init.input_base
        self.conv_base = init.conv_base
        self.int_base = init.int_base
        self.obj_base = init.obj_base
        self.fin_base = init.fin_base
        self.viz_base = init.viz_base
        self.log_base = init.log_base
        self.tmp_base = init.tmp_base
        self.abort_file = os.path.join(self.int_base, '.abort.tmp')

        self.obj_path = None
        self.obj_file = None
        self.fin_path = None
        self.fin_file = None
        self.viz_path = None

# ============================== SELECTION-ONLY FUNCTIONS ============================== #

    def import_int_file(self, init):
        """ Replaces path settings in imported image object with new settings
        NEED TO RE-DO LATER """

        if os.path.isfile(self.abort_file):
            self.fail = 'aborted'
            return self

        # Generate paths to output files
        self.params = init.params
        self.main_log = init.logfile
        self.input_base = init.input_base
        self.conv_base = init.conv_base
        self.int_base = init.int_base
        self.obj_base = init.obj_base
        self.fin_base = init.fin_base
        self.log_base = init.log_base
        self.viz_base = init.viz_base
        self.obj_path = misc.make_image_path(self.conv_img, self.input_base,
                                             self.obj_base)
        self.obj_file = os.path.abspath(
            os.path.join(
                self.obj_path,
                os.path.basename(self.conv_img).split('.')[0] + ".int"))
        self.fin_path = misc.make_image_path(self.conv_img, self.input_base,
                                             self.fin_base)
        self.log_path = misc.make_image_path(self.conv_img, self.input_base,
                                             self.log_base)
        self.fin_file = os.path.abspath(
            os.path.join(
                self.fin_path,
                os.path.basename(self.conv_img).split('.')[0] + "_int.pickle"))
        self.final['final'] = self.fin_file
        self.final['img'] = self.conv_img
        self.viz_path = misc.make_image_path(self.conv_img, self.input_base,
                                             self.viz_base)
        self.viz_file = os.path.join(
            self.viz_path,
            os.path.basename(self.conv_img).split('.')[0] + "_int.png")

        # Create actual folders (if necessary)
        try:
            if not os.path.isdir(self.obj_path):
                os.makedirs(self.obj_path)
            if not os.path.isdir(self.fin_path):
                os.makedirs(self.fin_path)
            if not os.path.isdir(self.viz_path):
                os.makedirs(self.viz_path)
        except OSError:
            pass

        # Grid search / integration log file
        self.int_log = os.path.join(
            self.log_path,
            os.path.basename(self.conv_img).split('.')[0] + '.tmp')

        # Reset status to 'grid search' to pick up at selection (if no fail)
        if self.fail == None:
            self.status = 'bypass grid search'

        return self

    def determine_gs_result_file(self):
        """ For 'selection-only' cctbx.xfel runs, determine where the image objects are """
        if self.params.cctbx.selection.select_only.grid_search_path != None:
            obj_path = os.path.abspath(
                self.params.cctbx.selection.select_only.grid_search_path)
        else:
            run_number = int(os.path.basename(self.int_base)) - 1
            obj_path = "{}/integration/{:03d}/image_objects"\
                      "".format(os.path.abspath(os.curdir), run_number)
        gs_result_file = os.path.join(obj_path,
                                      os.path.basename(self.obj_file))
        return gs_result_file

# =============================== IMAGE IMPORT FUNCTIONS =============================== #

    def load_image(self):
        """ Reads raw image file and extracts data for conversion into pickle
        format. Also estimates gain if turned on."""
        # Load raw image or image pickle

        try:
            with misc.Capturing() as junk_output:
                loaded_img = dxtbx.load(self.raw_img)
        except Exception, e:
            print 'IOTA IMPORT ERROR:', e
            loaded_img = None
            pass

        # Extract image information
        if loaded_img is not None:
            raw_data = loaded_img.get_raw_data()
            detector = loaded_img.get_detector()[0]
            beam = loaded_img.get_beam()
            scan = loaded_img.get_scan()
            distance = detector.get_distance()
            pixel_size = detector.get_pixel_size()[0]
            overload = detector.get_trusted_range()[1]
            wavelength = beam.get_wavelength()
            beam_x = detector.get_beam_centre(beam.get_s0())[0]
            beam_y = detector.get_beam_centre(beam.get_s0())[1]

            if scan is None:
                timestamp = None
                img_type = 'pickle'
            else:
                img_type = 'raw'
                msec, sec = math.modf(scan.get_epochs()[0])
                timestamp = evt_timestamp((sec, msec))

            # Assemble datapack
            data = dpack(data=raw_data,
                         distance=distance,
                         pixel_size=pixel_size,
                         wavelength=wavelength,
                         beam_center_x=beam_x,
                         beam_center_y=beam_y,
                         ccd_image_saturation=overload,
                         saturated_value=overload,
                         timestamp=timestamp)

            if scan is not None:
                osc_start, osc_range = scan.get_oscillation()
                if osc_start != osc_range:
                    data['OSC_START'] = 0  #osc_start
                    data['OSC_RANGE'] = 0  #osc_start
                    data['TIME'] = scan.get_exposure_times()[0]
        else:
            data = None
            img_type = 'not imported'

        # Estimate gain (or set gain to 1.00 if cannot calculate)
        # Cribbed from estimate_gain.py by Richard Gildea
        if self.params.advanced.estimate_gain:
            from dxtbx.datablock import DataBlockFactory
            from dials.command_line.estimate_gain import estimate_gain
            with misc.Capturing() as junk_output:
                try:
                    datablock = DataBlockFactory.from_filenames([self.raw_img
                                                                 ])[0]
                    imageset = datablock.extract_imagesets()[0]
                    self.gain = estimate_gain(imageset)
                except Exception, e:
                    self.gain = 1.0
Ejemplo n.º 47
0
def run(argv=None):
    if argv is None:
        argv = sys.argv[1:]

    command_line = (libtbx.option_parser.option_parser(
        usage=
        "%s [-v] [-c] [-s] [-w wavelength] [-d distance] [-p pixel_size] [-x beam_x] [-y beam_y] [-o overload] files"
        % libtbx.env.dispatcher_name
    ).option(
        None,
        "--verbose",
        "-v",
        action="store_true",
        default=False,
        dest="verbose",
        help="Print more information about progress"
    ).option(
        None,
        "--crop",
        "-c",
        action="store_true",
        default=False,
        dest="crop",
        help="Crop the image such that the beam center is in the middle"
    ).option(
        None,
        "--skip_converted",
        "-s",
        action="store_true",
        default=False,
        dest="skip_converted",
        help=
        "Skip converting if an image already exist that matches the destination file name"
    ).option(
        None,
        "--wavelength",
        "-w",
        type="float",
        default=None,
        dest="wavelength",
        help="Override the image's wavelength (angstroms)").option(
            None,
            "--distance",
            "-d",
            type="float",
            default=None,
            dest="distance",
            help="Override the detector distance (mm)").option(
                None,
                "--pixel_size",
                "-p",
                type="float",
                default=None,
                dest="pixel_size",
                help="Override the detector pixel size (mm)").option(
                    None,
                    "--beam_x",
                    "-x",
                    type="float",
                    default=None,
                    dest="beam_center_x",
                    help="Override the beam x position (pixels)").option(
                        None,
                        "--beam_y",
                        "-y",
                        type="float",
                        default=None,
                        dest="beam_center_y",
                        help="Override the beam y position (pixels)").option(
                            None,
                            "--overload",
                            "-o",
                            type="float",
                            default=None,
                            dest="overload",
                            help="Override the detector overload value (ADU)")
                    ).process(args=argv)

    paths = command_line.args
    if len(paths) <= 0:
        raise Usage("No files specified")

    for imgpath in paths:
        destpath = os.path.join(
            os.path.dirname(imgpath),
            os.path.splitext(os.path.basename(imgpath))[0] + ".pickle")
        if command_line.options.skip_converted and os.path.isfile(destpath):
            if command_line.options.verbose:
                print "Skipping %s, file exists" % imgpath
                continue

        if command_line.options.verbose:
            print "Converting %s to %s..." % (imgpath, destpath)

        try:
            img = dxtbx.load(imgpath)
        except IOError:
            img = None
            pass

        if img is None:
            import numpy as np
            try:
                raw_data = np.loadtxt(imgpath)

                from scitbx.array_family import flex
                raw_data = flex.double(raw_data.astype(np.double))
            except ValueError:
                raise Usage("Couldn't load %s, no supported readers" % imgpath)

            detector = None
            beam = None
            scan = None
        else:
            raw_data = img.get_raw_data()
            detector = img.get_detector()
            beam = img.get_beam()
            scan = img.get_scan()

        if detector is None:
            if command_line.options.distance is None:
                raise Usage("Can't get distance from image. Override with -d")
            if command_line.options.pixel_size is None:
                raise Usage(
                    "Can't get pixel size from image. Override with -p")
            if command_line.options.overload is None:
                raise Usage(
                    "Can't get overload value from image. Override with -o")
            distance = command_line.options.distance
            pixel_size = command_line.options.pixel_size
            overload = command_line.options.overload
        else:
            detector = detector[0]
            if command_line.options.distance is None:
                distance = detector.get_directed_distance()
            else:
                distance = command_line.options.distance

            if command_line.options.pixel_size is None:
                pixel_size = detector.get_pixel_size()[0]
            else:
                pixel_size = command_line.options.pixel_size

            if command_line.options.overload is None:
                overload = detector.get_trusted_range()[1]
            else:
                overload = command_line.options.overload

        if beam is None:
            if command_line.options.wavelength is None:
                raise Usage(
                    "Can't get wavelength from image. Override with -w")
            wavelength = command_line.options.wavelength
        else:
            if command_line.options.wavelength is None:
                wavelength = beam.get_wavelength()
            else:
                wavelength = command_line.options.wavelength

        if beam is None and detector is None:
            if command_line.options.beam_center_x is None:
                print "Can't get beam x position from image. Using image center. Override with -x"
                beam_x = raw_data.focus()[0] * pixel_size
            else:
                beam_x = command_line.options.beam_center_x * pixel_size

            if command_line.options.beam_center_y is None:
                print "Can't get beam y position from image. Using image center. Override with -y"
                beam_y = raw_data.focus()[1] * pixel_size
            else:
                beam_y = command_line.options.beam_center_y * pixel_size
        else:
            if command_line.options.beam_center_x is None:
                beam_x = detector.get_beam_centre(beam.get_s0())[0]
            else:
                beam_x = command_line.options.beam_center_x * pixel_size

            if command_line.options.beam_center_y is None:
                beam_y = detector.get_beam_centre(beam.get_s0())[1]
            else:
                beam_y = command_line.options.beam_center_y * pixel_size

        if scan is None:
            timestamp = None
        else:
            msec, sec = math.modf(scan.get_epochs()[0])
            timestamp = evt_timestamp((sec, msec))

        data = dpack(data=raw_data,
                     distance=distance,
                     pixel_size=pixel_size,
                     wavelength=wavelength,
                     beam_center_x=beam_x,
                     beam_center_y=beam_y,
                     ccd_image_saturation=overload,
                     saturated_value=overload,
                     timestamp=timestamp)

        if scan is not None:
            osc_start, osc_range = scan.get_oscillation()
            if osc_start != osc_range:
                data['OSC_START'] = osc_start
                data['OSC_RANGE'] = osc_range

                data['TIME'] = scan.get_exposure_times()[0]

        if command_line.options.crop:
            data = crop_image_pickle(data)
        easy_pickle.dump(destpath, data)
Ejemplo n.º 48
0
    def endjob(self, obj1, obj2=None):
        """The endjob() function writes the mean and standard deviation images
    to disk.

    @param evt Event object (psana only)
    @param env Environment object
    """
        if obj2 is None:
            env = obj1
        else:
            evt = obj1
            env = obj2

        stats = super(mod_average, self).endjob(env)
        if stats is None:
            return

        device = cspad_tbx.address_split(self.address)[2]
        if device == 'Andor':
            beam_center = (0, 0)  # XXX Fiction!
            pixel_size = 13.5e-3  # XXX Should not be hardcoded here!
            saturated_value = 10000
        elif device == 'Cspad' or device == 'Cspad2x2':
            beam_center = self.beam_center
            pixel_size = cspad_tbx.pixel_size
            saturated_value = cspad_tbx.cspad_saturated_value
        elif device == 'marccd':
            beam_center = tuple(t // 2 for t in d['mean_img'].focus())
            pixel_size = 0.079346
            saturated_value = 2**16 - 1

        if stats['nmemb'] > 0:
            if self.avg_dirname  is not None or \
               self.avg_basename is not None or \
               self._mean_out    is not None:
                d = cspad_tbx.dpack(active_areas=self.active_areas,
                                    address=self.address,
                                    beam_center_x=pixel_size * beam_center[0],
                                    beam_center_y=pixel_size * beam_center[1],
                                    data=stats['mean_img'],
                                    distance=stats['distance'],
                                    pixel_size=pixel_size,
                                    saturated_value=saturated_value,
                                    timestamp=cspad_tbx.evt_timestamp(
                                        stats['time']),
                                    wavelength=stats['wavelength'])
                if self._mean_out is not None:
                    p = cspad_tbx.dwritef2(d, self._mean_out)
                else:
                    p = cspad_tbx.dwritef(d, self.avg_dirname,
                                          self.avg_basename)
                self.logger.info("Average written to %s" % p)

            if self.stddev_dirname  is not None or \
               self.stddev_basename is not None or \
               self._std_out    is not None:
                d = cspad_tbx.dpack(active_areas=self.active_areas,
                                    address=self.address,
                                    beam_center_x=pixel_size * beam_center[0],
                                    beam_center_y=pixel_size * beam_center[1],
                                    data=stats['std_img'],
                                    distance=stats['distance'],
                                    pixel_size=pixel_size,
                                    saturated_value=saturated_value,
                                    timestamp=cspad_tbx.evt_timestamp(
                                        stats['time']),
                                    wavelength=stats['wavelength'])
                if self._std_out is not None:
                    p = cspad_tbx.dwritef2(d, self._std_out)
                else:
                    p = cspad_tbx.dwritef(d, self.stddev_dirname,
                                          self.stddev_basename)
                self.logger.info("Standard deviation written to %s" % p)

            if self.max_dirname  is not None or \
               self.max_basename is not None or \
               self._max_out    is not None:
                d = cspad_tbx.dpack(active_areas=self.active_areas,
                                    address=self.address,
                                    beam_center_x=pixel_size * beam_center[0],
                                    beam_center_y=pixel_size * beam_center[1],
                                    data=stats['max_img'],
                                    distance=stats['distance'],
                                    pixel_size=pixel_size,
                                    saturated_value=saturated_value,
                                    timestamp=cspad_tbx.evt_timestamp(
                                        stats['time']),
                                    wavelength=stats['wavelength'])
                if self._max_out is not None:
                    p = cspad_tbx.dwritef2(d, self._max_out)
                else:
                    p = cspad_tbx.dwritef(d, self.max_dirname,
                                          self.max_basename)
                self.logger.info("Max written to %s" % p)

        if stats['nfail'] == 0:
            self.logger.info("%d images processed" % stats['nmemb'])
        else:
            self.logger.warning("%d images processed, %d failed" %
                                (stats['nmemb'], stats['nfail']))
Ejemplo n.º 49
0
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.

    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(mod_image_dict, self).event(evt, env)
    if (evt.get("skip_event")):
      return

    if self.cspad_img is None:
      return

    # This module only applies to detectors for which a distance is
    # available.
    distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
    if distance is None:
      self.nfail += 1
      self.logger.warning("event(): no distance, shot skipped")
      evt.put(skip_event_flag(), "skip_event")
      return

    device = cspad_tbx.address_split(self.address)[2]

    self.logger.info("Subprocess %02d: process image #%05d @ %s" %
                     (env.subprocess(), self.nshots, self.timestamp))

    # See r17537 of mod_average.py.
    if device == 'Cspad':
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
    elif device == 'Rayonix':
      pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
      saturated_value = rayonix_tbx.rayonix_saturated_value
    elif device == 'marccd':
      pixel_size = evt.get("marccd_pixel_size")
      saturated_value = evt.get("marccd_saturated_value")
      if distance == 0:
        distance = evt.get("marccd_distance")

    d = cspad_tbx.dpack(
      active_areas=self.active_areas,
      address=self.address,
      beam_center_x=pixel_size * self.beam_center[0],
      beam_center_y=pixel_size * self.beam_center[1],
      data=self.cspad_img.iround(), # XXX ouch!
      distance=distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=self.timestamp,
      wavelength=self.wavelength)

    evt.put(d, self.m_out_key)

    # Diagnostic message emitted only when all the processing is done.
    if (env.subprocess() >= 0):
      self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
                       (env.subprocess(), self.nshots, self.timestamp))
    else:
      self.logger.info("Accepted #%05d @ %s" %
                       (self.nshots, self.timestamp))