示例#1
0
    def event(self, evt, env):
        """The event() function is called for every L1Accept transition.  It
    outputs the detector image associated with the event @p evt to the
    file system.

    @param evt Event data object, a configure object
    @param env Environment object
    """

        super(mod_dump_bitmap, self).event(evt, env)
        if (evt.get('skip_event')):
            return

        # Where the sample-detector distance is not available, set it to
        # zero.
        distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
        if distance is None:
            distance = 0

        # See r17537 of mod_average.py.
        device = cspad_tbx.address_split(self.address)[2]
        if device == 'Cspad':
            pixel_size = cspad_tbx.pixel_size
            saturated_value = cspad_tbx.cspad_saturated_value
        elif device == 'marccd':
            pixel_size = 0.079346
            saturated_value = 2**16 - 1

        from iotbx.detectors import FlexImage_d as FlexImage
        vendortype = device
        saturation = 65535
        flex_img = FlexImage(rawdata=self.cspad_img,
                             binning=self._binning,
                             vendortype=vendortype,
                             brightness=self._brightness,
                             saturation=saturated_value)

        flex_img.setWindow(0, 0, 1)
        flex_img.adjust(color_scheme=self._color_scheme)
        flex_img.prep_string()
        import Image
        # XXX is size//self._binning safe here?
        pil_img = Image.fromstring('RGB', (flex_img.size2() // self._binning,
                                           flex_img.size1() // self._binning),
                                   flex_img.export_string)

        # The output path should not contain any funny characters which may
        # not work in all environments.  This constructs a sequence number a
        # la evt_seqno() from the dictionary's timestamp.
        t = self.timestamp
        s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[
            20:23]

        path = os.path.join(self._dirname,
                            self._basename + s + '.' + self._ext)

        self._logger.info("Exporting %s" % path)
        tmp_stream = open(path, 'wb')
        pil_img.save(tmp_stream, format=self._format)
        tmp_stream.close()
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.
    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(mod_radial_average, self).event(evt, env)
    if (evt.get("skip_event")):
      return

    # This module only applies to detectors for which a distance is
    # available.
    distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
    if distance is None:
      self.nfail += 1
      self.logger.warning("event(): no distance, shot skipped")
      evt.put(skip_event_flag(), "skip_event")
      return

    # See r17537 of mod_average.py.
    device = cspad_tbx.address_split(self.address)[2]
    if device == 'Cspad':
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
    elif device == 'marccd':
      pixel_size = 0.079346
      saturated_value = 2**16 - 1

    d = cspad_tbx.dpack(
      active_areas=self.active_areas,
      address=self.address,
      beam_center_x=pixel_size * self.beam_center[0],
      beam_center_y=pixel_size * self.beam_center[1],
      data=self.cspad_img.iround(), # XXX ouch!
      distance=distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=self.timestamp,
      wavelength=self.wavelength,
      xtal_target=self.m_xtal_target)

    from xfel.command_line.radial_average import run
    args = [
      "file_path=XTC stream",
      "xfel_target=%s"%self.m_xtal_target,
      "verbose=False"
    ]

    t = self.timestamp
    s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[20:23]

    if self._dirname is not None:
      dest_path = os.path.join(self._dirname, self._basename + s + ".txt")
      args.append("output_file=%s"%dest_path)

    self.logger.info("Calculating radial average for image %s"%s)
    xvals, results = run(args, d)

    evt.put(xvals, "cctbx.xfel.radial_average.xvals")
    evt.put(results, "cctbx.xfel.radial_average.results")
示例#3
0
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.

    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(mod_image_dict, self).event(evt, env)
    if (evt.get("skip_event")):
      return

    # This module only applies to detectors for which a distance is
    # available.
    distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
    if distance is None:
      self.nfail += 1
      self.logger.warning("event(): no distance, shot skipped")
      evt.put(skip_event_flag(), "skip_event")
      return

    device = cspad_tbx.address_split(self.address)[2]

    self.logger.info("Subprocess %02d: process image #%05d @ %s" %
                     (env.subprocess(), self.nshots, self.timestamp))

    # See r17537 of mod_average.py.
    if device == 'Cspad':
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
    elif device == 'Rayonix':
      pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
      saturated_value = rayonix_tbx.rayonix_saturated_value
    elif device == 'marccd':
      pixel_size = evt.get("marccd_pixel_size")
      saturated_value = evt.get("marccd_saturated_value")
      if distance == 0:
        distance = evt.get("marccd_distance")

    d = cspad_tbx.dpack(
      active_areas=self.active_areas,
      address=self.address,
      beam_center_x=pixel_size * self.beam_center[0],
      beam_center_y=pixel_size * self.beam_center[1],
      data=self.cspad_img.iround(), # XXX ouch!
      distance=distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=self.timestamp,
      wavelength=self.wavelength)

    evt.put(d, self.m_out_key)

    # Diagnostic message emitted only when all the processing is done.
    if (env.subprocess() >= 0):
      self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
                       (env.subprocess(), self.nshots, self.timestamp))
    else:
      self.logger.info("Accepted #%05d @ %s" %
                       (self.nshots, self.timestamp))
示例#4
0
文件: mod_dump.py 项目: dials/cctbx
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.  It
    outputs the detector image associated with the event @p evt to the
    file system.

    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(mod_dump, self).event(evt, env)
    if (evt.get('skip_event')):
      return

    if self.cspad_img is None:
      print("No image to save for %s"%self.timestamp)
      return

    # Where the sample-detector distance is not available, set it to
    # zero.
    distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
    if distance is None:
      distance = 0

    # See r17537 of mod_average.py.
    device = cspad_tbx.address_split(self.address)[2]
    if device == 'Cspad':
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
      output_filename = self._basename
    elif device == 'Rayonix':
      pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
      saturated_value = rayonix_tbx.rayonix_saturated_value
      output_filename = self._basename
    elif device == 'marccd':
      if distance == 0:
        distance = evt.get('marccd_distance')
      pixel_size = 0.079346
      saturated_value = 2**16 - 1
      output_filename = self._basename + evt.get(str, 'mccd_name') + "_"

    d = cspad_tbx.dpack(
      active_areas=self.active_areas,
      address=self.address,
      beam_center_x=pixel_size * self.beam_center[0],
      beam_center_y=pixel_size * self.beam_center[1],
      data=self.cspad_img.iround(), # XXX ouch!
      distance=distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=self.timestamp,
      wavelength=self.wavelength)
    if self._format == "pickle":
      cspad_tbx.dwritef(d, self._dirname, output_filename)
    elif self._format == "tiff":
      cspad_tbx.write_tiff(d, self._dirname, output_filename)
    output_filename = None
示例#5
0
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.  It
    outputs the detector image associated with the event @p evt to the
    file system.

    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(mod_dump, self).event(evt, env)
    if (evt.get('skip_event')):
      return

    if self.cspad_img is None:
      print "No image to save for %s"%self.timestamp
      return

    # Where the sample-detector distance is not available, set it to
    # zero.
    distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
    if distance is None:
      distance = 0

    # See r17537 of mod_average.py.
    device = cspad_tbx.address_split(self.address)[2]
    if device == 'Cspad':
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
      output_filename = self._basename
    elif device == 'Rayonix':
      pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
      saturated_value = rayonix_tbx.rayonix_saturated_value
      output_filename = self._basename
    elif device == 'marccd':
      if distance == 0:
        distance = evt.get('marccd_distance')
      pixel_size = 0.079346
      saturated_value = 2**16 - 1
      output_filename = self._basename + evt.get(str, 'mccd_name') + "_"

    d = cspad_tbx.dpack(
      active_areas=self.active_areas,
      address=self.address,
      beam_center_x=pixel_size * self.beam_center[0],
      beam_center_y=pixel_size * self.beam_center[1],
      data=self.cspad_img.iround(), # XXX ouch!
      distance=distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=self.timestamp,
      wavelength=self.wavelength)
    if self._format == "pickle":
      cspad_tbx.dwritef(d, self._dirname, output_filename)
    elif self._format == "tiff":
      cspad_tbx.write_tiff(d, self._dirname, output_filename)
    output_filename = None
示例#6
0
  def __init__(self,
               address,
               n_collate   = None,
               n_update    = 120,
               common_mode_correction = "none",
               wait=None,
               photon_counting=False,
               sigma_scaling=False,
               **kwds):
    """The mod_view class constructor XXX.

    @param address         Full data source address of the DAQ device
    @param calib_dir       Directory with calibration information
    @param common_mode_correction The type of common mode correction to apply
    @param dark_path       Path to input average dark image
    @param dark_stddev     Path to input standard deviation dark
                           image, required if @p dark_path is given
    @param wait            Minimum time (in seconds) to wait on the current
                           image before moving on to the next
    @param n_collate       Number of shots to average, or <= 0 to
                           average all shots
    @param n_update        Number of shots between updates
    """

    super(mod_view, self).__init__(
      address=address,
      common_mode_correction=common_mode_correction,
      **kwds)

    self.detector = cspad_tbx.address_split(address)[0]
    self.nvalid   = 0
    self.ncollate = cspad_tbx.getOptInteger(n_collate)
    self.nupdate  = cspad_tbx.getOptInteger(n_update)
    self.photon_counting = cspad_tbx.getOptBool(photon_counting)
    self.sigma_scaling = cspad_tbx.getOptBool(sigma_scaling)
    if (self.ncollate is None):
      self.ncollate = self.nupdate
    if (self.ncollate > self.nupdate):
      self.ncollate = self.nupdate
      self.logger.warning("n_collate capped to %d" % self.nupdate)

    linger = True # XXX Make configurable
    wait = cspad_tbx.getOptFloat(wait)

    # Create a managed FIFO queue shared between the viewer and the
    # current process.  The current process will produce images, while
    # the viewer process will consume them.
    manager = multiprocessing.Manager()
    self._queue = manager.Queue()
    self._proc = multiprocessing.Process(
      target=_xray_frame_process, args=(self._queue, linger, wait))
    self._proc.start()

    self.n_shots = 0
示例#7
0
    def __init__(self,
                 address,
                 n_collate=None,
                 n_update=120,
                 common_mode_correction="none",
                 wait=None,
                 photon_counting=False,
                 sigma_scaling=False,
                 **kwds):
        """The mod_view class constructor XXX.

    @param address         Full data source address of the DAQ device
    @param calib_dir       Directory with calibration information
    @param common_mode_correction The type of common mode correction to apply
    @param dark_path       Path to input average dark image
    @param dark_stddev     Path to input standard deviation dark
                           image, required if @p dark_path is given
    @param wait            Minimum time (in seconds) to wait on the current
                           image before moving on to the next
    @param n_collate       Number of shots to average, or <= 0 to
                           average all shots
    @param n_update        Number of shots between updates
    """

        super(mod_view,
              self).__init__(address=address,
                             common_mode_correction=common_mode_correction,
                             **kwds)

        self.detector = cspad_tbx.address_split(address)[0]
        self.nvalid = 0
        self.ncollate = cspad_tbx.getOptInteger(n_collate)
        self.nupdate = cspad_tbx.getOptInteger(n_update)
        self.photon_counting = cspad_tbx.getOptBool(photon_counting)
        self.sigma_scaling = cspad_tbx.getOptBool(sigma_scaling)
        if (self.ncollate is None):
            self.ncollate = self.nupdate
        if (self.ncollate > self.nupdate):
            self.ncollate = self.nupdate
            self.logger.warning("n_collate capped to %d" % self.nupdate)

        linger = True  # XXX Make configurable
        wait = cspad_tbx.getOptFloat(wait)

        # Create a managed FIFO queue shared between the viewer and the
        # current process.  The current process will produce images, while
        # the viewer process will consume them.
        manager = multiprocessing.Manager()
        self._queue = manager.Queue()
        self._proc = multiprocessing.Process(target=_xray_frame_process,
                                             args=(self._queue, linger, wait))
        self._proc.start()

        self.n_shots = 0
示例#8
0
    def set_up_hitfinder(self):
        # See r17537 of mod_average.py.
        device = cspad_tbx.address_split(self.address)[2]
        if device == 'Cspad':
            img_dim = (1765, 1765)
            pixel_size = cspad_tbx.pixel_size
        elif device == 'marccd':
            img_dim = (4500, 4500)
            pixel_size = 0.079346
        elif device == 'Rayonix':
            img_dim = rayonix_tbx.get_rayonix_detector_dimensions(
                self.bin_size)
            pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
        else:
            raise RuntimeError("Unsupported device %s" % self.address)

        if self.beam_center is None:
            self.beam_center = [0, 0]

        self.hitfinder_d = cspad_tbx.dpack(
            active_areas=self.active_areas,
            beam_center_x=pixel_size * self.beam_center[0],
            beam_center_y=pixel_size * self.beam_center[1],
            data=flex.int(flex.grid(img_dim[0], img_dim[1]), 0),
            xtal_target=self.m_xtal_target)

        if device == 'Cspad':
            # Figure out which ASIC:s are on the central four sensors.  This
            # only applies to the CSPAD.
            assert len(self.active_areas) % 4 == 0
            distances = flex.double()
            for i in range(0, len(self.active_areas), 4):
                cenasic = (
                    (self.active_areas[i + 0] + self.active_areas[i + 2]) / 2,
                    (self.active_areas[i + 1] + self.active_areas[i + 3]) / 2)
                distances.append(
                    math.hypot(cenasic[0] - self.beam_center[0],
                               cenasic[1] - self.beam_center[1]))
            orders = flex.sort_permutation(distances)

            # Use the central 8 ASIC:s (central 4 sensors).
            flags = flex.int(len(self.active_areas) // 4, 0)
            for i in range(8):
                flags[orders[i]] = 1
            self.asic_filter = "distl.tile_flags=" + ",".join(
                ["%1d" % b for b in flags])

        elif device == 'marccd':
            # There is only one active area for the MAR CCD, so use it.
            self.asic_filter = "distl.tile_flags=1"
        elif device == 'Rayonix':
            # There is only one active area for the Rayonix, so use it.
            self.asic_filter = "distl.tile_flags=1"
示例#9
0
  def set_up_hitfinder(self):
    # See r17537 of mod_average.py.
    device = cspad_tbx.address_split(self.address)[2]
    if device == 'Cspad':
      img_dim = (1765, 1765)
      pixel_size = cspad_tbx.pixel_size
    elif device == 'marccd':
      img_dim = (4500, 4500)
      pixel_size = 0.079346
    elif device == 'Rayonix':
      img_dim = rayonix_tbx.get_rayonix_detector_dimensions(self.bin_size)
      pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
    else:
      raise RuntimeError("Unsupported device %s" % self.address)

    if self.beam_center is None:
      self.beam_center = [0,0]

    self.hitfinder_d = cspad_tbx.dpack(
      active_areas=self.active_areas,
      beam_center_x=pixel_size * self.beam_center[0],
      beam_center_y=pixel_size * self.beam_center[1],
      data=flex.int(flex.grid(img_dim[0], img_dim[1]), 0),
      xtal_target=self.m_xtal_target)

    if device == 'Cspad':
      # Figure out which ASIC:s are on the central four sensors.  This
      # only applies to the CSPAD.
      assert len(self.active_areas) % 4 == 0
      distances = flex.double()
      for i in range(0, len(self.active_areas), 4):
        cenasic = ((self.active_areas[i + 0] + self.active_areas[i + 2]) / 2,
                   (self.active_areas[i + 1] + self.active_areas[i + 3]) / 2)
        distances.append(math.hypot(cenasic[0] - self.beam_center[0],
                                    cenasic[1] - self.beam_center[1]))
      orders = flex.sort_permutation(distances)

      # Use the central 8 ASIC:s (central 4 sensors).
      flags = flex.int(len(self.active_areas) // 4, 0)
      for i in range(8):
        flags[orders[i]] = 1
      self.asic_filter = "distl.tile_flags=" + ",".join(
        ["%1d" % b for b in flags])

    elif device == 'marccd':
      # There is only one active area for the MAR CCD, so use it.
      self.asic_filter = "distl.tile_flags=1"
    elif device == 'Rayonix':
      # There is only one active area for the Rayonix, so use it.
      self.asic_filter = "distl.tile_flags=1"
示例#10
0
def average(argv=None):
  if argv == None:
    argv = sys.argv[1:]

  try:
    from mpi4py import MPI
  except ImportError:
    raise Sorry("MPI not found")

  command_line = (libtbx.option_parser.option_parser(
    usage="""
%s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f]

To write image pickles use -p, otherwise the program writes CSPAD CBFs.
Writing CBFs requires the geometry to be already deployed.

Examples:
cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571

Use one process on the current node to process all the events from run 25 of
experiment cxi49812, using a detz_offset of 571.

mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571

As above, using 16 cores on the current node.

bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812

As above, using the psanaq and 100 cores, putting the log in average.out and
the output images in the folder cxi49812.
""" % libtbx.env.dispatcher_name)
                .option(None, "--as_pickle", "-p",
                        action="store_true",
                        default=False,
                        dest="as_pickle",
                        help="Write results as image pickle files instead of cbf files")
                .option(None, "--config", "-c",
                        type="string",
                        default=None,
                        dest="config",
                        metavar="PATH",
                        help="psana config file")
                .option(None, "--experiment", "-x",
                        type="string",
                        default=None,
                        dest="experiment",
                        help="experiment name (eg cxi84914)")
                .option(None, "--run", "-r",
                        type="int",
                        default=None,
                        dest="run",
                        help="run number")
                .option(None, "--address", "-a",
                        type="string",
                        default="CxiDs2.0:Cspad.0",
                        dest="address",
                        help="detector address name (eg CxiDs2.0:Cspad.0)")
                .option(None, "--detz_offset", "-d",
                        type="float",
                        default=None,
                        dest="detz_offset",
                        help="offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)")
                .option(None, "--outputdir", "-o",
                        type="string",
                        default=".",
                        dest="outputdir",
                        metavar="PATH",
                        help="Optional path to output directory for output files")
                .option(None, "--averagebase", "-A",
                        type="string",
                        default="{experiment!l}_avg-r{run:04d}",
                        dest="averagepath",
                        metavar="PATH",
                        help="Path to output average image without extension. String substitution allowed")
                .option(None, "--stddevbase", "-S",
                        type="string",
                        default="{experiment!l}_stddev-r{run:04d}",
                        dest="stddevpath",
                        metavar="PATH",
                        help="Path to output standard deviation image without extension. String substitution allowed")
                .option(None, "--maxbase", "-M",
                        type="string",
                        default="{experiment!l}_max-r{run:04d}",
                        dest="maxpath",
                        metavar="PATH",
                        help="Path to output maximum projection image without extension. String substitution allowed")
                .option(None, "--numevents", "-n",
                        type="int",
                        default=None,
                        dest="numevents",
                        help="Maximum number of events to process. Default: all")
                .option(None, "--skipevents", "-s",
                        type="int",
                        default=0,
                        dest="skipevents",
                        help="Number of events in the beginning of the run to skip. Default: 0")
                .option(None, "--verbose", "-v",
                        action="store_true",
                        default=False,
                        dest="verbose",
                        help="Print more information about progress")
                .option(None, "--pickle-optical-metrology", "-m",
                        action="store_true",
                        default=False,
                        dest="pickle_optical_metrology",
                        help="If writing pickle files, use the optical metrology in the experiment's calib directory")
                .option(None, "--bin_size", "-b",
                        type="int",
                        default=None,
                        dest="bin_size",
                        help="Rayonix detector bin size")
                .option(None, "--override_beam_x", "-X",
                        type="float",
                        default=None,
                        dest="override_beam_x",
                        help="Rayonix detector beam center x coordinate")
                .option(None, "--override_beam_y", "-Y",
                        type="float",
                        default=None,
                        dest="override_beam_y",
                        help="Rayonix detector beam center y coordinate")
                .option(None, "--calib_dir", "-C",
                        type="string",
                        default=None,
                        dest="calib_dir",
                        metavar="PATH",
                        help="calibration directory")
                .option(None, "--xtc_dir", "-D",
                        type="string",
                        default=None,
                        dest="xtc_dir",
                        metavar="PATH",
                        help="xtc stream directory")
                .option(None, "--use_ffb", "-f",
                        action="store_true",
                        default=False,
                        dest="use_ffb",
                        help="Use the fast feedback filesystem at LCLS. Only for the active experiment!")
                ).process(args=argv)


  if len(command_line.args) > 0 or \
      command_line.options.as_pickle is None or \
      command_line.options.experiment is None or \
      command_line.options.run is None or \
      command_line.options.address is None or \
      command_line.options.detz_offset is None or \
      command_line.options.averagepath is None or \
      command_line.options.stddevpath is None or \
      command_line.options.maxpath is None or \
      command_line.options.pickle_optical_metrology is None:
    command_line.parser.show_help()
    return

  # set this to sys.maxint to analyze all events
  if command_line.options.numevents is None:
    maxevents = sys.maxint
  else:
    maxevents = command_line.options.numevents

  comm = MPI.COMM_WORLD
  rank = comm.Get_rank()
  size = comm.Get_size()

  if command_line.options.config is not None:
    psana.setConfigFile(command_line.options.config)
  dataset_name = "exp=%s:run=%d:idx"%(command_line.options.experiment, command_line.options.run)
  if command_line.options.xtc_dir is not None:
    if command_line.options.use_ffb:
      raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system")
    dataset_name += ":dir=%s"%command_line.options.xtc_dir
  elif command_line.options.use_ffb:
    # as ffb is only at SLAC, ok to hardcode /reg/d here
    dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc"%(command_line.options.experiment[0:3],command_line.options.experiment)
  ds = psana.DataSource(dataset_name)
  address = command_line.options.address
  src = psana.Source('DetInfo(%s)'%address)
  if not command_line.options.as_pickle:
    psana_det = psana.Detector(address, ds.env())

  nevent = np.array([0.])

  for run in ds.runs():
    runnumber = run.run()
    # list of all events
    if command_line.options.skipevents > 0:
      print "Skipping first %d events"%command_line.options.skipevents

    times = run.times()[command_line.options.skipevents:]
    nevents = min(len(times),maxevents)
    # chop the list into pieces, depending on rank.  This assigns each process
    # events such that the get every Nth event where N is the number of processes
    mytimes = [times[i] for i in xrange(nevents) if (i+rank)%size == 0]
    for i in xrange(len(mytimes)):
      if i%10==0: print 'Rank',rank,'processing event',rank*len(mytimes)+i,', ',i,'of',len(mytimes)
      evt = run.event(mytimes[i])
      #print "Event #",rank*mylength+i," has id:",evt.get(EventId)
      if 'Rayonix' in command_line.options.address:
        data = evt.get(Camera.FrameV1,src)
        if data is None:
          print "No data"
          continue
        data=data.data16().astype(np.float64)
      elif command_line.options.as_pickle:
        data = evt.get(psana.ndarray_float64_3, src, 'image0')
      else:
        # get numpy array, 32x185x388
        data = psana_det.calib(evt) # applies psana's complex run-dependent calibrations
      if data is None:
        print "No data"
        continue

      d = cspad_tbx.env_distance(address, run.env(), command_line.options.detz_offset)
      if d is None:
        print "No distance, skipping shot"
        continue
      if 'distance' in locals():
        distance += d
      else:
        distance = np.array([float(d)])

      w = cspad_tbx.evt_wavelength(evt)
      if w is None:
        print "No wavelength, skipping shot"
        continue
      if 'wavelength' in locals():
        wavelength += w
      else:
        wavelength = np.array([w])

      t = cspad_tbx.evt_time(evt)
      if t is None:
        print "No timestamp, skipping shot"
        continue
      if 'timestamp' in locals():
        timestamp += t[0] + (t[1]/1000)
      else:
        timestamp = np.array([t[0] + (t[1]/1000)])

      if 'sum' in locals():
        sum+=data
      else:
        sum=np.array(data, copy=True)
      if 'sumsq' in locals():
        sumsq+=data*data
      else:
        sumsq=data*data
      if 'maximum' in locals():
        maximum=np.maximum(maximum,data)
      else:
        maximum=np.array(data, copy=True)

      nevent += 1

  #sum the images across mpi cores
  if size > 1:
    print "Synchronizing rank", rank
  totevent = np.zeros(nevent.shape)
  comm.Reduce(nevent,totevent)

  if rank == 0 and totevent[0] == 0:
    raise Sorry("No events found in the run")

  sumall = np.zeros(sum.shape).astype(sum.dtype)
  comm.Reduce(sum,sumall)

  sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype)
  comm.Reduce(sumsq,sumsqall)

  maxall = np.zeros(maximum.shape).astype(maximum.dtype)
  comm.Reduce(maximum,maxall, op=MPI.MAX)

  waveall = np.zeros(wavelength.shape).astype(wavelength.dtype)
  comm.Reduce(wavelength,waveall)

  distall = np.zeros(distance.shape).astype(distance.dtype)
  comm.Reduce(distance,distall)

  timeall = np.zeros(timestamp.shape).astype(timestamp.dtype)
  comm.Reduce(timestamp,timeall)

  if rank==0:
    if size > 1:
      print "Synchronized"

    # Accumulating floating-point numbers introduces errors,
    # which may cause negative variances.  Since a two-pass
    # approach is unacceptable, the standard deviation is
    # clamped at zero.
    mean = sumall / float(totevent[0])
    variance = (sumsqall / float(totevent[0])) - (mean**2)
    variance[variance < 0] = 0
    stddev = np.sqrt(variance)

    wavelength = waveall[0] / totevent[0]
    distance = distall[0] / totevent[0]
    pixel_size = cspad_tbx.pixel_size
    saturated_value = cspad_tbx.cspad_saturated_value
    timestamp = timeall[0] / totevent[0]
    timestamp = (int(timestamp), timestamp % int(timestamp) * 1000)
    timestamp = cspad_tbx.evt_timestamp(timestamp)


    if command_line.options.as_pickle:
      extension = ".pickle"
    else:
      extension = ".cbf"

    dest_paths = [cspad_tbx.pathsubst(command_line.options.averagepath + extension, evt, ds.env()),
                  cspad_tbx.pathsubst(command_line.options.stddevpath  + extension, evt, ds.env()),
                  cspad_tbx.pathsubst(command_line.options.maxpath     + extension, evt, ds.env())]
    dest_paths = [os.path.join(command_line.options.outputdir, path) for path in dest_paths]
    if 'Rayonix' in command_line.options.address:
      from xfel.cxi.cspad_ana import rayonix_tbx
      pixel_size = rayonix_tbx.get_rayonix_pixel_size(command_line.options.bin_size)
      beam_center = [command_line.options.override_beam_x,command_line.options.override_beam_y]
      detector_dimensions = rayonix_tbx.get_rayonix_detector_dimensions(command_line.options.bin_size)
      active_areas = flex.int([0,0,detector_dimensions[0],detector_dimensions[1]])
      split_address = cspad_tbx.address_split(address)
      old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3]
      for data, path in zip([mean, stddev, maxall], dest_paths):
        print "Saving", path
        d = cspad_tbx.dpack(
            active_areas=active_areas,
            address=old_style_address,
            beam_center_x=pixel_size * beam_center[0],
            beam_center_y=pixel_size * beam_center[1],
            data=flex.double(data),
            distance=distance,
            pixel_size=pixel_size,
            saturated_value=rayonix_tbx.rayonix_saturated_value,
            timestamp=timestamp,
            wavelength=wavelength)
        easy_pickle.dump(path, d)
    elif command_line.options.as_pickle:
      split_address = cspad_tbx.address_split(address)
      old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3]

      xpp = 'xpp' in address.lower()
      if xpp:
        evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
        timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format
        from xfel.detector_formats import detector_format_version, reverse_timestamp
        from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
        version_lookup = detector_format_version(old_style_address, reverse_timestamp(timestamp)[0])
        assert version_lookup is not None
        active_areas = xpp_active_areas[version_lookup]['active_areas']
        beam_center = [1765 // 2, 1765 // 2]
      else:
        if command_line.options.calib_dir is not None:
          metro_path = command_line.options.calib_dir
        elif command_line.options.pickle_optical_metrology:
          from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path
          metro_path = get_calib_file_path(run.env(), address, run)
        else:
          metro_path = libtbx.env.find_in_repositories("xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
        sections = parse_calib.calib2sections(metro_path)
        beam_center, active_areas = cspad_tbx.cbcaa(
          cspad_tbx.getConfig(address, ds.env()), sections)

      class fake_quad(object):
        def __init__(self, q, d):
          self.q = q
          self.d = d

        def quad(self):
          return self.q

        def data(self):
          return self.d

      if xpp:
        quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)]
        mean = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
        mean = flex.double(mean.astype(np.float64))

        quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)]
        stddev = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
        stddev = flex.double(stddev.astype(np.float64))

        quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)]
        maxall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
        maxall = flex.double(maxall.astype(np.float64))
      else:
        quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)]
        mean = cspad_tbx.CsPadDetector(
          address, evt, ds.env(), sections, quads=quads)
        mean = flex.double(mean.astype(np.float64))

        quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)]
        stddev = cspad_tbx.CsPadDetector(
          address, evt, ds.env(), sections, quads=quads)
        stddev = flex.double(stddev.astype(np.float64))

        quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)]
        maxall = cspad_tbx.CsPadDetector(
          address, evt, ds.env(), sections, quads=quads)
        maxall = flex.double(maxall.astype(np.float64))

      for data, path in zip([mean, stddev, maxall], dest_paths):
        print "Saving", path

        d = cspad_tbx.dpack(
          active_areas=active_areas,
          address=old_style_address,
          beam_center_x=pixel_size * beam_center[0],
          beam_center_y=pixel_size * beam_center[1],
          data=data,
          distance=distance,
          pixel_size=pixel_size,
          saturated_value=saturated_value,
          timestamp=timestamp,
          wavelength=wavelength)

        easy_pickle.dump(path, d)
    else:
      # load a header only cspad cbf from the slac metrology
      from xfel.cftbx.detector import cspad_cbf_tbx
      import pycbf
      base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(run, address)
      if base_dxtbx is None:
        raise Sorry("Couldn't load calibration file for run %d"%run.run())

      for data, path in zip([mean, stddev, maxall], dest_paths):
        print "Saving", path

        cspad_img = cspad_cbf_tbx.format_object_from_data(base_dxtbx, data, distance, wavelength, timestamp, address)
        cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
          pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
示例#11
0
    def event(self, evt, env):
        """The event() function is called for every L1Accept transition.
    @param evt Event data object, a configure object
    @param env Environment object
    """

        super(mod_radial_average, self).event(evt, env)
        if (evt.get("skip_event")):
            return

        # This module only applies to detectors for which a distance is
        # available.
        distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
        if distance is None:
            self.nfail += 1
            self.logger.warning("event(): no distance, shot skipped")
            evt.put(skip_event_flag(), "skip_event")
            return

        # See r17537 of mod_average.py.
        device = cspad_tbx.address_split(self.address)[2]
        if device == 'Cspad':
            pixel_size = cspad_tbx.pixel_size
            saturated_value = cspad_tbx.cspad_saturated_value
        elif device == 'marccd':
            pixel_size = 0.079346
            saturated_value = 2**16 - 1
        elif device == 'Rayonix':
            pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
            saturated_value = rayonix_tbx.rayonix_saturated_value

        d = cspad_tbx.dpack(
            active_areas=self.active_areas,
            address=self.address,
            beam_center_x=pixel_size * self.beam_center[0],
            beam_center_y=pixel_size * self.beam_center[1],
            data=self.cspad_img.iround(),  # XXX ouch!
            distance=distance,
            pixel_size=pixel_size,
            saturated_value=saturated_value,
            timestamp=self.timestamp,
            wavelength=self.wavelength,
            xtal_target=self.m_xtal_target)

        from xfel.command_line.radial_average import run
        args = [
            "file_path=XTC stream",
            "xfel_target=%s" % self.m_xtal_target, "verbose=False"
        ]

        t = self.timestamp
        s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[
            20:23]

        if self._dirname is not None:
            dest_path = os.path.join(self._dirname,
                                     self._basename + s + ".txt")
            args.append("output_file=%s" % dest_path)

        self.logger.info("Calculating radial average for image %s" % s)
        xvals, results = run(args, d)

        evt.put(xvals, "cctbx.xfel.radial_average.xvals")
        evt.put(results, "cctbx.xfel.radial_average.results")

        def get_closest_idx(data, val):
            from scitbx.array_family import flex
            deltas = flex.abs(data - val)
            return flex.first_index(deltas, flex.min(deltas))

        if self._two_theta_low is not None:
            i_low = results[get_closest_idx(xvals, self._two_theta_low)]
            evt.put(i_low, "cctbx.xfel.radial_average.two_theta_low")

        if self._two_theta_high is not None:
            i_high = results[get_closest_idx(xvals, self._two_theta_high)]
            evt.put(i_high, "cctbx.xfel.radial_average.two_theta_high")
示例#12
0
  def __init__(self,
               address,
               avg_dirname=None,
               avg_basename=None,
               stddev_dirname=None,
               stddev_basename=None,
               max_dirname=None,
               max_basename=None,
               background_path=None,
               flags=None,
               hot_threshold=None,
               gain_threshold=None,
               noise_threshold=7,
               elastic_threshold=9,
               symnoise_threshold=4,
               **kwds):
    """
    @param address         Full data source address of the DAQ device
    @param avg_dirname     Directory portion of output average image
                           XXX mean
    @param avg_basename    Filename prefix of output average image XXX
                           mean
    @param flags inactive:  Eliminate the inactive pixels
                 noelastic: Eliminate elastic scattering
                 nohot:     Eliminate the hot pixels
                 nonoise:   Eliminate noisy pixels
                 symnoise:  Symmetrically eliminate noisy pixels
    @param stddev_dirname  Directory portion of output standard
                           deviation image XXX std
    @param stddev_basename Filename prefix of output standard
                           deviation image XXX std
    @param max_dirname     Directory portion of output maximum
                           projection image
    @param max_basename    Filename prefix of output maximum
                           projection image
     """

    super(average_mixin, self).__init__(
      address=address,
      **kwds
    )
    self.roi = None
    self.avg_basename = cspad_tbx.getOptString(avg_basename)
    self.avg_dirname = cspad_tbx.getOptString(avg_dirname)
    self.detector = cspad_tbx.address_split(address)[0]
    self.flags = cspad_tbx.getOptStrings(flags, default = [])
    self.stddev_basename = cspad_tbx.getOptString(stddev_basename)
    self.stddev_dirname = cspad_tbx.getOptString(stddev_dirname)
    self.max_basename = cspad_tbx.getOptString(max_basename)
    self.max_dirname = cspad_tbx.getOptString(max_dirname)
    self.background_path = cspad_tbx.getOptString(background_path)
    self.hot_threshold = cspad_tbx.getOptFloat(hot_threshold)
    self.gain_threshold = cspad_tbx.getOptFloat(gain_threshold)
    self.noise_threshold = cspad_tbx.getOptFloat(noise_threshold)
    self.elastic_threshold = cspad_tbx.getOptFloat(elastic_threshold)
    self.symnoise_threshold = cspad_tbx.getOptFloat(symnoise_threshold)

    if background_path is not None:
      background_dict = easy_pickle.load(background_path)
      self.background_img = background_dict['DATA']

    self._have_max = self.max_basename is not None or \
                     self.max_dirname is not None
    self._have_mean = self.avg_basename is not None or \
                      self.avg_dirname is not None
    self._have_std = self.stddev_basename is not None or \
                     self.stddev_dirname is not None

    # Start a server process which holds a set of Python objects that
    # other processes can manipulate using proxies.  The queues will
    # be used in endjob() to pass images between the worker processes,
    # and the lock will ensure the transfer is treated as a critical
    # section.  There is therefore the risk of a hang if the queues
    # cannot hold all the data one process will supply before another
    # empties it.
    #
    # In an attempt to alleviate this issue, separate queues are used
    # for the potentially big images.  The hope is to prevent
    # producers from blocking while consumers are locked out by using
    # more buffers.
    mgr = multiprocessing.Manager()
    self._lock = mgr.Lock()
    self._metadata = mgr.dict()
    self._queue_max = mgr.Queue()
    self._queue_sum = mgr.Queue()
    self._queue_ssq = mgr.Queue()
示例#13
0
    def event(self, evt, env):
        """The event() function is called for every L1Accept transition.
    XXX Since the viewer is now running in a parallel process, the
    averaging here is now the bottleneck.

    @param evt Event data object, a configure object
    @param env Environment object
    """
        from pyana.event import Event

        self.n_shots += 1

        super(mod_view, self).event(evt, env)
        if evt.status() != Event.Normal or evt.get(
                'skip_event'):  # XXX transition
            return

        # Get the distance for the detectors that should have it, and set
        # it to NaN for those that should not.
        if self.detector == 'CxiDs1' or \
           self.detector == 'CxiDsd' or \
           self.detector == 'XppGon':
            distance = cspad_tbx.env_distance(self.address, env,
                                              self._detz_offset)
            if distance is None:
                self.nfail += 1
                self.logger.warning("event(): no distance, shot skipped")
                evt.put(skip_event_flag(), "skip_event")
                return
        else:
            distance = float('nan')

        if not self._proc.is_alive():
            evt.setStatus(Event.Stop)

        # Early return if the next update to the viewer is more than
        # self.ncollate shots away.  XXX Since the common_mode.event()
        # function does quite a bit of processing, the savings are
        # probably not so big.
        next_update = (self.nupdate - 1) - (self.nshots - 1) % self.nupdate
        if (self.ncollate > 0 and next_update >= self.ncollate):
            return

        if self.sigma_scaling:
            self.do_sigma_scaling()

        if self.photon_counting:
            self.do_photon_counting()

        # Trim the disabled section from the Sc1 detector image.  XXX This
        # is a bit of a kludge, really.


#    if (self.address == "CxiSc1-0|Cspad2x2-0"):
#      self.cspad_img = self.cspad_img[185:2 * 185, :]

# Update the sum of the valid images, starting a new collation if
# appropriate.  This guarantees self.nvalid > 0.
        if (self.nvalid == 0
                or self.ncollate > 0 and self.nvalid >= self.ncollate):
            self.img_sum = self.cspad_img
            self.nvalid = 1
        else:
            self.img_sum += self.cspad_img
            self.nvalid += 1

        # Update the viewer to display the current average image, and
        # start a new collation, if appropriate.
        if (next_update == 0):
            from time import localtime, strftime

            time_str = strftime("%H:%M:%S", localtime(evt.getTime().seconds()))
            title = "r%04d@%s: average of %d last images on %s" \
                % (evt.run(), time_str, self.nvalid, self.address)

            # See also mod_average.py.
            device = cspad_tbx.address_split(self.address)[2]
            if device == 'Cspad':
                beam_center = self.beam_center
                pixel_size = cspad_tbx.pixel_size
                saturated_value = cspad_tbx.cspad_saturated_value

            elif device == 'marccd':
                beam_center = tuple(t // 2 for t in self.img_sum.focus())
                pixel_size = 0.079346
                saturated_value = 2**16 - 1

            # Wait for the viewer process to empty the queue before feeding
            # it a new image, and ensure not to hang if the viewer process
            # exits.  Because of multithreading/multiprocessing semantics,
            # self._queue.empty() is unreliable.
            fmt = _Format(BEAM_CENTER=beam_center,
                          DATA=self.img_sum / self.nvalid,
                          DETECTOR_ADDRESS=self.address,
                          DISTANCE=distance,
                          PIXEL_SIZE=pixel_size,
                          SATURATED_VALUE=saturated_value,
                          TIME_TUPLE=cspad_tbx.evt_time(evt),
                          WAVELENGTH=self.wavelength)

            while not self._queue.empty():
                if not self._proc.is_alive():
                    evt.setStatus(Event.Stop)
                    return
            while True:
                try:
                    self._queue.put((fmt, title), timeout=1)
                    break
                except Exception:
                    pass

            if (self.ncollate > 0):
                self.nvalid = 0
示例#14
0
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.
    XXX Since the viewer is now running in a parallel process, the
    averaging here is now the bottleneck.

    @param evt Event data object, a configure object
    @param env Environment object
    """
    from pyana.event import Event

    self.n_shots += 1

    super(mod_view, self).event(evt, env)
    if evt.status() != Event.Normal or evt.get('skip_event'): # XXX transition
      return

    # Get the distance for the detectors that should have it, and set
    # it to NaN for those that should not.
    if self.detector == 'CxiDs1' or \
       self.detector == 'CxiDsd' or \
       self.detector == 'XppGon':
      distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
      if distance is None:
        self.nfail += 1
        self.logger.warning("event(): no distance, shot skipped")
        evt.put(skip_event_flag(), "skip_event")
        return
    else:
      distance = float('nan')

    if not self._proc.is_alive():
      evt.setStatus(Event.Stop)

    # Early return if the next update to the viewer is more than
    # self.ncollate shots away.  XXX Since the common_mode.event()
    # function does quite a bit of processing, the savings are
    # probably not so big.
    next_update = (self.nupdate - 1) - (self.nshots - 1) % self.nupdate
    if (self.ncollate > 0 and next_update >= self.ncollate):
      return

    if self.sigma_scaling:
      self.do_sigma_scaling()

    if self.photon_counting:
      self.do_photon_counting()

    # Trim the disabled section from the Sc1 detector image.  XXX This
    # is a bit of a kludge, really.
#    if (self.address == "CxiSc1-0|Cspad2x2-0"):
#      self.cspad_img = self.cspad_img[185:2 * 185, :]

    # Update the sum of the valid images, starting a new collation if
    # appropriate.  This guarantees self.nvalid > 0.
    if (self.nvalid == 0 or self.ncollate > 0 and self.nvalid >= self.ncollate):
      self.img_sum = self.cspad_img
      self.nvalid  = 1
    else:
      self.img_sum += self.cspad_img
      self.nvalid  += 1

    # Update the viewer to display the current average image, and
    # start a new collation, if appropriate.
    if (next_update == 0):
      from time import localtime, strftime

      time_str = strftime("%H:%M:%S", localtime(evt.getTime().seconds()))
      title = "r%04d@%s: average of %d last images on %s" \
          % (evt.run(), time_str, self.nvalid, self.address)

      # See also mod_average.py.
      device = cspad_tbx.address_split(self.address)[2]
      if device == 'Cspad':
        beam_center = self.beam_center
        pixel_size = cspad_tbx.pixel_size
        saturated_value = cspad_tbx.cspad_saturated_value

      elif device == 'marccd':
        beam_center = tuple(t // 2 for t in self.img_sum.focus())
        pixel_size = 0.079346
        saturated_value = 2**16 - 1

      # Wait for the viewer process to empty the queue before feeding
      # it a new image, and ensure not to hang if the viewer process
      # exits.  Because of multithreading/multiprocessing semantics,
      # self._queue.empty() is unreliable.
      fmt = _Format(BEAM_CENTER=beam_center,
                      DATA=self.img_sum / self.nvalid,
                      DETECTOR_ADDRESS=self.address,
                      DISTANCE=distance,
                      PIXEL_SIZE=pixel_size,
                      SATURATED_VALUE=saturated_value,
                      TIME_TUPLE=cspad_tbx.evt_time(evt),
                      WAVELENGTH=self.wavelength)

      while not self._queue.empty():
        if not self._proc.is_alive():
          evt.setStatus(Event.Stop)
          return
      while True:
        try:
          self._queue.put((fmt, title), timeout=1)
          break
        except Exception:
          pass

      if (self.ncollate > 0):
        self.nvalid = 0
示例#15
0
  def endjob(self, obj1, obj2=None):
    """The endjob() function writes the mean and standard deviation images
    to disk.

    @param evt Event object (psana only)
    @param env Environment object
    """
    if obj2 is None:
      env = obj1
    else:
      evt = obj1
      env = obj2

    stats = super(mod_average, self).endjob(env)
    if stats is None:
      return

    device = cspad_tbx.address_split(self.address)[2]
    if device == 'Andor':
      beam_center = (0, 0) # XXX Fiction!
      pixel_size = 13.5e-3 # XXX Should not be hardcoded here!
      saturated_value = 10000
    elif device == 'Cspad' or device == 'Cspad2x2':
      beam_center = self.beam_center
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
    elif device == 'marccd':
      beam_center = tuple(t // 2 for t in d['mean_img'].focus())
      pixel_size = 0.079346
      saturated_value = 2**16 - 1

    if stats['nmemb'] > 0:
      if self.avg_dirname  is not None or \
         self.avg_basename is not None or \
         self._mean_out    is not None:
        d = cspad_tbx.dpack(
          active_areas=self.active_areas,
          address=self.address,
          beam_center_x=pixel_size * beam_center[0],
          beam_center_y=pixel_size * beam_center[1],
          data=stats['mean_img'],
          distance=stats['distance'],
          pixel_size=pixel_size,
          saturated_value=saturated_value,
          timestamp=cspad_tbx.evt_timestamp(stats['time']),
          wavelength=stats['wavelength'])
        if self._mean_out is not None:
          p = cspad_tbx.dwritef2(d, self._mean_out)
        else:
          p = cspad_tbx.dwritef(d, self.avg_dirname, self.avg_basename)
        self.logger.info("Average written to %s" % p)

      if self.stddev_dirname  is not None or \
         self.stddev_basename is not None or \
         self._std_out    is not None:
        d = cspad_tbx.dpack(
          active_areas=self.active_areas,
          address=self.address,
          beam_center_x=pixel_size * beam_center[0],
          beam_center_y=pixel_size * beam_center[1],
          data=stats['std_img'],
          distance=stats['distance'],
          pixel_size=pixel_size,
          saturated_value=saturated_value,
          timestamp=cspad_tbx.evt_timestamp(stats['time']),
          wavelength=stats['wavelength'])
        if self._std_out is not None:
          p = cspad_tbx.dwritef2(d, self._std_out)
        else:
          p = cspad_tbx.dwritef(d, self.stddev_dirname, self.stddev_basename)
        self.logger.info("Standard deviation written to %s" % p)

      if self.max_dirname  is not None or \
         self.max_basename is not None or \
         self._max_out    is not None:
        d = cspad_tbx.dpack(
          active_areas=self.active_areas,
          address=self.address,
          beam_center_x=pixel_size * beam_center[0],
          beam_center_y=pixel_size * beam_center[1],
          data=stats['max_img'],
          distance=stats['distance'],
          pixel_size=pixel_size,
          saturated_value=saturated_value,
          timestamp=cspad_tbx.evt_timestamp(stats['time']),
          wavelength=stats['wavelength'])
        if self._max_out is not None:
          p = cspad_tbx.dwritef2(d, self._max_out)
        else:
          p = cspad_tbx.dwritef(d, self.max_dirname, self.max_basename)
        self.logger.info("Max written to %s" % p)

    if stats['nfail'] == 0:
      self.logger.info("%d images processed" % stats['nmemb'])
    else:
      self.logger.warning(
        "%d images processed, %d failed" % (stats['nmemb'], stats['nfail']))
示例#16
0
  def __init__(self,
               address,
               calib_dir=None,
               common_mode_correction="none",
               photon_threshold=None,
               two_photon_threshold=None,
               dark_path=None,
               dark_stddev=None,
               mask_path=None,
               gain_map_path=None,
               gain_map_level=None,
               cache_image=True,
               roi=None,
               laser_1_status=None,
               laser_4_status=None,
               laser_wait_time=None,
               override_beam_x=None,
               override_beam_y=None,
               bin_size=None,
               crop_rayonix=False,
               **kwds):
    """The common_mode_correction class constructor stores the
    parameters passed from the pyana configuration file in instance
    variables.

    @param address         Full data source address of the DAQ device
    @param calib_dir       Directory with calibration information
    @param common_mode_correction The type of common mode correction to apply
    @param dark_path       Path to input average dark image
    @param dark_stddev     Path to input standard deviation dark
                           image, required if @p dark_path is given
    @param mask_path       Path to input mask.  Pixels to mask out should be set to -2
    @param gain_map_path   Path to input gain map.  Multiplied times the image.
    @param gain_map_level  If set, all the '1' pixels in the gain_map are set to this multiplier
                           and all the '0' pixels in the gain_map are set to '1'.  If not set,
                           use the values in the gain_map directly
    @param laser_1_status  0 or 1 to indicate that the laser should be off or on respectively
    @param laser_4_status  0 or 1 to indicate that the laser should be off or on respectively
    @param laser_wait_time Length of time in milliseconds to wait after a laser
                           change of status to begin accepting images again.
                           (rejection of images occurs immediately after status
                           change).
    @param override_beam_x override value for x coordinate of beam center in pixels
    @param override_beam_y override value for y coordinate of beam center in pixels
    @param bin_size bin size for rayonix detector used to determin pixel size
    @param crop_rayonix    whether to crop rayonix images such that image center is the beam center
    """

    # Cannot use the super().__init__() construct here, because
    # common_mode_correction refers to the argument, and not the
    # class.
    mod_event_info.__init__(self, address=address, **kwds)

    # The paths will be substituted in beginjob(), where evt and env
    # are available.
    self._dark_path = cspad_tbx.getOptString(dark_path)
    self._dark_stddev_path = cspad_tbx.getOptString(dark_stddev)
    self._gain_map_path = cspad_tbx.getOptString(gain_map_path)
    self._mask_path = cspad_tbx.getOptString(mask_path)

    self.gain_map_level = cspad_tbx.getOptFloat(gain_map_level)
    self.common_mode_correction = cspad_tbx.getOptString(common_mode_correction)
    self.photon_threshold = cspad_tbx.getOptFloat(photon_threshold)
    self.two_photon_threshold = cspad_tbx.getOptFloat(two_photon_threshold)
    self.cache_image = cspad_tbx.getOptBool(cache_image)
    self.filter_laser_1_status = cspad_tbx.getOptInteger(laser_1_status)
    self.filter_laser_4_status = cspad_tbx.getOptInteger(laser_4_status)
    if self.filter_laser_1_status is not None:
      self.filter_laser_1_status = bool(self.filter_laser_1_status)
    if self.filter_laser_4_status is not None:
      self.filter_laser_4_status = bool(self.filter_laser_4_status)
    self.filter_laser_wait_time = cspad_tbx.getOptInteger(laser_wait_time)
    self.override_beam_x = cspad_tbx.getOptFloat(override_beam_x)
    self.override_beam_y = cspad_tbx.getOptFloat(override_beam_y)
    self.bin_size = cspad_tbx.getOptInteger(bin_size)
    self.crop_rayonix = cspad_tbx.getOptBool(crop_rayonix)

    self.cspad_img = None # The current image - set by self.event()
    self.sum_common_mode = 0
    self.sumsq_common_mode = 0
    self.roi = cspad_tbx.getOptROI(roi) # used to ignore the signal region in chebyshev fit

    assert self.common_mode_correction in \
        ("gaussian", "mean", "median", "mode", "none", "chebyshev")

    # Get and parse metrology.
    self.sections = None
    device = cspad_tbx.address_split(self.address)[2]
    if device == 'Andor':
      self.sections = [] # XXX FICTION
    elif device == 'Cspad':
      if self.address == 'XppGon-0|Cspad-0':
        self.sections = [] # Not used for XPP
      else:
        self.sections = calib2sections(cspad_tbx.getOptString(calib_dir))
    elif device == 'Cspad2x2':
      # There is no metrology information for the Sc1 detector, so
      # make it up.  The sections are rotated by 90 degrees with
      # respect to the "standing up" convention.
      self.sections = [[Section(90, (185 / 2 + 0,   (2 * 194 + 3) / 2)),
                        Section(90, (185 / 2 + 185, (2 * 194 + 3) / 2))]]
    elif device == 'marccd':
      self.sections = [] # XXX FICTION
    elif device == 'pnCCD':
      self.sections = [] # XXX FICTION
    elif device == 'Rayonix':
      self.sections = [] # XXX FICTION
    elif device == 'Opal1000':
      self.sections = [] # XXX FICTION
    if self.sections is None:
      raise RuntimeError("Failed to load metrology")
示例#17
0
def average(argv=None):
    if argv == None:
        argv = sys.argv[1:]

    try:
        from mpi4py import MPI
    except ImportError:
        raise Sorry("MPI not found")

    command_line = (libtbx.option_parser.option_parser(usage="""
%s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] [-g gain_mask_value] [--min] [--minpath minpath]

To write image pickles use -p, otherwise the program writes CSPAD CBFs.
Writing CBFs requires the geometry to be already deployed.

Examples:
cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571

Use one process on the current node to process all the events from run 25 of
experiment cxi49812, using a detz_offset of 571.

mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571

As above, using 16 cores on the current node.

bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812

As above, using the psanaq and 100 cores, putting the log in average.out and
the output images in the folder cxi49812.
""" % libtbx.env.dispatcher_name).option(
        None,
        "--as_pickle",
        "-p",
        action="store_true",
        default=False,
        dest="as_pickle",
        help="Write results as image pickle files instead of cbf files"
    ).option(
        None,
        "--raw_data",
        "-R",
        action="store_true",
        default=False,
        dest="raw_data",
        help=
        "Disable psana corrections such as dark pedestal subtraction or common mode (cbf only)"
    ).option(
        None,
        "--background_pickle",
        "-B",
        default=None,
        dest="background_pickle",
        help=""
    ).option(
        None,
        "--config",
        "-c",
        type="string",
        default=None,
        dest="config",
        metavar="PATH",
        help="psana config file"
    ).option(
        None,
        "--experiment",
        "-x",
        type="string",
        default=None,
        dest="experiment",
        help="experiment name (eg cxi84914)"
    ).option(
        None,
        "--run",
        "-r",
        type="int",
        default=None,
        dest="run",
        help="run number"
    ).option(
        None,
        "--address",
        "-a",
        type="string",
        default="CxiDs2.0:Cspad.0",
        dest="address",
        help="detector address name (eg CxiDs2.0:Cspad.0)"
    ).option(
        None,
        "--detz_offset",
        "-d",
        type="float",
        default=None,
        dest="detz_offset",
        help=
        "offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)"
    ).option(
        None,
        "--outputdir",
        "-o",
        type="string",
        default=".",
        dest="outputdir",
        metavar="PATH",
        help="Optional path to output directory for output files"
    ).option(
        None,
        "--averagebase",
        "-A",
        type="string",
        default="{experiment!l}_avg-r{run:04d}",
        dest="averagepath",
        metavar="PATH",
        help=
        "Path to output average image without extension. String substitution allowed"
    ).option(
        None,
        "--stddevbase",
        "-S",
        type="string",
        default="{experiment!l}_stddev-r{run:04d}",
        dest="stddevpath",
        metavar="PATH",
        help=
        "Path to output standard deviation image without extension. String substitution allowed"
    ).option(
        None,
        "--maxbase",
        "-M",
        type="string",
        default="{experiment!l}_max-r{run:04d}",
        dest="maxpath",
        metavar="PATH",
        help=
        "Path to output maximum projection image without extension. String substitution allowed"
    ).option(
        None,
        "--numevents",
        "-n",
        type="int",
        default=None,
        dest="numevents",
        help="Maximum number of events to process. Default: all"
    ).option(
        None,
        "--skipevents",
        "-s",
        type="int",
        default=0,
        dest="skipevents",
        help="Number of events in the beginning of the run to skip. Default: 0"
    ).option(
        None,
        "--verbose",
        "-v",
        action="store_true",
        default=False,
        dest="verbose",
        help="Print more information about progress"
    ).option(
        None,
        "--pickle-optical-metrology",
        "-m",
        action="store_true",
        default=False,
        dest="pickle_optical_metrology",
        help=
        "If writing pickle files, use the optical metrology in the experiment's calib directory"
    ).option(
        None,
        "--bin_size",
        "-b",
        type="int",
        default=None,
        dest="bin_size",
        help="Rayonix detector bin size"
    ).option(
        None,
        "--override_beam_x",
        "-X",
        type="float",
        default=None,
        dest="override_beam_x",
        help="Rayonix detector beam center x coordinate"
    ).option(
        None,
        "--override_beam_y",
        "-Y",
        type="float",
        default=None,
        dest="override_beam_y",
        help="Rayonix detector beam center y coordinate"
    ).option(
        None,
        "--calib_dir",
        "-C",
        type="string",
        default=None,
        dest="calib_dir",
        metavar="PATH",
        help="calibration directory"
    ).option(
        None,
        "--pickle_calib_dir",
        "-P",
        type="string",
        default=None,
        dest="pickle_calib_dir",
        metavar="PATH",
        help=
        "pickle calibration directory specification. Replaces --calib_dir functionality."
    ).option(
        None,
        "--xtc_dir",
        "-D",
        type="string",
        default=None,
        dest="xtc_dir",
        metavar="PATH",
        help="xtc stream directory"
    ).option(
        None,
        "--use_ffb",
        "-f",
        action="store_true",
        default=False,
        dest="use_ffb",
        help=
        "Use the fast feedback filesystem at LCLS. Only for the active experiment!"
    ).option(
        None,
        "--gain_mask_value",
        "-g",
        type="float",
        default=None,
        dest="gain_mask_value",
        help=
        "Ratio between low and high gain pixels, if CSPAD in mixed-gain mode. Only used in CBF averaging mode."
    ).option(
        None,
        "--min",
        None,
        action="store_true",
        default=False,
        dest="do_minimum_projection",
        help="Output a minimum projection"
    ).option(
        None,
        "--minpath",
        None,
        type="string",
        default="{experiment!l}_min-r{run:04d}",
        dest="minpath",
        metavar="PATH",
        help=
        "Path to output minimum image without extension. String substitution allowed"
    )).process(args=argv)


    if len(command_line.args) > 0 or \
        command_line.options.as_pickle is None or \
        command_line.options.experiment is None or \
        command_line.options.run is None or \
        command_line.options.address is None or \
        command_line.options.detz_offset is None or \
        command_line.options.averagepath is None or \
        command_line.options.stddevpath is None or \
        command_line.options.maxpath is None or \
        command_line.options.pickle_optical_metrology is None:
        command_line.parser.show_help()
        return

    # set this to sys.maxint to analyze all events
    if command_line.options.numevents is None:
        maxevents = sys.maxsize
    else:
        maxevents = command_line.options.numevents

    comm = MPI.COMM_WORLD
    rank = comm.Get_rank()
    size = comm.Get_size()

    if command_line.options.config is not None:
        psana.setConfigFile(command_line.options.config)
    dataset_name = "exp=%s:run=%d:smd" % (command_line.options.experiment,
                                          command_line.options.run)
    if command_line.options.xtc_dir is not None:
        if command_line.options.use_ffb:
            raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system")
        dataset_name += ":dir=%s" % command_line.options.xtc_dir
    elif command_line.options.use_ffb:
        # as ffb is only at SLAC, ok to hardcode /reg/d here
        dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc" % (
            command_line.options.experiment[0:3],
            command_line.options.experiment)
    if command_line.options.calib_dir is not None:
        psana.setOption('psana.calib-dir', command_line.options.calib_dir)
    ds = psana.DataSource(dataset_name)
    address = command_line.options.address
    src = psana.Source('DetInfo(%s)' % address)
    nevent = np.array([0.])

    if command_line.options.background_pickle is not None:
        background = easy_pickle.load(
            command_line.options.background_pickle)['DATA'].as_numpy_array()

    for run in ds.runs():
        runnumber = run.run()

        if not command_line.options.as_pickle:
            psana_det = psana.Detector(address, ds.env())

        # list of all events
        if command_line.options.skipevents > 0:
            print("Skipping first %d events" % command_line.options.skipevents)
        elif "Rayonix" in command_line.options.address:
            print("Skipping first image in the Rayonix detector"
                  )  # Shuttering issue
            command_line.options.skipevents = 1

        for i, evt in enumerate(run.events()):
            if i % size != rank: continue
            if i < command_line.options.skipevents: continue
            if i >= maxevents: break
            if i % 10 == 0: print('Rank', rank, 'processing event', i)
            #print "Event #",rank*mylength+i," has id:",evt.get(EventId)
            if 'Rayonix' in command_line.options.address or 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
                data = evt.get(psana.Camera.FrameV1, src)
                if data is None:
                    print("No data")
                    continue
                data = data.data16().astype(np.float64)
            elif command_line.options.as_pickle:
                data = evt.get(psana.ndarray_float64_3, src, 'image0')
            else:
                # get numpy array, 32x185x388
                from xfel.cftbx.detector.cspad_cbf_tbx import get_psana_corrected_data
                if command_line.options.raw_data:
                    data = get_psana_corrected_data(psana_det,
                                                    evt,
                                                    use_default=False,
                                                    dark=False,
                                                    common_mode=None,
                                                    apply_gain_mask=False,
                                                    per_pixel_gain=False)
                else:
                    if command_line.options.gain_mask_value is None:
                        data = get_psana_corrected_data(psana_det,
                                                        evt,
                                                        use_default=True)
                    else:
                        data = get_psana_corrected_data(
                            psana_det,
                            evt,
                            use_default=False,
                            dark=True,
                            common_mode=None,
                            apply_gain_mask=True,
                            gain_mask_value=command_line.options.
                            gain_mask_value,
                            per_pixel_gain=False)

            if data is None:
                print("No data")
                continue

            if command_line.options.background_pickle is not None:
                data -= background

            if 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
                distance = np.array([0.0])
                wavelength = np.array([1.0])
            else:
                d = cspad_tbx.env_distance(address, run.env(),
                                           command_line.options.detz_offset)
                if d is None:
                    print("No distance, using distance",
                          command_line.options.detz_offset)
                    assert command_line.options.detz_offset is not None
                    if 'distance' not in locals():
                        distance = np.array([command_line.options.detz_offset])
                    else:
                        distance += command_line.options.detz_offset
                else:
                    if 'distance' in locals():
                        distance += d
                    else:
                        distance = np.array([float(d)])

                w = cspad_tbx.evt_wavelength(evt)
                if w is None:
                    print("No wavelength")
                    if 'wavelength' not in locals():
                        wavelength = np.array([1.0])
                else:
                    if 'wavelength' in locals():
                        wavelength += w
                    else:
                        wavelength = np.array([w])

            t = cspad_tbx.evt_time(evt)
            if t is None:
                print("No timestamp, skipping shot")
                continue
            if 'timestamp' in locals():
                timestamp += t[0] + (t[1] / 1000)
            else:
                timestamp = np.array([t[0] + (t[1] / 1000)])

            if 'sum' in locals():
                sum += data
            else:
                sum = np.array(data, copy=True)
            if 'sumsq' in locals():
                sumsq += data * data
            else:
                sumsq = data * data
            if 'maximum' in locals():
                maximum = np.maximum(maximum, data)
            else:
                maximum = np.array(data, copy=True)

            if command_line.options.do_minimum_projection:
                if 'minimum' in locals():
                    minimum = np.minimum(minimum, data)
                else:
                    minimum = np.array(data, copy=True)

            nevent += 1

    #sum the images across mpi cores
    if size > 1:
        print("Synchronizing rank", rank)
    totevent = np.zeros(nevent.shape)
    comm.Reduce(nevent, totevent)

    if rank == 0 and totevent[0] == 0:
        raise Sorry("No events found in the run")

    sumall = np.zeros(sum.shape).astype(sum.dtype)
    comm.Reduce(sum, sumall)

    sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype)
    comm.Reduce(sumsq, sumsqall)

    maxall = np.zeros(maximum.shape).astype(maximum.dtype)
    comm.Reduce(maximum, maxall, op=MPI.MAX)

    if command_line.options.do_minimum_projection:
        minall = np.zeros(maximum.shape).astype(minimum.dtype)
        comm.Reduce(minimum, minall, op=MPI.MIN)

    waveall = np.zeros(wavelength.shape).astype(wavelength.dtype)
    comm.Reduce(wavelength, waveall)

    distall = np.zeros(distance.shape).astype(distance.dtype)
    comm.Reduce(distance, distall)

    timeall = np.zeros(timestamp.shape).astype(timestamp.dtype)
    comm.Reduce(timestamp, timeall)

    if rank == 0:
        if size > 1:
            print("Synchronized")

        # Accumulating floating-point numbers introduces errors,
        # which may cause negative variances.  Since a two-pass
        # approach is unacceptable, the standard deviation is
        # clamped at zero.
        mean = sumall / float(totevent[0])
        variance = (sumsqall / float(totevent[0])) - (mean**2)
        variance[variance < 0] = 0
        stddev = np.sqrt(variance)

        wavelength = waveall[0] / totevent[0]
        distance = distall[0] / totevent[0]
        pixel_size = cspad_tbx.pixel_size
        saturated_value = cspad_tbx.cspad_saturated_value
        timestamp = timeall[0] / totevent[0]
        timestamp = (int(timestamp), timestamp % int(timestamp) * 1000)
        timestamp = cspad_tbx.evt_timestamp(timestamp)

        if command_line.options.as_pickle:
            extension = ".pickle"
        else:
            extension = ".cbf"

        dest_paths = [
            cspad_tbx.pathsubst(command_line.options.averagepath + extension,
                                evt, ds.env()),
            cspad_tbx.pathsubst(command_line.options.stddevpath + extension,
                                evt, ds.env()),
            cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt,
                                ds.env())
        ]
        if command_line.options.do_minimum_projection:
            dest_paths.append(
                cspad_tbx.pathsubst(command_line.options.minpath + extension,
                                    evt, ds.env()))

        dest_paths = [
            os.path.join(command_line.options.outputdir, path)
            for path in dest_paths
        ]
        if 'Rayonix' in command_line.options.address:
            all_data = [mean, stddev, maxall]
            if command_line.options.do_minimum_projection:
                all_data.append(minall)
            from xfel.cxi.cspad_ana import rayonix_tbx
            pixel_size = rayonix_tbx.get_rayonix_pixel_size(
                command_line.options.bin_size)
            beam_center = [
                command_line.options.override_beam_x,
                command_line.options.override_beam_y
            ]
            active_areas = flex.int([0, 0, mean.shape[1], mean.shape[0]])
            split_address = cspad_tbx.address_split(address)
            old_style_address = split_address[0] + "-" + split_address[
                1] + "|" + split_address[2] + "-" + split_address[3]
            for data, path in zip(all_data, dest_paths):
                print("Saving", path)
                d = cspad_tbx.dpack(
                    active_areas=active_areas,
                    address=old_style_address,
                    beam_center_x=pixel_size * beam_center[0],
                    beam_center_y=pixel_size * beam_center[1],
                    data=flex.double(data),
                    distance=distance,
                    pixel_size=pixel_size,
                    saturated_value=rayonix_tbx.rayonix_saturated_value,
                    timestamp=timestamp,
                    wavelength=wavelength)
                easy_pickle.dump(path, d)
        elif 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
            all_data = [mean, stddev, maxall]
            split_address = cspad_tbx.address_split(address)
            old_style_address = split_address[0] + "-" + split_address[
                1] + "|" + split_address[2] + "-" + split_address[3]
            if command_line.options.do_minimum_projection:
                all_data.append(minall)
            for data, path in zip(all_data, dest_paths):
                d = cspad_tbx.dpack(address=old_style_address,
                                    data=flex.double(data),
                                    distance=distance,
                                    pixel_size=0.1,
                                    timestamp=timestamp,
                                    wavelength=wavelength)
                print("Saving", path)
                easy_pickle.dump(path, d)
        elif command_line.options.as_pickle:
            split_address = cspad_tbx.address_split(address)
            old_style_address = split_address[0] + "-" + split_address[
                1] + "|" + split_address[2] + "-" + split_address[3]

            xpp = 'xpp' in address.lower()
            if xpp:
                evt_time = cspad_tbx.evt_time(
                    evt)  # tuple of seconds, milliseconds
                timestamp = cspad_tbx.evt_timestamp(
                    evt_time)  # human readable format
                from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
                from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
                version_lookup = detector_format_version(
                    old_style_address,
                    reverse_timestamp(timestamp)[0])
                assert version_lookup is not None
                active_areas = xpp_active_areas[version_lookup]['active_areas']
                beam_center = [1765 // 2, 1765 // 2]
            else:
                if command_line.options.pickle_calib_dir is not None:
                    metro_path = command_line.options.pickle_calib_dir
                elif command_line.options.pickle_optical_metrology:
                    from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path
                    metro_path = get_calib_file_path(run.env(), address, run)
                else:
                    metro_path = libtbx.env.find_in_repositories(
                        "xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
                sections = parse_calib.calib2sections(metro_path)
                beam_center, active_areas = cspad_tbx.cbcaa(
                    cspad_tbx.getConfig(address, ds.env()), sections)

            class fake_quad(object):
                def __init__(self, q, d):
                    self.q = q
                    self.d = d

                def quad(self):
                    return self.q

                def data(self):
                    return self.d

            if xpp:
                quads = [
                    fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
                    for i in range(4)
                ]
                mean = cspad_tbx.image_xpp(old_style_address,
                                           None,
                                           ds.env(),
                                           active_areas,
                                           quads=quads)
                mean = flex.double(mean.astype(np.float64))

                quads = [
                    fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
                    for i in range(4)
                ]
                stddev = cspad_tbx.image_xpp(old_style_address,
                                             None,
                                             ds.env(),
                                             active_areas,
                                             quads=quads)
                stddev = flex.double(stddev.astype(np.float64))

                quads = [
                    fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
                    for i in range(4)
                ]
                maxall = cspad_tbx.image_xpp(old_style_address,
                                             None,
                                             ds.env(),
                                             active_areas,
                                             quads=quads)
                maxall = flex.double(maxall.astype(np.float64))

                if command_line.options.do_minimum_projection:
                    quads = [
                        fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
                        for i in range(4)
                    ]
                    minall = cspad_tbx.image_xpp(old_style_address,
                                                 None,
                                                 ds.env(),
                                                 active_areas,
                                                 quads=quads)
                    minall = flex.double(minall.astype(np.float64))
            else:
                quads = [
                    fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
                    for i in range(4)
                ]
                mean = cspad_tbx.CsPadDetector(address,
                                               evt,
                                               ds.env(),
                                               sections,
                                               quads=quads)
                mean = flex.double(mean.astype(np.float64))

                quads = [
                    fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
                    for i in range(4)
                ]
                stddev = cspad_tbx.CsPadDetector(address,
                                                 evt,
                                                 ds.env(),
                                                 sections,
                                                 quads=quads)
                stddev = flex.double(stddev.astype(np.float64))

                quads = [
                    fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
                    for i in range(4)
                ]
                maxall = cspad_tbx.CsPadDetector(address,
                                                 evt,
                                                 ds.env(),
                                                 sections,
                                                 quads=quads)
                maxall = flex.double(maxall.astype(np.float64))

                if command_line.options.do_minimum_projection:
                    quads = [
                        fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
                        for i in range(4)
                    ]
                    minall = cspad_tbx.CsPadDetector(address,
                                                     evt,
                                                     ds.env(),
                                                     sections,
                                                     quads=quads)
                    minall = flex.double(minall.astype(np.float64))

            all_data = [mean, stddev, maxall]
            if command_line.options.do_minimum_projection:
                all_data.append(minall)

            for data, path in zip(all_data, dest_paths):
                print("Saving", path)

                d = cspad_tbx.dpack(active_areas=active_areas,
                                    address=old_style_address,
                                    beam_center_x=pixel_size * beam_center[0],
                                    beam_center_y=pixel_size * beam_center[1],
                                    data=data,
                                    distance=distance,
                                    pixel_size=pixel_size,
                                    saturated_value=saturated_value,
                                    timestamp=timestamp,
                                    wavelength=wavelength)

                easy_pickle.dump(path, d)
        else:
            # load a header only cspad cbf from the slac metrology
            from xfel.cftbx.detector import cspad_cbf_tbx
            import pycbf
            base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(
                run, address)
            if base_dxtbx is None:
                raise Sorry("Couldn't load calibration file for run %d" %
                            run.run())

            all_data = [mean, stddev, maxall]
            if command_line.options.do_minimum_projection:
                all_data.append(minall)

            for data, path in zip(all_data, dest_paths):
                print("Saving", path)
                cspad_img = cspad_cbf_tbx.format_object_from_data(
                    base_dxtbx,
                    data,
                    distance,
                    wavelength,
                    timestamp,
                    address,
                    round_to_int=False)
                cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
                  pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
示例#18
0
  def __init__(self,
               address,
               avg_dirname=None,
               avg_basename=None,
               stddev_dirname=None,
               stddev_basename=None,
               max_dirname=None,
               max_basename=None,
               background_path=None,
               flags=None,
               hot_threshold=None,
               gain_threshold=None,
               noise_threshold=7,
               elastic_threshold=9,
               symnoise_threshold=4,
               **kwds):
    """
    @param address         Full data source address of the DAQ device
    @param avg_dirname     Directory portion of output average image
                           XXX mean
    @param avg_basename    Filename prefix of output average image XXX
                           mean
    @param flags inactive:  Eliminate the inactive pixels
                 noelastic: Eliminate elastic scattering
                 nohot:     Eliminate the hot pixels
                 nonoise:   Eliminate noisy pixels
                 symnoise:  Symmetrically eliminate noisy pixels
    @param stddev_dirname  Directory portion of output standard
                           deviation image XXX std
    @param stddev_basename Filename prefix of output standard
                           deviation image XXX std
    @param max_dirname     Directory portion of output maximum
                           projection image
    @param max_basename    Filename prefix of output maximum
                           projection image
     """

    super(average_mixin, self).__init__(
      address=address,
      **kwds
    )
    self.roi = None
    self.avg_basename = cspad_tbx.getOptString(avg_basename)
    self.avg_dirname = cspad_tbx.getOptString(avg_dirname)
    self.detector = cspad_tbx.address_split(address)[0]
    self.flags = cspad_tbx.getOptStrings(flags, default = [])
    self.stddev_basename = cspad_tbx.getOptString(stddev_basename)
    self.stddev_dirname = cspad_tbx.getOptString(stddev_dirname)
    self.max_basename = cspad_tbx.getOptString(max_basename)
    self.max_dirname = cspad_tbx.getOptString(max_dirname)
    self.background_path = cspad_tbx.getOptString(background_path)
    self.hot_threshold = cspad_tbx.getOptFloat(hot_threshold)
    self.gain_threshold = cspad_tbx.getOptFloat(gain_threshold)
    self.noise_threshold = cspad_tbx.getOptFloat(noise_threshold)
    self.elastic_threshold = cspad_tbx.getOptFloat(elastic_threshold)
    self.symnoise_threshold = cspad_tbx.getOptFloat(symnoise_threshold)

    if background_path is not None:
      background_dict = easy_pickle.load(background_path)
      self.background_img = background_dict['DATA']

    self._have_max = self.max_basename is not None or \
                     self.max_dirname is not None
    self._have_mean = self.avg_basename is not None or \
                      self.avg_dirname is not None
    self._have_std = self.stddev_basename is not None or \
                     self.stddev_dirname is not None

    # Start a server process which holds a set of Python objects that
    # other processes can manipulate using proxies.  The queues will
    # be used in endjob() to pass images between the worker processes,
    # and the lock will ensure the transfer is treated as a critical
    # section.  There is therefore the risk of a hang if the queues
    # cannot hold all the data one process will supply before another
    # empties it.
    #
    # In an attempt to alleviate this issue, separate queues are used
    # for the potentially big images.  The hope is to prevent
    # producers from blocking while consumers are locked out by using
    # more buffers.
    mgr = multiprocessing.Manager()
    self._lock = mgr.Lock()
    self._metadata = mgr.dict()
    self._queue_max = mgr.Queue()
    self._queue_sum = mgr.Queue()
    self._queue_ssq = mgr.Queue()
示例#19
0
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.
    XXX more?

    Previously, common-mode correction was applied only after initial
    threshold filtering.  Since the common_mode class applies the
    (lengthy) common-mode correction immediately after reading the
    image from the stream, this optimisation is currently not
    (elegantly) doable.

    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(mod_hitfind, self).event(evt, env)
    if (evt.get("skip_event")):
      return

    # This module only applies to detectors for which a distance is
    # available.
    distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
    if distance is None:
      self.nfail += 1
      self.logger.warning("event(): no distance, shot skipped")
      evt.put(skip_event_flag(), "skip_event")
      return

    device = cspad_tbx.address_split(self.address)[2]

    # ***** HITFINDING ***** XXX For hitfinding it may be interesting
    # to look at the fraction of subzero pixels in the dark-corrected
    # image.
    if (self.m_threshold is not None):
      # If a threshold value is given it can be applied in one of three ways:
      #    1.  Apply it over the whole image
      if (self.m_roi is None and self.m_distl_min_peaks is None):
        vmax = flex.max(self.cspad_img)
        if (vmax < self.m_threshold):
          if not self.m_negate_hits:
            # Tell downstream modules to skip this event if the threshold was not met.
            evt.put(skip_event_flag(), "skip_event")
            return
        elif self.m_negate_hits:
          evt.put(skip_event_flag(), "skip_event")
          return

      #    2. Apply threshold over a rectangular region of interest.
      elif (self.m_roi is not None):
        vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
                                       self.m_roi[0]:self.m_roi[1]])
        if (vmax < self.m_threshold):
          if not self.m_negate_hits:
            evt.put(skip_event_flag(), "skip_event")
            return
        elif self.m_negate_hits:
          evt.put(skip_event_flag(), "skip_event")
          return

      #    3. Determine the spotfinder spots within the central ASICS, and accept the
      #       image as a hit if there are m_distl_min_peaks exceeding m_threshold.
      #       As a further requirement, the peaks must exceed 2.5 * the 90-percentile
      #       pixel value of the central ASICS.  This filter was added to avoid high-background
      #       false positives.
      elif (self.m_distl_min_peaks is not None):
        if device == 'marccd':
          self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
          self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
        elif device == 'Rayonix':
          self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
          self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]

        peak_heights,outvalue = self.distl_filter(
          self.address,
          self.cspad_img.iround(), # XXX correct?
          distance,
          self.timestamp,
          self.wavelength)
        if ('permissive' in self.m_distl_flags):
          number_of_accepted_peaks = (peak_heights > self.m_threshold).count(True)
        else:
          number_of_accepted_peaks = ((peak_heights > self.m_threshold).__and__(outvalue==0)).count(True)

        sec,ms = cspad_tbx.evt_time(evt)
        evt_time = sec + ms/1000
        self.stats_logger.info("BRAGG %.3f %d" %(evt_time, number_of_accepted_peaks))

        skip_event = False
        if number_of_accepted_peaks < self.m_distl_min_peaks:
          self.logger.info("Subprocess %02d: Spotfinder NO  HIT image #%05d @ %s; %d spots > %d" %(
            env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))

          if not self.m_negate_hits:
            skip_event = True
        else:
          self.logger.info("Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d" %(
            env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))

          if self.m_negate_hits:
            skip_event = True

        if skip_event:
          if self.m_db_logging:
            # log misses to the database
            self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, number_of_accepted_peaks, distance,
                              self.sifoil, self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, self.m_db_tags))
          evt.put(skip_event_flag(), "skip_event")
          return
        # the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
        # hit will not be logged in the db.
        evt.put(number_of_accepted_peaks, 'sfspots')

    self.logger.info("Subprocess %02d: process image #%05d @ %s" %
                     (env.subprocess(), self.nshots, self.timestamp))

    # See r17537 of mod_average.py.
    if device == 'Cspad':
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
    elif device == 'marccd':
      pixel_size = evt.get("marccd_pixel_size")
      saturated_value = evt.get("marccd_saturated_value")
    elif device == 'Rayonix':
      pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
      saturated_value = rayonix_tbx.rayonix_saturated_value

    d = cspad_tbx.dpack(
      active_areas=self.active_areas,
      address=self.address,
      beam_center_x=pixel_size * self.beam_center[0],
      beam_center_y=pixel_size * self.beam_center[1],
      data=self.cspad_img.iround(), # XXX ouch!
      distance=distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=self.timestamp,
      wavelength=self.wavelength,
      xtal_target=self.m_xtal_target)

    if (self.m_dispatch == "index"):
      import sys
      from xfel.cxi.integrate_image_api import integrate_one_image
      info = integrate_one_image(d,
                                 integration_dirname  = self.m_integration_dirname,
                                 integration_basename = self.m_integration_basename)
      sys.stdout = sys.__stdout__
      sys.stderr = sys.__stderr__

      indexed = info is not None
      if indexed and self.m_progress_logging:
        # integration pickle dictionary is available here as info.last_saved_best
        if info.last_saved_best["identified_isoform"] is not None:
          #print info.last_saved_best.keys()
          from cxi_xdr_xes.cftbx.cspad_ana import db
          dbobj = db.dbconnect(self.m_db_host, self.m_db_name, self.m_db_user, self.m_db_password)
          cursor = dbobj.cursor()
          if info.last_saved_best["identified_isoform"] in self.isoforms:
            PM, indices, miller_id = self.isoforms[info.last_saved_best["identified_isoform"]]
          else:
            from xfel.xpp.progress_support import progress_manager
            PM = progress_manager(info.last_saved_best,self.m_db_experiment_tag, self.m_trial_id, self.m_rungroup_id, evt.run())
            indices, miller_id = PM.get_HKL(cursor)
            # cache these as they don't change for a given isoform
            self.isoforms[info.last_saved_best["identified_isoform"]] = PM, indices, miller_id
          if self.m_sql_buffer_size > 1:
            self.queue_progress_entry(PM.scale_frame_detail(self.timestamp,cursor,do_inserts=False))
          else:
            PM.scale_frame_detail(self.timestamp,cursor,do_inserts=True)
            dbobj.commit()
            cursor.close()
            dbobj.close()

      if self.m_db_logging:
        sec,ms = cspad_tbx.evt_time(evt)
        evt_time = sec + ms/1000
        sfspots = evt.get('sfspots')
        if sfspots is None:
          if indexed:
            n_spots = len(info.spotfinder_results.images[info.frames[0]]['spots_total'])
          else:
            n_spots = 0
        else:
          n_spots = sfspots

        if indexed:
          mosaic_bloc_rotation = info.last_saved_best.get('ML_half_mosaicity_deg', [0])[0]
          mosaic_block_size = info.last_saved_best.get('ML_domain_size_ang', [0])[0]
          ewald_proximal_volume = info.last_saved_best.get('ewald_proximal_volume', [0])[0]

          obs = info.last_saved_best['observations'][0]
          cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell().parameters()
          pointgroup = info.last_saved_best['pointgroup']
          resolution = obs.d_min()
        else:
          mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
            cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0

        self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, n_spots, distance,
                          self.sifoil, self.wavelength, indexed, mosaic_bloc_rotation,
                          mosaic_block_size, ewald_proximal_volume, pointgroup, cell_a,
                          cell_b, cell_c, cell_alpha, cell_beta, cell_gamma, resolution,
                          self.m_db_tags))

      if (not indexed):
        evt.put(skip_event_flag(), "skip_event")
        return

    elif (self.m_dispatch == "nop"):
      pass

    elif (self.m_dispatch == "view"): #interactive image viewer

      args = ["indexing.data=dummy"]
      detector_format_version = detector_format_function(
        self.address, evt.GetTime())
      if detector_format_version is not None:
        args += ["distl.detector_format_version=%" % detector_format_version]

      from xfel.phil_preferences import load_cxi_phil
      horizons_phil = load_cxi_phil(self.m_xtal_target, args)
      horizons_phil.indexing.data = d

      from xfel.cxi import display_spots
      display_spots.parameters.horizons_phil = horizons_phil
      display_spots.wrapper_of_callback().display(horizons_phil.indexing.data)

    elif (self.m_dispatch == "spots"): #interactive spotfinder viewer

      args = ["indexing.data=dummy"]
      detector_format_version = detector_format_function(
        self.address, evt.GetTime())
      if detector_format_version is not None:
        args += ["distl.detector_format_version=%s" % detector_format_version]

      from xfel.phil_preferences import load_cxi_phil
      horizons_phil = load_cxi_phil(self.m_xtal_target, args)
      horizons_phil.indexing.data = d

      from xfel.cxi import display_spots
      display_spots.parameters.horizons_phil = horizons_phil

      from rstbx.new_horizons.index import pre_indexing_validation,pack_names
      pre_indexing_validation(horizons_phil)
      imagefile_arguments = pack_names(horizons_phil)
      horizons_phil.persist.show()
      from spotfinder.applications import signal_strength
      info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments)

      work = display_spots.wrapper_of_callback(info)
      work.display_with_callback(horizons_phil.indexing.data)

    elif (self.m_dispatch == "write_dict"):
      self.logger.warning(
        "event(): deprecated dispatch 'write_dict', use mod_dump instead")
      if (self.m_out_dirname  is not None or
          self.m_out_basename is not None):
        cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)

    # Diagnostic message emitted only when all the processing is done.
    if (env.subprocess() >= 0):
      self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
                       (env.subprocess(), self.nshots, self.timestamp))
    else:
      self.logger.info("Accepted #%05d @ %s" %
                       (self.nshots, self.timestamp))
示例#20
0
    def event(self, evt, env):
        """The event() function is called for every L1Accept transition.

    @param evt Event data object, a configure object
    @param env Environment object
    """
        super(common_mode_correction, self).event(evt, env)
        if (evt.get("skip_event")):
            return

        if not hasattr(self, 'active_areas') or self.active_areas is None or \
           not hasattr(self, 'beam_center')  or self.beam_center  is None:
            if self.address == 'XppGon-0|marccd-0':
                # The mod_mar module needs to have been called before this one
                # to set this up.  The MAR does not have a configure object.
                self.beam_center = evt.get("marccd_beam_center")
                self.active_areas = evt.get("marccd_active_areas")
            elif self.address == 'XppEndstation-0|Rayonix-0' or \
                 self.address == 'MfxEndstation-0|Rayonix-0':
                pass  # bc and aa set in the beginjob function
            elif self.address == 'XppGon-0|Cspad-0':
                # Load the active areas as determined from the optical metrology
                from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
                from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
                version_lookup = detector_format_version(
                    self.address,
                    reverse_timestamp(self.timestamp)[0])
                assert version_lookup is not None
                self.active_areas = xpp_active_areas[version_lookup][
                    'active_areas']
                self.beam_center = [1765 // 2, 1765 // 2]
            else:
                (self.beam_center, self.active_areas) = \
                  cspad_tbx.cbcaa(cspad_tbx.getConfig(self.address, env), self.sections)

        if self.filter_laser_1_status is not None:
            if (self.laser_1_status.status != self.filter_laser_1_status or
                (self.laser_1_ms_since_change is not None and
                 self.laser_1_ms_since_change < self.filter_laser_wait_time)):
                evt.put(skip_event_flag(), "skip_event")
                return
        if self.filter_laser_4_status is not None:
            if (self.laser_4_status.status != self.filter_laser_4_status or
                (self.laser_4_ms_since_change is not None and
                 self.laser_4_ms_since_change < self.filter_laser_wait_time)):
                evt.put(skip_event_flag(), "skip_event")
                return

        # Early return if the full detector image is already stored in the
        # event.  Otherwise, get it from the stream as a double-precision
        # floating-point flex array.  XXX It is probably not safe to key
        # the image on self.address, so we should come up with our own
        # namespace.  XXX Misnomer--could be CAMP, too

        self.cspad_img = evt.get(self.address)
        if self.cspad_img is not None:
            return
        if self.address == 'XppGon-0|Cspad-0':
            # Kludge until cspad_tbx.image() can be rewritten to handle the
            # XPP metrology.
            self.cspad_img = cspad_tbx.image_xpp(self.address, evt, env,
                                                 self.active_areas)
        elif self.address == 'XppEndstation-0|Rayonix-0' or \
             self.address == 'MfxEndstation-0|Rayonix-0':
            from psana import Source, Camera
            import numpy as np
            address = cspad_tbx.old_address_to_new_address(self.address)
            src = Source('DetInfo(%s)' % address)
            self.cspad_img = evt.get(Camera.FrameV1, src)
            if self.cspad_img is not None:
                self.cspad_img = self.cspad_img.data16().astype(np.float64)
        elif self.address == 'CxiDg3-0|Opal1000-0':
            if evt.getFrameValue(self.address) is not None:
                self.cspad_img = evt.getFrameValue(self.address).data()
        elif self.address == 'CxiEndstation-0|Opal1000-2':
            if evt.getFrameValue(self.address) is not None:
                self.cspad_img = evt.getFrameValue(self.address).data()
        elif self.address == 'FeeHxSpectrometer-0|Opal1000-1':
            if evt.getFrameValue(self.address) is not None:
                self.cspad_img = evt.getFrameValue(self.address).data()
        elif self.address == 'NoDetector-0|Cspad2x2-0':
            import numpy as np
            from pypdsdata import xtc
            test = []
            self.cspad_img = evt.get(xtc.TypeId.Type.Id_Cspad2x2Element,
                                     self.address).data()
            self.cspad_img = np.reshape(self.cspad_img, (370, 388))
        else:
            try:
                self.cspad_img = cspad_tbx.image(
                    self.address, cspad_tbx.getConfig(self.address, env), evt,
                    env, self.sections)
            except Exception as e:
                self.logger.error("Error reading image data: " + str(e))
                evt.put(skip_event_flag(), "skip_event")
                return

        if self.cspad_img is None:
            if cspad_tbx.address_split(self.address)[2] != 'Andor':
                self.nfail += 1
                self.logger.warning("event(): no image, shot skipped")
                evt.put(skip_event_flag(), "skip_event")
            return
        self.cspad_img = flex.double(self.cspad_img.astype(numpy.float64))
        # If a dark image was provided, subtract it from the image.  There
        # is no point in doing common-mode correction unless the dark
        # image was subtracted.
        if (self.dark_img is not None):
            self.cspad_img -= self.dark_img

            if (self.common_mode_correction != "none"):
                # Mask out inactive pixels prior to common mode correction.
                # Pixels are marked as inactive either due to low ADU values
                # or non-positive standard deviations in dark image.  XXX Make
                # the threshold tunable?
                cspad_mask = self.dark_mask.deep_copy()

                if self.roi is not None and self.common_mode_correction == "chebyshev":
                    roi_mask = cspad_mask[self.roi[2]:self.roi[3], :]
                    roi_mask = flex.bool(roi_mask.accessor(), False)
                    cspad_mask.matrix_paste_block_in_place(block=roi_mask,
                                                           i_row=self.roi[2],
                                                           i_column=0)

                # Extract each active section from the assembled detector
                # image and apply the common mode correction.  XXX Make up a
                # quadrant mask for the emission detector.  Needs to be
                # checked!
                config = cspad_tbx.getConfig(self.address, env)
                if len(self.sections) == 1:
                    q_mask = 1
                else:
                    q_mask = config.quadMask()
                for q in range(len(self.sections)):
                    if (not ((1 << q) & q_mask)):
                        continue

                    # XXX Make up section mask for the emission detector.  Needs
                    # to be checked!
                    import _pdsdata
                    if len(self.sections) == 1 and type(config) in (
                            _pdsdata.cspad2x2.ConfigV1,
                            _pdsdata.cspad2x2.ConfigV2):
                        s_mask = config.roiMask()
                    else:
                        s_mask = config.roiMask(q)
                    for s in range(len(self.sections[q])):
                        # XXX DAQ misconfiguration?  This mask appears not to work
                        # reliably for the Sc1 detector.
                        #            if (not((1 << s) & s_mask)):
                        #              continue
                        corners = self.sections[q][s].corners()
                        i_row = int(round(min(c[0] for c in corners)))
                        i_column = int(round(min(c[1] for c in corners)))
                        n_rows = int(round(max(c[0] for c in corners))) - i_row
                        n_columns = int(round(max(
                            c[1] for c in corners))) - i_column

                        section_img = self.cspad_img.matrix_copy_block(
                            i_row=i_row,
                            i_column=i_column,
                            n_rows=n_rows,
                            n_columns=n_columns)
                        section_mask = cspad_mask.matrix_copy_block(
                            i_row=i_row,
                            i_column=i_column,
                            n_rows=n_rows,
                            n_columns=n_columns)
                        section_stddev = self.dark_stddev.matrix_copy_block(
                            i_row=i_row,
                            i_column=i_column,
                            n_rows=n_rows,
                            n_columns=n_columns)

                        if section_mask.count(True) == 0: continue

                        if self.common_mode_correction == "chebyshev":
                            assert len(self.sections[q]) == 2
                            if s == 0:
                                section_imgs = [section_img]
                                section_masks = [section_mask]
                                i_rows = [i_row]
                                i_columns = [i_column]
                                continue
                            else:
                                section_imgs.append(section_img)
                                section_masks.append(section_mask)
                                i_rows.append(i_row)
                                i_columns.append(i_column)

                                chebyshev_corrected_imgs = self.chebyshev_common_mode(
                                    section_imgs, section_masks)
                                for i in range(2):
                                    section_imgs[i].as_1d().copy_selected(
                                        section_masks[i].as_1d().iselection(),
                                        chebyshev_corrected_imgs[i].as_1d())
                                    self.cspad_img.matrix_paste_block_in_place(
                                        block=section_imgs[i],
                                        i_row=i_rows[i],
                                        i_column=i_columns[i])

                        else:
                            common_mode = self.common_mode(
                                section_img, section_stddev, section_mask)
                            self.sum_common_mode += common_mode
                            self.sumsq_common_mode += common_mode**2

                            # Apply the common mode correction to the
                            # section, and paste it back into the image.
                            self.cspad_img.matrix_paste_block_in_place(
                                block=section_img - common_mode,
                                i_row=i_row,
                                i_column=i_column)

        if self.gain_map is not None:
            self.cspad_img *= self.gain_map

        if (self.mask_img is not None):
            sel = (self.mask_img == -2) | (self.mask_img
                                           == cspad_tbx.cspad_mask_value)
            self.cspad_img.set_selected(sel, cspad_tbx.cspad_mask_value)

        if (self.address == 'XppEndstation-0|Rayonix-0' or \
            self.address == 'MfxEndstation-0|Rayonix-0') and \
            self.crop_rayonix:
            # Crop the masked data so that the beam center is in the center of the image
            self.cspad_img = self.cspad_img[self.rayonix_crop_slice[0],
                                            self.rayonix_crop_slice[1]]

        if self.cache_image:
            # Store the image in the event.
            evt.put(self.cspad_img, self.address)
示例#21
0
    def event(self, evt, env):
        """The event() function is called for every L1Accept transition.
    XXX more?

    Previously, common-mode correction was applied only after initial
    threshold filtering.  Since the common_mode class applies the
    (lengthy) common-mode correction immediately after reading the
    image from the stream, this optimisation is currently not
    (elegantly) doable.

    @param evt Event data object, a configure object
    @param env Environment object
    """

        super(mod_hitfind, self).event(evt, env)
        if (evt.get("skip_event")):
            return

        # This module only applies to detectors for which a distance is
        # available.
        distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
        if distance is None:
            self.nfail += 1
            self.logger.warning("event(): no distance, shot skipped")
            evt.put(skip_event_flag(), "skip_event")
            return

        device = cspad_tbx.address_split(self.address)[2]

        # ***** HITFINDING ***** XXX For hitfinding it may be interesting
        # to look at the fraction of subzero pixels in the dark-corrected
        # image.
        if (self.m_threshold is not None):
            # If a threshold value is given it can be applied in one of three ways:
            #    1.  Apply it over the whole image
            if (self.m_roi is None and self.m_distl_min_peaks is None):
                vmax = flex.max(self.cspad_img)
                if (vmax < self.m_threshold):
                    if not self.m_negate_hits:
                        # Tell downstream modules to skip this event if the threshold was not met.
                        evt.put(skip_event_flag(), "skip_event")
                        return
                elif self.m_negate_hits:
                    evt.put(skip_event_flag(), "skip_event")
                    return

            #    2. Apply threshold over a rectangular region of interest.
            elif (self.m_roi is not None):
                vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
                                               self.m_roi[0]:self.m_roi[1]])
                if (vmax < self.m_threshold):
                    if not self.m_negate_hits:
                        evt.put(skip_event_flag(), "skip_event")
                        return
                elif self.m_negate_hits:
                    evt.put(skip_event_flag(), "skip_event")
                    return

            #    3. Determine the spotfinder spots within the central ASICS, and accept the
            #       image as a hit if there are m_distl_min_peaks exceeding m_threshold.
            #       As a further requirement, the peaks must exceed 2.5 * the 90-percentile
            #       pixel value of the central ASICS.  This filter was added to avoid high-background
            #       false positives.
            elif (self.m_distl_min_peaks is not None):
                if device == 'marccd':
                    self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
                    self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
                elif device == 'Rayonix':
                    self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
                    self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]

                peak_heights, outvalue = self.distl_filter(
                    self.address,
                    self.cspad_img.iround(),  # XXX correct?
                    distance,
                    self.timestamp,
                    self.wavelength)
                if ('permissive' in self.m_distl_flags):
                    number_of_accepted_peaks = (peak_heights >
                                                self.m_threshold).count(True)
                else:
                    number_of_accepted_peaks = ((
                        peak_heights > self.m_threshold).__and__(
                            outvalue == 0)).count(True)

                sec, ms = cspad_tbx.evt_time(evt)
                evt_time = sec + ms / 1000
                self.stats_logger.info("BRAGG %.3f %d" %
                                       (evt_time, number_of_accepted_peaks))

                skip_event = False
                if number_of_accepted_peaks < self.m_distl_min_peaks:
                    self.logger.info(
                        "Subprocess %02d: Spotfinder NO  HIT image #%05d @ %s; %d spots > %d"
                        % (env.subprocess(), self.nshots, self.timestamp,
                           number_of_accepted_peaks, self.m_threshold))

                    if not self.m_negate_hits:
                        skip_event = True
                else:
                    self.logger.info(
                        "Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d"
                        % (env.subprocess(), self.nshots, self.timestamp,
                           number_of_accepted_peaks, self.m_threshold))

                    if self.m_negate_hits:
                        skip_event = True

                if skip_event:
                    if self.m_db_logging:
                        # log misses to the database
                        self.queue_entry(
                            (self.trial, evt.run(), "%.3f" % evt_time,
                             number_of_accepted_peaks, distance, self.sifoil,
                             self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                             0, 0, self.m_db_tags))
                    evt.put(skip_event_flag(), "skip_event")
                    return
                # the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
                # hit will not be logged in the db.
                evt.put(number_of_accepted_peaks, 'sfspots')

        self.logger.info("Subprocess %02d: process image #%05d @ %s" %
                         (env.subprocess(), self.nshots, self.timestamp))

        # See r17537 of mod_average.py.
        if device == 'Cspad':
            pixel_size = cspad_tbx.pixel_size
            saturated_value = cspad_tbx.cspad_saturated_value
        elif device == 'marccd':
            pixel_size = evt.get("marccd_pixel_size")
            saturated_value = evt.get("marccd_saturated_value")
        elif device == 'Rayonix':
            pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
            saturated_value = rayonix_tbx.rayonix_saturated_value

        d = cspad_tbx.dpack(
            active_areas=self.active_areas,
            address=self.address,
            beam_center_x=pixel_size * self.beam_center[0],
            beam_center_y=pixel_size * self.beam_center[1],
            data=self.cspad_img.iround(),  # XXX ouch!
            distance=distance,
            pixel_size=pixel_size,
            saturated_value=saturated_value,
            timestamp=self.timestamp,
            wavelength=self.wavelength,
            xtal_target=self.m_xtal_target)

        if (self.m_dispatch == "index"):
            import sys
            from xfel.cxi.integrate_image_api import integrate_one_image
            info = integrate_one_image(
                d,
                integration_dirname=self.m_integration_dirname,
                integration_basename=self.m_integration_basename)
            sys.stdout = sys.__stdout__
            sys.stderr = sys.__stderr__

            indexed = info is not None and hasattr(info, 'spotfinder_results')
            if self.m_progress_logging:
                if self.m_db_version == 'v1':
                    if indexed:
                        # integration pickle dictionary is available here as info.last_saved_best
                        if info.last_saved_best[
                                "identified_isoform"] is not None:
                            #print info.last_saved_best.keys()
                            from cxi_xdr_xes.cftbx.cspad_ana import db
                            dbobj = db.dbconnect(self.m_db_host,
                                                 self.m_db_name,
                                                 self.m_db_user,
                                                 self.m_db_password)
                            cursor = dbobj.cursor()
                            if info.last_saved_best[
                                    "identified_isoform"] in self.isoforms:
                                PM, indices, miller_id = self.isoforms[
                                    info.last_saved_best["identified_isoform"]]
                            else:
                                from xfel.xpp.progress_support import progress_manager
                                PM = progress_manager(info.last_saved_best,
                                                      self.m_db_experiment_tag,
                                                      self.m_trial_id,
                                                      self.m_rungroup_id,
                                                      evt.run())
                                indices, miller_id = PM.get_HKL(cursor)
                                # cache these as they don't change for a given isoform
                                self.isoforms[info.last_saved_best[
                                    "identified_isoform"]] = PM, indices, miller_id
                            if self.m_sql_buffer_size > 1:
                                self.queue_progress_entry(
                                    PM.scale_frame_detail(self.timestamp,
                                                          cursor,
                                                          do_inserts=False))
                            else:
                                PM.scale_frame_detail(self.timestamp,
                                                      cursor,
                                                      do_inserts=True)
                                dbobj.commit()
                                cursor.close()
                                dbobj.close()
                elif self.m_db_version == 'v2':
                    key_low = 'cctbx.xfel.radial_average.two_theta_low'
                    key_high = 'cctbx.xfel.radial_average.two_theta_high'
                    tt_low = evt.get(key_low)
                    tt_high = evt.get(key_high)

                    from xfel.ui.db.dxtbx_db import log_frame
                    if indexed:
                        n_spots = len(info.spotfinder_results.images[
                            info.frames[0]]['spots_total'])
                    else:
                        sfspots = evt.get('sfspots')
                        if sfspots is None:
                            if info is None or not isinstance(info, int):
                                n_spots = 0
                            else:
                                n_spots = info
                        else:
                            n_spots = sfspots

                    if indexed:
                        known_setting = info.horizons_phil.known_setting
                        indexed_setting = info.organizer.info[
                            'best_integration']['counter']
                        if known_setting is None or known_setting == indexed_setting:
                            from xfel.command_line.frame_unpickler import construct_reflection_table_and_experiment_list
                            c = construct_reflection_table_and_experiment_list(
                                info.last_saved_best,
                                None,
                                pixel_size,
                                proceed_without_image=True)
                            c.assemble_experiments()
                            c.assemble_reflections()
                            log_frame(c.experiment_list, c.reflections,
                                      self.db_params, evt.run(), n_spots,
                                      self.timestamp, tt_low, tt_high)
                        else:
                            print(
                                "Not logging %s, wrong bravais setting (expecting %d, got %d)"
                                % (self.timestamp, known_setting,
                                   indexed_setting))
                    else:
                        log_frame(None, None, self.db_params, evt.run(),
                                  n_spots, self.timestamp, tt_low, tt_high)

            if self.m_db_logging:
                sec, ms = cspad_tbx.evt_time(evt)
                evt_time = sec + ms / 1000
                sfspots = evt.get('sfspots')
                if sfspots is None:
                    if indexed:
                        n_spots = len(info.spotfinder_results.images[
                            info.frames[0]]['spots_total'])
                    else:
                        n_spots = 0
                else:
                    n_spots = sfspots

                if indexed:
                    mosaic_bloc_rotation = info.last_saved_best.get(
                        'ML_half_mosaicity_deg', [0])[0]
                    mosaic_block_size = info.last_saved_best.get(
                        'ML_domain_size_ang', [0])[0]
                    ewald_proximal_volume = info.last_saved_best.get(
                        'ewald_proximal_volume', [0])[0]

                    obs = info.last_saved_best['observations'][0]
                    cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell(
                    ).parameters()
                    pointgroup = info.last_saved_best['pointgroup']
                    resolution = obs.d_min()
                else:
                    mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
                      cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0

                self.queue_entry(
                    (self.trial, evt.run(), "%.3f" % evt_time, n_spots,
                     distance, self.sifoil, self.wavelength, indexed,
                     mosaic_bloc_rotation, mosaic_block_size,
                     ewald_proximal_volume, pointgroup, cell_a, cell_b, cell_c,
                     cell_alpha, cell_beta, cell_gamma, resolution,
                     self.m_db_tags))

            if (not indexed):
                evt.put(skip_event_flag(), "skip_event")
                return

        elif (self.m_dispatch == "nop"):
            pass

        elif (self.m_dispatch == "view"):  #interactive image viewer

            args = ["indexing.data=dummy"]
            detector_format_version = detector_format_function(
                self.address, evt.GetTime())
            if detector_format_version is not None:
                args += [
                    "distl.detector_format_version=%" % detector_format_version
                ]

            from xfel.phil_preferences import load_cxi_phil
            horizons_phil = load_cxi_phil(self.m_xtal_target, args)
            horizons_phil.indexing.data = d

            from xfel.cxi import display_spots
            display_spots.parameters.horizons_phil = horizons_phil
            display_spots.wrapper_of_callback().display(
                horizons_phil.indexing.data)

        elif (self.m_dispatch == "spots"):  #interactive spotfinder viewer

            args = ["indexing.data=dummy"]
            detector_format_version = detector_format_function(
                self.address, evt.GetTime())
            if detector_format_version is not None:
                args += [
                    "distl.detector_format_version=%s" %
                    detector_format_version
                ]

            from xfel.phil_preferences import load_cxi_phil
            horizons_phil = load_cxi_phil(self.m_xtal_target, args)
            horizons_phil.indexing.data = d

            from xfel.cxi import display_spots
            display_spots.parameters.horizons_phil = horizons_phil

            from rstbx.new_horizons.index import pre_indexing_validation, pack_names
            pre_indexing_validation(horizons_phil)
            imagefile_arguments = pack_names(horizons_phil)
            horizons_phil.persist.show()
            from spotfinder.applications import signal_strength
            info = signal_strength.run_signal_strength_core(
                horizons_phil, imagefile_arguments)

            work = display_spots.wrapper_of_callback(info)
            work.display_with_callback(horizons_phil.indexing.data)

        elif (self.m_dispatch == "write_dict"):
            self.logger.warning(
                "event(): deprecated dispatch 'write_dict', use mod_dump instead"
            )
            if (self.m_out_dirname is not None
                    or self.m_out_basename is not None):
                cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)

        # Diagnostic message emitted only when all the processing is done.
        if (env.subprocess() >= 0):
            self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
                             (env.subprocess(), self.nshots, self.timestamp))
        else:
            self.logger.info("Accepted #%05d @ %s" %
                             (self.nshots, self.timestamp))
示例#22
0
class common_mode_correction(mod_event_info):
  """Dark subtraction and alternate implementation of common mode
  substituting for cspad_tbx.

  Known problems: the algorithm relies on a substantial part of the
  sensor having no signal, which is not the case if a water ring
  crosses the sensor.
  """


  def __init__(self,
               address,
               calib_dir=None,
               common_mode_correction="none",
               photon_threshold=None,
               two_photon_threshold=None,
               dark_path=None,
               dark_stddev=None,
               mask_path=None,
               gain_map_path=None,
               gain_map_level=None,
               cache_image=True,
               roi=None,
               laser_1_status=None,
               laser_4_status=None,
               laser_wait_time=None,
               override_beam_x=None,
               override_beam_y=None,
               bin_size=None,
               crop_rayonix=False,
               **kwds):
    """The common_mode_correction class constructor stores the
    parameters passed from the pyana configuration file in instance
    variables.

    @param address         Full data source address of the DAQ device
    @param calib_dir       Directory with calibration information
    @param common_mode_correction The type of common mode correction to apply
    @param dark_path       Path to input average dark image
    @param dark_stddev     Path to input standard deviation dark
                           image, required if @p dark_path is given
    @param mask_path       Path to input mask.  Pixels to mask out should be set to -2
    @param gain_map_path   Path to input gain map.  Multiplied times the image.
    @param gain_map_level  If set, all the '1' pixels in the gain_map are set to this multiplier
                           and all the '0' pixels in the gain_map are set to '1'.  If not set,
                           use the values in the gain_map directly
    @param laser_1_status  0 or 1 to indicate that the laser should be off or on respectively
    @param laser_4_status  0 or 1 to indicate that the laser should be off or on respectively
    @param laser_wait_time Length of time in milliseconds to wait after a laser
                           change of status to begin accepting images again.
                           (rejection of images occurs immediately after status
                           change).
    @param override_beam_x override value for x coordinate of beam center in pixels
    @param override_beam_y override value for y coordinate of beam center in pixels
    @param bin_size bin size for rayonix detector used to determin pixel size
    @param crop_rayonix    whether to crop rayonix images such that image center is the beam center
    """

    # Cannot use the super().__init__() construct here, because
    # common_mode_correction refers to the argument, and not the
    # class.
    mod_event_info.__init__(self, address=address, **kwds)

    # The paths will be substituted in beginjob(), where evt and env
    # are available.
    self._dark_path = cspad_tbx.getOptString(dark_path)
    self._dark_stddev_path = cspad_tbx.getOptString(dark_stddev)
    self._gain_map_path = cspad_tbx.getOptString(gain_map_path)
    self._mask_path = cspad_tbx.getOptString(mask_path)

    self.gain_map_level = cspad_tbx.getOptFloat(gain_map_level)
    self.common_mode_correction = cspad_tbx.getOptString(common_mode_correction)
    self.photon_threshold = cspad_tbx.getOptFloat(photon_threshold)
    self.two_photon_threshold = cspad_tbx.getOptFloat(two_photon_threshold)
    self.cache_image = cspad_tbx.getOptBool(cache_image)
    self.filter_laser_1_status = cspad_tbx.getOptInteger(laser_1_status)
    self.filter_laser_4_status = cspad_tbx.getOptInteger(laser_4_status)
    if self.filter_laser_1_status is not None:
      self.filter_laser_1_status = bool(self.filter_laser_1_status)
    if self.filter_laser_4_status is not None:
      self.filter_laser_4_status = bool(self.filter_laser_4_status)
    self.filter_laser_wait_time = cspad_tbx.getOptInteger(laser_wait_time)
    self.override_beam_x = cspad_tbx.getOptFloat(override_beam_x)
    self.override_beam_y = cspad_tbx.getOptFloat(override_beam_y)
    self.bin_size = cspad_tbx.getOptInteger(bin_size)
    self.crop_rayonix = cspad_tbx.getOptBool(crop_rayonix)

    self.cspad_img = None # The current image - set by self.event()
    self.sum_common_mode = 0
    self.sumsq_common_mode = 0
    self.roi = cspad_tbx.getOptROI(roi) # used to ignore the signal region in chebyshev fit

    assert self.common_mode_correction in \
        ("gaussian", "mean", "median", "mode", "none", "chebyshev")

    # Get and parse metrology.
    self.sections = None
    device = cspad_tbx.address_split(self.address)[2]
    if device == 'Andor':
      self.sections = [] # XXX FICTION
    elif device == 'Cspad':
      if self.address == 'XppGon-0|Cspad-0':
        self.sections = [] # Not used for XPP
      else:
        self.sections = calib2sections(cspad_tbx.getOptString(calib_dir))
    elif device == 'Cspad2x2':
      # There is no metrology information for the Sc1 detector, so
      # make it up.  The sections are rotated by 90 degrees with
      # respect to the "standing up" convention.
      self.sections = [[Section(90, (185 / 2 + 0,   (2 * 194 + 3) / 2)),
                        Section(90, (185 / 2 + 185, (2 * 194 + 3) / 2))]]
    elif device == 'marccd':
      self.sections = [] # XXX FICTION
    elif device == 'pnCCD':
      self.sections = [] # XXX FICTION
    elif device == 'Rayonix':
      self.sections = [] # XXX FICTION
    elif device == 'Opal1000':
      self.sections = [] # XXX FICTION
    if self.sections is None:
      raise RuntimeError("Failed to load metrology")


  def beginjob(self, evt, env):
    """The beginjob() function does one-time initialisation from
    event- or environment data.  It is called at an XTC configure
    transition.

    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(common_mode_correction, self).beginjob(evt, env)
    # Load the dark image and ensure it is signed and at least 32 bits
    # wide, since it will be used for differencing.  If a dark image
    # is provided, a standard deviation image is required, and all the
    # ADU scales must match up.
    #
    # XXX Can we zap all the ADU_SCALE stuff?
    #
    # XXX Do we really need to store the substituted values in the
    # instance here?  At least self._dark_path is referenced later on.
    #
    # Note that this will load the dark, standard deviation and gain
    # once (SEVERAL TIMES) for each process, but the gain is that we
    # can do substitutions.  But it is only done at the beginning of
    # the job.
    self.dark_img = None
    if self._dark_path is not None:
      self._dark_path = cspad_tbx.getOptEvalOrString(
        cspad_tbx.pathsubst(self._dark_path, evt, env))

      assert self._dark_stddev_path is not None
      dark_dict = easy_pickle.load(self._dark_path)
      #assert "ADU_SCALE" not in dark_dict # force use of recalculated dark
      self.dark_img = dark_dict['DATA']
      assert isinstance(self.dark_img, flex.double)

      self._dark_stddev_path = cspad_tbx.getOptEvalOrString(
        cspad_tbx.pathsubst(self._dark_stddev_path, evt, env))

      self.dark_stddev = easy_pickle.load(self._dark_stddev_path)['DATA']
      assert isinstance(self.dark_stddev, flex.double)
      self.dark_mask = (self.dark_stddev > 0)

    # Load the mask image and ensure it is signed and at least 32 bits
    # wide, since it will be used for differencing.
    self.gain_map = None
    if self._gain_map_path is not None:
      self._gain_map_path = cspad_tbx.getOptEvalOrString(
        cspad_tbx.pathsubst(self._gain_map_path, evt, env))
      self.gain_map = easy_pickle.load(self._gain_map_path)['DATA']
      if self.gain_map_level is not None:
        sel = flex.bool([bool(f) for f in self.gain_map])
        sel.reshape(flex.grid(self.gain_map.focus()))
        self.gain_map = self.gain_map.set_selected(~sel, self.gain_map_level)
        self.gain_map = self.gain_map.set_selected(sel, 1)
      assert isinstance(self.gain_map, flex.double)

    self.mask_img = None
    if self._mask_path is not None:
      self._mask_path = cspad_tbx.getOptEvalOrString(
        cspad_tbx.pathsubst(self._mask_path, evt, env))

      self.mask_img = easy_pickle.load(self._mask_path)['DATA']
      assert isinstance(self.mask_img, flex.double) \
        or   isinstance(self.mask_img, flex.int)

    if self.address == 'XppGon-0|marccd-0':
      #mod_mar.py will set these during its event function
      self.active_areas = None
      self.beam_center = None
    elif self.address == 'XppEndstation-0|Rayonix-0' or \
         self.address == 'MfxEndstation-0|Rayonix-0':
      assert self.override_beam_x is not None
      assert self.override_beam_y is not None
      from xfel.cxi.cspad_ana import rayonix_tbx
      maxx, maxy = rayonix_tbx.get_rayonix_detector_dimensions(self.bin_size)
      if self.crop_rayonix:
        bx = int(round(self.override_beam_x))
        by = int(round(self.override_beam_y))
        minsize = min([bx,by,maxx-bx,maxy-by])
        self.beam_center = minsize,minsize
        self.active_areas = flex.int([0,0,2*minsize,2*minsize])
        self.rayonix_crop_slice = slice(by-minsize,by+minsize), slice(bx-minsize,bx+minsize)
      else:
        self.beam_center = self.override_beam_x,self.override_beam_y
        self.active_areas = flex.int([0,0,maxx,maxy])
    elif self.address == 'XppGon-0|Cspad-0':
      evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
      timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format
      from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
      from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
      version_lookup = detector_format_version(self.address, reverse_timestamp(timestamp)[0])
      assert version_lookup is not None
      self.active_areas = xpp_active_areas[version_lookup]['active_areas']
      self.beam_center = [1765 // 2, 1765 // 2]
    else:
      (self.beam_center, self.active_areas) = cspad_tbx.cbcaa(
        cspad_tbx.getConfig(self.address, env), self.sections)

  def common_mode(self, img, stddev, mask):
    """The common_mode() function returns the mode of image stored in
    the array pointed to by @p img.  @p mask must be such that the @p
    stddev at the selected pixels is greater than zero.

    @param img    2D integer array of the image
    @param stddev 2D integer array of the standard deviation of each
                  pixel in @p img
    @param mask   2D Boolean array, @c True if the pixel is to be
                  included, @c False otherwise
    @return       Mode of the image, as a real number
    """

    # Flatten the image and take out inactive pixels XXX because we
    # cannot take means and medians of 2D arrays?
    img_1d = img.as_1d().select(mask.as_1d()).as_double()
    assert img_1d.size() > 0

    if (self.common_mode_correction == "mean"):
      # The common mode is approximated by the mean of the pixels with
      # signal-to-noise ratio less than a given threshold.  XXX Breaks
      # if the selection is empty!
      THRESHOLD_SNR = 2
      img_snr = img_1d / stddev.as_double().as_1d().select(mask.as_1d())
      return (flex.mean(img_1d.select(img_snr < THRESHOLD_SNR)))

    elif (self.common_mode_correction == "median"):
      return (flex.median(img_1d))

    # Identify the common-mode correction as the peak histogram of the
    # histogram of pixel values (the "standard" common-mode correction, as
    # previously implemented in this class).
    hist_min = -40
    hist_max = 40
    n_slots = 100

    hist = flex.histogram(img_1d, hist_min, hist_max, n_slots=n_slots)
    slots = hist.slots()
    i = flex.max_index(slots)
    common_mode = list(hist.slot_infos())[i].center()

    if (self.common_mode_correction == "mode"):
      return (common_mode)

    # Determine the common-mode correction from the peak of a single
    # Gaussian function fitted to the histogram.
    from scitbx.math.curve_fitting import single_gaussian_fit
    x = hist.slot_centers()
    y = slots.as_double()
    fit = single_gaussian_fit(x, y)
    scale, mu, sigma = fit.a, fit.b, fit.c
    self.logger.debug("fitted gaussian: mu=%.3f, sigma=%.3f" %(mu, sigma))
    mode = common_mode
    common_mode = mu
    if abs(mode-common_mode) > 1000: common_mode = mode # XXX
    self.logger.debug("delta common mode corrections: %.3f" %(mode-common_mode))

    if 0 and abs(mode-common_mode) > 0:
      #if 0 and skew > 0.5:
      # view histogram and fitted gaussian
      from numpy import exp
      from matplotlib import pyplot
      x_all = x
      n, bins, patches = pyplot.hist(section_img.as_1d().as_numpy_array(), bins=n_slots, range=(hist_min, hist_max))
      y_all = scale * flex.exp(-flex.pow2(x_all-mu) / (2 * sigma**2))
      scale = slots[flex.max_index(slots)]
      y_all *= scale/flex.max(y_all)
      pyplot.plot(x_all, y_all)
      pyplot.show()

    return (common_mode)


  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.

    @param evt Event data object, a configure object
    @param env Environment object
    """
    super(common_mode_correction, self).event(evt, env)
    if (evt.get("skip_event")):
      return

    if not hasattr(self, 'active_areas') or self.active_areas is None or \
       not hasattr(self, 'beam_center')  or self.beam_center  is None:
      if self.address == 'XppGon-0|marccd-0':
        # The mod_mar module needs to have been called before this one
        # to set this up.  The MAR does not have a configure object.
        self.beam_center = evt.get("marccd_beam_center")
        self.active_areas = evt.get("marccd_active_areas")
      elif self.address == 'XppEndstation-0|Rayonix-0' or \
           self.address == 'MfxEndstation-0|Rayonix-0':
        pass # bc and aa set in the beginjob function
      elif self.address == 'XppGon-0|Cspad-0':
        # Load the active areas as determined from the optical metrology
        from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
        from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
        version_lookup = detector_format_version(self.address, reverse_timestamp(self.timestamp)[0])
        assert version_lookup is not None
        self.active_areas = xpp_active_areas[version_lookup]['active_areas']
        self.beam_center = [1765 // 2, 1765 // 2]
      else:
        (self.beam_center, self.active_areas) = \
          cspad_tbx.cbcaa(cspad_tbx.getConfig(self.address, env), self.sections)

    if self.filter_laser_1_status is not None:
      if (self.laser_1_status.status != self.filter_laser_1_status or
          (self.laser_1_ms_since_change is not None and
           self.laser_1_ms_since_change < self.filter_laser_wait_time)):
        evt.put(skip_event_flag(), "skip_event")
        return
    if self.filter_laser_4_status is not None:
      if (self.laser_4_status.status != self.filter_laser_4_status or
          (self.laser_4_ms_since_change is not None and
           self.laser_4_ms_since_change < self.filter_laser_wait_time)):
        evt.put(skip_event_flag(), "skip_event")
        return

    # Early return if the full detector image is already stored in the
    # event.  Otherwise, get it from the stream as a double-precision
    # floating-point flex array.  XXX It is probably not safe to key
    # the image on self.address, so we should come up with our own
    # namespace.  XXX Misnomer--could be CAMP, too

    self.cspad_img = evt.get(self.address)
    if self.cspad_img is not None:
      return
    if self.address == 'XppGon-0|Cspad-0':
      # Kludge until cspad_tbx.image() can be rewritten to handle the
      # XPP metrology.
      self.cspad_img = cspad_tbx.image_xpp(
        self.address, evt, env, self.active_areas)
    elif self.address == 'XppEndstation-0|Rayonix-0' or \
         self.address == 'MfxEndstation-0|Rayonix-0':
      from psana import Source, Camera
      import numpy as np
      address = cspad_tbx.old_address_to_new_address(self.address)
      src=Source('DetInfo(%s)'%address)
      self.cspad_img = evt.get(Camera.FrameV1,src)
      if self.cspad_img is not None:
        self.cspad_img = self.cspad_img.data16().astype(np.float64)
    elif self.address=='CxiDg3-0|Opal1000-0':
      if evt.getFrameValue(self.address) is not None:
        self.cspad_img = evt.getFrameValue(self.address).data()
    elif self.address=='CxiEndstation-0|Opal1000-2':
      if evt.getFrameValue(self.address) is not None:
        self.cspad_img = evt.getFrameValue(self.address).data()
    elif self.address=='FeeHxSpectrometer-0|Opal1000-1':
      if evt.getFrameValue(self.address) is not None:
        self.cspad_img = evt.getFrameValue(self.address).data()
    elif self.address=='NoDetector-0|Cspad2x2-0':
        import numpy as np
        from pypdsdata import xtc
        test=[]
        self.cspad_img = evt.get(xtc.TypeId.Type.Id_Cspad2x2Element,self.address).data()
        self.cspad_img=np.reshape(self.cspad_img,(370, 388))
    else:
      try:
        self.cspad_img = cspad_tbx.image(
          self.address, cspad_tbx.getConfig(self.address, env),
          evt, env, self.sections)
      except Exception, e:
        self.logger.error("Error reading image data: " + str(e))
        evt.put(skip_event_flag(), "skip_event")
        return

    if self.cspad_img is None:
      if cspad_tbx.address_split(self.address)[2] != 'Andor':
        self.nfail += 1
        self.logger.warning("event(): no image, shot skipped")
        evt.put(skip_event_flag(), "skip_event")
      return
    self.cspad_img = flex.double(self.cspad_img.astype(numpy.float64))
    # If a dark image was provided, subtract it from the image.  There
    # is no point in doing common-mode correction unless the dark
    # image was subtracted.
    if (self.dark_img is not None):
      self.cspad_img -= self.dark_img

      if (self.common_mode_correction != "none"):
        # Mask out inactive pixels prior to common mode correction.
        # Pixels are marked as inactive either due to low ADU values
        # or non-positive standard deviations in dark image.  XXX Make
        # the threshold tunable?
        cspad_mask = self.dark_mask.deep_copy()

        if self.roi is not None and self.common_mode_correction == "chebyshev":
          roi_mask = cspad_mask[self.roi[2]:self.roi[3], :]
          roi_mask = flex.bool(roi_mask.accessor(), False)
          cspad_mask.matrix_paste_block_in_place(
            block=roi_mask,
            i_row=self.roi[2],
            i_column=0)

        # Extract each active section from the assembled detector
        # image and apply the common mode correction.  XXX Make up a
        # quadrant mask for the emission detector.  Needs to be
        # checked!
        config = cspad_tbx.getConfig(self.address, env)
        if len(self.sections) == 1:
          q_mask = 1
        else:
          q_mask = config.quadMask()
        for q in xrange(len(self.sections)):
          if (not((1 << q) & q_mask)):
            continue

          # XXX Make up section mask for the emission detector.  Needs
          # to be checked!
          import _pdsdata
          if len(self.sections) == 1 and type(config) in (
            _pdsdata.cspad2x2.ConfigV1, _pdsdata.cspad2x2.ConfigV2):
            s_mask = config.roiMask()
          else:
            s_mask = config.roiMask(q)
          for s in xrange(len(self.sections[q])):
            # XXX DAQ misconfiguration?  This mask appears not to work
            # reliably for the Sc1 detector.
#            if (not((1 << s) & s_mask)):
#              continue
            corners   = self.sections[q][s].corners()
            i_row     = int(round(min(c[0] for c in corners)))
            i_column  = int(round(min(c[1] for c in corners)))
            n_rows    = int(round(max(c[0] for c in corners))) - i_row
            n_columns = int(round(max(c[1] for c in corners))) - i_column

            section_img    = self.cspad_img.matrix_copy_block(
              i_row  = i_row,  i_column  = i_column,
              n_rows = n_rows, n_columns = n_columns)
            section_mask   = cspad_mask.matrix_copy_block(
              i_row  = i_row,  i_column  = i_column,
              n_rows = n_rows, n_columns = n_columns)
            section_stddev = self.dark_stddev.matrix_copy_block(
              i_row  = i_row,  i_column  = i_column,
              n_rows = n_rows, n_columns = n_columns)

            if section_mask.count(True) == 0: continue

            if self.common_mode_correction == "chebyshev":
              assert len(self.sections[q]) == 2
              if s == 0:
                section_imgs = [section_img]
                section_masks = [section_mask]
                i_rows = [i_row]
                i_columns = [i_column]
                continue
              else:
                section_imgs.append(section_img)
                section_masks.append(section_mask)
                i_rows.append(i_row)
                i_columns.append(i_column)

                chebyshev_corrected_imgs = self.chebyshev_common_mode(
                  section_imgs, section_masks)
                for i in range(2):
                  section_imgs[i].as_1d().copy_selected(
                    section_masks[i].as_1d().iselection(),
                    chebyshev_corrected_imgs[i].as_1d())
                  self.cspad_img.matrix_paste_block_in_place(
                    block=section_imgs[i],
                    i_row=i_rows[i],
                    i_column=i_columns[i])

            else:
              common_mode = self.common_mode(
                section_img, section_stddev, section_mask)
              self.sum_common_mode += common_mode
              self.sumsq_common_mode += common_mode**2

              # Apply the common mode correction to the
              # section, and paste it back into the image.
              self.cspad_img.matrix_paste_block_in_place(
                block    = section_img - common_mode,
                i_row    = i_row,
                i_column = i_column)

    if self.gain_map is not None:
      self.cspad_img *= self.gain_map

    if (self.mask_img is not None):
      sel = (self.mask_img == -2 )|(self.mask_img == cspad_tbx.cspad_mask_value)
      self.cspad_img.set_selected(sel, cspad_tbx.cspad_mask_value)

    if (self.address == 'XppEndstation-0|Rayonix-0' or \
        self.address == 'MfxEndstation-0|Rayonix-0') and \
        self.crop_rayonix:
      # Crop the masked data so that the beam center is in the center of the image
      self.cspad_img = self.cspad_img[self.rayonix_crop_slice[0], self.rayonix_crop_slice[1]]

    if self.cache_image:
      # Store the image in the event.
      evt.put(self.cspad_img, self.address)
示例#23
0
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.  It
    outputs the detector image associated with the event @p evt to the
    file system.

    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(mod_dump_bitmap, self).event(evt, env)
    if (evt.get('skip_event')):
      return

    # Where the sample-detector distance is not available, set it to
    # zero.
    distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
    if distance is None:
      distance = 0

    # See r17537 of mod_average.py.
    device = cspad_tbx.address_split(self.address)[2]
    if device == 'Cspad':
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
    elif device == 'marccd':
      pixel_size = 0.079346
      saturated_value = 2**16 - 1

    from iotbx.detectors import FlexImage_d as FlexImage
    vendortype = device
    saturation = 65535
    flex_img = FlexImage(
      rawdata=self.cspad_img,
      binning=self._binning,
      vendortype=vendortype,
      brightness=self._brightness,
      saturation=saturated_value)

    flex_img.setWindow(0, 0, 1)
    flex_img.adjust(color_scheme=self._color_scheme)
    flex_img.prep_string()
    import Image
    # XXX is size//self._binning safe here?
    pil_img = Image.fromstring(
      'RGB', (flex_img.size2()//self._binning,
              flex_img.size1()//self._binning),
      flex_img.export_string)

    # The output path should not contain any funny characters which may
    # not work in all environments.  This constructs a sequence number a
    # la evt_seqno() from the dictionary's timestamp.
    t = self.timestamp
    s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[20:23]

    path = os.path.join(
      self._dirname, self._basename + s + '.' + self._ext)

    self._logger.info("Exporting %s" %path)
    tmp_stream = open(path, 'wb')
    pil_img.save(tmp_stream, format=self._format)
    tmp_stream.close()
示例#24
0
    def endjob(self, obj1, obj2=None):
        """The endjob() function writes the mean and standard deviation images
    to disk.

    @param evt Event object (psana only)
    @param env Environment object
    """
        if obj2 is None:
            env = obj1
        else:
            evt = obj1
            env = obj2

        stats = super(mod_average, self).endjob(env)
        if stats is None:
            return

        device = cspad_tbx.address_split(self.address)[2]
        if device == 'Andor':
            beam_center = (0, 0)  # XXX Fiction!
            pixel_size = 13.5e-3  # XXX Should not be hardcoded here!
            saturated_value = 10000
        elif device == 'Cspad' or device == 'Cspad2x2':
            beam_center = self.beam_center
            pixel_size = cspad_tbx.pixel_size
            saturated_value = cspad_tbx.cspad_saturated_value
        elif device == 'marccd':
            beam_center = tuple(t // 2 for t in d['mean_img'].focus())
            pixel_size = 0.079346
            saturated_value = 2**16 - 1

        if stats['nmemb'] > 0:
            if self.avg_dirname  is not None or \
               self.avg_basename is not None or \
               self._mean_out    is not None:
                d = cspad_tbx.dpack(active_areas=self.active_areas,
                                    address=self.address,
                                    beam_center_x=pixel_size * beam_center[0],
                                    beam_center_y=pixel_size * beam_center[1],
                                    data=stats['mean_img'],
                                    distance=stats['distance'],
                                    pixel_size=pixel_size,
                                    saturated_value=saturated_value,
                                    timestamp=cspad_tbx.evt_timestamp(
                                        stats['time']),
                                    wavelength=stats['wavelength'])
                if self._mean_out is not None:
                    p = cspad_tbx.dwritef2(d, self._mean_out)
                else:
                    p = cspad_tbx.dwritef(d, self.avg_dirname,
                                          self.avg_basename)
                self.logger.info("Average written to %s" % p)

            if self.stddev_dirname  is not None or \
               self.stddev_basename is not None or \
               self._std_out    is not None:
                d = cspad_tbx.dpack(active_areas=self.active_areas,
                                    address=self.address,
                                    beam_center_x=pixel_size * beam_center[0],
                                    beam_center_y=pixel_size * beam_center[1],
                                    data=stats['std_img'],
                                    distance=stats['distance'],
                                    pixel_size=pixel_size,
                                    saturated_value=saturated_value,
                                    timestamp=cspad_tbx.evt_timestamp(
                                        stats['time']),
                                    wavelength=stats['wavelength'])
                if self._std_out is not None:
                    p = cspad_tbx.dwritef2(d, self._std_out)
                else:
                    p = cspad_tbx.dwritef(d, self.stddev_dirname,
                                          self.stddev_basename)
                self.logger.info("Standard deviation written to %s" % p)

            if self.max_dirname  is not None or \
               self.max_basename is not None or \
               self._max_out    is not None:
                d = cspad_tbx.dpack(active_areas=self.active_areas,
                                    address=self.address,
                                    beam_center_x=pixel_size * beam_center[0],
                                    beam_center_y=pixel_size * beam_center[1],
                                    data=stats['max_img'],
                                    distance=stats['distance'],
                                    pixel_size=pixel_size,
                                    saturated_value=saturated_value,
                                    timestamp=cspad_tbx.evt_timestamp(
                                        stats['time']),
                                    wavelength=stats['wavelength'])
                if self._max_out is not None:
                    p = cspad_tbx.dwritef2(d, self._max_out)
                else:
                    p = cspad_tbx.dwritef(d, self.max_dirname,
                                          self.max_basename)
                self.logger.info("Max written to %s" % p)

        if stats['nfail'] == 0:
            self.logger.info("%d images processed" % stats['nmemb'])
        else:
            self.logger.warning("%d images processed, %d failed" %
                                (stats['nmemb'], stats['nfail']))
示例#25
0
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.

    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(mod_image_dict, self).event(evt, env)
    if (evt.get("skip_event")):
      return

    if self.cspad_img is None:
      return

    # This module only applies to detectors for which a distance is
    # available.
    distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
    if distance is None:
      self.nfail += 1
      self.logger.warning("event(): no distance, shot skipped")
      evt.put(skip_event_flag(), "skip_event")
      return

    device = cspad_tbx.address_split(self.address)[2]

    self.logger.info("Subprocess %02d: process image #%05d @ %s" %
                     (env.subprocess(), self.nshots, self.timestamp))

    # See r17537 of mod_average.py.
    if device == 'Cspad':
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
    elif device == 'Rayonix':
      pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
      saturated_value = rayonix_tbx.rayonix_saturated_value
    elif device == 'marccd':
      pixel_size = evt.get("marccd_pixel_size")
      saturated_value = evt.get("marccd_saturated_value")
      if distance == 0:
        distance = evt.get("marccd_distance")

    d = cspad_tbx.dpack(
      active_areas=self.active_areas,
      address=self.address,
      beam_center_x=pixel_size * self.beam_center[0],
      beam_center_y=pixel_size * self.beam_center[1],
      data=self.cspad_img.iround(), # XXX ouch!
      distance=distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=self.timestamp,
      wavelength=self.wavelength)

    evt.put(d, self.m_out_key)

    # Diagnostic message emitted only when all the processing is done.
    if (env.subprocess() >= 0):
      self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
                       (env.subprocess(), self.nshots, self.timestamp))
    else:
      self.logger.info("Accepted #%05d @ %s" %
                       (self.nshots, self.timestamp))