Exemple #1
0
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.  It
    outputs the detector image associated with the event @p evt to the
    file system.

    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(mod_dump, self).event(evt, env)
    if (evt.get('skip_event')):
      return

    if self.cspad_img is None:
      print "No image to save for %s"%self.timestamp
      return

    # Where the sample-detector distance is not available, set it to
    # zero.
    distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
    if distance is None:
      distance = 0

    # See r17537 of mod_average.py.
    device = cspad_tbx.address_split(self.address)[2]
    if device == 'Cspad':
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
      output_filename = self._basename
    elif device == 'Rayonix':
      pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
      saturated_value = rayonix_tbx.rayonix_saturated_value
      output_filename = self._basename
    elif device == 'marccd':
      if distance == 0:
        distance = evt.get('marccd_distance')
      pixel_size = 0.079346
      saturated_value = 2**16 - 1
      output_filename = self._basename + evt.get(str, 'mccd_name') + "_"

    d = cspad_tbx.dpack(
      active_areas=self.active_areas,
      address=self.address,
      beam_center_x=pixel_size * self.beam_center[0],
      beam_center_y=pixel_size * self.beam_center[1],
      data=self.cspad_img.iround(), # XXX ouch!
      distance=distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=self.timestamp,
      wavelength=self.wavelength)
    if self._format == "pickle":
      cspad_tbx.dwritef(d, self._dirname, output_filename)
    elif self._format == "tiff":
      cspad_tbx.write_tiff(d, self._dirname, output_filename)
    output_filename = None
Exemple #2
0
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.  It
    outputs the detector image associated with the event @p evt to the
    file system.

    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(mod_dump, self).event(evt, env)
    if (evt.get('skip_event')):
      return

    if self.cspad_img is None:
      print("No image to save for %s"%self.timestamp)
      return

    # Where the sample-detector distance is not available, set it to
    # zero.
    distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
    if distance is None:
      distance = 0

    # See r17537 of mod_average.py.
    device = cspad_tbx.address_split(self.address)[2]
    if device == 'Cspad':
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
      output_filename = self._basename
    elif device == 'Rayonix':
      pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
      saturated_value = rayonix_tbx.rayonix_saturated_value
      output_filename = self._basename
    elif device == 'marccd':
      if distance == 0:
        distance = evt.get('marccd_distance')
      pixel_size = 0.079346
      saturated_value = 2**16 - 1
      output_filename = self._basename + evt.get(str, 'mccd_name') + "_"

    d = cspad_tbx.dpack(
      active_areas=self.active_areas,
      address=self.address,
      beam_center_x=pixel_size * self.beam_center[0],
      beam_center_y=pixel_size * self.beam_center[1],
      data=self.cspad_img.iround(), # XXX ouch!
      distance=distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=self.timestamp,
      wavelength=self.wavelength)
    if self._format == "pickle":
      cspad_tbx.dwritef(d, self._dirname, output_filename)
    elif self._format == "tiff":
      cspad_tbx.write_tiff(d, self._dirname, output_filename)
    output_filename = None
Exemple #3
0
    def endjob(self, obj1, obj2=None):
        """The endjob() function writes the mean and standard deviation images
    to disk.

    @param evt Event object (psana only)
    @param env Environment object
    """
        if obj2 is None:
            env = obj1
        else:
            evt = obj1
            env = obj2

        stats = super(mod_average, self).endjob(env)
        if stats is None:
            return

        device = cspad_tbx.address_split(self.address)[2]
        if device == 'Andor':
            beam_center = (0, 0)  # XXX Fiction!
            pixel_size = 13.5e-3  # XXX Should not be hardcoded here!
            saturated_value = 10000
        elif device == 'Cspad' or device == 'Cspad2x2':
            beam_center = self.beam_center
            pixel_size = cspad_tbx.pixel_size
            saturated_value = cspad_tbx.cspad_saturated_value
        elif device == 'marccd':
            beam_center = tuple(t // 2 for t in d['mean_img'].focus())
            pixel_size = 0.079346
            saturated_value = 2**16 - 1

        if stats['nmemb'] > 0:
            if self.avg_dirname  is not None or \
               self.avg_basename is not None or \
               self._mean_out    is not None:
                d = cspad_tbx.dpack(active_areas=self.active_areas,
                                    address=self.address,
                                    beam_center_x=pixel_size * beam_center[0],
                                    beam_center_y=pixel_size * beam_center[1],
                                    data=stats['mean_img'],
                                    distance=stats['distance'],
                                    pixel_size=pixel_size,
                                    saturated_value=saturated_value,
                                    timestamp=cspad_tbx.evt_timestamp(
                                        stats['time']),
                                    wavelength=stats['wavelength'])
                if self._mean_out is not None:
                    p = cspad_tbx.dwritef2(d, self._mean_out)
                else:
                    p = cspad_tbx.dwritef(d, self.avg_dirname,
                                          self.avg_basename)
                self.logger.info("Average written to %s" % p)

            if self.stddev_dirname  is not None or \
               self.stddev_basename is not None or \
               self._std_out    is not None:
                d = cspad_tbx.dpack(active_areas=self.active_areas,
                                    address=self.address,
                                    beam_center_x=pixel_size * beam_center[0],
                                    beam_center_y=pixel_size * beam_center[1],
                                    data=stats['std_img'],
                                    distance=stats['distance'],
                                    pixel_size=pixel_size,
                                    saturated_value=saturated_value,
                                    timestamp=cspad_tbx.evt_timestamp(
                                        stats['time']),
                                    wavelength=stats['wavelength'])
                if self._std_out is not None:
                    p = cspad_tbx.dwritef2(d, self._std_out)
                else:
                    p = cspad_tbx.dwritef(d, self.stddev_dirname,
                                          self.stddev_basename)
                self.logger.info("Standard deviation written to %s" % p)

            if self.max_dirname  is not None or \
               self.max_basename is not None or \
               self._max_out    is not None:
                d = cspad_tbx.dpack(active_areas=self.active_areas,
                                    address=self.address,
                                    beam_center_x=pixel_size * beam_center[0],
                                    beam_center_y=pixel_size * beam_center[1],
                                    data=stats['max_img'],
                                    distance=stats['distance'],
                                    pixel_size=pixel_size,
                                    saturated_value=saturated_value,
                                    timestamp=cspad_tbx.evt_timestamp(
                                        stats['time']),
                                    wavelength=stats['wavelength'])
                if self._max_out is not None:
                    p = cspad_tbx.dwritef2(d, self._max_out)
                else:
                    p = cspad_tbx.dwritef(d, self.max_dirname,
                                          self.max_basename)
                self.logger.info("Max written to %s" % p)

        if stats['nfail'] == 0:
            self.logger.info("%d images processed" % stats['nmemb'])
        else:
            self.logger.warning("%d images processed, %d failed" %
                                (stats['nmemb'], stats['nfail']))
Exemple #4
0
    def __init__(self, runs, output_dirname=".", roi=None):
        avg_basename = "avg_"
        stddev_basename = "stddev"
        self.sum_img = None
        self.sumsq_img = None
        self.nmemb = 0
        self.roi = cspad_tbx.getOptROI(roi)
        self.unbound_pixel_mask = cspad_unbound_pixel_mask()
        for i_run, run in enumerate(runs):
            run_scratch_dir = run
            result = finalise_one_run(run_scratch_dir)
            if result.sum_img is None: continue
            if self.sum_img is None:
                self.sum_img = result.sum_img
                self.sumsq_img = result.sumsq_img
            else:
                self.sum_img += result.sum_img
                self.sumsq_img += result.sumsq_img
            self.nmemb += result.nmemb

        self.avg_img = self.sum_img.as_double() / self.nmemb
        self.stddev_img = flex.sqrt(
            (self.sumsq_img.as_double() -
             self.sum_img.as_double() * self.avg_img) / (self.nmemb - 1))

        self.mask = flex.int(self.sum_img.accessor(), 0)
        self.mask.set_selected(self.sum_img == 0, 1)
        self.mask.set_selected(self.unbound_pixel_mask > 0, 1)

        if (output_dirname is not None and avg_basename is not None):
            if (not os.path.isdir(output_dirname)):
                os.makedirs(output_dirname)
            d = cspad_tbx.dpack(
                address='CxiSc1-0|Cspad2x2-0',
                data=self.avg_img,
                distance=1,
            )
            cspad_tbx.dwritef(d, output_dirname, avg_basename)
            d = cspad_tbx.dpack(
                address='CxiSc1-0|Cspad2x2-0',
                data=self.sum_img,
                distance=1,
            )
            cspad_tbx.dwritef(d, output_dirname, "sum_")
            if 1:
                output_image(self.avg_img, "%s/avg.png" % output_dirname)
                output_image(self.avg_img,
                             "%s/avg_inv.png" % output_dirname,
                             invert=True)

            if 1:
                output_matlab_form(self.sum_img, "%s/sum.m" % output_dirname)
                output_matlab_form(self.avg_img, "%s/avg.m" % output_dirname)
                output_matlab_form(self.stddev_img,
                                   "%s/stddev.m" % output_dirname)

        if (stddev_basename is not None):
            d = cspad_tbx.dpack(
                address='CxiSc1-0|Cspad2x2-0',
                data=self.stddev_img,
                distance=1,
            )
            cspad_tbx.dwritef(d, output_dirname, stddev_basename)

            # XXX we should really figure out automatically the area where the spectrum is
            #write an integrated spectrum from lines 186-227
            #spectrum_focus = self.sum_img.as_numpy_array()[186:228,:]
            img = self.sum_img
            if self.roi is None:
                spectrum_focus = img
                mask_focus = self.mask
            else:
                slices = (slice(self.roi[2],
                                self.roi[3]), slice(self.roi[0], self.roi[1]))
                spectrum_focus = img[slices]
                mask_focus = self.mask[slices]
            if False:
                from matplotlib import pylab
                pylab.imshow(spectrum_focus.as_numpy_array())
                pylab.show()

        output_spectrum(spectrum_focus,
                        mask_focus=mask_focus,
                        output_dirname=output_dirname)

        print "Total number of images used from %i runs: %i" % (i_run + 1,
                                                                self.nmemb)
        xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)

        print "Number of rows: %i" % spectrum_focus.all()[0]
        print "Estimated no. photons counted: %i" % flex.sum(spectrum_focus)
        print "Number of images used: %i" % flex.sum(
            pixel_histograms.histograms.values()[0].slots())

        d = cspad_tbx.dpack(
            address='CxiSc1-0|Cspad2x2-0',
            data=spectrum_focus,
            distance=1,
            ccd_image_saturation=2e8,  # XXX
        )
        if run is not None: runstr = "_%04d" % run
        else: runstr = ""
        cspad_tbx.dwritef(d, output_dirname, 'sum%s_' % runstr)

        if gain_map is None:
            gain_map = flex.double(gain_img.accessor(), 0)
            img_sel = (gain_img > 0).as_1d()
            d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0',
                                data=gain_img,
                                distance=1)
            cspad_tbx.dwritef(d, output_dirname, 'raw_gain_map_')
            gain_map.as_1d().set_selected(img_sel.iselection(),
                                          1 / gain_img.as_1d().select(img_sel))
            gain_map /= flex.mean(gain_map.as_1d().select(img_sel))
            d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0',
                                data=gain_map,
                                distance=1)
            cspad_tbx.dwritef(d, output_dirname, 'gain_map_')
Exemple #6
0
class xes_from_histograms(object):
    def __init__(self,
                 pixel_histograms,
                 output_dirname=".",
                 gain_map_path=None,
                 gain_map=None,
                 estimated_gain=30,
                 roi=None,
                 run=None):

        self.sum_img = flex.double(flex.grid(
            370, 391), 0)  # XXX define the image size some other way?
        gain_img = flex.double(self.sum_img.accessor(), 0)

        assert [gain_map, gain_map_path].count(None) > 0
        if gain_map_path is not None:
            d = easy_pickle.load(gain_map_path)
            gain_map = d["DATA"]

        mask = flex.int(self.sum_img.accessor(), 0)

        start_row = 370
        end_row = 0
        print len(pixel_histograms.histograms)

        pixels = list(pixel_histograms.pixels())
        n_pixels = len(pixels)
        if roi is not None:
            for k, (i, j) in enumerate(reversed(pixels)):
                if (i < roi[2] or i > roi[3] or j < roi[0] or j > roi[1]):
                    del pixels[n_pixels - k - 1]

        if gain_map is None:
            fixed_func = pixel_histograms.fit_one_histogram
        else:

            def fixed_func(pixel):
                return pixel_histograms.fit_one_histogram(pixel, n_gaussians=1)

        chi_squared_list = flex.double()

        for i, pixel in enumerate(pixels):
            #print i,pixel
            LEG = False
            start_row = min(start_row, pixel[0])
            end_row = max(end_row, pixel[0])
            n_photons = 0

            try:
                if LEG:
                    gaussians, two_photon_flag = pixel_histograms.fit_one_histogram(
                        pixel)
                alt_gaussians = pixel_histograms.fit_one_histogram_two_gaussians(
                    pixel)
            except ZeroDivisionError:
                print "HEY DIVIDE BY ZERO"
                #pixel_histograms.plot_combo(pixel, gaussians)
                mask[pixel] = 1
                continue
            except RuntimeError, e:
                print "Error fitting pixel %s" % str(pixel)
                print str(e)
                mask[pixel] = 1
                continue

            hist = pixel_histograms.histograms[pixel]

            if not LEG:
                gs = alt_gaussians[1].params
                fit_photons = gs[0] * gs[2] * math.sqrt(2. * math.pi)
                n_photons = int(round(fit_photons, 0))
                fit_interpretation = pixel_histograms.multiphoton_and_fit_residual(
                    pixel_histograms.histograms[pixel], alt_gaussians)
                multi_photons = fit_interpretation.get_multiphoton_count()
                total_photons = n_photons + multi_photons

                if False and n_photons < 0:  # Generally, do not mask negative values; if fit is still OK
                    print "\n%d pixel %s altrn %d photons from curvefitting" % (
                        i, pixel, n_photons)
                    pixel_histograms.plot_combo(
                        pixel,
                        alt_gaussians,
                        interpretation=fit_interpretation)
                    mask[
                        pixel] = 1  # do not mask out negative pixels if the Gaussian fit is good
                    continue

                chi_squared_list.append(fit_interpretation.chi_squared())
                suspect = False  # don't know the optimal statistical test.  Histograms vary primarily by total count & # photons
                if total_photons <= 3:
                    if fit_interpretation.chi_squared(
                    ) > 2.5 or fit_interpretation.quality_factor < 5:
                        suspect = True
                elif 3 < total_photons <= 10:
                    if fit_interpretation.chi_squared(
                    ) > 5 or fit_interpretation.quality_factor < 10:
                        suspect = True
                elif 10 < total_photons <= 33:
                    if fit_interpretation.chi_squared(
                    ) > 10 or fit_interpretation.quality_factor < 20:
                        suspect = True
                elif 33 < total_photons <= 100:
                    if fit_interpretation.chi_squared(
                    ) > 20 or fit_interpretation.quality_factor < 20:
                        suspect = True
                elif 100 < total_photons <= 330:
                    if fit_interpretation.chi_squared(
                    ) > 30 or fit_interpretation.quality_factor < 25:
                        suspect = True
                elif 330 < total_photons <= 1000:
                    if fit_interpretation.chi_squared(
                    ) > 40 or fit_interpretation.quality_factor < 30:
                        suspect = True
                elif 1000 < total_photons:
                    if fit_interpretation.chi_squared(
                    ) > 50 or fit_interpretation.quality_factor < 30:
                        suspect = True

                if suspect:
                    print "\n%d pixel %s Bad quality 0/1-photon fit" % (
                        i, pixel), fit_interpretation.quality_factor
                    print "   with chi-squared %10.5f" % fit_interpretation.chi_squared(
                    )
                    print "   Suspect", suspect
                    print "%d fit photons, %d total photons" % (n_photons,
                                                                total_photons)
                    #pixel_histograms.plot_combo(pixel, alt_gaussians,
                    #                            interpretation=fit_interpretation)
                    mask[pixel] = 1
                    continue

                self.sum_img[pixel] = n_photons + multi_photons

        mask.set_selected(self.sum_img == 0, 1)
        unbound_pixel_mask = xes_finalise.cspad_unbound_pixel_mask()
        mask.set_selected(unbound_pixel_mask > 0, 1)
        bad_pixel_mask = xes_finalise.cspad2x2_bad_pixel_mask_cxi_run7()
        mask.set_selected(bad_pixel_mask > 0, 1)

        for row in range(self.sum_img.all()[0]):
            self.sum_img[row:row + 1, :].count(0)

        spectrum_focus = self.sum_img[start_row:end_row, :]
        mask_focus = mask[start_row:end_row, :]

        spectrum_focus.set_selected(mask_focus > 0, 0)

        xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)

        print "Number of rows: %i" % spectrum_focus.all()[0]
        print "Estimated no. photons counted: %i" % flex.sum(spectrum_focus)
        print "Number of images used: %i" % flex.sum(
            pixel_histograms.histograms.values()[0].slots())

        d = cspad_tbx.dpack(
            address='CxiSc1-0|Cspad2x2-0',
            data=spectrum_focus,
            distance=1,
            ccd_image_saturation=2e8,  # XXX
        )
        if run is not None: runstr = "_%04d" % run
        else: runstr = ""
        cspad_tbx.dwritef(d, output_dirname, 'sum%s_' % runstr)

        plot_x, plot_y = xes_finalise.output_spectrum(
            spectrum_focus.iround(),
            mask_focus=mask_focus,
            output_dirname=output_dirname,
            run=run)
        self.spectrum = (plot_x, plot_y)
        self.spectrum_focus = spectrum_focus
        xes_finalise.output_matlab_form(
            spectrum_focus, "%s/sum%s.m" % (output_dirname, runstr))
        print output_dirname
        print "Average chi squared is", flex.mean(
            chi_squared_list), "on %d shots" % flex.sum(hist.slots())
Exemple #7
0
    def event(self, evt, env):
        """The event() function is called for every L1Accept transition.
    XXX more?

    Previously, common-mode correction was applied only after initial
    threshold filtering.  Since the common_mode class applies the
    (lengthy) common-mode correction immediately after reading the
    image from the stream, this optimisation is currently not
    (elegantly) doable.

    @param evt Event data object, a configure object
    @param env Environment object
    """

        super(mod_hitfind, self).event(evt, env)
        if (evt.get("skip_event")):
            return

        # This module only applies to detectors for which a distance is
        # available.
        distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
        if distance is None:
            self.nfail += 1
            self.logger.warning("event(): no distance, shot skipped")
            evt.put(skip_event_flag(), "skip_event")
            return

        device = cspad_tbx.address_split(self.address)[2]

        # ***** HITFINDING ***** XXX For hitfinding it may be interesting
        # to look at the fraction of subzero pixels in the dark-corrected
        # image.
        if (self.m_threshold is not None):
            # If a threshold value is given it can be applied in one of three ways:
            #    1.  Apply it over the whole image
            if (self.m_roi is None and self.m_distl_min_peaks is None):
                vmax = flex.max(self.cspad_img)
                if (vmax < self.m_threshold):
                    if not self.m_negate_hits:
                        # Tell downstream modules to skip this event if the threshold was not met.
                        evt.put(skip_event_flag(), "skip_event")
                        return
                elif self.m_negate_hits:
                    evt.put(skip_event_flag(), "skip_event")
                    return

            #    2. Apply threshold over a rectangular region of interest.
            elif (self.m_roi is not None):
                vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
                                               self.m_roi[0]:self.m_roi[1]])
                if (vmax < self.m_threshold):
                    if not self.m_negate_hits:
                        evt.put(skip_event_flag(), "skip_event")
                        return
                elif self.m_negate_hits:
                    evt.put(skip_event_flag(), "skip_event")
                    return

            #    3. Determine the spotfinder spots within the central ASICS, and accept the
            #       image as a hit if there are m_distl_min_peaks exceeding m_threshold.
            #       As a further requirement, the peaks must exceed 2.5 * the 90-percentile
            #       pixel value of the central ASICS.  This filter was added to avoid high-background
            #       false positives.
            elif (self.m_distl_min_peaks is not None):
                if device == 'marccd':
                    self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
                    self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
                elif device == 'Rayonix':
                    self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
                    self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]

                peak_heights, outvalue = self.distl_filter(
                    self.address,
                    self.cspad_img.iround(),  # XXX correct?
                    distance,
                    self.timestamp,
                    self.wavelength)
                if ('permissive' in self.m_distl_flags):
                    number_of_accepted_peaks = (peak_heights >
                                                self.m_threshold).count(True)
                else:
                    number_of_accepted_peaks = ((
                        peak_heights > self.m_threshold).__and__(
                            outvalue == 0)).count(True)

                sec, ms = cspad_tbx.evt_time(evt)
                evt_time = sec + ms / 1000
                self.stats_logger.info("BRAGG %.3f %d" %
                                       (evt_time, number_of_accepted_peaks))

                skip_event = False
                if number_of_accepted_peaks < self.m_distl_min_peaks:
                    self.logger.info(
                        "Subprocess %02d: Spotfinder NO  HIT image #%05d @ %s; %d spots > %d"
                        % (env.subprocess(), self.nshots, self.timestamp,
                           number_of_accepted_peaks, self.m_threshold))

                    if not self.m_negate_hits:
                        skip_event = True
                else:
                    self.logger.info(
                        "Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d"
                        % (env.subprocess(), self.nshots, self.timestamp,
                           number_of_accepted_peaks, self.m_threshold))

                    if self.m_negate_hits:
                        skip_event = True

                if skip_event:
                    if self.m_db_logging:
                        # log misses to the database
                        self.queue_entry(
                            (self.trial, evt.run(), "%.3f" % evt_time,
                             number_of_accepted_peaks, distance, self.sifoil,
                             self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                             0, 0, self.m_db_tags))
                    evt.put(skip_event_flag(), "skip_event")
                    return
                # the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
                # hit will not be logged in the db.
                evt.put(number_of_accepted_peaks, 'sfspots')

        self.logger.info("Subprocess %02d: process image #%05d @ %s" %
                         (env.subprocess(), self.nshots, self.timestamp))

        # See r17537 of mod_average.py.
        if device == 'Cspad':
            pixel_size = cspad_tbx.pixel_size
            saturated_value = cspad_tbx.cspad_saturated_value
        elif device == 'marccd':
            pixel_size = evt.get("marccd_pixel_size")
            saturated_value = evt.get("marccd_saturated_value")
        elif device == 'Rayonix':
            pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
            saturated_value = rayonix_tbx.rayonix_saturated_value

        d = cspad_tbx.dpack(
            active_areas=self.active_areas,
            address=self.address,
            beam_center_x=pixel_size * self.beam_center[0],
            beam_center_y=pixel_size * self.beam_center[1],
            data=self.cspad_img.iround(),  # XXX ouch!
            distance=distance,
            pixel_size=pixel_size,
            saturated_value=saturated_value,
            timestamp=self.timestamp,
            wavelength=self.wavelength,
            xtal_target=self.m_xtal_target)

        if (self.m_dispatch == "index"):
            import sys
            from xfel.cxi.integrate_image_api import integrate_one_image
            info = integrate_one_image(
                d,
                integration_dirname=self.m_integration_dirname,
                integration_basename=self.m_integration_basename)
            sys.stdout = sys.__stdout__
            sys.stderr = sys.__stderr__

            indexed = info is not None and hasattr(info, 'spotfinder_results')
            if self.m_progress_logging:
                if self.m_db_version == 'v1':
                    if indexed:
                        # integration pickle dictionary is available here as info.last_saved_best
                        if info.last_saved_best[
                                "identified_isoform"] is not None:
                            #print info.last_saved_best.keys()
                            from cxi_xdr_xes.cftbx.cspad_ana import db
                            dbobj = db.dbconnect(self.m_db_host,
                                                 self.m_db_name,
                                                 self.m_db_user,
                                                 self.m_db_password)
                            cursor = dbobj.cursor()
                            if info.last_saved_best[
                                    "identified_isoform"] in self.isoforms:
                                PM, indices, miller_id = self.isoforms[
                                    info.last_saved_best["identified_isoform"]]
                            else:
                                from xfel.xpp.progress_support import progress_manager
                                PM = progress_manager(info.last_saved_best,
                                                      self.m_db_experiment_tag,
                                                      self.m_trial_id,
                                                      self.m_rungroup_id,
                                                      evt.run())
                                indices, miller_id = PM.get_HKL(cursor)
                                # cache these as they don't change for a given isoform
                                self.isoforms[info.last_saved_best[
                                    "identified_isoform"]] = PM, indices, miller_id
                            if self.m_sql_buffer_size > 1:
                                self.queue_progress_entry(
                                    PM.scale_frame_detail(self.timestamp,
                                                          cursor,
                                                          do_inserts=False))
                            else:
                                PM.scale_frame_detail(self.timestamp,
                                                      cursor,
                                                      do_inserts=True)
                                dbobj.commit()
                                cursor.close()
                                dbobj.close()
                elif self.m_db_version == 'v2':
                    key_low = 'cctbx.xfel.radial_average.two_theta_low'
                    key_high = 'cctbx.xfel.radial_average.two_theta_high'
                    tt_low = evt.get(key_low)
                    tt_high = evt.get(key_high)

                    from xfel.ui.db.dxtbx_db import log_frame
                    if indexed:
                        n_spots = len(info.spotfinder_results.images[
                            info.frames[0]]['spots_total'])
                    else:
                        sfspots = evt.get('sfspots')
                        if sfspots is None:
                            if info is None or not isinstance(info, int):
                                n_spots = 0
                            else:
                                n_spots = info
                        else:
                            n_spots = sfspots

                    if indexed:
                        known_setting = info.horizons_phil.known_setting
                        indexed_setting = info.organizer.info[
                            'best_integration']['counter']
                        if known_setting is None or known_setting == indexed_setting:
                            from xfel.command_line.frame_unpickler import construct_reflection_table_and_experiment_list
                            c = construct_reflection_table_and_experiment_list(
                                info.last_saved_best,
                                None,
                                pixel_size,
                                proceed_without_image=True)
                            c.assemble_experiments()
                            c.assemble_reflections()
                            log_frame(c.experiment_list, c.reflections,
                                      self.db_params, evt.run(), n_spots,
                                      self.timestamp, tt_low, tt_high)
                        else:
                            print(
                                "Not logging %s, wrong bravais setting (expecting %d, got %d)"
                                % (self.timestamp, known_setting,
                                   indexed_setting))
                    else:
                        log_frame(None, None, self.db_params, evt.run(),
                                  n_spots, self.timestamp, tt_low, tt_high)

            if self.m_db_logging:
                sec, ms = cspad_tbx.evt_time(evt)
                evt_time = sec + ms / 1000
                sfspots = evt.get('sfspots')
                if sfspots is None:
                    if indexed:
                        n_spots = len(info.spotfinder_results.images[
                            info.frames[0]]['spots_total'])
                    else:
                        n_spots = 0
                else:
                    n_spots = sfspots

                if indexed:
                    mosaic_bloc_rotation = info.last_saved_best.get(
                        'ML_half_mosaicity_deg', [0])[0]
                    mosaic_block_size = info.last_saved_best.get(
                        'ML_domain_size_ang', [0])[0]
                    ewald_proximal_volume = info.last_saved_best.get(
                        'ewald_proximal_volume', [0])[0]

                    obs = info.last_saved_best['observations'][0]
                    cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell(
                    ).parameters()
                    pointgroup = info.last_saved_best['pointgroup']
                    resolution = obs.d_min()
                else:
                    mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
                      cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0

                self.queue_entry(
                    (self.trial, evt.run(), "%.3f" % evt_time, n_spots,
                     distance, self.sifoil, self.wavelength, indexed,
                     mosaic_bloc_rotation, mosaic_block_size,
                     ewald_proximal_volume, pointgroup, cell_a, cell_b, cell_c,
                     cell_alpha, cell_beta, cell_gamma, resolution,
                     self.m_db_tags))

            if (not indexed):
                evt.put(skip_event_flag(), "skip_event")
                return

        elif (self.m_dispatch == "nop"):
            pass

        elif (self.m_dispatch == "view"):  #interactive image viewer

            args = ["indexing.data=dummy"]
            detector_format_version = detector_format_function(
                self.address, evt.GetTime())
            if detector_format_version is not None:
                args += [
                    "distl.detector_format_version=%" % detector_format_version
                ]

            from xfel.phil_preferences import load_cxi_phil
            horizons_phil = load_cxi_phil(self.m_xtal_target, args)
            horizons_phil.indexing.data = d

            from xfel.cxi import display_spots
            display_spots.parameters.horizons_phil = horizons_phil
            display_spots.wrapper_of_callback().display(
                horizons_phil.indexing.data)

        elif (self.m_dispatch == "spots"):  #interactive spotfinder viewer

            args = ["indexing.data=dummy"]
            detector_format_version = detector_format_function(
                self.address, evt.GetTime())
            if detector_format_version is not None:
                args += [
                    "distl.detector_format_version=%s" %
                    detector_format_version
                ]

            from xfel.phil_preferences import load_cxi_phil
            horizons_phil = load_cxi_phil(self.m_xtal_target, args)
            horizons_phil.indexing.data = d

            from xfel.cxi import display_spots
            display_spots.parameters.horizons_phil = horizons_phil

            from rstbx.new_horizons.index import pre_indexing_validation, pack_names
            pre_indexing_validation(horizons_phil)
            imagefile_arguments = pack_names(horizons_phil)
            horizons_phil.persist.show()
            from spotfinder.applications import signal_strength
            info = signal_strength.run_signal_strength_core(
                horizons_phil, imagefile_arguments)

            work = display_spots.wrapper_of_callback(info)
            work.display_with_callback(horizons_phil.indexing.data)

        elif (self.m_dispatch == "write_dict"):
            self.logger.warning(
                "event(): deprecated dispatch 'write_dict', use mod_dump instead"
            )
            if (self.m_out_dirname is not None
                    or self.m_out_basename is not None):
                cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)

        # Diagnostic message emitted only when all the processing is done.
        if (env.subprocess() >= 0):
            self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
                             (env.subprocess(), self.nshots, self.timestamp))
        else:
            self.logger.info("Accepted #%05d @ %s" %
                             (self.nshots, self.timestamp))
Exemple #8
0
  def event(self, evt, env):
    """The event() function is called for every L1Accept transition.
    XXX more?

    Previously, common-mode correction was applied only after initial
    threshold filtering.  Since the common_mode class applies the
    (lengthy) common-mode correction immediately after reading the
    image from the stream, this optimisation is currently not
    (elegantly) doable.

    @param evt Event data object, a configure object
    @param env Environment object
    """

    super(mod_hitfind, self).event(evt, env)
    if (evt.get("skip_event")):
      return

    # This module only applies to detectors for which a distance is
    # available.
    distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
    if distance is None:
      self.nfail += 1
      self.logger.warning("event(): no distance, shot skipped")
      evt.put(skip_event_flag(), "skip_event")
      return

    device = cspad_tbx.address_split(self.address)[2]

    # ***** HITFINDING ***** XXX For hitfinding it may be interesting
    # to look at the fraction of subzero pixels in the dark-corrected
    # image.
    if (self.m_threshold is not None):
      # If a threshold value is given it can be applied in one of three ways:
      #    1.  Apply it over the whole image
      if (self.m_roi is None and self.m_distl_min_peaks is None):
        vmax = flex.max(self.cspad_img)
        if (vmax < self.m_threshold):
          if not self.m_negate_hits:
            # Tell downstream modules to skip this event if the threshold was not met.
            evt.put(skip_event_flag(), "skip_event")
            return
        elif self.m_negate_hits:
          evt.put(skip_event_flag(), "skip_event")
          return

      #    2. Apply threshold over a rectangular region of interest.
      elif (self.m_roi is not None):
        vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
                                       self.m_roi[0]:self.m_roi[1]])
        if (vmax < self.m_threshold):
          if not self.m_negate_hits:
            evt.put(skip_event_flag(), "skip_event")
            return
        elif self.m_negate_hits:
          evt.put(skip_event_flag(), "skip_event")
          return

      #    3. Determine the spotfinder spots within the central ASICS, and accept the
      #       image as a hit if there are m_distl_min_peaks exceeding m_threshold.
      #       As a further requirement, the peaks must exceed 2.5 * the 90-percentile
      #       pixel value of the central ASICS.  This filter was added to avoid high-background
      #       false positives.
      elif (self.m_distl_min_peaks is not None):
        if device == 'marccd':
          self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
          self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
        elif device == 'Rayonix':
          self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
          self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]

        peak_heights,outvalue = self.distl_filter(
          self.address,
          self.cspad_img.iround(), # XXX correct?
          distance,
          self.timestamp,
          self.wavelength)
        if ('permissive' in self.m_distl_flags):
          number_of_accepted_peaks = (peak_heights > self.m_threshold).count(True)
        else:
          number_of_accepted_peaks = ((peak_heights > self.m_threshold).__and__(outvalue==0)).count(True)

        sec,ms = cspad_tbx.evt_time(evt)
        evt_time = sec + ms/1000
        self.stats_logger.info("BRAGG %.3f %d" %(evt_time, number_of_accepted_peaks))

        skip_event = False
        if number_of_accepted_peaks < self.m_distl_min_peaks:
          self.logger.info("Subprocess %02d: Spotfinder NO  HIT image #%05d @ %s; %d spots > %d" %(
            env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))

          if not self.m_negate_hits:
            skip_event = True
        else:
          self.logger.info("Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d" %(
            env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))

          if self.m_negate_hits:
            skip_event = True

        if skip_event:
          if self.m_db_logging:
            # log misses to the database
            self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, number_of_accepted_peaks, distance,
                              self.sifoil, self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, self.m_db_tags))
          evt.put(skip_event_flag(), "skip_event")
          return
        # the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
        # hit will not be logged in the db.
        evt.put(number_of_accepted_peaks, 'sfspots')

    self.logger.info("Subprocess %02d: process image #%05d @ %s" %
                     (env.subprocess(), self.nshots, self.timestamp))

    # See r17537 of mod_average.py.
    if device == 'Cspad':
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
    elif device == 'marccd':
      pixel_size = evt.get("marccd_pixel_size")
      saturated_value = evt.get("marccd_saturated_value")
    elif device == 'Rayonix':
      pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
      saturated_value = rayonix_tbx.rayonix_saturated_value

    d = cspad_tbx.dpack(
      active_areas=self.active_areas,
      address=self.address,
      beam_center_x=pixel_size * self.beam_center[0],
      beam_center_y=pixel_size * self.beam_center[1],
      data=self.cspad_img.iround(), # XXX ouch!
      distance=distance,
      pixel_size=pixel_size,
      saturated_value=saturated_value,
      timestamp=self.timestamp,
      wavelength=self.wavelength,
      xtal_target=self.m_xtal_target)

    if (self.m_dispatch == "index"):
      import sys
      from xfel.cxi.integrate_image_api import integrate_one_image
      info = integrate_one_image(d,
                                 integration_dirname  = self.m_integration_dirname,
                                 integration_basename = self.m_integration_basename)
      sys.stdout = sys.__stdout__
      sys.stderr = sys.__stderr__

      indexed = info is not None
      if indexed and self.m_progress_logging:
        # integration pickle dictionary is available here as info.last_saved_best
        if info.last_saved_best["identified_isoform"] is not None:
          #print info.last_saved_best.keys()
          from cxi_xdr_xes.cftbx.cspad_ana import db
          dbobj = db.dbconnect(self.m_db_host, self.m_db_name, self.m_db_user, self.m_db_password)
          cursor = dbobj.cursor()
          if info.last_saved_best["identified_isoform"] in self.isoforms:
            PM, indices, miller_id = self.isoforms[info.last_saved_best["identified_isoform"]]
          else:
            from xfel.xpp.progress_support import progress_manager
            PM = progress_manager(info.last_saved_best,self.m_db_experiment_tag, self.m_trial_id, self.m_rungroup_id, evt.run())
            indices, miller_id = PM.get_HKL(cursor)
            # cache these as they don't change for a given isoform
            self.isoforms[info.last_saved_best["identified_isoform"]] = PM, indices, miller_id
          if self.m_sql_buffer_size > 1:
            self.queue_progress_entry(PM.scale_frame_detail(self.timestamp,cursor,do_inserts=False))
          else:
            PM.scale_frame_detail(self.timestamp,cursor,do_inserts=True)
            dbobj.commit()
            cursor.close()
            dbobj.close()

      if self.m_db_logging:
        sec,ms = cspad_tbx.evt_time(evt)
        evt_time = sec + ms/1000
        sfspots = evt.get('sfspots')
        if sfspots is None:
          if indexed:
            n_spots = len(info.spotfinder_results.images[info.frames[0]]['spots_total'])
          else:
            n_spots = 0
        else:
          n_spots = sfspots

        if indexed:
          mosaic_bloc_rotation = info.last_saved_best.get('ML_half_mosaicity_deg', [0])[0]
          mosaic_block_size = info.last_saved_best.get('ML_domain_size_ang', [0])[0]
          ewald_proximal_volume = info.last_saved_best.get('ewald_proximal_volume', [0])[0]

          obs = info.last_saved_best['observations'][0]
          cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell().parameters()
          pointgroup = info.last_saved_best['pointgroup']
          resolution = obs.d_min()
        else:
          mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
            cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0

        self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, n_spots, distance,
                          self.sifoil, self.wavelength, indexed, mosaic_bloc_rotation,
                          mosaic_block_size, ewald_proximal_volume, pointgroup, cell_a,
                          cell_b, cell_c, cell_alpha, cell_beta, cell_gamma, resolution,
                          self.m_db_tags))

      if (not indexed):
        evt.put(skip_event_flag(), "skip_event")
        return

    elif (self.m_dispatch == "nop"):
      pass

    elif (self.m_dispatch == "view"): #interactive image viewer

      args = ["indexing.data=dummy"]
      detector_format_version = detector_format_function(
        self.address, evt.GetTime())
      if detector_format_version is not None:
        args += ["distl.detector_format_version=%" % detector_format_version]

      from xfel.phil_preferences import load_cxi_phil
      horizons_phil = load_cxi_phil(self.m_xtal_target, args)
      horizons_phil.indexing.data = d

      from xfel.cxi import display_spots
      display_spots.parameters.horizons_phil = horizons_phil
      display_spots.wrapper_of_callback().display(horizons_phil.indexing.data)

    elif (self.m_dispatch == "spots"): #interactive spotfinder viewer

      args = ["indexing.data=dummy"]
      detector_format_version = detector_format_function(
        self.address, evt.GetTime())
      if detector_format_version is not None:
        args += ["distl.detector_format_version=%s" % detector_format_version]

      from xfel.phil_preferences import load_cxi_phil
      horizons_phil = load_cxi_phil(self.m_xtal_target, args)
      horizons_phil.indexing.data = d

      from xfel.cxi import display_spots
      display_spots.parameters.horizons_phil = horizons_phil

      from rstbx.new_horizons.index import pre_indexing_validation,pack_names
      pre_indexing_validation(horizons_phil)
      imagefile_arguments = pack_names(horizons_phil)
      horizons_phil.persist.show()
      from spotfinder.applications import signal_strength
      info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments)

      work = display_spots.wrapper_of_callback(info)
      work.display_with_callback(horizons_phil.indexing.data)

    elif (self.m_dispatch == "write_dict"):
      self.logger.warning(
        "event(): deprecated dispatch 'write_dict', use mod_dump instead")
      if (self.m_out_dirname  is not None or
          self.m_out_basename is not None):
        cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)

    # Diagnostic message emitted only when all the processing is done.
    if (env.subprocess() >= 0):
      self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
                       (env.subprocess(), self.nshots, self.timestamp))
    else:
      self.logger.info("Accepted #%05d @ %s" %
                       (self.nshots, self.timestamp))
Exemple #9
0
    def __init__(self,
                 pixel_histograms,
                 output_dirname=".",
                 gain_map_path=None,
                 gain_map=None,
                 method="photon_counting",
                 estimated_gain=30,
                 nproc=None,
                 photon_threshold=2 / 3,
                 roi=None,
                 run=None):
        assert method in ("sum_adu", "photon_counting")
        self.sum_img = flex.double(flex.grid(
            370, 391), 0)  # XXX define the image size some other way?
        gain_img = flex.double(self.sum_img.accessor(), 0)

        assert [gain_map, gain_map_path].count(None) > 0
        if gain_map_path is not None:
            d = easy_pickle.load(gain_map_path)
            gain_map = d["DATA"]

        two_photon_threshold = photon_threshold + 1

        mask = flex.int(self.sum_img.accessor(), 0)

        start_row = 370
        end_row = 0
        print(len(pixel_histograms.histograms))

        pixels = list(pixel_histograms.pixels())
        n_pixels = len(pixels)
        if roi is not None:
            for k, (i, j) in enumerate(reversed(pixels)):
                if (i < roi[2] or i > roi[3] or j < roi[0] or j > roi[1]):
                    del pixels[n_pixels - k - 1]

        if gain_map is None:
            fixed_func = pixel_histograms.fit_one_histogram
        else:

            def fixed_func(pixel):
                return pixel_histograms.fit_one_histogram(pixel, n_gaussians=1)

        results = None
        if nproc is None: nproc = easy_mp.Auto
        nproc = easy_mp.get_processes(nproc)
        print("nproc: ", nproc)

        stdout_and_results = easy_mp.pool_map(
            processes=nproc,
            fixed_func=fixed_func,
            args=pixels,
            func_wrapper="buffer_stdout_stderr")
        results = [r for so, r in stdout_and_results]

        gains = flex.double()

        for i, pixel in enumerate(pixels):
            start_row = min(start_row, pixel[0])
            end_row = max(end_row, pixel[0])
            n_photons = 0
            if results is None:
                # i.e. not multiprocessing
                try:
                    gaussians = pixel_histograms.fit_one_histogram(pixel)
                except RuntimeError as e:
                    print("Error fitting pixel %s" % str(pixel))
                    print(str(e))
                    mask[pixel] = 1
                    continue
            else:
                gaussians = results[i]
            hist = pixel_histograms.histograms[pixel]
            if gaussians is None:
                # Presumably the peak fitting failed in some way
                print("Skipping pixel %s" % str(pixel))
                continue
            zero_peak_diff = gaussians[0].params[1]
            if gain_map is None:
                try:
                    view_pixel_histograms.check_pixel_histogram_fit(
                        hist, gaussians)
                except view_pixel_histograms.PixelFitError as e:
                    print("PixelFitError:", str(pixel), str(e))
                    mask[pixel] = 1
                    continue
                gain = gaussians[1].params[1] - gaussians[0].params[1]
                gain_img[pixel] = gain
                gain_ratio = gain / estimated_gain
            else:
                gain = gain_map[pixel]
                if gain == 0:
                    print("bad gain!!!!!", pixel)
                    continue
                gain = 30 / gain
                gain_ratio = 1 / gain
            gains.append(gain)

            #for g in gaussians:
            #sigma = abs(g.params[2])
            #if sigma < 1 or sigma > 10:
            #print "bad sigma!!!!!", pixel, sigma
            #mask[pixel] = 1
            #continue
            if method == "sum_adu":
                sum_adu = 0
                one_photon_cutoff, two_photon_cutoff = [
                    (threshold * gain + zero_peak_diff)
                    for threshold in (photon_threshold, two_photon_threshold)
                ]
                i_one_photon_cutoff = hist.get_i_slot(one_photon_cutoff)
                slots = hist.slots().as_double()
                slot_centers = hist.slot_centers()
                slots -= gaussians[0](slot_centers)
                for j in range(i_one_photon_cutoff, len(slots)):
                    center = slot_centers[j]
                    sum_adu += slots[j] * (center - zero_peak_diff) * 30 / gain

                self.sum_img[pixel] = sum_adu
            elif method == "photon_counting":
                one_photon_cutoff, two_photon_cutoff = [
                    (threshold * gain + zero_peak_diff)
                    for threshold in (photon_threshold, two_photon_threshold)
                ]
                i_one_photon_cutoff = hist.get_i_slot(one_photon_cutoff)
                i_two_photon_cutoff = hist.get_i_slot(two_photon_cutoff)
                slots = hist.slots()
                for j in range(i_one_photon_cutoff, len(slots)):
                    if j == i_one_photon_cutoff:
                        center = hist.slot_centers()[j]
                        upper = center + 0.5 * hist.slot_width()
                        n_photons += int(
                            round((upper - one_photon_cutoff) /
                                  hist.slot_width() * slots[j]))
                    elif j == i_two_photon_cutoff:
                        center = hist.slot_centers()[j]
                        upper = center + 0.5 * hist.slot_width()
                        n_photons += 2 * int(
                            round((upper - two_photon_cutoff) /
                                  hist.slot_width() * slots[j]))
                    elif j < i_two_photon_cutoff:
                        n_photons += int(round(slots[j]))
                    else:
                        n_photons += 2 * int(round(slots[j]))
                self.sum_img[pixel] = n_photons

        stats = scitbx.math.basic_statistics(gains)
        print("gain statistics:")
        stats.show()

        mask.set_selected(self.sum_img == 0, 1)
        unbound_pixel_mask = xes_finalise.cspad_unbound_pixel_mask()
        mask.set_selected(unbound_pixel_mask > 0, 1)
        bad_pixel_mask = xes_finalise.cspad2x2_bad_pixel_mask_cxi_run7()
        mask.set_selected(bad_pixel_mask > 0, 1)

        for row in range(self.sum_img.all()[0]):
            self.sum_img[row:row + 1, :].count(0)

        spectrum_focus = self.sum_img[start_row:end_row, :]
        mask_focus = mask[start_row:end_row, :]

        spectrum_focus.set_selected(mask_focus > 0, 0)

        xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)

        print("Number of rows: %i" % spectrum_focus.all()[0])
        print("Estimated no. photons counted: %i" % flex.sum(spectrum_focus))
        print("Number of images used: %i" %
              flex.sum(pixel_histograms.histograms.values()[0].slots()))

        d = cspad_tbx.dpack(
            address='CxiSc1-0|Cspad2x2-0',
            data=spectrum_focus,
            distance=1,
            ccd_image_saturation=2e8,  # XXX
        )
        if run is not None: runstr = "_%04d" % run
        else: runstr = ""
        cspad_tbx.dwritef(d, output_dirname, 'sum%s_' % runstr)

        if gain_map is None:
            gain_map = flex.double(gain_img.accessor(), 0)
            img_sel = (gain_img > 0).as_1d()
            d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0',
                                data=gain_img,
                                distance=1)
            cspad_tbx.dwritef(d, output_dirname, 'raw_gain_map_')
            gain_map.as_1d().set_selected(img_sel.iselection(),
                                          1 / gain_img.as_1d().select(img_sel))
            gain_map /= flex.mean(gain_map.as_1d().select(img_sel))
            d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0',
                                data=gain_map,
                                distance=1)
            cspad_tbx.dwritef(d, output_dirname, 'gain_map_')

        plot_x, plot_y = xes_finalise.output_spectrum(
            spectrum_focus.iround(),
            mask_focus=mask_focus,
            output_dirname=output_dirname,
            run=run)
        self.spectrum = (plot_x, plot_y)
        self.spectrum_focus = spectrum_focus

        xes_finalise.output_matlab_form(
            spectrum_focus, "%s/sum%s.m" % (output_dirname, runstr))
        print(output_dirname)
Exemple #10
0
  def endjob(self, obj1, obj2=None):
    """The endjob() function writes the mean and standard deviation images
    to disk.

    @param evt Event object (psana only)
    @param env Environment object
    """
    if obj2 is None:
      env = obj1
    else:
      evt = obj1
      env = obj2

    stats = super(mod_average, self).endjob(env)
    if stats is None:
      return

    device = cspad_tbx.address_split(self.address)[2]
    if device == 'Andor':
      beam_center = (0, 0) # XXX Fiction!
      pixel_size = 13.5e-3 # XXX Should not be hardcoded here!
      saturated_value = 10000
    elif device == 'Cspad' or device == 'Cspad2x2':
      beam_center = self.beam_center
      pixel_size = cspad_tbx.pixel_size
      saturated_value = cspad_tbx.cspad_saturated_value
    elif device == 'marccd':
      beam_center = tuple(t // 2 for t in d['mean_img'].focus())
      pixel_size = 0.079346
      saturated_value = 2**16 - 1

    if stats['nmemb'] > 0:
      if self.avg_dirname  is not None or \
         self.avg_basename is not None or \
         self._mean_out    is not None:
        d = cspad_tbx.dpack(
          active_areas=self.active_areas,
          address=self.address,
          beam_center_x=pixel_size * beam_center[0],
          beam_center_y=pixel_size * beam_center[1],
          data=stats['mean_img'],
          distance=stats['distance'],
          pixel_size=pixel_size,
          saturated_value=saturated_value,
          timestamp=cspad_tbx.evt_timestamp(stats['time']),
          wavelength=stats['wavelength'])
        if self._mean_out is not None:
          p = cspad_tbx.dwritef2(d, self._mean_out)
        else:
          p = cspad_tbx.dwritef(d, self.avg_dirname, self.avg_basename)
        self.logger.info("Average written to %s" % p)

      if self.stddev_dirname  is not None or \
         self.stddev_basename is not None or \
         self._std_out    is not None:
        d = cspad_tbx.dpack(
          active_areas=self.active_areas,
          address=self.address,
          beam_center_x=pixel_size * beam_center[0],
          beam_center_y=pixel_size * beam_center[1],
          data=stats['std_img'],
          distance=stats['distance'],
          pixel_size=pixel_size,
          saturated_value=saturated_value,
          timestamp=cspad_tbx.evt_timestamp(stats['time']),
          wavelength=stats['wavelength'])
        if self._std_out is not None:
          p = cspad_tbx.dwritef2(d, self._std_out)
        else:
          p = cspad_tbx.dwritef(d, self.stddev_dirname, self.stddev_basename)
        self.logger.info("Standard deviation written to %s" % p)

      if self.max_dirname  is not None or \
         self.max_basename is not None or \
         self._max_out    is not None:
        d = cspad_tbx.dpack(
          active_areas=self.active_areas,
          address=self.address,
          beam_center_x=pixel_size * beam_center[0],
          beam_center_y=pixel_size * beam_center[1],
          data=stats['max_img'],
          distance=stats['distance'],
          pixel_size=pixel_size,
          saturated_value=saturated_value,
          timestamp=cspad_tbx.evt_timestamp(stats['time']),
          wavelength=stats['wavelength'])
        if self._max_out is not None:
          p = cspad_tbx.dwritef2(d, self._max_out)
        else:
          p = cspad_tbx.dwritef(d, self.max_dirname, self.max_basename)
        self.logger.info("Max written to %s" % p)

    if stats['nfail'] == 0:
      self.logger.info("%d images processed" % stats['nmemb'])
    else:
      self.logger.warning(
        "%d images processed, %d failed" % (stats['nmemb'], stats['nfail']))
  def __init__(self,
               runs,
               output_dirname=".",
               roi=None):
    avg_basename="avg_"
    stddev_basename="stddev"
    self.sum_img = None
    self.sumsq_img = None
    self.nmemb = 0
    self.roi = cspad_tbx.getOptROI(roi)
    self.unbound_pixel_mask = cspad_unbound_pixel_mask()
    for i_run, run in enumerate(runs):
      run_scratch_dir = run
      result = finalise_one_run(run_scratch_dir)
      if result.sum_img is None: continue
      if self.sum_img is None:
        self.sum_img = result.sum_img
        self.sumsq_img = result.sumsq_img
      else:
        self.sum_img += result.sum_img
        self.sumsq_img += result.sumsq_img
      self.nmemb += result.nmemb

    self.avg_img = self.sum_img.as_double() / self.nmemb
    self.stddev_img = flex.sqrt((self.sumsq_img.as_double() - self.sum_img.as_double() * self.avg_img) / (self.nmemb - 1))

    self.mask = flex.int(self.sum_img.accessor(), 0)
    self.mask.set_selected(self.sum_img == 0, 1)
    self.mask.set_selected(self.unbound_pixel_mask > 0, 1)

    if (output_dirname is not None and
        avg_basename is not None):
      if (not os.path.isdir(output_dirname)):
        os.makedirs(output_dirname)
      d = cspad_tbx.dpack(
        address='CxiSc1-0|Cspad2x2-0',
        data=self.avg_img,
        distance=1,
      )
      cspad_tbx.dwritef(d, output_dirname, avg_basename)
      d = cspad_tbx.dpack(
        address='CxiSc1-0|Cspad2x2-0',
        data=self.sum_img,
        distance=1,
      )
      cspad_tbx.dwritef(d, output_dirname, "sum_")
      if 1:
        output_image(self.avg_img, "%s/avg.png" %output_dirname)
        output_image(self.avg_img, "%s/avg_inv.png" %output_dirname, invert=True)

      if 1:
        output_matlab_form(self.sum_img, "%s/sum.m" %output_dirname)
        output_matlab_form(self.avg_img, "%s/avg.m" %output_dirname)
        output_matlab_form(self.stddev_img, "%s/stddev.m" %output_dirname)

    if (stddev_basename is not None):
      d = cspad_tbx.dpack(
        address='CxiSc1-0|Cspad2x2-0',
        data=self.stddev_img,
        distance=1,
      )
      cspad_tbx.dwritef(d, output_dirname, stddev_basename)

      # XXX we should really figure out automatically the area where the spectrum is
      #write an integrated spectrum from lines 186-227
      #spectrum_focus = self.sum_img.as_numpy_array()[186:228,:]
      img = self.sum_img
      if self.roi is None:
        spectrum_focus = img
        mask_focus = self.mask
      else:
        slices = (slice(self.roi[2],self.roi[3]), slice(self.roi[0],self.roi[1]))
        spectrum_focus = img[slices]
        mask_focus = self.mask[slices]
      if False:
        from matplotlib import pylab
        pylab.imshow(spectrum_focus.as_numpy_array())
        pylab.show()

    output_spectrum(spectrum_focus, mask_focus=mask_focus,
                    output_dirname=output_dirname)

    print "Total number of images used from %i runs: %i" %(i_run+1, self.nmemb)
    xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)

    print "Number of rows: %i" %spectrum_focus.all()[0]
    print "Estimated no. photons counted: %i" %flex.sum(spectrum_focus)
    print "Number of images used: %i" %flex.sum(
      pixel_histograms.histograms.values()[0].slots())

    d = cspad_tbx.dpack(
      address='CxiSc1-0|Cspad2x2-0',
      data=spectrum_focus,
      distance=1,
      ccd_image_saturation=2e8, # XXX
    )
    if run is not None: runstr="_%04d"%run
    else: runstr=""
    cspad_tbx.dwritef(d, output_dirname, 'sum%s_'%runstr)

    if gain_map is None:
      gain_map = flex.double(gain_img.accessor(), 0)
      img_sel = (gain_img > 0).as_1d()
      d = cspad_tbx.dpack(
        address='CxiSc1-0|Cspad2x2-0',
        data=gain_img,
        distance=1
      )
      cspad_tbx.dwritef(d, output_dirname, 'raw_gain_map_')
      gain_map.as_1d().set_selected(img_sel.iselection(), 1/gain_img.as_1d().select(img_sel))
      gain_map /= flex.mean(gain_map.as_1d().select(img_sel))
      d = cspad_tbx.dpack(
        address='CxiSc1-0|Cspad2x2-0',
        data=gain_map,