def beginjob(self, evt, env):
"""The beginjob() function does one-time initialisation from
event- or environment data. It is called at an XTC configure
transition.
@param evt Event data object, a configure object
@param env Environment object
"""
from os import makedirs, path
super(mod_ledge, self).beginjob(evt, env)
self._reset_counters()
self._nframes = 0
# XXX Don't really want to store the stream, but how to properly
# close it on exit?
self._table_path = cspad_tbx.pathsubst(self._table_path, evt, env)
if not path.isdir(path.dirname(self._table_path)):
makedirs(path.dirname(self._table_path))
self._stream_table = open(self._table_path, mode="wb", buffering=1)
# Initial camera settings requested by Rolf. The Andor's region
# of interest is defined by width, height, orgX, and orgY,
# independent of the binning. The camera has model number
# DO936N-M0W-#BN. Note that the configuration may change on an
# event-basis. XXX Does the width really correspond to X, and the
# height to Y?
config = cspad_tbx.getConfig(self.address, env)
if config is not None:
self.logger.info(
"Initial effective frame size: %dx%d"
% (config.width() // config.binX(), config.height() // config.binY())
)
self.logger.info(
"Initial region of interest: (%d, %d) - (%d, %d)"
% (config.orgX(), config.orgY(), config.orgX() + config.width(), config.orgY() + config.height())
)
if config.highCapacity() >= 0 and config.highCapacity() <= 1:
self.logger.info(
"Initial output mode: %s" % ("high sensitivity", "high capacity")[config.highCapacity()]
)
if config.readoutSpeedIndex() >= 0 and config.readoutSpeedIndex() <= 3:
self.logger.info(
"Initial pixel readout rate: %s" % ("5 MHz", "3 MHz", "1 MHz", "50 kHz")[config.readoutSpeedIndex()]
)
if config.gainIndex() >= 0 and config.gainIndex() <= 2:
self.logger.info("Initial pre-amplifier gain: %s" % ("1x", "2x", "4x")[config.gainIndex()])
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(common_mode_correction, self).event(evt, env)
if (evt.get("skip_event")):
return
if not hasattr(self, 'active_areas') or self.active_areas is None or \
not hasattr(self, 'beam_center') or self.beam_center is None:
if self.address == 'XppGon-0|marccd-0':
# The mod_mar module needs to have been called before this one
# to set this up. The MAR does not have a configure object.
self.beam_center = evt.get("marccd_beam_center")
self.active_areas = evt.get("marccd_active_areas")
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
pass # bc and aa set in the beginjob function
elif self.address == 'XppGon-0|Cspad-0':
# Load the active areas as determined from the optical metrology
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(self.address, reverse_timestamp(self.timestamp)[0])
assert version_lookup is not None
self.active_areas = xpp_active_areas[version_lookup]['active_areas']
self.beam_center = [1765 // 2, 1765 // 2]
else:
(self.beam_center, self.active_areas) = \
cspad_tbx.cbcaa(cspad_tbx.getConfig(self.address, env), self.sections)
if self.filter_laser_1_status is not None:
if (self.laser_1_status.status != self.filter_laser_1_status or
(self.laser_1_ms_since_change is not None and
self.laser_1_ms_since_change < self.filter_laser_wait_time)):
evt.put(skip_event_flag(), "skip_event")
return
if self.filter_laser_4_status is not None:
if (self.laser_4_status.status != self.filter_laser_4_status or
(self.laser_4_ms_since_change is not None and
self.laser_4_ms_since_change < self.filter_laser_wait_time)):
evt.put(skip_event_flag(), "skip_event")
return
# Early return if the full detector image is already stored in the
# event. Otherwise, get it from the stream as a double-precision
# floating-point flex array. XXX It is probably not safe to key
# the image on self.address, so we should come up with our own
# namespace. XXX Misnomer--could be CAMP, too
self.cspad_img = evt.get(self.address)
if self.cspad_img is not None:
return
if self.address == 'XppGon-0|Cspad-0':
# Kludge until cspad_tbx.image() can be rewritten to handle the
# XPP metrology.
self.cspad_img = cspad_tbx.image_xpp(
self.address, evt, env, self.active_areas)
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
from psana import Source, Camera
import numpy as np
address = cspad_tbx.old_address_to_new_address(self.address)
src=Source('DetInfo(%s)'%address)
self.cspad_img = evt.get(Camera.FrameV1,src)
if self.cspad_img is not None:
self.cspad_img = self.cspad_img.data16().astype(np.float64)
elif self.address=='CxiDg3-0|Opal1000-0':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address=='CxiEndstation-0|Opal1000-2':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address=='FeeHxSpectrometer-0|Opal1000-1':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address=='NoDetector-0|Cspad2x2-0':
import numpy as np
from pypdsdata import xtc
test=[]
self.cspad_img = evt.get(xtc.TypeId.Type.Id_Cspad2x2Element,self.address).data()
self.cspad_img=np.reshape(self.cspad_img,(370, 388))
else:
try:
self.cspad_img = cspad_tbx.image(
self.address, cspad_tbx.getConfig(self.address, env),
evt, env, self.sections)
except Exception, e:
self.logger.error("Error reading image data: " + str(e))
evt.put(skip_event_flag(), "skip_event")
return
class common_mode_correction(mod_event_info):
"""Dark subtraction and alternate implementation of common mode
substituting for cspad_tbx.
Known problems: the algorithm relies on a substantial part of the
sensor having no signal, which is not the case if a water ring
crosses the sensor.
"""
def __init__(self,
address,
calib_dir=None,
common_mode_correction="none",
photon_threshold=None,
two_photon_threshold=None,
dark_path=None,
dark_stddev=None,
mask_path=None,
gain_map_path=None,
gain_map_level=None,
cache_image=True,
roi=None,
laser_1_status=None,
laser_4_status=None,
laser_wait_time=None,
override_beam_x=None,
override_beam_y=None,
bin_size=None,
crop_rayonix=False,
**kwds):
"""The common_mode_correction class constructor stores the
parameters passed from the pyana configuration file in instance
variables.
@param address Full data source address of the DAQ device
@param calib_dir Directory with calibration information
@param common_mode_correction The type of common mode correction to apply
@param dark_path Path to input average dark image
@param dark_stddev Path to input standard deviation dark
image, required if @p dark_path is given
@param mask_path Path to input mask. Pixels to mask out should be set to -2
@param gain_map_path Path to input gain map. Multiplied times the image.
@param gain_map_level If set, all the '1' pixels in the gain_map are set to this multiplier
and all the '0' pixels in the gain_map are set to '1'. If not set,
use the values in the gain_map directly
@param laser_1_status 0 or 1 to indicate that the laser should be off or on respectively
@param laser_4_status 0 or 1 to indicate that the laser should be off or on respectively
@param laser_wait_time Length of time in milliseconds to wait after a laser
change of status to begin accepting images again.
(rejection of images occurs immediately after status
change).
@param override_beam_x override value for x coordinate of beam center in pixels
@param override_beam_y override value for y coordinate of beam center in pixels
@param bin_size bin size for rayonix detector used to determin pixel size
@param crop_rayonix whether to crop rayonix images such that image center is the beam center
"""
# Cannot use the super().__init__() construct here, because
# common_mode_correction refers to the argument, and not the
# class.
mod_event_info.__init__(self, address=address, **kwds)
# The paths will be substituted in beginjob(), where evt and env
# are available.
self._dark_path = cspad_tbx.getOptString(dark_path)
self._dark_stddev_path = cspad_tbx.getOptString(dark_stddev)
self._gain_map_path = cspad_tbx.getOptString(gain_map_path)
self._mask_path = cspad_tbx.getOptString(mask_path)
self.gain_map_level = cspad_tbx.getOptFloat(gain_map_level)
self.common_mode_correction = cspad_tbx.getOptString(common_mode_correction)
self.photon_threshold = cspad_tbx.getOptFloat(photon_threshold)
self.two_photon_threshold = cspad_tbx.getOptFloat(two_photon_threshold)
self.cache_image = cspad_tbx.getOptBool(cache_image)
self.filter_laser_1_status = cspad_tbx.getOptInteger(laser_1_status)
self.filter_laser_4_status = cspad_tbx.getOptInteger(laser_4_status)
if self.filter_laser_1_status is not None:
self.filter_laser_1_status = bool(self.filter_laser_1_status)
if self.filter_laser_4_status is not None:
self.filter_laser_4_status = bool(self.filter_laser_4_status)
self.filter_laser_wait_time = cspad_tbx.getOptInteger(laser_wait_time)
self.override_beam_x = cspad_tbx.getOptFloat(override_beam_x)
self.override_beam_y = cspad_tbx.getOptFloat(override_beam_y)
self.bin_size = cspad_tbx.getOptInteger(bin_size)
self.crop_rayonix = cspad_tbx.getOptBool(crop_rayonix)
self.cspad_img = None # The current image - set by self.event()
self.sum_common_mode = 0
self.sumsq_common_mode = 0
self.roi = cspad_tbx.getOptROI(roi) # used to ignore the signal region in chebyshev fit
assert self.common_mode_correction in \
("gaussian", "mean", "median", "mode", "none", "chebyshev")
# Get and parse metrology.
self.sections = None
device = cspad_tbx.address_split(self.address)[2]
if device == 'Andor':
self.sections = [] # XXX FICTION
elif device == 'Cspad':
if self.address == 'XppGon-0|Cspad-0':
self.sections = [] # Not used for XPP
else:
self.sections = calib2sections(cspad_tbx.getOptString(calib_dir))
elif device == 'Cspad2x2':
# There is no metrology information for the Sc1 detector, so
# make it up. The sections are rotated by 90 degrees with
# respect to the "standing up" convention.
self.sections = [[Section(90, (185 / 2 + 0, (2 * 194 + 3) / 2)),
Section(90, (185 / 2 + 185, (2 * 194 + 3) / 2))]]
elif device == 'marccd':
self.sections = [] # XXX FICTION
elif device == 'pnCCD':
self.sections = [] # XXX FICTION
elif device == 'Rayonix':
self.sections = [] # XXX FICTION
elif device == 'Opal1000':
self.sections = [] # XXX FICTION
if self.sections is None:
raise RuntimeError("Failed to load metrology")
def beginjob(self, evt, env):
"""The beginjob() function does one-time initialisation from
event- or environment data. It is called at an XTC configure
transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(common_mode_correction, self).beginjob(evt, env)
# Load the dark image and ensure it is signed and at least 32 bits
# wide, since it will be used for differencing. If a dark image
# is provided, a standard deviation image is required, and all the
# ADU scales must match up.
#
# XXX Can we zap all the ADU_SCALE stuff?
#
# XXX Do we really need to store the substituted values in the
# instance here? At least self._dark_path is referenced later on.
#
# Note that this will load the dark, standard deviation and gain
# once (SEVERAL TIMES) for each process, but the gain is that we
# can do substitutions. But it is only done at the beginning of
# the job.
self.dark_img = None
if self._dark_path is not None:
self._dark_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._dark_path, evt, env))
assert self._dark_stddev_path is not None
dark_dict = easy_pickle.load(self._dark_path)
#assert "ADU_SCALE" not in dark_dict # force use of recalculated dark
self.dark_img = dark_dict['DATA']
assert isinstance(self.dark_img, flex.double)
self._dark_stddev_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._dark_stddev_path, evt, env))
self.dark_stddev = easy_pickle.load(self._dark_stddev_path)['DATA']
assert isinstance(self.dark_stddev, flex.double)
self.dark_mask = (self.dark_stddev > 0)
# Load the mask image and ensure it is signed and at least 32 bits
# wide, since it will be used for differencing.
self.gain_map = None
if self._gain_map_path is not None:
self._gain_map_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._gain_map_path, evt, env))
self.gain_map = easy_pickle.load(self._gain_map_path)['DATA']
if self.gain_map_level is not None:
sel = flex.bool([bool(f) for f in self.gain_map])
sel.reshape(flex.grid(self.gain_map.focus()))
self.gain_map = self.gain_map.set_selected(~sel, self.gain_map_level)
self.gain_map = self.gain_map.set_selected(sel, 1)
assert isinstance(self.gain_map, flex.double)
self.mask_img = None
if self._mask_path is not None:
self._mask_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._mask_path, evt, env))
self.mask_img = easy_pickle.load(self._mask_path)['DATA']
assert isinstance(self.mask_img, flex.double) \
or isinstance(self.mask_img, flex.int)
if self.address == 'XppGon-0|marccd-0':
#mod_mar.py will set these during its event function
self.active_areas = None
self.beam_center = None
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
assert self.override_beam_x is not None
assert self.override_beam_y is not None
from xfel.cxi.cspad_ana import rayonix_tbx
maxx, maxy = rayonix_tbx.get_rayonix_detector_dimensions(self.bin_size)
if self.crop_rayonix:
bx = int(round(self.override_beam_x))
by = int(round(self.override_beam_y))
minsize = min([bx,by,maxx-bx,maxy-by])
self.beam_center = minsize,minsize
self.active_areas = flex.int([0,0,2*minsize,2*minsize])
self.rayonix_crop_slice = slice(by-minsize,by+minsize), slice(bx-minsize,bx+minsize)
else:
self.beam_center = self.override_beam_x,self.override_beam_y
self.active_areas = flex.int([0,0,maxx,maxy])
elif self.address == 'XppGon-0|Cspad-0':
evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(self.address, reverse_timestamp(timestamp)[0])
assert version_lookup is not None
self.active_areas = xpp_active_areas[version_lookup]['active_areas']
self.beam_center = [1765 // 2, 1765 // 2]
else:
(self.beam_center, self.active_areas) = cspad_tbx.cbcaa(
cspad_tbx.getConfig(self.address, env), self.sections)
def common_mode(self, img, stddev, mask):
"""The common_mode() function returns the mode of image stored in
the array pointed to by @p img. @p mask must be such that the @p
stddev at the selected pixels is greater than zero.
@param img 2D integer array of the image
@param stddev 2D integer array of the standard deviation of each
pixel in @p img
@param mask 2D Boolean array, @c True if the pixel is to be
included, @c False otherwise
@return Mode of the image, as a real number
"""
# Flatten the image and take out inactive pixels XXX because we
# cannot take means and medians of 2D arrays?
img_1d = img.as_1d().select(mask.as_1d()).as_double()
assert img_1d.size() > 0
if (self.common_mode_correction == "mean"):
# The common mode is approximated by the mean of the pixels with
# signal-to-noise ratio less than a given threshold. XXX Breaks
# if the selection is empty!
THRESHOLD_SNR = 2
img_snr = img_1d / stddev.as_double().as_1d().select(mask.as_1d())
return (flex.mean(img_1d.select(img_snr < THRESHOLD_SNR)))
elif (self.common_mode_correction == "median"):
return (flex.median(img_1d))
# Identify the common-mode correction as the peak histogram of the
# histogram of pixel values (the "standard" common-mode correction, as
# previously implemented in this class).
hist_min = -40
hist_max = 40
n_slots = 100
hist = flex.histogram(img_1d, hist_min, hist_max, n_slots=n_slots)
slots = hist.slots()
i = flex.max_index(slots)
common_mode = list(hist.slot_infos())[i].center()
if (self.common_mode_correction == "mode"):
return (common_mode)
# Determine the common-mode correction from the peak of a single
# Gaussian function fitted to the histogram.
from scitbx.math.curve_fitting import single_gaussian_fit
x = hist.slot_centers()
y = slots.as_double()
fit = single_gaussian_fit(x, y)
scale, mu, sigma = fit.a, fit.b, fit.c
self.logger.debug("fitted gaussian: mu=%.3f, sigma=%.3f" %(mu, sigma))
mode = common_mode
common_mode = mu
if abs(mode-common_mode) > 1000: common_mode = mode # XXX
self.logger.debug("delta common mode corrections: %.3f" %(mode-common_mode))
if 0 and abs(mode-common_mode) > 0:
#if 0 and skew > 0.5:
# view histogram and fitted gaussian
from numpy import exp
from matplotlib import pyplot
x_all = x
n, bins, patches = pyplot.hist(section_img.as_1d().as_numpy_array(), bins=n_slots, range=(hist_min, hist_max))
y_all = scale * flex.exp(-flex.pow2(x_all-mu) / (2 * sigma**2))
scale = slots[flex.max_index(slots)]
y_all *= scale/flex.max(y_all)
pyplot.plot(x_all, y_all)
pyplot.show()
return (common_mode)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(common_mode_correction, self).event(evt, env)
if (evt.get("skip_event")):
return
if not hasattr(self, 'active_areas') or self.active_areas is None or \
not hasattr(self, 'beam_center') or self.beam_center is None:
if self.address == 'XppGon-0|marccd-0':
# The mod_mar module needs to have been called before this one
# to set this up. The MAR does not have a configure object.
self.beam_center = evt.get("marccd_beam_center")
self.active_areas = evt.get("marccd_active_areas")
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
pass # bc and aa set in the beginjob function
elif self.address == 'XppGon-0|Cspad-0':
# Load the active areas as determined from the optical metrology
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(self.address, reverse_timestamp(self.timestamp)[0])
assert version_lookup is not None
self.active_areas = xpp_active_areas[version_lookup]['active_areas']
self.beam_center = [1765 // 2, 1765 // 2]
else:
(self.beam_center, self.active_areas) = \
cspad_tbx.cbcaa(cspad_tbx.getConfig(self.address, env), self.sections)
if self.filter_laser_1_status is not None:
if (self.laser_1_status.status != self.filter_laser_1_status or
(self.laser_1_ms_since_change is not None and
self.laser_1_ms_since_change < self.filter_laser_wait_time)):
evt.put(skip_event_flag(), "skip_event")
return
if self.filter_laser_4_status is not None:
if (self.laser_4_status.status != self.filter_laser_4_status or
(self.laser_4_ms_since_change is not None and
self.laser_4_ms_since_change < self.filter_laser_wait_time)):
evt.put(skip_event_flag(), "skip_event")
return
# Early return if the full detector image is already stored in the
# event. Otherwise, get it from the stream as a double-precision
# floating-point flex array. XXX It is probably not safe to key
# the image on self.address, so we should come up with our own
# namespace. XXX Misnomer--could be CAMP, too
self.cspad_img = evt.get(self.address)
if self.cspad_img is not None:
return
if self.address == 'XppGon-0|Cspad-0':
# Kludge until cspad_tbx.image() can be rewritten to handle the
# XPP metrology.
self.cspad_img = cspad_tbx.image_xpp(
self.address, evt, env, self.active_areas)
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
from psana import Source, Camera
import numpy as np
address = cspad_tbx.old_address_to_new_address(self.address)
src=Source('DetInfo(%s)'%address)
self.cspad_img = evt.get(Camera.FrameV1,src)
if self.cspad_img is not None:
self.cspad_img = self.cspad_img.data16().astype(np.float64)
elif self.address=='CxiDg3-0|Opal1000-0':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address=='CxiEndstation-0|Opal1000-2':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address=='FeeHxSpectrometer-0|Opal1000-1':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address=='NoDetector-0|Cspad2x2-0':
import numpy as np
from pypdsdata import xtc
test=[]
self.cspad_img = evt.get(xtc.TypeId.Type.Id_Cspad2x2Element,self.address).data()
self.cspad_img=np.reshape(self.cspad_img,(370, 388))
else:
try:
self.cspad_img = cspad_tbx.image(
self.address, cspad_tbx.getConfig(self.address, env),
evt, env, self.sections)
except Exception, e:
self.logger.error("Error reading image data: " + str(e))
evt.put(skip_event_flag(), "skip_event")
return
if self.cspad_img is None:
if cspad_tbx.address_split(self.address)[2] != 'Andor':
self.nfail += 1
self.logger.warning("event(): no image, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
self.cspad_img = flex.double(self.cspad_img.astype(numpy.float64))
# If a dark image was provided, subtract it from the image. There
# is no point in doing common-mode correction unless the dark
# image was subtracted.
if (self.dark_img is not None):
self.cspad_img -= self.dark_img
if (self.common_mode_correction != "none"):
# Mask out inactive pixels prior to common mode correction.
# Pixels are marked as inactive either due to low ADU values
# or non-positive standard deviations in dark image. XXX Make
# the threshold tunable?
cspad_mask = self.dark_mask.deep_copy()
if self.roi is not None and self.common_mode_correction == "chebyshev":
roi_mask = cspad_mask[self.roi[2]:self.roi[3], :]
roi_mask = flex.bool(roi_mask.accessor(), False)
cspad_mask.matrix_paste_block_in_place(
block=roi_mask,
i_row=self.roi[2],
i_column=0)
# Extract each active section from the assembled detector
# image and apply the common mode correction. XXX Make up a
# quadrant mask for the emission detector. Needs to be
# checked!
config = cspad_tbx.getConfig(self.address, env)
if len(self.sections) == 1:
q_mask = 1
else:
q_mask = config.quadMask()
for q in xrange(len(self.sections)):
if (not((1 << q) & q_mask)):
continue
# XXX Make up section mask for the emission detector. Needs
# to be checked!
import _pdsdata
if len(self.sections) == 1 and type(config) in (
_pdsdata.cspad2x2.ConfigV1, _pdsdata.cspad2x2.ConfigV2):
s_mask = config.roiMask()
else:
s_mask = config.roiMask(q)
for s in xrange(len(self.sections[q])):
# XXX DAQ misconfiguration? This mask appears not to work
# reliably for the Sc1 detector.
# if (not((1 << s) & s_mask)):
# continue
corners = self.sections[q][s].corners()
i_row = int(round(min(c[0] for c in corners)))
i_column = int(round(min(c[1] for c in corners)))
n_rows = int(round(max(c[0] for c in corners))) - i_row
n_columns = int(round(max(c[1] for c in corners))) - i_column
section_img = self.cspad_img.matrix_copy_block(
i_row = i_row, i_column = i_column,
n_rows = n_rows, n_columns = n_columns)
section_mask = cspad_mask.matrix_copy_block(
i_row = i_row, i_column = i_column,
n_rows = n_rows, n_columns = n_columns)
section_stddev = self.dark_stddev.matrix_copy_block(
i_row = i_row, i_column = i_column,
n_rows = n_rows, n_columns = n_columns)
if section_mask.count(True) == 0: continue
if self.common_mode_correction == "chebyshev":
assert len(self.sections[q]) == 2
if s == 0:
section_imgs = [section_img]
section_masks = [section_mask]
i_rows = [i_row]
i_columns = [i_column]
continue
else:
section_imgs.append(section_img)
section_masks.append(section_mask)
i_rows.append(i_row)
i_columns.append(i_column)
chebyshev_corrected_imgs = self.chebyshev_common_mode(
section_imgs, section_masks)
for i in range(2):
section_imgs[i].as_1d().copy_selected(
section_masks[i].as_1d().iselection(),
chebyshev_corrected_imgs[i].as_1d())
self.cspad_img.matrix_paste_block_in_place(
block=section_imgs[i],
i_row=i_rows[i],
i_column=i_columns[i])
else:
common_mode = self.common_mode(
section_img, section_stddev, section_mask)
self.sum_common_mode += common_mode
self.sumsq_common_mode += common_mode**2
# Apply the common mode correction to the
# section, and paste it back into the image.
self.cspad_img.matrix_paste_block_in_place(
block = section_img - common_mode,
i_row = i_row,
i_column = i_column)
if self.gain_map is not None:
self.cspad_img *= self.gain_map
if (self.mask_img is not None):
sel = (self.mask_img == -2 )|(self.mask_img == cspad_tbx.cspad_mask_value)
self.cspad_img.set_selected(sel, cspad_tbx.cspad_mask_value)
if (self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0') and \
self.crop_rayonix:
# Crop the masked data so that the beam center is in the center of the image
self.cspad_img = self.cspad_img[self.rayonix_crop_slice[0], self.rayonix_crop_slice[1]]
if self.cache_image:
# Store the image in the event.
evt.put(self.cspad_img, self.address)
def beginjob(self, evt, env):
"""The beginjob() function does one-time initialisation from
event- or environment data. It is called at an XTC configure
transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(common_mode_correction, self).beginjob(evt, env)
# Load the dark image and ensure it is signed and at least 32 bits
# wide, since it will be used for differencing. If a dark image
# is provided, a standard deviation image is required, and all the
# ADU scales must match up.
#
# XXX Can we zap all the ADU_SCALE stuff?
#
# XXX Do we really need to store the substituted values in the
# instance here? At least self._dark_path is referenced later on.
#
# Note that this will load the dark, standard deviation and gain
# once (SEVERAL TIMES) for each process, but the gain is that we
# can do substitutions. But it is only done at the beginning of
# the job.
self.dark_img = None
if self._dark_path is not None:
self._dark_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._dark_path, evt, env))
assert self._dark_stddev_path is not None
dark_dict = easy_pickle.load(self._dark_path)
#assert "ADU_SCALE" not in dark_dict # force use of recalculated dark
self.dark_img = dark_dict['DATA']
assert isinstance(self.dark_img, flex.double)
self._dark_stddev_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._dark_stddev_path, evt, env))
self.dark_stddev = easy_pickle.load(self._dark_stddev_path)['DATA']
assert isinstance(self.dark_stddev, flex.double)
self.dark_mask = (self.dark_stddev > 0)
# Load the mask image and ensure it is signed and at least 32 bits
# wide, since it will be used for differencing.
self.gain_map = None
if self._gain_map_path is not None:
self._gain_map_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._gain_map_path, evt, env))
self.gain_map = easy_pickle.load(self._gain_map_path)['DATA']
if self.gain_map_level is not None:
sel = flex.bool([bool(f) for f in self.gain_map])
sel.reshape(flex.grid(self.gain_map.focus()))
self.gain_map = self.gain_map.set_selected(~sel, self.gain_map_level)
self.gain_map = self.gain_map.set_selected(sel, 1)
assert isinstance(self.gain_map, flex.double)
self.mask_img = None
if self._mask_path is not None:
self._mask_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._mask_path, evt, env))
self.mask_img = easy_pickle.load(self._mask_path)['DATA']
assert isinstance(self.mask_img, flex.double) \
or isinstance(self.mask_img, flex.int)
if self.address == 'XppGon-0|marccd-0':
#mod_mar.py will set these during its event function
self.active_areas = None
self.beam_center = None
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
assert self.override_beam_x is not None
assert self.override_beam_y is not None
from xfel.cxi.cspad_ana import rayonix_tbx
maxx, maxy = rayonix_tbx.get_rayonix_detector_dimensions(self.bin_size)
if self.crop_rayonix:
bx = int(round(self.override_beam_x))
by = int(round(self.override_beam_y))
minsize = min([bx,by,maxx-bx,maxy-by])
self.beam_center = minsize,minsize
self.active_areas = flex.int([0,0,2*minsize,2*minsize])
self.rayonix_crop_slice = slice(by-minsize,by+minsize), slice(bx-minsize,bx+minsize)
else:
self.beam_center = self.override_beam_x,self.override_beam_y
self.active_areas = flex.int([0,0,maxx,maxy])
elif self.address == 'XppGon-0|Cspad-0':
evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(self.address, reverse_timestamp(timestamp)[0])
assert version_lookup is not None
self.active_areas = xpp_active_areas[version_lookup]['active_areas']
self.beam_center = [1765 // 2, 1765 // 2]
else:
(self.beam_center, self.active_areas) = cspad_tbx.cbcaa(
cspad_tbx.getConfig(self.address, env), self.sections)
def average(argv=None):
if argv == None:
argv = sys.argv[1:]
try:
from mpi4py import MPI
except ImportError:
raise Sorry("MPI not found")
command_line = (libtbx.option_parser.option_parser(usage="""
%s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] [-g gain_mask_value] [--min] [--minpath minpath]
To write image pickles use -p, otherwise the program writes CSPAD CBFs.
Writing CBFs requires the geometry to be already deployed.
Examples:
cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
Use one process on the current node to process all the events from run 25 of
experiment cxi49812, using a detz_offset of 571.
mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
As above, using 16 cores on the current node.
bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812
As above, using the psanaq and 100 cores, putting the log in average.out and
the output images in the folder cxi49812.
""" % libtbx.env.dispatcher_name).option(
None,
"--as_pickle",
"-p",
action="store_true",
default=False,
dest="as_pickle",
help="Write results as image pickle files instead of cbf files"
).option(
None,
"--raw_data",
"-R",
action="store_true",
default=False,
dest="raw_data",
help=
"Disable psana corrections such as dark pedestal subtraction or common mode (cbf only)"
).option(
None,
"--background_pickle",
"-B",
default=None,
dest="background_pickle",
help=""
).option(
None,
"--config",
"-c",
type="string",
default=None,
dest="config",
metavar="PATH",
help="psana config file"
).option(
None,
"--experiment",
"-x",
type="string",
default=None,
dest="experiment",
help="experiment name (eg cxi84914)"
).option(
None,
"--run",
"-r",
type="int",
default=None,
dest="run",
help="run number"
).option(
None,
"--address",
"-a",
type="string",
default="CxiDs2.0:Cspad.0",
dest="address",
help="detector address name (eg CxiDs2.0:Cspad.0)"
).option(
None,
"--detz_offset",
"-d",
type="float",
default=None,
dest="detz_offset",
help=
"offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)"
).option(
None,
"--outputdir",
"-o",
type="string",
default=".",
dest="outputdir",
metavar="PATH",
help="Optional path to output directory for output files"
).option(
None,
"--averagebase",
"-A",
type="string",
default="{experiment!l}_avg-r{run:04d}",
dest="averagepath",
metavar="PATH",
help=
"Path to output average image without extension. String substitution allowed"
).option(
None,
"--stddevbase",
"-S",
type="string",
default="{experiment!l}_stddev-r{run:04d}",
dest="stddevpath",
metavar="PATH",
help=
"Path to output standard deviation image without extension. String substitution allowed"
).option(
None,
"--maxbase",
"-M",
type="string",
default="{experiment!l}_max-r{run:04d}",
dest="maxpath",
metavar="PATH",
help=
"Path to output maximum projection image without extension. String substitution allowed"
).option(
None,
"--numevents",
"-n",
type="int",
default=None,
dest="numevents",
help="Maximum number of events to process. Default: all"
).option(
None,
"--skipevents",
"-s",
type="int",
default=0,
dest="skipevents",
help="Number of events in the beginning of the run to skip. Default: 0"
).option(
None,
"--verbose",
"-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress"
).option(
None,
"--pickle-optical-metrology",
"-m",
action="store_true",
default=False,
dest="pickle_optical_metrology",
help=
"If writing pickle files, use the optical metrology in the experiment's calib directory"
).option(
None,
"--bin_size",
"-b",
type="int",
default=None,
dest="bin_size",
help="Rayonix detector bin size"
).option(
None,
"--override_beam_x",
"-X",
type="float",
default=None,
dest="override_beam_x",
help="Rayonix detector beam center x coordinate"
).option(
None,
"--override_beam_y",
"-Y",
type="float",
default=None,
dest="override_beam_y",
help="Rayonix detector beam center y coordinate"
).option(
None,
"--calib_dir",
"-C",
type="string",
default=None,
dest="calib_dir",
metavar="PATH",
help="calibration directory"
).option(
None,
"--pickle_calib_dir",
"-P",
type="string",
default=None,
dest="pickle_calib_dir",
metavar="PATH",
help=
"pickle calibration directory specification. Replaces --calib_dir functionality."
).option(
None,
"--xtc_dir",
"-D",
type="string",
default=None,
dest="xtc_dir",
metavar="PATH",
help="xtc stream directory"
).option(
None,
"--use_ffb",
"-f",
action="store_true",
default=False,
dest="use_ffb",
help=
"Use the fast feedback filesystem at LCLS. Only for the active experiment!"
).option(
None,
"--gain_mask_value",
"-g",
type="float",
default=None,
dest="gain_mask_value",
help=
"Ratio between low and high gain pixels, if CSPAD in mixed-gain mode. Only used in CBF averaging mode."
).option(
None,
"--min",
None,
action="store_true",
default=False,
dest="do_minimum_projection",
help="Output a minimum projection"
).option(
None,
"--minpath",
None,
type="string",
default="{experiment!l}_min-r{run:04d}",
dest="minpath",
metavar="PATH",
help=
"Path to output minimum image without extension. String substitution allowed"
)).process(args=argv)
if len(command_line.args) > 0 or \
command_line.options.as_pickle is None or \
command_line.options.experiment is None or \
command_line.options.run is None or \
command_line.options.address is None or \
command_line.options.detz_offset is None or \
command_line.options.averagepath is None or \
command_line.options.stddevpath is None or \
command_line.options.maxpath is None or \
command_line.options.pickle_optical_metrology is None:
command_line.parser.show_help()
return
# set this to sys.maxint to analyze all events
if command_line.options.numevents is None:
maxevents = sys.maxsize
else:
maxevents = command_line.options.numevents
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if command_line.options.config is not None:
psana.setConfigFile(command_line.options.config)
dataset_name = "exp=%s:run=%d:smd" % (command_line.options.experiment,
command_line.options.run)
if command_line.options.xtc_dir is not None:
if command_line.options.use_ffb:
raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system")
dataset_name += ":dir=%s" % command_line.options.xtc_dir
elif command_line.options.use_ffb:
# as ffb is only at SLAC, ok to hardcode /reg/d here
dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc" % (
command_line.options.experiment[0:3],
command_line.options.experiment)
if command_line.options.calib_dir is not None:
psana.setOption('psana.calib-dir', command_line.options.calib_dir)
ds = psana.DataSource(dataset_name)
address = command_line.options.address
src = psana.Source('DetInfo(%s)' % address)
nevent = np.array([0.])
if command_line.options.background_pickle is not None:
background = easy_pickle.load(
command_line.options.background_pickle)['DATA'].as_numpy_array()
for run in ds.runs():
runnumber = run.run()
if not command_line.options.as_pickle:
psana_det = psana.Detector(address, ds.env())
# list of all events
if command_line.options.skipevents > 0:
print("Skipping first %d events" % command_line.options.skipevents)
elif "Rayonix" in command_line.options.address:
print("Skipping first image in the Rayonix detector"
) # Shuttering issue
command_line.options.skipevents = 1
for i, evt in enumerate(run.events()):
if i % size != rank: continue
if i < command_line.options.skipevents: continue
if i >= maxevents: break
if i % 10 == 0: print('Rank', rank, 'processing event', i)
#print "Event #",rank*mylength+i," has id:",evt.get(EventId)
if 'Rayonix' in command_line.options.address or 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
data = evt.get(psana.Camera.FrameV1, src)
if data is None:
print("No data")
continue
data = data.data16().astype(np.float64)
elif command_line.options.as_pickle:
data = evt.get(psana.ndarray_float64_3, src, 'image0')
else:
# get numpy array, 32x185x388
from xfel.cftbx.detector.cspad_cbf_tbx import get_psana_corrected_data
if command_line.options.raw_data:
data = get_psana_corrected_data(psana_det,
evt,
use_default=False,
dark=False,
common_mode=None,
apply_gain_mask=False,
per_pixel_gain=False)
else:
if command_line.options.gain_mask_value is None:
data = get_psana_corrected_data(psana_det,
evt,
use_default=True)
else:
data = get_psana_corrected_data(
psana_det,
evt,
use_default=False,
dark=True,
common_mode=None,
apply_gain_mask=True,
gain_mask_value=command_line.options.
gain_mask_value,
per_pixel_gain=False)
if data is None:
print("No data")
continue
if command_line.options.background_pickle is not None:
data -= background
if 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
distance = np.array([0.0])
wavelength = np.array([1.0])
else:
d = cspad_tbx.env_distance(address, run.env(),
command_line.options.detz_offset)
if d is None:
print("No distance, using distance",
command_line.options.detz_offset)
assert command_line.options.detz_offset is not None
if 'distance' not in locals():
distance = np.array([command_line.options.detz_offset])
else:
distance += command_line.options.detz_offset
else:
if 'distance' in locals():
distance += d
else:
distance = np.array([float(d)])
w = cspad_tbx.evt_wavelength(evt)
if w is None:
print("No wavelength")
if 'wavelength' not in locals():
wavelength = np.array([1.0])
else:
if 'wavelength' in locals():
wavelength += w
else:
wavelength = np.array([w])
t = cspad_tbx.evt_time(evt)
if t is None:
print("No timestamp, skipping shot")
continue
if 'timestamp' in locals():
timestamp += t[0] + (t[1] / 1000)
else:
timestamp = np.array([t[0] + (t[1] / 1000)])
if 'sum' in locals():
sum += data
else:
sum = np.array(data, copy=True)
if 'sumsq' in locals():
sumsq += data * data
else:
sumsq = data * data
if 'maximum' in locals():
maximum = np.maximum(maximum, data)
else:
maximum = np.array(data, copy=True)
if command_line.options.do_minimum_projection:
if 'minimum' in locals():
minimum = np.minimum(minimum, data)
else:
minimum = np.array(data, copy=True)
nevent += 1
#sum the images across mpi cores
if size > 1:
print("Synchronizing rank", rank)
totevent = np.zeros(nevent.shape)
comm.Reduce(nevent, totevent)
if rank == 0 and totevent[0] == 0:
raise Sorry("No events found in the run")
sumall = np.zeros(sum.shape).astype(sum.dtype)
comm.Reduce(sum, sumall)
sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype)
comm.Reduce(sumsq, sumsqall)
maxall = np.zeros(maximum.shape).astype(maximum.dtype)
comm.Reduce(maximum, maxall, op=MPI.MAX)
if command_line.options.do_minimum_projection:
minall = np.zeros(maximum.shape).astype(minimum.dtype)
comm.Reduce(minimum, minall, op=MPI.MIN)
waveall = np.zeros(wavelength.shape).astype(wavelength.dtype)
comm.Reduce(wavelength, waveall)
distall = np.zeros(distance.shape).astype(distance.dtype)
comm.Reduce(distance, distall)
timeall = np.zeros(timestamp.shape).astype(timestamp.dtype)
comm.Reduce(timestamp, timeall)
if rank == 0:
if size > 1:
print("Synchronized")
# Accumulating floating-point numbers introduces errors,
# which may cause negative variances. Since a two-pass
# approach is unacceptable, the standard deviation is
# clamped at zero.
mean = sumall / float(totevent[0])
variance = (sumsqall / float(totevent[0])) - (mean**2)
variance[variance < 0] = 0
stddev = np.sqrt(variance)
wavelength = waveall[0] / totevent[0]
distance = distall[0] / totevent[0]
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
timestamp = timeall[0] / totevent[0]
timestamp = (int(timestamp), timestamp % int(timestamp) * 1000)
timestamp = cspad_tbx.evt_timestamp(timestamp)
if command_line.options.as_pickle:
extension = ".pickle"
else:
extension = ".cbf"
dest_paths = [
cspad_tbx.pathsubst(command_line.options.averagepath + extension,
evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.stddevpath + extension,
evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt,
ds.env())
]
if command_line.options.do_minimum_projection:
dest_paths.append(
cspad_tbx.pathsubst(command_line.options.minpath + extension,
evt, ds.env()))
dest_paths = [
os.path.join(command_line.options.outputdir, path)
for path in dest_paths
]
if 'Rayonix' in command_line.options.address:
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
from xfel.cxi.cspad_ana import rayonix_tbx
pixel_size = rayonix_tbx.get_rayonix_pixel_size(
command_line.options.bin_size)
beam_center = [
command_line.options.override_beam_x,
command_line.options.override_beam_y
]
active_areas = flex.int([0, 0, mean.shape[1], mean.shape[0]])
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
for data, path in zip(all_data, dest_paths):
print("Saving", path)
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=flex.double(data),
distance=distance,
pixel_size=pixel_size,
saturated_value=rayonix_tbx.rayonix_saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
elif 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
all_data = [mean, stddev, maxall]
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
d = cspad_tbx.dpack(address=old_style_address,
data=flex.double(data),
distance=distance,
pixel_size=0.1,
timestamp=timestamp,
wavelength=wavelength)
print("Saving", path)
easy_pickle.dump(path, d)
elif command_line.options.as_pickle:
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
xpp = 'xpp' in address.lower()
if xpp:
evt_time = cspad_tbx.evt_time(
evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(
evt_time) # human readable format
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(
old_style_address,
reverse_timestamp(timestamp)[0])
assert version_lookup is not None
active_areas = xpp_active_areas[version_lookup]['active_areas']
beam_center = [1765 // 2, 1765 // 2]
else:
if command_line.options.pickle_calib_dir is not None:
metro_path = command_line.options.pickle_calib_dir
elif command_line.options.pickle_optical_metrology:
from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path
metro_path = get_calib_file_path(run.env(), address, run)
else:
metro_path = libtbx.env.find_in_repositories(
"xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
sections = parse_calib.calib2sections(metro_path)
beam_center, active_areas = cspad_tbx.cbcaa(
cspad_tbx.getConfig(address, ds.env()), sections)
class fake_quad(object):
def __init__(self, q, d):
self.q = q
self.d = d
def quad(self):
return self.q
def data(self):
return self.d
if xpp:
quads = [
fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
mean = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [
fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
stddev = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [
fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
maxall = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [
fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
minall = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
minall = flex.double(minall.astype(np.float64))
else:
quads = [
fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
mean = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [
fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
stddev = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [
fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
maxall = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [
fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
minall = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
minall = flex.double(minall.astype(np.float64))
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print("Saving", path)
d = cspad_tbx.dpack(active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=data,
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
else:
# load a header only cspad cbf from the slac metrology
from xfel.cftbx.detector import cspad_cbf_tbx
import pycbf
base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(
run, address)
if base_dxtbx is None:
raise Sorry("Couldn't load calibration file for run %d" %
run.run())
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print("Saving", path)
cspad_img = cspad_cbf_tbx.format_object_from_data(
base_dxtbx,
data,
distance,
wavelength,
timestamp,
address,
round_to_int=False)
cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(common_mode_correction, self).event(evt, env)
if (evt.get("skip_event")):
return
if not hasattr(self, 'active_areas') or self.active_areas is None or \
not hasattr(self, 'beam_center') or self.beam_center is None:
if self.address == 'XppGon-0|marccd-0':
# The mod_mar module needs to have been called before this one
# to set this up. The MAR does not have a configure object.
self.beam_center = evt.get("marccd_beam_center")
self.active_areas = evt.get("marccd_active_areas")
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
pass # bc and aa set in the beginjob function
elif self.address == 'XppGon-0|Cspad-0':
# Load the active areas as determined from the optical metrology
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(
self.address,
reverse_timestamp(self.timestamp)[0])
assert version_lookup is not None
self.active_areas = xpp_active_areas[version_lookup][
'active_areas']
self.beam_center = [1765 // 2, 1765 // 2]
else:
(self.beam_center, self.active_areas) = \
cspad_tbx.cbcaa(cspad_tbx.getConfig(self.address, env), self.sections)
if self.filter_laser_1_status is not None:
if (self.laser_1_status.status != self.filter_laser_1_status or
(self.laser_1_ms_since_change is not None and
self.laser_1_ms_since_change < self.filter_laser_wait_time)):
evt.put(skip_event_flag(), "skip_event")
return
if self.filter_laser_4_status is not None:
if (self.laser_4_status.status != self.filter_laser_4_status or
(self.laser_4_ms_since_change is not None and
self.laser_4_ms_since_change < self.filter_laser_wait_time)):
evt.put(skip_event_flag(), "skip_event")
return
# Early return if the full detector image is already stored in the
# event. Otherwise, get it from the stream as a double-precision
# floating-point flex array. XXX It is probably not safe to key
# the image on self.address, so we should come up with our own
# namespace. XXX Misnomer--could be CAMP, too
self.cspad_img = evt.get(self.address)
if self.cspad_img is not None:
return
if self.address == 'XppGon-0|Cspad-0':
# Kludge until cspad_tbx.image() can be rewritten to handle the
# XPP metrology.
self.cspad_img = cspad_tbx.image_xpp(self.address, evt, env,
self.active_areas)
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
from psana import Source, Camera
import numpy as np
address = cspad_tbx.old_address_to_new_address(self.address)
src = Source('DetInfo(%s)' % address)
self.cspad_img = evt.get(Camera.FrameV1, src)
if self.cspad_img is not None:
self.cspad_img = self.cspad_img.data16().astype(np.float64)
elif self.address == 'CxiDg3-0|Opal1000-0':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address == 'CxiEndstation-0|Opal1000-2':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address == 'FeeHxSpectrometer-0|Opal1000-1':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address == 'NoDetector-0|Cspad2x2-0':
import numpy as np
from pypdsdata import xtc
test = []
self.cspad_img = evt.get(xtc.TypeId.Type.Id_Cspad2x2Element,
self.address).data()
self.cspad_img = np.reshape(self.cspad_img, (370, 388))
else:
try:
self.cspad_img = cspad_tbx.image(
self.address, cspad_tbx.getConfig(self.address, env), evt,
env, self.sections)
except Exception as e:
self.logger.error("Error reading image data: " + str(e))
evt.put(skip_event_flag(), "skip_event")
return
if self.cspad_img is None:
if cspad_tbx.address_split(self.address)[2] != 'Andor':
self.nfail += 1
self.logger.warning("event(): no image, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
self.cspad_img = flex.double(self.cspad_img.astype(numpy.float64))
# If a dark image was provided, subtract it from the image. There
# is no point in doing common-mode correction unless the dark
# image was subtracted.
if (self.dark_img is not None):
self.cspad_img -= self.dark_img
if (self.common_mode_correction != "none"):
# Mask out inactive pixels prior to common mode correction.
# Pixels are marked as inactive either due to low ADU values
# or non-positive standard deviations in dark image. XXX Make
# the threshold tunable?
cspad_mask = self.dark_mask.deep_copy()
if self.roi is not None and self.common_mode_correction == "chebyshev":
roi_mask = cspad_mask[self.roi[2]:self.roi[3], :]
roi_mask = flex.bool(roi_mask.accessor(), False)
cspad_mask.matrix_paste_block_in_place(block=roi_mask,
i_row=self.roi[2],
i_column=0)
# Extract each active section from the assembled detector
# image and apply the common mode correction. XXX Make up a
# quadrant mask for the emission detector. Needs to be
# checked!
config = cspad_tbx.getConfig(self.address, env)
if len(self.sections) == 1:
q_mask = 1
else:
q_mask = config.quadMask()
for q in range(len(self.sections)):
if (not ((1 << q) & q_mask)):
continue
# XXX Make up section mask for the emission detector. Needs
# to be checked!
import _pdsdata
if len(self.sections) == 1 and type(config) in (
_pdsdata.cspad2x2.ConfigV1,
_pdsdata.cspad2x2.ConfigV2):
s_mask = config.roiMask()
else:
s_mask = config.roiMask(q)
for s in range(len(self.sections[q])):
# XXX DAQ misconfiguration? This mask appears not to work
# reliably for the Sc1 detector.
# if (not((1 << s) & s_mask)):
# continue
corners = self.sections[q][s].corners()
i_row = int(round(min(c[0] for c in corners)))
i_column = int(round(min(c[1] for c in corners)))
n_rows = int(round(max(c[0] for c in corners))) - i_row
n_columns = int(round(max(
c[1] for c in corners))) - i_column
section_img = self.cspad_img.matrix_copy_block(
i_row=i_row,
i_column=i_column,
n_rows=n_rows,
n_columns=n_columns)
section_mask = cspad_mask.matrix_copy_block(
i_row=i_row,
i_column=i_column,
n_rows=n_rows,
n_columns=n_columns)
section_stddev = self.dark_stddev.matrix_copy_block(
i_row=i_row,
i_column=i_column,
n_rows=n_rows,
n_columns=n_columns)
if section_mask.count(True) == 0: continue
if self.common_mode_correction == "chebyshev":
assert len(self.sections[q]) == 2
if s == 0:
section_imgs = [section_img]
section_masks = [section_mask]
i_rows = [i_row]
i_columns = [i_column]
continue
else:
section_imgs.append(section_img)
section_masks.append(section_mask)
i_rows.append(i_row)
i_columns.append(i_column)
chebyshev_corrected_imgs = self.chebyshev_common_mode(
section_imgs, section_masks)
for i in range(2):
section_imgs[i].as_1d().copy_selected(
section_masks[i].as_1d().iselection(),
chebyshev_corrected_imgs[i].as_1d())
self.cspad_img.matrix_paste_block_in_place(
block=section_imgs[i],
i_row=i_rows[i],
i_column=i_columns[i])
else:
common_mode = self.common_mode(
section_img, section_stddev, section_mask)
self.sum_common_mode += common_mode
self.sumsq_common_mode += common_mode**2
# Apply the common mode correction to the
# section, and paste it back into the image.
self.cspad_img.matrix_paste_block_in_place(
block=section_img - common_mode,
i_row=i_row,
i_column=i_column)
if self.gain_map is not None:
self.cspad_img *= self.gain_map
if (self.mask_img is not None):
sel = (self.mask_img == -2) | (self.mask_img
== cspad_tbx.cspad_mask_value)
self.cspad_img.set_selected(sel, cspad_tbx.cspad_mask_value)
if (self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0') and \
self.crop_rayonix:
# Crop the masked data so that the beam center is in the center of the image
self.cspad_img = self.cspad_img[self.rayonix_crop_slice[0],
self.rayonix_crop_slice[1]]
if self.cache_image:
# Store the image in the event.
evt.put(self.cspad_img, self.address)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
For now, log error and set bogus value to allow stuff to continue
-- must check for the bogosity later
XXX The dead time of the detector complicates checking how often
things are updated! Move this to the ring buffer?
@param evt Event data object, a configure object
@param env Environment object
"""
from pyana.event import Event
from acqiris_ext import acqiris_integrate, apd_hitfind
super(mod_ledge, self).event(evt, env)
if evt.status() != Event.Normal:
pass # XXX return -- Never skip because arrays will end up
# different length, so ignore this?
# Get the time of the event, in fractional seconds since the
# epoch. This is needed for all subsequent history-keeping, and
# is hence determined first. XXX Is history-keeping even
# justified?
time = cspad_tbx.evt_time(evt)
if time is None:
time = float("nan")
else:
time = time[0] + time[1] / 1e3
self._timestamp.append(time)
# The repetition rate is currently just used for sanity checking.
repetition_rate = cspad_tbx.evt_repetition_rate(evt)
if repetition_rate is None:
repetition_rate = float("nan")
self._repetition_rate.append(repetition_rate)
# Get the I0. No need to warn about it here, it will be done once
# the image is written out.
I0 = cspad_tbx.evt_pulse_energy(evt)
if I0 is None:
I0 = float("nan")
self._I0.append(I0)
# Get the FEE energy. Average the two readings before and after
# attenuation separately. XXX What are the units? It look like
# it could be mJ?
fee_before = 0.5 * sum(evt.getFeeGasDet()[0:2])
if fee_before is None:
fee_before = float("nan")
self._fee_before.append(fee_before)
fee_after = 0.5 * sum(evt.getFeeGasDet()[2:4])
if fee_after is None:
fee_after = float("nan")
self._fee_after.append(fee_after)
# XXX Just a check: this is what xtcexplorer does:
fee_energy = evt.get(xtc.TypeId.Type.Id_FEEGasDetEnergy)
if fee_energy is not None:
assert (
evt.getFeeGasDet()[0] == fee_energy.f_11_ENRC
and evt.getFeeGasDet()[1] == fee_energy.f_12_ENRC
and evt.getFeeGasDet()[2] == fee_energy.f_21_ENRC
and evt.getFeeGasDet()[3] == fee_energy.f_22_ENRC
)
"""
# For Bill: expect 84240 data points for r0054
#
# grep "^BILL_POINT" | cut -d' ' -f2,3,4,5,6 > t.dat
# gnuplot> m=0.1 ; k=-0.01e-8; f(x) = k * x + m
# gnuplot> fit f(x) "t.dat" using ($3):($5) via k,m
if not hasattr(self, '_gmd_seqno'):
self._gmd_seqno = 0
gmd = evt.get(key=xtc.TypeId.Type.Id_GMD)
if gmd is None:
return
acq_apd = evt.getAcqValue('SxrEndstation-0|Acqiris-1', 0, env)
if acq_apd is not None and acq_apd.waveform() is not None:
w = acq_apd.waveform()
baseline = numpy.mean(w[0:(w.shape[0] / 5)])
peak = numpy.min(w[(w.shape[0] / 5):w.shape[0]])
self._gmd_seqno += 1
print "BILL_POINT %d %s %s %s %s" % (self._gmd_seqno,
repr(gmd.fBgValuePerSample),
repr(gmd.fCorrectedSumPerPulse),
repr(gmd.fRelativeEnergyPerPulse),
repr(peak - baseline))
return
"""
"""
# XXX Record injector motion--note that they cannot be added--see
# Ray's email.
injector_micos_xyz = cspad_tbx.env_pv3_get(
env,
['SXR:EXP:MZM:%02d:ENCPOSITIONGET' % i for i in [1, 2, 3]])
if injector_micos_xyz is None:
self.logger.error("No micos injector motor positions")
injector_micos_xyz = (float('nan'), float('nan'), float('nan'))
self._injector_micos_xyz.append(injector_micos_xyz)
injector_rough_xyz = cspad_tbx.env_pv3_get(
env,
['SXR:EXP:MMS:%02d.RBV' % i for i in [1, 2, 3]])
if injector_rough_xyz is None:
self.logger.error("No rough injector motor positions")
injector_rough_xyz = (float('nan'), float('nan'), float('nan'))
self._injector_rough_xyz.append(injector_rough_xyz)
# Injector power supplies XXX There is a third PSU, no?
#
# The -5kV supply
# SXR:EXP:SHV:VHS6:CH0:VoltageMeasure
# SXR:EXP:SHV:VHS6:CH0:CurrentMeasure
#
# The plus 5kV supply
# SXR:EXP:SHV:VHS2:CH0:VoltageMeasure
# SXR:EXP:SHV:VHS2:CH0:CurrentMeasure
injector_plus_current = cspad_tbx.env_pv1_get(
env, 'SXR:EXP:SHV:VHS6:CH0:CurrentMeasure')
if injector_plus_current is None:
self.logger.error("No plus-motor current")
injector_plus_current = -1
self._injector_plus_current.append(injector_plus_current)
injector_plus_voltage = cspad_tbx.env_pv1_get(
env, 'SXR:EXP:SHV:VHS6:CH0:VoltageMeasure')
if injector_plus_voltage is None:
self.logger.error("No plus-motor voltage")
injector_plus_voltage = -1
self._injector_plus_voltage.append(injector_plus_voltage)
injector_minus_current = cspad_tbx.env_pv1_get(
env, 'SXR:EXP:SHV:VHS2:CH0:CurrentMeasure')
if injector_minus_current is None:
self.logger.error("No minus-motor current")
injector_minus_current = -1
self._injector_minus_current.append(injector_minus_current)
injector_minus_voltage = cspad_tbx.env_pv1_get(
env, 'SXR:EXP:SHV:VHS2:CH0:VoltageMeasure')
if injector_minus_voltage is None:
self.logger.error("No minus-motor voltage")
injector_minus_voltage = -1
self._injector_minus_voltage.append(injector_minus_voltage)
"""
"""
# The spectrometer motor positions are just used for sanity
# checking.
spectrometer_xyz = cspad_tbx.env_spectrometer_xyz_sxr(env)
if spectrometer_xyz is None:
self.logger.error("No spectrometer motor positions")
spectrometer_xyz = (float('nan'), float('nan'), float('nan'))
self._spectrometer_xyz.append(spectrometer_xyz)
"""
# Get the pulse energy after monochromator, and fall back on the
# pre-monochromator energy if the former is absent. Record in
# list for mean and stddev. XXX Verify that the wavelength after
# the monochromator is updated at around 1 Hz.
#
# For the publication an offset and scale were calibrated.
wavelength = cspad_tbx.env_wavelength_sxr(evt, env)
if wavelength is None:
wavelength = cspad_tbx.evt_wavelength(evt)
if wavelength is None:
energy = float("nan")
else:
energy = 12398.4187 / wavelength
self._energy.append(energy)
self._history_energy.push(time, energy) # XXX Not necessary?!
"""
# Laser shutters XXX need to sort out laser numbering XXX Laser
# power stuff? XXX Position of polarizer/analyser
shutters = cspad_tbx.env_laser_shutters(env)
#print "Got shutters", shutters
"""
# Read out the diode traces from the via the Acqiris. XXX In any
# case, the APD and the more sensitive Opto Diode in the monitor
# tank (i.e. the transmission diode) should be anti-correlated, so
# check it! The entire trace always covers 10 us. XXX Could this
# be figured out from xtc.TypeId.Type.Id_AcqConfig?
#
# XXX This appears to be suboptimal: look at the
# skewness-transform for the APD to sort this out.
acq_apd = evt.getAcqValue("SxrEndstation-0|Acqiris-1", 0, env)
acq_apd_integral = float("nan")
if acq_apd is not None:
waveform = acq_apd.waveform()
if waveform is not None:
# With a 40k-point trace, one should integrate from 18200 to
# 18400.
waveform = waveform.flatten()
nmemb = len(waveform) // 200
if nmemb > 0:
acq_apd_integral = acqiris_integrate(flex.double(waveform), 91 * nmemb, 100 * nmemb, nmemb)
self._acq_apd_integral.append(acq_apd_integral)
if evt.expNum() == 208:
# Opto diode address for L632.
acq_opto_diode = evt.getAcqValue("SxrEndstation-0|Acqiris-1", 1, env)
elif evt.expNum() == 363:
# Opto diode address for LB68.
acq_opto_diode = evt.getAcqValue("SxrEndstation-0|Acqiris-2", 2, env)
acq_opto_diode_integral = float("nan")
if acq_opto_diode is not None:
waveform = acq_opto_diode.waveform()
if waveform is not None:
# With a 40k-point trace, one should integrate from 16000 to
# 24000. With a 20k-point trace, a suitable integration
# region is bounded by 8000 and 12000. There is no need for
# thresholding, because the integral of the Opto Diode will
# not be used for hit finding. XXX What are the "misses" we
# record on the Opto Diode? XXX The direct beam is completely
# gone after it hits the sample, because soft X-rays.
waveform = waveform.flatten()
nmemb = len(waveform) // 5
if nmemb > 0:
acq_opto_diode_integral = acqiris_integrate(flex.double(waveform), 2 * nmemb, 4 * nmemb, nmemb)
self._acq_opto_diode_integral.append(acq_opto_diode_integral)
# Sanity check: verify that the timestamps for the two Acqiris
# traces are similar enough.
if acq_apd is not None and acq_opto_diode is not None:
assert (
len(acq_apd.timestamps()) == len(acq_opto_diode.timestamps())
and numpy.any(numpy.abs(acq_apd.timestamps() - acq_opto_diode.timestamps())) < 1e-6
)
# self.logger.info("DIODE INTEGRALS: %f %f %f" % (I0, acq_apd_integral, acq_opto_diode_integral))
"""
import matplotlib.pyplot as plt
hit_array_apd = apd_hitfind(
flex.double(acq_apd.waveform()),
len(acq_apd.waveform()) // 5)
hit_array_opto_diode = apd_hitfind(
flex.double(acq_opto_diode.waveform()),
len(acq_opto_diode.waveform()) // 5)
fig = plt.figure()
ax = fig.add_subplot(111)
#ax.plot(
# range(len(acq_apd.timestamps())), acq_apd.waveform())
ax.plot(
range(len(acq_opto_diode.timestamps())), acq_opto_diode.waveform()[0, :])
plt.show()
fig = plt.figure()
ax = fig.add_subplot(111)
#ax.plot(
# acq_apd.timestamps()[0:len(hit_array_apd)], hit_array)
ax.plot(
acq_opto_diode.timestamps()[0:len(hit_array_opto_diode)], hit_array)
plt.show()
"""
# Determine whether the beam hit the sample, and register the
# outcome. If not using any diodes for hit-finding, every shot is
# assumed to be a hit. XXX Unfortunately, this crucial piece is
# very unreliable. The threshold for the APD needs to be
# verified--inspect all the histograms. XXX hitfind_flags is
# probable better as a module parameter.
# hitfind_flags = 0x3
hitfind_flags = 0
hit = False
if not hitfind_flags:
hit = True
elif hitfind_flags & 0x1 and acq_apd_integral > 0.2:
hit = True
self._hit.append(hit)
# Always proceed all the way through (even if some shots have
# invalid values of e.g. I0) because images are precious. XXX
# Must reset counters before returning! XXX What about skipping
# all of the above if display is True?
if self.cspad_img is not None:
self._nframes += 1
"""
# The spectrometer should not move!
t = (self._spectrometer_xyz -
self._spectrometer_xyz.mean()).rms_length()
print "Spectrometer displacement", t
# Fine/rough motor position deviations from the mean. See Ray's
# email.
t = (self._injector_micos_xyz -
self._injector_micos_xyz.mean()).rms_length()
print "Injector micos displacement", t
t = (self._injector_rough_xyz -
self._injector_rough_xyz.mean()).rms_length()
print "Injector rough displacement", t
# Injector motor position means and deviations
if self._injector_plus_current.size() > 1:
t = flex.mean_and_variance(self._injector_plus_current)
print "Injector plus current mean %10e stddev %10e" % \
(t.mean(), t.unweighted_sample_standard_deviation())
if self._injector_plus_voltage.size() > 1:
t = flex.mean_and_variance(self._injector_plus_voltage)
print "Injector plus voltage mean %10e stddev %10e" % \
(t.mean(), t.unweighted_sample_standard_deviation())
if self._injector_minus_current.size() > 1:
t = flex.mean_and_variance(self._injector_minus_current)
print "Injector minus current mean %10e stddev %10e" % \
(t.mean(), t.unweighted_sample_standard_deviation())
if self._injector_minus_voltage.size() > 1:
t = flex.mean_and_variance(self._injector_minus_voltage)
print "Injector minus voltage mean %10e stddev %10e" % \
(t.mean(), t.unweighted_sample_standard_deviation())
"""
# Energy statistics are collected from all shots, regardless of
# whether they are hits or not. Since this statistic mentions
# the frame number, it should be reported first. XXX The energy
# should have a really small standard deviation. Check
# self._energy.size() and self._history_energy.frequency() XXX
# verify that it works for one data point.
(energy_mean, energy_stddev, energy_nmemb, n) = self._filtered_stats(
lambda x: not math.isnan(x) and x > 0, self._energy
)
if n > 0:
self.logger.warning("%d shots have undefined energy" % n)
(I0_mean, I0_stddev, I0_nmemb, n) = self._filtered_stats(lambda x: not math.isnan(x), self._I0)
if n > 0:
self.logger.warning("%d shots have undefined I0" % n)
self.logger.info(
"Frame %d: E=%.3f+/-%.3f (N=%d) I0=%.0f+/-%.0f (N=%d)"
% (self._nframes, energy_mean, energy_stddev, energy_nmemb, I0_mean, I0_stddev, I0_nmemb)
)
# Sanity check: unless changed while integrating the frame, the
# repetition rate should have a standard deviation of zero.
dt = self._timestamp[-1] - self._timestamp[0]
rr_mean = rr_observed = rr_stddev = 0
if dt > 0:
rr_observed = (len(self._timestamp) - 1) / dt
rr = filter(lambda x: not math.isnan(x) and x > 0, self._repetition_rate)
if len(rr) > 1:
rr_stats = flex.mean_and_variance(flex.double(rr))
rr_mean = rr_stats.mean()
rr_stddev = rr_stats.unweighted_sample_standard_deviation()
self.logger.info(
"Repetition rate: %.3f Hz (observed), %.3f+/-%.3f Hz (expected)" % (rr_observed, rr_mean, rr_stddev)
)
# Compare observed and configured exposure time.
config = cspad_tbx.getConfig(self.address, env)
exposure_time = 0
if config is not None and dt > 0 and len(self._timestamp) > 0:
exposure_time = dt * (len(self._timestamp) + 1) / len(self._timestamp)
self.logger.info(
"Exposure time: %.3f s (observed), %.3f s (configured)" % (exposure_time, config.exposureTime())
)
# Compute the leading dead time, the time between starting the
# readout of the previous frame and the arrival of the shot
# immediately following it. This is an interesting statistic,
# no matter what. XXX Maybe look at its distribution?
dead_time = 0
if rr_observed > 0 and hasattr(self, "_previous_readout_time"):
dead_time = self._timestamp[0] - self._previous_readout_time - 1 / rr_observed
if math.isnan(dead_time):
dead_time = 0
self.logger.info("Dead time: %.3f s" % dead_time)
self._previous_readout_time = self._timestamp[-1]
assert time == self._timestamp[-1] # XXX ZAP once one run survives it!
# Flag blank images (i.e. images that had no hits), because
# these may interesting for background subtraction.
hits = self._hit.count(True)
self.logger.info("Hit rate: %d/%d (%.2f%%)" % (hits, self._hit.size(), 100 * hits / self._hit.size()))
if hits == 0:
self.logger.info("Frame %d is blank" % self._nframes)
# Get the normalisation factor by summing up I0 for all hits.
# Invalid and non-positive values of I0 are treated as zeroes.
# XXX Make this kind of summing a function of its own.
I0 = sum(filter(lambda x: not math.isnan(x) and x > 0, self._I0.select(self._hit)))
I0_all = sum(filter(lambda x: not math.isnan(x) and x > 0, self._I0))
fee_before_all = sum(filter(lambda x: not math.isnan(x) and x > 0, self._fee_before))
fee_after_all = sum(filter(lambda x: not math.isnan(x) and x > 0, self._fee_after))
# Register the template to the image and locate the regions of
# interest based on the registration parameters. XXX Should
# also give contrast: fit 2D-Gaussian to peak and report its
# standard deviations and fit?
if self._template is not None:
gamma = lewis(self._template, self.cspad_img)
p = flex.max_index(gamma)
peak = (
p // gamma.focus()[1] - self._template.focus()[0] + 1,
p % gamma.focus()[1] - self._template.focus()[1] + 1,
)
# """
### REFERENCE CHECK ###
from os.path import dirname, isdir, join
from scipy import io
mat_dirname = dirname(cspad_tbx.pathsubst(self._mat_path, evt, env, frame_number=self._nframes))
if not isdir(mat_dirname):
makedirs(mat_dirname)
io.savemat(
file_name=join(mat_dirname, "cross-check-%05d.mat" % self._nframes),
mdict=dict(
image=self.cspad_img.as_numpy_array(),
template=self._template.as_numpy_array(),
gamma=gamma.as_numpy_array(),
peak=numpy.array(peak),
),
appendmat=False,
do_compression=True,
oned_as="column",
)
return
### REFERENCE CHECK ###
# """
else:
# Alternative: position everything with respect to the frame
# origin.
peak = (0, 0)
# XXX Come up with a better way to handle the offsets! They
# really do depend on the template, and should therefore be
# packaged with it.
self.logger.info("Template registration anchor point (%d, %d)" % (peak[0], peak[1]))
roi = []
if evt.expNum() == 208:
# Regions of interest for L632 (experiment number 208). XXX
# Could perhaps migrate the template matching here instead?
# The left, middle, and right manganese signals. XXX Extend the
# rightmost ROI three pixels in upward direction (see runs 145
# and onwards, also note narrower slit)?
roi.append((peak[0] + 59, peak[1] - 24, 12, 5))
roi.append((peak[0] + 61, peak[1] + 28, 12, 4))
roi.append((peak[0] + 61, peak[1] + 79, 12, 5))
# Two background regions between the manganese spots, with the
# same total area as the signal.
roi.append((peak[0] + 62, peak[1] + 1, 8, 8))
roi.append((peak[0] + 63, peak[1] + 51, 8, 8))
# The left and right direct reflections from the Si substrate
# (i.e. the areas between the zone plates). These were the
# features used for template registration.
roi.append((peak[0], peak[1], 40, 10))
roi.append((peak[0], peak[1] + 50, 40, 9))
# Spot between the direct reflections. XXX What is this?
roi.append((peak[0] + 1, peak[1] + 23, 22, 13))
# The horizontal slit, where the direct reflection occurs. This
# is fixed. XXX Verify this!
roi.append((22, 0, 41, 128))
# Background stripe, below the manganese spots. This is fixed
# to the bottom of the detector.
roi.append((104, 0, 20, 128))
elif evt.expNum() == 363:
# Regions of interest for LB68 (experiment number 363).
# 0-pixel are active, 255-pixel are inactive
from scipy.misc import imread
# Dec 5, 2013 (09:00 - 21:00): initial estimates from r0010
"""
roi.append((peak[0] + 14, peak[1] + 138 + 23, 25, 50 - 25))
roi.append((peak[0] + 45, peak[1] + 138 + 23, 25, 50 - 25))
roi.append((peak[0] + 78, peak[1] + 137 + 23, 25, 50 - 25))
roi.append((peak[0] + 111, peak[1] + 137 + 23, 25, 50 - 25))
roi.append((peak[0] + 144, peak[1] + 137 + 23, 25, 50 - 25))
roi.append((peak[0] + 177, peak[1] + 136 + 23, 25, 50 - 25))
roi.append((peak[0] + 210, peak[1] + 136 + 23, 25, 50 - 25))
roi.append((peak[0] + 243, peak[1] + 136 + 23, 25, 50 - 25))
roi.append((peak[0] + 278, peak[1] + 135 + 23, 25, 50 - 25))
roi.append((peak[0] + 312, peak[1] + 135 + 23, 25, 50 - 25))
roi.append((peak[0] + 344, peak[1] + 135 + 23, 25, 50 - 25))
roi.append((peak[0] + 376, peak[1] + 135 + 23, 25, 50 - 25))
roi.append((peak[0] + 408, peak[1] + 135 + 23, 25, 50 - 25))
roi.append((peak[0] + 442, peak[1] + 135 + 23, 25, 50 - 25))
roi.append((peak[0] + 475, peak[1] + 135 + 23, 25, 50 - 25))
"""
# Dec 6, 2013 (09:00 - 21:00): rough estimates
"""
roi.append((peak[0] + 0, peak[1] + 25, 512, 25)) # bkg
roi.append((peak[0] + 0, peak[1] + 135, 512, 25)) # oxygen
roi.append((peak[0] + 0, peak[1] + 160, 512, 25)) # signal
roi.append((peak[0] + 0, peak[1] + 300, 512, 130)) # zeroth order
"""
# Dec 7, 2013 (09:00 - 21:00): overlap between oxygen and
# signal. Will loose some signal.
"""
roi.append((peak[0] + 0, peak[1] + 25, 512, 25)) # bkg
roi.append((peak[0] + 0, peak[1] + 135, 512, 50)) # oxygen
roi.append((peak[0] + 0, peak[1] + 185, 512, 40)) # signal
roi.append((peak[0] + 0, peak[1] + 270, 512, 170)) # zeroth order
"""
"""
# Dec 7 2013 (09:00 - 21:00): binary masks stored in PNG
# images.
roi.append((peak[0] + 0, peak[1] + 25, 512, 25)) # bkg
roi.append((peak[0] + 0, peak[1] + 135, 512, 25)) # oxygen
#roi_image = flex.float(
# imread('/reg/neh/home1/hattne/myrelease/LB68-r0039-max-mask.png',
# flatten=True))
#roi_image = flex.float(
# imread('/reg/neh/home1/hattne/myrelease/LB68-r0039-std-mask.png',
# flatten=True))
roi_image = flex.float(
imread('/reg/neh/home1/hattne/myrelease/LB68-r0052-avg-mask.png',
flatten=True))
roi_image = (255 - roi_image)
#roi.append((0, 0, self.cspad_img.focus()[0], self.cspad_img.focus()[1]))
roi.append(roi_image)
roi.append((peak[0] + 0, peak[1] + 270, 512, 170)) # zeroth order
"""
# Dec 9, 2013 (09:00 - 21:00)
# """
roi.append((peak[0] + 0, peak[1] + 25, 512, 25)) # bkg
roi.append((peak[0] + 0, peak[1] + 135, 512, 25)) # oxygen
# roi.append((peak[0] + 0, peak[1] + 160, 512, 25)) # signal
roi_image = flex.float(imread("/reg/neh/home1/hattne/myrelease/LB68-r0067-max-mask.png", flatten=True))
roi.append(roi_image)
roi.append((peak[0] + 0, peak[1] + 240, 512, 180)) # zeroth order
# """
else:
self.logger.error(
"No regions of interest for %s (experiment number %d)" % (env.experiment(), evt.expNum())
)
# Clip the regions of interest to the actual image. If the ROI
# does not overlap with the image at all, set its width and
# height to zero. XXX Do the integration here as well?
for i in range(len(roi)):
if not isinstance(roi[i], tuple):
continue
r = roi[i]
if (
r[0] + r[2] < 0
or r[0] >= self.cspad_img.focus()[0]
or r[1] + r[3] < 0
or r[1] >= self.cspad_img.focus()[1]
):
roi[i] = (r[0], r[1], 0, 0)
continue
r = roi[i]
if r[0] < 0:
roi[i] = (0, r[1], r[2] + r[0], r[3])
r = roi[i]
if r[1] < 0:
roi[i] = (r[0], 0, r[2], r[3] + r[1])
r = roi[i]
if r[0] + r[2] > self.cspad_img.focus()[0]:
roi[i] = (r[0], r[1], self.cspad_img.focus()[0] - r[0], r[3])
r = roi[i]
if r[1] + r[3] > self.cspad_img.focus()[1]:
roi[i] = (r[0], r[1], r[2], self.cspad_img.focus()[1] - r[1])
# Sum up intensities in all regions of interest, and keep track
# of the actual number of pixels summed. The common_mode module
# takes care of dark-subtraction. XXX Would like to estimate
# sigma for spot, like in spotfinder/LABELIT.
I = flex.double(len(roi))
I_nmemb = flex.int(len(roi))
for i in range(len(roi)):
if isinstance(roi[i], flex.float):
sel = roi[i].as_1d() < 128
I[i] = flex.sum(self.cspad_img.as_1d().select(sel))
I_nmemb[i] = sel.count(True)
continue
if roi[i][2] <= 0 or roi[i][3] <= 0:
I[i] = 0
I_nmemb[i] = 0
else:
I[i] = flex.sum(
self.cspad_img.matrix_copy_block(
i_row=roi[i][0], i_column=roi[i][1], n_rows=roi[i][2], n_columns=roi[i][3]
)
)
I_nmemb[i] = roi[i][2] * roi[i][3]
"""
# Sanity check: white out the region of interest.
self.cspad_img.matrix_paste_block_in_place(
block=flex.double(flex.grid(roi[i][2], roi[i][3])),
i_row=roi[i][0],
i_column=roi[i][1])
"""
acq_apd_sum = sum(filter(lambda x: not math.isnan(x) and x > 0, self._acq_apd_integral.select(self._hit)))
acq_opto_diode_sum = sum(
filter(lambda x: not math.isnan(x) and x > 0, self._acq_opto_diode_integral.select(self._hit))
)
acq_apd_sum_all = sum(filter(lambda x: not math.isnan(x) and x > 0, self._acq_apd_integral))
acq_opto_diode_sum_all = sum(filter(lambda x: not math.isnan(x) and x > 0, self._acq_opto_diode_integral))
# Append the data point to the stream: shots, hits, energy, and
# I. XXX OrderedDict requires Python 2.7, could fall back on
# regular Dict at the price of non-deterministic column order.
from collections import OrderedDict
csv_dict = OrderedDict(
[
("n_frames", self._hit.size()),
("n_hits", hits),
("I0", I0),
("I0_all", I0_all),
("fee_before_all", fee_before_all),
("fee_after_all", fee_after_all),
("energy_mean", energy_mean),
("acq_apd_sum", acq_apd_sum),
("acq_apd_sum_all", acq_apd_sum_all),
("acq_opto_diode_sum", acq_opto_diode_sum),
("acq_opto_diode_sum_all", acq_opto_diode_sum_all),
]
)
for (i, item) in enumerate(zip(roi, I, I_nmemb)):
key = "roi_" + ("bkg", "oxygen", "manganese", "zeroth_order")[i]
csv_dict["%s_nmemb" % key] = item[2]
if isinstance(item[0], tuple):
csv_dict["%s_ss_start" % key] = item[0][0]
csv_dict["%s_fs_start" % key] = item[0][1]
csv_dict["%s_ss_size" % key] = item[0][2]
csv_dict["%s_fs_size" % key] = item[0][3]
else:
csv_dict["%s_ss_start" % key] = 0
csv_dict["%s_fs_start" % key] = 0
csv_dict["%s_ss_size" % key] = item[0].focus()[0]
csv_dict["%s_fs_size" % key] = item[0].focus()[1]
csv_dict["%s_I" % key] = item[1]
# XXX assert that keys match up with what's in the file already?
# Or exploit the error-reporting mechanism already implemented?
# Write the header. XXX How to control the order of the
# columns?
if not hasattr(self, "_csv"):
from csv import DictWriter
self._csv = DictWriter(self._stream_table, csv_dict.keys())
self._csv.writerow({key: key for key in csv_dict.keys()})
self._csv.writerow(csv_dict)
# Output the non-normalised image and all other relevant data to
# a binary MATLAB file. XXX What if scipy is not available?
from os import makedirs, path
from scipy import io
mat_path = cspad_tbx.pathsubst(self._mat_path, evt, env, frame_number=self._nframes)
if not path.isdir(path.dirname(mat_path)):
makedirs(path.dirname(mat_path))
io.savemat(
file_name=mat_path,
mdict=dict(
DATA=self.cspad_img.as_numpy_array(),
DIODES=numpy.array((acq_apd_sum, acq_apd_sum_all, acq_opto_diode_sum, acq_opto_diode_sum_all)),
ENERGY=energy_mean,
HITS=numpy.array((hits, self._hit.size())),
I0=numpy.array((I0, I0_all)),
INTENSITIES=numpy.array(I),
ROIS=numpy.array([r for r in roi if isinstance(r, tuple)]),
),
appendmat=False,
do_compression=True,
oned_as="column",
)
# Optionally update the image in the viewer. See mod_view.
if self._display:
from time import localtime, strftime
# Copy over regions of interest to shared multiprocessing
# array. XXX Flip to honour wxPython convention.
for i in range(len(roi)):
if not isinstance(roi[i], tuple):
continue
self._roi[4 * i + 0] = roi[i][1]
self._roi[4 * i + 1] = roi[i][0]
self._roi[4 * i + 2] = roi[i][3]
self._roi[4 * i + 3] = roi[i][2]
time_str = strftime("%H:%M:%S", localtime(evt.getTime().seconds()))
title = "r%04d@%s: frame %d on %s" % (evt.run(), time_str, self._nframes, self.address)
# XXX No distance in the Andor experiment. So don't bother
# with the fictional beam center, distance, and saturation
# value? See also mod_average.endjob()
img_obj = (
dict(
BEAM_CENTER=(0, 0),
DATA=self.cspad_img,
DETECTOR_ADDRESS=self.address,
DISTANCE=10, # XXX Evil kludge to keep dxtbx happy!
PIXEL_SIZE=13.5e-3, # XXX Hard-coded, again!
SATURATED_VALUE=10000,
TIME_TUPLE=cspad_tbx.evt_time(evt),
WAVELENGTH=12398.4187 / energy,
),
title,
)
while not self._queue.empty():
if not self._proc.is_alive():
evt.setStatus(Event.Stop)
return
while True:
try:
self._queue.put(img_obj, timeout=1)
break
except Exception: # Queue.Full:
pass
self._reset_counters()
return
def average(argv=None):
if argv == None:
argv = sys.argv[1:]
try:
from mpi4py import MPI
except ImportError:
raise Sorry("MPI not found")
command_line = (libtbx.option_parser.option_parser(
usage="""
%s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f]
To write image pickles use -p, otherwise the program writes CSPAD CBFs.
Writing CBFs requires the geometry to be already deployed.
Examples:
cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
Use one process on the current node to process all the events from run 25 of
experiment cxi49812, using a detz_offset of 571.
mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
As above, using 16 cores on the current node.
bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812
As above, using the psanaq and 100 cores, putting the log in average.out and
the output images in the folder cxi49812.
""" % libtbx.env.dispatcher_name)
.option(None, "--as_pickle", "-p",
action="store_true",
default=False,
dest="as_pickle",
help="Write results as image pickle files instead of cbf files")
.option(None, "--config", "-c",
type="string",
default=None,
dest="config",
metavar="PATH",
help="psana config file")
.option(None, "--experiment", "-x",
type="string",
default=None,
dest="experiment",
help="experiment name (eg cxi84914)")
.option(None, "--run", "-r",
type="int",
default=None,
dest="run",
help="run number")
.option(None, "--address", "-a",
type="string",
default="CxiDs2.0:Cspad.0",
dest="address",
help="detector address name (eg CxiDs2.0:Cspad.0)")
.option(None, "--detz_offset", "-d",
type="float",
default=None,
dest="detz_offset",
help="offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)")
.option(None, "--outputdir", "-o",
type="string",
default=".",
dest="outputdir",
metavar="PATH",
help="Optional path to output directory for output files")
.option(None, "--averagebase", "-A",
type="string",
default="{experiment!l}_avg-r{run:04d}",
dest="averagepath",
metavar="PATH",
help="Path to output average image without extension. String substitution allowed")
.option(None, "--stddevbase", "-S",
type="string",
default="{experiment!l}_stddev-r{run:04d}",
dest="stddevpath",
metavar="PATH",
help="Path to output standard deviation image without extension. String substitution allowed")
.option(None, "--maxbase", "-M",
type="string",
default="{experiment!l}_max-r{run:04d}",
dest="maxpath",
metavar="PATH",
help="Path to output maximum projection image without extension. String substitution allowed")
.option(None, "--numevents", "-n",
type="int",
default=None,
dest="numevents",
help="Maximum number of events to process. Default: all")
.option(None, "--skipevents", "-s",
type="int",
default=0,
dest="skipevents",
help="Number of events in the beginning of the run to skip. Default: 0")
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
.option(None, "--pickle-optical-metrology", "-m",
action="store_true",
default=False,
dest="pickle_optical_metrology",
help="If writing pickle files, use the optical metrology in the experiment's calib directory")
.option(None, "--bin_size", "-b",
type="int",
default=None,
dest="bin_size",
help="Rayonix detector bin size")
.option(None, "--override_beam_x", "-X",
type="float",
default=None,
dest="override_beam_x",
help="Rayonix detector beam center x coordinate")
.option(None, "--override_beam_y", "-Y",
type="float",
default=None,
dest="override_beam_y",
help="Rayonix detector beam center y coordinate")
.option(None, "--calib_dir", "-C",
type="string",
default=None,
dest="calib_dir",
metavar="PATH",
help="calibration directory")
.option(None, "--xtc_dir", "-D",
type="string",
default=None,
dest="xtc_dir",
metavar="PATH",
help="xtc stream directory")
.option(None, "--use_ffb", "-f",
action="store_true",
default=False,
dest="use_ffb",
help="Use the fast feedback filesystem at LCLS. Only for the active experiment!")
).process(args=argv)
if len(command_line.args) > 0 or \
command_line.options.as_pickle is None or \
command_line.options.experiment is None or \
command_line.options.run is None or \
command_line.options.address is None or \
command_line.options.detz_offset is None or \
command_line.options.averagepath is None or \
command_line.options.stddevpath is None or \
command_line.options.maxpath is None or \
command_line.options.pickle_optical_metrology is None:
command_line.parser.show_help()
return
# set this to sys.maxint to analyze all events
if command_line.options.numevents is None:
maxevents = sys.maxint
else:
maxevents = command_line.options.numevents
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if command_line.options.config is not None:
psana.setConfigFile(command_line.options.config)
dataset_name = "exp=%s:run=%d:idx"%(command_line.options.experiment, command_line.options.run)
if command_line.options.xtc_dir is not None:
if command_line.options.use_ffb:
raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system")
dataset_name += ":dir=%s"%command_line.options.xtc_dir
elif command_line.options.use_ffb:
# as ffb is only at SLAC, ok to hardcode /reg/d here
dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc"%(command_line.options.experiment[0:3],command_line.options.experiment)
ds = psana.DataSource(dataset_name)
address = command_line.options.address
src = psana.Source('DetInfo(%s)'%address)
if not command_line.options.as_pickle:
psana_det = psana.Detector(address, ds.env())
nevent = np.array([0.])
for run in ds.runs():
runnumber = run.run()
# list of all events
if command_line.options.skipevents > 0:
print "Skipping first %d events"%command_line.options.skipevents
times = run.times()[command_line.options.skipevents:]
nevents = min(len(times),maxevents)
# chop the list into pieces, depending on rank. This assigns each process
# events such that the get every Nth event where N is the number of processes
mytimes = [times[i] for i in xrange(nevents) if (i+rank)%size == 0]
for i in xrange(len(mytimes)):
if i%10==0: print 'Rank',rank,'processing event',rank*len(mytimes)+i,', ',i,'of',len(mytimes)
evt = run.event(mytimes[i])
#print "Event #",rank*mylength+i," has id:",evt.get(EventId)
if 'Rayonix' in command_line.options.address:
data = evt.get(Camera.FrameV1,src)
if data is None:
print "No data"
continue
data=data.data16().astype(np.float64)
elif command_line.options.as_pickle:
data = evt.get(psana.ndarray_float64_3, src, 'image0')
else:
# get numpy array, 32x185x388
data = psana_det.calib(evt) # applies psana's complex run-dependent calibrations
if data is None:
print "No data"
continue
d = cspad_tbx.env_distance(address, run.env(), command_line.options.detz_offset)
if d is None:
print "No distance, skipping shot"
continue
if 'distance' in locals():
distance += d
else:
distance = np.array([float(d)])
w = cspad_tbx.evt_wavelength(evt)
if w is None:
print "No wavelength, skipping shot"
continue
if 'wavelength' in locals():
wavelength += w
else:
wavelength = np.array([w])
t = cspad_tbx.evt_time(evt)
if t is None:
print "No timestamp, skipping shot"
continue
if 'timestamp' in locals():
timestamp += t[0] + (t[1]/1000)
else:
timestamp = np.array([t[0] + (t[1]/1000)])
if 'sum' in locals():
sum+=data
else:
sum=np.array(data, copy=True)
if 'sumsq' in locals():
sumsq+=data*data
else:
sumsq=data*data
if 'maximum' in locals():
maximum=np.maximum(maximum,data)
else:
maximum=np.array(data, copy=True)
nevent += 1
#sum the images across mpi cores
if size > 1:
print "Synchronizing rank", rank
totevent = np.zeros(nevent.shape)
comm.Reduce(nevent,totevent)
if rank == 0 and totevent[0] == 0:
raise Sorry("No events found in the run")
sumall = np.zeros(sum.shape).astype(sum.dtype)
comm.Reduce(sum,sumall)
sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype)
comm.Reduce(sumsq,sumsqall)
maxall = np.zeros(maximum.shape).astype(maximum.dtype)
comm.Reduce(maximum,maxall, op=MPI.MAX)
waveall = np.zeros(wavelength.shape).astype(wavelength.dtype)
comm.Reduce(wavelength,waveall)
distall = np.zeros(distance.shape).astype(distance.dtype)
comm.Reduce(distance,distall)
timeall = np.zeros(timestamp.shape).astype(timestamp.dtype)
comm.Reduce(timestamp,timeall)
if rank==0:
if size > 1:
print "Synchronized"
# Accumulating floating-point numbers introduces errors,
# which may cause negative variances. Since a two-pass
# approach is unacceptable, the standard deviation is
# clamped at zero.
mean = sumall / float(totevent[0])
variance = (sumsqall / float(totevent[0])) - (mean**2)
variance[variance < 0] = 0
stddev = np.sqrt(variance)
wavelength = waveall[0] / totevent[0]
distance = distall[0] / totevent[0]
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
timestamp = timeall[0] / totevent[0]
timestamp = (int(timestamp), timestamp % int(timestamp) * 1000)
timestamp = cspad_tbx.evt_timestamp(timestamp)
if command_line.options.as_pickle:
extension = ".pickle"
else:
extension = ".cbf"
dest_paths = [cspad_tbx.pathsubst(command_line.options.averagepath + extension, evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.stddevpath + extension, evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt, ds.env())]
dest_paths = [os.path.join(command_line.options.outputdir, path) for path in dest_paths]
if 'Rayonix' in command_line.options.address:
from xfel.cxi.cspad_ana import rayonix_tbx
pixel_size = rayonix_tbx.get_rayonix_pixel_size(command_line.options.bin_size)
beam_center = [command_line.options.override_beam_x,command_line.options.override_beam_y]
detector_dimensions = rayonix_tbx.get_rayonix_detector_dimensions(command_line.options.bin_size)
active_areas = flex.int([0,0,detector_dimensions[0],detector_dimensions[1]])
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3]
for data, path in zip([mean, stddev, maxall], dest_paths):
print "Saving", path
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=flex.double(data),
distance=distance,
pixel_size=pixel_size,
saturated_value=rayonix_tbx.rayonix_saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
elif command_line.options.as_pickle:
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3]
xpp = 'xpp' in address.lower()
if xpp:
evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format
from xfel.detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(old_style_address, reverse_timestamp(timestamp)[0])
assert version_lookup is not None
active_areas = xpp_active_areas[version_lookup]['active_areas']
beam_center = [1765 // 2, 1765 // 2]
else:
if command_line.options.calib_dir is not None:
metro_path = command_line.options.calib_dir
elif command_line.options.pickle_optical_metrology:
from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path
metro_path = get_calib_file_path(run.env(), address, run)
else:
metro_path = libtbx.env.find_in_repositories("xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
sections = parse_calib.calib2sections(metro_path)
beam_center, active_areas = cspad_tbx.cbcaa(
cspad_tbx.getConfig(address, ds.env()), sections)
class fake_quad(object):
def __init__(self, q, d):
self.q = q
self.d = d
def quad(self):
return self.q
def data(self):
return self.d
if xpp:
quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)]
mean = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
mean = flex.double(mean.astype(np.float64))
quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)]
stddev = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)]
maxall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
maxall = flex.double(maxall.astype(np.float64))
else:
quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)]
mean = cspad_tbx.CsPadDetector(
address, evt, ds.env(), sections, quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)]
stddev = cspad_tbx.CsPadDetector(
address, evt, ds.env(), sections, quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)]
maxall = cspad_tbx.CsPadDetector(
address, evt, ds.env(), sections, quads=quads)
maxall = flex.double(maxall.astype(np.float64))
for data, path in zip([mean, stddev, maxall], dest_paths):
print "Saving", path
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=data,
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
else:
# load a header only cspad cbf from the slac metrology
from xfel.cftbx.detector import cspad_cbf_tbx
import pycbf
base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(run, address)
if base_dxtbx is None:
raise Sorry("Couldn't load calibration file for run %d"%run.run())
for data, path in zip([mean, stddev, maxall], dest_paths):
print "Saving", path
cspad_img = cspad_cbf_tbx.format_object_from_data(base_dxtbx, data, distance, wavelength, timestamp, address)
cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)