class TailCutsDataVolumeReducer(DataVolumeReducer):
"""
Reduce the time integrated shower image in 3 Steps:
1) Select pixels with tailcuts_clean.
2) Add iteratively all pixels with Signal S >= boundary_thresh
with ctapipe module dilate until no new pixels were added.
3) Adding new pixels with dilate to get more conservative.
"""
n_end_dilates = IntTelescopeParameter(
default_value=1,
help="Number of how many times to dilate at the end.").tag(config=True)
do_boundary_dilation = BoolTelescopeParameter(
default_value=True,
help="If set to 'False', the iteration steps in 2) are skipped and"
"normal TailcutCleaning is used.",
).tag(config=True)
def select_pixels(self, waveforms, telid=None, selected_gain_channel=None):
camera_geom = self.subarray.tel[telid].camera.geometry
# Pulse-integrate waveforms
charge, _ = self.image_extractor(
waveforms,
telid=telid,
selected_gain_channel=selected_gain_channel)
# 1) Step: TailcutCleaning at first
mask = self.cleaner(telid, charge)
pixels_above_boundary_thresh = (
charge >= self.cleaner.boundary_threshold_pe.tel[telid])
mask_in_loop = np.array([])
# 2) Step: Add iteratively all pixels with Signal
# S > boundary_thresh with ctapipe module
# 'dilate' until no new pixels were added.
while (not np.array_equal(mask, mask_in_loop)
and self.do_boundary_dilation.tel[telid]):
mask_in_loop = mask
mask = dilate(camera_geom, mask) & pixels_above_boundary_thresh
# 3) Step: Adding Pixels with 'dilate' to get more conservative.
for _ in range(self.n_end_dilates.tel[telid]):
mask = dilate(camera_geom, mask)
return mask
class TwoPassWindowSum(ImageExtractor):
"""Extractor based on [1]_ which integrates the waveform a second time using
a time-gradient linear fit. This is in particular the version implemented
in the CTA-MARS analysis pipeline [2]_.
Notes
-----
#. slide a 3-samples window through the waveform, finding max counts sum;
the range of the sliding is the one allowing extension from 3 to 5;
add 1 sample on each side and integrate charge in the 5-sample window;
time is obtained as a charge-weighted average of the sample numbers;
No information from neighboouring pixels is used.
#. Preliminary image cleaning via simple tailcut with minimum number
of core neighbours set at 1,
#. Only the biggest cluster of pixels is kept.
#. Parametrize following Hillas approach only if the resulting image has 3
or more pixels.
#. Do a linear fit of pulse time vs. distance along major image axis
(CTA-MARS uses ROOT "robust" fit option,
aka Least Trimmed Squares, to get rid of far outliers - this should
be implemented in 'timing_parameters', e.g scipy.stats.siegelslopes).
#. For all pixels except the core ones in the main island, integrate
the waveform once more, in a fixed window of 5 samples set at the time
"predicted" by the linear time fit.
If the predicted time for a pixel leads to a window outside the readout
window, then integrate the last (or first) 5 samples.
#. The result is an image with main-island core pixels calibrated with a
1st pass and non-core pixels re-calibrated with a 2nd pass.
References
----------
.. [1] J. Holder et al., Astroparticle Physics, 25, 6, 391 (2006)
.. [2] https://forge.in2p3.fr/projects/step-by-step-reference-mars-analysis/wiki
"""
# Get thresholds for core-pixels depending on telescope type.
# WARNING: default values are not yet optimized
core_threshold = FloatTelescopeParameter(
default_value=[
("type", "*", 6.0),
("type", "LST*", 6.0),
("type", "MST*", 8.0),
("type", "SST*", 4.0),
],
help="Picture threshold for internal tail-cuts pass",
).tag(config=True)
disable_second_pass = Bool(
default_value=False,
help="only run the first pass of the extractor, for debugging purposes",
).tag(config=True)
apply_integration_correction = BoolTelescopeParameter(
default_value=True,
help="Apply the integration window correction").tag(config=True)
@lru_cache(maxsize=4096)
def _calculate_correction(self, telid, width, shift):
"""Obtain the correction for the integration window specified for each
pixel.
The TwoPassWindowSum image extractor applies potentially different
parameters for the integration window to each pixel, depending on the
position of the peak. It has been decided to apply gain selection
directly here. For basic definitions look at the documentation of
`integration_correction`.
Parameters
----------
telid : int
Index of the telescope in use.
width : int
Width of the integration window in samples
shift : int
Window shift to the left of the pulse peak in samples
Returns
-------
correction : ndarray
Value of the pixel-wise gain-selected integration correction.
"""
readout = self.subarray.tel[telid].camera.readout
# Calculate correction of first pixel for both channels
return integration_correction(
readout.reference_pulse_shape,
readout.reference_pulse_sample_width.to_value("ns"),
(1 / readout.sampling_rate).to_value("ns"),
width,
shift,
)
def _apply_first_pass(self, waveforms,
telid) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
"""
Execute step 1.
Parameters
----------
waveforms : array of size (N_pixels, N_samples)
DL0-level waveforms of one event.
telid : int
Index of the telescope.
Returns
-------
charge : array_like
Integrated charge per pixel.
Shape: (n_pix)
pulse_time : array_like
Samples in which the waveform peak has been recognized.
Shape: (n_pix)
"""
# STEP 1
# Starting from DL0, the channel is already selected (if more than one)
# event.dl0.tel[tel_id].waveform object has shape (N_pixels, N_samples)
# For each pixel, we slide a 3-samples window through the
# waveform summing each time the ADC counts contained within it.
peak_search_window_width = 3
sums = convolve1d(waveforms,
np.ones(peak_search_window_width),
axis=1,
mode="nearest")
# 'sums' has now still shape of (N_pixels, N_samples)
# each element is the center-sample of each 3-samples sliding window
# For each pixel, we check where the peak search window encountered
# the maximum number of ADC counts.
# We want to stop before the edge of the readout window in order to
# later extend the search window to a 1+3+1 integration window.
# Since in 'sums' the peak index corresponds to the center of the
# search window, we shift it on the right by 2 samples so to get the
# correspondent sample index in each waveform.
peak_index = np.argmax(sums[:, 2:-2], axis=1) + 2
# Now peak_index has the shape of (N_pixels).
# The final 5-samples window will be 1+3+1, centered on the 3-samples
# window in which the highest amount of ADC counts has been found
window_width = peak_search_window_width + 2
window_shift = 2
# this function is applied to all pixels together
charge_1stpass, pulse_time_1stpass = extract_around_peak(
waveforms,
peak_index,
window_width,
window_shift,
self.sampling_rate[telid],
)
# Get integration correction factors
if self.apply_integration_correction.tel[telid]:
correction = self._calculate_correction(telid, window_width,
window_shift)
else:
correction = np.ones(waveforms.shape[0])
return charge_1stpass, pulse_time_1stpass, correction
def _apply_second_pass(
self,
waveforms,
telid,
selected_gain_channel,
charge_1stpass_uncorrected,
pulse_time_1stpass,
correction,
) -> Tuple[np.ndarray, np.ndarray]:
"""
Follow steps from 2 to 7.
Parameters
----------
waveforms : array of shape (N_pixels, N_samples)
DL0-level waveforms of one event.
telid : int
Index of the telescope.
selected_gain_channel: array of shape (N_channels, N_pixels)
Array containing the index of the selected gain channel for each
pixel (0 for low gain, 1 for high gain).
charge_1stpass_uncorrected : array of shape N_pixels
Pixel charges reconstructed with the 1st pass, but not corrected.
pulse_time_1stpass : array of shape N_pixels
Pixel-wise pulse times reconstructed with the 1st pass.
correction: array of shape N_pixels
Charge correction from 1st pass.
Returns
-------
charge : array_like
Integrated charge per pixel.
Note that in the case of a very bright full-camera image this can
coincide the 1st pass information.
Also in the case of very dim images the 1st pass will be recycled,
but in this case the resulting image should be discarded
from further analysis.
Shape: (n_pix)
pulse_time : array_like
Samples in which the waveform peak has been recognized.
Same specifications as above.
Shape: (n_pix)
"""
# STEP 2
# Apply correction to 1st pass charges
charge_1stpass = charge_1stpass_uncorrected * correction[
selected_gain_channel]
# Set thresholds for core-pixels depending on telescope
core_th = self.core_threshold.tel[telid]
# Boundary thresholds will be half of core thresholds.
# Preliminary image cleaning with simple two-level tail-cut
camera_geometry = self.subarray.tel[telid].camera.geometry
mask_1 = tailcuts_clean(
camera_geometry,
charge_1stpass,
picture_thresh=core_th,
boundary_thresh=core_th / 2,
keep_isolated_pixels=False,
min_number_picture_neighbors=1,
)
image_1 = charge_1stpass.copy()
image_1[~mask_1] = 0
# STEP 3
# find all islands using this cleaning
num_islands, labels = number_of_islands(camera_geometry, mask_1)
if num_islands == 0:
image_2 = image_1.copy() # no islands = image unchanged
else:
# ...find the biggest one
mask_biggest = largest_island(labels)
image_2 = image_1.copy()
image_2[~mask_biggest] = 0
# Indexes of pixels that will need the 2nd pass
non_core_pixels_ids = np.where(image_2 < core_th)[0]
non_core_pixels_mask = image_2 < core_th
# STEP 4
# if the resulting image has less then 3 pixels
# or there are more than 3 pixels but all contain a number of
# photoelectrons above the core threshold
if np.count_nonzero(image_2) < 3:
# we return the 1st pass information
# NOTE: In this case, the image was not bright enough!
# We should label it as "bad and NOT use it"
return charge_1stpass, pulse_time_1stpass
elif len(non_core_pixels_ids) == 0:
# Since all reconstructed charges are above the core threshold,
# there is no need to perform the 2nd pass.
# We return the 1st pass information.
# NOTE: In this case, even if this is 1st pass information,
# the image is actually very bright! We should label it as "good"!
return charge_1stpass, pulse_time_1stpass
# otherwise we proceed by parametrizing the image
hillas = hillas_parameters(camera_geometry, image_2)
# STEP 5
# linear fit of pulse time vs. distance along major image axis
# using only the main island surviving the preliminary
# image cleaning
timing = timing_parameters(camera_geometry, image_2,
pulse_time_1stpass, hillas)
# If the fit returns nan
if np.isnan(timing.slope):
return charge_1stpass, pulse_time_1stpass
# get projected distances along main image axis
long, _ = camera_to_shower_coordinates(camera_geometry.pix_x,
camera_geometry.pix_y, hillas.x,
hillas.y, hillas.psi)
# get the predicted times as a linear relation
predicted_pulse_times = (timing.slope * long[non_core_pixels_ids] +
timing.intercept)
predicted_peaks = np.zeros(len(predicted_pulse_times))
# Convert time in ns to sample index using the sampling rate from
# the readout.
# Approximate the value obtained to nearest integer, then cast to
# int64 otherwise 'extract_around_peak' complains.
sampling_rate = self.sampling_rate[telid]
np.rint(predicted_pulse_times.value * sampling_rate, predicted_peaks)
predicted_peaks = predicted_peaks.astype(np.int64)
# Due to the fit these peak indexes can now be also outside of the
# readout window, so later we check for this.
# STEP 6
# select only the waveforms correspondent to the non-core pixels
# of the main island survived from the 1st pass image cleaning
non_core_waveforms = waveforms[non_core_pixels_ids]
# Build 'width' and 'shift' arrays that adapt on the position of the
# window along each waveform
# As before we will integrate the charge in a 5-sample window centered
# on the peak
window_width_default = 5
window_shift_default = 2
# now let's deal with some edge cases: the predicted peak falls before
# or after the readout window:
peak_before_window = predicted_peaks < 0
peak_after_window = predicted_peaks > (non_core_waveforms.shape[1] - 1)
# BUT, if the resulting 5-samples window falls outside of the readout
# window then we take the first (or last) 5 samples
window_width_before = 5
window_shift_before = 0
# in the case where the window is after, shift backward
window_width_after = 5
window_shift_after = 5
# and put them together:
window_widths = np.full(non_core_waveforms.shape[0],
window_width_default)
window_widths[peak_before_window] = window_width_before
window_widths[peak_after_window] = window_width_after
window_shifts = np.full(non_core_waveforms.shape[0],
window_shift_default)
window_shifts[peak_before_window] = window_shift_before
window_shifts[peak_after_window] = window_shift_after
# Now we can also (re)define the patological predicted times
# because (we needed them to define the corrispective widths
# and shifts)
# set sample to 0 (beginning of the waveform) if predicted time
# falls before
predicted_peaks[predicted_peaks < 0] = 0
# set sample to max-1 (first sample has index 0)
# if predicted time falls after
predicted_peaks[predicted_peaks > (waveforms.shape[1] - 1)] = (
waveforms.shape[1] - 1)
# re-calibrate non-core pixels using the fixed 5-samples window
charge_no_core, pulse_times_no_core = extract_around_peak(
non_core_waveforms,
predicted_peaks,
window_widths,
window_shifts,
self.sampling_rate[telid],
)
if self.apply_integration_correction.tel[telid]:
# Modify integration correction factors only for non-core pixels
# now we compute 3 corrections for the default, before, and after cases:
correction = self._calculate_correction(
telid, window_width_default, window_shift_default
)[selected_gain_channel][non_core_pixels_mask]
correction_before = self._calculate_correction(
telid, window_width_before, window_shift_before
)[selected_gain_channel][non_core_pixels_mask]
correction_after = self._calculate_correction(
telid, window_width_after, window_shift_after
)[selected_gain_channel][non_core_pixels_mask]
correction[peak_before_window] = correction_before[
peak_before_window]
correction[peak_after_window] = correction_after[peak_after_window]
charge_no_core *= correction
# STEP 7
# Combine core and non-core pixels in the final output
# this is the biggest cluster from the cleaned image
# it contains the core pixels (which we leave untouched)
# plus possibly some non-core pixels
charge_2ndpass = image_2.copy()
# Now we overwrite the charges of all non-core pixels in the camera
# plus all those pixels which didn't survive the preliminary
# cleaning.
# We apply also their corrections.
charge_2ndpass[non_core_pixels_mask] = charge_no_core
# Same approach for the pulse times
pulse_time_2ndpass = pulse_time_1stpass # core + non-core pixels
pulse_time_2ndpass[
non_core_pixels_mask] = pulse_times_no_core # non-core pixels
return charge_2ndpass, pulse_time_2ndpass
def __call__(self, waveforms, telid, selected_gain_channel):
"""
Call this ImageExtractor.
Parameters
----------
waveforms : array of shape (N_pixels, N_samples)
DL0-level waveforms of one event.
telid : int
Index of the telescope.
selected_gain_channel: array of shape (N_channels, N_pixels)
Array containing the index of the selected gain channel for each
pixel (0 for low gain, 1 for high gain).
Returns
-------
charge : array_like
Integrated charge per pixel.
Shape: (n_pix)
pulse_time : array_like
Samples in which the waveform peak has been recognized.
Shape: (n_pix)
"""
charge1, pulse_time1, correction1 = self._apply_first_pass(
waveforms, telid)
# FIXME: properly make sure that output is 32Bit instead of downcasting here
if self.disable_second_pass:
return (
(charge1 *
correction1[selected_gain_channel]).astype("float32"),
pulse_time1.astype("float32"),
)
charge2, pulse_time2 = self._apply_second_pass(waveforms, telid,
selected_gain_channel,
charge1, pulse_time1,
correction1)
# FIXME: properly make sure that output is 32Bit instead of downcasting here
return charge2.astype("float32"), pulse_time2.astype("float32")
class NeighborPeakWindowSum(ImageExtractor):
"""
Extractor which sums in a window about the
peak defined by the wavefroms in neighboring pixels.
"""
window_width = IntTelescopeParameter(
default_value=7,
help="Define the width of the integration window").tag(config=True)
window_shift = IntTelescopeParameter(
default_value=3,
help="Define the shift of the integration window "
"from the peak_index (peak_index - shift)",
).tag(config=True)
lwt = IntTelescopeParameter(
default_value=0,
help="Weight of the local pixel (0: peak from neighbors only, "
"1: local pixel counts as much as any neighbor)",
).tag(config=True)
apply_integration_correction = BoolTelescopeParameter(
default_value=True,
help="Apply the integration window correction").tag(config=True)
@lru_cache(maxsize=128)
def _calculate_correction(self, telid):
"""
Calculate the correction for the extracted change such that the value
returned would equal 1 for a noise-less unit pulse.
This method is decorated with @lru_cache to ensure it is only
calculated once per telescope.
Parameters
----------
telid : int
Returns
-------
correction : ndarray
The correction to apply to an extracted charge using this ImageExtractor
Has size n_channels, as a different correction value might be required
for different gain channels.
"""
readout = self.subarray.tel[telid].camera.readout
return integration_correction(
readout.reference_pulse_shape,
readout.reference_pulse_sample_width.to_value("ns"),
(1 / readout.sampling_rate).to_value("ns"),
self.window_width.tel[telid],
self.window_shift.tel[telid],
)
def __call__(self, waveforms, telid, selected_gain_channel):
neighbors = self.subarray.tel[
telid].camera.geometry.neighbor_matrix_sparse
average_wfs = neighbor_average_waveform(
waveforms,
neighbors_indices=neighbors.indices,
neighbors_indptr=neighbors.indptr,
lwt=self.lwt.tel[telid],
)
peak_index = average_wfs.argmax(axis=-1)
charge, peak_time = extract_around_peak(
waveforms,
peak_index,
self.window_width.tel[telid],
self.window_shift.tel[telid],
self.sampling_rate[telid],
)
if self.apply_integration_correction.tel[telid]:
charge *= self._calculate_correction(
telid=telid)[selected_gain_channel]
return charge, peak_time
class LocalPeakWindowSum(ImageExtractor):
"""
Extractor which sums in a window about the
peak in each pixel's waveform.
"""
window_width = IntTelescopeParameter(
default_value=7,
help="Define the width of the integration window").tag(config=True)
window_shift = IntTelescopeParameter(
default_value=3,
help="Define the shift of the integration window"
"from the peak_index (peak_index - shift)",
).tag(config=True)
apply_integration_correction = BoolTelescopeParameter(
default_value=True,
help="Apply the integration window correction").tag(config=True)
@lru_cache(maxsize=128)
def _calculate_correction(self, telid):
"""
Calculate the correction for the extracted change such that the value
returned would equal 1 for a noise-less unit pulse.
This method is decorated with @lru_cache to ensure it is only
calculated once per telescope.
Parameters
----------
telid : int
Returns
-------
correction : ndarray
The correction to apply to an extracted charge using this ImageExtractor
Has size n_channels, as a different correction value might be required
for different gain channels.
"""
readout = self.subarray.tel[telid].camera.readout
return integration_correction(
readout.reference_pulse_shape,
readout.reference_pulse_sample_width.to_value("ns"),
(1 / readout.sampling_rate).to_value("ns"),
self.window_width.tel[telid],
self.window_shift.tel[telid],
)
def __call__(self, waveforms, telid, selected_gain_channel):
peak_index = waveforms.argmax(axis=-1).astype(np.int)
charge, peak_time = extract_around_peak(
waveforms,
peak_index,
self.window_width.tel[telid],
self.window_shift.tel[telid],
self.sampling_rate[telid],
)
if self.apply_integration_correction.tel[telid]:
charge *= self._calculate_correction(
telid=telid)[selected_gain_channel]
return charge, peak_time
class SlidingWindowMaxSum(ImageExtractor):
"""
Sliding window extractor that maximizes the signal in window_width consecutive slices.
"""
window_width = IntTelescopeParameter(
default_value=7, help="Define the width of the integration window"
).tag(config=True)
apply_integration_correction = BoolTelescopeParameter(
default_value=True, help="Apply the integration window correction"
).tag(config=True)
@lru_cache(maxsize=128)
def _calculate_correction(self, telid):
"""
Calculate the correction for the extracted charge such that the value
returned would equal 1 for a noise-less unit pulse.
This method is decorated with @lru_cache to ensure it is only
calculated once per telescope.
The same procedure as for the actual SlidingWindowMaxSum extractor is used, but
on the reference pulse_shape (that is also more finely binned)
Parameters
----------
telid : int
Returns
-------
correction : ndarray
The correction to apply to an extracted charge using this ImageExtractor
Has size n_channels, as a different correction value might be required
for different gain channels.
"""
readout = self.subarray.tel[telid].camera.readout
# compute the number of slices to integrate in the pulse template
width_shape = int(
round(
(
self.window_width.tel[telid]
/ readout.sampling_rate
/ readout.reference_pulse_sample_width
)
.to("")
.value
)
)
n_channels = len(readout.reference_pulse_shape)
correction = np.ones(n_channels, dtype=np.float)
for ichannel, pulse_shape in enumerate(readout.reference_pulse_shape):
# apply the same method as sliding window to find the highest sum
cwf = np.cumsum(pulse_shape)
# add zero at the begining so it is easier to substract the two arrays later
cwf = np.concatenate((np.zeros(1), cwf))
sums = cwf[width_shape:] - cwf[:-width_shape]
maxsum = np.max(sums)
correction[ichannel] = np.sum(pulse_shape) / maxsum
return correction
def __call__(self, waveforms, telid, selected_gain_channel):
charge, peak_time = extract_sliding_window(
waveforms, self.window_width.tel[telid], self.sampling_rate_ghz[telid]
)
if self.apply_integration_correction.tel[telid]:
charge *= self._calculate_correction(telid=telid)[selected_gain_channel]
return charge, peak_time
class TailCutsDataVolumeReducer(DataVolumeReducer):
"""
Reduce the time integrated shower image in 3 Steps:
1) Select pixels with tailcuts_clean.
2) Add iteratively all pixels with Signal S >= boundary_thresh
with ctapipe module dilate until no new pixels were added.
3) Adding new pixels with dilate to get more conservative.
Attributes
----------
image_extractor_type: String
Name of the image_extractor to be used.
n_end_dilates: IntTelescopeParameter
Number of how many times to dilate at the end.
do_boundary_dilation: BoolTelescopeParameter
If set to 'False', the iteration steps in 2) are skipped and
normal TailcutCleaning is used.
"""
image_extractor_type = TelescopeParameter(
trait=create_class_enum_trait(ImageExtractor,
default_value="NeighborPeakWindowSum"),
default_value="NeighborPeakWindowSum",
help="Name of the ImageExtractor subclass to be used.",
).tag(config=True)
n_end_dilates = IntTelescopeParameter(
default_value=1,
help="Number of how many times to dilate at the end.").tag(config=True)
do_boundary_dilation = BoolTelescopeParameter(
default_value=True,
help="If set to 'False', the iteration steps in 2) are skipped and"
"normal TailcutCleaning is used.",
).tag(config=True)
def __init__(
self,
subarray,
config=None,
parent=None,
cleaner=None,
image_extractor=None,
**kwargs,
):
"""
Parameters
----------
subarray: ctapipe.instrument.SubarrayDescription
Description of the subarray
config: traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
kwargs
"""
super().__init__(config=config,
parent=parent,
subarray=subarray,
**kwargs)
if cleaner is None:
self.cleaner = TailcutsImageCleaner(parent=self,
subarray=self.subarray)
else:
self.cleaner = cleaner
self.image_extractors = {}
if image_extractor is None:
for (_, _, name) in self.image_extractor_type:
self.image_extractors[name] = ImageExtractor.from_name(
name, subarray=self.subarray, parent=self)
else:
name = image_extractor.__class__.__name__
self.image_extractor_type = [("type", "*", name)]
self.image_extractors[name] = image_extractor
def select_pixels(self, waveforms, telid=None, selected_gain_channel=None):
camera_geom = self.subarray.tel[telid].camera.geometry
# Pulse-integrate waveforms
extractor = self.image_extractors[self.image_extractor_type.tel[telid]]
charge, _ = extractor(waveforms,
telid=telid,
selected_gain_channel=selected_gain_channel)
# 1) Step: TailcutCleaning at first
mask = self.cleaner(telid, charge)
pixels_above_boundary_thresh = (
charge >= self.cleaner.boundary_threshold_pe.tel[telid])
mask_in_loop = np.array([])
# 2) Step: Add iteratively all pixels with Signal
# S > boundary_thresh with ctapipe module
# 'dilate' until no new pixels were added.
while (not np.array_equal(mask, mask_in_loop)
and self.do_boundary_dilation.tel[telid]):
mask_in_loop = mask
mask = dilate(camera_geom, mask) & pixels_above_boundary_thresh
# 3) Step: Adding Pixels with 'dilate' to get more conservative.
for _ in range(self.n_end_dilates.tel[telid]):
mask = dilate(camera_geom, mask)
return mask
class CameraCalibrator(TelescopeComponent):
"""
Calibrator to handle the full camera calibration chain, in order to fill
the DL1 data level in the event container.
Attributes
----------
data_volume_reducer_type: str
The name of the DataVolumeReducer subclass to be used
for data volume reduction
image_extractor_type: str
The name of the ImageExtractor subclass to be used for image extraction
"""
data_volume_reducer_type = create_class_enum_trait(
DataVolumeReducer,
default_value="NullDataVolumeReducer").tag(config=True)
image_extractor_type = TelescopeParameter(
trait=create_class_enum_trait(ImageExtractor,
default_value="NeighborPeakWindowSum"),
default_value="NeighborPeakWindowSum",
help="Name of the ImageExtractor subclass to be used.",
).tag(config=True)
apply_waveform_time_shift = BoolTelescopeParameter(
default_value=False,
help=("Apply waveform time shift corrections."
" The minimal integer shift to synchronize waveforms is applied"
" before peak extraction if this option is True"),
).tag(config=True)
apply_peak_time_shift = BoolTelescopeParameter(
default_value=True,
help=
("Apply peak time shift corrections."
" Apply the remaining absolute and fractional time shift corrections"
" to the peak time after pulse extraction."
" If `apply_waveform_time_shift` is False, this will apply the full time shift"
),
).tag(config=True)
def __init__(
self,
subarray,
config=None,
parent=None,
image_extractor=None,
data_volume_reducer=None,
**kwargs,
):
"""
Parameters
----------
subarray: ctapipe.instrument.SubarrayDescription
Description of the subarray. Provides information about the
camera which are useful in calibration. Also required for
configuring the TelescopeParameter traitlets.
config: traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
This is mutually exclusive with passing a ``parent``.
parent: ctapipe.core.Component or ctapipe.core.Tool
Parent of this component in the configuration hierarchy,
this is mutually exclusive with passing ``config``
data_volume_reducer: ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use.
This is used to override the options from the config system
and to enable passing a preconfigured reducer.
image_extractor: ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, the default via the
configuration system will be constructed.
"""
super().__init__(subarray=subarray,
config=config,
parent=parent,
**kwargs)
self.subarray = subarray
self._r1_empty_warn = False
self._dl0_empty_warn = False
self.image_extractors = {}
if image_extractor is None:
for (_, _, name) in self.image_extractor_type:
self.image_extractors[name] = ImageExtractor.from_name(
name, subarray=self.subarray, parent=self)
else:
name = image_extractor.__class__.__name__
self.image_extractor_type = [("type", "*", name)]
self.image_extractors[name] = image_extractor
if data_volume_reducer is None:
self.data_volume_reducer = DataVolumeReducer.from_name(
self.data_volume_reducer_type,
subarray=self.subarray,
parent=self)
else:
self.data_volume_reducer = data_volume_reducer
def _check_r1_empty(self, waveforms):
if waveforms is None:
if not self._r1_empty_warn:
warnings.warn("Encountered an event with no R1 data. "
"DL0 is unchanged in this circumstance.")
self._r1_empty_warn = True
return True
else:
return False
def _check_dl0_empty(self, waveforms):
if waveforms is None:
if not self._dl0_empty_warn:
warnings.warn("Encountered an event with no DL0 data. "
"DL1 is unchanged in this circumstance.")
self._dl0_empty_warn = True
return True
else:
return False
def _calibrate_dl0(self, event, telid):
waveforms = event.r1.tel[telid].waveform
selected_gain_channel = event.r1.tel[telid].selected_gain_channel
if self._check_r1_empty(waveforms):
return
reduced_waveforms_mask = self.data_volume_reducer(
waveforms,
telid=telid,
selected_gain_channel=selected_gain_channel)
waveforms_copy = waveforms.copy()
waveforms_copy[~reduced_waveforms_mask] = 0
event.dl0.tel[telid].waveform = waveforms_copy
event.dl0.tel[telid].selected_gain_channel = selected_gain_channel
def _calibrate_dl1(self, event, telid):
waveforms = event.dl0.tel[telid].waveform
selected_gain_channel = event.dl0.tel[telid].selected_gain_channel
dl1_calib = event.calibration.tel[telid].dl1
if self._check_dl0_empty(waveforms):
return
selected_gain_channel = event.r1.tel[telid].selected_gain_channel
time_shift = event.calibration.tel[telid].dl1.time_shift
readout = self.subarray.tel[telid].camera.readout
n_pixels, n_samples = waveforms.shape
# subtract any remaining pedestal before extraction
if dl1_calib.pedestal_offset is not None:
# this copies intentionally, we don't want to modify the dl0 data
# waveforms have shape (n_pixel, n_samples), pedestals (n_pixels, )
waveforms = waveforms - dl1_calib.pedestal_offset[:, np.newaxis]
if n_samples == 1:
# To handle ASTRI and dst
# TODO: Improved handling of ASTRI and dst
# - dst with custom EventSource?
# - Read into dl1 container directly?
# - Don't do anything if dl1 container already filled
# - Update on SST review decision
charge = waveforms[..., 0].astype(np.float32)
peak_time = np.zeros(n_pixels, dtype=np.float32)
else:
# shift waveforms if time_shift calibration is available
if time_shift is not None:
if self.apply_waveform_time_shift.tel[telid]:
sampling_rate = readout.sampling_rate.to_value(u.GHz)
time_shift_samples = time_shift * sampling_rate
waveforms, remaining_shift = shift_waveforms(
waveforms, time_shift_samples)
remaining_shift /= sampling_rate
else:
remaining_shift = time_shift
extractor = self.image_extractors[
self.image_extractor_type.tel[telid]]
charge, peak_time = extractor(
waveforms,
telid=telid,
selected_gain_channel=selected_gain_channel)
# correct non-integer remainder of the shift if given
if self.apply_peak_time_shift.tel[telid] and time_shift is not None:
peak_time -= remaining_shift
# Calibrate extracted charge
charge *= dl1_calib.relative_factor / dl1_calib.absolute_factor
event.dl1.tel[telid].image = charge
event.dl1.tel[telid].peak_time = peak_time
def __call__(self, event):
"""
Perform the full camera calibration from R1 to DL1. Any calibration
relating to data levels before the data level the file is read into
will be skipped.
Parameters
----------
event : container
A `~ctapipe.containers.ArrayEventContainer` event container
"""
# TODO: How to handle different calibrations depending on telid?
tel = event.r1.tel or event.dl0.tel or event.dl1.tel
for telid in tel.keys():
self._calibrate_dl0(event, telid)
self._calibrate_dl1(event, telid)
class TwoPassWindowSum(ImageExtractor):
"""Extractor based on [1]_ which integrates the waveform a second time using
a time-gradient linear fit. This is in particular the version implemented
in the CTA-MARS analysis pipeline [2]_.
Notes
-----
#. slide a 3-samples window through the waveform, finding max counts sum;
the range of the sliding is the one allowing extension from 3 to 5;
add 1 sample on each side and integrate charge in the 5-sample window;
time is obtained as a charge-weighted average of the sample numbers;
No information from neighboouring pixels is used.
#. Preliminary image cleaning via simple tailcut with minimum number
of core neighbours set at 1,
#. Only the brightest cluster of pixels is kept.
#. Parametrize following Hillas approach only if the resulting image has 3
or more pixels.
#. Do a linear fit of pulse time vs. distance along major image axis
(CTA-MARS uses ROOT "robust" fit option,
aka Least Trimmed Squares, to get rid of far outliers - this should
be implemented in 'timing_parameters', e.g scipy.stats.siegelslopes).
#. For all pixels except the core ones in the main island, integrate
the waveform once more, in a fixed window of 5 samples set at the time
"predicted" by the linear time fit.
If the predicted time for a pixel leads to a window outside the readout
window, then integrate the last (or first) 5 samples.
#. The result is an image with main-island core pixels calibrated with a
1st pass and non-core pixels re-calibrated with a 2nd pass.
References
----------
.. [1] J. Holder et al., Astroparticle Physics, 25, 6, 391 (2006)
.. [2] https://forge.in2p3.fr/projects/step-by-step-reference-mars-analysis/wiki
"""
# Get thresholds for core-pixels depending on telescope type.
# WARNING: default values are not yet optimized
core_threshold = FloatTelescopeParameter(
default_value=[
("type", "*", 6.0),
("type", "LST*", 6.0),
("type", "MST*", 8.0),
("type", "SST*", 4.0),
],
help="Picture threshold for internal tail-cuts pass",
).tag(config=True)
disable_second_pass = Bool(
default_value=False,
help="only run the first pass of the extractor, for debugging purposes",
).tag(config=True)
apply_integration_correction = BoolTelescopeParameter(
default_value=True,
help="Apply the integration window correction").tag(config=True)
@lru_cache(maxsize=4096)
def _calculate_correction(self, telid, width, shift):
"""Obtain the correction for the integration window specified for each
pixel.
The TwoPassWindowSum image extractor applies potentially different
parameters for the integration window to each pixel, depending on the
position of the peak. It has been decided to apply gain selection
directly here. For basic definitions look at the documentation of
`integration_correction`.
Parameters
----------
telid : int
Index of the telescope in use.
width : int
Width of the integration window in samples
shift : int
Window shift to the left of the pulse peak in samples
Returns
-------
correction : ndarray
Value of the pixel-wise gain-selected integration correction.
"""
readout = self.subarray.tel[telid].camera.readout
# Calculate correction of first pixel for both channels
return integration_correction(
readout.reference_pulse_shape,
readout.reference_pulse_sample_width.to_value("ns"),
(1 / readout.sampling_rate).to_value("ns"),
width,
shift,
)
def _apply_first_pass(self, waveforms,
telid) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
"""
Execute step 1.
Parameters
----------
waveforms : array of size (N_pixels, N_samples)
DL0-level waveforms of one event.
telid : int
Index of the telescope.
Returns
-------
charge : array_like
Integrated charge per pixel.
Shape: (n_pix)
pulse_time : array_like
Samples in which the waveform peak has been recognized.
Shape: (n_pix)
correction : ndarray
pixel-wise integration correction
"""
# STEP 1
# Starting from DL0, the channel is already selected (if more than one)
# event.dl0.tel[tel_id].waveform object has shape (N_pixels, N_samples)
# For each pixel, we slide a 3-samples window through the
# waveform summing each time the ADC counts contained within it.
peak_search_window_width = 3
sums = convolve1d(waveforms,
np.ones(peak_search_window_width),
axis=1,
mode="nearest")
# 'sums' has now still shape of (N_pixels, N_samples)
# each element is the center-sample of each 3-samples sliding window
# For each pixel, we check where the peak search window encountered
# the maximum number of ADC counts.
# We want to stop before the edge of the readout window in order to
# later extend the search window to a 1+3+1 integration window.
# Since in 'sums' the peak index corresponds to the center of the
# search window, we shift it on the right by 2 samples so to get the
# correspondent sample index in each waveform.
peak_index = np.argmax(sums[:, 2:-2], axis=1) + 2
# Now peak_index has the shape of (N_pixels).
# The final 5-samples window will be 1+3+1, centered on the 3-samples
# window in which the highest amount of ADC counts has been found
window_width = peak_search_window_width + 2
window_shift = 2
# this function is applied to all pixels together
charge_1stpass, pulse_time_1stpass = extract_around_peak(
waveforms,
peak_index,
window_width,
window_shift,
self.sampling_rate_ghz[telid],
)
# Get integration correction factors
if self.apply_integration_correction.tel[telid]:
correction = self._calculate_correction(telid, window_width,
window_shift)
else:
correction = np.ones(waveforms.shape[0])
return charge_1stpass, pulse_time_1stpass, correction
def _apply_second_pass(
self,
waveforms,
telid,
selected_gain_channel,
charge_1stpass_uncorrected,
pulse_time_1stpass,
correction,
) -> Tuple[np.ndarray, np.ndarray]:
"""
Follow steps from 2 to 7.
Parameters
----------
waveforms : array of shape (N_pixels, N_samples)
DL0-level waveforms of one event.
telid : int
Index of the telescope.
selected_gain_channel: array of shape (N_channels, N_pixels)
Array containing the index of the selected gain channel for each
pixel (0 for low gain, 1 for high gain).
charge_1stpass_uncorrected : array of shape N_pixels
Pixel charges reconstructed with the 1st pass, but not corrected.
pulse_time_1stpass : array of shape N_pixels
Pixel-wise pulse times reconstructed with the 1st pass.
correction: array of shape N_pixels
Charge correction from 1st pass.
Returns
-------
charge : array_like
Integrated charge per pixel.
Note that in the case of a very bright full-camera image this can
coincide the 1st pass information.
Also in the case of very dim images the 1st pass will be recycled,
but in this case the resulting image should be discarded
from further analysis.
Shape: (n_pix)
pulse_time : array_like
Samples in which the waveform peak has been recognized.
Same specifications as above.
Shape: (n_pix)
is_valid: bool
True=second-pass succeeded, False=second-pass failed, first pass used
"""
# STEP 2
# Apply correction to 1st pass charges
charge_1stpass = charge_1stpass_uncorrected * correction[
selected_gain_channel]
# Set thresholds for core-pixels depending on telescope
core_th = self.core_threshold.tel[telid]
# Boundary thresholds will be half of core thresholds.
# Preliminary image cleaning with simple two-level tail-cut
camera_geometry = self.subarray.tel[telid].camera.geometry
mask_clean = tailcuts_clean(
camera_geometry,
charge_1stpass,
picture_thresh=core_th,
boundary_thresh=core_th / 2,
keep_isolated_pixels=False,
min_number_picture_neighbors=1,
)
# STEP 3
# find all islands using this cleaning
num_islands, labels = number_of_islands(camera_geometry, mask_clean)
if num_islands > 0:
# ...find the brightest one
mask_brightest_island = brightest_island(num_islands, labels,
charge_1stpass)
else:
mask_brightest_island = mask_clean
# for all pixels except the core ones in the main island of the
# preliminary image, the waveform will be integrated once more (2nd pass)
mask_2nd_pass = ~mask_brightest_island | (mask_brightest_island &
(charge_1stpass < core_th))
# STEP 4
# if the resulting image has less then 3 pixels
if np.count_nonzero(mask_brightest_island) < 3:
# we return the 1st pass information
return charge_1stpass, pulse_time_1stpass, False
# otherwise we proceed by parametrizing the image
camera_geometry_brightest = camera_geometry[mask_brightest_island]
charge_brightest = charge_1stpass[mask_brightest_island]
hillas = hillas_parameters(camera_geometry_brightest, charge_brightest)
# STEP 5
# linear fit of pulse time vs. distance along major image axis
# using only the main island surviving the preliminary
# image cleaning
timing = timing_parameters(
geom=camera_geometry_brightest,
image=charge_brightest,
peak_time=pulse_time_1stpass[mask_brightest_island],
hillas_parameters=hillas,
)
# If the fit returns nan
if np.isnan(timing.slope):
return charge_1stpass, pulse_time_1stpass, False
# get projected distances along main image axis
longitude, _ = camera_to_shower_coordinates(camera_geometry.pix_x,
camera_geometry.pix_y,
hillas.x, hillas.y,
hillas.psi)
# get the predicted times as a linear relation
predicted_pulse_times = (timing.slope * longitude[mask_2nd_pass] +
timing.intercept)
# Convert time in ns to sample index using the sampling rate from
# the readout.
# Approximate the value obtained to nearest integer, then cast to
# int64 otherwise 'extract_around_peak' complains.
predicted_peaks = np.rint(predicted_pulse_times.value *
self.sampling_rate_ghz[telid]).astype(
np.int64)
# Due to the fit these peak indexes can lead to an integration window
# outside the readout window, so in the next step we check for this.
# STEP 6
# select all pixels except the core ones in the main island
waveforms_to_repass = waveforms[mask_2nd_pass]
# Build 'width' and 'shift' arrays that adapt on the position of the
# window along each waveform
# As before we will integrate the charge in a 5-sample window centered
# on the peak
window_width_default = 5
window_shift_default = 2
# first we find where the integration window edges WOULD BE
integration_windows_start = predicted_peaks - window_shift_default
integration_windows_end = integration_windows_start + window_width_default
# then we define 2 possible edge cases
# the predicted integration window falls before the readout window
integration_before_readout = integration_windows_start < 0
# or after
integration_after_readout = integration_windows_end > (
waveforms_to_repass.shape[1] - 1)
# If the resulting 5-samples window falls before the readout
# window we take the first 5 samples
window_width_before = 5
window_shift_before = 0
# If the resulting 5-samples window falls after the readout
# window we take the last 5 samples
window_width_after = 6
window_shift_after = 4
# put all values of widths and shifts for 2nd pass pixels together
window_widths = np.full(waveforms_to_repass.shape[0],
window_width_default)
window_widths[integration_before_readout] = window_width_before
window_widths[integration_after_readout] = window_width_after
window_shifts = np.full(waveforms_to_repass.shape[0],
window_shift_default)
window_shifts[integration_before_readout] = window_shift_before
window_shifts[integration_after_readout] = window_shift_after
# Now we have to (re)define the pathological predicted times for which
# - either the peak itself falls outside of the readout window
# - or is within the first or last 2 samples (so that at least 1 sample
# of the integration window is outside of the readout window)
# We place them at the first or last sample, so the special window
# widhts and shifts that we defined earlier put the integration window
# for these 2 cases either in the first 5 samples or the last
# set sample to 0 (beginning of the waveform)
# if predicted time falls before
# but also if it's so near the edge that the integration window falls
# outside
predicted_peaks[predicted_peaks < 2] = 0
# set sample to max-1 (first sample has index 0)
# if predicted time falls after
predicted_peaks[predicted_peaks > (waveforms_to_repass.shape[1] -
3)] = (
waveforms_to_repass.shape[1] -
1)
# re-calibrate 2nd pass pixels using the fixed 5-samples window
reintegrated_charge, reestimated_pulse_times = extract_around_peak(
waveforms_to_repass,
predicted_peaks,
window_widths,
window_shifts,
self.sampling_rate_ghz[telid],
)
if self.apply_integration_correction.tel[telid]:
# Modify integration correction factors only for non-core pixels
# now we compute 3 corrections for the default, before, and after cases:
correction = self._calculate_correction(
telid, window_width_default,
window_shift_default)[selected_gain_channel][mask_2nd_pass]
correction_before = self._calculate_correction(
telid, window_width_before,
window_shift_before)[selected_gain_channel][mask_2nd_pass]
correction_after = self._calculate_correction(
telid, window_width_after,
window_shift_after)[selected_gain_channel][mask_2nd_pass]
correction[integration_before_readout] = correction_before[
integration_before_readout]
correction[integration_after_readout] = correction_after[
integration_after_readout]
reintegrated_charge *= correction
# STEP 7
# Combine in the final output with,
# - core pixels from the main cluster
# - rest of the pixels which have been passed a second time
# Start from a copy of the 1st pass charge
charge_2ndpass = charge_1stpass.copy()
# Overwrite the charges of pixels marked for second pass
# leaving untouched the core pixels of the main island
# from the preliminary (cleaned) image
charge_2ndpass[mask_2nd_pass] = reintegrated_charge
# Same approach for the pulse times
pulse_time_2ndpass = pulse_time_1stpass.copy()
pulse_time_2ndpass[mask_2nd_pass] = reestimated_pulse_times
return charge_2ndpass, pulse_time_2ndpass, True
def __call__(self, waveforms, telid, selected_gain_channel):
charge1, pulse_time1, correction1 = self._apply_first_pass(
waveforms, telid)
# FIXME: properly make sure that output is 32Bit instead of downcasting here
if self.disable_second_pass:
return DL1CameraContainer(
image=(charge1 *
correction1[selected_gain_channel]).astype("float32"),
peak_time=pulse_time1.astype("float32"),
is_valid=True,
)
charge2, pulse_time2, is_valid = self._apply_second_pass(
waveforms, telid, selected_gain_channel, charge1, pulse_time1,
correction1)
# FIXME: properly make sure that output is 32Bit instead of downcasting here
return DL1CameraContainer(
image=charge2.astype("float32"),
peak_time=pulse_time2.astype("float32"),
is_valid=is_valid,
)
class GlobalPeakWindowSum(ImageExtractor):
"""
Extractor which sums in a window about the
peak from the global average waveform.
To reduce the influence of noise pixels, the average can be calculated
only on the ``pixel_fraction`` brightest pixels.
The "brightest" pixels are determined by sorting the waveforms by their
maximum value.
"""
window_width = IntTelescopeParameter(
default_value=7,
help="Define the width of the integration window").tag(config=True)
window_shift = IntTelescopeParameter(
default_value=3,
help="Define the shift of the integration window from the peak_index "
"(peak_index - shift)",
).tag(config=True)
apply_integration_correction = BoolTelescopeParameter(
default_value=True,
help="Apply the integration window correction").tag(config=True)
pixel_fraction = FloatTelescopeParameter(
default_value=1.0,
help=
("Fraction of pixels to use for finding the integration window."
" By default, the full camera is used."
" If fraction is smaller 1, only the brightest pixels will be averaged"
" to find the peak position"),
).tag(config=True)
@lru_cache(maxsize=128)
def _calculate_correction(self, telid):
"""
Calculate the correction for the extracted change such that the value
returned would equal 1 for a noise-less unit pulse.
This method is decorated with @lru_cache to ensure it is only
calculated once per telescope.
Parameters
----------
telid : int
Returns
-------
correction : ndarray
The correction to apply to an extracted charge using this ImageExtractor
Has size n_channels, as a different correction value might be required
for different gain channels.
"""
readout = self.subarray.tel[telid].camera.readout
return integration_correction(
readout.reference_pulse_shape,
readout.reference_pulse_sample_width.to_value("ns"),
(1 / readout.sampling_rate).to_value("ns"),
self.window_width.tel[telid],
self.window_shift.tel[telid],
)
def __call__(self, waveforms, telid, selected_gain_channel):
if self.pixel_fraction.tel[telid] == 1.0:
# average over pixels then argmax over samples
peak_index = waveforms.mean(axis=-2).argmax()
else:
n_pixels = int(self.pixel_fraction.tel[telid] *
waveforms.shape[-2])
brightest = np.argsort(waveforms.max(axis=-1))[..., -n_pixels:]
# average over brightest pixels then argmax over samples
peak_index = waveforms[brightest].mean(axis=-2).argmax()
charge, peak_time = extract_around_peak(
waveforms,
peak_index,
self.window_width.tel[telid],
self.window_shift.tel[telid],
self.sampling_rate_ghz[telid],
)
if self.apply_integration_correction.tel[telid]:
charge *= self._calculate_correction(
telid=telid)[selected_gain_channel]
return DL1CameraContainer(image=charge,
peak_time=peak_time,
is_valid=True)
class FixedWindowSum(ImageExtractor):
"""
Extractor that sums within a fixed window defined by the user.
"""
peak_index = IntTelescopeParameter(
default_value=0,
help="Manually select index where the peak is located").tag(
config=True)
window_width = IntTelescopeParameter(
default_value=7,
help="Define the width of the integration window").tag(config=True)
window_shift = IntTelescopeParameter(
default_value=0,
help="Define the shift of the integration window from the peak_index "
"(peak_index - shift)",
).tag(config=True)
apply_integration_correction = BoolTelescopeParameter(
default_value=True,
help="Apply the integration window correction").tag(config=True)
@lru_cache(maxsize=128)
def _calculate_correction(self, telid):
"""
Calculate the correction for the extracted change such that the value
returned would equal 1 for a noise-less unit pulse.
This method is decorated with @lru_cache to ensure it is only
calculated once per telescope.
Parameters
----------
telid : int
Returns
-------
correction : ndarray
The correction to apply to an extracted charge using this ImageExtractor
Has size n_channels, as a different correction value might be required
for different gain channels.
"""
readout = self.subarray.tel[telid].camera.readout
return integration_correction(
readout.reference_pulse_shape,
readout.reference_pulse_sample_width.to_value("ns"),
(1 / readout.sampling_rate).to_value("ns"),
self.window_width.tel[telid],
self.window_shift.tel[telid],
)
def __call__(self, waveforms, telid, selected_gain_channel):
charge, peak_time = extract_around_peak(
waveforms,
self.peak_index.tel[telid],
self.window_width.tel[telid],
self.window_shift.tel[telid],
self.sampling_rate_ghz[telid],
)
if self.apply_integration_correction.tel[telid]:
charge *= self._calculate_correction(
telid=telid)[selected_gain_channel]
return DL1CameraContainer(image=charge,
peak_time=peak_time,
is_valid=True)