def __init__(self, subarray, **kwargs):
"""
Parameters
----------
reducer_product : ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use. If None, then
NullDataVolumeReducer will be used by default, and waveforms
will not be reduced.
extractor_product : ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, then LocalPeakWindowSum
will be used by default.
calibration_path :
Path to LST calibration file to get the pedestal and flat-field corrections
kwargs
"""
super().__init__(subarray, **kwargs)
# load the waveform charge extractor
self.image_extractor = ImageExtractor.from_name(self.extractor_product,
subarray=self.subarray,
config=self.config)
self.log.info(f"extractor {self.extractor_product}")
print("EXTRACTOR", self.image_extractor)
self.data_volume_reducer = DataVolumeReducer.from_name(
self.reducer_product, subarray=self.subarray, config=self.config)
self.log.info(f" {self.reducer_product}")
# declare gain selector if the threshold is defined
if self.gain_threshold:
self.gain_selector = gainselection.ThresholdGainSelector(
threshold=self.gain_threshold)
# declare time calibrator if correction file exist
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
raise IOError(
f"Time calibration file {self.time_calibration_path} not found!"
)
# calibration data container
self.mon_data = MonitoringContainer()
# initialize the MonitoringContainer() for the moment it reads it from a hdf5 file
self._initialize_correction()
self.log.info(f"Global charge scale {self.charge_scale}")
def __init__(self, subarray, **kwargs):
"""Calculates flat-field parameters from flasher data
based on the best algorithm described by S. Fegan in MST-CAM-TN-0060 (eq. 19)
Pixels are defined as outliers on the base of a cut on the pixel charge median
over the full sample distribution and the pixel signal time inside the
waveform time
Parameters:
----------
charge_cut_outliers : List[2]
Interval of accepted charge values (fraction with respect to camera median value)
time_cut_outliers : List[2]
Interval (in waveform samples) of accepted time values
"""
super().__init__(subarray, **kwargs)
self.log.info("Used events statistics : %d", self.sample_size)
# members to keep state in calculate_relative_gain()
self.num_events_seen = 0
self.time_start = None # trigger time of first event in sample
self.trigger_time = None # trigger time of present event
self.charge_medians = None # med. charge in camera per event in sample
self.charges = None # charge per event in sample
self.arrival_times = None # arrival time per event in sample
self.sample_masked_pixels = None # masked pixels per event in sample
if self.time_calibration_path is None:
self.time_corrector = None
else:
# look for calibration path otherwise
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
msg = f"Time calibration file {self.time_calibration_path} not found!"
raise IOError(msg)
class LSTCameraCalibrator(CameraCalibrator):
"""
Calibrator to handle the LST camera calibration chain, in order to fill
the DL1 data level in the event container.
"""
extractor_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
reducer_product = Unicode(
'NullDataVolumeReducer',
help='Name of the DataVolumeReducer to use').tag(config=True)
calibration_path = Path(
exists=True, directory_ok=False,
help='Path to LST calibration file').tag(config=True)
time_calibration_path = Path(
exists=True,
directory_ok=False,
help='Path to drs4 time calibration file').tag(config=True)
time_sampling_correction_path = Path(
exists=True,
directory_ok=False,
help='Path to time sampling correction file',
allow_none=True,
).tag(config=True)
allowed_tels = List(
[1], help='List of telescope to be calibrated').tag(config=True)
gain_threshold = Int(
4094, allow_none=True,
help='Threshold for the gain selection in ADC').tag(config=True)
charge_scale = List(
[1, 1],
help='Multiplicative correction factor for charge estimation [HG,LG]'
).tag(config=True)
def __init__(self, subarray, **kwargs):
"""
Parameters
----------
reducer_product : ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use. If None, then
NullDataVolumeReducer will be used by default, and waveforms
will not be reduced.
extractor_product : ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, then LocalPeakWindowSum
will be used by default.
calibration_path :
Path to LST calibration file to get the pedestal and flat-field corrections
kwargs
"""
super().__init__(subarray, **kwargs)
# load the waveform charge extractor
self.image_extractor = ImageExtractor.from_name(self.extractor_product,
subarray=self.subarray,
config=self.config)
self.log.info(f"extractor {self.extractor_product}")
print("EXTRACTOR", self.image_extractor)
self.data_volume_reducer = DataVolumeReducer.from_name(
self.reducer_product, subarray=self.subarray, config=self.config)
self.log.info(f" {self.reducer_product}")
# declare gain selector if the threshold is defined
if self.gain_threshold:
self.gain_selector = gainselection.ThresholdGainSelector(
threshold=self.gain_threshold)
# declare time calibrator if correction file exist
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
raise IOError(
f"Time calibration file {self.time_calibration_path} not found!"
)
# declare the charge sampling corrector
if self.time_sampling_correction_path is not None:
# search the file in resources if not found
if not os.path.exists(self.time_sampling_correction_path):
self.time_sampling_correction_path = resource_filename(
'lstchain',
f"resources/{self.time_sampling_correction_path}")
if os.path.exists(self.time_sampling_correction_path):
self.time_sampling_corrector = TimeSamplingCorrection(
time_sampling_correction_path=self.
time_sampling_correction_path)
else:
raise IOError(
f"Sampling correction file {self.time_sampling_correction_path} not found!"
)
else:
self.time_sampling_corrector = None
# calibration data container
self.mon_data = MonitoringContainer()
# initialize the MonitoringContainer() for the moment it reads it from a hdf5 file
self._initialize_correction()
self.log.info(f"Global charge scale {self.charge_scale}")
def _initialize_correction(self):
"""
Read the correction from hdf5 calibration file
"""
self.log.info(f"read {self.calibration_path}")
try:
with HDF5TableReader(self.calibration_path) as h5_table:
for telid in self.allowed_tels:
# read the calibration data
table = '/tel_' + str(telid) + '/calibration'
next(
h5_table.read(table,
self.mon_data.tel[telid].calibration))
# read pedestal data
table = '/tel_' + str(telid) + '/pedestal'
next(
h5_table.read(table,
self.mon_data.tel[telid].pedestal))
# read flat-field data
table = '/tel_' + str(telid) + '/flatfield'
next(
h5_table.read(table,
self.mon_data.tel[telid].flatfield))
# read the pixel_status container
table = '/tel_' + str(telid) + '/pixel_status'
next(
h5_table.read(table,
self.mon_data.tel[telid].pixel_status))
except Exception:
self.log.exception(
f"Problem in reading calibration file {self.calibration_path}")
raise
def _calibrate_dl0(self, event, telid):
"""
create dl0 level, with gain-selected and calibrated waveform
"""
waveforms = event.r1.tel[telid].waveform
if self._check_r1_empty(waveforms):
return
# if not already done, initialize the event monitoring containers
if event.mon.tel[telid].calibration.dc_to_pe is None:
event.mon.tel[telid].calibration = self.mon_data.tel[
telid].calibration
event.mon.tel[telid].flatfield = self.mon_data.tel[telid].flatfield
event.mon.tel[telid].pedestal = self.mon_data.tel[telid].pedestal
event.mon.tel[telid].pixel_status = self.mon_data.tel[
telid].pixel_status
#
# subtract the pedestal per sample and multiply for the calibration coefficients
#
calibrated_waveform = (
(waveforms - self.mon_data.tel[telid].calibration.
pedestal_per_sample[:, :, np.newaxis]) *
self.mon_data.tel[telid].calibration.dc_to_pe[:, :,
np.newaxis]).astype(
np.float32)
# If requested, perform gain selection (this will be done by the EvB in future)
# find the gain selection mask
if waveforms.ndim == 3:
# if threshold defined, perform gain selection
if self.gain_threshold:
gain_mask = self.gain_selector(waveforms)
# select the samples
calibrated_waveform = calibrated_waveform[
gain_mask, np.arange(waveforms.shape[1])]
else:
# keep both HG and LG
gain_mask = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype=np.int64)
gain_mask[1] = 1
else:
# gain selection already performed in EvB: (0=HG, 1=LG)
gain_mask = event.lst.tel[telid].evt.pixel_status >> 2 & 1
# remember the calibrated and gain selected waveform
# (this should be the r1 waveform to be compliant with ctapipe (?))
event.dl0.tel[telid].waveform = calibrated_waveform
# remember which channel has been selected
event.r1.tel[telid].selected_gain_channel = gain_mask
event.dl0.tel[telid].selected_gain_channel = gain_mask
def _calibrate_dl1(self, event, telid):
"""
create calibrated dl1 image and calibrate it
"""
n_pixels = self.subarray.tels[telid].camera.geometry.n_pixels
# copy the waveform be cause I do not want to change it
waveforms = np.copy(event.dl0.tel[telid].waveform)
gain_mask = event.dl0.tel[telid].selected_gain_channel
if self._check_dl0_empty(waveforms):
return
# In case of no gain selection the selected gain channels are [0,0,..][1,1,..]
no_gain_selection = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype=np.int64)
no_gain_selection[1] = 1
# correct the dl0 waveform for the sampling time corrections
if self.time_sampling_corrector:
waveforms *= self.time_sampling_corrector.get_corrections(
event, telid)[gain_mask, np.arange(n_pixels)]
# extract the charge
charge, peak_time = self.image_extractor(waveforms, telid, gain_mask)
# correct charge for global scale
corrected_charge = charge * np.array(self.charge_scale,
dtype=np.float32)[gain_mask]
# correct time with drs4 correction if available
if self.time_corrector:
peak_time_drs4_corrected = (
peak_time - self.time_corrector.get_pulse_time_corrections(
event)[gain_mask, np.arange(n_pixels)])
# add flat-fielding time correction
peak_time_ff_corrected = (
peak_time_drs4_corrected +
self.mon_data.tel[telid].calibration.time_correction.value[
gain_mask, np.arange(n_pixels)])
# fill dl1 container
event.dl1.tel[telid].image = corrected_charge
event.dl1.tel[telid].peak_time = peak_time_ff_corrected.astype(
np.float32)
class FlasherFlatFieldCalculator(FlatFieldCalculator):
"""Calculates flat-field parameters from flasher data
based on the best algorithm described by S. Fegan in MST-CAM-TN-0060 (eq. 19)
Pixels are defined as outliers on the base of a cut on the pixel charge median
over the full sample distribution and the pixel signal time inside the
waveform time
Parameters:
----------
charge_cut_outliers : List[2]
Interval of accepted charge values (fraction with respect to camera median value)
time_cut_outliers : List[2]
Interval (in waveform samples) of accepted time values
"""
charge_median_cut_outliers = List(
[-0.3, 0.3],
help=
'Interval of accepted charge values (fraction with respect to camera median value)'
).tag(config=True)
time_cut_outliers = List(
[0, 60],
help='Interval (in waveform samples) of accepted time values').tag(
config=True)
charge_std_cut_outliers = List(
[-3, 3],
help=
'Interval (number of std) of accepted charge standard deviation around camera median value'
).tag(config=True)
time_calibration_path = Unicode(
None, allow_none=True,
help='Path to drs4 time calibration file').tag(config=True)
def __init__(self, subarray, **kwargs):
"""Calculates flat-field parameters from flasher data
based on the best algorithm described by S. Fegan in MST-CAM-TN-0060 (eq. 19)
Pixels are defined as outliers on the base of a cut on the pixel charge median
over the full sample distribution and the pixel signal time inside the
waveform time
Parameters:
----------
charge_cut_outliers : List[2]
Interval of accepted charge values (fraction with respect to camera median value)
time_cut_outliers : List[2]
Interval (in waveform samples) of accepted time values
"""
super().__init__(subarray, **kwargs)
self.log.info("Used events statistics : %d", self.sample_size)
# members to keep state in calculate_relative_gain()
self.num_events_seen = 0
self.time_start = None # trigger time of first event in sample
self.trigger_time = None # trigger time of present event
self.charge_medians = None # med. charge in camera per event in sample
self.charges = None # charge per event in sample
self.arrival_times = None # arrival time per event in sample
self.sample_masked_pixels = None # masked pixels per event in sample
if self.time_calibration_path is None:
self.time_corrector = None
else:
# look for calibration path otherwise
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
msg = f"Time calibration file {self.time_calibration_path} not found!"
raise IOError(msg)
def _extract_charge(self, event):
"""
Extract the charge and the time from a calibration event
Parameters
----------
event : general event container
"""
waveforms = event.r1.tel[self.tel_id].waveform
no_gain_selection = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype=np.int)
# Extract charge and time
charge = 0
peak_pos = 0
if self.extractor:
charge, peak_pos = self.extractor(waveforms, self.tel_id,
no_gain_selection)
# correct time with drs4 correction if available
if self.time_corrector:
peak_pos = self.time_corrector.get_corr_pulse(event, peak_pos)
return charge, peak_pos
def calculate_relative_gain(self, event):
"""
calculate the flatfield statistical values
and fill mon.tel[tel_id].flatfield container
Parameters
----------
event : general event container
Returns: True if the mon.tel[tel_id].flatfield is updated, False otherwise
"""
# initialize the np array at each cycle
waveform = event.r1.tel[self.tel_id].waveform
# re-initialize counter
if self.num_events_seen == self.sample_size:
self.num_events_seen = 0
pixel_mask = np.logical_or(
event.mon.tel[self.tel_id].pixel_status.hardware_failing_pixels,
event.mon.tel[self.tel_id].pixel_status.flatfield_failing_pixels)
# real data
if event.meta['origin'] != 'hessio':
self.trigger_time = event.r1.tel[self.tel_id].trigger_time
else: # patches for MC data
if event.trig.tels_with_trigger:
self.trigger_time = event.trig.gps_time.unix
else:
self.trigger_time = 0
if self.num_events_seen == 0:
self.time_start = self.trigger_time
self.setup_sample_buffers(waveform, self.sample_size)
# extract the charge of the event and
# the peak position (assumed as time for the moment)
charge, arrival_time = self._extract_charge(event)
# correct pulse time with drs4 corrections
self.collect_sample(charge, pixel_mask, arrival_time)
sample_age = self.trigger_time - self.time_start
# check if to create a calibration event
if (self.num_events_seen > 0
and (sample_age > self.sample_duration
or self.num_events_seen == self.sample_size)):
# update the monitoring container
self.store_results(event)
return True
else:
return False
def store_results(self, event):
"""
Store stastical results in monitoring container
Parameters
----------
event : general event container
"""
if self.num_events_seen == 0:
raise ValueError(
"No flat-field events in statistics, zero results")
container = event.mon.tel[self.tel_id].flatfield
# mask the part of the array not filled
self.sample_masked_pixels[self.num_events_seen:] = 1
relative_gain_results = self.calculate_relative_gain_results(
self.charge_medians, self.charges, self.sample_masked_pixels)
time_results = self.calculate_time_results(self.arrival_times,
self.sample_masked_pixels,
self.time_start,
self.trigger_time)
result = {
'n_events': self.num_events_seen,
**relative_gain_results,
**time_results,
}
for key, value in result.items():
setattr(container, key, value)
# update the flatfield mask
ff_charge_failing_pixels = np.logical_or(
container.charge_median_outliers, container.charge_std_outliers)
event.mon.tel[self.tel_id].pixel_status.flatfield_failing_pixels = \
np.logical_or(ff_charge_failing_pixels, container.time_median_outliers)
def setup_sample_buffers(self, waveform, sample_size):
"""Initialize sample buffers"""
n_channels = waveform.shape[0]
n_pix = waveform.shape[1]
shape = (sample_size, n_channels, n_pix)
self.charge_medians = np.zeros((sample_size, n_channels))
self.charges = np.zeros(shape)
self.arrival_times = np.zeros(shape)
self.sample_masked_pixels = np.zeros(shape)
def collect_sample(self, charge, pixel_mask, arrival_time):
"""Collect the sample data"""
# extract the charge of the event and
# the peak position (assumed as time for the moment)
good_charge = np.ma.array(charge, mask=pixel_mask)
charge_median = np.ma.median(good_charge, axis=1)
self.charges[self.num_events_seen] = charge
self.arrival_times[self.num_events_seen] = arrival_time
self.sample_masked_pixels[self.num_events_seen] = pixel_mask
self.charge_medians[self.num_events_seen] = charge_median
self.num_events_seen += 1
def calculate_time_results(
self,
trace_time,
masked_pixels_of_sample,
time_start,
trigger_time,
):
"""Calculate and return the time results """
masked_trace_time = np.ma.array(trace_time,
mask=masked_pixels_of_sample)
# median over the sample per pixel
pixel_median = np.ma.median(masked_trace_time, axis=0)
# mean over the sample per pixel
pixel_mean = np.ma.mean(masked_trace_time, axis=0)
# std over the sample per pixel
pixel_std = np.ma.std(masked_trace_time, axis=0)
# median of the median over the camera
median_of_pixel_median = np.ma.median(pixel_median, axis=1)
# time outliers from median
relative_median = pixel_median - median_of_pixel_median[:, np.newaxis]
time_median_outliers = np.logical_or(
pixel_median < self.time_cut_outliers[0],
pixel_median > self.time_cut_outliers[1])
return {
'sample_time': (trigger_time - time_start) / 2 * u.s,
'sample_time_min': time_start * u.s,
'sample_time_max': trigger_time * u.s,
'time_mean': np.ma.getdata(pixel_mean) * u.ns,
'time_median': np.ma.getdata(pixel_median) * u.ns,
'time_std': np.ma.getdata(pixel_std) * u.ns,
'relative_time_median': np.ma.getdata(relative_median) * u.ns,
'time_median_outliers': np.ma.getdata(time_median_outliers),
}
def calculate_relative_gain_results(
self,
event_median,
trace_integral,
masked_pixels_of_sample,
):
"""Calculate and return the sample statistics"""
masked_trace_integral = np.ma.array(trace_integral,
mask=masked_pixels_of_sample)
# median over the sample per pixel
pixel_median = np.ma.median(masked_trace_integral, axis=0)
# mean over the sample per pixel
pixel_mean = np.ma.mean(masked_trace_integral, axis=0)
# std over the sample per pixel
pixel_std = np.ma.std(masked_trace_integral, axis=0)
# median of the median over the camera
median_of_pixel_median = np.ma.median(pixel_median, axis=1)
# median of the std over the camera
median_of_pixel_std = np.ma.median(pixel_std, axis=1)
# std of the std over camera
std_of_pixel_std = np.ma.std(pixel_std, axis=1)
# relative gain
relative_gain_event = masked_trace_integral / event_median[:, :,
np.newaxis]
# outliers from median
charge_deviation = pixel_median - median_of_pixel_median[:, np.newaxis]
charge_median_outliers = (np.logical_or(
charge_deviation < self.charge_median_cut_outliers[0] *
median_of_pixel_median[:, np.newaxis],
charge_deviation > self.charge_median_cut_outliers[1] *
median_of_pixel_median[:, np.newaxis]))
# outliers from standard deviation
deviation = pixel_std - median_of_pixel_std[:, np.newaxis]
charge_std_outliers = (np.logical_or(
deviation <
self.charge_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis],
deviation >
self.charge_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis]))
return {
'relative_gain_median':
np.ma.getdata(np.ma.median(relative_gain_event, axis=0)),
'relative_gain_mean':
np.ma.getdata(np.ma.mean(relative_gain_event, axis=0)),
'relative_gain_std':
np.ma.getdata(np.ma.std(relative_gain_event, axis=0)),
'charge_median':
np.ma.getdata(pixel_median),
'charge_mean':
np.ma.getdata(pixel_mean),
'charge_std':
np.ma.getdata(pixel_std),
'charge_std_outliers':
np.ma.getdata(charge_std_outliers),
'charge_median_outliers':
np.ma.getdata(charge_median_outliers),
}
class LSTCameraCalibrator(CameraCalibrator):
"""
Calibrator to handle the LST camera calibration chain, in order to fill
the DL1 data level in the event container.
"""
extractor_product = Unicode(
'NeighborPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
reducer_product = Unicode(
'NullDataVolumeReducer',
help='Name of the DataVolumeReducer to use').tag(config=True)
calibration_path = Unicode(
'', allow_none=True,
help='Path to LST calibration file').tag(config=True)
time_calibration_path = Unicode(
'', allow_none=True,
help='Path to drs4 time calibration file').tag(config=True)
allowed_tels = List(
[1], help='List of telescope to be calibrated').tag(config=True)
gain_threshold = Int(
4094, allow_none=True,
help='Threshold for the gain selection in ADC').tag(config=True)
def __init__(self, **kwargs):
"""
Parameters
----------
reducer_product : ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use. If None, then
NullDataVolumeReducer will be used by default, and waveforms
will not be reduced.
extractor_product : ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, then NeighborPeakWindowSum
will be used by default.
calibration_path :
Path to LST calibration file to get the pedestal and flat-field corrections
kwargs
"""
super().__init__(**kwargs)
# load the waveform charge extractor
self.image_extractor = ImageExtractor.from_name(self.extractor_product,
config=self.config)
self.log.info(f"extractor {self.extractor_product}")
self.data_volume_reducer = DataVolumeReducer.from_name(
self.reducer_product, config=self.config)
self.log.info(f" {self.reducer_product}")
# declare gain selector if the threshold is defined
if self.gain_threshold:
self.gain_selector = gainselection.ThresholdGainSelector(
threshold=self.gain_threshold)
# declare time calibrator if correction file exist
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
self.time_corrector = None
self.log.info(
f"File {self.time_calibration_path} not found. No drs4 time corrections"
)
# calibration data container
self.mon_data = MonitoringContainer()
# initialize the MonitoringContainer() for the moment it reads it from a hdf5 file
self._initialize_correction()
def _initialize_correction(self):
"""
Read the correction from hdf5 calibration file
"""
self.mon_data.tels_with_data = self.allowed_tels
self.log.info(f"read {self.calibration_path}")
try:
with HDF5TableReader(self.calibration_path) as h5_table:
assert h5_table._h5file.isopen == True
for telid in self.allowed_tels:
# read the calibration data for the moment only one event
table = '/tel_' + str(telid) + '/calibration'
next(
h5_table.read(table,
self.mon_data.tel[telid].calibration))
# eliminate inf values (should be done probably before)
dc_to_pe = self.mon_data.tel[telid].calibration.dc_to_pe
dc_to_pe[np.isinf(dc_to_pe)] = 0
self.log.info(
f"read {self.mon_data.tel[telid].calibration.dc_to_pe}"
)
except:
self.log.error(
f"Problem in reading calibration file {self.calibration_path}")
def _calibrate_dl0(self, event, telid):
"""
create dl0 level, for the moment copy the r1
"""
waveforms = event.r1.tel[telid].waveform
if self._check_r1_empty(waveforms):
return
event.dl0.event_id = event.r1.event_id
event.mon.tel[telid].calibration = self.mon_data.tel[telid].calibration
# subtract the pedestal per sample (should we do it?) and multiply for the calibration coefficients
#
event.dl0.tel[telid].waveform = (
(event.r1.tel[telid].waveform - self.mon_data.tel[telid].
calibration.pedestal_per_sample[:, :, np.newaxis]) *
self.mon_data.tel[telid].calibration.dc_to_pe[:, :, np.newaxis])
def _calibrate_dl1(self, event, telid):
"""
create calibrated dl1 image and calibrate it
"""
waveforms = event.dl0.tel[telid].waveform
if self._check_dl0_empty(waveforms):
return
if self.image_extractor.requires_neighbors():
camera = event.inst.subarray.tel[telid].camera
self.image_extractor.neighbors = camera.neighbor_matrix_where
charge, pulse_time = self.image_extractor(waveforms)
# correct time with drs4 correction if available
if self.time_corrector:
pulse_corr_array = self.time_corrector.get_corr_pulse(
event, pulse_time)
# otherwise use the ff time correction (not drs4 corrected)
else:
pulse_corr_array = pulse_time + self.mon_data.tel[
telid].calibration.time_correction
# perform the gain selection if the threshold is defined
if self.gain_threshold:
waveforms, gain_mask = self.gain_selector(
event.r1.tel[telid].waveform)
event.dl1.tel[telid].image = charge[gain_mask,
np.arange(charge.shape[1])]
event.dl1.tel[telid].pulse_time = pulse_corr_array[
gain_mask, np.arange(pulse_corr_array.shape[1])]
# remember the mask in the lst pixel_status array (this info is missing for the moment in the
# r1 container). I follow the prescription given in the document
# "R1 & DL0 Telescope Event Interfaces and Prototype Evaluation" of K. Kosack
# bit 2 = LG
gain_mask *= 4
# bit 3 = HG
gain_mask[np.where(gain_mask == 0)] = 8
# bit 1 = pixel broken pixel (coming from the EvB)
gain_mask += event.lst.tel[telid].evt.pixel_status >> 1 & 1
# update pixel status
event.lst.tel[telid].evt.pixel_status = gain_mask
# if threshold == None
else:
event.dl1.tel[telid].image = charge
event.dl1.tel[telid].pulse_time = pulse_corr_array
class LSTCameraCalibrator(CameraCalibrator):
"""
Calibrator to handle the LST camera calibration chain, in order to fill
the DL1 data level in the event container.
"""
extractor_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
reducer_product = Unicode(
'NullDataVolumeReducer',
help='Name of the DataVolumeReducer to use').tag(config=True)
calibration_path = Unicode(
'', help='Path to LST calibration file').tag(config=True)
time_calibration_path = Unicode(
'', help='Path to drs4 time calibration file').tag(config=True)
allowed_tels = List(
[1], help='List of telescope to be calibrated').tag(config=True)
gain_threshold = Int(
4094, allow_none=True,
help='Threshold for the gain selection in ADC').tag(config=True)
charge_scale = List(
[1, 1],
help='Multiplicative correction factor for charge estimation [HG,LG]'
).tag(config=True)
def __init__(self, subarray, **kwargs):
"""
Parameters
----------
reducer_product : ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use. If None, then
NullDataVolumeReducer will be used by default, and waveforms
will not be reduced.
extractor_product : ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, then LocalPeakWindowSum
will be used by default.
calibration_path :
Path to LST calibration file to get the pedestal and flat-field corrections
kwargs
"""
super().__init__(subarray, **kwargs)
# load the waveform charge extractor
self.image_extractor = ImageExtractor.from_name(self.extractor_product,
subarray=self.subarray,
config=self.config)
self.log.info(f"extractor {self.extractor_product}")
print("EXTRACTOR", self.image_extractor)
self.data_volume_reducer = DataVolumeReducer.from_name(
self.reducer_product, subarray=self.subarray, config=self.config)
self.log.info(f" {self.reducer_product}")
# declare gain selector if the threshold is defined
if self.gain_threshold:
self.gain_selector = gainselection.ThresholdGainSelector(
threshold=self.gain_threshold)
# declare time calibrator if correction file exist
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
raise IOError(
f"Time calibration file {self.time_calibration_path} not found!"
)
# calibration data container
self.mon_data = MonitoringContainer()
# initialize the MonitoringContainer() for the moment it reads it from a hdf5 file
self._initialize_correction()
self.log.info(f"Global charge scale {self.charge_scale}")
def _initialize_correction(self):
"""
Read the correction from hdf5 calibration file
"""
self.mon_data.tels_with_data = self.allowed_tels
self.log.info(f"read {self.calibration_path}")
try:
with HDF5TableReader(self.calibration_path) as h5_table:
for telid in self.allowed_tels:
# read the calibration data
table = '/tel_' + str(telid) + '/calibration'
next(
h5_table.read(table,
self.mon_data.tel[telid].calibration))
# read pedestal data
table = '/tel_' + str(telid) + '/pedestal'
next(
h5_table.read(table,
self.mon_data.tel[telid].pedestal))
# read flat-field data
table = '/tel_' + str(telid) + '/flatfield'
next(
h5_table.read(table,
self.mon_data.tel[telid].flatfield))
# read the pixel_status container
table = '/tel_' + str(telid) + '/pixel_status'
next(
h5_table.read(table,
self.mon_data.tel[telid].pixel_status))
except Exception:
self.log.exception(
f"Problem in reading calibration file {self.calibration_path}")
raise
def _calibrate_dl0(self, event, telid):
"""
create dl0 level, for the moment copy the r1
"""
waveforms = event.r1.tel[telid].waveform
if self._check_r1_empty(waveforms):
return
# if not already done, initialize the event monitoring containers
if event.mon.tel[telid].calibration.dc_to_pe is None:
event.mon.tel[telid].calibration = self.mon_data.tel[
telid].calibration
event.mon.tel[telid].flatfield = self.mon_data.tel[telid].flatfield
event.mon.tel[telid].pedestal = self.mon_data.tel[telid].pedestal
event.mon.tel[telid].pixel_status = self.mon_data.tel[
telid].pixel_status
#
# subtract the pedestal per sample and multiply for the calibration coefficients
#
event.dl0.tel[telid].waveform = (
(waveforms - self.mon_data.tel[telid].calibration.
pedestal_per_sample[:, :, np.newaxis]) *
self.mon_data.tel[telid].calibration.dc_to_pe[:, :,
np.newaxis]).astype(
np.float32)
def _calibrate_dl1(self, event, telid):
"""
create calibrated dl1 image and calibrate it
"""
waveforms = event.dl0.tel[telid].waveform
if self._check_dl0_empty(waveforms):
return
# for the moment we do the gain selection afterwards
# use gain mask without gain selection
# TBD: - perform calibration of the R1 waveform (not DL1)
# - gain selection before charge integration
# In case of no gain selection the selected gain channels are [0,0,..][1,1,..]
no_gain_selection = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype=np.int)
no_gain_selection[1] = 1
charge = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype='float32')
peak_time = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype='float32')
# image extraction for each channel:
for i in range(waveforms.shape[0]):
charge[i], peak_time[i] = self.image_extractor(
waveforms[i], telid, no_gain_selection[i])
# correct charge for global scale
corrected_charge = charge * np.array(self.charge_scale,
dtype=np.float32)[:, np.newaxis]
# correct time with drs4 correction if available
if self.time_corrector:
peak_time = self.time_corrector.get_corr_pulse(event, peak_time)
# add flat-fielding time correction
peak_time_ff_corrected = peak_time + self.mon_data.tel[
telid].calibration.time_correction.value
# perform the gain selection if the threshold is defined
if self.gain_threshold:
gain_mask = self.gain_selector(event.r1.tel[telid].waveform)
event.dl1.tel[telid].image = corrected_charge[
gain_mask, np.arange(charge.shape[1])]
event.dl1.tel[telid].peak_time = \
peak_time_ff_corrected[gain_mask, np.arange(peak_time_ff_corrected.shape[1])].astype(np.float32)
# remember which channel has been selected
event.r1.tel[telid].selected_gain_channel = gain_mask
# if threshold == None
else:
event.dl1.tel[telid].image = corrected_charge
event.dl1.tel[telid].peak_time = peak_time_ff_corrected
class LSTCameraCalibrator(CameraCalibrator):
"""
Calibrator to handle the LST camera calibration chain, in order to fill
the DL1 data level in the event container.
"""
extractor_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
reducer_product = Unicode(
'NullDataVolumeReducer',
help='Name of the DataVolumeReducer to use').tag(config=True)
calibration_path = Unicode(
'', help='Path to LST calibration file').tag(config=True)
time_calibration_path = Unicode(
'', help='Path to drs4 time calibration file').tag(config=True)
allowed_tels = List(
[1], help='List of telescope to be calibrated').tag(config=True)
gain_threshold = Int(
4094, allow_none=True,
help='Threshold for the gain selection in ADC').tag(config=True)
def __init__(self, **kwargs):
"""
Parameters
----------
reducer_product : ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use. If None, then
NullDataVolumeReducer will be used by default, and waveforms
will not be reduced.
extractor_product : ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, then LocalPeakWindowSum
will be used by default.
calibration_path :
Path to LST calibration file to get the pedestal and flat-field corrections
kwargs
"""
super().__init__(**kwargs)
# load the waveform charge extractor
self.image_extractor = ImageExtractor.from_name(self.extractor_product,
config=self.config)
self.log.info(f"extractor {self.extractor_product}")
print("EXTRACTOR", self.image_extractor)
self.data_volume_reducer = DataVolumeReducer.from_name(
self.reducer_product, config=self.config)
self.log.info(f" {self.reducer_product}")
# declare gain selector if the threshold is defined
if self.gain_threshold:
self.gain_selector = gainselection.ThresholdGainSelector(
threshold=self.gain_threshold)
# declare time calibrator if correction file exist
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
raise IOError(
f"Time calibration file {self.time_calibration_path} not found!"
)
# calibration data container
self.mon_data = MonitoringContainer()
# initialize the MonitoringContainer() for the moment it reads it from a hdf5 file
self._initialize_correction()
def _initialize_correction(self):
"""
Read the correction from hdf5 calibration file
"""
self.mon_data.tels_with_data = self.allowed_tels
self.log.info(f"read {self.calibration_path}")
try:
with HDF5TableReader(self.calibration_path) as h5_table:
for telid in self.allowed_tels:
# read the calibration data
table = '/tel_' + str(telid) + '/calibration'
next(
h5_table.read(table,
self.mon_data.tel[telid].calibration))
# read pedestal data
table = '/tel_' + str(telid) + '/pedestal'
next(
h5_table.read(table,
self.mon_data.tel[telid].pedestal))
# read flat-field data
table = '/tel_' + str(telid) + '/flatfield'
next(
h5_table.read(table,
self.mon_data.tel[telid].flatfield))
# read the pixel_status container
table = '/tel_' + str(telid) + '/pixel_status'
next(
h5_table.read(table,
self.mon_data.tel[telid].pixel_status))
except Exception:
self.log.exception(
f"Problem in reading calibration file {self.calibration_path}")
raise
def _calibrate_dl0(self, event, telid):
"""
create dl0 level, for the moment copy the r1
"""
waveforms = event.r1.tel[telid].waveform
if self._check_r1_empty(waveforms):
return
event.dl0.event_id = event.r1.event_id
# if not already done, initialize the event monitoring containers
if event.mon.tel[telid].calibration.dc_to_pe is None:
event.mon.tel[telid].calibration = self.mon_data.tel[
telid].calibration
event.mon.tel[telid].flatfield = self.mon_data.tel[telid].flatfield
event.mon.tel[telid].pedestal = self.mon_data.tel[telid].pedestal
event.mon.tel[telid].pixel_status = self.mon_data.tel[
telid].pixel_status
#
# subtract the pedestal per sample and multiply for the calibration coefficients
#
event.dl0.tel[telid].waveform = (
(waveforms - self.mon_data.tel[telid].calibration.
pedestal_per_sample[:, :, np.newaxis]) *
self.mon_data.tel[telid].calibration.dc_to_pe[:, :, np.newaxis])
def _calibrate_dl1(self, event, telid):
"""
create calibrated dl1 image and calibrate it
"""
waveforms = event.dl0.tel[telid].waveform
if self._check_dl0_empty(waveforms):
return
if self.image_extractor.requires_neighbors():
camera = event.inst.subarray.tel[telid].camera
self.image_extractor.neighbors = camera.neighbor_matrix_where
charge, pulse_time = self.image_extractor(waveforms)
# correct time with drs4 correction if available
if self.time_corrector:
pulse_time = self.time_corrector.get_corr_pulse(event, pulse_time)
# add flat-fielding time correction
pulse_time_ff_corrected = pulse_time + self.mon_data.tel[
telid].calibration.time_correction
# perform the gain selection if the threshold is defined
if self.gain_threshold:
waveforms, gain_mask = self.gain_selector(
event.r1.tel[telid].waveform)
event.dl1.tel[telid].image = charge[gain_mask,
np.arange(charge.shape[1])]
event.dl1.tel[telid].pulse_time = pulse_time_ff_corrected[
gain_mask,
np.arange(pulse_time_ff_corrected.shape[1])]
# remember which channel has been selected
event.r1.tel[telid].selected_gain_channel = gain_mask
# if threshold == None
else:
event.dl1.tel[telid].image = charge
event.dl1.tel[telid].pulse_time = pulse_time_ff_corrected