def test_telescope_parameter_patterns():
""" Test validation of TelescopeParameters"""
with pytest.raises(ValueError):
TelescopeParameter(dtype="notatype")
class SomeComponent(Component):
tel_param = TelescopeParameter()
tel_param_int = IntTelescopeParameter()
comp = SomeComponent()
# single value allowed (converted to ("default","",val) )
comp.tel_param = 4.5
assert list(comp.tel_param)[0][2] == 4.5
comp.tel_param = [("type", "*", 1.0), ("type", "*LSTCam", 16.0),
("id", 16, 10.0)]
with pytest.raises(TraitError):
comp.tel_param = [("badcommand", "", 1.0)]
with pytest.raises(TraitError):
comp.tel_param = [("type", 12, 1.5)] # bad argument
with pytest.raises(TraitError):
comp.tel_param_int = [("type", "LST_LST_LSTCam", 1.5)] # not int
comp.tel_param_int = [("type", "LST_LST_LSTCam", 1)]
with pytest.raises(TraitError):
comp.tel_param_int = [("*", 5)] # wrong number of args
with pytest.raises(TraitError):
comp.tel_param_int = [(12, "", 5)] # command not string
class SomeComponent(TelescopeComponent):
path = TelescopeParameter(
Path(exists=True,
directory_ok=False,
allow_none=True,
default_value=None),
default_value=None,
allow_none=True,
)
def test_telescope_parameter_patterns(mock_subarray):
""" Test validation of TelescopeParameters"""
with pytest.raises(TypeError):
TelescopeParameter(trait=int)
with pytest.raises(TypeError):
TelescopeParameter(trait=Int)
class SomeComponent(TelescopeComponent):
tel_param = TelescopeParameter(
Float(default_value=0.0, allow_none=True))
tel_param_int = IntTelescopeParameter()
comp = SomeComponent(mock_subarray)
# single value allowed (converted to ("default","",val) )
comp.tel_param = 4.5
assert list(comp.tel_param)[0][2] == 4.5
comp.tel_param = [("type", "*", 1.0), ("type", "*LSTCam", 16.0),
("id", 16, 10.0)]
with pytest.raises(TraitError):
comp.tel_param = [("badcommand", "", 1.0)]
with pytest.raises(TraitError):
comp.tel_param = [("type", 12, 1.5)] # bad argument
with pytest.raises(TraitError):
comp.tel_param_int = [("type", "LST_LST_LSTCam", 1.5)] # not int
comp.tel_param_int = [("type", "LST_LST_LSTCam", 1)]
with pytest.raises(TraitError):
comp.tel_param_int = [("*", 5)] # wrong number of args
with pytest.raises(TraitError):
comp.tel_param_int = [(12, "", 5)] # command not string
class SomeComponent(Component):
tel_param = TelescopeParameter()
tel_param_int = IntTelescopeParameter()
class SomeComponent(TelescopeComponent):
path = TelescopeParameter(Path(), default_value=None).tag(config=True)
val = TelescopeParameter(Float(), default_value=1.0).tag(config=True)
class SomeComponent(TelescopeComponent):
path = TelescopeParameter(Path(exists=True, directory_ok=False))
class SomeComponent(TelescopeComponent):
tel_param = TelescopeParameter(Float(default_value=0.0, allow_none=True))
tel_param_int = IntTelescopeParameter()
class TailCutsDataVolumeReducer(DataVolumeReducer):
"""
Reduce the time integrated shower image in 3 Steps:
1) Select pixels with tailcuts_clean.
2) Add iteratively all pixels with Signal S >= boundary_thresh
with ctapipe module dilate until no new pixels were added.
3) Adding new pixels with dilate to get more conservative.
Attributes
----------
image_extractor_type: String
Name of the image_extractor to be used.
n_end_dilates: IntTelescopeParameter
Number of how many times to dilate at the end.
do_boundary_dilation: BoolTelescopeParameter
If set to 'False', the iteration steps in 2) are skipped and
normal TailcutCleaning is used.
"""
image_extractor_type = TelescopeParameter(
trait=create_class_enum_trait(ImageExtractor,
default_value="NeighborPeakWindowSum"),
default_value="NeighborPeakWindowSum",
help="Name of the ImageExtractor subclass to be used.",
).tag(config=True)
n_end_dilates = IntTelescopeParameter(
default_value=1,
help="Number of how many times to dilate at the end.").tag(config=True)
do_boundary_dilation = BoolTelescopeParameter(
default_value=True,
help="If set to 'False', the iteration steps in 2) are skipped and"
"normal TailcutCleaning is used.",
).tag(config=True)
def __init__(
self,
subarray,
config=None,
parent=None,
cleaner=None,
image_extractor=None,
**kwargs,
):
"""
Parameters
----------
subarray: ctapipe.instrument.SubarrayDescription
Description of the subarray
config: traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
kwargs
"""
super().__init__(config=config,
parent=parent,
subarray=subarray,
**kwargs)
if cleaner is None:
self.cleaner = TailcutsImageCleaner(parent=self,
subarray=self.subarray)
else:
self.cleaner = cleaner
self.image_extractors = {}
if image_extractor is None:
for (_, _, name) in self.image_extractor_type:
self.image_extractors[name] = ImageExtractor.from_name(
name, subarray=self.subarray, parent=self)
else:
name = image_extractor.__class__.__name__
self.image_extractor_type = [("type", "*", name)]
self.image_extractors[name] = image_extractor
def select_pixels(self, waveforms, telid=None, selected_gain_channel=None):
camera_geom = self.subarray.tel[telid].camera.geometry
# Pulse-integrate waveforms
extractor = self.image_extractors[self.image_extractor_type.tel[telid]]
charge, _ = extractor(waveforms,
telid=telid,
selected_gain_channel=selected_gain_channel)
# 1) Step: TailcutCleaning at first
mask = self.cleaner(telid, charge)
pixels_above_boundary_thresh = (
charge >= self.cleaner.boundary_threshold_pe.tel[telid])
mask_in_loop = np.array([])
# 2) Step: Add iteratively all pixels with Signal
# S > boundary_thresh with ctapipe module
# 'dilate' until no new pixels were added.
while (not np.array_equal(mask, mask_in_loop)
and self.do_boundary_dilation.tel[telid]):
mask_in_loop = mask
mask = dilate(camera_geom, mask) & pixels_above_boundary_thresh
# 3) Step: Adding Pixels with 'dilate' to get more conservative.
for _ in range(self.n_end_dilates.tel[telid]):
mask = dilate(camera_geom, mask)
return mask
class CameraCalibrator(TelescopeComponent):
"""
Calibrator to handle the full camera calibration chain, in order to fill
the DL1 data level in the event container.
Attributes
----------
data_volume_reducer_type: str
The name of the DataVolumeReducer subclass to be used
for data volume reduction
image_extractor_type: str
The name of the ImageExtractor subclass to be used for image extraction
"""
data_volume_reducer_type = create_class_enum_trait(
DataVolumeReducer,
default_value="NullDataVolumeReducer").tag(config=True)
image_extractor_type = TelescopeParameter(
trait=create_class_enum_trait(ImageExtractor,
default_value="NeighborPeakWindowSum"),
default_value="NeighborPeakWindowSum",
help="Name of the ImageExtractor subclass to be used.",
).tag(config=True)
apply_waveform_time_shift = BoolTelescopeParameter(
default_value=False,
help=("Apply waveform time shift corrections."
" The minimal integer shift to synchronize waveforms is applied"
" before peak extraction if this option is True"),
).tag(config=True)
apply_peak_time_shift = BoolTelescopeParameter(
default_value=True,
help=
("Apply peak time shift corrections."
" Apply the remaining absolute and fractional time shift corrections"
" to the peak time after pulse extraction."
" If `apply_waveform_time_shift` is False, this will apply the full time shift"
),
).tag(config=True)
def __init__(
self,
subarray,
config=None,
parent=None,
image_extractor=None,
data_volume_reducer=None,
**kwargs,
):
"""
Parameters
----------
subarray: ctapipe.instrument.SubarrayDescription
Description of the subarray. Provides information about the
camera which are useful in calibration. Also required for
configuring the TelescopeParameter traitlets.
config: traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
This is mutually exclusive with passing a ``parent``.
parent: ctapipe.core.Component or ctapipe.core.Tool
Parent of this component in the configuration hierarchy,
this is mutually exclusive with passing ``config``
data_volume_reducer: ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use.
This is used to override the options from the config system
and to enable passing a preconfigured reducer.
image_extractor: ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, the default via the
configuration system will be constructed.
"""
super().__init__(subarray=subarray,
config=config,
parent=parent,
**kwargs)
self.subarray = subarray
self._r1_empty_warn = False
self._dl0_empty_warn = False
self.image_extractors = {}
if image_extractor is None:
for (_, _, name) in self.image_extractor_type:
self.image_extractors[name] = ImageExtractor.from_name(
name, subarray=self.subarray, parent=self)
else:
name = image_extractor.__class__.__name__
self.image_extractor_type = [("type", "*", name)]
self.image_extractors[name] = image_extractor
if data_volume_reducer is None:
self.data_volume_reducer = DataVolumeReducer.from_name(
self.data_volume_reducer_type,
subarray=self.subarray,
parent=self)
else:
self.data_volume_reducer = data_volume_reducer
def _check_r1_empty(self, waveforms):
if waveforms is None:
if not self._r1_empty_warn:
warnings.warn("Encountered an event with no R1 data. "
"DL0 is unchanged in this circumstance.")
self._r1_empty_warn = True
return True
else:
return False
def _check_dl0_empty(self, waveforms):
if waveforms is None:
if not self._dl0_empty_warn:
warnings.warn("Encountered an event with no DL0 data. "
"DL1 is unchanged in this circumstance.")
self._dl0_empty_warn = True
return True
else:
return False
def _calibrate_dl0(self, event, telid):
waveforms = event.r1.tel[telid].waveform
selected_gain_channel = event.r1.tel[telid].selected_gain_channel
if self._check_r1_empty(waveforms):
return
reduced_waveforms_mask = self.data_volume_reducer(
waveforms,
telid=telid,
selected_gain_channel=selected_gain_channel)
waveforms_copy = waveforms.copy()
waveforms_copy[~reduced_waveforms_mask] = 0
event.dl0.tel[telid].waveform = waveforms_copy
event.dl0.tel[telid].selected_gain_channel = selected_gain_channel
def _calibrate_dl1(self, event, telid):
waveforms = event.dl0.tel[telid].waveform
selected_gain_channel = event.dl0.tel[telid].selected_gain_channel
dl1_calib = event.calibration.tel[telid].dl1
if self._check_dl0_empty(waveforms):
return
selected_gain_channel = event.r1.tel[telid].selected_gain_channel
time_shift = event.calibration.tel[telid].dl1.time_shift
readout = self.subarray.tel[telid].camera.readout
n_pixels, n_samples = waveforms.shape
# subtract any remaining pedestal before extraction
if dl1_calib.pedestal_offset is not None:
# this copies intentionally, we don't want to modify the dl0 data
# waveforms have shape (n_pixel, n_samples), pedestals (n_pixels, )
waveforms = waveforms - dl1_calib.pedestal_offset[:, np.newaxis]
if n_samples == 1:
# To handle ASTRI and dst
# TODO: Improved handling of ASTRI and dst
# - dst with custom EventSource?
# - Read into dl1 container directly?
# - Don't do anything if dl1 container already filled
# - Update on SST review decision
charge = waveforms[..., 0].astype(np.float32)
peak_time = np.zeros(n_pixels, dtype=np.float32)
else:
# shift waveforms if time_shift calibration is available
if time_shift is not None:
if self.apply_waveform_time_shift.tel[telid]:
sampling_rate = readout.sampling_rate.to_value(u.GHz)
time_shift_samples = time_shift * sampling_rate
waveforms, remaining_shift = shift_waveforms(
waveforms, time_shift_samples)
remaining_shift /= sampling_rate
else:
remaining_shift = time_shift
extractor = self.image_extractors[
self.image_extractor_type.tel[telid]]
charge, peak_time = extractor(
waveforms,
telid=telid,
selected_gain_channel=selected_gain_channel)
# correct non-integer remainder of the shift if given
if self.apply_peak_time_shift.tel[telid] and time_shift is not None:
peak_time -= remaining_shift
# Calibrate extracted charge
charge *= dl1_calib.relative_factor / dl1_calib.absolute_factor
event.dl1.tel[telid].image = charge
event.dl1.tel[telid].peak_time = peak_time
def __call__(self, event):
"""
Perform the full camera calibration from R1 to DL1. Any calibration
relating to data levels before the data level the file is read into
will be skipped.
Parameters
----------
event : container
A `~ctapipe.containers.ArrayEventContainer` event container
"""
# TODO: How to handle different calibrations depending on telid?
tel = event.r1.tel or event.dl0.tel or event.dl1.tel
for telid in tel.keys():
self._calibrate_dl0(event, telid)
self._calibrate_dl1(event, telid)
class LSTR0Corrections(TelescopeComponent):
"""
The base R0-level calibrator. Changes the r0 container.
The R0 calibrator performs the camera-specific R0 calibration that is
usually performed on the raw data by the camera server.
This calibrator exists in lstchain for testing and prototyping purposes.
"""
offset = IntTelescopeParameter(
default_value=0,
help=(
'Define offset to be subtracted from the waveform *additionally*'
' to the drs4 pedestal offset. This only needs to be given when'
' the drs4 pedestal calibration is not applied or the offset of the'
' drs4 run is different from the data run'
)
).tag(config=True)
r1_sample_start = IntTelescopeParameter(
default_value=3,
help='Start sample for r1 waveform',
allow_none=True,
).tag(config=True)
r1_sample_end = IntTelescopeParameter(
default_value=39,
help='End sample for r1 waveform',
allow_none=True,
).tag(config=True)
drs4_pedestal_path = TelescopeParameter(
trait=Path(exists=True, directory_ok=False),
allow_none=True,
default_value=None,
help=(
'Path to the LST pedestal file'
', required when `apply_drs4_pedestal_correction=True`'
),
).tag(config=True)
calibration_path = Path(
exists=True, directory_ok=False,
help='Path to LST calibration file',
).tag(config=True)
drs4_time_calibration_path = TelescopeParameter(
trait=Path(exists=True, directory_ok=False),
help='Path to the time calibration file',
default_value=None,
allow_none=True,
).tag(config=True)
calib_scale_high_gain = FloatTelescopeParameter(
default_value=1.0,
help='High gain waveform is multiplied by this number'
).tag(config=True)
calib_scale_low_gain = FloatTelescopeParameter(
default_value=1.0,
help='Low gain waveform is multiplied by this number'
).tag(config=True)
select_gain = Bool(
default_value=True,
help='Set to False to keep both gains.'
).tag(config=True)
apply_drs4_pedestal_correction = Bool(
default_value=True,
help=(
'Set to False to disable drs4 pedestal correction.'
' Providing the drs4_pedestal_path is required to perform this calibration'
),
).tag(config=True)
apply_timelapse_correction = Bool(
default_value=True,
help='Set to False to disable drs4 timelapse correction'
).tag(config=True)
apply_spike_correction = Bool(
default_value=True,
help='Set to False to disable drs4 spike correction'
).tag(config=True)
add_calibration_timeshift = Bool(
default_value=True,
help=(
'If true, time correction from the calibration'
' file is added to calibration.dl1.time'
),
).tag(config=True)
gain_selection_threshold = Float(
default_value=3500,
help='Threshold for the ThresholdGainSelector.'
).tag(config=True)
def __init__(self, subarray, config=None, parent=None, **kwargs):
"""
The R0 calibrator for LST data.
Fill the r1 container.
Parameters
----------
"""
super().__init__(
subarray=subarray, config=config, parent=parent, **kwargs
)
self.mon_data = None
self.last_readout_time = {}
self.first_cap = {}
self.first_cap_old = {}
self.fbn = {}
self.fan = {}
for tel_id in self.subarray.tel:
shape = (N_GAINS, N_PIXELS, N_CAPACITORS_PIXEL)
self.last_readout_time[tel_id] = np.zeros(shape, dtype='uint64')
shape = (N_GAINS, N_PIXELS)
self.first_cap[tel_id] = np.zeros(shape, dtype=int)
self.first_cap_old[tel_id] = np.zeros(shape, dtype=int)
if self.select_gain:
self.gain_selector = ThresholdGainSelector(
threshold=self.gain_selection_threshold,
parent=self
)
else:
self.gain_selector = None
if self.calibration_path is not None:
self.mon_data = self._read_calibration_file(self.calibration_path)
def apply_drs4_corrections(self, event: LSTArrayEventContainer):
self.update_first_capacitors(event)
for tel_id, r0 in event.r0.tel.items():
r1 = event.r1.tel[tel_id]
# If r1 was not yet filled, copy of r0 converted
if r1.waveform is None:
r1.waveform = r0.waveform
# float32 can represent all values of uint16 exactly,
# so this does not loose precision.
r1.waveform = r1.waveform.astype(np.float32, copy=False)
# apply drs4 corrections
if self.apply_drs4_pedestal_correction:
self.subtract_pedestal(event, tel_id)
if self.apply_timelapse_correction:
self.time_lapse_corr(event, tel_id)
if self.apply_spike_correction:
self.interpolate_spikes(event, tel_id)
# remove samples at beginning / end of waveform
start = self.r1_sample_start.tel[tel_id]
end = self.r1_sample_end.tel[tel_id]
r1.waveform = r1.waveform[..., start:end]
if self.offset.tel[tel_id] != 0:
r1.waveform -= self.offset.tel[tel_id]
mon = event.mon.tel[tel_id]
if r1.selected_gain_channel is None:
r1.waveform[mon.pixel_status.hardware_failing_pixels] = 0.0
else:
broken = mon.pixel_status.hardware_failing_pixels[r1.selected_gain_channel, PIXEL_INDEX]
r1.waveform[broken] = 0.0
def update_first_capacitors(self, event: LSTArrayEventContainer):
for tel_id, lst in event.lst.tel.items():
self.first_cap_old[tel_id] = self.first_cap[tel_id]
self.first_cap[tel_id] = get_first_capacitors_for_pixels(
lst.evt.first_capacitor_id,
lst.svc.pixel_ids,
)
def calibrate(self, event: LSTArrayEventContainer):
for tel_id in event.r0.tel:
r1 = event.r1.tel[tel_id]
# if `apply_drs4_corrections` is False, we did not fill in the
# waveform yet.
if r1.waveform is None:
r1.waveform = event.r0.tel[tel_id].waveform
r1.waveform = r1.waveform.astype(np.float32, copy=False)
# do gain selection before converting to pe
# like eventbuilder will do
if self.select_gain and r1.selected_gain_channel is None:
r1.selected_gain_channel = self.gain_selector(r1.waveform)
r1.waveform = r1.waveform[r1.selected_gain_channel, PIXEL_INDEX]
# apply monitoring data corrections,
# subtract pedestal and convert to pe
if self.mon_data is not None:
calibration = self.mon_data.tel[tel_id].calibration
convert_to_pe(
waveform=r1.waveform,
calibration=calibration,
selected_gain_channel=r1.selected_gain_channel
)
broken_pixels = event.mon.tel[tel_id].pixel_status.hardware_failing_pixels
if r1.selected_gain_channel is None:
r1.waveform[broken_pixels] = 0.0
else:
r1.waveform[broken_pixels[r1.selected_gain_channel, PIXEL_INDEX]] = 0.0
# store calibration data needed for dl1 calibration in ctapipe
# first drs4 time shift (zeros if no calib file was given)
time_shift = self.get_drs4_time_correction(
tel_id, self.first_cap[tel_id],
selected_gain_channel=r1.selected_gain_channel,
)
# time shift from flat fielding
if self.mon_data is not None and self.add_calibration_timeshift:
time_corr = self.mon_data.tel[tel_id].calibration.time_correction
# time_shift is subtracted in ctapipe,
# but time_correction should be added
if r1.selected_gain_channel is not None:
time_shift -= time_corr[r1.selected_gain_channel, PIXEL_INDEX].to_value(u.ns)
else:
time_shift -= time_corr.to_value(u.ns)
event.calibration.tel[tel_id].dl1.time_shift = time_shift
# needed for charge scaling in ctpaipe dl1 calib
if r1.selected_gain_channel is not None:
relative_factor = np.empty(N_PIXELS)
relative_factor[r1.selected_gain_channel == HIGH_GAIN] = self.calib_scale_high_gain.tel[tel_id]
relative_factor[r1.selected_gain_channel == LOW_GAIN] = self.calib_scale_low_gain.tel[tel_id]
else:
relative_factor = np.empty((N_GAINS, N_PIXELS))
relative_factor[HIGH_GAIN] = self.calib_scale_high_gain.tel[tel_id]
relative_factor[LOW_GAIN] = self.calib_scale_low_gain.tel[tel_id]
event.calibration.tel[tel_id].dl1.relative_factor = relative_factor
@staticmethod
def _read_calibration_file(path):
"""
Read the correction from hdf5 calibration file
"""
mon = MonitoringContainer()
with tables.open_file(path) as f:
tel_ids = [
int(key[4:]) for key in f.root._v_children.keys()
if key.startswith('tel_')
]
for tel_id in tel_ids:
with HDF5TableReader(path) as h5_table:
base = f'/tel_{tel_id}'
# read the calibration data
table = base + '/calibration'
next(h5_table.read(table, mon.tel[tel_id].calibration))
# read pedestal data
table = base + '/pedestal'
next(h5_table.read(table, mon.tel[tel_id].pedestal))
# read flat-field data
table = base + '/flatfield'
next(h5_table.read(table, mon.tel[tel_id].flatfield))
# read the pixel_status container
table = base + '/pixel_status'
next(h5_table.read(table, mon.tel[tel_id].pixel_status))
return mon
@staticmethod
def load_drs4_time_calibration_file(path):
"""
Function to load calibration file.
"""
with tables.open_file(path, 'r') as f:
fan = f.root.fan[:]
fbn = f.root.fbn[:]
return fan, fbn
def load_drs4_time_calibration_file_for_tel(self, tel_id):
self.fan[tel_id], self.fbn[tel_id] = self.load_drs4_time_calibration_file(
self.drs4_time_calibration_path.tel[tel_id]
)
def get_drs4_time_correction(self, tel_id, first_capacitors, selected_gain_channel=None):
"""
Return pulse time after time correction.
"""
if self.drs4_time_calibration_path.tel[tel_id] is None:
if selected_gain_channel is None:
return np.zeros((N_GAINS, N_PIXELS))
else:
return np.zeros(N_PIXELS)
# load calib file if not already done
if tel_id not in self.fan:
self.load_drs4_time_calibration_file_for_tel(tel_id)
if selected_gain_channel is not None:
return calc_drs4_time_correction_gain_selected(
first_capacitors,
selected_gain_channel,
self.fan[tel_id],
self.fbn[tel_id],
)
else:
return calc_drs4_time_correction_both_gains(
first_capacitors,
self.fan[tel_id],
self.fbn[tel_id],
)
@staticmethod
@lru_cache(maxsize=4)
def _get_drs4_pedestal_data(path):
"""
Function to load pedestal file.
To make boundary conditions unnecessary,
the first N_SAMPLES values are repeated at the end of the array
The result is cached so we can repeatedly call this method
using the configured path without reading it each time.
"""
if path is None:
raise ValueError(
"DRS4 pedestal correction requested"
" but no file provided for telescope"
)
pedestal_data = np.empty(
(N_GAINS, N_PIXELS_MODULE * N_MODULES, N_CAPACITORS_PIXEL + N_SAMPLES),
dtype=np.int16
)
with fits.open(path) as f:
pedestal_data[:, :, :N_CAPACITORS_PIXEL] = f[1].data
pedestal_data[:, :, N_CAPACITORS_PIXEL:] = pedestal_data[:, :, :N_SAMPLES]
return pedestal_data
def subtract_pedestal(self, event, tel_id):
"""
Subtract cell offset using pedestal file.
Fill the R1 container.
Parameters
----------
event : `ctapipe` event-container
tel_id : id of the telescope
"""
pedestal = self._get_drs4_pedestal_data(
self.drs4_pedestal_path.tel[tel_id]
)
if event.r1.tel[tel_id].selected_gain_channel is None:
subtract_pedestal(
event.r1.tel[tel_id].waveform,
self.first_cap[tel_id],
pedestal,
)
else:
subtract_pedestal_gain_selected(
event.r1.tel[tel_id].waveform,
self.first_cap[tel_id],
pedestal,
event.r1.tel[tel_id].selected_gain_channel,
)
def time_lapse_corr(self, event, tel_id):
"""
Perform time lapse baseline corrections.
Fill the R1 container or modifies R0 container.
Parameters
----------
event : `ctapipe` event-container
tel_id : id of the telescope
"""
lst = event.lst.tel[tel_id]
# If R1 container exists, update it inplace
if isinstance(event.r1.tel[tel_id].waveform, np.ndarray):
container = event.r1.tel[tel_id]
else:
# Modify R0 container. This is to create pedestal files.
container = event.r0.tel[tel_id]
waveform = container.waveform.copy()
# We have 2 functions: one for data from 2018/10/10 to 2019/11/04 and
# one for data from 2019/11/05 (from Run 1574) after update firmware.
# The old readout (before 2019/11/05) is shifted by 1 cell.
run_id = event.lst.tel[tel_id].svc.configuration_id
# not yet gain selected
if event.r1.tel[tel_id].selected_gain_channel is None:
apply_timelapse_correction(
waveform=waveform,
local_clock_counter=lst.evt.local_clock_counter,
first_capacitors=self.first_cap[tel_id],
last_readout_time=self.last_readout_time[tel_id],
expected_pixels_id=lst.svc.pixel_ids,
run_id=run_id,
)
else:
apply_timelapse_correction_gain_selected(
waveform=waveform,
local_clock_counter=lst.evt.local_clock_counter,
first_capacitors=self.first_cap[tel_id],
last_readout_time=self.last_readout_time[tel_id],
expected_pixels_id=lst.svc.pixel_ids,
selected_gain_channel=event.r1.tel[tel_id].selected_gain_channel,
run_id=run_id,
)
container.waveform = waveform
def interpolate_spikes(self, event, tel_id):
"""
Interpolates spike A & B.
Fill the R1 container.
Parameters
----------
event : `ctapipe` event-container
tel_id : id of the telescope
"""
run_id = event.lst.tel[tel_id].svc.configuration_id
r1 = event.r1.tel[tel_id]
if r1.selected_gain_channel is None:
interpolate_spikes(
waveform=r1.waveform,
first_capacitors=self.first_cap[tel_id],
previous_first_capacitors=self.first_cap_old[tel_id],
run_id=run_id,
)
else:
interpolate_spikes_gain_selected(
waveform=r1.waveform,
first_capacitors=self.first_cap[tel_id],
previous_first_capacitors=self.first_cap_old[tel_id],
selected_gain_channel=r1.selected_gain_channel,
run_id=run_id,
)
class EventTimeCalculator(TelescopeComponent):
'''
Class to calculate event times from low-level counter information.
Also keeps track of "UCTS jumps", where UCTS info goes missing for
a certain event and all following info has to be shifted.
There are several sources of timing information in LST raw data.
Each dragon module has two high precision counters, which however only
give a relative time.
Same is true for the TIB.
The only precise absolute timestamp is the UCTS timestamp.
However, at least during the commissioning, UCTS was/is not reliable
enough to only use the UCTS timestamp.
Instead, we calculate an absolute timestamp by using one valid pair
of dragon counter / ucts timestamp and then use the relative time elapsed
from this reference using the dragon counter.
For runs where no such UCTS reference exists, for example because UCTS
was completely unavailable, we use the start of run timestamp from the
camera configuration.
This will however result in imprecises timestamps off by several seconds.
These might be good enough for interpolating pointing information but
are only precise for relative time changes, i.e. not suitable for pulsar
analysis or matching events with MAGIC.
Extracting the reference will only work reliably for the first subrun
for ucts.
Using svc.date is only possible for the first subrun and will raise an erorr
if the event id of the first event seen by the time calculator is not 1.
'''
timestamp = TelescopeParameter(
trait=Enum(['ucts', 'dragon']),
default_value='dragon',
help=
('Source of the timestamp. UCTS is simplest and most precise,'
' unfortunately it is not yet reliable, instead the time is calculated'
' by default using the relative dragon board counters with a reference'
' pair of counter / time. See the `dragon_reference_time` and'
' `dragon_reference_counter` traitlets')).tag(config=True)
dragon_reference_time = TelescopeParameter(
Int(allow_none=True),
default_value=None,
help='Reference timestamp for the dragon time calculation in ns').tag(
config=True)
dragon_reference_counter = TelescopeParameter(
Int(allow_none=True),
help=
'Dragon board counter value of a valid ucts/dragon counter combination',
default_value=None,
).tag(config=True)
dragon_module_id = TelescopeParameter(
Int(allow_none=True),
default_value=None,
help='Module id used to calculate dragon time.',
).tag(config=True)
run_summary_path = TelescopeParameter(
Path(exists=True, directory_ok=False),
default_value=None,
help=('Path to the run summary for the correct night.'
' If given, dragon reference counters are read from this file.'
' Explicitly given values override values read from the file.'
)).tag(config=True)
extract_reference = Bool(
default_value=True,
help=(
'If true, extract the reference values from the first event.'
'This will only work for the first file of a run, due to the '
'UCTS jumps when UCTS is available or because svc.date gives only '
'the start of the run, not the start of each file (subrun) ')).tag(
config=True)
def __init__(self,
subarray,
run_id,
expected_modules_id,
config=None,
parent=None,
**kwargs):
'''Initialize EventTimeCalculator'''
super().__init__(subarray=subarray,
config=config,
parent=parent,
**kwargs)
self.previous_ucts_timestamps = defaultdict(deque)
self.previous_ucts_trigger_types = defaultdict(deque)
# we cannot __setitem__ telescope lookup values, so we store them
# in non-trait private values
self._has_dragon_reference = {}
self._dragon_reference_time = {}
self._dragon_reference_counter = {}
self._dragon_module_index = {}
self.detected_jumps = defaultdict(list)
for tel_id in self.subarray.tel:
if self.run_summary_path.tel[tel_id] is not None:
run_summary = read_run_summary(
self.run_summary_path.tel[tel_id])
row = run_summary.loc[run_id]
self._has_dragon_reference[tel_id] = True
self._dragon_reference_time[tel_id] = np.uint64(
row['dragon_reference_time'])
self._dragon_reference_counter[tel_id] = np.uint64(
row['dragon_reference_counter'])
self._dragon_module_index[tel_id] = row[
'dragon_reference_module_index']
if row['dragon_reference_source'] == 'run_start':
self.log.warning(
'Dragon reference source is run_start, '
'times will be imprecise by several seconds')
else:
self._has_dragon_reference[tel_id] = (
self.dragon_reference_time.tel[tel_id] is not None
and self.dragon_reference_counter.tel[tel_id] is not None
and self.dragon_module_id.tel[tel_id] is not None)
if not self._has_dragon_reference[
tel_id] and not self.extract_reference:
raise ValueError(
'No dragon reference values given and extract_reference=False'
)
# set values from traitlets, overrides values from files if both given
if self.dragon_reference_counter.tel[tel_id] is not None:
self._dragon_reference_counter[tel_id] = np.uint64(
self.dragon_reference_counter.tel[tel_id])
if self.dragon_reference_time.tel[tel_id] is not None:
self._dragon_reference_time[tel_id] = np.uint64(
self.dragon_reference_time.tel[tel_id])
if self.dragon_module_id.tel[tel_id] is not None:
module_id = self.dragon_module_id.tel[tel_id]
module_index = module_id_to_index(expected_modules_id,
module_id)
self._dragon_module_index[tel_id] = module_index
def __call__(self, tel_id, event):
lst = event.lst.tel[tel_id]
ucts_available = bool(lst.evt.extdevices_presence & 2)
ucts_timestamp = lst.evt.ucts_timestamp
# first event and values not passed
if self.extract_reference and not self._has_dragon_reference[tel_id]:
# use first working module if none is specified
if tel_id not in self._dragon_module_index:
self._dragon_module_index[tel_id] = np.where(
lst.evt.module_status != 0)[0][0]
module_index = self._dragon_module_index[tel_id]
self._dragon_reference_counter[tel_id] = combine_counters(
lst.evt.pps_counter[module_index],
lst.evt.tenMHz_counter[module_index])
if not ucts_available:
source = 'svc.date'
if event.index.event_id != 1:
raise ValueError('Can only use run start timestamp'
' as reference for the first subrun')
self.log.warning(
f'Cannot calculate a precise timestamp for obs_id={event.index.obs_id}'
f', tel_id={tel_id}. UCTS unavailable.')
# convert runstart from UTC to tai
run_start = Time(lst.svc.date, format='unix')
self._dragon_reference_time[tel_id] = np.uint64(
S_TO_NS * run_start.unix_tai)
else:
source = 'ucts'
self._dragon_reference_time[tel_id] = ucts_timestamp
if event.index.event_id != 1:
self.log.warning(
'Calculating time reference values not from first event.'
' This might result in wrong timestamps due to UCTS jumps'
)
self.log.critical(
f'Using event {event.index.event_id} as time reference for dragon.'
f' timestamp: {self._dragon_reference_time[tel_id]} from {source}'
f' counter: {self._dragon_reference_counter[tel_id]}')
self._has_dragon_reference[tel_id] = True
# Dragon timestamp based on the reference timestamp
module_index = self._dragon_module_index[tel_id]
dragon_timestamp = calc_dragon_time(
pps_counter=lst.evt.pps_counter[module_index],
tenMHz_counter=lst.evt.tenMHz_counter[module_index],
reference_time=self._dragon_reference_time[tel_id],
reference_counter=self._dragon_reference_counter[tel_id],
)
# if ucts is not available, there is nothing more we have to do
# and dragon time is our only option
if not ucts_available:
return time_from_unix_tai_ns(dragon_timestamp)
# Due to a DAQ bug, sometimes there are 'jumps' in the
# UCTS info in the raw files. After one such jump,
# all the UCTS info attached to an event actually
# corresponds to the next event. This one-event
# shift stays like that until there is another jump
# (then it becomes a 2-event shift and so on). We will
# keep track of those jumps, by storing the UCTS info
# of the previously read events in the list
# previous_ucts_time_unix. The list has one element
# for each of the jumps, so if there has been just
# one jump we have the UCTS info of the previous
# event only (which truly corresponds to the
# current event). If there have been n jumps, we keep
# the past n events. The info to be used for
# the current event is always the first element of
# the array, previous_ucts_time_unix[0], whereas the
# current event's (wrong) ucts info is placed last in
# the array. Each time the first array element is
# used, it is removed and the rest move up in the
# list. We have another similar array for the trigger
# types, previous_ucts_trigger_type
ucts_trigger_type = lst.evt.ucts_trigger_type
if len(self.previous_ucts_timestamps[tel_id]) > 0:
# put the current values last in the queue, for later use:
self.previous_ucts_timestamps[tel_id].append(ucts_timestamp)
self.previous_ucts_trigger_types[tel_id].append(ucts_trigger_type)
# get the correct time for the current event from the queue
ucts_timestamp = self.previous_ucts_timestamps[tel_id].popleft()
ucts_trigger_type = self.previous_ucts_trigger_types[
tel_id].popleft()
lst.evt.ucts_trigger_type = ucts_trigger_type
lst.evt.ucts_timestamp = ucts_timestamp
# Now check consistency of UCTS and Dragon times. If
# UCTS time is ahead of Dragon time by more than
# 1 us, most likely the UCTS info has been
# lost for this event (i.e. there has been another
# 'jump' of those described above), and the one we have
# actually corresponds to the next event. So we put it
# back first in the list, to assign it to the next
# event. We also move the other elements down in the
# list, which will now be one element longer.
# We leave the current event with the same time,
# which will be approximately correct (depending on
# event rate), and set its ucts_trigger_type to -1,
# which will tell us a jump happened and hence this
# event does not have proper UCTS info.
delta = abs_diff(ucts_timestamp, dragon_timestamp)
if delta > 1e3:
self.log.warning(f'Found UCTS jump in event {event.index.event_id}'
f', dragon time: {dragon_timestamp:d}'
f', delta: {delta / 1000:.0f} µs')
self.previous_ucts_timestamps[tel_id].appendleft(ucts_timestamp)
self.previous_ucts_trigger_types[tel_id].appendleft(
ucts_trigger_type)
self.detected_jumps[tel_id].append(
(event.count, event.index.event_id, delta))
lst.evt.ucts_jump = True
# fall back to dragon time / tib trigger
lst.evt.ucts_timestamp = dragon_timestamp
ucts_timestamp = dragon_timestamp
tib_available = lst.evt.extdevices_presence & 1
if tib_available:
lst.evt.ucts_trigger_type = lst.evt.tib_masked_trigger
else:
self.log.warning(
'Detected ucts jump but not tib trigger info available'
', event will have no trigger information')
lst.evt.ucts_trigger_type = 0
# Select the timestamps to be used for pointing interpolation
if self.timestamp.tel[tel_id] == "dragon":
return time_from_unix_tai_ns(dragon_timestamp)
return time_from_unix_tai_ns(ucts_timestamp)
class SomeComponent(TelescopeComponent):
path = TelescopeParameter(Path(allow_none=True, default_value=None),
default_value=None).tag(config=True)
val = TelescopeParameter(Float(), default_value=1.0).tag(config=True)
flag = TelescopeParameter(Bool(), default_value=True).tag(config=True)
class PointingSource(TelescopeComponent):
"""Provides access to pointing information stored in LST drive reports."""
drive_report_path = TelescopeParameter(
trait=Path(exists=True, directory_ok=False),
help='Path to the LST drive report file',
default_value=None,
).tag(config=True)
def __init__(self, subarray, config=None, parent=None, **kwargs):
'''Initialize PointingSource'''
super().__init__(subarray, config=config, parent=parent, **kwargs)
self.drive_report = {}
self.interp_az = {}
self.interp_alt = {}
self.interp_ra = {}
self.interp_dec = {}
@staticmethod
def _read_drive_report(path):
"""
Read a drive report into an astropy table
Parameters:
-----------
str: drive report file
Returns:
data:`~astropy.table.Table`
A table of drive reports
"""
data = Table.read(
path, format='ascii', delimiter=' ',
header_start=None,
data_start=0,
names=[
'weekday', 'month', 'day', 'time', 'year', 'unix_time',
'Az', 'azimuth_avg', 'azimuth_min', 'azimuth_max', 'azimuth_std',
'El', 'zenith_avg', 'zenith_min', 'zenith_max', 'zenith_std',
'Ra', 'target_ra', 'Dec', 'target_dec',
]
)
return data
def _read_drive_report_for_tel(self, tel_id):
path = self.drive_report_path.tel[tel_id]
if path is None:
raise ValueError(f'No drive report given for telescope {tel_id}')
self.log.info(f'Loading drive report "{path}" for tel_id={tel_id}')
self.drive_report[tel_id] = self._read_drive_report(path)
self.interp_az[tel_id] = interp1d(
self.drive_report[tel_id]['unix_time'],
self.drive_report[tel_id]['azimuth_avg'],
)
self.interp_alt[tel_id] = interp1d(
self.drive_report[tel_id]['unix_time'],
90 - self.drive_report[tel_id]['zenith_avg'],
)
self.interp_ra[tel_id] = interp1d(
self.drive_report[tel_id]['unix_time'],
self.drive_report[tel_id]['target_ra'],
)
self.interp_dec[tel_id] = interp1d(
self.drive_report[tel_id]['unix_time'],
self.drive_report[tel_id]['target_dec'],
)
def get_pointing_position_altaz(self, tel_id, time):
"""
Calculating pointing positions by interpolation
Parameters:
-----------
time: array
times from events
Drivereport: Container
a container filled with drive information
"""
if tel_id not in self.drive_report:
self._read_drive_report_for_tel(tel_id)
alt = u.Quantity(self.interp_alt[tel_id](time.unix), u.deg)
az = u.Quantity(self.interp_az[tel_id](time.unix), u.deg)
return TelescopePointingContainer(
altitude=alt.to(u.rad),
azimuth=az.to(u.rad),
)
def get_pointing_position_icrs(self, tel_id, time):
if tel_id not in self.drive_report:
self._read_drive_report_for_tel(tel_id)
ra = u.Quantity(self.interp_ra[tel_id](time.unix), u.deg)
dec = u.Quantity(self.interp_dec[tel_id](time.unix), u.deg)
# drive reports contain 0 / 0 if not tracking ICRS coordinates
# TODO: hope we never really observe ra=0°, dec=0°
if ra != 0.0 and dec != 0.0:
return ra, dec
return NAN_ANGLE, NAN_ANGLE