def test_telescope_parameter_patterns(): """ Test validation of TelescopeParameters""" with pytest.raises(ValueError): TelescopeParameter(dtype="notatype") class SomeComponent(Component): tel_param = TelescopeParameter() tel_param_int = IntTelescopeParameter() comp = SomeComponent() # single value allowed (converted to ("default","",val) ) comp.tel_param = 4.5 assert list(comp.tel_param)[0][2] == 4.5 comp.tel_param = [("type", "*", 1.0), ("type", "*LSTCam", 16.0), ("id", 16, 10.0)] with pytest.raises(TraitError): comp.tel_param = [("badcommand", "", 1.0)] with pytest.raises(TraitError): comp.tel_param = [("type", 12, 1.5)] # bad argument with pytest.raises(TraitError): comp.tel_param_int = [("type", "LST_LST_LSTCam", 1.5)] # not int comp.tel_param_int = [("type", "LST_LST_LSTCam", 1)] with pytest.raises(TraitError): comp.tel_param_int = [("*", 5)] # wrong number of args with pytest.raises(TraitError): comp.tel_param_int = [(12, "", 5)] # command not string
class SomeComponent(TelescopeComponent): path = TelescopeParameter( Path(exists=True, directory_ok=False, allow_none=True, default_value=None), default_value=None, allow_none=True, )
def test_telescope_parameter_patterns(mock_subarray): """ Test validation of TelescopeParameters""" with pytest.raises(TypeError): TelescopeParameter(trait=int) with pytest.raises(TypeError): TelescopeParameter(trait=Int) class SomeComponent(TelescopeComponent): tel_param = TelescopeParameter( Float(default_value=0.0, allow_none=True)) tel_param_int = IntTelescopeParameter() comp = SomeComponent(mock_subarray) # single value allowed (converted to ("default","",val) ) comp.tel_param = 4.5 assert list(comp.tel_param)[0][2] == 4.5 comp.tel_param = [("type", "*", 1.0), ("type", "*LSTCam", 16.0), ("id", 16, 10.0)] with pytest.raises(TraitError): comp.tel_param = [("badcommand", "", 1.0)] with pytest.raises(TraitError): comp.tel_param = [("type", 12, 1.5)] # bad argument with pytest.raises(TraitError): comp.tel_param_int = [("type", "LST_LST_LSTCam", 1.5)] # not int comp.tel_param_int = [("type", "LST_LST_LSTCam", 1)] with pytest.raises(TraitError): comp.tel_param_int = [("*", 5)] # wrong number of args with pytest.raises(TraitError): comp.tel_param_int = [(12, "", 5)] # command not string
class SomeComponent(Component): tel_param = TelescopeParameter() tel_param_int = IntTelescopeParameter()
class SomeComponent(TelescopeComponent): path = TelescopeParameter(Path(), default_value=None).tag(config=True) val = TelescopeParameter(Float(), default_value=1.0).tag(config=True)
class SomeComponent(TelescopeComponent): path = TelescopeParameter(Path(exists=True, directory_ok=False))
class SomeComponent(TelescopeComponent): tel_param = TelescopeParameter(Float(default_value=0.0, allow_none=True)) tel_param_int = IntTelescopeParameter()
class TailCutsDataVolumeReducer(DataVolumeReducer): """ Reduce the time integrated shower image in 3 Steps: 1) Select pixels with tailcuts_clean. 2) Add iteratively all pixels with Signal S >= boundary_thresh with ctapipe module dilate until no new pixels were added. 3) Adding new pixels with dilate to get more conservative. Attributes ---------- image_extractor_type: String Name of the image_extractor to be used. n_end_dilates: IntTelescopeParameter Number of how many times to dilate at the end. do_boundary_dilation: BoolTelescopeParameter If set to 'False', the iteration steps in 2) are skipped and normal TailcutCleaning is used. """ image_extractor_type = TelescopeParameter( trait=create_class_enum_trait(ImageExtractor, default_value="NeighborPeakWindowSum"), default_value="NeighborPeakWindowSum", help="Name of the ImageExtractor subclass to be used.", ).tag(config=True) n_end_dilates = IntTelescopeParameter( default_value=1, help="Number of how many times to dilate at the end.").tag(config=True) do_boundary_dilation = BoolTelescopeParameter( default_value=True, help="If set to 'False', the iteration steps in 2) are skipped and" "normal TailcutCleaning is used.", ).tag(config=True) def __init__( self, subarray, config=None, parent=None, cleaner=None, image_extractor=None, **kwargs, ): """ Parameters ---------- subarray: ctapipe.instrument.SubarrayDescription Description of the subarray config: traitlets.loader.Config Configuration specified by config file or cmdline arguments. Used to set traitlet values. Set to None if no configuration to pass. kwargs """ super().__init__(config=config, parent=parent, subarray=subarray, **kwargs) if cleaner is None: self.cleaner = TailcutsImageCleaner(parent=self, subarray=self.subarray) else: self.cleaner = cleaner self.image_extractors = {} if image_extractor is None: for (_, _, name) in self.image_extractor_type: self.image_extractors[name] = ImageExtractor.from_name( name, subarray=self.subarray, parent=self) else: name = image_extractor.__class__.__name__ self.image_extractor_type = [("type", "*", name)] self.image_extractors[name] = image_extractor def select_pixels(self, waveforms, telid=None, selected_gain_channel=None): camera_geom = self.subarray.tel[telid].camera.geometry # Pulse-integrate waveforms extractor = self.image_extractors[self.image_extractor_type.tel[telid]] charge, _ = extractor(waveforms, telid=telid, selected_gain_channel=selected_gain_channel) # 1) Step: TailcutCleaning at first mask = self.cleaner(telid, charge) pixels_above_boundary_thresh = ( charge >= self.cleaner.boundary_threshold_pe.tel[telid]) mask_in_loop = np.array([]) # 2) Step: Add iteratively all pixels with Signal # S > boundary_thresh with ctapipe module # 'dilate' until no new pixels were added. while (not np.array_equal(mask, mask_in_loop) and self.do_boundary_dilation.tel[telid]): mask_in_loop = mask mask = dilate(camera_geom, mask) & pixels_above_boundary_thresh # 3) Step: Adding Pixels with 'dilate' to get more conservative. for _ in range(self.n_end_dilates.tel[telid]): mask = dilate(camera_geom, mask) return mask
class CameraCalibrator(TelescopeComponent): """ Calibrator to handle the full camera calibration chain, in order to fill the DL1 data level in the event container. Attributes ---------- data_volume_reducer_type: str The name of the DataVolumeReducer subclass to be used for data volume reduction image_extractor_type: str The name of the ImageExtractor subclass to be used for image extraction """ data_volume_reducer_type = create_class_enum_trait( DataVolumeReducer, default_value="NullDataVolumeReducer").tag(config=True) image_extractor_type = TelescopeParameter( trait=create_class_enum_trait(ImageExtractor, default_value="NeighborPeakWindowSum"), default_value="NeighborPeakWindowSum", help="Name of the ImageExtractor subclass to be used.", ).tag(config=True) apply_waveform_time_shift = BoolTelescopeParameter( default_value=False, help=("Apply waveform time shift corrections." " The minimal integer shift to synchronize waveforms is applied" " before peak extraction if this option is True"), ).tag(config=True) apply_peak_time_shift = BoolTelescopeParameter( default_value=True, help= ("Apply peak time shift corrections." " Apply the remaining absolute and fractional time shift corrections" " to the peak time after pulse extraction." " If `apply_waveform_time_shift` is False, this will apply the full time shift" ), ).tag(config=True) def __init__( self, subarray, config=None, parent=None, image_extractor=None, data_volume_reducer=None, **kwargs, ): """ Parameters ---------- subarray: ctapipe.instrument.SubarrayDescription Description of the subarray. Provides information about the camera which are useful in calibration. Also required for configuring the TelescopeParameter traitlets. config: traitlets.loader.Config Configuration specified by config file or cmdline arguments. Used to set traitlet values. This is mutually exclusive with passing a ``parent``. parent: ctapipe.core.Component or ctapipe.core.Tool Parent of this component in the configuration hierarchy, this is mutually exclusive with passing ``config`` data_volume_reducer: ctapipe.image.reducer.DataVolumeReducer The DataVolumeReducer to use. This is used to override the options from the config system and to enable passing a preconfigured reducer. image_extractor: ctapipe.image.extractor.ImageExtractor The ImageExtractor to use. If None, the default via the configuration system will be constructed. """ super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.subarray = subarray self._r1_empty_warn = False self._dl0_empty_warn = False self.image_extractors = {} if image_extractor is None: for (_, _, name) in self.image_extractor_type: self.image_extractors[name] = ImageExtractor.from_name( name, subarray=self.subarray, parent=self) else: name = image_extractor.__class__.__name__ self.image_extractor_type = [("type", "*", name)] self.image_extractors[name] = image_extractor if data_volume_reducer is None: self.data_volume_reducer = DataVolumeReducer.from_name( self.data_volume_reducer_type, subarray=self.subarray, parent=self) else: self.data_volume_reducer = data_volume_reducer def _check_r1_empty(self, waveforms): if waveforms is None: if not self._r1_empty_warn: warnings.warn("Encountered an event with no R1 data. " "DL0 is unchanged in this circumstance.") self._r1_empty_warn = True return True else: return False def _check_dl0_empty(self, waveforms): if waveforms is None: if not self._dl0_empty_warn: warnings.warn("Encountered an event with no DL0 data. " "DL1 is unchanged in this circumstance.") self._dl0_empty_warn = True return True else: return False def _calibrate_dl0(self, event, telid): waveforms = event.r1.tel[telid].waveform selected_gain_channel = event.r1.tel[telid].selected_gain_channel if self._check_r1_empty(waveforms): return reduced_waveforms_mask = self.data_volume_reducer( waveforms, telid=telid, selected_gain_channel=selected_gain_channel) waveforms_copy = waveforms.copy() waveforms_copy[~reduced_waveforms_mask] = 0 event.dl0.tel[telid].waveform = waveforms_copy event.dl0.tel[telid].selected_gain_channel = selected_gain_channel def _calibrate_dl1(self, event, telid): waveforms = event.dl0.tel[telid].waveform selected_gain_channel = event.dl0.tel[telid].selected_gain_channel dl1_calib = event.calibration.tel[telid].dl1 if self._check_dl0_empty(waveforms): return selected_gain_channel = event.r1.tel[telid].selected_gain_channel time_shift = event.calibration.tel[telid].dl1.time_shift readout = self.subarray.tel[telid].camera.readout n_pixels, n_samples = waveforms.shape # subtract any remaining pedestal before extraction if dl1_calib.pedestal_offset is not None: # this copies intentionally, we don't want to modify the dl0 data # waveforms have shape (n_pixel, n_samples), pedestals (n_pixels, ) waveforms = waveforms - dl1_calib.pedestal_offset[:, np.newaxis] if n_samples == 1: # To handle ASTRI and dst # TODO: Improved handling of ASTRI and dst # - dst with custom EventSource? # - Read into dl1 container directly? # - Don't do anything if dl1 container already filled # - Update on SST review decision charge = waveforms[..., 0].astype(np.float32) peak_time = np.zeros(n_pixels, dtype=np.float32) else: # shift waveforms if time_shift calibration is available if time_shift is not None: if self.apply_waveform_time_shift.tel[telid]: sampling_rate = readout.sampling_rate.to_value(u.GHz) time_shift_samples = time_shift * sampling_rate waveforms, remaining_shift = shift_waveforms( waveforms, time_shift_samples) remaining_shift /= sampling_rate else: remaining_shift = time_shift extractor = self.image_extractors[ self.image_extractor_type.tel[telid]] charge, peak_time = extractor( waveforms, telid=telid, selected_gain_channel=selected_gain_channel) # correct non-integer remainder of the shift if given if self.apply_peak_time_shift.tel[telid] and time_shift is not None: peak_time -= remaining_shift # Calibrate extracted charge charge *= dl1_calib.relative_factor / dl1_calib.absolute_factor event.dl1.tel[telid].image = charge event.dl1.tel[telid].peak_time = peak_time def __call__(self, event): """ Perform the full camera calibration from R1 to DL1. Any calibration relating to data levels before the data level the file is read into will be skipped. Parameters ---------- event : container A `~ctapipe.containers.ArrayEventContainer` event container """ # TODO: How to handle different calibrations depending on telid? tel = event.r1.tel or event.dl0.tel or event.dl1.tel for telid in tel.keys(): self._calibrate_dl0(event, telid) self._calibrate_dl1(event, telid)
class LSTR0Corrections(TelescopeComponent): """ The base R0-level calibrator. Changes the r0 container. The R0 calibrator performs the camera-specific R0 calibration that is usually performed on the raw data by the camera server. This calibrator exists in lstchain for testing and prototyping purposes. """ offset = IntTelescopeParameter( default_value=0, help=( 'Define offset to be subtracted from the waveform *additionally*' ' to the drs4 pedestal offset. This only needs to be given when' ' the drs4 pedestal calibration is not applied or the offset of the' ' drs4 run is different from the data run' ) ).tag(config=True) r1_sample_start = IntTelescopeParameter( default_value=3, help='Start sample for r1 waveform', allow_none=True, ).tag(config=True) r1_sample_end = IntTelescopeParameter( default_value=39, help='End sample for r1 waveform', allow_none=True, ).tag(config=True) drs4_pedestal_path = TelescopeParameter( trait=Path(exists=True, directory_ok=False), allow_none=True, default_value=None, help=( 'Path to the LST pedestal file' ', required when `apply_drs4_pedestal_correction=True`' ), ).tag(config=True) calibration_path = Path( exists=True, directory_ok=False, help='Path to LST calibration file', ).tag(config=True) drs4_time_calibration_path = TelescopeParameter( trait=Path(exists=True, directory_ok=False), help='Path to the time calibration file', default_value=None, allow_none=True, ).tag(config=True) calib_scale_high_gain = FloatTelescopeParameter( default_value=1.0, help='High gain waveform is multiplied by this number' ).tag(config=True) calib_scale_low_gain = FloatTelescopeParameter( default_value=1.0, help='Low gain waveform is multiplied by this number' ).tag(config=True) select_gain = Bool( default_value=True, help='Set to False to keep both gains.' ).tag(config=True) apply_drs4_pedestal_correction = Bool( default_value=True, help=( 'Set to False to disable drs4 pedestal correction.' ' Providing the drs4_pedestal_path is required to perform this calibration' ), ).tag(config=True) apply_timelapse_correction = Bool( default_value=True, help='Set to False to disable drs4 timelapse correction' ).tag(config=True) apply_spike_correction = Bool( default_value=True, help='Set to False to disable drs4 spike correction' ).tag(config=True) add_calibration_timeshift = Bool( default_value=True, help=( 'If true, time correction from the calibration' ' file is added to calibration.dl1.time' ), ).tag(config=True) gain_selection_threshold = Float( default_value=3500, help='Threshold for the ThresholdGainSelector.' ).tag(config=True) def __init__(self, subarray, config=None, parent=None, **kwargs): """ The R0 calibrator for LST data. Fill the r1 container. Parameters ---------- """ super().__init__( subarray=subarray, config=config, parent=parent, **kwargs ) self.mon_data = None self.last_readout_time = {} self.first_cap = {} self.first_cap_old = {} self.fbn = {} self.fan = {} for tel_id in self.subarray.tel: shape = (N_GAINS, N_PIXELS, N_CAPACITORS_PIXEL) self.last_readout_time[tel_id] = np.zeros(shape, dtype='uint64') shape = (N_GAINS, N_PIXELS) self.first_cap[tel_id] = np.zeros(shape, dtype=int) self.first_cap_old[tel_id] = np.zeros(shape, dtype=int) if self.select_gain: self.gain_selector = ThresholdGainSelector( threshold=self.gain_selection_threshold, parent=self ) else: self.gain_selector = None if self.calibration_path is not None: self.mon_data = self._read_calibration_file(self.calibration_path) def apply_drs4_corrections(self, event: LSTArrayEventContainer): self.update_first_capacitors(event) for tel_id, r0 in event.r0.tel.items(): r1 = event.r1.tel[tel_id] # If r1 was not yet filled, copy of r0 converted if r1.waveform is None: r1.waveform = r0.waveform # float32 can represent all values of uint16 exactly, # so this does not loose precision. r1.waveform = r1.waveform.astype(np.float32, copy=False) # apply drs4 corrections if self.apply_drs4_pedestal_correction: self.subtract_pedestal(event, tel_id) if self.apply_timelapse_correction: self.time_lapse_corr(event, tel_id) if self.apply_spike_correction: self.interpolate_spikes(event, tel_id) # remove samples at beginning / end of waveform start = self.r1_sample_start.tel[tel_id] end = self.r1_sample_end.tel[tel_id] r1.waveform = r1.waveform[..., start:end] if self.offset.tel[tel_id] != 0: r1.waveform -= self.offset.tel[tel_id] mon = event.mon.tel[tel_id] if r1.selected_gain_channel is None: r1.waveform[mon.pixel_status.hardware_failing_pixels] = 0.0 else: broken = mon.pixel_status.hardware_failing_pixels[r1.selected_gain_channel, PIXEL_INDEX] r1.waveform[broken] = 0.0 def update_first_capacitors(self, event: LSTArrayEventContainer): for tel_id, lst in event.lst.tel.items(): self.first_cap_old[tel_id] = self.first_cap[tel_id] self.first_cap[tel_id] = get_first_capacitors_for_pixels( lst.evt.first_capacitor_id, lst.svc.pixel_ids, ) def calibrate(self, event: LSTArrayEventContainer): for tel_id in event.r0.tel: r1 = event.r1.tel[tel_id] # if `apply_drs4_corrections` is False, we did not fill in the # waveform yet. if r1.waveform is None: r1.waveform = event.r0.tel[tel_id].waveform r1.waveform = r1.waveform.astype(np.float32, copy=False) # do gain selection before converting to pe # like eventbuilder will do if self.select_gain and r1.selected_gain_channel is None: r1.selected_gain_channel = self.gain_selector(r1.waveform) r1.waveform = r1.waveform[r1.selected_gain_channel, PIXEL_INDEX] # apply monitoring data corrections, # subtract pedestal and convert to pe if self.mon_data is not None: calibration = self.mon_data.tel[tel_id].calibration convert_to_pe( waveform=r1.waveform, calibration=calibration, selected_gain_channel=r1.selected_gain_channel ) broken_pixels = event.mon.tel[tel_id].pixel_status.hardware_failing_pixels if r1.selected_gain_channel is None: r1.waveform[broken_pixels] = 0.0 else: r1.waveform[broken_pixels[r1.selected_gain_channel, PIXEL_INDEX]] = 0.0 # store calibration data needed for dl1 calibration in ctapipe # first drs4 time shift (zeros if no calib file was given) time_shift = self.get_drs4_time_correction( tel_id, self.first_cap[tel_id], selected_gain_channel=r1.selected_gain_channel, ) # time shift from flat fielding if self.mon_data is not None and self.add_calibration_timeshift: time_corr = self.mon_data.tel[tel_id].calibration.time_correction # time_shift is subtracted in ctapipe, # but time_correction should be added if r1.selected_gain_channel is not None: time_shift -= time_corr[r1.selected_gain_channel, PIXEL_INDEX].to_value(u.ns) else: time_shift -= time_corr.to_value(u.ns) event.calibration.tel[tel_id].dl1.time_shift = time_shift # needed for charge scaling in ctpaipe dl1 calib if r1.selected_gain_channel is not None: relative_factor = np.empty(N_PIXELS) relative_factor[r1.selected_gain_channel == HIGH_GAIN] = self.calib_scale_high_gain.tel[tel_id] relative_factor[r1.selected_gain_channel == LOW_GAIN] = self.calib_scale_low_gain.tel[tel_id] else: relative_factor = np.empty((N_GAINS, N_PIXELS)) relative_factor[HIGH_GAIN] = self.calib_scale_high_gain.tel[tel_id] relative_factor[LOW_GAIN] = self.calib_scale_low_gain.tel[tel_id] event.calibration.tel[tel_id].dl1.relative_factor = relative_factor @staticmethod def _read_calibration_file(path): """ Read the correction from hdf5 calibration file """ mon = MonitoringContainer() with tables.open_file(path) as f: tel_ids = [ int(key[4:]) for key in f.root._v_children.keys() if key.startswith('tel_') ] for tel_id in tel_ids: with HDF5TableReader(path) as h5_table: base = f'/tel_{tel_id}' # read the calibration data table = base + '/calibration' next(h5_table.read(table, mon.tel[tel_id].calibration)) # read pedestal data table = base + '/pedestal' next(h5_table.read(table, mon.tel[tel_id].pedestal)) # read flat-field data table = base + '/flatfield' next(h5_table.read(table, mon.tel[tel_id].flatfield)) # read the pixel_status container table = base + '/pixel_status' next(h5_table.read(table, mon.tel[tel_id].pixel_status)) return mon @staticmethod def load_drs4_time_calibration_file(path): """ Function to load calibration file. """ with tables.open_file(path, 'r') as f: fan = f.root.fan[:] fbn = f.root.fbn[:] return fan, fbn def load_drs4_time_calibration_file_for_tel(self, tel_id): self.fan[tel_id], self.fbn[tel_id] = self.load_drs4_time_calibration_file( self.drs4_time_calibration_path.tel[tel_id] ) def get_drs4_time_correction(self, tel_id, first_capacitors, selected_gain_channel=None): """ Return pulse time after time correction. """ if self.drs4_time_calibration_path.tel[tel_id] is None: if selected_gain_channel is None: return np.zeros((N_GAINS, N_PIXELS)) else: return np.zeros(N_PIXELS) # load calib file if not already done if tel_id not in self.fan: self.load_drs4_time_calibration_file_for_tel(tel_id) if selected_gain_channel is not None: return calc_drs4_time_correction_gain_selected( first_capacitors, selected_gain_channel, self.fan[tel_id], self.fbn[tel_id], ) else: return calc_drs4_time_correction_both_gains( first_capacitors, self.fan[tel_id], self.fbn[tel_id], ) @staticmethod @lru_cache(maxsize=4) def _get_drs4_pedestal_data(path): """ Function to load pedestal file. To make boundary conditions unnecessary, the first N_SAMPLES values are repeated at the end of the array The result is cached so we can repeatedly call this method using the configured path without reading it each time. """ if path is None: raise ValueError( "DRS4 pedestal correction requested" " but no file provided for telescope" ) pedestal_data = np.empty( (N_GAINS, N_PIXELS_MODULE * N_MODULES, N_CAPACITORS_PIXEL + N_SAMPLES), dtype=np.int16 ) with fits.open(path) as f: pedestal_data[:, :, :N_CAPACITORS_PIXEL] = f[1].data pedestal_data[:, :, N_CAPACITORS_PIXEL:] = pedestal_data[:, :, :N_SAMPLES] return pedestal_data def subtract_pedestal(self, event, tel_id): """ Subtract cell offset using pedestal file. Fill the R1 container. Parameters ---------- event : `ctapipe` event-container tel_id : id of the telescope """ pedestal = self._get_drs4_pedestal_data( self.drs4_pedestal_path.tel[tel_id] ) if event.r1.tel[tel_id].selected_gain_channel is None: subtract_pedestal( event.r1.tel[tel_id].waveform, self.first_cap[tel_id], pedestal, ) else: subtract_pedestal_gain_selected( event.r1.tel[tel_id].waveform, self.first_cap[tel_id], pedestal, event.r1.tel[tel_id].selected_gain_channel, ) def time_lapse_corr(self, event, tel_id): """ Perform time lapse baseline corrections. Fill the R1 container or modifies R0 container. Parameters ---------- event : `ctapipe` event-container tel_id : id of the telescope """ lst = event.lst.tel[tel_id] # If R1 container exists, update it inplace if isinstance(event.r1.tel[tel_id].waveform, np.ndarray): container = event.r1.tel[tel_id] else: # Modify R0 container. This is to create pedestal files. container = event.r0.tel[tel_id] waveform = container.waveform.copy() # We have 2 functions: one for data from 2018/10/10 to 2019/11/04 and # one for data from 2019/11/05 (from Run 1574) after update firmware. # The old readout (before 2019/11/05) is shifted by 1 cell. run_id = event.lst.tel[tel_id].svc.configuration_id # not yet gain selected if event.r1.tel[tel_id].selected_gain_channel is None: apply_timelapse_correction( waveform=waveform, local_clock_counter=lst.evt.local_clock_counter, first_capacitors=self.first_cap[tel_id], last_readout_time=self.last_readout_time[tel_id], expected_pixels_id=lst.svc.pixel_ids, run_id=run_id, ) else: apply_timelapse_correction_gain_selected( waveform=waveform, local_clock_counter=lst.evt.local_clock_counter, first_capacitors=self.first_cap[tel_id], last_readout_time=self.last_readout_time[tel_id], expected_pixels_id=lst.svc.pixel_ids, selected_gain_channel=event.r1.tel[tel_id].selected_gain_channel, run_id=run_id, ) container.waveform = waveform def interpolate_spikes(self, event, tel_id): """ Interpolates spike A & B. Fill the R1 container. Parameters ---------- event : `ctapipe` event-container tel_id : id of the telescope """ run_id = event.lst.tel[tel_id].svc.configuration_id r1 = event.r1.tel[tel_id] if r1.selected_gain_channel is None: interpolate_spikes( waveform=r1.waveform, first_capacitors=self.first_cap[tel_id], previous_first_capacitors=self.first_cap_old[tel_id], run_id=run_id, ) else: interpolate_spikes_gain_selected( waveform=r1.waveform, first_capacitors=self.first_cap[tel_id], previous_first_capacitors=self.first_cap_old[tel_id], selected_gain_channel=r1.selected_gain_channel, run_id=run_id, )
class EventTimeCalculator(TelescopeComponent): ''' Class to calculate event times from low-level counter information. Also keeps track of "UCTS jumps", where UCTS info goes missing for a certain event and all following info has to be shifted. There are several sources of timing information in LST raw data. Each dragon module has two high precision counters, which however only give a relative time. Same is true for the TIB. The only precise absolute timestamp is the UCTS timestamp. However, at least during the commissioning, UCTS was/is not reliable enough to only use the UCTS timestamp. Instead, we calculate an absolute timestamp by using one valid pair of dragon counter / ucts timestamp and then use the relative time elapsed from this reference using the dragon counter. For runs where no such UCTS reference exists, for example because UCTS was completely unavailable, we use the start of run timestamp from the camera configuration. This will however result in imprecises timestamps off by several seconds. These might be good enough for interpolating pointing information but are only precise for relative time changes, i.e. not suitable for pulsar analysis or matching events with MAGIC. Extracting the reference will only work reliably for the first subrun for ucts. Using svc.date is only possible for the first subrun and will raise an erorr if the event id of the first event seen by the time calculator is not 1. ''' timestamp = TelescopeParameter( trait=Enum(['ucts', 'dragon']), default_value='dragon', help= ('Source of the timestamp. UCTS is simplest and most precise,' ' unfortunately it is not yet reliable, instead the time is calculated' ' by default using the relative dragon board counters with a reference' ' pair of counter / time. See the `dragon_reference_time` and' ' `dragon_reference_counter` traitlets')).tag(config=True) dragon_reference_time = TelescopeParameter( Int(allow_none=True), default_value=None, help='Reference timestamp for the dragon time calculation in ns').tag( config=True) dragon_reference_counter = TelescopeParameter( Int(allow_none=True), help= 'Dragon board counter value of a valid ucts/dragon counter combination', default_value=None, ).tag(config=True) dragon_module_id = TelescopeParameter( Int(allow_none=True), default_value=None, help='Module id used to calculate dragon time.', ).tag(config=True) run_summary_path = TelescopeParameter( Path(exists=True, directory_ok=False), default_value=None, help=('Path to the run summary for the correct night.' ' If given, dragon reference counters are read from this file.' ' Explicitly given values override values read from the file.' )).tag(config=True) extract_reference = Bool( default_value=True, help=( 'If true, extract the reference values from the first event.' 'This will only work for the first file of a run, due to the ' 'UCTS jumps when UCTS is available or because svc.date gives only ' 'the start of the run, not the start of each file (subrun) ')).tag( config=True) def __init__(self, subarray, run_id, expected_modules_id, config=None, parent=None, **kwargs): '''Initialize EventTimeCalculator''' super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.previous_ucts_timestamps = defaultdict(deque) self.previous_ucts_trigger_types = defaultdict(deque) # we cannot __setitem__ telescope lookup values, so we store them # in non-trait private values self._has_dragon_reference = {} self._dragon_reference_time = {} self._dragon_reference_counter = {} self._dragon_module_index = {} self.detected_jumps = defaultdict(list) for tel_id in self.subarray.tel: if self.run_summary_path.tel[tel_id] is not None: run_summary = read_run_summary( self.run_summary_path.tel[tel_id]) row = run_summary.loc[run_id] self._has_dragon_reference[tel_id] = True self._dragon_reference_time[tel_id] = np.uint64( row['dragon_reference_time']) self._dragon_reference_counter[tel_id] = np.uint64( row['dragon_reference_counter']) self._dragon_module_index[tel_id] = row[ 'dragon_reference_module_index'] if row['dragon_reference_source'] == 'run_start': self.log.warning( 'Dragon reference source is run_start, ' 'times will be imprecise by several seconds') else: self._has_dragon_reference[tel_id] = ( self.dragon_reference_time.tel[tel_id] is not None and self.dragon_reference_counter.tel[tel_id] is not None and self.dragon_module_id.tel[tel_id] is not None) if not self._has_dragon_reference[ tel_id] and not self.extract_reference: raise ValueError( 'No dragon reference values given and extract_reference=False' ) # set values from traitlets, overrides values from files if both given if self.dragon_reference_counter.tel[tel_id] is not None: self._dragon_reference_counter[tel_id] = np.uint64( self.dragon_reference_counter.tel[tel_id]) if self.dragon_reference_time.tel[tel_id] is not None: self._dragon_reference_time[tel_id] = np.uint64( self.dragon_reference_time.tel[tel_id]) if self.dragon_module_id.tel[tel_id] is not None: module_id = self.dragon_module_id.tel[tel_id] module_index = module_id_to_index(expected_modules_id, module_id) self._dragon_module_index[tel_id] = module_index def __call__(self, tel_id, event): lst = event.lst.tel[tel_id] ucts_available = bool(lst.evt.extdevices_presence & 2) ucts_timestamp = lst.evt.ucts_timestamp # first event and values not passed if self.extract_reference and not self._has_dragon_reference[tel_id]: # use first working module if none is specified if tel_id not in self._dragon_module_index: self._dragon_module_index[tel_id] = np.where( lst.evt.module_status != 0)[0][0] module_index = self._dragon_module_index[tel_id] self._dragon_reference_counter[tel_id] = combine_counters( lst.evt.pps_counter[module_index], lst.evt.tenMHz_counter[module_index]) if not ucts_available: source = 'svc.date' if event.index.event_id != 1: raise ValueError('Can only use run start timestamp' ' as reference for the first subrun') self.log.warning( f'Cannot calculate a precise timestamp for obs_id={event.index.obs_id}' f', tel_id={tel_id}. UCTS unavailable.') # convert runstart from UTC to tai run_start = Time(lst.svc.date, format='unix') self._dragon_reference_time[tel_id] = np.uint64( S_TO_NS * run_start.unix_tai) else: source = 'ucts' self._dragon_reference_time[tel_id] = ucts_timestamp if event.index.event_id != 1: self.log.warning( 'Calculating time reference values not from first event.' ' This might result in wrong timestamps due to UCTS jumps' ) self.log.critical( f'Using event {event.index.event_id} as time reference for dragon.' f' timestamp: {self._dragon_reference_time[tel_id]} from {source}' f' counter: {self._dragon_reference_counter[tel_id]}') self._has_dragon_reference[tel_id] = True # Dragon timestamp based on the reference timestamp module_index = self._dragon_module_index[tel_id] dragon_timestamp = calc_dragon_time( pps_counter=lst.evt.pps_counter[module_index], tenMHz_counter=lst.evt.tenMHz_counter[module_index], reference_time=self._dragon_reference_time[tel_id], reference_counter=self._dragon_reference_counter[tel_id], ) # if ucts is not available, there is nothing more we have to do # and dragon time is our only option if not ucts_available: return time_from_unix_tai_ns(dragon_timestamp) # Due to a DAQ bug, sometimes there are 'jumps' in the # UCTS info in the raw files. After one such jump, # all the UCTS info attached to an event actually # corresponds to the next event. This one-event # shift stays like that until there is another jump # (then it becomes a 2-event shift and so on). We will # keep track of those jumps, by storing the UCTS info # of the previously read events in the list # previous_ucts_time_unix. The list has one element # for each of the jumps, so if there has been just # one jump we have the UCTS info of the previous # event only (which truly corresponds to the # current event). If there have been n jumps, we keep # the past n events. The info to be used for # the current event is always the first element of # the array, previous_ucts_time_unix[0], whereas the # current event's (wrong) ucts info is placed last in # the array. Each time the first array element is # used, it is removed and the rest move up in the # list. We have another similar array for the trigger # types, previous_ucts_trigger_type ucts_trigger_type = lst.evt.ucts_trigger_type if len(self.previous_ucts_timestamps[tel_id]) > 0: # put the current values last in the queue, for later use: self.previous_ucts_timestamps[tel_id].append(ucts_timestamp) self.previous_ucts_trigger_types[tel_id].append(ucts_trigger_type) # get the correct time for the current event from the queue ucts_timestamp = self.previous_ucts_timestamps[tel_id].popleft() ucts_trigger_type = self.previous_ucts_trigger_types[ tel_id].popleft() lst.evt.ucts_trigger_type = ucts_trigger_type lst.evt.ucts_timestamp = ucts_timestamp # Now check consistency of UCTS and Dragon times. If # UCTS time is ahead of Dragon time by more than # 1 us, most likely the UCTS info has been # lost for this event (i.e. there has been another # 'jump' of those described above), and the one we have # actually corresponds to the next event. So we put it # back first in the list, to assign it to the next # event. We also move the other elements down in the # list, which will now be one element longer. # We leave the current event with the same time, # which will be approximately correct (depending on # event rate), and set its ucts_trigger_type to -1, # which will tell us a jump happened and hence this # event does not have proper UCTS info. delta = abs_diff(ucts_timestamp, dragon_timestamp) if delta > 1e3: self.log.warning(f'Found UCTS jump in event {event.index.event_id}' f', dragon time: {dragon_timestamp:d}' f', delta: {delta / 1000:.0f} µs') self.previous_ucts_timestamps[tel_id].appendleft(ucts_timestamp) self.previous_ucts_trigger_types[tel_id].appendleft( ucts_trigger_type) self.detected_jumps[tel_id].append( (event.count, event.index.event_id, delta)) lst.evt.ucts_jump = True # fall back to dragon time / tib trigger lst.evt.ucts_timestamp = dragon_timestamp ucts_timestamp = dragon_timestamp tib_available = lst.evt.extdevices_presence & 1 if tib_available: lst.evt.ucts_trigger_type = lst.evt.tib_masked_trigger else: self.log.warning( 'Detected ucts jump but not tib trigger info available' ', event will have no trigger information') lst.evt.ucts_trigger_type = 0 # Select the timestamps to be used for pointing interpolation if self.timestamp.tel[tel_id] == "dragon": return time_from_unix_tai_ns(dragon_timestamp) return time_from_unix_tai_ns(ucts_timestamp)
class SomeComponent(TelescopeComponent): path = TelescopeParameter(Path(allow_none=True, default_value=None), default_value=None).tag(config=True) val = TelescopeParameter(Float(), default_value=1.0).tag(config=True) flag = TelescopeParameter(Bool(), default_value=True).tag(config=True)
class PointingSource(TelescopeComponent): """Provides access to pointing information stored in LST drive reports.""" drive_report_path = TelescopeParameter( trait=Path(exists=True, directory_ok=False), help='Path to the LST drive report file', default_value=None, ).tag(config=True) def __init__(self, subarray, config=None, parent=None, **kwargs): '''Initialize PointingSource''' super().__init__(subarray, config=config, parent=parent, **kwargs) self.drive_report = {} self.interp_az = {} self.interp_alt = {} self.interp_ra = {} self.interp_dec = {} @staticmethod def _read_drive_report(path): """ Read a drive report into an astropy table Parameters: ----------- str: drive report file Returns: data:`~astropy.table.Table` A table of drive reports """ data = Table.read( path, format='ascii', delimiter=' ', header_start=None, data_start=0, names=[ 'weekday', 'month', 'day', 'time', 'year', 'unix_time', 'Az', 'azimuth_avg', 'azimuth_min', 'azimuth_max', 'azimuth_std', 'El', 'zenith_avg', 'zenith_min', 'zenith_max', 'zenith_std', 'Ra', 'target_ra', 'Dec', 'target_dec', ] ) return data def _read_drive_report_for_tel(self, tel_id): path = self.drive_report_path.tel[tel_id] if path is None: raise ValueError(f'No drive report given for telescope {tel_id}') self.log.info(f'Loading drive report "{path}" for tel_id={tel_id}') self.drive_report[tel_id] = self._read_drive_report(path) self.interp_az[tel_id] = interp1d( self.drive_report[tel_id]['unix_time'], self.drive_report[tel_id]['azimuth_avg'], ) self.interp_alt[tel_id] = interp1d( self.drive_report[tel_id]['unix_time'], 90 - self.drive_report[tel_id]['zenith_avg'], ) self.interp_ra[tel_id] = interp1d( self.drive_report[tel_id]['unix_time'], self.drive_report[tel_id]['target_ra'], ) self.interp_dec[tel_id] = interp1d( self.drive_report[tel_id]['unix_time'], self.drive_report[tel_id]['target_dec'], ) def get_pointing_position_altaz(self, tel_id, time): """ Calculating pointing positions by interpolation Parameters: ----------- time: array times from events Drivereport: Container a container filled with drive information """ if tel_id not in self.drive_report: self._read_drive_report_for_tel(tel_id) alt = u.Quantity(self.interp_alt[tel_id](time.unix), u.deg) az = u.Quantity(self.interp_az[tel_id](time.unix), u.deg) return TelescopePointingContainer( altitude=alt.to(u.rad), azimuth=az.to(u.rad), ) def get_pointing_position_icrs(self, tel_id, time): if tel_id not in self.drive_report: self._read_drive_report_for_tel(tel_id) ra = u.Quantity(self.interp_ra[tel_id](time.unix), u.deg) dec = u.Quantity(self.interp_dec[tel_id](time.unix), u.deg) # drive reports contain 0 / 0 if not tracking ICRS coordinates # TODO: hope we never really observe ra=0°, dec=0° if ra != 0.0 and dec != 0.0: return ra, dec return NAN_ANGLE, NAN_ANGLE