class ChargeResolutionViewer(Tool): name = "ChargeResolutionViewer" description = "Plot charge resolutions generated by " "ChargeResolutionCalculator." input_files = List( Path(exists=True, directory_ok=False), None, help="Input HDF5 files produced by ChargeResolutionCalculator", ).tag(config=True) aliases = Dict( dict( f="ChargeResolutionViewer.input_files", B="ChargeResolutionPlotter.n_bins", o="ChargeResolutionPlotter.output_path", )) classes = List([ChargeResolutionPlotter]) def __init__(self, **kwargs): super().__init__(**kwargs) self.plotter = None def setup(self): self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]" self.plotter = ChargeResolutionPlotter(parent=self) def start(self): for fp in self.input_files: self.plotter.plot_camera(fp) def finish(self): q = np.arange(1, 1000) self.plotter.plot_poisson(q) self.plotter.plot_requirement(q) self.plotter.save()
class SomeComponent(TelescopeComponent): path = TelescopeParameter( Path(exists=True, directory_ok=False, allow_none=True, default_value=None), default_value=None, allow_none=True, )
class SingleTelEventDisplay(Tool): name = "ctapipe-display-televents" description = Unicode(__doc__) infile = Path(help="input file to read", exists=True, directory_ok=False).tag(config=True) tel = Int(help="Telescope ID to display", default_value=0).tag(config=True) write = Bool(help="Write out images to PNG files", default_value=False).tag(config=True) clean = Bool(help="Apply image cleaning", default_value=False).tag(config=True) hillas = Bool(help="Apply and display Hillas parametrization", default_value=False).tag(config=True) samples = Bool(help="Show each sample", default_value=False).tag(config=True) display = Bool(help="Display results in interactive window", default_value=True).tag(config=True) delay = Float(help="delay between events in s", default_value=0.01, min=0.001).tag(config=True) progress = Bool(help="display progress bar", default_value=True).tag(config=True) aliases = Dict({ "infile": "SingleTelEventDisplay.infile", "tel": "SingleTelEventDisplay.tel", "max-events": "EventSource.max_events", "write": "SingleTelEventDisplay.write", "clean": "SingleTelEventDisplay.clean", "hillas": "SingleTelEventDisplay.hillas", "samples": "SingleTelEventDisplay.samples", "display": "SingleTelEventDisplay.display", "delay": "SingleTelEventDisplay.delay", "progress": "SingleTelEventDisplay.progress", }) classes = List([EventSource, CameraCalibrator]) def __init__(self, **kwargs): super().__init__(**kwargs) def setup(self): print("TOLLES INFILE", self.infile) self.event_source = EventSource.from_url(self.infile, parent=self) self.event_source.allowed_tels = {self.tel} self.calibrator = CameraCalibrator(parent=self, subarray=self.event_source.subarray) self.log.info(f"SELECTING EVENTS FROM TELESCOPE {self.tel}") def start(self): disp = None for event in tqdm( self.event_source, desc=f"Tel{self.tel}", total=self.event_source.max_events, disable=~self.progress, ): self.log.debug(event.trigger) self.log.debug(f"Energy: {event.simulation.shower.energy}") self.calibrator(event) if disp is None: geom = self.event_source.subarray.tel[self.tel].camera.geometry self.log.info(geom) disp = CameraDisplay(geom) # disp.enable_pixel_picker() disp.add_colorbar() if self.display: plt.show(block=False) # display the event disp.axes.set_title("CT{:03d} ({}), event {:06d}".format( self.tel, geom.camera_name, event.index.event_id)) if self.samples: # display time-varying event data = event.dl0.tel[self.tel].waveform for ii in range(data.shape[1]): disp.image = data[:, ii] disp.set_limits_percent(70) plt.suptitle(f"Sample {ii:03d}") if self.display: plt.pause(self.delay) if self.write: plt.savefig( f"CT{self.tel:03d}_EV{event.index.event_id:10d}" f"_S{ii:02d}.png") else: # display integrated event: im = event.dl1.tel[self.tel].image if self.clean: mask = tailcuts_clean(geom, im, picture_thresh=10, boundary_thresh=7) im[~mask] = 0.0 disp.image = im if self.hillas: try: ellipses = disp.axes.findobj(Ellipse) if len(ellipses) > 0: ellipses[0].remove() params = hillas_parameters(geom, image=im) disp.overlay_moments(params, color="pink", lw=3, with_label=False) except HillasParameterizationError: pass if self.display: plt.pause(self.delay) if self.write: plt.savefig( f"CT{self.tel:03d}_EV{event.index.event_id:010d}.png") self.log.info("FINISHED READING DATA FILE") if disp is None: self.log.warning( "No events for tel {} were found in {}. Try a " "different EventIO file or another telescope".format( self.tel, self.infile))
class DumpInstrumentTool(Tool): description = Unicode(__doc__) name = "ctapipe-dump-instrument" infile = Path(exists=True, help="input simtelarray file").tag(config=True) format = Enum( ["fits", "ecsv", "hdf5"], default_value="fits", help="Format of output file", config=True, ) aliases = Dict( dict(infile="DumpInstrumentTool.infile", format="DumpInstrumentTool.format")) def setup(self): with event_source(self.infile) as source: self.subarray = source.subarray def start(self): if self.format == "hdf5": self.subarray.to_hdf("subarray.h5") else: self.write_camera_geometries() self.write_optics_descriptions() self.write_subarray_description() def finish(self): pass @staticmethod def _get_file_format_info(format_name, table_type, table_name): """ returns file extension + dict of required parameters for Table.write""" if format_name == "fits": return "fits.gz", dict() elif format_name == "ecsv": return "ecsv.txt", dict(format="ascii.ecsv") else: raise NameError(f"format {format_name} not supported") def write_camera_geometries(self): cam_types = get_camera_types(self.subarray) self.subarray.info(printer=self.log.info) for cam_name in cam_types: ext, args = self._get_file_format_info(self.format, "CAMGEOM", cam_name) self.log.debug(f"writing {cam_name}") tel_id = cam_types[cam_name].pop() geom = self.subarray.tel[tel_id].camera.geometry table = geom.to_table() table.meta["SOURCE"] = str(self.infile) filename = f"{cam_name}.camgeom.{ext}" try: table.write(filename, **args) Provenance().add_output_file(filename, "dl0.tel.svc.camera") except IOError as err: self.log.warning( "couldn't write camera definition '%s' because: %s", filename, err) def write_optics_descriptions(self): sub = self.subarray ext, args = self._get_file_format_info(self.format, sub.name, "optics") tab = sub.to_table(kind="optics") tab.meta["SOURCE"] = str(self.infile) filename = f"{sub.name}.optics.{ext}" try: tab.write(filename, **args) Provenance().add_output_file(filename, "dl0.sub.svc.optics") except IOError as err: self.log.warning( "couldn't write optics description '%s' because: %s", filename, err) def write_subarray_description(self): sub = self.subarray ext, args = self._get_file_format_info(self.format, sub.name, "subarray") tab = sub.to_table(kind="subarray") tab.meta["SOURCE"] = str(self.infile) filename = f"{sub.name}.subarray.{ext}" try: tab.write(filename, **args) Provenance().add_output_file(filename, "dl0.sub.svc.subarray") except IOError as err: self.log.warning( "couldn't write subarray description '%s' because: %s", filename, err)
class CalibrationCalculator(Component): """ Parent class for the camera calibration calculators. Fills the MonitoringCameraContainer on the base of calibration events Parameters ---------- flatfield_calculator: lstchain.calib.camera.flatfield The flatfield to use. If None, then FlatFieldCalculator will be used by default. pedestal_calculator: lstchain.calib.camera.pedestal The pedestal to use. If None, then PedestalCalculator will be used by default. kwargs """ systematic_correction_path = Path( default_value=None, allow_none=True, exists=True, directory_ok=False, help='Path to systematic correction file ', ).tag(config=True) squared_excess_noise_factor = Float( 1.222, help='Excess noise factor squared: 1+ Var(gain)/Mean(Gain)**2').tag( config=True) pedestal_product = traits.create_class_enum_trait( PedestalCalculator, default_value='PedestalIntegrator') flatfield_product = traits.create_class_enum_trait( FlatFieldCalculator, default_value='FlasherFlatFieldCalculator') classes = ([FlatFieldCalculator, PedestalCalculator] + traits.classes_with_traits(FlatFieldCalculator) + traits.classes_with_traits(PedestalCalculator)) def __init__(self, subarray, parent=None, config=None, **kwargs): """ Parent class for the camera calibration calculators. Fills the MonitoringCameraContainer on the base of calibration events Parameters ---------- flatfield_calculator: lstchain.calib.camera.flatfield The flatfield to use. If None, then FlatFieldCalculator will be used by default. pedestal_calculator: lstchain.calib.camera.pedestal The pedestal to use. If None, then PedestalCalculator will be used by default. """ super().__init__(parent=parent, config=config, **kwargs) if self.squared_excess_noise_factor <= 0: msg = "Argument squared_excess_noise_factor must have a positive value" raise ValueError(msg) self.flatfield = FlatFieldCalculator.from_name(self.flatfield_product, parent=self, subarray=subarray) self.pedestal = PedestalCalculator.from_name(self.pedestal_product, parent=self, subarray=subarray) msg = "tel_id not the same for all calibration components" if self.pedestal.tel_id != self.flatfield.tel_id: raise ValueError(msg) self.tel_id = self.flatfield.tel_id # load systematic correction term B self.quadratic_term = 0 if self.systematic_correction_path is not None: try: with h5py.File(self.systematic_correction_path, 'r') as hf: self.quadratic_term = np.array(hf['B_term']) except: raise IOError( f"Problem in reading quadratic term file {self.systematic_correction_path}" ) self.log.debug(f"{self.pedestal}") self.log.debug(f"{self.flatfield}")
class C(Component): thepath = Path(exists=True, directory_ok=False)
class C2(Component): thepath = Path(exists=True)
class SomeComponent(TelescopeComponent): path = TelescopeParameter(Path(), default_value=None).tag(config=True) val = TelescopeParameter(Float(), default_value=1.0).tag(config=True)
class ChargeResolutionPlotter(Component): output_path = Path(help='Output path to save the plot.').tag(config=True) n_bins = Int( 40, help='Number of bins for collecting true charges and combining ' 'their resolution').tag(config=True) def __init__(self, config=None, parent=None, **kwargs): """ Plots the charge resolution HDF5 file obtained from `ctapipe.analysis.camera.charge_resolution`. Also contains the equation from which the charge resolution requirement is defined, which can be plotted alongside the charge resolution curve. `ctapipe.tools.plot_charge_resolution` demonstrated the use of this component. Parameters ---------- config : traitlets.loader.Config Configuration specified by config file or cmdline arguments. Used to set traitlet values. Set to None if no configuration to pass. tool : ctapipe.core.Tool Tool executable that is calling this component. Passes the correct logger to the component. Set to None if no Tool to pass. kwargs """ super().__init__(config=config, parent=parent, **kwargs) self._df_pixel = None self._df_camera = None self.fig = plt.figure(figsize=(12, 7.42)) self.ax = self.fig.add_subplot(111) self.ax.set_xlabel("True Charge (p.e.)") self.ax.set_ylabel( r"Fractional Charge Resolution $\frac{{\sigma_Q}}{{Q}}$") if not self.output_path: raise ValueError("Output path must be specified") def _set_file(self, path): """ Reads the charge resolution DataFrames from the file and assigns it to this class ready for plotting. Parameters ---------- path : str Path to the charge resolution HDF5 file """ with pd.HDFStore(path, 'r') as store: self._df_pixel = store['charge_resolution_pixel'] self._df_camera = store['charge_resolution_camera'] def _plot(self, x, y, **kwargs): """ Plot the given points onto the figure Parameters ---------- x : ndarray y : ndarray xerr : ndarray yerr : ndarray label : str """ defaults = dict(mew=1, capsize=1, elinewidth=0.5, markersize=2, linewidth=0.5, fmt='.') kwargs = {**defaults, **kwargs} (_, caps, _) = self.ax.errorbar(x, y, **kwargs) for cap in caps: cap.set_markeredgewidth(0.5) def plot_average(self, path, label='', **kwargs): """ Plot the average and standard deviation of the charge resolution across the pixels of the camera. Parameters ---------- path : str Path to the charge resolution HDF5 file label : str Label for the figure's legend kwargs """ self._set_file(path) df_binned = bin_dataframe(self._df_pixel, self.n_bins) agg = {'charge_resolution': ['mean', 'std'], 'true': 'mean'} df_agg = df_binned.groupby(['bin']).agg(agg) x = df_agg['true']['mean'].values y = df_agg['charge_resolution']['mean'].values yerr = df_agg['charge_resolution']['std'].values self._plot(x, y, yerr=yerr, label=label, **kwargs) def plot_pixel(self, path, pixel, label='', **kwargs): """ Plot a single pixel's charge resolution. The yerr represents the amount of entries. Parameters ---------- path : str Path to the charge resolution HDF5 file pixel : int Pixel index to plot label : str Label for the figure's legend kwargs """ self._set_file(path) df_p = self._df_pixel.loc[self._df_pixel['pixel'] == pixel] df_binned = bin_dataframe(df_p, self.n_bins) agg = {'charge_resolution': 'mean', 'true': 'mean', 'n': 'sum'} df_agg = df_binned.groupby(['bin']).agg(agg) x = df_agg['true'].values y = df_agg['charge_resolution'].values yerr = 1 / np.sqrt(df_agg['n'].values) self._plot(x, y, yerr=yerr, label=label, **kwargs) def plot_camera(self, path, label='', **kwargs): """ Plot the charge resolution for the entire camera. The yerr represents the amount of entries. Parameters ---------- path : str Path to the charge resolution HDF5 file label : str Label for the figure's legend kwargs """ self._set_file(path) df_binned = bin_dataframe(self._df_camera, self.n_bins) agg = {'charge_resolution': 'mean', 'true': 'mean', 'n': 'sum'} df_agg = df_binned.groupby(['bin']).agg(agg) x = df_agg['true'].values y = df_agg['charge_resolution'].values yerr = 1 / np.sqrt(df_agg['n'].values) self._plot(x, y, yerr=yerr, label=label, **kwargs) def _finish(self): """ Perform the finishing touches to the figure before saving. """ self.ax.set_xscale('log') self.ax.get_xaxis().set_major_formatter( FuncFormatter(lambda x, _: f'{x:g}')) self.ax.set_yscale('log') self.ax.get_yaxis().set_major_formatter( FuncFormatter(lambda y, _: f'{y:g}')) def save(self): """ Save the figure to the path defined by the `output_path` trait """ self._finish() output_dir = os.path.dirname(self.output_path) if not os.path.exists(output_dir): print(f"Creating directory: {output_dir}") os.makedirs(output_dir) self.fig.savefig(self.output_path, bbox_inches='tight') print(f"Figure saved to: {self.output_path}") Provenance().add_output_file(self.output_path) plt.close(self.fig) @staticmethod def limit_curves(q, nsb, t_w, n_e, sigma_g, enf): """ Equation for calculating the Goal and Requirement curves, as defined in SCI-MC/121113. https://portal.cta-observatory.org/recordscentre/Records/SCI/ SCI-MC/measurment_errors_system_performance_1YQCBC.pdf Parameters ---------- q : ndarray Number of photoeletrons (variable). nsb : float Number of NSB photons. t_w : float Effective signal readout window size. n_e : float Electronic noise sigma_g : float Multiplicative errors of the gain. enf : float Excess noise factor. """ sigma_0 = np.sqrt(nsb * t_w + n_e**2) sigma_enf = 1 + enf sigma_q = np.sqrt(sigma_0**2 + sigma_enf**2 * q + sigma_g**2 * q**2) return sigma_q / q @staticmethod def requirement(q): """ CTA requirement curve. Parameters ---------- q : ndarray Number of photoeletrons """ nsb = 0.125 t_w = 15 n_e = 0.87 sigma_g = 0.1 enf = 0.2 defined_npe = 1000 lc = __class__.limit_curves requirement = lc(q, nsb, t_w, n_e, sigma_g, enf) requirement[q > defined_npe] = np.nan return requirement @staticmethod def poisson(q): """ Poisson limit curve. Parameters ---------- q : ndarray Number of photoeletrons """ poisson = np.sqrt(q) / q return poisson def plot_requirement(self, q): """ Plot the CTA requirement curve onto the figure. Parameters ---------- q : ndarray Charges to evaluate the requirement curve at """ req = self.requirement(q) self.ax.plot(q, req, '--', color='black', label="Requirement") def plot_poisson(self, q): """ Plot the Poisson limit curve onto the figure. Parameters ---------- q : ndarray Charges to evaluate the limit at """ poisson = self.poisson(q) self.ax.plot(q, poisson, '--', color='grey', label="Poisson")
class EventTimeCalculator(TelescopeComponent): ''' Class to calculate event times from low-level counter information. Also keeps track of "UCTS jumps", where UCTS info goes missing for a certain event and all following info has to be shifted. There are several sources of timing information in LST raw data. Each dragon module has two high precision counters, which however only give a relative time. Same is true for the TIB. The only precise absolute timestamp is the UCTS timestamp. However, at least during the commissioning, UCTS was/is not reliable enough to only use the UCTS timestamp. Instead, we calculate an absolute timestamp by using one valid pair of dragon counter / ucts timestamp and then use the relative time elapsed from this reference using the dragon counter. For runs where no such UCTS reference exists, for example because UCTS was completely unavailable, we use the start of run timestamp from the camera configuration. This will however result in imprecises timestamps off by several seconds. These might be good enough for interpolating pointing information but are only precise for relative time changes, i.e. not suitable for pulsar analysis or matching events with MAGIC. Extracting the reference will only work reliably for the first subrun for ucts. Using svc.date is only possible for the first subrun and will raise an erorr if the event id of the first event seen by the time calculator is not 1. ''' timestamp = TelescopeParameter( trait=Enum(['ucts', 'dragon']), default_value='dragon', help= ('Source of the timestamp. UCTS is simplest and most precise,' ' unfortunately it is not yet reliable, instead the time is calculated' ' by default using the relative dragon board counters with a reference' ' pair of counter / time. See the `dragon_reference_time` and' ' `dragon_reference_counter` traitlets')).tag(config=True) dragon_reference_time = TelescopeParameter( Int(allow_none=True), default_value=None, help='Reference timestamp for the dragon time calculation in ns').tag( config=True) dragon_reference_counter = TelescopeParameter( Int(allow_none=True), help= 'Dragon board counter value of a valid ucts/dragon counter combination', default_value=None, ).tag(config=True) dragon_module_id = TelescopeParameter( Int(allow_none=True), default_value=None, help='Module id used to calculate dragon time.', ).tag(config=True) run_summary_path = TelescopeParameter( Path(exists=True, directory_ok=False), default_value=None, help=('Path to the run summary for the correct night.' ' If given, dragon reference counters are read from this file.' ' Explicitly given values override values read from the file.' )).tag(config=True) extract_reference = Bool( default_value=True, help=( 'If true, extract the reference values from the first event.' 'This will only work for the first file of a run, due to the ' 'UCTS jumps when UCTS is available or because svc.date gives only ' 'the start of the run, not the start of each file (subrun) ')).tag( config=True) def __init__(self, subarray, run_id, expected_modules_id, config=None, parent=None, **kwargs): '''Initialize EventTimeCalculator''' super().__init__(subarray=subarray, config=config, parent=parent, **kwargs) self.previous_ucts_timestamps = defaultdict(deque) self.previous_ucts_trigger_types = defaultdict(deque) # we cannot __setitem__ telescope lookup values, so we store them # in non-trait private values self._has_dragon_reference = {} self._dragon_reference_time = {} self._dragon_reference_counter = {} self._dragon_module_index = {} self.detected_jumps = defaultdict(list) for tel_id in self.subarray.tel: if self.run_summary_path.tel[tel_id] is not None: run_summary = read_run_summary( self.run_summary_path.tel[tel_id]) row = run_summary.loc[run_id] self._has_dragon_reference[tel_id] = True self._dragon_reference_time[tel_id] = np.uint64( row['dragon_reference_time']) self._dragon_reference_counter[tel_id] = np.uint64( row['dragon_reference_counter']) self._dragon_module_index[tel_id] = row[ 'dragon_reference_module_index'] if row['dragon_reference_source'] == 'run_start': self.log.warning( 'Dragon reference source is run_start, ' 'times will be imprecise by several seconds') else: self._has_dragon_reference[tel_id] = ( self.dragon_reference_time.tel[tel_id] is not None and self.dragon_reference_counter.tel[tel_id] is not None and self.dragon_module_id.tel[tel_id] is not None) if not self._has_dragon_reference[ tel_id] and not self.extract_reference: raise ValueError( 'No dragon reference values given and extract_reference=False' ) # set values from traitlets, overrides values from files if both given if self.dragon_reference_counter.tel[tel_id] is not None: self._dragon_reference_counter[tel_id] = np.uint64( self.dragon_reference_counter.tel[tel_id]) if self.dragon_reference_time.tel[tel_id] is not None: self._dragon_reference_time[tel_id] = np.uint64( self.dragon_reference_time.tel[tel_id]) if self.dragon_module_id.tel[tel_id] is not None: module_id = self.dragon_module_id.tel[tel_id] module_index = module_id_to_index(expected_modules_id, module_id) self._dragon_module_index[tel_id] = module_index def __call__(self, tel_id, event): lst = event.lst.tel[tel_id] ucts_available = bool(lst.evt.extdevices_presence & 2) ucts_timestamp = lst.evt.ucts_timestamp # first event and values not passed if self.extract_reference and not self._has_dragon_reference[tel_id]: # use first working module if none is specified if tel_id not in self._dragon_module_index: self._dragon_module_index[tel_id] = np.where( lst.evt.module_status != 0)[0][0] module_index = self._dragon_module_index[tel_id] self._dragon_reference_counter[tel_id] = combine_counters( lst.evt.pps_counter[module_index], lst.evt.tenMHz_counter[module_index]) if not ucts_available: source = 'svc.date' if event.index.event_id != 1: raise ValueError('Can only use run start timestamp' ' as reference for the first subrun') self.log.warning( f'Cannot calculate a precise timestamp for obs_id={event.index.obs_id}' f', tel_id={tel_id}. UCTS unavailable.') # convert runstart from UTC to tai run_start = Time(lst.svc.date, format='unix') self._dragon_reference_time[tel_id] = np.uint64( S_TO_NS * run_start.unix_tai) else: source = 'ucts' self._dragon_reference_time[tel_id] = ucts_timestamp if event.index.event_id != 1: self.log.warning( 'Calculating time reference values not from first event.' ' This might result in wrong timestamps due to UCTS jumps' ) self.log.critical( f'Using event {event.index.event_id} as time reference for dragon.' f' timestamp: {self._dragon_reference_time[tel_id]} from {source}' f' counter: {self._dragon_reference_counter[tel_id]}') self._has_dragon_reference[tel_id] = True # Dragon timestamp based on the reference timestamp module_index = self._dragon_module_index[tel_id] dragon_timestamp = calc_dragon_time( pps_counter=lst.evt.pps_counter[module_index], tenMHz_counter=lst.evt.tenMHz_counter[module_index], reference_time=self._dragon_reference_time[tel_id], reference_counter=self._dragon_reference_counter[tel_id], ) # if ucts is not available, there is nothing more we have to do # and dragon time is our only option if not ucts_available: return time_from_unix_tai_ns(dragon_timestamp) # Due to a DAQ bug, sometimes there are 'jumps' in the # UCTS info in the raw files. After one such jump, # all the UCTS info attached to an event actually # corresponds to the next event. This one-event # shift stays like that until there is another jump # (then it becomes a 2-event shift and so on). We will # keep track of those jumps, by storing the UCTS info # of the previously read events in the list # previous_ucts_time_unix. The list has one element # for each of the jumps, so if there has been just # one jump we have the UCTS info of the previous # event only (which truly corresponds to the # current event). If there have been n jumps, we keep # the past n events. The info to be used for # the current event is always the first element of # the array, previous_ucts_time_unix[0], whereas the # current event's (wrong) ucts info is placed last in # the array. Each time the first array element is # used, it is removed and the rest move up in the # list. We have another similar array for the trigger # types, previous_ucts_trigger_type ucts_trigger_type = lst.evt.ucts_trigger_type if len(self.previous_ucts_timestamps[tel_id]) > 0: # put the current values last in the queue, for later use: self.previous_ucts_timestamps[tel_id].append(ucts_timestamp) self.previous_ucts_trigger_types[tel_id].append(ucts_trigger_type) # get the correct time for the current event from the queue ucts_timestamp = self.previous_ucts_timestamps[tel_id].popleft() ucts_trigger_type = self.previous_ucts_trigger_types[ tel_id].popleft() lst.evt.ucts_trigger_type = ucts_trigger_type lst.evt.ucts_timestamp = ucts_timestamp # Now check consistency of UCTS and Dragon times. If # UCTS time is ahead of Dragon time by more than # 1 us, most likely the UCTS info has been # lost for this event (i.e. there has been another # 'jump' of those described above), and the one we have # actually corresponds to the next event. So we put it # back first in the list, to assign it to the next # event. We also move the other elements down in the # list, which will now be one element longer. # We leave the current event with the same time, # which will be approximately correct (depending on # event rate), and set its ucts_trigger_type to -1, # which will tell us a jump happened and hence this # event does not have proper UCTS info. delta = abs_diff(ucts_timestamp, dragon_timestamp) if delta > 1e3: self.log.warning(f'Found UCTS jump in event {event.index.event_id}' f', dragon time: {dragon_timestamp:d}' f', delta: {delta / 1000:.0f} µs') self.previous_ucts_timestamps[tel_id].appendleft(ucts_timestamp) self.previous_ucts_trigger_types[tel_id].appendleft( ucts_trigger_type) self.detected_jumps[tel_id].append( (event.count, event.index.event_id, delta)) lst.evt.ucts_jump = True # fall back to dragon time / tib trigger lst.evt.ucts_timestamp = dragon_timestamp ucts_timestamp = dragon_timestamp tib_available = lst.evt.extdevices_presence & 1 if tib_available: lst.evt.ucts_trigger_type = lst.evt.tib_masked_trigger else: self.log.warning( 'Detected ucts jump but not tib trigger info available' ', event will have no trigger information') lst.evt.ucts_trigger_type = 0 # Select the timestamps to be used for pointing interpolation if self.timestamp.tel[tel_id] == "dragon": return time_from_unix_tai_ns(dragon_timestamp) return time_from_unix_tai_ns(ucts_timestamp)
class DRS4PedestalAndSpikeHeight(Tool): name = 'lstchain_create_drs4_pedestal_file' output_path = Path( directory_ok=False, help= 'Path for the output hdf5 file of pedestal baseline and spike heights', ).tag(config=True) skip_samples_front = Integer( default_value=10, help='Do not include first N samples in pedestal calculation').tag( config=True) skip_samples_end = Integer( default_value=1, help='Do not include last N samples in pedestal calculation').tag( config=True) progress_bar = Bool(help="Show progress bar during processing", default_value=True).tag(config=True) full_statistics = Bool(help=( "If True, write spike{1,2,3} mean, count, std for each capacitor." " Otherwise, only mean spike height for each gain, pixel is written"), default_value=False).tag(config=True) overwrite = Bool(help=("If true, overwrite output without asking," " else fail if output file already exists"), default_value=False).tag(config=True) aliases = { ('i', 'input'): 'LSTEventSource.input_url', ('o', 'output'): 'DRS4PedestalAndSpikeHeight.output_path', ('m', 'max-events'): 'LSTEventSource.max_events', } flags = { **flag( "overwrite", "DRS4PedestalAndSpikeHeight.overwrite", "Overwrite output file if it exists", "Fail if output file already exists", ), **flag( "progress", "DRS4PedestalAndSpikeHeight.progress_bar", "Show a progress bar during event processing", "Do not show a progress bar during event processing", ), **flag( "full-statistics", "DRS4PedestalAndSpikeHeight.full_statistics", "Whether to write the full statistics about spikes or not", ), } classes = [LSTEventSource] def setup(self): self.output_path = self.output_path.expanduser().resolve() if self.output_path.exists(): if self.overwrite: self.log.warning("Overwriting %s", self.output_path) self.output_path.unlink() else: raise ToolConfigurationError( f"Output file {self.output_path} exists" ", use the `overwrite` option or choose another `output_path` " ) self.log.debug("output path: %s", self.output_path) Provenance().add_output_file(str(self.output_path), role="DL1/Event") self.source = LSTEventSource( parent=self, pointing_information=False, trigger_information=False, ) # set some config options, these are necessary for this tool, # so we set them here and not via the config system self.source.r0_r1_calibrator.r1_sample_start = 0 self.source.r0_r1_calibrator.r1_sample_end = N_SAMPLES self.source.r0_r1_calibrator.offset = 0 self.source.r0_r1_calibrator.apply_spike_correction = False self.source.r0_r1_calibrator.apply_timelapse_correction = True self.source.r0_r1_calibrator.apply_drs4_pedestal_correction = False n_stats = N_GAINS * N_PIXELS * N_CAPACITORS_PIXEL self.baseline_stats = OnlineStats(n_stats) self.spike0_stats = OnlineStats(n_stats) self.spike1_stats = OnlineStats(n_stats) self.spike2_stats = OnlineStats(n_stats) def start(self): tel_id = self.source.tel_id for event in tqdm(self.source, disable=not self.progress_bar): fill_stats( event.r1.tel[tel_id].waveform, self.source.r0_r1_calibrator.first_cap[tel_id], self.source.r0_r1_calibrator.first_cap_old[tel_id], self.source.r0_r1_calibrator.last_readout_time[tel_id], self.baseline_stats, self.spike0_stats, self.spike1_stats, self.spike2_stats, skip_samples_front=self.skip_samples_front, skip_samples_end=self.skip_samples_end, ) def mean_spike_height(self): '''Calculate mean spike height for each gain, pixel''' shape = (N_GAINS, N_PIXELS, N_CAPACITORS_PIXEL) mean_baseline = self.baseline_stats.mean.reshape(shape) spike_heights = np.full((N_GAINS, N_PIXELS, 3), np.nan, dtype=np.float32) for i in range(3): stats = getattr(self, f'spike{i}_stats') counts = stats.counts.reshape(shape) spike_height = stats.mean.reshape(shape) - mean_baseline spike_height[counts == 0] = 0 # np.ma does not raise an error if the weights sum to 0 mean_height = np.ma.average(spike_height, weights=counts, axis=2) # convert masked array to dense, replacing invalid values with nan spike_heights[:, :, i] = mean_height.filled(np.nan) unknown_spike_heights = np.isnan(spike_heights).any(axis=2) n_unknown_spike_heights = np.count_nonzero(unknown_spike_heights) if n_unknown_spike_heights > 0: self.log.warning( f'Could not determine spike height for {n_unknown_spike_heights} channels' ) self.log.warning( f'Gain, pixel: {np.nonzero(unknown_spike_heights)}') # replace any unknown pixels with the mean over the camera camera_mean_spike_height = np.nanmean(spike_heights, axis=(0, 1)) self.log.warning( f'Using camera mean of {camera_mean_spike_height} for these pixels' ) spike_heights[unknown_spike_heights] = camera_mean_spike_height return spike_heights def finish(self): tel_id = self.source.tel_id self.log.info('Writing output to %s', self.output_path) key = f'r1/monitoring/drs4_baseline/tel_{tel_id:03d}' shape = (N_GAINS, N_PIXELS, N_CAPACITORS_PIXEL) baseline_mean = self.baseline_stats.mean.reshape(shape) baseline_std = self.baseline_stats.std.reshape(shape) baseline_counts = self.baseline_stats.counts.reshape(shape).astype( np.uint16) n_negative = np.count_nonzero(baseline_mean < 0) if n_negative > 0: gain, pixel, capacitor = np.nonzero(baseline_mean < 0) self.log.critical( f'{n_negative} baseline values are smaller than 0') self.log.info("Gain | Pixel | Capacitor | Baseline ") for g, p, c in zip(gain, pixel, capacitor): self.log.info( f"{g:4d} | {p:4d} | {c:9d} | {baseline_mean[g][p][c]:6.1f}" ) n_small = np.count_nonzero(baseline_mean < 25) if n_small > 0: gain, pixel, capacitor = np.nonzero(baseline_mean < 25) self.log.warning(f'{n_small} baseline values are smaller than 25') self.log.info("Gain | Pixel | Capacitor | Baseline ") for g, p, c in zip(gain, pixel, capacitor): self.log.info( f"{g:4d} | {p:4d} | {c:9d} | {baseline_mean[g][p][c]:6.1f}" ) # Convert baseline mean and spike heights to uint16, handle missing # values and values smaller 0, larger maxint baseline_mean = convert_to_uint16(baseline_mean) baseline_std = convert_to_uint16(baseline_std) spike_height = convert_to_uint16(self.mean_spike_height()) with HDF5TableWriter(self.output_path) as writer: Provenance().add_output_file(str(self.output_path)) drs4_calibration = DRS4CalibrationContainer( baseline_mean=baseline_mean, baseline_std=baseline_std, baseline_counts=baseline_counts, spike_height=spike_height, ) writer.write(key, drs4_calibration)
class C(Component): path = Path(exists=True, allow_none=True, default_value=None)
class SomeComponent(TelescopeComponent): path = TelescopeParameter(Path(allow_none=True, default_value=None), default_value=None).tag(config=True) val = TelescopeParameter(Float(), default_value=1.0).tag(config=True) flag = TelescopeParameter(Bool(), default_value=True).tag(config=True)
class C(Component): path = Path(allow_none=False)
class C(Component): thepath = Path()
class EventSource(Component): """ Parent class for EventFileReaders of different sources. A new EventFileReader should be created for each type of event file read into ctapipe, e.g. sim_telarray files are read by the `SimTelEventSource`. EventFileReader provides a common high-level interface for accessing event information from different data sources (simulation or different camera file formats). Creating an EventFileReader for a new file format ensures that data can be accessed in a common way, irregardless of the file format. EventFileReader itself is an abstract class. To use an EventFileReader you must use a subclass that is relevant for the file format you are reading (for example you must use `ctapipe.io.SimTelEventSource` to read a hessio format file). Alternatively you can use `event_source()` to automatically select the correct EventFileReader subclass for the file format you wish to read. To create an instance of an EventFileReader you must pass the traitlet configuration (containing the input_url) and the `ctapipe.core.tool.Tool`. Therefore from inside a Tool you would do: >>> event_source = EventSource(self.config, self) An example of how to use `ctapipe.core.tool.Tool` and `event_source()` can be found in ctapipe/tools/display_dl1.py. However if you are not inside a Tool, you can still create an instance and supply an input_url via: >>> event_source = EventSource( input_url="/path/to/file") To loop through the events in a file: >>> event_source = EventSource( input_url="/path/to/file") >>> for event in event_source: >>> print(event.count) **NOTE**: Every time a new loop is started through the event_source, it restarts from the first event. Alternatively one can use EventFileReader in a `with` statement to ensure the correct cleanups are performed when you are finished with the event_source: >>> with EventSource( input_url="/path/to/file") as event_source: >>> for event in event_source: >>> print(event.count) **NOTE**: The "event" that is returned from the generator is a pointer. Any operation that progresses that instance of the generator further will change the data pointed to by "event". If you wish to ensure a particular event is kept, you should perform a `event_copy = copy.deepcopy(event)`. Attributes ---------- input_url : str Path to the input event file. max_events : int Maximum number of events to loop through in generator """ input_url = Path( directory_ok=False, exists=True, help="Path to the input file containing events.").tag(config=True) max_events = Int( None, allow_none=True, help="Maximum number of events that will be read from the file", ).tag(config=True) allowed_tels = Set( default_value=None, allow_none=True, help=("list of allowed tel_ids, others will be ignored. " "If None, all telescopes in the input stream " "will be included")).tag(config=True) def __init__(self, input_url=None, config=None, parent=None, **kwargs): """ Class to handle generic input files. Enables obtaining the "source" generator, regardless of the type of file (either hessio or camera file). Parameters ---------- config : traitlets.loader.Config Configuration specified by config file or cmdline arguments. Used to set traitlet values. Set to None if no configuration to pass. tool : ctapipe.core.Tool Tool executable that is calling this component. Passes the correct logger to the component. Set to None if no Tool to pass. kwargs """ super().__init__(config=config, parent=parent, input_url=input_url, **kwargs) self.metadata = dict(is_simulation=False) self.log.info(f"INPUT PATH = {self.input_url}") if self.max_events: self.log.info(f"Max events being read = {self.max_events}") Provenance().add_input_file(str(self.input_url), role="DL0/Event") @staticmethod @abstractmethod def is_compatible(file_path): """ Abstract method to be defined in child class. Perform a set of checks to see if the input file is compatible with this file event_source. Parameters ---------- file_path : str File path to the event file. Returns ------- compatible : bool True if file is compatible, False if it is incompatible """ @property def is_stream(self): """ Bool indicating if input is a stream. If it is then it is incompatible with `ctapipe.io.eventseeker.EventSeeker`. TODO: Define a method to detect if it is a stream Returns ------- bool If True, then input is a stream. """ return False @property @abstractmethod def subarray(self): """ Obtain the subarray from the EventSource Returns ------- ctapipe.instrument.SubarrayDecription """ @property @abstractmethod def is_simulation(self): """ Weither the currently opened file is simulated Returns ------- bool """ @property @abstractmethod def datalevels(self): """ The datalevels provided by this event source Returns ------- tuple[ctapipe.io.DataLevel] """ @property @abstractmethod def obs_id(self): """ The current observation id Returns ------- int """ @abstractmethod def _generator(self): """ Abstract method to be defined in child class. Generator where the filling of the `ctapipe.containers` occurs. Returns ------- generator """ def __iter__(self): """ Generator that iterates through `_generator`, but keeps track of `self.max_events`. Returns ------- generator """ for event in self._generator(): yield event if self.max_events and event.count >= self.max_events - 1: break def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass @classmethod def from_url(cls, input_url, **kwargs): """ Find compatible EventSource for input_url via the `is_compatible` method of the EventSource Parameters ---------- input_url : str Filename or URL pointing to an event file kwargs Named arguments for the EventSource Returns ------- instance Instance of a compatible EventSource subclass """ if input_url == "" or input_url is None: raise ToolConfigurationError( "EventSource: No input_url was specified") # validate input url with the traitel validate method # to make sure it's compatible and to raise the correct error input_url = EventSource.input_url.validate(obj=None, value=input_url) detect_and_import_io_plugins() available_classes = non_abstract_children(cls) for subcls in available_classes: if subcls.is_compatible(input_url): return subcls(input_url=input_url, **kwargs) raise ValueError("Cannot find compatible EventSource for \n" "\turl:{}\n" "in available EventSources:\n" "\t{}".format(input_url, [c.__name__ for c in available_classes])) @classmethod def from_config(cls, config=None, parent=None, **kwargs): """ Find compatible EventSource for the EventSource.input_url traitlet specified via the config. This method is typically used in Tools, where the input_url is chosen via the command line using the traitlet configuration system. Parameters ---------- config : traitlets.config.loader.Config Configuration created in the Tool kwargs Named arguments for the EventSource Returns ------- instance Instance of a compatible EventSource subclass """ if config is None and parent is None: raise ValueError('One of config or parent must be provided') if config is not None and parent is not None: raise ValueError('Only one of config or parent must be provided') if config is None: config = parent.config if isinstance(config.EventSource.input_url, LazyConfigValue): config.EventSource.input_url = cls.input_url.default_value return event_source(config.EventSource.input_url, config=config, **kwargs)
class SomeComponent(TelescopeComponent): path = TelescopeParameter(Path(exists=True, directory_ok=False))
class FlasherFlatFieldCalculator(FlatFieldCalculator): """Calculates flat-field parameters from flasher data based on the best algorithm described by S. Fegan in MST-CAM-TN-0060 (eq. 19) Pixels are defined as outliers on the base of a cut on the pixel charge median over the full sample distribution and the pixel signal time inside the waveform time Parameters: ---------- charge_cut_outliers : List[2] Interval of accepted charge values (fraction with respect to camera median value) time_cut_outliers : List[2] Interval (in waveform samples) of accepted time values """ charge_median_cut_outliers = List( [-0.3, 0.3], help= 'Interval of accepted charge values (fraction with respect to camera median value)' ).tag(config=True) charge_std_cut_outliers = List( [-3, 3], help= 'Interval (number of std) of accepted charge standard deviation around camera median value' ).tag(config=True) time_cut_outliers = List( [0, 60], help="Interval (in waveform samples) of accepted time values").tag( config=True) time_sampling_correction_path = Path( exists=True, directory_ok=False, help='Path to time sampling correction file').tag(config=True) def __init__(self, subarray, **kwargs): """Calculates flat-field parameters from flasher data based on the best algorithm described by S. Fegan in MST-CAM-TN-0060 (eq. 19) Pixels are defined as outliers on the base of a cut on the pixel charge median over the full sample distribution and the pixel signal time inside the waveform time Parameters: ---------- charge_cut_outliers : List[2] Interval of accepted charge values (fraction with respect to camera median value) time_cut_outliers : List[2] Interval (in waveform samples) of accepted time values """ super().__init__(subarray, **kwargs) self.log.info("Used events statistics : %d", self.sample_size) # members to keep state in calculate_relative_gain() self.num_events_seen = 0 self.time_start = None # trigger time of first event in sample self.trigger_time = None # trigger time of present event self.charge_medians = None # med. charge in camera per event in sample self.charges = None # charge per event in sample self.arrival_times = None # arrival time per event in sample self.sample_masked_pixels = None # masked pixels per event in sample # declare the charge sampling corrector if self.time_sampling_correction_path is not None: self.time_sampling_corrector = TimeSamplingCorrection( time_sampling_correction_path=self. time_sampling_correction_path) else: self.time_sampling_corrector = None def _extract_charge(self, event): """ Extract the charge and the time from a calibration event Parameters ---------- event : general event container """ # copy the waveform be cause we do not want to change it for the moment waveforms = np.copy(event.r1.tel[self.tel_id].waveform) # In case of no gain selection the selected gain channels are [0,0,..][1,1,..] no_gain_selection = np.zeros((waveforms.shape[0], waveforms.shape[1]), dtype=np.int64) no_gain_selection[1] = 1 n_pixels = 1855 # correct the r1 waveform for the sampling time corrections if self.time_sampling_corrector: waveforms *= (self.time_sampling_corrector.get_corrections( event, self.tel_id)[no_gain_selection, np.arange(n_pixels)]) # Extract charge and time charge = 0 peak_pos = 0 if self.extractor: charge, peak_pos = self.extractor(waveforms, self.tel_id, no_gain_selection) # shift the time if time shift is already defined # (e.g. drs4 waveform time shifts for LST) time_shift = event.calibration.tel[self.tel_id].dl1.time_shift if time_shift is not None: peak_pos -= time_shift return charge, peak_pos def calculate_relative_gain(self, event): """ calculate the flatfield statistical values and fill mon.tel[tel_id].flatfield container Parameters ---------- event : general event container Returns: True if the mon.tel[tel_id].flatfield is updated, False otherwise """ # initialize the np array at each cycle waveform = event.r1.tel[self.tel_id].waveform # re-initialize counter if self.num_events_seen == self.sample_size: self.num_events_seen = 0 pixel_mask = np.logical_or( event.mon.tel[self.tel_id].pixel_status.hardware_failing_pixels, event.mon.tel[self.tel_id].pixel_status.flatfield_failing_pixels) # real data if event.meta['origin'] != 'hessio': self.trigger_time = event.trigger.time if self.num_events_seen == 0: self.time_start = self.trigger_time self.setup_sample_buffers(waveform, self.sample_size) # extract the charge of the event and # the peak position (assumed as time for the moment) charge, arrival_time = self._extract_charge(event) self.collect_sample(charge, pixel_mask, arrival_time) sample_age = self.trigger_time - self.time_start # check if to create a calibration event if (self.num_events_seen > 0 and (sample_age > self.sample_duration or self.num_events_seen == self.sample_size)): # update the monitoring container self.store_results(event) return True else: return False def store_results(self, event): """ Store statistical results in monitoring container Parameters ---------- event : general event container """ if self.num_events_seen == 0: raise ValueError( "No flat-field events in statistics, zero results") container = event.mon.tel[self.tel_id].flatfield # mask the part of the array not filled self.sample_masked_pixels[self.num_events_seen:] = 1 relative_gain_results = self.calculate_relative_gain_results( self.charge_medians, self.charges, self.sample_masked_pixels) time_results = self.calculate_time_results(self.arrival_times, self.sample_masked_pixels, self.time_start, self.trigger_time) result = { 'n_events': self.num_events_seen, **relative_gain_results, **time_results, } for key, value in result.items(): setattr(container, key, value) # update the flatfield mask ff_charge_failing_pixels = np.logical_or( container.charge_median_outliers, container.charge_std_outliers) event.mon.tel[self.tel_id].pixel_status.flatfield_failing_pixels = \ np.logical_or(ff_charge_failing_pixels, container.time_median_outliers) def setup_sample_buffers(self, waveform, sample_size): """Initialize sample buffers""" n_channels = waveform.shape[0] n_pix = waveform.shape[1] shape = (sample_size, n_channels, n_pix) self.charge_medians = np.zeros((sample_size, n_channels)) self.charges = np.zeros(shape) self.arrival_times = np.zeros(shape) self.sample_masked_pixels = np.zeros(shape) def collect_sample(self, charge, pixel_mask, arrival_time): """Collect the sample data""" # extract the charge of the event and # the peak position (assumed as time for the moment) good_charge = np.ma.array(charge, mask=pixel_mask) charge_median = np.ma.median(good_charge, axis=1) self.charges[self.num_events_seen] = charge self.arrival_times[self.num_events_seen] = arrival_time self.sample_masked_pixels[self.num_events_seen] = pixel_mask self.charge_medians[self.num_events_seen] = charge_median self.num_events_seen += 1 def calculate_time_results( self, trace_time, masked_pixels_of_sample, time_start, trigger_time, ): """Calculate and return the time results """ masked_trace_time = np.ma.array(trace_time, mask=masked_pixels_of_sample) # median over the sample per pixel pixel_median = np.ma.median(masked_trace_time, axis=0) # mean over the sample per pixel pixel_mean = np.ma.mean(masked_trace_time, axis=0) # std over the sample per pixel pixel_std = np.ma.std(masked_trace_time, axis=0) # median of the median over the camera median_of_pixel_median = np.ma.median(pixel_median, axis=1) # time outliers from median relative_median = pixel_median - median_of_pixel_median[:, np.newaxis] time_median_outliers = np.logical_or( pixel_median < self.time_cut_outliers[0], pixel_median > self.time_cut_outliers[1]) return { 'sample_time': (trigger_time - time_start).value / 2 * u.s, 'sample_time_min': time_start.value * u.s, 'sample_time_max': trigger_time.value * u.s, 'time_mean': np.ma.getdata(pixel_mean) * u.ns, 'time_median': np.ma.getdata(pixel_median) * u.ns, 'time_std': np.ma.getdata(pixel_std) * u.ns, 'relative_time_median': np.ma.getdata(relative_median) * u.ns, 'time_median_outliers': np.ma.getdata(time_median_outliers), } def calculate_relative_gain_results( self, event_median, trace_integral, masked_pixels_of_sample, ): """Calculate and return the sample statistics""" masked_trace_integral = np.ma.array(trace_integral, mask=masked_pixels_of_sample) # median over the sample per pixel pixel_median = np.ma.median(masked_trace_integral, axis=0) # mean over the sample per pixel pixel_mean = np.ma.mean(masked_trace_integral, axis=0) # std over the sample per pixel pixel_std = np.ma.std(masked_trace_integral, axis=0) # median of the median over the camera median_of_pixel_median = np.ma.median(pixel_median, axis=1) # median of the std over the camera median_of_pixel_std = np.ma.median(pixel_std, axis=1) # std of the std over camera std_of_pixel_std = np.ma.std(pixel_std, axis=1) # relative gain relative_gain_event = masked_trace_integral / event_median[:, :, np.newaxis] # outliers from median charge_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] charge_median_outliers = (np.logical_or( charge_deviation < self.charge_median_cut_outliers[0] * median_of_pixel_median[:, np.newaxis], charge_deviation > self.charge_median_cut_outliers[1] * median_of_pixel_median[:, np.newaxis])) # outliers from standard deviation deviation = pixel_std - median_of_pixel_std[:, np.newaxis] charge_std_outliers = (np.logical_or( deviation < self.charge_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], deviation > self.charge_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis])) return { 'relative_gain_median': np.ma.getdata(np.ma.median(relative_gain_event, axis=0)), 'relative_gain_mean': np.ma.getdata(np.ma.mean(relative_gain_event, axis=0)), 'relative_gain_std': np.ma.getdata(np.ma.std(relative_gain_event, axis=0)), 'charge_median': np.ma.getdata(pixel_median), 'charge_mean': np.ma.getdata(pixel_mean), 'charge_std': np.ma.getdata(pixel_std), 'charge_std_outliers': np.ma.getdata(charge_std_outliers), 'charge_median_outliers': np.ma.getdata(charge_median_outliers), }
class C1(Component): p = Path(exists=False)
class LSTR0Corrections(TelescopeComponent): """ The base R0-level calibrator. Changes the r0 container. The R0 calibrator performs the camera-specific R0 calibration that is usually performed on the raw data by the camera server. This calibrator exists in lstchain for testing and prototyping purposes. """ offset = IntTelescopeParameter( default_value=0, help=( 'Define offset to be subtracted from the waveform *additionally*' ' to the drs4 pedestal offset. This only needs to be given when' ' the drs4 pedestal calibration is not applied or the offset of the' ' drs4 run is different from the data run' ) ).tag(config=True) r1_sample_start = IntTelescopeParameter( default_value=3, help='Start sample for r1 waveform', allow_none=True, ).tag(config=True) r1_sample_end = IntTelescopeParameter( default_value=39, help='End sample for r1 waveform', allow_none=True, ).tag(config=True) drs4_pedestal_path = TelescopeParameter( trait=Path(exists=True, directory_ok=False), allow_none=True, default_value=None, help=( 'Path to the LST pedestal file' ', required when `apply_drs4_pedestal_correction=True`' ), ).tag(config=True) calibration_path = Path( exists=True, directory_ok=False, help='Path to LST calibration file', ).tag(config=True) drs4_time_calibration_path = TelescopeParameter( trait=Path(exists=True, directory_ok=False), help='Path to the time calibration file', default_value=None, allow_none=True, ).tag(config=True) calib_scale_high_gain = FloatTelescopeParameter( default_value=1.0, help='High gain waveform is multiplied by this number' ).tag(config=True) calib_scale_low_gain = FloatTelescopeParameter( default_value=1.0, help='Low gain waveform is multiplied by this number' ).tag(config=True) select_gain = Bool( default_value=True, help='Set to False to keep both gains.' ).tag(config=True) apply_drs4_pedestal_correction = Bool( default_value=True, help=( 'Set to False to disable drs4 pedestal correction.' ' Providing the drs4_pedestal_path is required to perform this calibration' ), ).tag(config=True) apply_timelapse_correction = Bool( default_value=True, help='Set to False to disable drs4 timelapse correction' ).tag(config=True) apply_spike_correction = Bool( default_value=True, help='Set to False to disable drs4 spike correction' ).tag(config=True) add_calibration_timeshift = Bool( default_value=True, help=( 'If true, time correction from the calibration' ' file is added to calibration.dl1.time' ), ).tag(config=True) gain_selection_threshold = Float( default_value=3500, help='Threshold for the ThresholdGainSelector.' ).tag(config=True) def __init__(self, subarray, config=None, parent=None, **kwargs): """ The R0 calibrator for LST data. Fill the r1 container. Parameters ---------- """ super().__init__( subarray=subarray, config=config, parent=parent, **kwargs ) self.mon_data = None self.last_readout_time = {} self.first_cap = {} self.first_cap_old = {} self.fbn = {} self.fan = {} for tel_id in self.subarray.tel: shape = (N_GAINS, N_PIXELS, N_CAPACITORS_PIXEL) self.last_readout_time[tel_id] = np.zeros(shape, dtype='uint64') shape = (N_GAINS, N_PIXELS) self.first_cap[tel_id] = np.zeros(shape, dtype=int) self.first_cap_old[tel_id] = np.zeros(shape, dtype=int) if self.select_gain: self.gain_selector = ThresholdGainSelector( threshold=self.gain_selection_threshold, parent=self ) else: self.gain_selector = None if self.calibration_path is not None: self.mon_data = self._read_calibration_file(self.calibration_path) def apply_drs4_corrections(self, event: LSTArrayEventContainer): self.update_first_capacitors(event) for tel_id, r0 in event.r0.tel.items(): r1 = event.r1.tel[tel_id] # If r1 was not yet filled, copy of r0 converted if r1.waveform is None: r1.waveform = r0.waveform # float32 can represent all values of uint16 exactly, # so this does not loose precision. r1.waveform = r1.waveform.astype(np.float32, copy=False) # apply drs4 corrections if self.apply_drs4_pedestal_correction: self.subtract_pedestal(event, tel_id) if self.apply_timelapse_correction: self.time_lapse_corr(event, tel_id) if self.apply_spike_correction: self.interpolate_spikes(event, tel_id) # remove samples at beginning / end of waveform start = self.r1_sample_start.tel[tel_id] end = self.r1_sample_end.tel[tel_id] r1.waveform = r1.waveform[..., start:end] if self.offset.tel[tel_id] != 0: r1.waveform -= self.offset.tel[tel_id] mon = event.mon.tel[tel_id] if r1.selected_gain_channel is None: r1.waveform[mon.pixel_status.hardware_failing_pixels] = 0.0 else: broken = mon.pixel_status.hardware_failing_pixels[r1.selected_gain_channel, PIXEL_INDEX] r1.waveform[broken] = 0.0 def update_first_capacitors(self, event: LSTArrayEventContainer): for tel_id, lst in event.lst.tel.items(): self.first_cap_old[tel_id] = self.first_cap[tel_id] self.first_cap[tel_id] = get_first_capacitors_for_pixels( lst.evt.first_capacitor_id, lst.svc.pixel_ids, ) def calibrate(self, event: LSTArrayEventContainer): for tel_id in event.r0.tel: r1 = event.r1.tel[tel_id] # if `apply_drs4_corrections` is False, we did not fill in the # waveform yet. if r1.waveform is None: r1.waveform = event.r0.tel[tel_id].waveform r1.waveform = r1.waveform.astype(np.float32, copy=False) # do gain selection before converting to pe # like eventbuilder will do if self.select_gain and r1.selected_gain_channel is None: r1.selected_gain_channel = self.gain_selector(r1.waveform) r1.waveform = r1.waveform[r1.selected_gain_channel, PIXEL_INDEX] # apply monitoring data corrections, # subtract pedestal and convert to pe if self.mon_data is not None: calibration = self.mon_data.tel[tel_id].calibration convert_to_pe( waveform=r1.waveform, calibration=calibration, selected_gain_channel=r1.selected_gain_channel ) broken_pixels = event.mon.tel[tel_id].pixel_status.hardware_failing_pixels if r1.selected_gain_channel is None: r1.waveform[broken_pixels] = 0.0 else: r1.waveform[broken_pixels[r1.selected_gain_channel, PIXEL_INDEX]] = 0.0 # store calibration data needed for dl1 calibration in ctapipe # first drs4 time shift (zeros if no calib file was given) time_shift = self.get_drs4_time_correction( tel_id, self.first_cap[tel_id], selected_gain_channel=r1.selected_gain_channel, ) # time shift from flat fielding if self.mon_data is not None and self.add_calibration_timeshift: time_corr = self.mon_data.tel[tel_id].calibration.time_correction # time_shift is subtracted in ctapipe, # but time_correction should be added if r1.selected_gain_channel is not None: time_shift -= time_corr[r1.selected_gain_channel, PIXEL_INDEX].to_value(u.ns) else: time_shift -= time_corr.to_value(u.ns) event.calibration.tel[tel_id].dl1.time_shift = time_shift # needed for charge scaling in ctpaipe dl1 calib if r1.selected_gain_channel is not None: relative_factor = np.empty(N_PIXELS) relative_factor[r1.selected_gain_channel == HIGH_GAIN] = self.calib_scale_high_gain.tel[tel_id] relative_factor[r1.selected_gain_channel == LOW_GAIN] = self.calib_scale_low_gain.tel[tel_id] else: relative_factor = np.empty((N_GAINS, N_PIXELS)) relative_factor[HIGH_GAIN] = self.calib_scale_high_gain.tel[tel_id] relative_factor[LOW_GAIN] = self.calib_scale_low_gain.tel[tel_id] event.calibration.tel[tel_id].dl1.relative_factor = relative_factor @staticmethod def _read_calibration_file(path): """ Read the correction from hdf5 calibration file """ mon = MonitoringContainer() with tables.open_file(path) as f: tel_ids = [ int(key[4:]) for key in f.root._v_children.keys() if key.startswith('tel_') ] for tel_id in tel_ids: with HDF5TableReader(path) as h5_table: base = f'/tel_{tel_id}' # read the calibration data table = base + '/calibration' next(h5_table.read(table, mon.tel[tel_id].calibration)) # read pedestal data table = base + '/pedestal' next(h5_table.read(table, mon.tel[tel_id].pedestal)) # read flat-field data table = base + '/flatfield' next(h5_table.read(table, mon.tel[tel_id].flatfield)) # read the pixel_status container table = base + '/pixel_status' next(h5_table.read(table, mon.tel[tel_id].pixel_status)) return mon @staticmethod def load_drs4_time_calibration_file(path): """ Function to load calibration file. """ with tables.open_file(path, 'r') as f: fan = f.root.fan[:] fbn = f.root.fbn[:] return fan, fbn def load_drs4_time_calibration_file_for_tel(self, tel_id): self.fan[tel_id], self.fbn[tel_id] = self.load_drs4_time_calibration_file( self.drs4_time_calibration_path.tel[tel_id] ) def get_drs4_time_correction(self, tel_id, first_capacitors, selected_gain_channel=None): """ Return pulse time after time correction. """ if self.drs4_time_calibration_path.tel[tel_id] is None: if selected_gain_channel is None: return np.zeros((N_GAINS, N_PIXELS)) else: return np.zeros(N_PIXELS) # load calib file if not already done if tel_id not in self.fan: self.load_drs4_time_calibration_file_for_tel(tel_id) if selected_gain_channel is not None: return calc_drs4_time_correction_gain_selected( first_capacitors, selected_gain_channel, self.fan[tel_id], self.fbn[tel_id], ) else: return calc_drs4_time_correction_both_gains( first_capacitors, self.fan[tel_id], self.fbn[tel_id], ) @staticmethod @lru_cache(maxsize=4) def _get_drs4_pedestal_data(path): """ Function to load pedestal file. To make boundary conditions unnecessary, the first N_SAMPLES values are repeated at the end of the array The result is cached so we can repeatedly call this method using the configured path without reading it each time. """ if path is None: raise ValueError( "DRS4 pedestal correction requested" " but no file provided for telescope" ) pedestal_data = np.empty( (N_GAINS, N_PIXELS_MODULE * N_MODULES, N_CAPACITORS_PIXEL + N_SAMPLES), dtype=np.int16 ) with fits.open(path) as f: pedestal_data[:, :, :N_CAPACITORS_PIXEL] = f[1].data pedestal_data[:, :, N_CAPACITORS_PIXEL:] = pedestal_data[:, :, :N_SAMPLES] return pedestal_data def subtract_pedestal(self, event, tel_id): """ Subtract cell offset using pedestal file. Fill the R1 container. Parameters ---------- event : `ctapipe` event-container tel_id : id of the telescope """ pedestal = self._get_drs4_pedestal_data( self.drs4_pedestal_path.tel[tel_id] ) if event.r1.tel[tel_id].selected_gain_channel is None: subtract_pedestal( event.r1.tel[tel_id].waveform, self.first_cap[tel_id], pedestal, ) else: subtract_pedestal_gain_selected( event.r1.tel[tel_id].waveform, self.first_cap[tel_id], pedestal, event.r1.tel[tel_id].selected_gain_channel, ) def time_lapse_corr(self, event, tel_id): """ Perform time lapse baseline corrections. Fill the R1 container or modifies R0 container. Parameters ---------- event : `ctapipe` event-container tel_id : id of the telescope """ lst = event.lst.tel[tel_id] # If R1 container exists, update it inplace if isinstance(event.r1.tel[tel_id].waveform, np.ndarray): container = event.r1.tel[tel_id] else: # Modify R0 container. This is to create pedestal files. container = event.r0.tel[tel_id] waveform = container.waveform.copy() # We have 2 functions: one for data from 2018/10/10 to 2019/11/04 and # one for data from 2019/11/05 (from Run 1574) after update firmware. # The old readout (before 2019/11/05) is shifted by 1 cell. run_id = event.lst.tel[tel_id].svc.configuration_id # not yet gain selected if event.r1.tel[tel_id].selected_gain_channel is None: apply_timelapse_correction( waveform=waveform, local_clock_counter=lst.evt.local_clock_counter, first_capacitors=self.first_cap[tel_id], last_readout_time=self.last_readout_time[tel_id], expected_pixels_id=lst.svc.pixel_ids, run_id=run_id, ) else: apply_timelapse_correction_gain_selected( waveform=waveform, local_clock_counter=lst.evt.local_clock_counter, first_capacitors=self.first_cap[tel_id], last_readout_time=self.last_readout_time[tel_id], expected_pixels_id=lst.svc.pixel_ids, selected_gain_channel=event.r1.tel[tel_id].selected_gain_channel, run_id=run_id, ) container.waveform = waveform def interpolate_spikes(self, event, tel_id): """ Interpolates spike A & B. Fill the R1 container. Parameters ---------- event : `ctapipe` event-container tel_id : id of the telescope """ run_id = event.lst.tel[tel_id].svc.configuration_id r1 = event.r1.tel[tel_id] if r1.selected_gain_channel is None: interpolate_spikes( waveform=r1.waveform, first_capacitors=self.first_cap[tel_id], previous_first_capacitors=self.first_cap_old[tel_id], run_id=run_id, ) else: interpolate_spikes_gain_selected( waveform=r1.waveform, first_capacitors=self.first_cap[tel_id], previous_first_capacitors=self.first_cap_old[tel_id], selected_gain_channel=r1.selected_gain_channel, run_id=run_id, )
class C1(Component): thepath = Path(exists=False)
class SimpleEventWriter(Tool): name = "ctapipe-simple-event-writer" description = Unicode(__doc__) infile = Path( default_value=get_dataset_path( "lst_prod3_calibration_and_mcphotons.simtel.zst"), help="input file to read", directory_ok=False, exists=True, ).tag(config=True) outfile = Path(help="output file name", directory_ok=False, default_value="output.h5").tag(config=True) progress = Bool(help="display progress bar", default_value=True).tag(config=True) aliases = Dict({ "infile": "EventSource.input_url", "outfile": "SimpleEventWriter.outfile", "max-events": "EventSource.max_events", "progress": "SimpleEventWriter.progress", }) classes = List([EventSource, CameraCalibrator]) def setup(self): self.log.info("Configure EventSource...") self.event_source = EventSource.from_url(self.infile, parent=self) self.calibrator = CameraCalibrator(subarray=self.event_source.subarray, parent=self) self.writer = HDF5TableWriter(filename=self.outfile, group_name="image_infos", overwrite=True, parent=self) def start(self): self.log.info("Loop on events...") for event in tqdm( self.event_source, desc="EventWriter", total=self.event_source.max_events, disable=~self.progress, ): self.calibrator(event) for tel_id in event.dl0.tel.keys(): geom = self.event_source.subarray.tel[tel_id].camera.geometry dl1_tel = event.dl1.tel[tel_id] # Image cleaning image = dl1_tel.image # Waiting for automatic gain selection mask = tailcuts_clean(geom, image, picture_thresh=10, boundary_thresh=5) cleaned = image.copy() cleaned[~mask] = 0 # Image parametrisation params = hillas_parameters(geom, cleaned) # Save Ids, MC infos and Hillas informations self.writer.write(geom.camera_name, [event.r0, event.simulation.shower, params]) def finish(self): self.log.info("End of job.") self.writer.close()
class C(Component): thepath = Path(exists=True, file_ok=False)
class DumpTriggersTool(Tool): description = Unicode(__doc__) name = 'ctapipe-dump-triggers' # ============================================= # configuration parameters: # ============================================= infile = Path(exists=True, directory_ok=False, help='input simtelarray file').tag(config=True) outfile = Path( default_value='triggers.fits', directory_ok=False, help='output filename (*.fits, *.h5)', ).tag(config=True) overwrite = Bool(False, help="overwrite existing output file").tag(config=True) # ============================================= # map low-level options to high-level command-line options # ============================================= aliases = Dict({ 'infile': 'DumpTriggersTool.infile', 'outfile': 'DumpTriggersTool.outfile' }) flags = Dict({ 'overwrite': ({ 'DumpTriggersTool': { 'overwrite': True } }, 'Enable overwriting of output file') }) examples = ('ctapipe-dump-triggers --infile gamma.simtel.gz ' '--outfile trig.fits --overwrite' '\n\n' 'If you want to see more output, use --log_level=DEBUG') # ============================================= # The methods of the Tool (initialize, start, finish): # ============================================= def add_event_to_table(self, event): """ add the current hessio event to a row in the `self.events` table """ time = event.trigger.time if self._prev_time is None: self._prev_time = time if self._current_starttime is None: self._current_starttime = time relative_time = time - self._current_starttime delta_t = time - self._prev_time self._prev_time = time # build the trigger pattern as a fixed-length array # (better for storage in FITS format) # trigtels = event.get_telescope_with_data_list() trigtels = event.dl0.tels_with_data self._current_trigpattern[:] = 0 # zero the trigger pattern self._current_trigpattern[list(trigtels)] = 1 # set the triggered tels # to 1 # insert the row into the table self.events.add_row( (event.index.event_id, relative_time.sec, delta_t.sec, len(trigtels), self._current_trigpattern)) def setup(self): """ setup function, called before `start()` """ if self.infile == '': raise ToolConfigurationError( "No 'infile' parameter was specified. ") self.events = Table( names=['EVENT_ID', 'T_REL', 'DELTA_T', 'N_TRIG', 'TRIGGERED_TELS'], dtype=[np.int64, np.float64, np.float64, np.int32, np.uint8]) self.events['TRIGGERED_TELS'].shape = (0, MAX_TELS) self.events['T_REL'].unit = u.s self.events['T_REL'].description = 'Time relative to first event' self.events['DELTA_T'].unit = u.s self.events.meta['INPUT'] = self.infile self._current_trigpattern = np.zeros(MAX_TELS) self._current_starttime = None self._prev_time = None def start(self): """ main event loop """ with event_source(self.infile) as source: for event in source: self.add_event_to_table(event) def finish(self): """ finish up and write out results (called automatically after `start()`) """ # write out the final table try: if '.fits' in self.outfile.suffixes: self.events.write(self.outfile, overwrite=self.overwrite) elif self.outfile.suffix in ('.hdf5', '.h5', '.hdf'): self.events.write(self.outfile, path='/events', overwrite=self.overwrite) else: self.events.write(self.outfile) Provenance().add_output_file(self.outfile) except IOError as err: self.log.warning("Couldn't write output (%s)", err) self.log.info('\n %s', self.events)
class LSTCameraCalibrator(CameraCalibrator): """ Calibrator to handle the LST camera calibration chain, in order to fill the DL1 data level in the event container. """ extractor_product = Unicode( 'LocalPeakWindowSum', help='Name of the charge extractor to be used').tag(config=True) reducer_product = Unicode( 'NullDataVolumeReducer', help='Name of the DataVolumeReducer to use').tag(config=True) calibration_path = Path( exists=True, directory_ok=False, help='Path to LST calibration file').tag(config=True) time_calibration_path = Path( exists=True, directory_ok=False, help='Path to drs4 time calibration file').tag(config=True) time_sampling_correction_path = Path( exists=True, directory_ok=False, help='Path to time sampling correction file', allow_none=True, ).tag(config=True) allowed_tels = List( [1], help='List of telescope to be calibrated').tag(config=True) gain_threshold = Int( 4094, allow_none=True, help='Threshold for the gain selection in ADC').tag(config=True) charge_scale = List( [1, 1], help='Multiplicative correction factor for charge estimation [HG,LG]' ).tag(config=True) def __init__(self, subarray, **kwargs): """ Parameters ---------- reducer_product : ctapipe.image.reducer.DataVolumeReducer The DataVolumeReducer to use. If None, then NullDataVolumeReducer will be used by default, and waveforms will not be reduced. extractor_product : ctapipe.image.extractor.ImageExtractor The ImageExtractor to use. If None, then LocalPeakWindowSum will be used by default. calibration_path : Path to LST calibration file to get the pedestal and flat-field corrections kwargs """ super().__init__(subarray, **kwargs) # load the waveform charge extractor self.image_extractor = ImageExtractor.from_name(self.extractor_product, subarray=self.subarray, config=self.config) self.log.info(f"extractor {self.extractor_product}") print("EXTRACTOR", self.image_extractor) self.data_volume_reducer = DataVolumeReducer.from_name( self.reducer_product, subarray=self.subarray, config=self.config) self.log.info(f" {self.reducer_product}") # declare gain selector if the threshold is defined if self.gain_threshold: self.gain_selector = gainselection.ThresholdGainSelector( threshold=self.gain_threshold) # declare time calibrator if correction file exist if os.path.exists(self.time_calibration_path): self.time_corrector = PulseTimeCorrection( calib_file_path=self.time_calibration_path) else: raise IOError( f"Time calibration file {self.time_calibration_path} not found!" ) # declare the charge sampling corrector if self.time_sampling_correction_path is not None: # search the file in resources if not found if not os.path.exists(self.time_sampling_correction_path): self.time_sampling_correction_path = resource_filename( 'lstchain', f"resources/{self.time_sampling_correction_path}") if os.path.exists(self.time_sampling_correction_path): self.time_sampling_corrector = TimeSamplingCorrection( time_sampling_correction_path=self. time_sampling_correction_path) else: raise IOError( f"Sampling correction file {self.time_sampling_correction_path} not found!" ) else: self.time_sampling_corrector = None # calibration data container self.mon_data = MonitoringContainer() # initialize the MonitoringContainer() for the moment it reads it from a hdf5 file self._initialize_correction() self.log.info(f"Global charge scale {self.charge_scale}") def _initialize_correction(self): """ Read the correction from hdf5 calibration file """ self.log.info(f"read {self.calibration_path}") try: with HDF5TableReader(self.calibration_path) as h5_table: for telid in self.allowed_tels: # read the calibration data table = '/tel_' + str(telid) + '/calibration' next( h5_table.read(table, self.mon_data.tel[telid].calibration)) # read pedestal data table = '/tel_' + str(telid) + '/pedestal' next( h5_table.read(table, self.mon_data.tel[telid].pedestal)) # read flat-field data table = '/tel_' + str(telid) + '/flatfield' next( h5_table.read(table, self.mon_data.tel[telid].flatfield)) # read the pixel_status container table = '/tel_' + str(telid) + '/pixel_status' next( h5_table.read(table, self.mon_data.tel[telid].pixel_status)) except Exception: self.log.exception( f"Problem in reading calibration file {self.calibration_path}") raise def _calibrate_dl0(self, event, telid): """ create dl0 level, with gain-selected and calibrated waveform """ waveforms = event.r1.tel[telid].waveform if self._check_r1_empty(waveforms): return # if not already done, initialize the event monitoring containers if event.mon.tel[telid].calibration.dc_to_pe is None: event.mon.tel[telid].calibration = self.mon_data.tel[ telid].calibration event.mon.tel[telid].flatfield = self.mon_data.tel[telid].flatfield event.mon.tel[telid].pedestal = self.mon_data.tel[telid].pedestal event.mon.tel[telid].pixel_status = self.mon_data.tel[ telid].pixel_status # # subtract the pedestal per sample and multiply for the calibration coefficients # calibrated_waveform = ( (waveforms - self.mon_data.tel[telid].calibration. pedestal_per_sample[:, :, np.newaxis]) * self.mon_data.tel[telid].calibration.dc_to_pe[:, :, np.newaxis]).astype( np.float32) # If requested, perform gain selection (this will be done by the EvB in future) # find the gain selection mask if waveforms.ndim == 3: # if threshold defined, perform gain selection if self.gain_threshold: gain_mask = self.gain_selector(waveforms) # select the samples calibrated_waveform = calibrated_waveform[ gain_mask, np.arange(waveforms.shape[1])] else: # keep both HG and LG gain_mask = np.zeros((waveforms.shape[0], waveforms.shape[1]), dtype=np.int64) gain_mask[1] = 1 else: # gain selection already performed in EvB: (0=HG, 1=LG) gain_mask = event.lst.tel[telid].evt.pixel_status >> 2 & 1 # remember the calibrated and gain selected waveform # (this should be the r1 waveform to be compliant with ctapipe (?)) event.dl0.tel[telid].waveform = calibrated_waveform # remember which channel has been selected event.r1.tel[telid].selected_gain_channel = gain_mask event.dl0.tel[telid].selected_gain_channel = gain_mask def _calibrate_dl1(self, event, telid): """ create calibrated dl1 image and calibrate it """ n_pixels = self.subarray.tels[telid].camera.geometry.n_pixels # copy the waveform be cause I do not want to change it waveforms = np.copy(event.dl0.tel[telid].waveform) gain_mask = event.dl0.tel[telid].selected_gain_channel if self._check_dl0_empty(waveforms): return # In case of no gain selection the selected gain channels are [0,0,..][1,1,..] no_gain_selection = np.zeros((waveforms.shape[0], waveforms.shape[1]), dtype=np.int64) no_gain_selection[1] = 1 # correct the dl0 waveform for the sampling time corrections if self.time_sampling_corrector: waveforms *= self.time_sampling_corrector.get_corrections( event, telid)[gain_mask, np.arange(n_pixels)] # extract the charge charge, peak_time = self.image_extractor(waveforms, telid, gain_mask) # correct charge for global scale corrected_charge = charge * np.array(self.charge_scale, dtype=np.float32)[gain_mask] # correct time with drs4 correction if available if self.time_corrector: peak_time_drs4_corrected = ( peak_time - self.time_corrector.get_pulse_time_corrections( event)[gain_mask, np.arange(n_pixels)]) # add flat-fielding time correction peak_time_ff_corrected = ( peak_time_drs4_corrected + self.mon_data.tel[telid].calibration.time_correction.value[ gain_mask, np.arange(n_pixels)]) # fill dl1 container event.dl1.tel[telid].image = corrected_charge event.dl1.tel[telid].peak_time = peak_time_ff_corrected.astype( np.float32)
class PedestalIntegrator(PedestalCalculator): """Calculates pedestal parameters integrating the charge of pedestal events: the pedestal value corresponds to the charge estimated with the selected charge extractor The pixels are set as outliers on the base of a cut on the pixel charge median over the pedestal sample and the pixel charge standard deviation over the pedestal sample with respect to the camera median values Parameters: ---------- charge_median_cut_outliers : List[2] Interval (number of std) of accepted charge values around camera median value charge_std_cut_outliers : List[2] Interval (number of std) of accepted charge standard deviation around camera median value """ charge_median_cut_outliers = List( [-3, 3], help= 'Interval (number of std) of accepted charge values around camera median value' ).tag(config=True) charge_std_cut_outliers = List( [-3, 3], help= 'Interval (number of std) of accepted charge standard deviation around camera median value' ).tag(config=True) time_sampling_correction_path = Path( default_value=None, allow_none=True, directory_ok=False, help='Path to time sampling correction file', ).tag(config=True) def __init__(self, subarray, **kwargs): """Calculates pedestal parameters integrating the charge of pedestal events: the pedestal value corresponds to the charge estimated with the selected charge extractor The pixels are set as outliers on the base of a cut on the pixel charge median over the pedestal sample and the pixel charge standard deviation over the pedestal sample with respect to the camera median values Parameters: ---------- charge_median_cut_outliers : List[2] Interval (number of std) of accepted charge values around camera median value charge_std_cut_outliers : List[2] Interval (number of std) of accepted charge standard deviation around camera median value """ super().__init__(subarray, **kwargs) self.log.info("Used events statistics : %d", self.sample_size) # members to keep state in calculate_relative_gain() self.num_events_seen = 0 self.time_start = None # trigger time of first event in sample self.trigger_time = None # trigger time of present event self.charge_medians = None # med. charge in camera per event in sample self.charges = None # charge per event in sample self.sample_masked_pixels = None # pixels tp be masked per event in sample # declare the charge sampling corrector if self.time_sampling_correction_path is not None: self.time_sampling_corrector = TimeSamplingCorrection( time_sampling_correction_path=self. time_sampling_correction_path) else: self.time_sampling_corrector = None # fix for broken extractor setup in ctapipe baseclass self.extractor = ImageExtractor.from_name(self.charge_product, parent=self, subarray=subarray) def _extract_charge(self, event): """ Extract the charge and the time from a pedestal event Parameters ---------- event : general event container """ # copy the waveform be cause we do not want to change it for the moment waveforms = np.copy(event.r1.tel[self.tel_id].waveform) # pedestal event do not have gain selection no_gain_selection = np.zeros((waveforms.shape[0], waveforms.shape[1]), dtype=np.int64) no_gain_selection[1] = 1 n_pixels = 1855 # correct the r1 waveform for the sampling time corrections if self.time_sampling_corrector: waveforms *= (self.time_sampling_corrector.get_corrections( event, self.tel_id)[no_gain_selection, np.arange(n_pixels)]) # Extract charge and time charge = 0 peak_pos = 0 if self.extractor: charge, peak_pos = self.extractor(waveforms, self.tel_id, no_gain_selection) return charge, peak_pos def calculate_pedestals(self, event): """ calculate the pedestal statistical values from the charge extracted from pedestal events and fill the mon.tel[tel_id].pedestal container Parameters ---------- event : general event container Returns: True if the mon.tel[tel_id].pedestal is updated, False otherwise """ # initialize the np array at each cycle waveform = event.r1.tel[self.tel_id].waveform # re-initialize counter if self.num_events_seen == self.sample_size: self.num_events_seen = 0 pixel_mask = event.mon.tel[ self.tel_id].pixel_status.hardware_failing_pixels self.trigger_time = event.trigger.time if self.num_events_seen == 0: self.time_start = self.trigger_time self.setup_sample_buffers(waveform, self.sample_size) # extract the charge of the event and # the peak position (assumed as time for the moment) charge = self._extract_charge(event)[0] self.collect_sample(charge, pixel_mask) sample_age = (self.trigger_time - self.time_start).to_value(u.s) # check if to create a calibration event if (self.num_events_seen > 0 and (sample_age > self.sample_duration or self.num_events_seen == self.sample_size)): # update the monitoring container self.store_results(event) return True else: return False def store_results(self, event): """ Store statistical results in monitoring container Parameters ---------- event : general event container """ # something wrong if you are here and no statistic is there if self.num_events_seen == 0: raise ValueError("No pedestal events in statistics, zero results") container = event.mon.tel[self.tel_id].pedestal # mask the part of the array not filled self.sample_masked_pixels[self.num_events_seen:] = 1 pedestal_results = calculate_pedestal_results( self, self.charges, self.sample_masked_pixels, ) time_results = calculate_time_results( self.time_start, self.trigger_time, ) result = { 'n_events': self.num_events_seen, **pedestal_results, **time_results, } for key, value in result.items(): setattr(container, key, value) # update pedestal mask event.mon.tel[self.tel_id].pixel_status.pedestal_failing_pixels = \ np.logical_or(container.charge_median_outliers, container.charge_std_outliers) def setup_sample_buffers(self, waveform, sample_size): """Initialize sample buffers""" n_channels = waveform.shape[0] n_pix = waveform.shape[1] shape = (sample_size, n_channels, n_pix) self.charge_medians = np.zeros((sample_size, n_channels)) self.charges = np.zeros(shape) self.sample_masked_pixels = np.zeros(shape) def collect_sample(self, charge, pixel_mask): """Collect the sample data""" good_charge = np.ma.array(charge, mask=pixel_mask) charge_median = np.ma.median(good_charge, axis=1) self.charges[self.num_events_seen] = charge self.sample_masked_pixels[self.num_events_seen] = pixel_mask self.charge_medians[self.num_events_seen] = charge_median self.num_events_seen += 1
class EventSource(Component): """ Parent class for EventSources. EventSources read input files and generate `ArrayEvents` when iterated over. A new EventSource should be created for each type of event file read into ctapipe, e.g. sim_telarray files are read by the `SimTelEventSource`. EventSource provides a common high-level interface for accessing event information from different data sources (simulation or different camera file formats). Creating an EventSource for a new file format or other event source ensures that data can be accessed in a common way, irregardless of the file format or data origin. EventSource itself is an abstract class, but will create an appropriate subclass if a compatible source is found for the given ``input_url``. >>> dataset = get_dataset_path('gamma_test_large.simtel.gz') >>> event_source = EventSource(input_url=dataset) <ctapipe.io.simteleventsource.SimTelEventSource at ...> An ``EventSource`` can also be created through the configuration system, by passing ``config`` or ``parent`` as appropriate. E.g. if using ``EventSource`` inside of a ``Tool``, you would do: >>> self.event_source = EventSource(parent=self) To loop through the events in a file: >>> event_source = EventSource(input_url="/path/to/file") >>> for event in event_source: >>> print(event.count) **NOTE**: Every time a new loop is started through the event_source, it tries to restart from the first event, which might not be supported by the event source. It is encouraged to use ``EventSource`` in a context manager to ensure the correct cleanups are performed when you are finished with the event_source: >>> with EventSource(input_url="/path/to/file") as event_source: >>> for event in event_source: >>> print(event.count) **NOTE**: For effiency reasons, most sources only use a single ``ArrayEvent`` instance and update it with new data on iteration, which might lead to surprising behaviour if you want to access multiple events at the same time. To keep an event and prevent its data from being overwritten with the next event's data, perform a deepcopy: ``some_special_event = copy.deepcopy(event)``. Attributes ---------- input_url : str Path to the input event file. max_events : int Maximum number of events to loop through in generator allowed_tels: Set[int] or None Ids of the telescopes to be included in the data. If given, only this subset of telescopes will be present in the generated events. If None, all available telescopes are used. """ input_url = Path( directory_ok=False, exists=True, help="Path to the input file containing events.", ).tag(config=True) max_events = Int( None, allow_none=True, help="Maximum number of events that will be read from the file", ).tag(config=True) allowed_tels = Set( default_value=None, allow_none=True, help=( "list of allowed tel_ids, others will be ignored. " "If None, all telescopes in the input stream " "will be included" ), ).tag(config=True) def __new__(cls, input_url=None, config=None, parent=None, **kwargs): """ Returns a compatible subclass for given input url, either directly or via config / parent """ # needed to break recursion, as __new__ of subclass will also # call this method if cls is not EventSource: return super().__new__(cls) # check we have at least one of these to be able to determine the subclass if input_url is None and config is None and parent is None: raise ValueError("One of `input_url`, `config`, `parent` is required") if input_url is None: input_url = cls._find_input_url_in_config(config=config, parent=parent) subcls = cls._find_compatible_source(input_url) return super().__new__(subcls) def __init__(self, input_url=None, config=None, parent=None, **kwargs): """ Class to handle generic input files. Enables obtaining the "source" generator, regardless of the type of file (either hessio or camera file). Parameters ---------- config : traitlets.loader.Config Configuration specified by config file or cmdline arguments. Used to set traitlet values. Set to None if no configuration to pass. tool : ctapipe.core.Tool Tool executable that is calling this component. Passes the correct logger to the component. Set to None if no Tool to pass. kwargs """ # traitlets differentiates between not getting the kwarg # and getting the kwarg with a None value. # the latter overrides the value in the config with None, the former # enables getting it from the config. if input_url is not None: kwargs["input_url"] = input_url super().__init__(config=config, parent=parent, **kwargs) self.metadata = dict(is_simulation=False) self.log.info(f"INPUT PATH = {self.input_url}") if self.max_events: self.log.info(f"Max events being read = {self.max_events}") Provenance().add_input_file(str(self.input_url), role="DL0/Event") @staticmethod @abstractmethod def is_compatible(file_path): """ Abstract method to be defined in child class. Perform a set of checks to see if the input file is compatible with this file event_source. Parameters ---------- file_path : str File path to the event file. Returns ------- compatible : bool True if file is compatible, False if it is incompatible """ @property def is_stream(self): """ Bool indicating if input is a stream. If it is then it is incompatible with `ctapipe.io.eventseeker.EventSeeker`. TODO: Define a method to detect if it is a stream Returns ------- bool If True, then input is a stream. """ return False @property @abstractmethod def subarray(self): """ Obtain the subarray from the EventSource Returns ------- ctapipe.instrument.SubarrayDecription """ @property @abstractmethod def is_simulation(self): """ Weither the currently opened file is simulated Returns ------- bool """ @property @abstractmethod def datalevels(self): """ The datalevels provided by this event source Returns ------- tuple[ctapipe.io.DataLevel] """ def has_any_datalevel(self, datalevels): """ Check if any of `datalevels` is in self.datalevels Parameters: ----------- datalevels: Iterable Iterable of datalevels """ return any(dl in self.datalevels for dl in datalevels) @property @abstractmethod def obs_ids(self): """ The observation ids of the runs located in the file Unmerged files should only contain a single obs id. Returns ------- list[int] """ @abstractmethod def _generator(self): """ Abstract method to be defined in child class. Generator where the filling of the `ctapipe.containers` occurs. Returns ------- generator """ def __iter__(self): """ Generator that iterates through `_generator`, but keeps track of `self.max_events`. Returns ------- generator """ for event in self._generator(): yield event if self.max_events and event.count >= self.max_events - 1: break def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass @classmethod def _find_compatible_source(cls, input_url): if input_url == "" or input_url is None: raise ToolConfigurationError("EventSource: No input_url was specified") # validate input url with the traitel validate method # to make sure it's compatible and to raise the correct error input_url = EventSource.input_url.validate(obj=None, value=input_url) available_classes = non_abstract_children(cls) for subcls in available_classes: if subcls.is_compatible(input_url): return subcls raise ValueError( "Cannot find compatible EventSource for \n" "\turl:{}\n" "in available EventSources:\n" "\t{}".format(input_url, [c.__name__ for c in available_classes]) ) @classmethod def from_url(cls, input_url, **kwargs): """ Find compatible EventSource for input_url via the `is_compatible` method of the EventSource Parameters ---------- input_url : str Filename or URL pointing to an event file kwargs Named arguments for the EventSource Returns ------- instance Instance of a compatible EventSource subclass """ subcls = cls._find_compatible_source(input_url) return subcls(input_url=input_url, **kwargs) @classmethod def _find_input_url_in_config(cls, config=None, parent=None): if config is None and parent is None: raise ValueError("One of config or parent must be provided") if config is not None and parent is not None: raise ValueError("Only one of config or parent must be provided") input_url = None # config was passed if config is not None: if not isinstance(config.input_url, LazyConfigValue): input_url = config.input_url elif not isinstance(config.EventSource.input_url, LazyConfigValue): input_url = config.EventSource.input_url else: input_url = cls.input_url.default_value # parent was passed else: # first look at appropriate position in the config hierarcy input_url = find_config_in_hierarchy(parent, "EventSource", "input_url") # if not found, check top level if isinstance(input_url, LazyConfigValue): if not isinstance(parent.config.EventSource.input_url, LazyConfigValue): input_url = parent.config.EventSource.input_url else: input_url = cls.input_url.default_value return input_url @classmethod def from_config(cls, config=None, parent=None, **kwargs): """ Find compatible EventSource for the EventSource.input_url traitlet specified via the config. This method is typically used in Tools, where the input_url is chosen via the command line using the traitlet configuration system. Parameters ---------- config : traitlets.config.loader.Config Configuration created in the Tool kwargs Named arguments for the EventSource Returns ------- instance Instance of a compatible EventSource subclass """ input_url = cls._find_input_url_in_config(config=config, parent=parent) return cls.from_url(input_url, config=config, parent=parent, **kwargs)
class ChargeResolutionGenerator(Tool): name = "ChargeResolutionGenerator" description = ("Calculate the Charge Resolution from a sim_telarray " "simulation and store within a HDF5 file.") telescopes = List( Int(), None, allow_none=True, help= "Telescopes to include from the event file. Default = All telescopes", ).tag(config=True) output_path = Path( default_value="charge_resolution.h5", directory_ok=False, help="Path to store the output HDF5 file", ).tag(config=True) aliases = Dict( dict( f="EventSource.input_url", max_events="SimTelEventSource.max_events", T="EventSource.allowed_tels", O="ChargeResolutionGenerator.output_path", )) classes = traits.classes_with_traits( EventSource) + traits.classes_with_traits(ImageExtractor) def __init__(self, **kwargs): super().__init__(**kwargs) self.eventsource = None self.calibrator = None self.calculator = None def setup(self): self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]" self.eventsource = EventSource(parent=self) self.calibrator = CameraCalibrator(parent=self, subarray=self.eventsource.subarray) self.calculator = ChargeResolutionCalculator() def start(self): desc = "Extracting Charge Resolution" for event in tqdm(self.eventsource, desc=desc): self.calibrator(event) # Check events have true charge included if event.count == 0: try: pe = list(event.simulation.tel.values())[0].true_image if np.all(pe == 0): raise KeyError except KeyError: self.log.exception("Source does not contain true charge!") raise for mc, dl1 in zip(event.simulation.tel.values(), event.dl1.tel.values()): true_charge = mc.true_image measured_charge = dl1.image pixels = np.arange(measured_charge.size) self.calculator.add(pixels, true_charge, measured_charge) def finish(self): df_p, df_c = self.calculator.finish() output_directory = os.path.dirname(self.output_path) if not os.path.exists(output_directory): self.log.info(f"Creating directory: {output_directory}") os.makedirs(output_directory) with pd.HDFStore(self.output_path, "w") as store: store["charge_resolution_pixel"] = df_p store["charge_resolution_camera"] = df_c self.log.info("Created charge resolution file: {}".format( self.output_path)) Provenance().add_output_file(self.output_path)
class ImageSumDisplayerTool(Tool): description = Unicode(__doc__) name = "ctapipe-display-imagesum" infile = Path( help="input simtelarray file", default_value=get_dataset_path("gamma_test_large.simtel.gz"), exists=True, directory_ok=False, ).tag(config=True) telgroup = Integer(help="telescope group number", default_value=1).tag(config=True) max_events = Integer( help="stop after this many events if non-zero", default_value=0, min=0 ).tag(config=True) output_suffix = Unicode( help="suffix (file extension) of output " "filenames to write images " "to (no writing is done if blank). " "Images will be named [EVENTID][suffix]", default_value="", ).tag(config=True) aliases = Dict( { "infile": "ImageSumDisplayerTool.infile", "telgroup": "ImageSumDisplayerTool.telgroup", "max-events": "ImageSumDisplayerTool.max_events", "output-suffix": "ImageSumDisplayerTool.output_suffix", } ) classes = [CameraCalibrator, SimTelEventSource] def setup(self): # load up the telescope types table (need to first open a file, a bit of # a hack until a proper instrument module exists) and select only the # telescopes with the same camera type # make sure gzip files are seekable self.reader = SimTelEventSource( input_url=self.infile, max_events=self.max_events, back_seekable=True ) camtypes = self.reader.subarray.to_table().group_by("camera_type") self.reader.subarray.info(printer=self.log.info) group = camtypes.groups[self.telgroup] self._selected_tels = list(group["tel_id"].data) self._base_tel = self._selected_tels[0] self.log.info( "Telescope group %d: %s", self.telgroup, str(self.reader.subarray.tel[self._selected_tels[0]]), ) self.log.info(f"SELECTED TELESCOPES:{self._selected_tels}") self.calibrator = CameraCalibrator(parent=self, subarray=self.reader.subarray) self.reader = SimTelEventSource( input_url=self.infile, max_events=self.max_events, back_seekable=True, allowed_tels=set(self._selected_tels), ) def start(self): geom = None imsum = None disp = None for event in self.reader: self.calibrator(event) if geom is None: geom = self.reader.subarray.tel[self._base_tel].camera.geometry imsum = np.zeros(shape=geom.pix_x.shape, dtype=np.float64) disp = CameraDisplay(geom, title=geom.camera_name) disp.add_colorbar() disp.cmap = "viridis" if len(event.dl0.tel.keys()) <= 2: continue imsum[:] = 0 for telid in event.dl0.tel.keys(): imsum += event.dl1.tel[telid].image self.log.info( "event={} ntels={} energy={}".format( event.index.event_id, len(event.dl0.tel.keys()), event.simulation.shower.energy, ) ) disp.image = imsum plt.pause(0.1) if self.output_suffix != "": filename = "{:020d}{}".format(event.index.event_id, self.output_suffix) self.log.info(f"saving: '{filename}'") plt.savefig(filename)
class C(Component): path = Path(default_value=None, allow_none=False)