class ImageSumDisplayerTool(Tool): description = Unicode(__doc__) name = "ctapipe-image-sum-display" infile = Unicode( help='input simtelarray file', default="/Users/kosack/Data/CTA/Prod3/gamma.simtel.gz").tag( config=True) telgroup = Integer(help='telescope group number', default=1).tag(config=True) aliases = Dict({ 'infile': 'ImageSumDisplayerTool.infile', 'telgroup': 'ImageSumDisplayerTool.telgroup' }) def setup(self): # load up the telescope types table (need to first open a file, a bit of # a hack until a proper insturment module exists) and select only the # telescopes with the same camera type data = next(hessio_event_source(self.infile, max_events=1)) camtypes = get_camera_types(data.inst) group = camtypes.groups[self.telgroup] self._selected_tels = group['tel_id'].data self._base_tel = self._selected_tels[0] self.log.info("Telescope group %d: %s with %s camera", self.telgroup, group[0]['tel_type'], group[0]['cam_type']) self.log.info("SELECTED TELESCOPES:{}".format(self._selected_tels)) def start(self): geom = None imsum = None disp = None for data in hessio_event_source(self.infile, allowed_tels=self._selected_tels, max_events=None): if geom is None: x, y = data.inst.pixel_pos[self._base_tel] flen = data.inst.optical_foclen[self._base_tel] geom = CameraGeometry.guess(x, y, flen) imsum = np.zeros(shape=x.shape, dtype=np.float) disp = CameraDisplay(geom, title=geom.cam_id) disp.add_colorbar() disp.cmap = 'viridis' if len(data.r0.tels_with_data) <= 2: continue imsum[:] = 0 for telid in data.r0.tels_with_data: imsum += data.r0.tel[telid].adc_sums[0] self.log.info("event={} ntels={} energy={}" \ .format(data.r0.event_id, len(data.r0.tels_with_data), data.mc.energy)) disp.image = imsum plt.pause(0.1)
class DumpInstrumentTool(Tool): description = Unicode(__doc__) name = 'ctapipe-dump-instrument' infile = Unicode(help='input simtelarray file').tag(config=True) format = Enum(['fits', 'ecsv', 'hdf5'], default_value='fits', help='Format ' 'of ' 'output ' 'file', config=True) aliases = Dict( dict(infile='DumpInstrumentTool.infile', format='DumpInstrumentTool.format')) def setup(self): source = hessio_event_source(self.infile) data = next(source) # get one event, so the instrument table is # filled in self.inst = data.inst # keep a pointer to the instrument stuff pass def start(self): self.write_camera_geometries() def finish(self): pass def _get_file_format_info(self, format_name, table_type, cam_name): """ returns file extension + dict of required parameters for Table.write""" if format_name == 'fits': return 'fits.gz', dict() elif format_name == 'ecsv': return 'ecsv.txt', dict(format='ascii.ecsv') elif format_name == 'hdf5': return 'h5', dict(path="/" + table_type + "/" + cam_name) else: raise NameError("format not supported") def write_camera_geometries(self): cam_types = get_camera_types(self.inst) print_camera_types(self.inst, printer=self.log.info) for cam_name in cam_types: ext, args = self._get_file_format_info(self.format, 'CAMGEOM', cam_name) self.log.debug("writing {}".format(cam_name)) tel_id = cam_types[cam_name].pop() pix = self.inst.pixel_pos[tel_id] flen = self.inst.optical_foclen[tel_id] geom = CameraGeometry.guess(*pix, flen) table = geom.to_table() table.meta['SOURCE'] = self.infile table.write("{}.camgeom.{}".format(cam_name, ext), **args)
class PedestalCalculator(Component): """ Parent class for the pedestal calculators. Fills the MON.pedestal container on the base of pedestal events (preliminary version) """ tel_id = Int( 0, help='id of the telescope to calculate the pedestal values').tag( config=True) sample_duration = Int(60, help='sample duration in seconds').tag(config=True) sample_size = Int(10000, help='sample size').tag(config=True) n_channels = Int(2, help='number of channels to be treated').tag(config=True) charge_cut_outliers = List( [3, 3], help='Interval (number of std) of accepted charge values').tag( config=True) charge_std_cut_outliers = List( [3, 3], help= 'Interval (number of std) of accepted charge standard deviation values' ).tag(config=True) charge_product = Unicode( 'LocalPeakIntegrator', help='Name of the charge extractor to be used').tag(config=True) def __init__(self, **kwargs): """ Parent class for pedestal calculators. Fills the MON.pedestal container. Parameters ---------- config : traitlets.loader.Config Configuration specified by config file or cmdline arguments. Used to set traitlet values. Set to None if no configuration to pass. tool : ctapipe.core.Tool Tool executable that is calling this component. Passes the correct logger to the component. Set to None if no Tool to pass. kwargs """ super().__init__(**kwargs) # initialize the output self.container = PedestalContainer() # load the waveform charge extractor self.extractor = ChargeExtractor.from_name(self.charge_product, config=self.config) self.log.info(f"extractor {self.extractor}") @abstractmethod def calculate_pedestals(self, event): """calculate relative gain from event
class FlatFieldCalculator(Component): """ Parent class for the flat field calculators. Fills the MON.flatfield container. """ tel_id = Int( 0, help='id of the telescope to calculate the flat-field coefficients' ).tag(config=True) sample_duration = Int(60, help='sample duration in seconds').tag(config=True) sample_size = Int(10000, help='sample size').tag(config=True) n_channels = Int(2, help='number of channels to be treated').tag(config=True) charge_cut_outliers = List( [-0.3, 0.3], help= 'Interval of accepted charge values (fraction with respect to camera median value)' ).tag(config=True) time_cut_outliers = List( [10, 30], help='Interval (in samples) of accepted time values').tag(config=True) charge_product = Unicode( 'LocalPeakWindowSum', help='Name of the charge extractor to be used').tag(config=True) def __init__(self, **kwargs): """ Parent class for the flat field calculators. Fills the flatfield container. Parameters ---------- config : traitlets.loader.Config Configuration specified by config file or cmdline arguments. Used to set traitlet values. Set to None if no configuration to pass. tool : ctapipe.core.Tool Tool executable that is calling this component. Passes the correct logger to the component. Set to None if no Tool to pass. kwargs """ super().__init__(**kwargs) # load the waveform charge extractor self.extractor = ImageExtractor.from_name(self.charge_product, config=self.config) self.log.info(f"extractor {self.extractor}") @abstractmethod def calculate_relative_gain(self, event): """calculate relative gain from event
class MakeHistFirstCap(Tool): name = "make-hist-first-cap" infile = Unicode(help='input LST file', default="").tag(config=True) max_events = Integer(help='stop after this many events if non-zero', default_value=0, min=0).tag(config=True) output_suffix = Unicode(help='suffix (file extension) of output ' 'filenames to write images ' 'to (no writing is done if blank). ' 'Images will be named [EVENTID][suffix]', default_value="").tag(config=True) aliases = Dict({ 'infile': 'MakeHistFirstCap.infile', 'max-events': 'MakeHistFirstCap.max_events', 'output-suffix': 'MakeHistFirstCap.output_suffix' }) def setup(self): # load LST data self.log.info("Read file:{}".format(self.infile)) self.reader = LSTEventSource(input_url=self.infile, max_events=self.max_events) def start(self): fc = [] for event in self.reader: fc.extend(get_first_capacitor_array(event)) plt.figure() plt.hist(fc, bins=4096) plt.ylabel("Number") plt.xlabel("Stop Cell") plt.show()
class EventSource(Component): """ Parent class for EventFileReaders of different sources. A new EventFileReader should be created for each type of event file read into ctapipe, e.g. sim_telarray files are read by the `SimTelEventSource`. EventFileReader provides a common high-level interface for accessing event information from different data sources (simulation or different camera file formats). Creating an EventFileReader for a new file format ensures that data can be accessed in a common way, irregardless of the file format. EventFileReader itself is an abstract class. To use an EventFileReader you must use a subclass that is relevant for the file format you are reading (for example you must use `ctapipe.io.SimTelEventSource` to read a hessio format file). Alternatively you can use `event_source()` to automatically select the correct EventFileReader subclass for the file format you wish to read. To create an instance of an EventFileReader you must pass the traitlet configuration (containing the input_url) and the `ctapipe.core.tool.Tool`. Therefore from inside a Tool you would do: >>> event_source = EventSource(self.config, self) An example of how to use `ctapipe.core.tool.Tool` and `event_source()` can be found in ctapipe/tools/display_dl1.py. However if you are not inside a Tool, you can still create an instance and supply an input_url via: >>> event_source = EventSource( input_url="/path/to/file") To loop through the events in a file: >>> event_source = EventSource( input_url="/path/to/file") >>> for event in event_source: >>> print(event.count) **NOTE**: Every time a new loop is started through the event_source, it restarts from the first event. Alternatively one can use EventFileReader in a `with` statement to ensure the correct cleanups are performed when you are finished with the event_source: >>> with EventSource( input_url="/path/to/file") as event_source: >>> for event in event_source: >>> print(event.count) **NOTE**: The "event" that is returned from the generator is a pointer. Any operation that progresses that instance of the generator further will change the data pointed to by "event". If you wish to ensure a particular event is kept, you should perform a `event_copy = copy.deepcopy(event)`. Attributes ---------- input_url : str Path to the input event file. max_events : int Maximum number of events to loop through in generator metadata : dict A dictionary containing the metadata of the file. This could include: * is_simulation (bool indicating if the file contains simulated events) * Telescope:Camera names (list if file contains multiple) * Information in the file header * Observation ID """ input_url = Unicode( "", help="Path to the input file containing events.").tag(config=True) max_events = Int( None, allow_none=True, help="Maximum number of events that will be read from the file", ).tag(config=True) allowed_tels = Set( help=("list of allowed tel_ids, others will be ignored. " "If left empty, all telescopes in the input stream " "will be included")).tag(config=True) def __init__(self, config=None, parent=None, **kwargs): """ Class to handle generic input files. Enables obtaining the "source" generator, regardless of the type of file (either hessio or camera file). Parameters ---------- config : traitlets.loader.Config Configuration specified by config file or cmdline arguments. Used to set traitlet values. Set to None if no configuration to pass. tool : ctapipe.core.Tool Tool executable that is calling this component. Passes the correct logger to the component. Set to None if no Tool to pass. kwargs """ super().__init__(config=config, parent=parent, **kwargs) self.metadata = dict(is_simulation=False) input_url: Path = Path(self.input_url).expanduser() if not input_url.exists: raise FileNotFoundError(f"file path does not exist: '{input_url}'") self.log.info(f"INPUT PATH = {input_url}") if self.max_events: self.log.info(f"Max events being read = {self.max_events}") Provenance().add_input_file(input_url, role="DL0/Event") @staticmethod @abstractmethod def is_compatible(file_path): """ Abstract method to be defined in child class. Perform a set of checks to see if the input file is compatible with this file event_source. Parameters ---------- file_path : str File path to the event file. Returns ------- compatible : bool True if file is compatible, False if it is incompatible """ @property def is_stream(self): """ Bool indicating if input is a stream. If it is then it is incompatible with `ctapipe.io.eventseeker.EventSeeker`. TODO: Define a method to detect if it is a stream Returns ------- bool If True, then input is a stream. """ return False @property @abstractmethod def subarray(self): """ Obtain the subarray from the EventSource Returns ------- ctapipe.instrument.SubarrayDecription """ @abstractmethod def _generator(self): """ Abstract method to be defined in child class. Generator where the filling of the `ctapipe.containers` occurs. Returns ------- generator """ def __iter__(self): """ Generator that iterates through `_generator`, but keeps track of `self.max_events`. Returns ------- generator """ for event in self._generator(): yield event if self.max_events and event.count >= self.max_events - 1: break def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass @classmethod def from_url(cls, input_url, **kwargs): """ Find compatible EventSource for input_url via the `is_compatible` method of the EventSource Parameters ---------- input_url : str Filename or URL pointing to an event file kwargs Named arguments for the EventSource Returns ------- instance Instance of a compatible EventSource subclass """ if input_url == "" or input_url is None: raise ToolConfigurationError( "EventSource: No input_url was specified") detect_and_import_io_plugins() available_classes = non_abstract_children(cls) for subcls in available_classes: if subcls.is_compatible(input_url): return subcls(input_url=input_url, **kwargs) raise ValueError("Cannot find compatible EventSource for \n" "\turl:{}\n" "in available EventSources:\n" "\t{}".format(input_url, [c.__name__ for c in available_classes])) @classmethod def from_config(cls, config=None, parent=None, **kwargs): """ Find compatible EventSource for the EventSource.input_url traitlet specified via the config. This method is typically used in Tools, where the input_url is chosen via the command line using the traitlet configuration system. Parameters ---------- config : traitlets.config.loader.Config Configuration created in the Tool kwargs Named arguments for the EventSource Returns ------- instance Instance of a compatible EventSource subclass """ if config is None: config = parent.config if isinstance(config.EventSource.input_url, LazyConfigValue): config.EventSource.input_url = cls.input_url.default_value elif not isinstance(config.EventSource.input_url, str): raise TraitError("Wrong type specified for input_url traitlet") return event_source(config.EventSource.input_url, config=config, **kwargs)
class SingleTelEventDisplay(Tool): name = "ctapipe-display-televents" description = Unicode(__doc__) infile = Path(help="input file to read", exists=True, directory_ok=False).tag(config=True) tel = Int(help="Telescope ID to display", default_value=0).tag(config=True) write = Bool(help="Write out images to PNG files", default_value=False).tag(config=True) clean = Bool(help="Apply image cleaning", default_value=False).tag(config=True) hillas = Bool(help="Apply and display Hillas parametrization", default_value=False).tag(config=True) samples = Bool(help="Show each sample", default_value=False).tag(config=True) display = Bool(help="Display results in interactive window", default_value=True).tag(config=True) delay = Float(help="delay between events in s", default_value=0.01, min=0.001).tag(config=True) progress = Bool(help="display progress bar", default_value=True).tag(config=True) aliases = Dict({ "infile": "SingleTelEventDisplay.infile", "tel": "SingleTelEventDisplay.tel", "max-events": "EventSource.max_events", "write": "SingleTelEventDisplay.write", "clean": "SingleTelEventDisplay.clean", "hillas": "SingleTelEventDisplay.hillas", "samples": "SingleTelEventDisplay.samples", "display": "SingleTelEventDisplay.display", "delay": "SingleTelEventDisplay.delay", "progress": "SingleTelEventDisplay.progress", }) classes = List([EventSource, CameraCalibrator]) def __init__(self, **kwargs): super().__init__(**kwargs) def setup(self): print("TOLLES INFILE", self.infile) self.event_source = EventSource.from_url(self.infile, parent=self) self.event_source.allowed_tels = {self.tel} self.calibrator = CameraCalibrator(parent=self, subarray=self.event_source.subarray) self.log.info(f"SELECTING EVENTS FROM TELESCOPE {self.tel}") def start(self): disp = None for event in tqdm( self.event_source, desc=f"Tel{self.tel}", total=self.event_source.max_events, disable=~self.progress, ): self.log.debug(event.trigger) self.log.debug(f"Energy: {event.simulation.shower.energy}") self.calibrator(event) if disp is None: geom = self.event_source.subarray.tel[self.tel].camera.geometry self.log.info(geom) disp = CameraDisplay(geom) # disp.enable_pixel_picker() disp.add_colorbar() if self.display: plt.show(block=False) # display the event disp.axes.set_title("CT{:03d} ({}), event {:06d}".format( self.tel, geom.camera_name, event.index.event_id)) if self.samples: # display time-varying event data = event.dl0.tel[self.tel].waveform for ii in range(data.shape[1]): disp.image = data[:, ii] disp.set_limits_percent(70) plt.suptitle(f"Sample {ii:03d}") if self.display: plt.pause(self.delay) if self.write: plt.savefig( f"CT{self.tel:03d}_EV{event.index.event_id:10d}" f"_S{ii:02d}.png") else: # display integrated event: im = event.dl1.tel[self.tel].image if self.clean: mask = tailcuts_clean(geom, im, picture_thresh=10, boundary_thresh=7) im[~mask] = 0.0 disp.image = im if self.hillas: try: ellipses = disp.axes.findobj(Ellipse) if len(ellipses) > 0: ellipses[0].remove() params = hillas_parameters(geom, image=im) disp.overlay_moments(params, color="pink", lw=3, with_label=False) except HillasParameterizationError: pass if self.display: plt.pause(self.delay) if self.write: plt.savefig( f"CT{self.tel:03d}_EV{event.index.event_id:010d}.png") self.log.info("FINISHED READING DATA FILE") if disp is None: self.log.warning( "No events for tel {} were found in {}. Try a " "different EventIO file or another telescope".format( self.tel, self.infile))
class PulseTimeCorrection(Component): """ The PulseTimeCorrection class to correct time pulse using Fourier series expansion. """ tel_id = Int(1, help='id of the telescope to calibrate').tag(config=True) n_capacitors = Int( 1024, help='number of capacitors (1024 or 4096)').tag(config=True) calib_file_path = Unicode( '', allow_none=True, help='Path to the time calibration file').tag(config=True) def __init__(self, **kwargs): super().__init__(**kwargs) self.n_harmonics = None self.fan_array = None # array to store cos coeff for Fourier series expansion self.fbn_array = None # array to store sin coeff for Fourier series expansion self.first_cap_array = np.zeros((n_modules, n_gain, n_channel)) self.load_calib_file() def load_calib_file(self): """ Function to load calibration file. """ try: with h5py.File(self.calib_file_path, 'r') as hf: self.n_harmonics = hf["/"].attrs['n_harm'] fan = hf.get('fan') self.fan_array = np.array(fan) fbn = hf.get('fbn') self.fbn_array = np.array(fbn) except: self.log.error( f"Problem in reading time from calibration file {self.calib_file_path}" ) def get_corr_pulse(self, event, pulse): """ Return pulse time after time correction. Parameters ---------- event : `ctapipe` event-container pulse : ndarray pulse time in each pixel. Stored in a numpy array of shape (2, 1855). """ pixel_ids = event.lst.tel[self.tel_id].svc.pixel_ids n_modules_from_event = event.lst.tel[self.tel_id].svc.num_modules pulse_corr = np.empty((n_gain, n_pixels)) for nr in prange(0, n_modules_from_event): self.first_cap_array[nr, :, :] = self.get_first_capacitor( event, nr) self.get_corr_pulse_jit(pulse, pulse_corr, pixel_ids, self.first_cap_array, self.fan_array, self.fbn_array, self.n_harmonics, self.n_capacitors) return pulse_corr @staticmethod @njit(parallel=True) def get_corr_pulse_jit(pulse, pulse_corr, pixel_ids, first_capacitor, fan_array, fbn_array, n_harmonics, n_cap): """ Numba function for pulse time correction. Parameters ---------- pulse : ndarray Pulse time stored in a numpy array of shape (n_gain, n_pixels). pulse_corr : ndarray Pulse correction time stored in a numpy array of shape (n_gain, n_pixels). pixel_ids: ndarray Array stored expected pixel id (n_pixels). first_capacitor : ndarray Value of first capacitor stored in a numpy array of shape (n_clus, n_gain, n_pix). fan_array : ndarray Array to store coeff for Fourier series expansion stored in a numpy array of shape (n_gain, n_pixels, n_harmonics). fbn_array : ndarray Array to store coeff for Fourier series expansion stored in a numpy array of shape (n_gain, n_pixels, n_harmonics). n_harmonics : int Number of harmonics """ for gain in prange(0, n_gain): for nr in prange(0, n_modules): for pix in prange(0, n_channel): fc = first_capacitor[nr, gain, pix] pixel = pixel_ids[nr * 7 + pix] pulse_corr[gain, pixel] = pulse[gain, pixel] - get_corr_time_jit( fc % n_cap, fan_array[gain, pixel], fbn_array[gain, pixel], n_harmonics, n_cap) def get_first_capacitor(self, event, nr): """ Get first capacitor values from event for nr module. Parameters ---------- event : `ctapipe` event-container nr_module : number of module tel_id : id of the telescope """ fc = np.zeros((n_gain, n_channel)) first_cap = event.lst.tel[ self.tel_id].evt.first_capacitor_id[nr * 8:(nr + 1) * 8] # First capacitor order according Dragon v5 board data format for i, j in zip([0, 1, 2, 3, 4, 5, 6], [0, 0, 1, 1, 2, 2, 3]): fc[high_gain, i] = first_cap[j] for i, j in zip([0, 1, 2, 3, 4, 5, 6], [4, 4, 5, 5, 6, 6, 7]): fc[low_gain, i] = first_cap[j] return fc
class DumpInstrumentTool(Tool): description = Unicode(__doc__) name = "ctapipe-dump-instrument" infile = Path(exists=True, help="input simtelarray file").tag(config=True) format = Enum( ["fits", "ecsv", "hdf5"], default_value="fits", help="Format of output file", config=True, ) aliases = Dict( dict(infile="DumpInstrumentTool.infile", format="DumpInstrumentTool.format")) def setup(self): with event_source(self.infile) as source: self.subarray = source.subarray def start(self): if self.format == "hdf5": self.subarray.to_hdf("subarray.h5") else: self.write_camera_geometries() self.write_optics_descriptions() self.write_subarray_description() def finish(self): pass @staticmethod def _get_file_format_info(format_name, table_type, table_name): """ returns file extension + dict of required parameters for Table.write""" if format_name == "fits": return "fits.gz", dict() elif format_name == "ecsv": return "ecsv.txt", dict(format="ascii.ecsv") else: raise NameError(f"format {format_name} not supported") def write_camera_geometries(self): cam_types = get_camera_types(self.subarray) self.subarray.info(printer=self.log.info) for cam_name in cam_types: ext, args = self._get_file_format_info(self.format, "CAMGEOM", cam_name) self.log.debug(f"writing {cam_name}") tel_id = cam_types[cam_name].pop() geom = self.subarray.tel[tel_id].camera.geometry table = geom.to_table() table.meta["SOURCE"] = str(self.infile) filename = f"{cam_name}.camgeom.{ext}" try: table.write(filename, **args) Provenance().add_output_file(filename, "dl0.tel.svc.camera") except IOError as err: self.log.warning( "couldn't write camera definition '%s' because: %s", filename, err) def write_optics_descriptions(self): sub = self.subarray ext, args = self._get_file_format_info(self.format, sub.name, "optics") tab = sub.to_table(kind="optics") tab.meta["SOURCE"] = str(self.infile) filename = f"{sub.name}.optics.{ext}" try: tab.write(filename, **args) Provenance().add_output_file(filename, "dl0.sub.svc.optics") except IOError as err: self.log.warning( "couldn't write optics description '%s' because: %s", filename, err) def write_subarray_description(self): sub = self.subarray ext, args = self._get_file_format_info(self.format, sub.name, "subarray") tab = sub.to_table(kind="subarray") tab.meta["SOURCE"] = str(self.infile) filename = f"{sub.name}.subarray.{ext}" try: tab.write(filename, **args) Provenance().add_output_file(filename, "dl0.sub.svc.subarray") except IOError as err: self.log.warning( "couldn't write subarray description '%s' because: %s", filename, err)
class TimeCorrectionCalculate(Component): """ The TimeCorrectionCalculate class to create h5py file with coefficients for time correction curve of chip DRS4. Description of this method: "Analysis techniques and performance of the Domino Ring Sampler version 4 based readout for the MAGIC telescopes [arxiv:1305.1007] """ minimum_charge = Float( 200, help='Cut on charge. Default 200 ADC').tag(config=True) tel_id = Int(1, help='Id of the telescope to calibrate').tag(config=True) n_combine = Int( 8, help='How many capacitors are combines in a single bin. Default 8' ).tag(config=True) n_harmonics = Int( 16, help='Number of harmonic for Fourier series expansion. Default 16' ).tag(config=True) n_capacitors = Int( 1024, help='Number of capacitors (1024 or 4096). Default 1024.').tag( config=True) charge_product = Unicode( 'LocalPeakWindowSum', help='Name of the charge extractor to be used').tag(config=True) calib_file_path = Unicode( '', allow_none=True, help='Path to the time calibration file').tag(config=True) def __init__(self, **kwargs): super().__init__(**kwargs) self.n_bins = int(self.n_capacitors / self.n_combine) self.mean_values_per_bin = np.zeros((n_gain, n_pixels, self.n_bins)) self.entries_per_bin = np.zeros((n_gain, n_pixels, self.n_bins)) self.first_cap_array = np.zeros((n_modules, n_gain, n_channel)) # load the waveform charge extractor self.extractor = ImageExtractor.from_name(self.charge_product, config=self.config) self.log.info(f"extractor {self.extractor}") self.sum_events = 0 def calibrate_pulse_time(self, event): """ Fill bins using time pulse from LocalPeakWindowSum. Parameters ---------- event : `ctapipe` event-container """ if event.r1.tel[self.tel_id].trigger_type == 1: for nr_module in prange(0, n_modules): self.first_cap_array[ nr_module, :, :] = self.get_first_capacitor( event, nr_module) pixel_ids = event.lst.tel[self.tel_id].svc.pixel_ids charge, pulse_time = self.extractor( event.r1.tel[self.tel_id].waveform) self.calib_pulse_time_jit(charge, pulse_time, pixel_ids, self.first_cap_array, self.mean_values_per_bin, self.entries_per_bin, n_cap=self.n_capacitors, n_combine=self.n_combine, min_charge=self.minimum_charge) self.sum_events += 1 @jit(parallel=True) def calib_pulse_time_jit(self, charge, pulse_time, pixel_ids, first_cap_array, mean_values_per_bin, entries_per_bin, n_cap, n_combine, min_charge): """ Numba function for calibration pulse time. Parameters ---------- pulse : ndarray Pulse time stored in a numpy array of shape (n_gain, n_pixels). charge : ndarray Charge in each pixel. (n_gain, n_pixels). pixel_ids: ndarray Array stored expected pixel id (n_pixels). first_cap_array : ndarray Value of first capacitor stored in a numpy array of shape (n_clus, n_gain, n_pix). mean_values_per_bin : ndarray Array to fill using pulse time stored in a numpy array of shape (n_gain, n_pixels, n_bins). entries_per_bin : ndarray Array to store number of entries per bin stored in a numpy array of shape (n_gain, n_pixels, n_bins). n_cap : int Number of capacitors n_combine : int Number of combine capacitors in a single bin """ for nr_module in prange(0, n_modules): for gain in prange(0, n_gain): for pix in prange(0, n_channel): pixel = pixel_ids[nr_module * 7 + pix] if charge[gain, pixel] > min_charge: # cut change fc = first_cap_array[nr_module, :, :] first_cap = (fc[gain, pix]) % n_cap bin = int(first_cap / n_combine) mean_values_per_bin[gain, pixel, bin] += pulse_time[gain, pixel] entries_per_bin[gain, pixel, bin] += 1 def finalize(self): if np.sum(self.entries_per_bin == 0) > 0: raise RuntimeError( "Not enough events to coverage all capacitor. " "Please use more events to time calibration file.") else: self.mean_values_per_bin = self.mean_values_per_bin / self.entries_per_bin self.save_to_hdf5_file() def fit(self, pixel_id, gain): """ Fit data bins using Fourier series expansion Parameters ---------- pixel_id : ndarray Array stored expected pixel id of shape (n_pixels). gain: int 0 for high gain, 1 for low gain """ self.pos = np.zeros(self.n_bins) for i in range(0, self.n_bins): self.pos[i] = (i + 0.5) * self.n_combine self.fan = np.zeros(self.n_harmonics) # cos coeff self.fbn = np.zeros(self.n_harmonics) # sin coeff for n in range(0, self.n_harmonics): self.integrate_with_trig(self.pos, self.mean_values_per_bin[gain, pixel_id], n, self.fan, self.fbn) def integrate_with_trig(self, x, y, n, an, bn): """ Function to expanding into Fourier series Parameters ---------- x : ndarray Array stored position in DRS4 ring of shape (n_bins). y: ndarray Array stored mean pulse time per bin of shape (n_bins) n : int n harmonic an: ndarray Array to fill with cos coeff of shape (n_harmonics) bn: ndarray Array to fill with sin coeff of shape (n_harmonics) """ suma = 0 sumb = 0 for i in range(0, self.n_bins): suma += y[i] * self.n_combine * np.cos( 2 * np.pi * n * (x[i] / float(self.n_capacitors))) sumb += y[i] * self.n_combine * np.sin( 2 * np.pi * n * (x[i] / float(self.n_capacitors))) an[n] = suma * (2. / (self.n_bins * self.n_combine)) bn[n] = sumb * (2. / (self.n_bins * self.n_combine)) def get_first_capacitor(self, event, nr): fc = np.zeros((n_gain, n_channel)) first_cap = event.lst.tel[ self.tel_id].evt.first_capacitor_id[nr * 8:(nr + 1) * 8] # First capacitor order according Dragon v5 board data format for i, j in zip([0, 1, 2, 3, 4, 5, 6], [0, 0, 1, 1, 2, 2, 3]): fc[high_gain, i] = first_cap[j] for i, j in zip([0, 1, 2, 3, 4, 5, 6], [4, 4, 5, 5, 6, 6, 7]): fc[low_gain, i] = first_cap[j] return fc def save_to_hdf5_file(self): """ Function to save Fourier series expansion coeff into hdf5 file """ fan_array = np.zeros((n_gain, n_pixels, self.n_harmonics)) fbn_array = np.zeros((n_gain, n_pixels, self.n_harmonics)) for pix_id in range(0, n_pixels): self.fit(pix_id, gain=high_gain) fan_array[high_gain, pix_id, :] = self.fan fbn_array[high_gain, pix_id, :] = self.fbn self.fit(pix_id, gain=low_gain) fan_array[low_gain, pix_id, :] = self.fan fbn_array[low_gain, pix_id, :] = self.fbn try: hf = h5py.File(self.calib_file_path, 'w') hf.create_dataset('fan', data=fan_array) hf.create_dataset('fbn', data=fbn_array) hf.attrs['n_events'] = self.sum_events hf.attrs['n_harm'] = self.n_harmonics except Exception as err: print("FAILED!", err) hf.close()
class LSTCameraCalibrator(CameraCalibrator): """ Calibrator to handle the LST camera calibration chain, in order to fill the DL1 data level in the event container. """ extractor_product = Unicode( 'NeighborPeakWindowSum', help='Name of the charge extractor to be used').tag(config=True) reducer_product = Unicode( 'NullDataVolumeReducer', help='Name of the DataVolumeReducer to use').tag(config=True) calibration_path = Unicode( '', allow_none=True, help='Path to LST calibration file').tag(config=True) time_calibration_path = Unicode( '', allow_none=True, help='Path to drs4 time calibration file').tag(config=True) allowed_tels = List( [1], help='List of telescope to be calibrated').tag(config=True) gain_threshold = Int( 4094, allow_none=True, help='Threshold for the gain selection in ADC').tag(config=True) def __init__(self, **kwargs): """ Parameters ---------- reducer_product : ctapipe.image.reducer.DataVolumeReducer The DataVolumeReducer to use. If None, then NullDataVolumeReducer will be used by default, and waveforms will not be reduced. extractor_product : ctapipe.image.extractor.ImageExtractor The ImageExtractor to use. If None, then NeighborPeakWindowSum will be used by default. calibration_path : Path to LST calibration file to get the pedestal and flat-field corrections kwargs """ super().__init__(**kwargs) # load the waveform charge extractor self.image_extractor = ImageExtractor.from_name(self.extractor_product, config=self.config) self.log.info(f"extractor {self.extractor_product}") self.data_volume_reducer = DataVolumeReducer.from_name( self.reducer_product, config=self.config) self.log.info(f" {self.reducer_product}") # declare gain selector if the threshold is defined if self.gain_threshold: self.gain_selector = gainselection.ThresholdGainSelector( threshold=self.gain_threshold) # declare time calibrator if correction file exist if os.path.exists(self.time_calibration_path): self.time_corrector = PulseTimeCorrection( calib_file_path=self.time_calibration_path) else: self.time_corrector = None self.log.info( f"File {self.time_calibration_path} not found. No drs4 time corrections" ) # calibration data container self.mon_data = MonitoringContainer() # initialize the MonitoringContainer() for the moment it reads it from a hdf5 file self._initialize_correction() def _initialize_correction(self): """ Read the correction from hdf5 calibration file """ self.mon_data.tels_with_data = self.allowed_tels self.log.info(f"read {self.calibration_path}") try: with HDF5TableReader(self.calibration_path) as h5_table: assert h5_table._h5file.isopen == True for telid in self.allowed_tels: # read the calibration data for the moment only one event table = '/tel_' + str(telid) + '/calibration' next( h5_table.read(table, self.mon_data.tel[telid].calibration)) # eliminate inf values (should be done probably before) dc_to_pe = self.mon_data.tel[telid].calibration.dc_to_pe dc_to_pe[np.isinf(dc_to_pe)] = 0 self.log.info( f"read {self.mon_data.tel[telid].calibration.dc_to_pe}" ) except: self.log.error( f"Problem in reading calibration file {self.calibration_path}") def _calibrate_dl0(self, event, telid): """ create dl0 level, for the moment copy the r1 """ waveforms = event.r1.tel[telid].waveform if self._check_r1_empty(waveforms): return event.dl0.event_id = event.r1.event_id event.mon.tel[telid].calibration = self.mon_data.tel[telid].calibration # subtract the pedestal per sample (should we do it?) and multiply for the calibration coefficients # event.dl0.tel[telid].waveform = ( (event.r1.tel[telid].waveform - self.mon_data.tel[telid]. calibration.pedestal_per_sample[:, :, np.newaxis]) * self.mon_data.tel[telid].calibration.dc_to_pe[:, :, np.newaxis]) def _calibrate_dl1(self, event, telid): """ create calibrated dl1 image and calibrate it """ waveforms = event.dl0.tel[telid].waveform if self._check_dl0_empty(waveforms): return if self.image_extractor.requires_neighbors(): camera = event.inst.subarray.tel[telid].camera self.image_extractor.neighbors = camera.neighbor_matrix_where charge, pulse_time = self.image_extractor(waveforms) # correct time with drs4 correction if available if self.time_corrector: pulse_corr_array = self.time_corrector.get_corr_pulse( event, pulse_time) # otherwise use the ff time correction (not drs4 corrected) else: pulse_corr_array = pulse_time + self.mon_data.tel[ telid].calibration.time_correction # perform the gain selection if the threshold is defined if self.gain_threshold: waveforms, gain_mask = self.gain_selector( event.r1.tel[telid].waveform) event.dl1.tel[telid].image = charge[gain_mask, np.arange(charge.shape[1])] event.dl1.tel[telid].pulse_time = pulse_corr_array[ gain_mask, np.arange(pulse_corr_array.shape[1])] # remember the mask in the lst pixel_status array (this info is missing for the moment in the # r1 container). I follow the prescription given in the document # "R1 & DL0 Telescope Event Interfaces and Prototype Evaluation" of K. Kosack # bit 2 = LG gain_mask *= 4 # bit 3 = HG gain_mask[np.where(gain_mask == 0)] = 8 # bit 1 = pixel broken pixel (coming from the EvB) gain_mask += event.lst.tel[telid].evt.pixel_status >> 1 & 1 # update pixel status event.lst.tel[telid].evt.pixel_status = gain_mask # if threshold == None else: event.dl1.tel[telid].image = charge event.dl1.tel[telid].pulse_time = pulse_corr_array
class LSTR0Corrections(CameraR0Calibrator): """ The R0 calibrator class for LST Camera. """ pedestal_path = Unicode( '', allow_none=True, help='Path to the LST pedestal binary file').tag(config=True) def __init__(self, config=None, tool=None, offset=300, **kwargs): """ The R0 calibrator for LST data. Change the r0 container. Parameters ---------- config : traitlets.loader.Config Configuration specified by config file or cmdline arguments. Used to set traitlet values. Set to None if no configuration to pass. tool : ctapipe.core.Tool Tool executable that is calling this component. Passes the correct logger to the component. Set to None if no Tool to pass. kwargs """ super().__init__(config=config, tool=tool, **kwargs) self.telid = 0 self.n_module = 265 self.n_gain = 2 self.n_pix = 7 self.size4drs = 4 * 1024 self.roisize = 40 self.offset = offset self.high_gain = 0 self.low_gain = 1 self.pedestal_value_array = np.zeros( (self.n_gain, self.n_pix * self.n_module, self.size4drs + 40), dtype=np.int16) self.first_cap_array = np.zeros( (self.n_module, self.n_gain, self.n_pix)) self.first_cap_time_lapse_array = np.zeros( (self.n_module, self.n_gain, self.n_pix)) self.last_reading_time_array = np.zeros( (self.n_module, self.n_gain, self.n_pix, self.size4drs)) self.first_cap_array_spike = np.zeros( (self.n_module, self.n_gain, self.n_pix)) self.first_cap_old_array = np.zeros( (self.n_module, self.n_gain, self.n_pix)) self._load_calib() def subtract_pedestal(self, event): """ Subtract cell offset using pedestal file. Change the R0 container. Parameters ---------- event : `ctapipe` event-container """ n_modules = event.lst.tel[0].svc.num_modules for nr_module in range(0, n_modules): self.first_cap_array[nr_module, :, :] = self._get_first_capacitor( event, nr_module) expected_pixel_id = event.lst.tel[0].svc.pixel_ids event.r0.tel[self.telid].waveform[:, :, :] = subtract_pedestal_jit( event.r0.tel[self.telid].waveform, expected_pixel_id, self.first_cap_array, self.pedestal_value_array, n_modules) event.r0.tel[self.telid].waveform.astype(np.uint16) def time_lapse_corr(self, event): """ Perform time lapse baseline corrections. Change the R0 container. Parameters ---------- event : `ctapipe` event-container """ expected_pixel_id = event.lst.tel[0].svc.pixel_ids local_clock_list = event.lst.tel[0].evt.local_clock_counter n_modules = event.lst.tel[0].svc.num_modules for nr_module in range(0, n_modules): self.first_cap_time_lapse_array[ nr_module, :, :] = self._get_first_capacitor(event, nr_module) do_time_lapse_corr(event.r0.tel[0].waveform, expected_pixel_id, local_clock_list, self.first_cap_time_lapse_array, self.last_reading_time_array, n_modules) event.r0.tel[self.telid].waveform.astype(np.uint16) def interpolate_spikes(self, event): """ Interpolates spike A & B. Change the R0 container. Parameters ---------- event : `ctapipe` event-container """ self.first_cap_old_array[:, :, :] = self.first_cap_array_spike[:, :, :] n_modules = event.lst.tel[0].svc.num_modules for nr_module in range(0, n_modules): self.first_cap_array_spike[ nr_module, :, :] = self._get_first_capacitor(event, nr_module) waveform = event.r0.tel[0].waveform[:, :, :] expected_pixel_id = event.lst.tel[0].svc.pixel_ids wf = waveform.copy() wf = wf.astype('int16') event.r0.tel[0].waveform = self.interpolate_pseudo_pulses( wf, expected_pixel_id, self.first_cap_array_spike, self.first_cap_old_array, n_modules) event.r0.tel[self.telid].waveform.astype(np.uint16) @staticmethod @jit(parallel=True) def interpolate_pseudo_pulses(waveform, expected_pixel_id, fc, fc_old, n_modules): """ Interpolate Spike A & B. Change waveform array. Parameters ---------- waveform : ndarray Waveform stored in a numpy array of shape (n_gain, n_pix, n_samples). expected_pixel_id: ndarray Array stored expected pixel id (n_pix*n_modules). fc : ndarray Value of first capacitor stored in a numpy array of shape (n_clus, n_gain, n_pix). fc_old : ndarray Value of first capacitor from previous event stored in a numpy array of shape (n_clus, n_gain, n_pix). n_modules : int Number of modules """ roisize = 40 size4drs = 4096 n_gain = 2 n_pix = 7 for nr_module in prange(0, n_modules): for gain in prange(0, n_gain): for pix in prange(0, n_pix): for k in prange(0, 4): # looking for spike A first case abspos = int(1024 - roisize - 2 - fc_old[nr_module, gain, pix] + k * 1024 + size4drs) spike_A_position = int( (abspos - fc[nr_module, gain, pix] + size4drs) % size4drs) if (spike_A_position > 2 and spike_A_position < 38): pixel = expected_pixel_id[nr_module * 7 + pix] interpolate_spike_A(waveform, gain, spike_A_position, pixel) # looking for spike A second case abspos = int(roisize - 2 + fc_old[nr_module, gain, pix] + k * 1024 + size4drs) spike_A_position = int( (abspos - fc[nr_module, gain, pix] + size4drs) % size4drs) if (spike_A_position > 2 and spike_A_position < 38): pixel = expected_pixel_id[nr_module * 7 + pix] interpolate_spike_A(waveform, gain, spike_A_position, pixel) # looking for spike B spike_b_position = int( (fc_old[nr_module, gain, pix] - 1 - fc[nr_module, gain, pix] + 2 * size4drs) % size4drs) if spike_b_position < roisize - 1: pixel = expected_pixel_id[nr_module * 7 + pix] interpolate_spike_B(waveform, gain, spike_b_position, pixel) return waveform def _load_calib(self): """ Function to load pedestal file. """ if self.pedestal_path: with fits.open(self.pedestal_path) as f: pedestal_data = np.int16(f[1].data) self.pedestal_value_array[:, :, :self. size4drs] = pedestal_data - self.offset self.pedestal_value_array[:, :, self.size4drs:self.size4drs + 40] \ = pedestal_data[:, :, 0:40] - self.offset def _get_first_capacitor(self, event, nr_module): """ Get first capacitor values from event for nr module. Parameters ---------- event : `ctapipe` event-container nr_module : number of module """ fc = np.zeros((2, 7)) first_cap = event.lst.tel[0].evt.first_capacitor_id[nr_module * 8:(nr_module + 1) * 8] # First capacitor order according Dragon v5 board data format for i, j in zip([0, 1, 2, 3, 4, 5, 6], [0, 0, 1, 1, 2, 2, 3]): fc[self.high_gain, i] = first_cap[j] for i, j in zip([0, 1, 2, 3, 4, 5, 6], [4, 4, 5, 5, 6, 6, 7]): fc[self.low_gain, i] = first_cap[j] return fc
class TimeSamplingCorrection(Component): """ The PulseTimeCorrection class to correct time pulse using Fourier series expansion. """ time_sampling_correction_path = Unicode( '', help='Path to the waveform sampling correction file', allow_none = True, ).tag(config=True) def __init__(self, **kwargs): super().__init__(**kwargs) self.time_sampling_coefficients = None self.load_sampling_coefficient_file() def load_sampling_coefficient_file(self): """ Function to load sampling coefficient file. """ try: with h5py.File(self.time_sampling_correction_path, 'r') as hf: self.time_sampling_coefficients = np.array(hf['sampling_interval_coefficient']) except: self.log.error(f"Problem in reading sampling coefficient file {self.time_sampling_correction_path}") def get_corrections(self, event, telid): """ Get the time/charge sampling corrections for one event and one telescope Parameters ---------- event: general event container telid: id of the telescope Return ------ sampling_corrections: np.array (n_gains, n_pixels, n_samples) with the correction factors """ n_gains = 2 n_modules = 265 # number of modules in LST's camera. n_pixel_per_module = 7 # number of pixels per one module. size_drs4 = 1024 n_pixels = 1855 roi = 36 # shift the first capacitor to the value used in the r1 r1_roi_start = 3 fc_all = event.lst.tel[telid].evt.first_capacitor_id # get the first capacitor per pixel and gain fc = np.zeros((n_gains, n_pixels), dtype=np.int16) for k in range(n_modules): for n in range(n_pixel_per_module): fc[0][k * 7 + n] = fc_all[8 * k + n // 2] fc[1][k * 7 + n] = fc_all[8 * k + n // 2 + 4] # reorder first capacitor as in waveform fc_ordered = np.zeros((n_gains, n_pixels), dtype=np.int16) fc_ordered[:, event.lst.tel[telid].svc.pixel_ids] = fc # shift the first capacitor to the value used in the r1 fc_ordered = fc_ordered + r1_roi_start # first capacitor in the drs4 fc_drs4 = (fc_ordered[:, :]) % size_drs4 # how many slices to the end of buffer fc_to_last = size_drs4 - fc_drs4[:, :] # initialize the charge correction vector to one sampling_corrections = np.ones((n_gains, n_pixels, roi)) # loop over the gains and pixels for gain in range(n_gains): for pix in range(n_pixels): # if I am at the end of the 1024 if 0 < fc_to_last[gain, pix] < roi: # I complete the buffer sampling_corrections[gain, pix, :fc_to_last[gain, pix]] = ( sampling_corrections[gain, pix,:fc_to_last[gain, pix]] * self.time_sampling_coefficients[gain, pix, fc_drs4[gain, pix]:]) # I start again from the beginning of the buffer sampling_corrections[gain, pix, fc_to_last[gain, pix]:] = ( sampling_corrections[gain, pix,fc_to_last[gain, pix]:] * self.time_sampling_coefficients[gain, pix,:roi - fc_to_last[gain, pix]]) else: sampling_corrections[gain, pix, :] = ( sampling_corrections[gain, pix] * self.time_sampling_coefficients[gain, pix, fc_drs4[gain, pix]:fc_drs4[gain, pix] + roi]) return sampling_corrections
class FlatFieldCalculator(Component): """ Parent class for the flat-field calculators. Fills the MonitoringCameraContainer.FlatfieldContainer on the base of a given flat-field event sample. The sample is defined by a maximal interval of time (sample_duration) or a minimal number of events (sample_duration). The calculator is supposed to be called in an event loop, extract and collect the event charge and fill the PedestalContainer Parameters ---------- tel_id : int id of the telescope (default 0) sample_duration : int interval of time (s) used to gather the pedestal statistics sample_size : int number of pedestal events requested for the statistics n_channels : int number of waveform channel to be considered charge_product : str Name of the charge extractor to be used config : traitlets.loader.Config Configuration specified by config file or cmdline arguments. Used to set traitlet values. Set to None if no configuration to pass. kwargs """ tel_id = Int( 0, help="id of the telescope to calculate the flat-field coefficients" ).tag(config=True) sample_duration = Int(60, help="sample duration in seconds").tag(config=True) sample_size = Int(10000, help="sample size").tag(config=True) n_channels = Int(2, help="number of channels to be treated").tag(config=True) charge_product = Unicode( "LocalPeakWindowSum", help="Name of the charge extractor to be used" ).tag(config=True) def __init__(self, subarray, **kwargs): """ Parent class for the flat-field calculators. Fills the MonitoringCameraContainer.FlatfieldContainer on the base of a given flat-field event sample. The sample is defined by a maximal interval of time (sample_duration) or a minimal number of events (sample_duration). The calculator is supposed to be called in an event loop, extract and collect the event charge and fill the PedestalContainer Parameters ---------- subarray: ctapipe.instrument.SubarrayDescription Description of the subarray tel_id : int id of the telescope (default 0) sample_duration : int interval of time (s) used to gather the pedestal statistics sample_size : int number of pedestal events requested for the statistics n_channels : int number of waveform channel to be considered charge_product : str Name of the charge extractor to be used config : traitlets.loader.Config Configuration specified by config file or cmdline arguments. Used to set traitlet values. Set to None if no configuration to pass. kwargs """ super().__init__(**kwargs) # load the waveform charge extractor self.extractor = ImageExtractor.from_name( self.charge_product, config=self.config, subarray=subarray ) self.log.info(f"extractor {self.extractor}") @abstractmethod def calculate_relative_gain(self, event): """
class FlasherFlatFieldCalculator(FlatFieldCalculator): """Calculates flat-field parameters from flasher data based on the best algorithm described by S. Fegan in MST-CAM-TN-0060 (eq. 19) Pixels are defined as outliers on the base of a cut on the pixel charge median over the full sample distribution and the pixel signal time inside the waveform time Parameters: ---------- charge_cut_outliers : List[2] Interval of accepted charge values (fraction with respect to camera median value) time_cut_outliers : List[2] Interval (in waveform samples) of accepted time values """ charge_median_cut_outliers = List( [-0.3, 0.3], help= 'Interval of accepted charge values (fraction with respect to camera median value)' ).tag(config=True) time_cut_outliers = List( [0, 60], help='Interval (in waveform samples) of accepted time values').tag( config=True) charge_std_cut_outliers = List( [-3, 3], help= 'Interval (number of std) of accepted charge standard deviation around camera median value' ).tag(config=True) time_calibration_path = Unicode( None, allow_none=True, help='Path to drs4 time calibration file').tag(config=True) def __init__(self, subarray, **kwargs): """Calculates flat-field parameters from flasher data based on the best algorithm described by S. Fegan in MST-CAM-TN-0060 (eq. 19) Pixels are defined as outliers on the base of a cut on the pixel charge median over the full sample distribution and the pixel signal time inside the waveform time Parameters: ---------- charge_cut_outliers : List[2] Interval of accepted charge values (fraction with respect to camera median value) time_cut_outliers : List[2] Interval (in waveform samples) of accepted time values """ super().__init__(subarray, **kwargs) self.log.info("Used events statistics : %d", self.sample_size) # members to keep state in calculate_relative_gain() self.num_events_seen = 0 self.time_start = None # trigger time of first event in sample self.trigger_time = None # trigger time of present event self.charge_medians = None # med. charge in camera per event in sample self.charges = None # charge per event in sample self.arrival_times = None # arrival time per event in sample self.sample_masked_pixels = None # masked pixels per event in sample if self.time_calibration_path is None: self.time_corrector = None else: # look for calibration path otherwise if os.path.exists(self.time_calibration_path): self.time_corrector = PulseTimeCorrection( calib_file_path=self.time_calibration_path) else: msg = f"Time calibration file {self.time_calibration_path} not found!" raise IOError(msg) def _extract_charge(self, event): """ Extract the charge and the time from a calibration event Parameters ---------- event : general event container """ waveforms = event.r1.tel[self.tel_id].waveform no_gain_selection = np.zeros((waveforms.shape[0], waveforms.shape[1]), dtype=np.int) # Extract charge and time charge = 0 peak_pos = 0 if self.extractor: charge, peak_pos = self.extractor(waveforms, self.tel_id, no_gain_selection) # correct time with drs4 correction if available if self.time_corrector: peak_pos = self.time_corrector.get_corr_pulse(event, peak_pos) return charge, peak_pos def calculate_relative_gain(self, event): """ calculate the flatfield statistical values and fill mon.tel[tel_id].flatfield container Parameters ---------- event : general event container Returns: True if the mon.tel[tel_id].flatfield is updated, False otherwise """ # initialize the np array at each cycle waveform = event.r1.tel[self.tel_id].waveform # re-initialize counter if self.num_events_seen == self.sample_size: self.num_events_seen = 0 pixel_mask = np.logical_or( event.mon.tel[self.tel_id].pixel_status.hardware_failing_pixels, event.mon.tel[self.tel_id].pixel_status.flatfield_failing_pixels) # real data if event.meta['origin'] != 'hessio': self.trigger_time = event.r1.tel[self.tel_id].trigger_time else: # patches for MC data if event.trig.tels_with_trigger: self.trigger_time = event.trig.gps_time.unix else: self.trigger_time = 0 if self.num_events_seen == 0: self.time_start = self.trigger_time self.setup_sample_buffers(waveform, self.sample_size) # extract the charge of the event and # the peak position (assumed as time for the moment) charge, arrival_time = self._extract_charge(event) # correct pulse time with drs4 corrections self.collect_sample(charge, pixel_mask, arrival_time) sample_age = self.trigger_time - self.time_start # check if to create a calibration event if (self.num_events_seen > 0 and (sample_age > self.sample_duration or self.num_events_seen == self.sample_size)): # update the monitoring container self.store_results(event) return True else: return False def store_results(self, event): """ Store stastical results in monitoring container Parameters ---------- event : general event container """ if self.num_events_seen == 0: raise ValueError( "No flat-field events in statistics, zero results") container = event.mon.tel[self.tel_id].flatfield # mask the part of the array not filled self.sample_masked_pixels[self.num_events_seen:] = 1 relative_gain_results = self.calculate_relative_gain_results( self.charge_medians, self.charges, self.sample_masked_pixels) time_results = self.calculate_time_results(self.arrival_times, self.sample_masked_pixels, self.time_start, self.trigger_time) result = { 'n_events': self.num_events_seen, **relative_gain_results, **time_results, } for key, value in result.items(): setattr(container, key, value) # update the flatfield mask ff_charge_failing_pixels = np.logical_or( container.charge_median_outliers, container.charge_std_outliers) event.mon.tel[self.tel_id].pixel_status.flatfield_failing_pixels = \ np.logical_or(ff_charge_failing_pixels, container.time_median_outliers) def setup_sample_buffers(self, waveform, sample_size): """Initialize sample buffers""" n_channels = waveform.shape[0] n_pix = waveform.shape[1] shape = (sample_size, n_channels, n_pix) self.charge_medians = np.zeros((sample_size, n_channels)) self.charges = np.zeros(shape) self.arrival_times = np.zeros(shape) self.sample_masked_pixels = np.zeros(shape) def collect_sample(self, charge, pixel_mask, arrival_time): """Collect the sample data""" # extract the charge of the event and # the peak position (assumed as time for the moment) good_charge = np.ma.array(charge, mask=pixel_mask) charge_median = np.ma.median(good_charge, axis=1) self.charges[self.num_events_seen] = charge self.arrival_times[self.num_events_seen] = arrival_time self.sample_masked_pixels[self.num_events_seen] = pixel_mask self.charge_medians[self.num_events_seen] = charge_median self.num_events_seen += 1 def calculate_time_results( self, trace_time, masked_pixels_of_sample, time_start, trigger_time, ): """Calculate and return the time results """ masked_trace_time = np.ma.array(trace_time, mask=masked_pixels_of_sample) # median over the sample per pixel pixel_median = np.ma.median(masked_trace_time, axis=0) # mean over the sample per pixel pixel_mean = np.ma.mean(masked_trace_time, axis=0) # std over the sample per pixel pixel_std = np.ma.std(masked_trace_time, axis=0) # median of the median over the camera median_of_pixel_median = np.ma.median(pixel_median, axis=1) # time outliers from median relative_median = pixel_median - median_of_pixel_median[:, np.newaxis] time_median_outliers = np.logical_or( pixel_median < self.time_cut_outliers[0], pixel_median > self.time_cut_outliers[1]) return { 'sample_time': (trigger_time - time_start) / 2 * u.s, 'sample_time_min': time_start * u.s, 'sample_time_max': trigger_time * u.s, 'time_mean': np.ma.getdata(pixel_mean) * u.ns, 'time_median': np.ma.getdata(pixel_median) * u.ns, 'time_std': np.ma.getdata(pixel_std) * u.ns, 'relative_time_median': np.ma.getdata(relative_median) * u.ns, 'time_median_outliers': np.ma.getdata(time_median_outliers), } def calculate_relative_gain_results( self, event_median, trace_integral, masked_pixels_of_sample, ): """Calculate and return the sample statistics""" masked_trace_integral = np.ma.array(trace_integral, mask=masked_pixels_of_sample) # median over the sample per pixel pixel_median = np.ma.median(masked_trace_integral, axis=0) # mean over the sample per pixel pixel_mean = np.ma.mean(masked_trace_integral, axis=0) # std over the sample per pixel pixel_std = np.ma.std(masked_trace_integral, axis=0) # median of the median over the camera median_of_pixel_median = np.ma.median(pixel_median, axis=1) # median of the std over the camera median_of_pixel_std = np.ma.median(pixel_std, axis=1) # std of the std over camera std_of_pixel_std = np.ma.std(pixel_std, axis=1) # relative gain relative_gain_event = masked_trace_integral / event_median[:, :, np.newaxis] # outliers from median charge_deviation = pixel_median - median_of_pixel_median[:, np.newaxis] charge_median_outliers = (np.logical_or( charge_deviation < self.charge_median_cut_outliers[0] * median_of_pixel_median[:, np.newaxis], charge_deviation > self.charge_median_cut_outliers[1] * median_of_pixel_median[:, np.newaxis])) # outliers from standard deviation deviation = pixel_std - median_of_pixel_std[:, np.newaxis] charge_std_outliers = (np.logical_or( deviation < self.charge_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis], deviation > self.charge_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis])) return { 'relative_gain_median': np.ma.getdata(np.ma.median(relative_gain_event, axis=0)), 'relative_gain_mean': np.ma.getdata(np.ma.mean(relative_gain_event, axis=0)), 'relative_gain_std': np.ma.getdata(np.ma.std(relative_gain_event, axis=0)), 'charge_median': np.ma.getdata(pixel_median), 'charge_mean': np.ma.getdata(pixel_mean), 'charge_std': np.ma.getdata(pixel_std), 'charge_std_outliers': np.ma.getdata(charge_std_outliers), 'charge_median_outliers': np.ma.getdata(charge_median_outliers), }
class CleanigPedestalImage(Component): """ Class to chceck pedestal image """ tel_id = Int(1, help='Id of the telescope to calibrate').tag(config=True) charge_product = Unicode( 'LocalPeakWindowSum', help='Name of the charge extractor to be used').tag(config=True) calib_file = Unicode('', allow_none=True, help='Path to the calibration file').tag(config=True) calib_time_file = Unicode( '', allow_none=True, help="Path to the time calibration file").tag(config=True) def __init__(self, **kwargs): super().__init__(**kwargs) self.cleaning_parameters = self.config["tailcut"] print(self.config) self.r1_dl1_calibrator = LSTCameraCalibrator( calibration_path=self.calib_file, time_calibration_path=self.calib_time_file, extractor_product=self.charge_product, config=self.config, gain_threshold=Config( self.config).gain_selector_config['threshold'], allowed_tels=[1]) def run(self, list_of_file, max_events): signal_place_after_clean = np.zeros(1855) sum_ped_ev = 0 alive_ped_ev = 0 for input_file in list_of_file: print(input_file) r0_r1_calibrator = LSTR0Corrections(pedestal_path=None, r1_sample_start=3, r1_sample_end=39) reader = LSTEventSource(input_url=input_file, max_events=max_events) for i, ev in enumerate(reader): r0_r1_calibrator.calibrate(ev) if i % 10000 == 0: print(ev.r0.event_id) if ev.lst.tel[1].evt.tib_masked_trigger == 32: sum_ped_ev += 1 self.r1_dl1_calibrator(ev) img = ev.dl1.tel[1].image geom = ev.inst.subarray.tel[1].camera clean = tailcuts_clean(geom, img, **self.cleaning_parameters) cleaned = img.copy() cleaned[~clean] = 0.0 signal_place_after_clean[np.where(clean == True)] += 1 if np.sum(cleaned > 0) > 0: alive_ped_ev += 1 fig, ax = plt.subplots(figsize=(10, 8)) geom = ev.inst.subarray.tel[1].camera disp0 = CameraDisplay(geom, ax=ax) disp0.image = signal_place_after_clean / sum_ped_ev disp0.add_colorbar(ax=ax, label="N times signal remain after cleaning [%]") disp0.cmap = 'gnuplot2' ax.set_title("{} \n {}/{}".format( input_file.split("/")[-1][8:21], alive_ped_ev, sum_ped_ev), fontsize=25) print("{}/{}".format(alive_ped_ev, sum_ped_ev)) ax.set_xlabel(" ") ax.set_ylabel(" ") plt.tight_layout() plt.show() def remove_star_and_run(self, list_of_file, max_events, noise_pixels_id_list): signal_place_after_clean = np.zeros(1855) sum_ped_ev = 0 alive_ped_ev = 0 for input_file in list_of_file: print(input_file) r0_r1_calibrator = LSTR0Corrections(pedestal_path=None, r1_sample_start=3, r1_sample_end=39) reader = LSTEventSource(input_url=input_file, max_events=max_events) for i, ev in enumerate(reader): r0_r1_calibrator.calibrate(ev) if i % 10000 == 0: print(ev.r0.event_id) if ev.lst.tel[1].evt.tib_masked_trigger == 32: sum_ped_ev += 1 self.r1_dl1_calibrator(ev) img = ev.dl1.tel[1].image img[noise_pixels_id_list] = 0 geom = ev.inst.subarray.tel[1].camera clean = tailcuts_clean(geom, img, **self.cleaning_parameters) cleaned = img.copy() cleaned[~clean] = 0.0 signal_place_after_clean[np.where(clean == True)] += 1 if np.sum(cleaned > 0) > 0: alive_ped_ev += 1 fig, ax = plt.subplots(figsize=(10, 8)) geom = ev.inst.subarray.tel[1].camera disp0 = CameraDisplay(geom, ax=ax) disp0.image = signal_place_after_clean / sum_ped_ev disp0.highlight_pixels(noise_pixels_id_list, linewidth=3) disp0.add_colorbar(ax=ax, label="N times signal remain after cleaning [%]") disp0.cmap = 'gnuplot2' ax.set_title("{} \n {}/{}".format( input_file.split("/")[-1][8:21], alive_ped_ev, sum_ped_ev), fontsize=25) print("{}/{}".format(alive_ped_ev, sum_ped_ev)) ax.set_xlabel(" ") ax.set_ylabel(" ") plt.tight_layout() plt.show() def plot_camera_display(self, image, input_file, noise_pixels_id_list, alive_ped_ev, sum_ped_ev): fig, ax = plt.subplots(figsize=(10, 8)) geom = CameraGeometry.from_name('LSTCam-003') disp0 = CameraDisplay(geom, ax=ax) disp0.image = image disp0.highlight_pixels(noise_pixels_id_list, linewidth=3) disp0.add_colorbar(ax=ax, label="N times signal remain after cleaning [%]") disp0.cmap = 'gnuplot2' ax.set_title("{} \n {}/{}".format( input_file.split("/")[-1][8:21], alive_ped_ev, sum_ped_ev), fontsize=25) print("{}/{}".format(alive_ped_ev, sum_ped_ev)) ax.set_xlabel(" ") ax.set_ylabel(" ") plt.tight_layout() plt.show() def check_interleave_pedestal_cleaning(self, list_of_file, max_events, sigma, dl1_file): high_gain = 0 ped_mean_pe, ped_rms_pe = get_bias_and_rms(dl1_file) bad_pixel_ids = np.where(ped_rms_pe[1, high_gain, :] == 0)[0] print(bad_pixel_ids) th = get_threshold(ped_mean_pe[1, high_gain, :], ped_rms_pe[1, high_gain, :], sigma) make_camera_binary_image(th, sigma, self.cleaning_parameters['picture_thresh'], bad_pixel_ids) signal_place_after_clean = np.zeros(1855) sum_ped_ev = 0 alive_ped_ev = 0 for input_file in list_of_file: print(input_file) r0_r1_calibrator = LSTR0Corrections(pedestal_path=None, r1_sample_start=3, r1_sample_end=39) reader = LSTEventSource(input_url=input_file, max_events=max_events) for i, ev in enumerate(reader): r0_r1_calibrator.calibrate(ev) if i % 10000 == 0: print(ev.r0.event_id) if ev.lst.tel[1].evt.tib_masked_trigger == 32: sum_ped_ev += 1 self.r1_dl1_calibrator(ev) img = ev.dl1.tel[1].image img[bad_pixel_ids] = 0 geom = ev.inst.subarray.tel[1].camera clean = tailcuts_pedestal_clean(geom, img, th, **self.cleaning_parameters) cleaned = img.copy() cleaned[~clean] = 0.0 signal_place_after_clean[np.where(clean == True)] += 1 if np.sum(cleaned > 0) > 0: alive_ped_ev += 1 noise_remain = signal_place_after_clean / sum_ped_ev self.plot_camera_display(noise_remain, input_file, bad_pixel_ids, alive_ped_ev, sum_ped_ev)
class LSTR0Corrections(CameraR0Calibrator): """ The R0 calibrator class for LST Camera. """ pedestal_path = Unicode( '', allow_none=True, help='Path to the LST pedestal binary file').tag(config=True) def __init__(self, **kwargs): """ The R0 calibrator for LST data. Fill the r1 container. Parameters ---------- config : traitlets.loader.Config Configuration specified by config file or cmdline arguments. Used to set traitlet values. Set to None if no configuration to pass. tool : ctapipe.core.Tool Tool executable that is calling this component. Passes the correct logger to the component. Set to None if no Tool to pass. kwargs """ super().__init__(**kwargs) self.n_module = 265 self.n_gain = 2 self.n_pix = 7 self.size4drs = 4 * 1024 self.roisize = 40 self.high_gain = 0 self.low_gain = 1 self.last_run_with_old_firmware = 1574 self.pedestal_value_array = np.zeros( (self.n_gain, self.n_pix * self.n_module, self.size4drs + 40), dtype=np.int16) self.first_cap_array = np.zeros( (self.n_module, self.n_gain, self.n_pix)) self.first_cap_time_lapse_array = np.zeros( (self.n_module, self.n_gain, self.n_pix)) self.last_reading_time_array = np.zeros( (self.n_module, self.n_gain, self.n_pix, self.size4drs)) self.first_cap_array_spike = np.zeros( (self.n_module, self.n_gain, self.n_pix)) self.first_cap_old_array = np.zeros( (self.n_module, self.n_gain, self.n_pix)) self._load_calib() def calibrate(self, event): for tel_id in event.r0.tels_with_data: self.subtract_pedestal(event, tel_id) self.time_lapse_corr(event, tel_id) self.interpolate_spikes(event, tel_id) event.r1.tel[tel_id].trigger_type = event.r0.tel[ tel_id].trigger_type event.r1.tel[tel_id].trigger_time = event.r0.tel[ tel_id].trigger_time samples = event.r1.tel[tel_id].waveform[:, :, self.r1_sample_start:self. r1_sample_end] event.r1.tel[tel_id].waveform = (samples - self.offset).astype( np.float32) def subtract_pedestal(self, event, tel_id): """ Subtract cell offset using pedestal file. Fill the R1 container. Parameters ---------- event : `ctapipe` event-container tel_id : id of the telescope """ n_modules = event.lst.tel[tel_id].svc.num_modules for nr_module in range(0, n_modules): self.first_cap_array[nr_module, :, :] = self._get_first_capacitor( event, nr_module, tel_id) expected_pixel_id = event.lst.tel[tel_id].svc.pixel_ids samples = event.r0.tel[tel_id].waveform.astype(np.float32) samples = subtract_pedestal_jit(samples, expected_pixel_id, self.first_cap_array, self.pedestal_value_array, n_modules) event.r1.tel[self.tel_id].trigger_type = event.r0.tel[ self.tel_id].trigger_type event.r1.tel[self.tel_id].trigger_time = event.r1.tel[ self.tel_id].trigger_time event.r1.tel[self.tel_id].waveform = samples[:, :, :] def time_lapse_corr(self, event, tel_id): """ Perform time lapse baseline corrections. Fill the R1 container or modifies R0 container. Parameters ---------- event : `ctapipe` event-container tel_id : id of the telescope """ run_id = event.lst.tel[tel_id].svc.configuration_id expected_pixel_id = event.lst.tel[tel_id].svc.pixel_ids local_clock_list = event.lst.tel[tel_id].evt.local_clock_counter n_modules = event.lst.tel[tel_id].svc.num_modules for nr_module in range(0, n_modules): self.first_cap_time_lapse_array[ nr_module, :, :] = self._get_first_capacitor( event, nr_module, tel_id) #If R1 container exist modifies it if isinstance(event.r1.tel[self.tel_id].waveform, np.ndarray): samples = event.r1.tel[self.tel_id].waveform # We have 2 functions: one for data from 2018/10/10 to 2019/11/04 and # one for data from 2019/11/05 (from Run 1574) after update firmware. # The old readout (before 2019/11/05) is shifted by 1 cell. if run_id > self.last_run_with_old_firmware: do_time_lapse_corr(samples, expected_pixel_id, local_clock_list, self.first_cap_time_lapse_array, self.last_reading_time_array, n_modules) else: do_time_lapse_corr_data_from_20181010_to_20191104( samples, expected_pixel_id, local_clock_list, self.first_cap_time_lapse_array, self.last_reading_time_array, n_modules) event.r1.tel[self.tel_id].trigger_type = event.r0.tel[ self.tel_id].trigger_type event.r1.tel[self.tel_id].trigger_time = event.r0.tel[ self.tel_id].trigger_time event.r1.tel[self.tel_id].waveform = samples[:, :, :] else: # Modifies R0 container. This is for create pedestal file. samples = np.copy(event.r0.tel[self.tel_id].waveform) if run_id > self.last_run_with_old_firmware: do_time_lapse_corr(samples, expected_pixel_id, local_clock_list, self.first_cap_time_lapse_array, self.last_reading_time_array, n_modules) else: do_time_lapse_corr_data_from_20181010_to_20191104( samples, expected_pixel_id, local_clock_list, self.first_cap_time_lapse_array, self.last_reading_time_array, n_modules) event.r0.tel[self.tel_id].waveform = samples[:, :, :] def interpolate_spikes(self, event, tel_id): """ Interpolates spike A & B. Fill the R1 container. Parameters ---------- event : `ctapipe` event-container tel_id : id of the telescope """ run_id = event.lst.tel[tel_id].svc.configuration_id self.first_cap_old_array[:, :, :] = self.first_cap_array_spike[:, :, :] n_modules = event.lst.tel[tel_id].svc.num_modules for nr_module in range(0, n_modules): self.first_cap_array_spike[ nr_module, :, :] = self._get_first_capacitor( event, nr_module, tel_id) # Interpolate spikes should be done after pedestal subtraction and time lapse correction. if isinstance(event.r1.tel[tel_id].waveform, np.ndarray): waveform = event.r1.tel[tel_id].waveform[:, :, :] expected_pixel_id = event.lst.tel[tel_id].svc.pixel_ids samples = waveform.copy() # We have 2 functions: one for data from 2018/10/10 to 2019/11/04 and # one for data from 2019/11/05 (from Run 1574) after update firmware. # The old readout (before 2019/11/05) is shifted by 1 cell. if run_id > self.last_run_with_old_firmware: event.r1.tel[ self.tel_id].waveform = self.interpolate_pseudo_pulses( samples, expected_pixel_id, self.first_cap_array_spike, self.first_cap_old_array, n_modules) else: event.r1.tel[self.tel_id].waveform = \ self.interpolate_pseudo_pulses_data_from_20181010_to_20191104(samples, expected_pixel_id, self.first_cap_array_spike, self.first_cap_old_array, n_modules) event.r1.tel[self.tel_id].trigger_type = event.r0.tel[ self.tel_id].trigger_type event.r1.tel[self.tel_id].trigger_time = event.r0.tel[ self.tel_id].trigger_time @staticmethod @jit(parallel=True) def interpolate_pseudo_pulses(waveform, expected_pixel_id, fc, fc_old, n_modules): """ Interpolate Spike A & B. Change waveform array. Parameters ---------- waveform : ndarray Waveform stored in a numpy array of shape (n_gain, n_pix, n_samples). expected_pixel_id: ndarray Array stored expected pixel id (n_pix*n_modules). fc : ndarray Value of first capacitor stored in a numpy array of shape (n_clus, n_gain, n_pix). fc_old : ndarray Value of first capacitor from previous event stored in a numpy array of shape (n_clus, n_gain, n_pix). n_modules : int Number of modules """ roi_size = 40 size1drs = 1024 size4drs = 4096 n_gain = 2 n_pix = 7 for nr_module in prange(0, n_modules): for gain in prange(0, n_gain): for pix in prange(0, n_pix): for k in prange(0, 4): # looking for spike A first case abspos = int(size1drs + 1 - roi_size - 2 - fc_old[nr_module, gain, pix] + k * size1drs + size4drs) spike_A_position = int( (abspos - fc[nr_module, gain, pix] + size4drs) % size4drs) if (spike_A_position > 2 and spike_A_position < roi_size - 2): # The correction is only needed for even # last capacitor (lc) in the first half of the # DRS4 ring if ((fc_old[nr_module, gain, pix] + (roi_size - 1)) % 2 == 0 and (fc_old[nr_module, gain, pix] + (roi_size - 1)) % size1drs <= size1drs // 2 - 1): pixel = expected_pixel_id[nr_module * 7 + pix] interpolate_spike_A(waveform, gain, spike_A_position, pixel) # looking for spike A second case abspos = int(roi_size - 1 + fc_old[nr_module, gain, pix] + k * size1drs) spike_A_position = int( (abspos - fc[nr_module, gain, pix] + size4drs) % size4drs) if (spike_A_position > 2 and spike_A_position < (roi_size - 2)): # The correction is only needed for even last capacitor (lc) in the # first half of the DRS4 ring if ((fc_old[nr_module, gain, pix] + (roi_size - 1)) % 2 == 0 and (fc_old[nr_module, gain, pix] + (roi_size - 1)) % size1drs <= size1drs // 2 - 1): pixel = expected_pixel_id[nr_module * 7 + pix] interpolate_spike_A(waveform, gain, spike_A_position, pixel) return waveform @staticmethod @jit(parallel=True) def interpolate_pseudo_pulses_data_from_20181010_to_20191104( waveform, expected_pixel_id, fc, fc_old, n_modules): """ Interpolate Spike A & B. This is function for data from 2018/10/10 to 2019/11/04 with old firmware. Change waveform array. Parameters ---------- waveform : ndarray Waveform stored in a numpy array of shape (n_gain, n_pix, n_samples). expected_pixel_id: ndarray Array stored expected pixel id (n_pix*n_modules). fc : ndarray Value of first capacitor stored in a numpy array of shape (n_clus, n_gain, n_pix). fc_old : ndarray Value of first capacitor from previous event stored in a numpy array of shape (n_clus, n_gain, n_pix). n_modules : int Number of modules """ roi_size = 40 size1drs = 1024 size4drs = 4096 n_gain = 2 n_pix = 7 for nr_module in prange(0, n_modules): for gain in prange(0, n_gain): for pix in prange(0, n_pix): for k in prange(0, 4): # looking for spike A first case abspos = int(size1drs - roi_size - 2 - fc_old[nr_module, gain, pix] + k * size1drs + size4drs) spike_A_position = int( (abspos - fc[nr_module, gain, pix] + size4drs) % size4drs) if (spike_A_position > 2 and spike_A_position < roi_size - 2): # The correction is only needed for even # last capacitor (lc) in the first half of the # DRS4 ring if ((fc_old[nr_module, gain, pix] + (roi_size - 1)) % 2 == 0 and (fc_old[nr_module, gain, pix] + (roi_size - 1)) % size1drs <= size1drs // 2 - 2): pixel = expected_pixel_id[nr_module * 7 + pix] interpolate_spike_A(waveform, gain, spike_A_position, pixel) # looking for spike A second case abspos = int(roi_size - 2 + fc_old[nr_module, gain, pix] + k * size1drs) spike_A_position = int( (abspos - fc[nr_module, gain, pix] + size4drs) % size4drs) if (spike_A_position > 2 and spike_A_position < (roi_size - 2)): # The correction is only needed for even last capacitor (lc) in the # first half of the DRS4 ring if ((fc_old[nr_module, gain, pix] + (roi_size - 1)) % 2 == 0 and (fc_old[nr_module, gain, pix] + (roi_size - 1)) % size1drs <= size1drs // 2 - 2): pixel = expected_pixel_id[nr_module * 7 + pix] interpolate_spike_A(waveform, gain, spike_A_position, pixel) return waveform def _load_calib(self): """ Function to load pedestal file. """ if self.pedestal_path: with fits.open(self.pedestal_path) as f: pedestal_data = np.int16(f[1].data) self.pedestal_value_array[:, :, :self.size4drs] = \ pedestal_data - self.offset self.pedestal_value_array[:, :, self.size4drs:self.size4drs + 40] \ = pedestal_data[:, :, 0:40] - self.offset def _get_first_capacitor(self, event, nr_module, tel_id): """ Get first capacitor values from event for nr module. Parameters ---------- event : `ctapipe` event-container nr_module : number of module tel_id : id of the telescope """ fc = np.zeros((2, 7)) first_cap = event.lst.tel[tel_id].evt.first_capacitor_id[nr_module * 8:(nr_module + 1) * 8] # First capacitor order according Dragon v5 board data format for i, j in zip([0, 1, 2, 3, 4, 5, 6], [0, 0, 1, 1, 2, 2, 3]): fc[self.high_gain, i] = first_cap[j] for i, j in zip([0, 1, 2, 3, 4, 5, 6], [4, 4, 5, 5, 6, 6, 7]): fc[self.low_gain, i] = first_cap[j] return fc
class SingleTelEventDisplay(Tool): name = "ctapipe-display-televents" description = Unicode(__doc__) infile = Unicode(help="input file to read", default='').tag(config=True) tel = Int(help='Telescope ID to display', default=0).tag(config=True) channel = Integer(help="channel number to display", min=0, max=1).tag(config=True) write = Bool(help="Write out images to PNG files", default=False).tag(config=True) clean = Bool(help="Apply image cleaning", default=False).tag(config=True) hillas = Bool(help="Apply and display Hillas parametrization", default=False).tag(config=True) samples = Bool(help="Show each sample", default=False).tag(config=True) display = Bool(help="Display results in interactive window", default_value=True).tag(config=True) delay = Float(help='delay between events in s', default_value=0.01, min=0.001).tag(config=True) progress = Bool(help='display progress bar', default_value=True).tag(config=True) aliases = Dict({ 'infile': 'SingleTelEventDisplay.infile', 'tel': 'SingleTelEventDisplay.tel', 'max-events': 'EventSource.max_events', 'channel': 'SingleTelEventDisplay.channel', 'write': 'SingleTelEventDisplay.write', 'clean': 'SingleTelEventDisplay.clean', 'hillas': 'SingleTelEventDisplay.hillas', 'samples': 'SingleTelEventDisplay.samples', 'display': 'SingleTelEventDisplay.display', 'delay': 'SingleTelEventDisplay.delay', 'progress': 'SingleTelEventDisplay.progress' }) classes = List([EventSource, CameraCalibrator]) def __init__(self, **kwargs): super().__init__(**kwargs) def setup(self): print('TOLLES INFILE', self.infile) self.event_source = EventSource.from_url(self.infile, parent=self) self.event_source.allowed_tels = { self.tel, } self.calibrator = CameraCalibrator(parent=self) self.log.info(f'SELECTING EVENTS FROM TELESCOPE {self.tel}') def start(self): disp = None for event in tqdm(self.event_source, desc=f'Tel{self.tel}', total=self.event_source.max_events, disable=~self.progress): self.log.debug(event.trig) self.log.debug(f"Energy: {event.mc.energy}") self.calibrator(event) if disp is None: geom = event.inst.subarray.tel[self.tel].camera self.log.info(geom) disp = CameraDisplay(geom) # disp.enable_pixel_picker() disp.add_colorbar() if self.display: plt.show(block=False) # display the event disp.axes.set_title('CT{:03d} ({}), event {:06d}'.format( self.tel, geom.cam_id, event.r0.event_id)) if self.samples: # display time-varying event data = event.dl0.tel[self.tel].waveform[self.channel] for ii in range(data.shape[1]): disp.image = data[:, ii] disp.set_limits_percent(70) plt.suptitle(f"Sample {ii:03d}") if self.display: plt.pause(self.delay) if self.write: plt.savefig( f'CT{self.tel:03d}_EV{event.r0.event_id:10d}' f'_S{ii:02d}.png') else: # display integrated event: im = event.dl1.tel[self.tel].image[self.channel] if self.clean: mask = tailcuts_clean(geom, im, picture_thresh=10, boundary_thresh=7) im[~mask] = 0.0 disp.image = im if self.hillas: try: ellipses = disp.axes.findobj(Ellipse) if len(ellipses) > 0: ellipses[0].remove() params = hillas_parameters(geom, image=im) disp.overlay_moments(params, color='pink', lw=3, with_label=False) except HillasParameterizationError: pass if self.display: plt.pause(self.delay) if self.write: plt.savefig( f'CT{self.tel:03d}_EV{event.r0.event_id:010d}.png') self.log.info("FINISHED READING DATA FILE") if disp is None: self.log.warning( 'No events for tel {} were found in {}. Try a ' 'different EventIO file or another telescope'.format( self.tel, self.infile), )
class LSTCameraCalibrator(CameraCalibrator): """ Calibrator to handle the LST camera calibration chain, in order to fill the DL1 data level in the event container. """ extractor_product = Unicode( 'NeighborPeakWindowSum', help='Name of the charge extractor to be used').tag(config=True) reducer_product = Unicode( 'NullDataVolumeReducer', help='Name of the DataVolumeReducer to use').tag(config=True) calibration_path = Unicode( '', allow_none=True, help='Path to LST calibration file').tag(config=True) allowed_tels = List( [1], help='List of telescope to be calibrated').tag(config=True) def __init__(self, **kwargs): """ Parameters ---------- reducer_product : ctapipe.image.reducer.DataVolumeReducer The DataVolumeReducer to use. If None, then NullDataVolumeReducer will be used by default, and waveforms will not be reduced. extractor_product : ctapipe.image.extractor.ImageExtractor The ImageExtractor to use. If None, then NeighborPeakWindowSum will be used by default. calibration_path : Path to LST calibration file to get the pedestal and flat-field corrections kwargs """ super().__init__(**kwargs) # load the waveform charge extractor self.image_extractor = ImageExtractor.from_name(self.extractor_product, config=self.config) self.log.info(f"extractor {self.extractor_product}") self.data_volume_reducer = DataVolumeReducer.from_name( self.reducer_product, config=self.config) self.log.info(f" {self.reducer_product}") # calibration data container self.mon_data = MonitoringContainer() # initialize the MonitoringContainer() for the moment it reads it from a hdf5 file self._initialize_correction() def _initialize_correction(self): """ Read the correction from hdf5 calibration file """ self.mon_data.tels_with_data = self.allowed_tels self.log.info(f"read {self.calibration_path}") try: with HDF5TableReader(self.calibration_path) as h5_table: assert h5_table._h5file.isopen == True for telid in self.allowed_tels: # read the calibration data for the moment only one event table = '/tel_' + str(telid) + '/calibration' next( h5_table.read(table, self.mon_data.tel[telid].calibration)) # eliminate inf values (should be done probably before) dc_to_pe = self.mon_data.tel[telid].calibration.dc_to_pe dc_to_pe[np.isinf(dc_to_pe)] = 0 self.log.info( f"read {self.mon_data.tel[telid].calibration.dc_to_pe}" ) except: self.log.error( f"Problem in reading calibration file {self.calibration_path}") def _calibrate_dl0(self, event, telid): """ create dl0 level, for the moment copy the r1 """ waveforms = event.r1.tel[telid].waveform if self._check_r1_empty(waveforms): return event.dl0.event_id = event.r1.event_id event.mon.tel[telid].calibration = self.mon_data.tel[telid].calibration # subtract the pedestal per sample (should we do it?) and multiply for the calibration coefficients # event.dl0.tel[telid].waveform = ( (event.r1.tel[telid].waveform - self.mon_data.tel[telid]. calibration.pedestal_per_sample[:, :, np.newaxis]) * self.mon_data.tel[telid].calibration.dc_to_pe[:, :, np.newaxis]) def _calibrate_dl1(self, event, telid): """ create calibrated dl1 image and calibrate it """ waveforms = event.dl0.tel[telid].waveform if self._check_dl0_empty(waveforms): return if self.image_extractor.requires_neighbors(): camera = event.inst.subarray.tel[telid].camera self.image_extractor.neighbors = camera.neighbor_matrix_where charge, pulse_time = self.image_extractor(waveforms) event.dl0.event_id = event.r1.event_id event.dl1.tel[telid].image = charge event.dl1.tel[telid].pulse_time = pulse_time + self.mon_data.tel[ telid].calibration.time_correction
class TailCutsDataVolumeReducer(DataVolumeReducer): """ Reduce the time integrated shower image in 3 Steps: 1) Select pixels with tailcuts_clean. 2) Add iteratively all pixels with Signal S >= boundary_thresh with ctapipe module dilate until no new pixels were added. 3) Adding new pixels with dilate to get more conservative. Attributes ---------- image_extractor_type: String Name of the image_extractor to be used. n_end_dilates: IntTelescopeParameter Number of how many times to dilate at the end. do_boundary_dilation: BoolTelescopeParameter If set to 'False', the iteration steps in 2) are skipped and normal TailcutCleaning is used. """ image_extractor_type = Unicode( default_value="NeighborPeakWindowSum", help="Name of the image_extractor" "to be used.", ).tag(config=True) n_end_dilates = IntTelescopeParameter( default_value=1, help="Number of how many times to dilate at the end.").tag(config=True) do_boundary_dilation = BoolTelescopeParameter( default_value=True, help="If set to 'False', the iteration steps in 2) are skipped and" "normal TailcutCleaning is used.", ).tag(config=True) def __init__(self, subarray, config=None, parent=None, **kwargs): """ Parameters ---------- subarray: ctapipe.instrument.SubarrayDescription Description of the subarray config: traitlets.loader.Config Configuration specified by config file or cmdline arguments. Used to set traitlet values. Set to None if no configuration to pass. kwargs """ super().__init__(config=config, parent=parent, subarray=subarray, **kwargs) self.cleaner = TailcutsImageCleaner(parent=self, subarray=self.subarray) self.image_extractor = ImageExtractor.from_name( self.image_extractor_type, subarray=self.subarray, parent=self) def select_pixels(self, waveforms, telid=None, selected_gain_channel=None): camera_geom = self.subarray.tel[telid].camera.geometry # Pulse-integrate waveforms charge, _ = self.image_extractor( waveforms, telid=telid, selected_gain_channel=selected_gain_channel) # 1) Step: TailcutCleaning at first mask = self.cleaner(telid, charge) pixels_above_boundary_thresh = ( charge >= self.cleaner.boundary_threshold_pe.tel[telid]) mask_in_loop = np.array([]) # 2) Step: Add iteratively all pixels with Signal # S > boundary_thresh with ctapipe module # 'dilate' until no new pixels were added. while (not np.array_equal(mask, mask_in_loop) and self.do_boundary_dilation.tel[telid]): mask_in_loop = mask mask = dilate(camera_geom, mask) & pixels_above_boundary_thresh # 3) Step: Adding Pixels with 'dilate' to get more conservative. for _ in range(self.n_end_dilates.tel[telid]): mask = dilate(camera_geom, mask) return mask
class ImageSumDisplayerTool(Tool): description = Unicode(__doc__) name = "ctapipe-display-imagesum" infile = Unicode( help='input simtelarray file', default="/Users/kosack/Data/CTA/Prod3/gamma.simtel.gz").tag( config=True) telgroup = Integer(help='telescope group number', default=1).tag(config=True) max_events = Integer(help='stop after this many events if non-zero', default_value=0, min=0).tag(config=True) output_suffix = Unicode(help='suffix (file extension) of output ' 'filenames to write images ' 'to (no writing is done if blank). ' 'Images will be named [EVENTID][suffix]', default_value="").tag(config=True) aliases = Dict({ 'infile': 'ImageSumDisplayerTool.infile', 'telgroup': 'ImageSumDisplayerTool.telgroup', 'max-events': 'ImageSumDisplayerTool.max_events', 'output-suffix': 'ImageSumDisplayerTool.output_suffix' }) classes = List([CameraCalibrator, SimTelEventSource]) def setup(self): # load up the telescope types table (need to first open a file, a bit of # a hack until a proper insturment module exists) and select only the # telescopes with the same camera type # make sure gzip files are seekable self.reader = SimTelEventSource(input_url=self.infile, max_events=self.max_events, back_seekable=True) for event in self.reader: camtypes = event.inst.subarray.to_table().group_by('camera_type') event.inst.subarray.info(printer=self.log.info) break group = camtypes.groups[self.telgroup] self._selected_tels = list(group['tel_id'].data) self._base_tel = self._selected_tels[0] self.log.info("Telescope group %d: %s", self.telgroup, str(event.inst.subarray.tel[self._selected_tels[0]])) self.log.info(f"SELECTED TELESCOPES:{self._selected_tels}") self.calibrator = CameraCalibrator(parent=self) self.reader.allowed_tels = self._selected_tels def start(self): geom = None imsum = None disp = None for event in self.reader: self.calibrator(event) if geom is None: geom = event.inst.subarray.tel[self._base_tel].camera imsum = np.zeros(shape=geom.pix_x.shape, dtype=np.float) disp = CameraDisplay(geom, title=geom.cam_id) disp.add_colorbar() disp.cmap = 'viridis' if len(event.dl0.tels_with_data) <= 2: continue imsum[:] = 0 for telid in event.dl0.tels_with_data: imsum += event.dl1.tel[telid].image self.log.info("event={} ntels={} energy={}".format( event.r0.event_id, len(event.dl0.tels_with_data), event.mc.energy)) disp.image = imsum plt.pause(0.1) if self.output_suffix is not "": filename = "{:020d}{}".format(event.r0.event_id, self.output_suffix) self.log.info(f"saving: '{filename}'") plt.savefig(filename)
class DumpTriggersTool(Tool): description = Unicode(__doc__) # ============================================= # configuration parameters: # ============================================= infile = Unicode(help='input simtelarray file').tag(config=True, allow_none=False) outfile = Unicode('triggers.fits', help='output filename (*.fits, *.h5)').tag(config=True) overwrite = Bool(False, help="overwrite existing output file").tag(config=True) # ============================================= # map low-level options to high-level command-line options # ============================================= aliases = Dict({ 'infile': 'DumpTriggersTool.infile', 'outfile': 'DumpTriggersTool.outfile' }) flags = Dict({ 'overwrite': ({ 'DumpTriggersTool': { 'overwrite': True } }, 'Enable overwriting of output file') }) examples = ('ctapipe-dump-triggers --infile gamma.simtel.gz ' '--outfile trig.fits --overwrite' '\n\n' 'If you want to see more output, use --log_level=DEBUG') # ============================================= # The methods of the Tool (initialize, start, finish): # ============================================= def add_event_to_table(self, event_id): """ add the current pyhessio event to a row in the `self.events` table """ ts, tns = pyhessio.get_central_event_gps_time() gpstime = Time(ts * u.s, tns * u.ns, format='gps', scale='utc') if self._prev_gpstime is None: self._prev_gpstime = gpstime if self._current_starttime is None: self._current_starttime = gpstime relative_time = gpstime - self._current_starttime delta_t = gpstime - self._prev_gpstime self._prev_gpstime = gpstime # build the trigger pattern as a fixed-length array # (better for storage in FITS format) trigtels = pyhessio.get_telescope_with_data_list() self._current_trigpattern[:] = 0 # zero the trigger pattern self._current_trigpattern[trigtels] = 1 # set the triggered tels to 1 # insert the row into the table self.events.add_row((event_id, relative_time.sec, delta_t.sec, len(trigtels), self._current_trigpattern)) def setup(self): """ setup function, called before `start()` """ if self.infile == '': raise ValueError("No 'infile' parameter was specified. " "Use --help for info") self.events = Table( names=['EVENT_ID', 'T_REL', 'DELTA_T', 'N_TRIG', 'TRIGGERED_TELS'], dtype=[np.int64, np.float64, np.float64, np.int32, np.uint8]) self.events['TRIGGERED_TELS'].shape = (0, MAX_TELS) self.events['T_REL'].unit = u.s self.events['T_REL'].description = 'Time relative to first event' self.events['DELTA_T'].unit = u.s self.events.meta['INPUT'] = self.infile self._current_trigpattern = np.zeros(MAX_TELS) self._current_starttime = None self._prev_gpstime = None pyhessio.file_open(self.infile) def start(self): """ main event loop """ for run_id, event_id in pyhessio.move_to_next_event(): self.add_event_to_table(event_id) def finish(self): """ finish up and write out results (called automatically after `start()`) """ pyhessio.close_file() # write out the final table if self.outfile.endswith('fits') or self.outfile.endswith('fits.gz'): self.events.write(self.outfile, overwrite=self.overwrite) elif self.outfile.endswith('h5'): self.events.write(self.outfile, path='/events', overwrite=self.overwrite) else: self.events.write(self.outfile) self.log.info("Table written to '{}'".format(self.outfile)) self.log.info('\n %s', self.events)
class DumpInstrumentTool(Tool): description = Unicode(__doc__) name = 'ctapipe-dump-instrument' infile = Unicode(help='input simtelarray file').tag(config=True) format = Enum(['fits', 'ecsv', 'hdf5'], default_value='fits', help='Format of output file', config=True) aliases = Dict(dict(infile='DumpInstrumentTool.infile', format='DumpInstrumentTool.format')) def setup(self): with event_source(self.infile) as source: data = next(iter(source)) # get one event, so the instrument table is there self.inst = data.inst # keep a reference to the instrument stuff def start(self): self.write_camera_geometries() self.write_optics_descriptions() self.write_subarray_description() def finish(self): pass @staticmethod def _get_file_format_info(format_name, table_type, table_name): """ returns file extension + dict of required parameters for Table.write""" if format_name == 'fits': return 'fits.gz', dict() elif format_name == 'ecsv': return 'ecsv.txt', dict(format='ascii.ecsv') elif format_name == 'hdf5': return 'h5', dict(path="/" + table_type + "/" + table_name) else: raise NameError("format not supported") def write_camera_geometries(self): cam_types = get_camera_types(self.inst.subarray) self.inst.subarray.info(printer=self.log.info) for cam_name in cam_types: ext, args = self._get_file_format_info(self.format, 'CAMGEOM', cam_name) self.log.debug("writing {}".format(cam_name)) tel_id = cam_types[cam_name].pop() geom = self.inst.subarray.tel[tel_id].camera table = geom.to_table() table.meta['SOURCE'] = self.infile filename = "{}.camgeom.{}".format(cam_name, ext) try: table.write(filename, **args) Provenance().add_output_file(filename, 'dl0.tel.svc.camera') except IOError as err: self.log.warn("couldn't write camera definition '%s' because: " "%s", filename, err) def write_optics_descriptions(self): sub = self.inst.subarray ext, args = self._get_file_format_info(self.format, sub.name, 'optics') tab = sub.to_table(kind='optics') tab.meta['SOURCE'] = self.infile filename = '{}.optics.{}'.format(sub.name, ext) try: tab.write(filename, **args) Provenance().add_output_file(filename, 'dl0.sub.svc.optics') except IOError as err: self.log.warn("couldn't write optics description '%s' because: " "%s", filename, err) def write_subarray_description(self): sub = self.inst.subarray ext, args = self._get_file_format_info(self.format, sub.name, 'subarray') tab = sub.to_table(kind='subarray') tab.meta['SOURCE'] = self.infile filename = '{}.subarray.{}'.format(sub.name, ext) try: tab.write(filename, **args) Provenance().add_output_file(filename, 'dl0.sub.svc.subarray') except IOError as err: self.log.warn("couldn't write subarray description '%s' because: " "%s", filename, err)
class PedestalCalculator(Component): """ Parent class for the pedestal calculators. Fills the MonitoringCameraContainer.PedestalContainer on the base of a given pedestal sample. The sample is defined by a maximal interval of time (sample_duration) or a minimal number of events (sample_duration). The calculator is supposed to act in an event loop, extract and collect the event charge and fill the PedestalContainer Parameters ---------- tel_id : int id of the telescope (default 0) sample_duration : int interval of time (s) used to gather the pedestal statistics sample_size : int number of pedestal events requested for the statistics n_channels : int number of waveform channel to be considered charge_product : str Name of the charge extractor to be used config : traitlets.loader.Config Configuration specified by config file or cmdline arguments. Used to set traitlet values. Set to None if no configuration to pass. kwargs """ tel_id = Int( 0, help='id of the telescope to calculate the pedestal values' ).tag(config=True) sample_duration = Int( 60, help='sample duration in seconds' ).tag(config=True) sample_size = Int( 10000, help='sample size' ).tag(config=True) n_channels = Int( 2, help='number of channels to be treated' ).tag(config=True) charge_product = Unicode( 'FixedWindowSum', help='Name of the charge extractor to be used' ).tag(config=True) def __init__( self, **kwargs ): """ Parent class for the pedestal calculators. Fills the MonitoringCameraContainer.PedestalContainer on the base of a given pedestal sample. The sample is defined by a maximal interval of time (sample_duration) or a minimal number of events (sample_duration). The calculator is supposed to act in an event loop, extract and collect the event charge and fill the PedestalContainer Parameters ---------- tel_id : int id of the telescope (default 0) sample_duration : int interval of time (s) used to gather the pedestal statistics sample_size : int number of pedestal events requested for the statistics n_channels : int number of waveform channel to be considered charge_product : str Name of the charge extractor to be used config : traitlets.loader.Config Configuration specified by config file or cmdline arguments. Used to set traitlet values. Set to None if no configuration to pass. kwargs """ super().__init__(**kwargs) # load the waveform charge extractor self.extractor = ImageExtractor.from_name( self.charge_product, config=self.config ) self.log.info(f"extractor {self.extractor}") @abstractmethod def calculate_pedestals(self, event): """
class LSTCameraCalibrator(CameraCalibrator): """ Calibrator to handle the LST camera calibration chain, in order to fill the DL1 data level in the event container. """ extractor_product = Unicode( 'LocalPeakWindowSum', help='Name of the charge extractor to be used').tag(config=True) reducer_product = Unicode( 'NullDataVolumeReducer', help='Name of the DataVolumeReducer to use').tag(config=True) calibration_path = Path( exists=True, directory_ok=False, help='Path to LST calibration file').tag(config=True) time_calibration_path = Path( exists=True, directory_ok=False, help='Path to drs4 time calibration file').tag(config=True) time_sampling_correction_path = Path( exists=True, directory_ok=False, help='Path to time sampling correction file', allow_none=True, ).tag(config=True) allowed_tels = List( [1], help='List of telescope to be calibrated').tag(config=True) gain_threshold = Int( 4094, allow_none=True, help='Threshold for the gain selection in ADC').tag(config=True) charge_scale = List( [1, 1], help='Multiplicative correction factor for charge estimation [HG,LG]' ).tag(config=True) def __init__(self, subarray, **kwargs): """ Parameters ---------- reducer_product : ctapipe.image.reducer.DataVolumeReducer The DataVolumeReducer to use. If None, then NullDataVolumeReducer will be used by default, and waveforms will not be reduced. extractor_product : ctapipe.image.extractor.ImageExtractor The ImageExtractor to use. If None, then LocalPeakWindowSum will be used by default. calibration_path : Path to LST calibration file to get the pedestal and flat-field corrections kwargs """ super().__init__(subarray, **kwargs) # load the waveform charge extractor self.image_extractor = ImageExtractor.from_name(self.extractor_product, subarray=self.subarray, config=self.config) self.log.info(f"extractor {self.extractor_product}") print("EXTRACTOR", self.image_extractor) self.data_volume_reducer = DataVolumeReducer.from_name( self.reducer_product, subarray=self.subarray, config=self.config) self.log.info(f" {self.reducer_product}") # declare gain selector if the threshold is defined if self.gain_threshold: self.gain_selector = gainselection.ThresholdGainSelector( threshold=self.gain_threshold) # declare time calibrator if correction file exist if os.path.exists(self.time_calibration_path): self.time_corrector = PulseTimeCorrection( calib_file_path=self.time_calibration_path) else: raise IOError( f"Time calibration file {self.time_calibration_path} not found!" ) # declare the charge sampling corrector if self.time_sampling_correction_path is not None: # search the file in resources if not found if not os.path.exists(self.time_sampling_correction_path): self.time_sampling_correction_path = resource_filename( 'lstchain', f"resources/{self.time_sampling_correction_path}") if os.path.exists(self.time_sampling_correction_path): self.time_sampling_corrector = TimeSamplingCorrection( time_sampling_correction_path=self. time_sampling_correction_path) else: raise IOError( f"Sampling correction file {self.time_sampling_correction_path} not found!" ) else: self.time_sampling_corrector = None # calibration data container self.mon_data = MonitoringContainer() # initialize the MonitoringContainer() for the moment it reads it from a hdf5 file self._initialize_correction() self.log.info(f"Global charge scale {self.charge_scale}") def _initialize_correction(self): """ Read the correction from hdf5 calibration file """ self.log.info(f"read {self.calibration_path}") try: with HDF5TableReader(self.calibration_path) as h5_table: for telid in self.allowed_tels: # read the calibration data table = '/tel_' + str(telid) + '/calibration' next( h5_table.read(table, self.mon_data.tel[telid].calibration)) # read pedestal data table = '/tel_' + str(telid) + '/pedestal' next( h5_table.read(table, self.mon_data.tel[telid].pedestal)) # read flat-field data table = '/tel_' + str(telid) + '/flatfield' next( h5_table.read(table, self.mon_data.tel[telid].flatfield)) # read the pixel_status container table = '/tel_' + str(telid) + '/pixel_status' next( h5_table.read(table, self.mon_data.tel[telid].pixel_status)) except Exception: self.log.exception( f"Problem in reading calibration file {self.calibration_path}") raise def _calibrate_dl0(self, event, telid): """ create dl0 level, with gain-selected and calibrated waveform """ waveforms = event.r1.tel[telid].waveform if self._check_r1_empty(waveforms): return # if not already done, initialize the event monitoring containers if event.mon.tel[telid].calibration.dc_to_pe is None: event.mon.tel[telid].calibration = self.mon_data.tel[ telid].calibration event.mon.tel[telid].flatfield = self.mon_data.tel[telid].flatfield event.mon.tel[telid].pedestal = self.mon_data.tel[telid].pedestal event.mon.tel[telid].pixel_status = self.mon_data.tel[ telid].pixel_status # # subtract the pedestal per sample and multiply for the calibration coefficients # calibrated_waveform = ( (waveforms - self.mon_data.tel[telid].calibration. pedestal_per_sample[:, :, np.newaxis]) * self.mon_data.tel[telid].calibration.dc_to_pe[:, :, np.newaxis]).astype( np.float32) # If requested, perform gain selection (this will be done by the EvB in future) # find the gain selection mask if waveforms.ndim == 3: # if threshold defined, perform gain selection if self.gain_threshold: gain_mask = self.gain_selector(waveforms) # select the samples calibrated_waveform = calibrated_waveform[ gain_mask, np.arange(waveforms.shape[1])] else: # keep both HG and LG gain_mask = np.zeros((waveforms.shape[0], waveforms.shape[1]), dtype=np.int64) gain_mask[1] = 1 else: # gain selection already performed in EvB: (0=HG, 1=LG) gain_mask = event.lst.tel[telid].evt.pixel_status >> 2 & 1 # remember the calibrated and gain selected waveform # (this should be the r1 waveform to be compliant with ctapipe (?)) event.dl0.tel[telid].waveform = calibrated_waveform # remember which channel has been selected event.r1.tel[telid].selected_gain_channel = gain_mask event.dl0.tel[telid].selected_gain_channel = gain_mask def _calibrate_dl1(self, event, telid): """ create calibrated dl1 image and calibrate it """ n_pixels = self.subarray.tels[telid].camera.geometry.n_pixels # copy the waveform be cause I do not want to change it waveforms = np.copy(event.dl0.tel[telid].waveform) gain_mask = event.dl0.tel[telid].selected_gain_channel if self._check_dl0_empty(waveforms): return # In case of no gain selection the selected gain channels are [0,0,..][1,1,..] no_gain_selection = np.zeros((waveforms.shape[0], waveforms.shape[1]), dtype=np.int64) no_gain_selection[1] = 1 # correct the dl0 waveform for the sampling time corrections if self.time_sampling_corrector: waveforms *= self.time_sampling_corrector.get_corrections( event, telid)[gain_mask, np.arange(n_pixels)] # extract the charge charge, peak_time = self.image_extractor(waveforms, telid, gain_mask) # correct charge for global scale corrected_charge = charge * np.array(self.charge_scale, dtype=np.float32)[gain_mask] # correct time with drs4 correction if available if self.time_corrector: peak_time_drs4_corrected = ( peak_time - self.time_corrector.get_pulse_time_corrections( event)[gain_mask, np.arange(n_pixels)]) # add flat-fielding time correction peak_time_ff_corrected = ( peak_time_drs4_corrected + self.mon_data.tel[telid].calibration.time_correction.value[ gain_mask, np.arange(n_pixels)]) # fill dl1 container event.dl1.tel[telid].image = corrected_charge event.dl1.tel[telid].peak_time = peak_time_ff_corrected.astype( np.float32)
class SimpleEventWriter(Tool): name = "ctapipe-simple-event-writer" description = Unicode(__doc__) infile = Path( default_value=get_dataset_path( "lst_prod3_calibration_and_mcphotons.simtel.zst"), help="input file to read", directory_ok=False, exists=True, ).tag(config=True) outfile = Path(help="output file name", directory_ok=False, default_value="output.h5").tag(config=True) progress = Bool(help="display progress bar", default_value=True).tag(config=True) aliases = Dict({ "infile": "EventSource.input_url", "outfile": "SimpleEventWriter.outfile", "max-events": "EventSource.max_events", "progress": "SimpleEventWriter.progress", }) classes = List([EventSource, CameraCalibrator]) def setup(self): self.log.info("Configure EventSource...") self.event_source = EventSource.from_url(self.infile, parent=self) self.calibrator = CameraCalibrator(subarray=self.event_source.subarray, parent=self) self.writer = HDF5TableWriter(filename=self.outfile, group_name="image_infos", overwrite=True, parent=self) def start(self): self.log.info("Loop on events...") for event in tqdm( self.event_source, desc="EventWriter", total=self.event_source.max_events, disable=~self.progress, ): self.calibrator(event) for tel_id in event.dl0.tel.keys(): geom = self.event_source.subarray.tel[tel_id].camera.geometry dl1_tel = event.dl1.tel[tel_id] # Image cleaning image = dl1_tel.image # Waiting for automatic gain selection mask = tailcuts_clean(geom, image, picture_thresh=10, boundary_thresh=5) cleaned = image.copy() cleaned[~mask] = 0 # Image parametrisation params = hillas_parameters(geom, cleaned) # Save Ids, MC infos and Hillas informations self.writer.write(geom.camera_name, [event.r0, event.simulation.shower, params]) def finish(self): self.log.info("End of job.") self.writer.close()
class SimpleEventWriter(Tool): name = 'ctapipe-simple-event-writer' description = Unicode(__doc__) infile = Unicode(help='input file to read', default='').tag(config=True) outfile = Unicode(help='output file name', default_value='output.h5').tag(config=True) progress = Bool(help='display progress bar', default_value=True).tag(config=True) aliases = Dict({ 'infile': 'EventSource.input_url', 'outfile': 'SimpleEventWriter.outfile', 'max-events': 'EventSource.max_events', 'progress': 'SimpleEventWriter.progress' }) classes = List([EventSource, CameraCalibrator, CutFlow]) def setup(self): self.log.info('Configure EventSource...') self.event_source = self.add_component( EventSource.from_config(config=self.config, parent=self)) self.calibrator = self.add_component(CameraCalibrator(parent=self)) self.writer = self.add_component( HDF5TableWriter(filename=self.outfile, group_name='image_infos', overwrite=True)) # Define Pre-selection for images preselcuts = self.config['Preselect'] self.image_cutflow = CutFlow('Image preselection') self.image_cutflow.set_cuts( dict(no_sel=None, n_pixel=lambda s: np.count_nonzero(s) < preselcuts['n_pixel'][ 'min'], image_amplitude=lambda q: q < preselcuts['image_amplitude'][ 'min'])) # Define Pre-selection for events self.event_cutflow = CutFlow('Event preselection') self.event_cutflow.set_cuts(dict(no_sel=None)) def start(self): self.log.info('Loop on events...') for event in tqdm(self.event_source, desc='EventWriter', total=self.event_source.max_events, disable=~self.progress): self.event_cutflow.count('no_sel') self.calibrator(event) for tel_id in event.dl0.tels_with_data: self.image_cutflow.count('no_sel') camera = event.inst.subarray.tel[tel_id].camera dl1_tel = event.dl1.tel[tel_id] # Image cleaning image = dl1_tel.image # Waiting for automatic gain selection mask = tailcuts_clean(camera, image, picture_thresh=10, boundary_thresh=5) cleaned = image.copy() cleaned[~mask] = 0 # Preselection cuts if self.image_cutflow.cut('n_pixel', cleaned): continue if self.image_cutflow.cut('image_amplitude', np.sum(cleaned)): continue # Image parametrisation params = hillas_parameters(camera, cleaned) # Save Ids, MC infos and Hillas informations self.writer.write(camera.cam_id, [event.r0, event.mc, params]) def finish(self): self.log.info('End of job.') self.image_cutflow() self.event_cutflow() self.writer.close()
class DumpTriggersTool(Tool): description = Unicode(__doc__) name = 'ctapipe-dump-triggers' # ============================================= # configuration parameters: # ============================================= infile = Path(exists=True, directory_ok=False, help='input simtelarray file').tag(config=True) outfile = Path( default_value='triggers.fits', directory_ok=False, help='output filename (*.fits, *.h5)', ).tag(config=True) overwrite = Bool(False, help="overwrite existing output file").tag(config=True) # ============================================= # map low-level options to high-level command-line options # ============================================= aliases = Dict({ 'infile': 'DumpTriggersTool.infile', 'outfile': 'DumpTriggersTool.outfile' }) flags = Dict({ 'overwrite': ({ 'DumpTriggersTool': { 'overwrite': True } }, 'Enable overwriting of output file') }) examples = ('ctapipe-dump-triggers --infile gamma.simtel.gz ' '--outfile trig.fits --overwrite' '\n\n' 'If you want to see more output, use --log_level=DEBUG') # ============================================= # The methods of the Tool (initialize, start, finish): # ============================================= def add_event_to_table(self, event): """ add the current hessio event to a row in the `self.events` table """ time = event.trigger.time if self._prev_time is None: self._prev_time = time if self._current_starttime is None: self._current_starttime = time relative_time = time - self._current_starttime delta_t = time - self._prev_time self._prev_time = time # build the trigger pattern as a fixed-length array # (better for storage in FITS format) # trigtels = event.get_telescope_with_data_list() trigtels = event.dl0.tels_with_data self._current_trigpattern[:] = 0 # zero the trigger pattern self._current_trigpattern[list(trigtels)] = 1 # set the triggered tels # to 1 # insert the row into the table self.events.add_row( (event.index.event_id, relative_time.sec, delta_t.sec, len(trigtels), self._current_trigpattern)) def setup(self): """ setup function, called before `start()` """ if self.infile == '': raise ToolConfigurationError( "No 'infile' parameter was specified. ") self.events = Table( names=['EVENT_ID', 'T_REL', 'DELTA_T', 'N_TRIG', 'TRIGGERED_TELS'], dtype=[np.int64, np.float64, np.float64, np.int32, np.uint8]) self.events['TRIGGERED_TELS'].shape = (0, MAX_TELS) self.events['T_REL'].unit = u.s self.events['T_REL'].description = 'Time relative to first event' self.events['DELTA_T'].unit = u.s self.events.meta['INPUT'] = self.infile self._current_trigpattern = np.zeros(MAX_TELS) self._current_starttime = None self._prev_time = None def start(self): """ main event loop """ with event_source(self.infile) as source: for event in source: self.add_event_to_table(event) def finish(self): """ finish up and write out results (called automatically after `start()`) """ # write out the final table try: if '.fits' in self.outfile.suffixes: self.events.write(self.outfile, overwrite=self.overwrite) elif self.outfile.suffix in ('.hdf5', '.h5', '.hdf'): self.events.write(self.outfile, path='/events', overwrite=self.overwrite) else: self.events.write(self.outfile) Provenance().add_output_file(self.outfile) except IOError as err: self.log.warning("Couldn't write output (%s)", err) self.log.info('\n %s', self.events)
class ImageSumDisplayerTool(Tool): description = Unicode(__doc__) name = "ctapipe-display-imagesum" infile = Path( help="input simtelarray file", default_value=get_dataset_path("gamma_test_large.simtel.gz"), exists=True, directory_ok=False, ).tag(config=True) telgroup = Integer(help="telescope group number", default_value=1).tag(config=True) max_events = Integer( help="stop after this many events if non-zero", default_value=0, min=0 ).tag(config=True) output_suffix = Unicode( help="suffix (file extension) of output " "filenames to write images " "to (no writing is done if blank). " "Images will be named [EVENTID][suffix]", default_value="", ).tag(config=True) aliases = Dict( { "infile": "ImageSumDisplayerTool.infile", "telgroup": "ImageSumDisplayerTool.telgroup", "max-events": "ImageSumDisplayerTool.max_events", "output-suffix": "ImageSumDisplayerTool.output_suffix", } ) classes = [CameraCalibrator, SimTelEventSource] def setup(self): # load up the telescope types table (need to first open a file, a bit of # a hack until a proper instrument module exists) and select only the # telescopes with the same camera type # make sure gzip files are seekable self.reader = SimTelEventSource( input_url=self.infile, max_events=self.max_events, back_seekable=True ) camtypes = self.reader.subarray.to_table().group_by("camera_type") self.reader.subarray.info(printer=self.log.info) group = camtypes.groups[self.telgroup] self._selected_tels = list(group["tel_id"].data) self._base_tel = self._selected_tels[0] self.log.info( "Telescope group %d: %s", self.telgroup, str(self.reader.subarray.tel[self._selected_tels[0]]), ) self.log.info(f"SELECTED TELESCOPES:{self._selected_tels}") self.calibrator = CameraCalibrator(parent=self, subarray=self.reader.subarray) self.reader = SimTelEventSource( input_url=self.infile, max_events=self.max_events, back_seekable=True, allowed_tels=set(self._selected_tels), ) def start(self): geom = None imsum = None disp = None for event in self.reader: self.calibrator(event) if geom is None: geom = self.reader.subarray.tel[self._base_tel].camera.geometry imsum = np.zeros(shape=geom.pix_x.shape, dtype=np.float64) disp = CameraDisplay(geom, title=geom.camera_name) disp.add_colorbar() disp.cmap = "viridis" if len(event.dl0.tel.keys()) <= 2: continue imsum[:] = 0 for telid in event.dl0.tel.keys(): imsum += event.dl1.tel[telid].image self.log.info( "event={} ntels={} energy={}".format( event.index.event_id, len(event.dl0.tel.keys()), event.simulation.shower.energy, ) ) disp.image = imsum plt.pause(0.1) if self.output_suffix != "": filename = "{:020d}{}".format(event.index.event_id, self.output_suffix) self.log.info(f"saving: '{filename}'") plt.savefig(filename)
class PointingPosition(Component): """ Pointion position of telescopes """ drive_path = Unicode( '', allow_none=True, help='Path to the LST drive report file' ).tag(config=True) tel_id = Int( 0, help='id of the telescope to take drive report for' ).tag(config=True) def _read_drive_report(self): """ Reading drive reports Parameters: ----------- str: drive report file Returns: data:`~astropy.table.Table` A table of drive reports """ self.log.info("Drive report file:", self.drive_path) if self.drive_path: data = ascii.read(self.drive_path) # Renaming the columns, since the drive report doesn't contain # these information it self data['col6'].name = 'time' data['col8'].name = 'azimuth_avg' data['col13'].name = 'zenith_avg' return data else: raise Exception("No drive report file found") def cal_pointingposition(self, ev_time, drive_data): """ Calculating pointing positions by interpolation Parameters: ----------- time: array times from events Drivereport: Container a container filled with drive information """ drive_container = LSTDriveContainer() drive_container.time = drive_data['time'].data drive_container.azimuth_avg = drive_data['azimuth_avg'].data drive_container.altitude_avg = 90.0 - drive_data['zenith_avg'].data xp = drive_container.time lower_drive_time = xp[xp < ev_time].max() upper_drive_time = xp[xp > ev_time].min() time_in_window = (xp >= lower_drive_time) & (xp <= upper_drive_time) run_times = xp[time_in_window] if len(run_times) > 1: run_azimuth = drive_container.azimuth_avg[time_in_window] run_altitude = drive_container.altitude_avg[time_in_window] ev_azimuth = np.interp(ev_time, run_times, run_azimuth) * u.deg ev_altitude = np.interp(ev_time, run_times, run_altitude) * u.deg return ev_azimuth, ev_altitude else: raise Exception("No drive time in the range of event times")