class SingleTelEventDisplay(Tool):
    name = "ctapipe-display-televents"
    description = Unicode(__doc__)

    infile = Path(help="input file to read", exists=True,
                  directory_ok=False).tag(config=True)
    tel = Int(help="Telescope ID to display", default_value=0).tag(config=True)
    write = Bool(help="Write out images to PNG files",
                 default_value=False).tag(config=True)
    clean = Bool(help="Apply image cleaning",
                 default_value=False).tag(config=True)
    hillas = Bool(help="Apply and display Hillas parametrization",
                  default_value=False).tag(config=True)
    samples = Bool(help="Show each sample",
                   default_value=False).tag(config=True)
    display = Bool(help="Display results in interactive window",
                   default_value=True).tag(config=True)
    delay = Float(help="delay between events in s",
                  default_value=0.01,
                  min=0.001).tag(config=True)
    progress = Bool(help="display progress bar",
                    default_value=True).tag(config=True)

    aliases = Dict({
        "infile": "SingleTelEventDisplay.infile",
        "tel": "SingleTelEventDisplay.tel",
        "max-events": "EventSource.max_events",
        "write": "SingleTelEventDisplay.write",
        "clean": "SingleTelEventDisplay.clean",
        "hillas": "SingleTelEventDisplay.hillas",
        "samples": "SingleTelEventDisplay.samples",
        "display": "SingleTelEventDisplay.display",
        "delay": "SingleTelEventDisplay.delay",
        "progress": "SingleTelEventDisplay.progress",
    })

    classes = List([EventSource, CameraCalibrator])

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def setup(self):
        print("TOLLES INFILE", self.infile)
        self.event_source = EventSource.from_url(self.infile, parent=self)
        self.event_source.allowed_tels = {self.tel}

        self.calibrator = CameraCalibrator(parent=self,
                                           subarray=self.event_source.subarray)
        self.log.info(f"SELECTING EVENTS FROM TELESCOPE {self.tel}")

    def start(self):

        disp = None

        for event in tqdm(
                self.event_source,
                desc=f"Tel{self.tel}",
                total=self.event_source.max_events,
                disable=~self.progress,
        ):

            self.log.debug(event.trigger)
            self.log.debug(f"Energy: {event.simulation.shower.energy}")

            self.calibrator(event)

            if disp is None:
                geom = self.event_source.subarray.tel[self.tel].camera.geometry
                self.log.info(geom)
                disp = CameraDisplay(geom)
                # disp.enable_pixel_picker()
                disp.add_colorbar()
                if self.display:
                    plt.show(block=False)

            # display the event
            disp.axes.set_title("CT{:03d} ({}), event {:06d}".format(
                self.tel, geom.camera_name, event.index.event_id))

            if self.samples:
                # display time-varying event
                data = event.dl0.tel[self.tel].waveform
                for ii in range(data.shape[1]):
                    disp.image = data[:, ii]
                    disp.set_limits_percent(70)
                    plt.suptitle(f"Sample {ii:03d}")
                    if self.display:
                        plt.pause(self.delay)
                    if self.write:
                        plt.savefig(
                            f"CT{self.tel:03d}_EV{event.index.event_id:10d}"
                            f"_S{ii:02d}.png")
            else:
                # display integrated event:
                im = event.dl1.tel[self.tel].image

                if self.clean:
                    mask = tailcuts_clean(geom,
                                          im,
                                          picture_thresh=10,
                                          boundary_thresh=7)
                    im[~mask] = 0.0

                disp.image = im

                if self.hillas:
                    try:
                        ellipses = disp.axes.findobj(Ellipse)
                        if len(ellipses) > 0:
                            ellipses[0].remove()

                        params = hillas_parameters(geom, image=im)
                        disp.overlay_moments(params,
                                             color="pink",
                                             lw=3,
                                             with_label=False)
                    except HillasParameterizationError:
                        pass

                if self.display:
                    plt.pause(self.delay)
                if self.write:
                    plt.savefig(
                        f"CT{self.tel:03d}_EV{event.index.event_id:010d}.png")

        self.log.info("FINISHED READING DATA FILE")

        if disp is None:
            self.log.warning(
                "No events for tel {} were found in {}. Try a "
                "different EventIO file or another telescope".format(
                    self.tel, self.infile))
Esempio n. 2
0
class EventViewer(Component):
    """
    Event viewer class built on top of the other plotters to allow a single view of both the camera images
    and the projected Hillas parameters for a single event. Can be further modified to show the reconstructed
    shower direction and core position if needed. Plus further info
    """
    name = 'EventViewer'
    test = Bool(True, help='').tag(config=True)

    def __init__(self, draw_hillas_planes=False):
        """

        Parameters
        ----------
        draw_hillas_planes: bool
            Determines whether a projection of the Hillas parameters in the nominal and tilted systems should be drawn
        """
        self.array_view = None
        self.nominal_view = None

        self.geom = dict()
        self.cam_display = dict()
        self.draw_hillas_planes = draw_hillas_planes

    def draw_source(self, source):
        """
        Loop over events and draw each

        Parameters
        ----------
        source: ctapipe source object

        Returns
        -------
            None
        """
        for event in source:
            self.draw_event(event)

        return

    def draw_event(self, event, hillas_parameters=None):
        """
        Draw display for a given event

        Parameters
        ----------
        event: ctapipe event object
        hillas_parameters: dict
            Dictionary of Hillas parameters (in nominal system)

        Returns
        -------
            None
        """
        tel_list = event.r0.tels_with_data
        images = event.dl1

        # First close any plots that already exist
        plt.close()
        ntels = len(tel_list)

        fig = plt.figure(figsize=(20, 20 * 0.66))

        # If we want to draw the Hillas parameters in different planes we need to split our figure
        if self.draw_hillas_planes:
            y_axis_split = 2
        else:
            y_axis_split = 1

        outer_grid = gridspec.GridSpec(1,
                                       y_axis_split,
                                       width_ratios=[y_axis_split, 1])
        # Create a square grid for camera images
        nn = int(ceil(sqrt(ntels)))
        nx = nn
        ny = nn

        while nx * ny >= ntels:
            ny -= 1
        ny += 1
        while nx * ny >= ntels:
            nx -= 1
        nx += 1

        camera_grid = gridspec.GridSpecFromSubplotSpec(
            ny, nx, subplot_spec=outer_grid[0])

        # Loop over camera images of all telescopes and create plots
        for ii, tel_id in zip(range(ntels), tel_list):

            # Cache of camera geometries, this may go away soon
            if tel_id not in self.geom:
                self.geom[tel_id] = CameraGeometry.guess(
                    event.inst.pixel_pos[tel_id][0],
                    event.inst.pixel_pos[tel_id][1],
                    event.inst.optical_foclen[tel_id])

            ax = plt.subplot(camera_grid[ii])
            self.get_camera_view(tel_id, images.tel[tel_id].image[0], ax)

        # If we want to draw the Hillas parameters in different planes we need to make a couple more viewers
        if self.draw_hillas_planes:
            # Split the second sub figure into two further figures
            reco_grid = gridspec.GridSpecFromSubplotSpec(
                2, 1, subplot_spec=outer_grid[1])
            # Create plot of telescope positions at ground level
            array = ArrayPlotter(
                telescopes=tel_list,
                instrument=event.inst,  # system=tilted_system,
                ax=plt.subplot(reco_grid[0]))
            # Draw MC position (this should change later)
            array.draw_position(event.mc.core_x,
                                event.mc.core_y,
                                use_centre=True)
            array.draw_array(((-300, 300), (-300, 300)))

            # If we have valid Hillas parameters we should draw them in the Nominal system
            if hillas_parameters is not None:
                array.overlay_hillas(hillas_parameters, draw_axes=True)

                nominal = NominalPlotter(hillas_parameters=hillas_parameters,
                                         draw_axes=True,
                                         ax=plt.subplot(reco_grid[1]))
                nominal.draw_array()

        plt.show()

        return

    def get_camera_view(self, tel_id, image, axis):
        """
        Create camera viewer for a given camera image

        Parameters
        ----------
        tel_id: int
            Telescope ID number
        image: ndarray
            Array of calibrated pixel intensities
        axis: matplotlib axis
            Axis on which to draw plot

        Returns
        -------
            Camera display
        """
        #if tel_id not in self.cam_display:
        # Argh this is annoying, for some reason we cannot cahe the displays
        self.cam_display[tel_id] = visualization.CameraDisplay(
            self.geom[tel_id], title="CT{0}".format(tel_id))
        self.cam_display[tel_id].add_colorbar()
        self.cam_display[tel_id].pixels.set_antialiaseds(False)
        self.cam_display[tel_id].autoupdate = True
        self.cam_display[tel_id].cmap = "viridis"

        self.cam_display[tel_id].ax = axis
        self.cam_display[tel_id].image = image

        self.cam_display[tel_id].set_limits_percent(95)

        return self.cam_display[tel_id]
Esempio n. 3
0
class SimpleEventWriter(Tool):
    name = 'ctapipe-simple-event-writer'
    description = Unicode(__doc__)

    infile = Unicode(help='input file to read', default='').tag(config=True)
    outfile = Unicode(help='output file name',
                      default_value='output.h5').tag(config=True)
    progress = Bool(help='display progress bar',
                    default_value=True).tag(config=True)

    aliases = Dict({
        'infile': 'EventSource.input_url',
        'outfile': 'SimpleEventWriter.outfile',
        'max-events': 'EventSource.max_events',
        'progress': 'SimpleEventWriter.progress'
    })
    classes = List([EventSource, CameraCalibrator, CutFlow])

    def setup(self):
        self.log.info('Configure EventSource...')

        self.event_source = self.add_component(
            EventSource.from_config(config=self.config, parent=self))

        self.calibrator = self.add_component(CameraCalibrator(parent=self))

        self.writer = self.add_component(
            HDF5TableWriter(filename=self.outfile,
                            group_name='image_infos',
                            overwrite=True))

        # Define Pre-selection for images
        preselcuts = self.config['Preselect']
        self.image_cutflow = CutFlow('Image preselection')
        self.image_cutflow.set_cuts(
            dict(no_sel=None,
                 n_pixel=lambda s: np.count_nonzero(s) < preselcuts['n_pixel'][
                     'min'],
                 image_amplitude=lambda q: q < preselcuts['image_amplitude'][
                     'min']))

        # Define Pre-selection for events
        self.event_cutflow = CutFlow('Event preselection')
        self.event_cutflow.set_cuts(dict(no_sel=None))

    def start(self):
        self.log.info('Loop on events...')

        for event in tqdm(self.event_source,
                          desc='EventWriter',
                          total=self.event_source.max_events,
                          disable=~self.progress):

            self.event_cutflow.count('no_sel')
            self.calibrator(event)

            for tel_id in event.dl0.tels_with_data:
                self.image_cutflow.count('no_sel')

                camera = event.inst.subarray.tel[tel_id].camera
                dl1_tel = event.dl1.tel[tel_id]

                # Image cleaning
                image = dl1_tel.image  # Waiting for automatic gain selection
                mask = tailcuts_clean(camera,
                                      image,
                                      picture_thresh=10,
                                      boundary_thresh=5)
                cleaned = image.copy()
                cleaned[~mask] = 0

                # Preselection cuts
                if self.image_cutflow.cut('n_pixel', cleaned):
                    continue
                if self.image_cutflow.cut('image_amplitude', np.sum(cleaned)):
                    continue

                # Image parametrisation
                params = hillas_parameters(camera, cleaned)

                # Save Ids, MC infos and Hillas informations
                self.writer.write(camera.cam_id, [event.r0, event.mc, params])

    def finish(self):
        self.log.info('End of job.')

        self.image_cutflow()
        self.event_cutflow()
        self.writer.close()
Esempio n. 4
0
class DumpTriggersTool(Tool):
    description = Unicode(__doc__)
    name = 'ctapipe-dump-triggers'

    # =============================================
    # configuration parameters:
    # =============================================

    infile = Path(exists=True,
                  directory_ok=False,
                  help='input simtelarray file').tag(config=True)

    outfile = Path(
        default_value='triggers.fits',
        directory_ok=False,
        help='output filename (*.fits, *.h5)',
    ).tag(config=True)

    overwrite = Bool(False,
                     help="overwrite existing output file").tag(config=True)

    # =============================================
    # map low-level options to high-level command-line options
    # =============================================

    aliases = Dict({
        'infile': 'DumpTriggersTool.infile',
        'outfile': 'DumpTriggersTool.outfile'
    })

    flags = Dict({
        'overwrite': ({
            'DumpTriggersTool': {
                'overwrite': True
            }
        }, 'Enable overwriting of output file')
    })

    examples = ('ctapipe-dump-triggers --infile gamma.simtel.gz '
                '--outfile trig.fits --overwrite'
                '\n\n'
                'If you want to see more output, use --log_level=DEBUG')

    # =============================================
    # The methods of the Tool (initialize, start, finish):
    # =============================================

    def add_event_to_table(self, event):
        """
        add the current hessio event to a row in the `self.events` table
        """
        time = event.trigger.time

        if self._prev_time is None:
            self._prev_time = time

        if self._current_starttime is None:
            self._current_starttime = time

        relative_time = time - self._current_starttime
        delta_t = time - self._prev_time
        self._prev_time = time

        # build the trigger pattern as a fixed-length array
        # (better for storage in FITS format)
        # trigtels = event.get_telescope_with_data_list()
        trigtels = event.dl0.tels_with_data
        self._current_trigpattern[:] = 0  # zero the trigger pattern
        self._current_trigpattern[list(trigtels)] = 1  # set the triggered tels
        # to 1

        # insert the row into the table
        self.events.add_row(
            (event.index.event_id, relative_time.sec, delta_t.sec,
             len(trigtels), self._current_trigpattern))

    def setup(self):
        """ setup function, called before `start()` """

        if self.infile == '':
            raise ToolConfigurationError(
                "No 'infile' parameter was specified. ")

        self.events = Table(
            names=['EVENT_ID', 'T_REL', 'DELTA_T', 'N_TRIG', 'TRIGGERED_TELS'],
            dtype=[np.int64, np.float64, np.float64, np.int32, np.uint8])

        self.events['TRIGGERED_TELS'].shape = (0, MAX_TELS)
        self.events['T_REL'].unit = u.s
        self.events['T_REL'].description = 'Time relative to first event'
        self.events['DELTA_T'].unit = u.s
        self.events.meta['INPUT'] = self.infile

        self._current_trigpattern = np.zeros(MAX_TELS)
        self._current_starttime = None
        self._prev_time = None

    def start(self):
        """ main event loop """
        with event_source(self.infile) as source:
            for event in source:
                self.add_event_to_table(event)

    def finish(self):
        """
        finish up and write out results (called automatically after
        `start()`)
        """
        # write out the final table
        try:
            if '.fits' in self.outfile.suffixes:
                self.events.write(self.outfile, overwrite=self.overwrite)
            elif self.outfile.suffix in ('.hdf5', '.h5', '.hdf'):
                self.events.write(self.outfile,
                                  path='/events',
                                  overwrite=self.overwrite)
            else:
                self.events.write(self.outfile)

            Provenance().add_output_file(self.outfile)
        except IOError as err:
            self.log.warning("Couldn't write output (%s)", err)

        self.log.info('\n %s', self.events)
class SingleTelEventDisplay(Tool):
    name = "ctapipe-display-televents"
    description = Unicode(__doc__)

    infile = Unicode(help="input file to read", default='').tag(config=True)
    tel = Int(help='Telescope ID to display', default=0).tag(config=True)
    channel = Integer(help="channel number to display", min=0,
                      max=1).tag(config=True)
    write = Bool(help="Write out images to PNG files",
                 default=False).tag(config=True)
    clean = Bool(help="Apply image cleaning", default=False).tag(config=True)
    hillas = Bool(help="Apply and display Hillas parametrization",
                  default=False).tag(config=True)
    samples = Bool(help="Show each sample", default=False).tag(config=True)
    display = Bool(help="Display results in interactive window",
                   default_value=True).tag(config=True)
    delay = Float(help='delay between events in s',
                  default_value=0.01,
                  min=0.001).tag(config=True)
    progress = Bool(help='display progress bar',
                    default_value=True).tag(config=True)

    aliases = Dict({
        'infile': 'SingleTelEventDisplay.infile',
        'tel': 'SingleTelEventDisplay.tel',
        'max-events': 'EventSource.max_events',
        'channel': 'SingleTelEventDisplay.channel',
        'write': 'SingleTelEventDisplay.write',
        'clean': 'SingleTelEventDisplay.clean',
        'hillas': 'SingleTelEventDisplay.hillas',
        'samples': 'SingleTelEventDisplay.samples',
        'display': 'SingleTelEventDisplay.display',
        'delay': 'SingleTelEventDisplay.delay',
        'progress': 'SingleTelEventDisplay.progress'
    })

    classes = List([EventSource, CameraCalibrator])

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def setup(self):
        print('TOLLES INFILE', self.infile)
        self.event_source = EventSource.from_url(self.infile, parent=self)
        self.event_source.allowed_tels = {
            self.tel,
        }

        self.calibrator = CameraCalibrator(parent=self)

        self.log.info(f'SELECTING EVENTS FROM TELESCOPE {self.tel}')

    def start(self):

        disp = None

        for event in tqdm(self.event_source,
                          desc=f'Tel{self.tel}',
                          total=self.event_source.max_events,
                          disable=~self.progress):

            self.log.debug(event.trig)
            self.log.debug(f"Energy: {event.mc.energy}")

            self.calibrator(event)

            if disp is None:
                geom = event.inst.subarray.tel[self.tel].camera
                self.log.info(geom)
                disp = CameraDisplay(geom)
                # disp.enable_pixel_picker()
                disp.add_colorbar()
                if self.display:
                    plt.show(block=False)

            # display the event
            disp.axes.set_title('CT{:03d} ({}), event {:06d}'.format(
                self.tel, geom.cam_id, event.r0.event_id))

            if self.samples:
                # display time-varying event
                data = event.dl0.tel[self.tel].waveform[self.channel]
                for ii in range(data.shape[1]):
                    disp.image = data[:, ii]
                    disp.set_limits_percent(70)
                    plt.suptitle(f"Sample {ii:03d}")
                    if self.display:
                        plt.pause(self.delay)
                    if self.write:
                        plt.savefig(
                            f'CT{self.tel:03d}_EV{event.r0.event_id:10d}'
                            f'_S{ii:02d}.png')
            else:
                # display integrated event:
                im = event.dl1.tel[self.tel].image[self.channel]

                if self.clean:
                    mask = tailcuts_clean(geom,
                                          im,
                                          picture_thresh=10,
                                          boundary_thresh=7)
                    im[~mask] = 0.0

                disp.image = im

                if self.hillas:
                    try:
                        ellipses = disp.axes.findobj(Ellipse)
                        if len(ellipses) > 0:
                            ellipses[0].remove()

                        params = hillas_parameters(geom, image=im)
                        disp.overlay_moments(params,
                                             color='pink',
                                             lw=3,
                                             with_label=False)
                    except HillasParameterizationError:
                        pass

                if self.display:
                    plt.pause(self.delay)
                if self.write:
                    plt.savefig(
                        f'CT{self.tel:03d}_EV{event.r0.event_id:010d}.png')

        self.log.info("FINISHED READING DATA FILE")

        if disp is None:
            self.log.warning(
                'No events for tel {} were found in {}. Try a '
                'different EventIO file or another telescope'.format(
                    self.tel, self.infile), )
Esempio n. 6
0
class LSTEventSource(EventSource):
    """EventSource for LST r0 data."""

    n_gains = Int(
        2,
        help='Number of gains at r0/r1 level'
    ).tag(config=True)

    baseline = Int(
        400,
        help='r0 waveform baseline (default from EvB v3)'
    ).tag(config=True)

    multi_streams = Bool(
        True,
        help='Read in parallel all streams '
    ).tag(config=True)

    def __init__(self, **kwargs):
        """
        Constructor
        Parameters
        ----------
        n_gains = number of gains expected in input file

        baseline = baseline to be subtracted at r1 level (not used for the moment)

        multi_streams = enable the reading of input files from all streams

        config: traitlets.loader.Config
            Configuration specified by config file or cmdline arguments.
            Used to set traitlet values.
            Set to None if no configuration to pass.\
        kwargs: dict
            Additional parameters to be passed.
            NOTE: The file mask of the data to read can be passed with
            the 'input_url' parameter.
        """

        super().__init__(**kwargs)

        if self.multi_streams:
            # test how many streams are there:
            # file name must be [stream name]Run[all the rest]
            # All the files with the same [all the rest] are opened

            path, name = os.path.split(os.path.abspath(self.input_url))
            if 'Run' in name:
                stream, run = name.split('Run', 1)
            else:
                run = name

            ls = listdir(path)
            self.file_list = []

            for file_name in ls:
                if run in file_name:
                    full_name = os.path.join(path, file_name)
                    self.file_list.append(full_name)

        else:
            self.file_list = [self.input_url]

        self.multi_file = MultiFiles(self.file_list)
        self.geometry_version = 4

        self.camera_config = self.multi_file.camera_config
        self.log.info(
            "Read {} input files".format(
                self.multi_file.num_inputs()
            )
        )
        self.tel_id = self.camera_config.telescope_id
        self._subarray = self.create_subarray(self.tel_id)
        self.n_camera_pixels = self.subarray.tel[self.tel_id].camera.n_pixels

    @property
    def subarray(self):
        return self._subarray

    @property
    def is_simulation(self):
        return False

    @property
    def obs_id(self):
        # currently no obs id is available from the input files
        return self.camera_config.configuration_id

    @property
    def datalevels(self):
        return (DataLevel.R0, )

    def rewind(self):
        self.multi_file.rewind()

    def create_subarray(self, tel_id=1):
        """
        Obtain the subarray from the EventSource
        Returns
        -------
        ctapipe.instrument.SubarrayDecription
        """

        # camera info from LSTCam-[geometry_version].camgeom.fits.gz file
        camera = load_camera_geometry(version=self.geometry_version)

        tel_descr = TelescopeDescription(
            name='LST', tel_type='LST', optics=OPTICS, camera=camera
        )

        tels = {tel_id: tel_descr}

        # LSTs telescope position taken from MC from the moment
        tel_pos = {tel_id: [50., 50., 16] * u.m}

        subarray = SubarrayDescription("LST1 subarray")
        subarray.tels = tels
        subarray.positions = tel_pos

        return subarray

    def _generator(self):

        # container for LST data
        self.data = LSTDataContainer()
        self.data.meta['input_url'] = self.input_url
        self.data.meta['max_events'] = self.max_events
        self.data.meta['origin'] = 'LSTCAM'

        # fill LST data from the CameraConfig table
        self.fill_lst_service_container_from_zfile()

        # initialize general monitoring container
        self.initialize_mon_container()

        # loop on events
        for count, event in enumerate(self.multi_file):

            self.data.count = count
            self.data.index.event_id = event.event_id
            self.data.index.obs_id = self.obs_id

            # fill specific LST event data
            self.fill_lst_event_container_from_zfile(event)

            # fill general monitoring data
            self.fill_mon_container_from_zfile(event)

            # fill general R0 data
            self.fill_r0_container_from_zfile(event)

            yield self.data

    @staticmethod
    def is_compatible(file_path):
        from astropy.io import fits
        try:
            # The file contains two tables:
            #  1: CameraConfig
            #  2: Events
            h = fits.open(file_path)[2].header
            ttypes = [
                h[x] for x in h.keys() if 'TTYPE' in x
            ]
        except OSError:
            # not even a fits file
            return False

        except IndexError:
            # A fits file of a different format
            return False

        is_protobuf_zfits_file = (
            (h['XTENSION'] == 'BINTABLE') and
            (h['EXTNAME'] == 'Events') and
            (h['ZTABLE'] is True) and
            (h['ORIGIN'] == 'CTA') and
            (h['PBFHEAD'] == 'R1.CameraEvent')
        )

        is_lst_file = 'lstcam_counters' in ttypes
        return is_protobuf_zfits_file & is_lst_file

    def fill_lst_service_container_from_zfile(self):
        """
        Fill LSTServiceContainer with specific LST service data data
        (from the CameraConfig table of zfit file)

        """

        self.data.lst.tels_with_data = [self.tel_id, ]
        svc_container = self.data.lst.tel[self.tel_id].svc

        svc_container.telescope_id = self.tel_id
        svc_container.cs_serial = self.camera_config.cs_serial
        svc_container.configuration_id = self.camera_config.configuration_id
        svc_container.date = self.camera_config.date
        svc_container.num_pixels = self.camera_config.num_pixels
        svc_container.num_samples = self.camera_config.num_samples
        svc_container.pixel_ids = self.camera_config.expected_pixels_id
        svc_container.data_model_version = self.camera_config.data_model_version
        svc_container.num_modules = self.camera_config.lstcam.num_modules
        svc_container.module_ids = self.camera_config.lstcam.expected_modules_id
        svc_container.idaq_version = self.camera_config.lstcam.idaq_version
        svc_container.cdhs_version = self.camera_config.lstcam.cdhs_version
        svc_container.algorithms = self.camera_config.lstcam.algorithms
        svc_container.pre_proc_algorithms = self.camera_config.lstcam.pre_proc_algorithms

    def fill_lst_event_container_from_zfile(self, event):
        """
        Fill LSTEventContainer with specific LST service data
        (from the Event table of zfit file)

        """

        event_container = self.data.lst.tel[self.tel_id].evt

        event_container.configuration_id = event.configuration_id
        event_container.event_id = event.event_id
        event_container.tel_event_id = event.tel_event_id
        event_container.pixel_status = event.pixel_status
        event_container.ped_id = event.ped_id
        event_container.module_status = event.lstcam.module_status
        event_container.extdevices_presence = event.lstcam.extdevices_presence

        # if TIB data are there
        if event_container.extdevices_presence & 1:
            # unpack TIB data
            rec_fmt = '=IHIBB'
            unpacked_tib = struct.unpack(rec_fmt, event.lstcam.tib_data)
            event_container.tib_event_counter = unpacked_tib[0]
            event_container.tib_pps_counter = unpacked_tib[1]
            event_container.tib_tenMHz_counter = unpacked_tib[2]
            event_container.tib_stereo_pattern = unpacked_tib[3]
            event_container.tib_masked_trigger = unpacked_tib[4]

        # if UCTS data are there
        if event_container.extdevices_presence & 2:

            if int(self.data.lst.tel[self.tel_id].svc.idaq_version) > 37201:

                # unpack UCTS-CDTS data (new version)
                rec_fmt = '=QIIIIIBBBBI'
                unpacked_cdts = struct.unpack(rec_fmt, event.lstcam.cdts_data)
                event_container.ucts_timestamp = unpacked_cdts[0]
                event_container.ucts_address = unpacked_cdts[1]        # new
                event_container.ucts_event_counter = unpacked_cdts[2]
                event_container.ucts_busy_counter = unpacked_cdts[3]   # new
                event_container.ucts_pps_counter = unpacked_cdts[4]
                event_container.ucts_clock_counter = unpacked_cdts[5]
                event_container.ucts_trigger_type = unpacked_cdts[6]
                event_container.ucts_white_rabbit_status = unpacked_cdts[7]
                event_container.ucts_stereo_pattern = unpacked_cdts[8] # new
                event_container.ucts_num_in_bunch = unpacked_cdts[9]   # new
                event_container.ucts_cdts_version = unpacked_cdts[10]  # new

            else:
                # unpack UCTS-CDTS data (old version)
                rec_fmt = '=IIIQQBBB'
                unpacked_cdts =  struct.unpack(rec_fmt, event.lstcam.cdts_data)
                event_container.ucts_event_counter = unpacked_cdts[0]
                event_container.ucts_pps_counter = unpacked_cdts[1]
                event_container.ucts_clock_counter = unpacked_cdts[2]
                event_container.ucts_timestamp = unpacked_cdts[3]
                event_container.ucts_camera_timestamp = unpacked_cdts[4]
                event_container.ucts_trigger_type = unpacked_cdts[5]
                event_container.ucts_white_rabbit_status = unpacked_cdts[6]

        # if SWAT data are there
        if event_container.extdevices_presence & 4:
            # unpack SWAT data
            rec_fmt = '=QIIBBIBI'
            unpacked_swat = struct.unpack(rec_fmt, event.lstcam.swat_data)
            event_container.swat_timestamp = unpacked_swat[0]
            event_container.swat_counter1 = unpacked_swat[1]
            event_container.swat_counter2 = unpacked_swat[2]
            event_container.swat_event_type = unpacked_swat[3]
            event_container.swat_camera_flag = unpacked_swat[4]
            event_container.swat_camera_event_num = unpacked_swat[5]
            event_container.swat_array_flag = unpacked_swat[6]
            event_container.swat_array_event_num = unpacked_swat[7]

        # unpack Dragon counters
        rec_fmt = '=HIIIQ'
        rec_len = struct.calcsize(rec_fmt)
        rec_unpack = struct.Struct(rec_fmt).unpack_from

        event_container.pps_counter = np.zeros(self.camera_config.lstcam.num_modules)
        event_container.tenMHz_counter = np.zeros(self.camera_config.lstcam.num_modules)
        event_container.event_counter = np.zeros(self.camera_config.lstcam.num_modules)
        event_container.trigger_counter = np.zeros(self.camera_config.lstcam.num_modules)
        event_container.local_clock_counter = np.zeros(self.camera_config.lstcam.num_modules)
        for mod in range(self.camera_config.lstcam.num_modules):

            words=event.lstcam.counters[mod*rec_len:(mod+1)*rec_len]
            unpacked_counter = rec_unpack(words)
            event_container.pps_counter[mod] = unpacked_counter[0]
            event_container.tenMHz_counter[mod] = unpacked_counter[1]
            event_container.event_counter[mod] = unpacked_counter[2]
            event_container.trigger_counter[mod] = unpacked_counter[3]
            event_container.local_clock_counter[mod] = unpacked_counter[4]

        event_container.chips_flags = event.lstcam.chips_flags
        event_container.first_capacitor_id = event.lstcam.first_capacitor_id
        event_container.drs_tag_status = event.lstcam.drs_tag_status
        event_container.drs_tag = event.lstcam.drs_tag

    def fill_r0_camera_container_from_zfile(self, r0_container, event):
        """
        Fill with R0CameraContainer
        """

        # look for correct trigger_time (TAI time in s), first in UCTS and then in TIB
        #if self.data.lst.tel[self.tel_id].evt.ucts_timestamp > 0:
        #    r0_container.trigger_time = self.data.lst.tel[self.tel_id].evt.ucts_timestamp/1e9

        # consider for the moment only TIB time since UCTS seems not correct
        #if self.data.lst.tel[self.tel_id].evt.tib_pps_counter > 0:
        #    r0_container.trigger_time = (
        #        self.data.lst.tel[self.tel_id].svc.date +
        #        self.data.lst.tel[self.tel_id].evt.tib_pps_counter +
        #        self.data.lst.tel[self.tel_id].evt.tib_tenMHz_counter * 10**(-7))
        #else:
        #    r0_container.trigger_time = 0

        #consider for the moment trigger time from central dragon module
        module_rank = np.where(self.data.lst.tel[self.tel_id].svc.module_ids == 132)
        r0_container.trigger_time = (
                    self.data.lst.tel[self.tel_id].svc.date +
                    self.data.lst.tel[self.tel_id].evt.pps_counter[module_rank] +
                    self.data.lst.tel[self.tel_id].evt.tenMHz_counter[module_rank] * 10**(-7))

        # look for correct trigger type first in UCTS and then in TIB
        #if self.data.lst.tel[self.tel_id].evt.ucts_trigger_type > 0:
        #    r0_container.trigger_type = self.data.lst.tel[self.tel_id].evt.ucts_trigger_type

        # consider for the moment only TIB trigger since UCTS seems not correct
        if self.data.lst.tel[self.tel_id].evt.tib_masked_trigger > 0:
            r0_container.trigger_type = self.data.lst.tel[self.tel_id].evt.tib_masked_trigger
        else:
            r0_container.trigger_type = -1

        # verify the number of gains
        if event.waveform.shape[0] != self.camera_config.num_pixels * self.camera_config.num_samples * self.n_gains:
            raise ValueError(f"Number of gains not correct, waveform shape is {event.waveform.shape[0]}"
                             f" instead of "
                             f"{self.camera_config.num_pixels * self.camera_config.num_samples * self.n_gains}")

        reshaped_waveform = np.array(
            event.waveform
        ).reshape(
            self.n_gains,
            self.camera_config.num_pixels,
            self.camera_config.num_samples
        )

        # initialize the waveform container to zero
        r0_container.waveform = np.zeros([self.n_gains, self.n_camera_pixels,
                                          self.camera_config.num_samples])

        # re-order the waveform following the expected_pixels_id values
        # (rank = pixel id)
        r0_container.waveform[:, self.camera_config.expected_pixels_id, :] =\
            reshaped_waveform

    def fill_r0_container_from_zfile(self, event):
        """
        Fill with R0Container

        """
        container = self.data.r0

        container.tels_with_data = [self.tel_id, ]
        r0_camera_container = container.tel[self.tel_id]
        self.fill_r0_camera_container_from_zfile(
            r0_camera_container,
            event
        )

    def initialize_mon_container(self):
        """
        Fill with MonitoringContainer.
        For the moment, initialize only the PixelStatusContainer

        """
        container = self.data.mon
        container.tels_with_data = [self.tel_id, ]
        mon_camera_container = container.tel[self.tel_id]

        # initialize the container
        status_container = PixelStatusContainer()
        status_container.hardware_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)
        status_container.pedestal_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)
        status_container.flatfield_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)

        mon_camera_container.pixel_status = status_container

    def fill_mon_container_from_zfile(self, event):
        """
        Fill with MonitoringContainer.
        For the moment, initialize only the PixelStatusContainer

        """

        status_container = self.data.mon.tel[self.tel_id].pixel_status

        # reorder the array
        pixel_status = np.zeros(self.n_camera_pixels)
        pixel_status[self.camera_config.expected_pixels_id] = event.pixel_status
        status_container.hardware_failing_pixels[:] = pixel_status == 0
Esempio n. 7
0
class SimpleEventWriter(Tool):
    name = "ctapipe-simple-event-writer"
    description = Unicode(__doc__)

    infile = Path(
        default_value=get_dataset_path(
            "lst_prod3_calibration_and_mcphotons.simtel.zst"),
        help="input file to read",
        directory_ok=False,
        exists=True,
    ).tag(config=True)
    outfile = Path(help="output file name",
                   directory_ok=False,
                   default_value="output.h5").tag(config=True)
    progress = Bool(help="display progress bar",
                    default_value=True).tag(config=True)

    aliases = Dict({
        "infile": "EventSource.input_url",
        "outfile": "SimpleEventWriter.outfile",
        "max-events": "EventSource.max_events",
        "progress": "SimpleEventWriter.progress",
    })
    classes = List([EventSource, CameraCalibrator])

    def setup(self):
        self.log.info("Configure EventSource...")

        self.event_source = EventSource.from_url(self.infile, parent=self)
        self.calibrator = CameraCalibrator(subarray=self.event_source.subarray,
                                           parent=self)
        self.writer = HDF5TableWriter(filename=self.outfile,
                                      group_name="image_infos",
                                      overwrite=True,
                                      parent=self)

    def start(self):
        self.log.info("Loop on events...")

        for event in tqdm(
                self.event_source,
                desc="EventWriter",
                total=self.event_source.max_events,
                disable=~self.progress,
        ):

            self.calibrator(event)

            for tel_id in event.dl0.tel.keys():

                geom = self.event_source.subarray.tel[tel_id].camera.geometry
                dl1_tel = event.dl1.tel[tel_id]

                # Image cleaning
                image = dl1_tel.image  # Waiting for automatic gain selection
                mask = tailcuts_clean(geom,
                                      image,
                                      picture_thresh=10,
                                      boundary_thresh=5)
                cleaned = image.copy()
                cleaned[~mask] = 0

                # Image parametrisation
                params = hillas_parameters(geom, cleaned)

                # Save Ids, MC infos and Hillas informations
                self.writer.write(geom.camera_name,
                                  [event.r0, event.simulation.shower, params])

    def finish(self):
        self.log.info("End of job.")
        self.writer.close()
Esempio n. 8
0
class LSTR0Corrections(TelescopeComponent):
    """
    The base R0-level calibrator. Changes the r0 container.

    The R0 calibrator performs the camera-specific R0 calibration that is
    usually performed on the raw data by the camera server.
    This calibrator exists in lstchain for testing and prototyping purposes.
    """
    offset = IntTelescopeParameter(
        default_value=0,
        help=(
            'Define offset to be subtracted from the waveform *additionally*'
            ' to the drs4 pedestal offset. This only needs to be given when'
            ' the drs4 pedestal calibration is not applied or the offset of the'
            ' drs4 run is different from the data run'
        )
    ).tag(config=True)

    r1_sample_start = IntTelescopeParameter(
        default_value=3,
        help='Start sample for r1 waveform',
        allow_none=True,
    ).tag(config=True)

    r1_sample_end = IntTelescopeParameter(
        default_value=39,
        help='End sample for r1 waveform',
        allow_none=True,
    ).tag(config=True)

    drs4_pedestal_path = TelescopeParameter(
        trait=Path(exists=True, directory_ok=False),
        allow_none=True,
        default_value=None,
        help=(
            'Path to the LST pedestal file'
            ', required when `apply_drs4_pedestal_correction=True`'
        ),
    ).tag(config=True)

    calibration_path = Path(
        exists=True, directory_ok=False,
        help='Path to LST calibration file',
    ).tag(config=True)

    drs4_time_calibration_path = TelescopeParameter(
        trait=Path(exists=True, directory_ok=False),
        help='Path to the time calibration file',
        default_value=None,
        allow_none=True,
    ).tag(config=True)

    calib_scale_high_gain = FloatTelescopeParameter(
        default_value=1.0,
        help='High gain waveform is multiplied by this number'
    ).tag(config=True)

    calib_scale_low_gain = FloatTelescopeParameter(
        default_value=1.0,
        help='Low gain waveform is multiplied by this number'
    ).tag(config=True)

    select_gain = Bool(
        default_value=True,
        help='Set to False to keep both gains.'
    ).tag(config=True)

    apply_drs4_pedestal_correction = Bool(
        default_value=True,
        help=(
            'Set to False to disable drs4 pedestal correction.'
            ' Providing the drs4_pedestal_path is required to perform this calibration'
        ),
    ).tag(config=True)

    apply_timelapse_correction = Bool(
        default_value=True,
        help='Set to False to disable drs4 timelapse correction'
    ).tag(config=True)

    apply_spike_correction = Bool(
        default_value=True,
        help='Set to False to disable drs4 spike correction'
    ).tag(config=True)

    add_calibration_timeshift = Bool(
        default_value=True,
        help=(
            'If true, time correction from the calibration'
            ' file is added to calibration.dl1.time'
        ),
    ).tag(config=True)

    gain_selection_threshold = Float(
        default_value=3500,
        help='Threshold for the ThresholdGainSelector.'
    ).tag(config=True)

    def __init__(self, subarray, config=None, parent=None, **kwargs):
        """
        The R0 calibrator for LST data.
        Fill the r1 container.

        Parameters
        ----------
        """
        super().__init__(
            subarray=subarray, config=config, parent=parent, **kwargs
        )

        self.mon_data = None
        self.last_readout_time = {}
        self.first_cap = {}
        self.first_cap_old = {}
        self.fbn = {}
        self.fan = {}

        for tel_id in self.subarray.tel:
            shape = (N_GAINS, N_PIXELS, N_CAPACITORS_PIXEL)
            self.last_readout_time[tel_id] = np.zeros(shape, dtype='uint64')

            shape = (N_GAINS, N_PIXELS)
            self.first_cap[tel_id] = np.zeros(shape, dtype=int)
            self.first_cap_old[tel_id] = np.zeros(shape, dtype=int)

        if self.select_gain:
            self.gain_selector = ThresholdGainSelector(
                threshold=self.gain_selection_threshold,
                parent=self
            )
        else:
            self.gain_selector = None

        if self.calibration_path is not None:
            self.mon_data = self._read_calibration_file(self.calibration_path)

    def apply_drs4_corrections(self, event: LSTArrayEventContainer):
        self.update_first_capacitors(event)

        for tel_id, r0 in event.r0.tel.items():
            r1 = event.r1.tel[tel_id]
            # If r1 was not yet filled, copy of r0 converted
            if r1.waveform is None:
                r1.waveform = r0.waveform

            # float32 can represent all values of uint16 exactly,
            # so this does not loose precision.
            r1.waveform = r1.waveform.astype(np.float32, copy=False)

            # apply drs4 corrections
            if self.apply_drs4_pedestal_correction:
                self.subtract_pedestal(event, tel_id)

            if self.apply_timelapse_correction:
                self.time_lapse_corr(event, tel_id)

            if self.apply_spike_correction:
                self.interpolate_spikes(event, tel_id)

            # remove samples at beginning / end of waveform
            start = self.r1_sample_start.tel[tel_id]
            end = self.r1_sample_end.tel[tel_id]
            r1.waveform = r1.waveform[..., start:end]

            if self.offset.tel[tel_id] != 0:
                r1.waveform -= self.offset.tel[tel_id]

            mon = event.mon.tel[tel_id]
            if r1.selected_gain_channel is None:
                r1.waveform[mon.pixel_status.hardware_failing_pixels] = 0.0
            else:
                broken = mon.pixel_status.hardware_failing_pixels[r1.selected_gain_channel, PIXEL_INDEX]
                r1.waveform[broken] = 0.0


    def update_first_capacitors(self, event: LSTArrayEventContainer):
        for tel_id, lst in event.lst.tel.items():
            self.first_cap_old[tel_id] = self.first_cap[tel_id]
            self.first_cap[tel_id] = get_first_capacitors_for_pixels(
                lst.evt.first_capacitor_id,
                lst.svc.pixel_ids,
            )

    def calibrate(self, event: LSTArrayEventContainer):
        for tel_id in event.r0.tel:
            r1 = event.r1.tel[tel_id]
            # if `apply_drs4_corrections` is False, we did not fill in the
            # waveform yet.
            if r1.waveform is None:
                r1.waveform = event.r0.tel[tel_id].waveform

            r1.waveform = r1.waveform.astype(np.float32, copy=False)

            # do gain selection before converting to pe
            # like eventbuilder will do
            if self.select_gain and r1.selected_gain_channel is None:
                r1.selected_gain_channel = self.gain_selector(r1.waveform)
                r1.waveform = r1.waveform[r1.selected_gain_channel, PIXEL_INDEX]

            # apply monitoring data corrections,
            # subtract pedestal and convert to pe
            if self.mon_data is not None:
                calibration = self.mon_data.tel[tel_id].calibration
                convert_to_pe(
                    waveform=r1.waveform,
                    calibration=calibration,
                    selected_gain_channel=r1.selected_gain_channel
                )

            broken_pixels = event.mon.tel[tel_id].pixel_status.hardware_failing_pixels
            if r1.selected_gain_channel is None:
                r1.waveform[broken_pixels] = 0.0
            else:
                r1.waveform[broken_pixels[r1.selected_gain_channel, PIXEL_INDEX]] = 0.0

            # store calibration data needed for dl1 calibration in ctapipe
            # first drs4 time shift (zeros if no calib file was given)
            time_shift = self.get_drs4_time_correction(
                tel_id, self.first_cap[tel_id],
                selected_gain_channel=r1.selected_gain_channel,
            )

            # time shift from flat fielding
            if self.mon_data is not None and self.add_calibration_timeshift:
                time_corr = self.mon_data.tel[tel_id].calibration.time_correction
                # time_shift is subtracted in ctapipe,
                # but time_correction should be added
                if r1.selected_gain_channel is not None:
                    time_shift -= time_corr[r1.selected_gain_channel, PIXEL_INDEX].to_value(u.ns)
                else:
                    time_shift -= time_corr.to_value(u.ns)

            event.calibration.tel[tel_id].dl1.time_shift = time_shift

            # needed for charge scaling in ctpaipe dl1 calib
            if r1.selected_gain_channel is not None:
                relative_factor = np.empty(N_PIXELS)
                relative_factor[r1.selected_gain_channel == HIGH_GAIN] = self.calib_scale_high_gain.tel[tel_id]
                relative_factor[r1.selected_gain_channel == LOW_GAIN] = self.calib_scale_low_gain.tel[tel_id]
            else:
                relative_factor = np.empty((N_GAINS, N_PIXELS))
                relative_factor[HIGH_GAIN] = self.calib_scale_high_gain.tel[tel_id]
                relative_factor[LOW_GAIN] = self.calib_scale_low_gain.tel[tel_id]

            event.calibration.tel[tel_id].dl1.relative_factor = relative_factor

    @staticmethod
    def _read_calibration_file(path):
        """
        Read the correction from hdf5 calibration file
        """
        mon = MonitoringContainer()

        with tables.open_file(path) as f:
            tel_ids = [
                int(key[4:]) for key in f.root._v_children.keys()
                if key.startswith('tel_')
            ]

        for tel_id in tel_ids:
            with HDF5TableReader(path) as h5_table:
                base = f'/tel_{tel_id}'
                # read the calibration data
                table = base + '/calibration'
                next(h5_table.read(table, mon.tel[tel_id].calibration))

                # read pedestal data
                table = base + '/pedestal'
                next(h5_table.read(table, mon.tel[tel_id].pedestal))

                # read flat-field data
                table = base + '/flatfield'
                next(h5_table.read(table, mon.tel[tel_id].flatfield))

                # read the pixel_status container
                table = base + '/pixel_status'
                next(h5_table.read(table, mon.tel[tel_id].pixel_status))

        return mon

    @staticmethod
    def load_drs4_time_calibration_file(path):
        """
        Function to load calibration file.
        """
        with tables.open_file(path, 'r') as f:
            fan = f.root.fan[:]
            fbn = f.root.fbn[:]

        return fan, fbn

    def load_drs4_time_calibration_file_for_tel(self, tel_id):
        self.fan[tel_id], self.fbn[tel_id] = self.load_drs4_time_calibration_file(
            self.drs4_time_calibration_path.tel[tel_id]
        )

    def get_drs4_time_correction(self, tel_id, first_capacitors, selected_gain_channel=None):
        """
        Return pulse time after time correction.
        """

        if self.drs4_time_calibration_path.tel[tel_id] is None:
            if selected_gain_channel is None:
                return np.zeros((N_GAINS, N_PIXELS))
            else:
                return np.zeros(N_PIXELS)

        # load calib file if not already done
        if tel_id not in self.fan:
            self.load_drs4_time_calibration_file_for_tel(tel_id)

        if selected_gain_channel is not None:
            return calc_drs4_time_correction_gain_selected(
                first_capacitors,
                selected_gain_channel,
                self.fan[tel_id],
                self.fbn[tel_id],
            )
        else:
            return calc_drs4_time_correction_both_gains(
                first_capacitors,
                self.fan[tel_id],
                self.fbn[tel_id],
            )

    @staticmethod
    @lru_cache(maxsize=4)
    def _get_drs4_pedestal_data(path):
        """
        Function to load pedestal file.

        To make boundary conditions unnecessary,
        the first N_SAMPLES values are repeated at the end of the array

        The result is cached so we can repeatedly call this method
        using the configured path without reading it each time.
        """
        if path is None:
            raise ValueError(
                "DRS4 pedestal correction requested"
                " but no file provided for telescope"
            )

        pedestal_data = np.empty(
            (N_GAINS, N_PIXELS_MODULE * N_MODULES, N_CAPACITORS_PIXEL + N_SAMPLES),
            dtype=np.int16
        )
        with fits.open(path) as f:
            pedestal_data[:, :, :N_CAPACITORS_PIXEL] = f[1].data

        pedestal_data[:, :, N_CAPACITORS_PIXEL:] = pedestal_data[:, :, :N_SAMPLES]

        return pedestal_data

    def subtract_pedestal(self, event, tel_id):
        """
        Subtract cell offset using pedestal file.
        Fill the R1 container.
        Parameters
        ----------
        event : `ctapipe` event-container
        tel_id : id of the telescope
        """
        pedestal = self._get_drs4_pedestal_data(
            self.drs4_pedestal_path.tel[tel_id]
        )

        if event.r1.tel[tel_id].selected_gain_channel is None:
            subtract_pedestal(
                event.r1.tel[tel_id].waveform,
                self.first_cap[tel_id],
                pedestal,
            )
        else:
            subtract_pedestal_gain_selected(
                event.r1.tel[tel_id].waveform,
                self.first_cap[tel_id],
                pedestal,
                event.r1.tel[tel_id].selected_gain_channel,
            )


    def time_lapse_corr(self, event, tel_id):
        """
        Perform time lapse baseline corrections.
        Fill the R1 container or modifies R0 container.
        Parameters
        ----------
        event : `ctapipe` event-container
        tel_id : id of the telescope
        """
        lst = event.lst.tel[tel_id]

        # If R1 container exists, update it inplace
        if isinstance(event.r1.tel[tel_id].waveform, np.ndarray):
            container = event.r1.tel[tel_id]
        else:
            # Modify R0 container. This is to create pedestal files.
            container = event.r0.tel[tel_id]

        waveform = container.waveform.copy()

        # We have 2 functions: one for data from 2018/10/10 to 2019/11/04 and
        # one for data from 2019/11/05 (from Run 1574) after update firmware.
        # The old readout (before 2019/11/05) is shifted by 1 cell.
        run_id = event.lst.tel[tel_id].svc.configuration_id

        # not yet gain selected
        if event.r1.tel[tel_id].selected_gain_channel is None:
            apply_timelapse_correction(
                waveform=waveform,
                local_clock_counter=lst.evt.local_clock_counter,
                first_capacitors=self.first_cap[tel_id],
                last_readout_time=self.last_readout_time[tel_id],
                expected_pixels_id=lst.svc.pixel_ids,
                run_id=run_id,
            )
        else:
            apply_timelapse_correction_gain_selected(
                waveform=waveform,
                local_clock_counter=lst.evt.local_clock_counter,
                first_capacitors=self.first_cap[tel_id],
                last_readout_time=self.last_readout_time[tel_id],
                expected_pixels_id=lst.svc.pixel_ids,
                selected_gain_channel=event.r1.tel[tel_id].selected_gain_channel,
                run_id=run_id,
            )

        container.waveform = waveform

    def interpolate_spikes(self, event, tel_id):
        """
        Interpolates spike A & B.
        Fill the R1 container.
        Parameters
        ----------
        event : `ctapipe` event-container
        tel_id : id of the telescope
        """
        run_id = event.lst.tel[tel_id].svc.configuration_id

        r1 = event.r1.tel[tel_id]
        if r1.selected_gain_channel is None:
            interpolate_spikes(
                waveform=r1.waveform,
                first_capacitors=self.first_cap[tel_id],
                previous_first_capacitors=self.first_cap_old[tel_id],
                run_id=run_id,
            )
        else:
            interpolate_spikes_gain_selected(
                waveform=r1.waveform,
                first_capacitors=self.first_cap[tel_id],
                previous_first_capacitors=self.first_cap_old[tel_id],
                selected_gain_channel=r1.selected_gain_channel,
                run_id=run_id,
            )
Esempio n. 9
0
class DumpTriggersTool(Tool):
    description = Unicode(__doc__)

    # =============================================
    # configuration parameters:
    # =============================================

    infile = Unicode(help='input simtelarray file').tag(config=True,
                                                        allow_none=False)

    outfile = Unicode('triggers.fits',
                      help='output filename (*.fits, *.h5)').tag(config=True)

    overwrite = Bool(False,
                     help="overwrite existing output file").tag(config=True)

    # =============================================
    # map low-level options to high-level command-line options
    # =============================================

    aliases = Dict({
        'infile': 'DumpTriggersTool.infile',
        'outfile': 'DumpTriggersTool.outfile'
    })

    flags = Dict({
        'overwrite': ({
            'DumpTriggersTool': {
                'overwrite': True
            }
        }, 'Enable overwriting of output file')
    })

    examples = ('ctapipe-dump-triggers --infile gamma.simtel.gz '
                '--outfile trig.fits --overwrite'
                '\n\n'
                'If you want to see more output, use --log_level=DEBUG')

    # =============================================
    # The methods of the Tool (initialize, start, finish):
    # =============================================

    def add_event_to_table(self, event_id):
        """
        add the current pyhessio event to a row in the `self.events` table
        """
        ts, tns = pyhessio.get_central_event_gps_time()
        gpstime = Time(ts * u.s, tns * u.ns, format='gps', scale='utc')

        if self._prev_gpstime is None:
            self._prev_gpstime = gpstime

        if self._current_starttime is None:
            self._current_starttime = gpstime

        relative_time = gpstime - self._current_starttime
        delta_t = gpstime - self._prev_gpstime
        self._prev_gpstime = gpstime

        # build the trigger pattern as a fixed-length array
        # (better for storage in FITS format)
        trigtels = pyhessio.get_telescope_with_data_list()
        self._current_trigpattern[:] = 0  # zero the trigger pattern
        self._current_trigpattern[trigtels] = 1  # set the triggered tels to 1

        # insert the row into the table
        self.events.add_row((event_id, relative_time.sec, delta_t.sec,
                             len(trigtels), self._current_trigpattern))

    def setup(self):
        """ setup function, called before `start()` """

        if self.infile == '':
            raise ValueError("No 'infile' parameter was specified. "
                             "Use --help for info")

        self.events = Table(
            names=['EVENT_ID', 'T_REL', 'DELTA_T', 'N_TRIG', 'TRIGGERED_TELS'],
            dtype=[np.int64, np.float64, np.float64, np.int32, np.uint8])

        self.events['TRIGGERED_TELS'].shape = (0, MAX_TELS)
        self.events['T_REL'].unit = u.s
        self.events['T_REL'].description = 'Time relative to first event'
        self.events['DELTA_T'].unit = u.s
        self.events.meta['INPUT'] = self.infile

        self._current_trigpattern = np.zeros(MAX_TELS)
        self._current_starttime = None
        self._prev_gpstime = None

        pyhessio.file_open(self.infile)

    def start(self):
        """ main event loop """

        for run_id, event_id in pyhessio.move_to_next_event():
            self.add_event_to_table(event_id)

    def finish(self):
        """
        finish up and write out results (called automatically after
        `start()`)
        """
        pyhessio.close_file()

        # write out the final table
        if self.outfile.endswith('fits') or self.outfile.endswith('fits.gz'):
            self.events.write(self.outfile, overwrite=self.overwrite)
        elif self.outfile.endswith('h5'):
            self.events.write(self.outfile,
                              path='/events',
                              overwrite=self.overwrite)
        else:
            self.events.write(self.outfile)

        self.log.info("Table written to '{}'".format(self.outfile))
        self.log.info('\n %s', self.events)
Esempio n. 10
0
class LSTEventSource(EventSource):
    """EventSource for LST r0 data."""

    n_gains = Int(
        2,
        help='Number of gains at r0/r1 level'
    ).tag(config=True)

    baseline = Int(
        400,
        help='r0 waveform baseline (default from EvB v3)'
    ).tag(config=True)

    multi_streams = Bool(
        True,
        help='Read in parallel all streams '
    ).tag(config=True)

    def __init__(self, **kwargs):
        """
        Constructor
        Parameters
        ----------
        n_gains = number of gains expected in input file

        baseline = baseline to be subtracted at r1 level (not used for the moment)

        multi_streams = enable the reading of input files from all streams

        config: traitlets.loader.Config
            Configuration specified by config file or cmdline arguments.
            Used to set traitlet values.
            Set to None if no configuration to pass.\
        kwargs: dict
            Additional parameters to be passed.
            NOTE: The file mask of the data to read can be passed with
            the 'input_url' parameter.
        """

        # EventSource can not handle file wild cards as input_url
        # To overcome this we substitute the input_url with first file matching
        # the specified file mask (copied from  MAGICEventSourceROOT).


        super().__init__(**kwargs)

        if self.multi_streams:
            # test how many streams are there:
            # file name must be [stream name]Run[all the rest]
            # All the files with the same [all the rest] are opened

            if '/' in self.input_url:
                dir, name = self.input_url.rsplit('/', 1)
            else:
                dir = getcwd()
                name = self.input_url

            if 'Run' in name:
                stream, run = name.split('Run', 1)
            else:
                run = name

            ls = listdir(dir)
            self.file_list = []

            for file_name in ls:
                if run in file_name:
                    full_name = dir + '/' + file_name
                    self.file_list.append(full_name)
                    Provenance().add_input_file(full_name, role='dl0.sub.evt')
        else:
            self.file_list = [self.input_url]

        self.multi_file = MultiFiles(self.file_list)

        self.camera_config = self.multi_file.camera_config
        self.log.info(
            "Read {} input files".format(
                self.multi_file.num_inputs()
            )
        )

    def rewind(self):
        self.multi_file.rewind()

    def _generator(self):

        # container for LST data
        self.data = LSTDataContainer()
        self.data.meta['input_url'] = self.input_url
        self.data.meta['max_events'] = self.max_events
        self.data.meta['origin'] = 'LSTCAM'

        # fill LST data from the CameraConfig table
        self.fill_lst_service_container_from_zfile()

        # Instrument information
        for tel_id in self.data.lst.tels_with_data:

            assert (tel_id == 0 or tel_id == 1) # only LST1 (for the moment id = 0)

            # optics info from standard optics.fits.gz file
            optics = OpticsDescription.from_name("LST")

            # camera info from LSTCam-[geometry_version].camgeom.fits.gz file
            geometry_version = 2
            camera = CameraGeometry.from_name("LSTCam", geometry_version)

            tel_descr = TelescopeDescription(
                name='LST', tel_type='LST', optics=optics, camera=camera
            )

            self.n_camera_pixels = tel_descr.camera.n_pixels
            tels = {tel_id: tel_descr}

            # LSTs telescope position taken from MC from the moment
            tel_pos = {tel_id: [50., 50., 16] * u.m}

        subarray = SubarrayDescription("LST1 subarray")
        subarray.tels = tels
        subarray.positions = tel_pos

        self.data.inst.subarray = subarray

        # initialize general monitoring container
        self.initialize_mon_container()

        # loop on events
        for count, event in enumerate(self.multi_file):

            self.data.count = count

            # fill specific LST event data
            self.fill_lst_event_container_from_zfile(event)

            # fill general monitoring data
            self.fill_mon_container_from_zfile(event)

            # fill general R0 data
            self.fill_r0_container_from_zfile(event)

            yield self.data

    @staticmethod
    def is_compatible(file_path):
        from astropy.io import fits
        try:
            # The file contains two tables:
            #  1: CameraConfig
            #  2: Events
            h = fits.open(file_path)[2].header
            ttypes = [
                h[x] for x in h.keys() if 'TTYPE' in x
            ]
        except OSError:
            # not even a fits file
            return False

        except IndexError:
            # A fits file of a different format
            return False

        is_protobuf_zfits_file = (
            (h['XTENSION'] == 'BINTABLE') and
            (h['EXTNAME'] == 'Events') and
            (h['ZTABLE'] is True) and
            (h['ORIGIN'] == 'CTA') and
            (h['PBFHEAD'] == 'R1.CameraEvent')
        )

        is_lst_file = 'lstcam_counters' in ttypes
        return is_protobuf_zfits_file & is_lst_file

    def fill_lst_service_container_from_zfile(self):
        """
        Fill LSTServiceContainer with specific LST service data data
        (from the CameraConfig table of zfit file)

        """

        self.data.lst.tels_with_data = [self.camera_config.telescope_id, ]
        svc_container = self.data.lst.tel[self.camera_config.telescope_id].svc

        svc_container.telescope_id = self.camera_config.telescope_id
        svc_container.cs_serial = self.camera_config.cs_serial
        svc_container.configuration_id = self.camera_config.configuration_id
        svc_container.date = self.camera_config.date
        svc_container.num_pixels = self.camera_config.num_pixels
        svc_container.num_samples = self.camera_config.num_samples
        svc_container.pixel_ids = self.camera_config.expected_pixels_id
        svc_container.data_model_version = self.camera_config.data_model_version

        svc_container.num_modules = self.camera_config.lstcam.num_modules
        svc_container.module_ids = self.camera_config.lstcam.expected_modules_id
        svc_container.idaq_version = self.camera_config.lstcam.idaq_version
        svc_container.cdhs_version = self.camera_config.lstcam.cdhs_version
        svc_container.algorithms = self.camera_config.lstcam.algorithms
        svc_container.pre_proc_algorithms = self.camera_config.lstcam.pre_proc_algorithms

    def fill_lst_event_container_from_zfile(self, event):
        """
        Fill LSTEventContainer with specific LST service data
        (from the Event table of zfit file)

        """

        event_container = self.data.lst.tel[self.camera_config.telescope_id].evt

        event_container.configuration_id = event.configuration_id
        event_container.event_id = event.event_id
        event_container.tel_event_id = event.tel_event_id
        event_container.pixel_status = event.pixel_status
        event_container.ped_id = event.ped_id
        event_container.module_status = event.lstcam.module_status
        event_container.extdevices_presence = event.lstcam.extdevices_presence

        # unpack TIB data
        rec_fmt = '=IHIBB'
        unpacked_tib = struct.unpack(rec_fmt, event.lstcam.tib_data)
        event_container.tib_event_counter = unpacked_tib[0]
        event_container.tib_pps_counter = unpacked_tib[1]
        event_container.tib_tenMHz_counter = unpacked_tib[2]
        event_container.tib_stereo_pattern = unpacked_tib[3]
        event_container.tib_masked_trigger = unpacked_tib[4]
        event_container.swat_data = event.lstcam.swat_data

        # unpack CDTS data
        rec_fmt = '=IIIQQBBB'
        unpacked_cdts =  struct.unpack(rec_fmt, event.lstcam.cdts_data)
        event_container.ucts_event_counter = unpacked_cdts[0]
        event_container.ucts_pps_counter = unpacked_cdts[1]
        event_container.ucts_clock_counter = unpacked_cdts[2]
        event_container.ucts_timestamp = unpacked_cdts[3]
        event_container.ucts_camera_timestamp = unpacked_cdts[4]
        event_container.ucts_trigger_type = unpacked_cdts[5]
        event_container.ucts_white_rabbit_status = unpacked_cdts[6]

        # unpack Dragon counters
        rec_fmt = '=HIIIQ'
        rec_len = struct.calcsize(rec_fmt)
        rec_unpack = struct.Struct(rec_fmt).unpack_from

        event_container.pps_counter = np.zeros(self.camera_config.lstcam.num_modules)
        event_container.tenMHz_counter = np.zeros(self.camera_config.lstcam.num_modules)
        event_container.event_counter = np.zeros(self.camera_config.lstcam.num_modules)
        event_container.trigger_counter = np.zeros(self.camera_config.lstcam.num_modules)
        event_container.local_clock_counter = np.zeros(self.camera_config.lstcam.num_modules)
        for mod in range(self.camera_config.lstcam.num_modules):

            words=event.lstcam.counters[mod*rec_len:(mod+1)*rec_len]
            unpacked_counter = rec_unpack(words)
            event_container.pps_counter[mod] = unpacked_counter[0]
            event_container.tenMHz_counter[mod] = unpacked_counter[1]
            event_container.event_counter[mod] = unpacked_counter[2]
            event_container.trigger_counter[mod] = unpacked_counter[3]
            event_container.local_clock_counter[mod] = unpacked_counter[4]

        event_container.chips_flags = event.lstcam.chips_flags
        event_container.first_capacitor_id = event.lstcam.first_capacitor_id
        event_container.drs_tag_status = event.lstcam.drs_tag_status
        event_container.drs_tag = event.lstcam.drs_tag

    def fill_r0_camera_container_from_zfile(self, r0_container, event):
        """
        Fill with R0CameraContainer

        """


        # temporary patch to have an event time set
        r0_container.trigger_time = (
            self.data.lst.tel[self.camera_config.telescope_id].evt.tib_pps_counter +
            self.data.lst.tel[self.camera_config.telescope_id].evt.tib_tenMHz_counter * 10**(-7))

        if r0_container.trigger_time is None:
            r0_container.trigger_time = 0
        #r0_container.trigger_type = event.trigger_type
        r0_container.trigger_type = self.data.lst.tel[self.camera_config.telescope_id].evt.tib_masked_trigger

        # verify the number of gains
        if event.waveform.shape[0] != self.camera_config.num_pixels * self.camera_config.num_samples * self.n_gains:
            raise ValueError(f"Number of gains not correct, waveform shape is {event.waveform.shape[0]}"
                             f" instead of "
                             f"{self.camera_config.num_pixels * self.camera_config.num_samples * self.n_gains}")

        reshaped_waveform = np.array(
            event.waveform
        ).reshape(
            self.n_gains,
            self.camera_config.num_pixels,
            self.camera_config.num_samples
        )

        # initialize the waveform container to zero
        r0_container.waveform = np.zeros([self.n_gains, self.n_camera_pixels,
                                          self.camera_config.num_samples])

        # re-order the waveform following the expected_pixels_id values
        # (rank = pixel id)
        r0_container.waveform[:, self.camera_config.expected_pixels_id, :] =\
            reshaped_waveform

    def fill_r0_container_from_zfile(self, event):
        """
        Fill with R0Container

        """
        container = self.data.r0

        container.obs_id = -1
        container.event_id = event.event_id

        container.tels_with_data = [self.camera_config.telescope_id, ]
        r0_camera_container = container.tel[self.camera_config.telescope_id]
        self.fill_r0_camera_container_from_zfile(
            r0_camera_container,
            event
        )

    def initialize_mon_container(self):
        """
        Fill with MonitoringContainer.
        For the moment, initialize only the PixelStatusContainer

        """
        container = self.data.mon
        container.tels_with_data = [self.camera_config.telescope_id, ]
        mon_camera_container = container.tel[self.camera_config.telescope_id]

        # initialize the container
        status_container = PixelStatusContainer()
        status_container.hardware_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)
        status_container.pedestal_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)
        status_container.flatfield_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)

        mon_camera_container.pixel_status = status_container

    def fill_mon_container_from_zfile(self, event):
        """
        Fill with MonitoringContainer.
        For the moment, initialize only the PixelStatusContainer

        """

        status_container = self.data.mon.tel[self.camera_config.telescope_id].pixel_status

        # reorder the array
        pixel_status = np.zeros(self.n_camera_pixels)
        pixel_status[self.camera_config.expected_pixels_id] = event.pixel_status
        status_container.hardware_failing_pixels[:] = pixel_status == 0
Esempio n. 11
0
class ChargeResolutionCalculator(Component):
    """
    Class to handle the calculation of Charge Resolution.

    Attributes
    ----------
    max_pe : int
        Maximum pe to calculate the charge resolution up to.
    sum_dict : dict
        Dictionary to store the running sum for each true charge.
    n_dict : dict
        Dictionary to store the running number for each true charge.
    variation_hist_nbins : float
        Number of bins for the variation histogram.
    variation_hist_range : list
        X and Y range for the variation histogram.
    variation_hist : `np.histogram2d`
    variation_xedges : ndarray
        Edges of the X bins for the variation histogram.
    variation_yedges : ndarray
        Edges of the Y bins for the variation histogram.
    """
    max_pe = Int(2000,
                 help='Maximum pe to calculate the charge resolution '
                 'up to').tag(config=True)
    binning = Int(60,
                  allow_none=True,
                  help='Number of bins for the Charge Resolution. If None, '
                  'no binning is performed.').tag(config=True)
    log_bins = Bool(True,
                    help='Bin the x axis linearly instead of '
                    'logarithmic.').tag(config=True)

    def __init__(self, config=None, tool=None, **kwargs):
        """
        Calculator of charge resolution.

        Parameters
        ----------
        config : traitlets.loader.Config
            Configuration specified by config file or cmdline arguments.
            Used to set traitlet values.
            Set to None if no configuration to pass.
        tool : ctapipe.core.Tool
            Tool executable that is calling this component.
            Passes the correct logger to the component.
            Set to None if no Tool to pass.
        kwargs
        """
        super().__init__(config=config, parent=tool, **kwargs)

        self.sum_array = np.zeros(self.max_pe)
        self.n_array = np.zeros(self.max_pe)
        self.sum_dict = {}
        self.n_dict = {}

        self.variation_hist_nbins = log10(self.max_pe) * 50
        self.variation_hist_range = [[log10(1), log10(self.max_pe)],
                                     [log10(1), log10(self.max_pe)]]
        h, xedges, yedges = np.histogram2d([np.nan], [np.nan],
                                           bins=self.variation_hist_nbins,
                                           range=self.variation_hist_range)
        self.variation_hist = h
        self.variation_xedges = xedges
        self.variation_yedges = yedges

        self.storage_arrays = [
            'max_pe', 'sum_array', 'n_array', 'variation_hist_nbins',
            'variation_hist_range', 'variation_hist', 'variation_xedges',
            'variation_yedges'
        ]

    def add_charges(self, true_charge, measured_charge):
        """
        Fill the class parameters with a numpy array of true charge and
        measured (calibrated) charge from an event. The two arrays must be the
        same size.

        Parameters
        ----------
        true_charge : ndarray
            Array of true (MC) charge.
            Obtained from event.mc.tel[telid].image[channel]
        measured_charge : ndarray
            Array of measured (dl1 calibrated) charge.
            Obtained from event.mc.tel[tel_id].photo_electron_image
        """
        above_0 = (measured_charge > 0) & (true_charge > 0)
        x = true_charge[above_0]
        y = measured_charge[above_0]
        h, _, _ = np.histogram2d(np.log10(y),
                                 np.log10(x),
                                 bins=self.variation_hist_nbins,
                                 range=self.variation_hist_range)
        self.variation_hist += h

        in_range = (true_charge > 0) & (true_charge <= self.max_pe)
        true_q = true_charge[in_range]
        measured_q = measured_charge[in_range]
        np.add.at(self.sum_array, true_q - 1, np.power(measured_q - true_q, 2))
        np.add.at(self.n_array, true_q - 1, 1)

    def get_charge_resolution(self):
        """
        Calculate and obtain the charge resolution graph arrays.

        Returns
        -------
        true_charge : ndarray
            The X axis true charges.
        chargeres : ndarray
            The Y axis charge resolution values.
        chargeres_error : ndarray
            The error on the charge resolution.
        scaled_chargeres : ndarray
            The Y axis charge resolution divided by the Goal.
        scaled_chargeres_error : ndarray
            The error on the charge resolution divided by the Goal.
        """
        self.log.debug('[chargeres] Calculating charge resolution')

        n_1 = self.n_array > 0
        n = self.n_array[n_1]
        true = (np.arange(self.max_pe) + 1)[n_1]
        sum_ = self.sum_array[n_1]

        res = np.sqrt((sum_ / n) + true) / true
        res_error = res * (1 / np.sqrt(2 * n))

        scale = self.goal(true)
        scaled_res = res / scale
        scaled_res_error = res_error / scale

        if self.binning is not None:
            x = true
            if self.log_bins:
                x = np.log10(true)

            def binning(array):
                return bs(x, array, 'mean', bins=self.binning)

            def sum_errors(array):
                return np.sqrt(np.sum(np.power(array, 2))) / array.size

            def bin_errors(array):
                return bs(x, array, sum_errors, bins=self.binning)

            true, _, _ = binning(true)
            res, _, _ = binning(res)
            res_error, _, _ = bin_errors(res_error)
            scaled_res, _, _ = binning(scaled_res)
            scaled_res_error, _, _ = bin_errors(scaled_res_error)

        return true, res, res_error, scaled_res, scaled_res_error

    @staticmethod
    def limit_curves(npe, n_nsb, n_add, enf, sigma2):
        """
        Equation for calculating the Goal and Requirement curves, as defined
        in SCI-MC/121113.
        https://portal.cta-observatory.org/recordscentre/Records/SCI/
        SCI-MC/measurment_errors_system_performance_1YQCBC.pdf

        Parameters
        ----------
        npe : ndarray
            Number of photoeletrons (variable).
        n_nsb : float
            Number of NSB photons.
        n_add : float
            Number of photoelectrons from additional noise sources.
        enf : float
            Excess noise factor.
        sigma2 : float
            Percentage ofmultiplicative errors.
        """
        return (np.sqrt((n_nsb + n_add) + np.power(enf, 2) * npe +
                        np.power(sigma2 * npe, 2)) / npe).astype(float)

    @staticmethod
    def requirement(npe):
        """
        CTA requirement curve.

        Parameters
        ----------
        npe : ndarray
            Number of photoeletrons (variable).
        """
        n_nsb = sqrt(4.0 + 3.0)
        n_add = 0
        enf = 1.2
        sigma2 = 0.1
        defined_npe = 1000

        # If npe is not an array, temporarily convert it to one
        npe = np.array([npe])

        lc = ChargeResolutionCalculator.limit_curves
        requirement = lc(npe, n_nsb, n_add, enf, sigma2)
        requirement[npe > defined_npe] = np.nan

        return requirement[0]

    @staticmethod
    def goal(npe):
        """
        CTA goal curve.

        Parameters
        ----------
        npe : ndarray
            Number of photoeletrons (variable).
        """
        n_nsb = 2
        n_add = 0
        enf = 1.1152
        sigma2 = 0.05
        defined_npe = 2000

        # If npe is not an array, temporarily convert it to one
        npe = np.array([npe])

        lc = ChargeResolutionCalculator.limit_curves
        goal = lc(npe, n_nsb, n_add, enf, sigma2)
        goal[npe > defined_npe] = np.nan

        return goal[0]

    @staticmethod
    def poisson(npe):
        """
        Poisson limit curve.

        Parameters
        ----------
        npe : ndarray
            Number of photoeletrons (variable).
        """
        # If npe is not an array, temporarily convert it to one
        npe = np.array([npe])
        poisson = np.sqrt(npe) / npe

        return poisson[0]

    def save(self, path):
        output_dir = dirname(path)
        if not exists(output_dir):
            self.log.info("[output] Creating directory: {}".format(output_dir))
            makedirs(output_dir)
        self.log.info("Saving Charge Resolution file: {}".format(path))

        with open_file(path, mode="w", title="ChargeResolutionFile") as f:
            group = f.create_group("/", 'ChargeResolution', '')
            for arr in self.storage_arrays:
                f.create_array(group, arr, getattr(self, arr), arr)

    def load(self, path):
        self.log.info("Loading Charge Resolution file: {}".format(path))
        with open_file(path, mode="r") as f:
            for arr in self.storage_arrays:
                setattr(self, arr, f.get_node("/ChargeResolution", arr).read())
Esempio n. 12
0
 class SomeComponent(TelescopeComponent):
     path = TelescopeParameter(Path(), default_value=None).tag(config=True)
     val = TelescopeParameter(Float(), default_value=1.0).tag(config=True)
     flag = TelescopeParameter(Bool(), default_value=True).tag(config=True)
class DRS4PedestalAndSpikeHeight(Tool):
    name = 'lstchain_create_drs4_pedestal_file'

    output_path = Path(
        directory_ok=False,
        help=
        'Path for the output hdf5 file of pedestal baseline and spike heights',
    ).tag(config=True)
    skip_samples_front = Integer(
        default_value=10,
        help='Do not include first N samples in pedestal calculation').tag(
            config=True)
    skip_samples_end = Integer(
        default_value=1,
        help='Do not include last N samples in pedestal calculation').tag(
            config=True)

    progress_bar = Bool(help="Show progress bar during processing",
                        default_value=True).tag(config=True)

    full_statistics = Bool(help=(
        "If True, write spike{1,2,3} mean, count, std for each capacitor."
        " Otherwise, only mean spike height for each gain, pixel is written"),
                           default_value=False).tag(config=True)

    overwrite = Bool(help=("If true, overwrite output without asking,"
                           " else fail if output file already exists"),
                     default_value=False).tag(config=True)

    aliases = {
        ('i', 'input'): 'LSTEventSource.input_url',
        ('o', 'output'): 'DRS4PedestalAndSpikeHeight.output_path',
        ('m', 'max-events'): 'LSTEventSource.max_events',
    }

    flags = {
        **flag(
            "overwrite",
            "DRS4PedestalAndSpikeHeight.overwrite",
            "Overwrite output file if it exists",
            "Fail if output file already exists",
        ),
        **flag(
            "progress",
            "DRS4PedestalAndSpikeHeight.progress_bar",
            "Show a progress bar during event processing",
            "Do not show a progress bar during event processing",
        ),
        **flag(
            "full-statistics",
            "DRS4PedestalAndSpikeHeight.full_statistics",
            "Whether to write the full statistics about spikes or not",
        ),
    }

    classes = [LSTEventSource]

    def setup(self):
        self.output_path = self.output_path.expanduser().resolve()
        if self.output_path.exists():
            if self.overwrite:
                self.log.warning("Overwriting %s", self.output_path)
                self.output_path.unlink()
            else:
                raise ToolConfigurationError(
                    f"Output file {self.output_path} exists"
                    ", use the `overwrite` option or choose another `output_path` "
                )
        self.log.debug("output path: %s", self.output_path)
        Provenance().add_output_file(str(self.output_path), role="DL1/Event")

        self.source = LSTEventSource(
            parent=self,
            pointing_information=False,
            trigger_information=False,
        )

        # set some config options, these are necessary for this tool,
        # so we set them here and not via the config system
        self.source.r0_r1_calibrator.r1_sample_start = 0
        self.source.r0_r1_calibrator.r1_sample_end = N_SAMPLES

        self.source.r0_r1_calibrator.offset = 0
        self.source.r0_r1_calibrator.apply_spike_correction = False
        self.source.r0_r1_calibrator.apply_timelapse_correction = True
        self.source.r0_r1_calibrator.apply_drs4_pedestal_correction = False

        n_stats = N_GAINS * N_PIXELS * N_CAPACITORS_PIXEL
        self.baseline_stats = OnlineStats(n_stats)
        self.spike0_stats = OnlineStats(n_stats)
        self.spike1_stats = OnlineStats(n_stats)
        self.spike2_stats = OnlineStats(n_stats)

    def start(self):
        tel_id = self.source.tel_id

        for event in tqdm(self.source, disable=not self.progress_bar):
            fill_stats(
                event.r1.tel[tel_id].waveform,
                self.source.r0_r1_calibrator.first_cap[tel_id],
                self.source.r0_r1_calibrator.first_cap_old[tel_id],
                self.source.r0_r1_calibrator.last_readout_time[tel_id],
                self.baseline_stats,
                self.spike0_stats,
                self.spike1_stats,
                self.spike2_stats,
                skip_samples_front=self.skip_samples_front,
                skip_samples_end=self.skip_samples_end,
            )

    def mean_spike_height(self):
        '''Calculate mean spike height for each gain, pixel'''
        shape = (N_GAINS, N_PIXELS, N_CAPACITORS_PIXEL)
        mean_baseline = self.baseline_stats.mean.reshape(shape)
        spike_heights = np.full((N_GAINS, N_PIXELS, 3),
                                np.nan,
                                dtype=np.float32)

        for i in range(3):
            stats = getattr(self, f'spike{i}_stats')
            counts = stats.counts.reshape(shape)
            spike_height = stats.mean.reshape(shape) - mean_baseline
            spike_height[counts == 0] = 0

            # np.ma does not raise an error if the weights sum to 0
            mean_height = np.ma.average(spike_height, weights=counts, axis=2)
            # convert masked array to dense, replacing invalid values with nan
            spike_heights[:, :, i] = mean_height.filled(np.nan)

        unknown_spike_heights = np.isnan(spike_heights).any(axis=2)
        n_unknown_spike_heights = np.count_nonzero(unknown_spike_heights)

        if n_unknown_spike_heights > 0:
            self.log.warning(
                f'Could not determine spike height for {n_unknown_spike_heights} channels'
            )
            self.log.warning(
                f'Gain, pixel: {np.nonzero(unknown_spike_heights)}')

            # replace any unknown pixels with the mean over the camera
            camera_mean_spike_height = np.nanmean(spike_heights, axis=(0, 1))
            self.log.warning(
                f'Using camera mean of {camera_mean_spike_height} for these pixels'
            )
            spike_heights[unknown_spike_heights] = camera_mean_spike_height

        return spike_heights

    def finish(self):
        tel_id = self.source.tel_id
        self.log.info('Writing output to %s', self.output_path)
        key = f'r1/monitoring/drs4_baseline/tel_{tel_id:03d}'

        shape = (N_GAINS, N_PIXELS, N_CAPACITORS_PIXEL)
        baseline_mean = self.baseline_stats.mean.reshape(shape)
        baseline_std = self.baseline_stats.std.reshape(shape)
        baseline_counts = self.baseline_stats.counts.reshape(shape).astype(
            np.uint16)

        n_negative = np.count_nonzero(baseline_mean < 0)
        if n_negative > 0:
            gain, pixel, capacitor = np.nonzero(baseline_mean < 0)
            self.log.critical(
                f'{n_negative} baseline values are smaller than 0')
            self.log.info("Gain | Pixel | Capacitor | Baseline ")
            for g, p, c in zip(gain, pixel, capacitor):
                self.log.info(
                    f"{g:4d} | {p:4d} | {c:9d} | {baseline_mean[g][p][c]:6.1f}"
                )

        n_small = np.count_nonzero(baseline_mean < 25)
        if n_small > 0:
            gain, pixel, capacitor = np.nonzero(baseline_mean < 25)
            self.log.warning(f'{n_small} baseline values are smaller than 25')
            self.log.info("Gain | Pixel | Capacitor | Baseline ")
            for g, p, c in zip(gain, pixel, capacitor):
                self.log.info(
                    f"{g:4d} | {p:4d} | {c:9d} | {baseline_mean[g][p][c]:6.1f}"
                )

        # Convert baseline mean and spike heights to uint16, handle missing
        # values and values smaller 0, larger maxint
        baseline_mean = convert_to_uint16(baseline_mean)
        baseline_std = convert_to_uint16(baseline_std)
        spike_height = convert_to_uint16(self.mean_spike_height())

        with HDF5TableWriter(self.output_path) as writer:
            Provenance().add_output_file(str(self.output_path))

            drs4_calibration = DRS4CalibrationContainer(
                baseline_mean=baseline_mean,
                baseline_std=baseline_std,
                baseline_counts=baseline_counts,
                spike_height=spike_height,
            )

            writer.write(key, drs4_calibration)
Esempio n. 14
0
class LSTEventSource(EventSource):
    """
    EventSource for LST R0 data.
    """
    multi_streams = Bool(True,
                         help='Read in parallel all streams ').tag(config=True)

    min_flatfield_adc = Float(
        default_value=3000.0,
        help=
        ('Events with that have more than ``min_flatfield_pixel_fraction``'
         ' of the pixels inside [``min_flatfield_adc``, ``max_flatfield_adc``]'
         ' get tagged as EventType.FLATFIELD'),
    ).tag(config=True)

    max_flatfield_adc = Float(
        default_value=12000.0,
        help=
        ('Events with that have more than ``min_flatfield_pixel_fraction``'
         ' of the pixels inside [``min_flatfield_adc``, ``max_flatfield_adc``]'
         ' get tagged as EventType.FLATFIELD'),
    ).tag(config=True)

    min_flatfield_pixel_fraction = Float(
        default_value=0.8,
        help=(
            'Events with that have more than ``min_flatfield_pixel_fraction``'
            ' of the pixels inside [``min_flatfield_pe``, ``max_flatfield_pe``]'
            ' get tagged as EventType.FLATFIELD'),
    ).tag(config=True)

    default_trigger_type = Enum(
        ['ucts', 'tib'],
        default_value='ucts',
        help=
        ('Default source for trigger type information.'
         ' For older data, tib might be the better choice but for data newer'
         ' than 2020-06-25, ucts is the preferred option. The source will still'
         ' fallback to the other device if the chosen default device is not '
         ' available')).tag(config=True)

    use_flatfield_heuristic = Bool(
        default_value=True,
        help=('If true, try to identify flat field events independent of the'
              ' trigger type in the event. See option ``min_flatfield_adc``'),
    ).tag(config=True)

    calibrate_flatfields_and_pedestals = Bool(
        default_value=True,
        help='If True, flat field and pedestal events are also calibrated.'
    ).tag(config=True)

    apply_drs4_corrections = Bool(
        default_value=True,
        help=(
            'Apply DRS4 corrections.'
            ' If True, this will fill R1 waveforms with the corrections applied'
            ' Use the options for the LSTR0Corrections to configure which'
            ' corrections are applied'),
    ).tag(config=True)

    trigger_information = Bool(
        default_value=True, help='Fill trigger information.').tag(config=True)
    pointing_information = Bool(
        default_value=True,
        help=('Fill pointing information.'
              ' Requires specifying `PointingSource.drive_report_path`'),
    ).tag(config=True)

    classes = [PointingSource, EventTimeCalculator, LSTR0Corrections]

    def __init__(self, input_url=None, **kwargs):
        '''
        Create a new LSTEventSource.

        Parameters
        ----------
        input_url: Path
            Path to or url understood by ``ctapipe.core.traits.Path``.
            If ``multi_streams`` is ``True``, the source will try to read all
            streams matching the given ``input_url``
        **kwargs:
            Any of the traitlets. See ``LSTEventSource.class_print_help``
        '''
        super().__init__(input_url=input_url, **kwargs)

        if self.multi_streams:
            # test how many streams are there:
            # file name must be [stream name]Run[all the rest]
            # All the files with the same [all the rest] are opened

            path, name = os.path.split(os.path.abspath(self.input_url))
            if 'Run' in name:
                _, run = name.split('Run', 1)
            else:
                run = name

            ls = listdir(path)
            self.file_list = []

            for file_name in ls:
                if run in file_name:
                    full_name = os.path.join(path, file_name)
                    self.file_list.append(full_name)

        else:
            self.file_list = [self.input_url]

        self.multi_file = MultiFiles(self.file_list)
        self.geometry_version = 4

        self.camera_config = self.multi_file.camera_config
        self.log.info("Read {} input files".format(
            self.multi_file.num_inputs()))
        self.tel_id = self.camera_config.telescope_id
        self._subarray = self.create_subarray(self.geometry_version,
                                              self.tel_id)
        self.r0_r1_calibrator = LSTR0Corrections(subarray=self._subarray,
                                                 parent=self)
        self.time_calculator = EventTimeCalculator(
            subarray=self.subarray,
            run_id=self.camera_config.configuration_id,
            expected_modules_id=self.camera_config.lstcam.expected_modules_id,
            parent=self,
        )
        self.pointing_source = PointingSource(subarray=self.subarray,
                                              parent=self)
        self.lst_service = self.fill_lst_service_container(
            self.tel_id, self.camera_config)

    @property
    def subarray(self):
        return self._subarray

    @property
    def is_simulation(self):
        return False

    @property
    def obs_ids(self):
        # currently no obs id is available from the input files
        return [
            self.camera_config.configuration_id,
        ]

    @property
    def datalevels(self):
        if self.r0_r1_calibrator.calibration_path is not None:
            return (DataLevel.R0, DataLevel.R1)
        return (DataLevel.R0, )

    def rewind(self):
        self.multi_file.rewind()

    @staticmethod
    def create_subarray(geometry_version, tel_id=1):
        """
        Obtain the subarray from the EventSource
        Returns
        -------
        ctapipe.instrument.SubarrayDescription
        """

        # camera info from LSTCam-[geometry_version].camgeom.fits.gz file
        camera_geom = load_camera_geometry(version=geometry_version)

        # get info on the camera readout:
        daq_time_per_sample, pulse_shape_time_step, pulse_shapes = read_pulse_shapes(
        )

        camera_readout = CameraReadout(
            'LSTCam',
            1 / daq_time_per_sample,
            pulse_shapes,
            pulse_shape_time_step,
        )

        camera = CameraDescription('LSTCam', camera_geom, camera_readout)

        lst_tel_descr = TelescopeDescription(name='LST',
                                             tel_type='LST',
                                             optics=OPTICS,
                                             camera=camera)

        tel_descriptions = {tel_id: lst_tel_descr}

        # LSTs telescope position taken from MC from the moment
        tel_positions = {tel_id: [50., 50., 16] * u.m}

        subarray = SubarrayDescription(
            name=f"LST-{tel_id} subarray",
            tel_descriptions=tel_descriptions,
            tel_positions=tel_positions,
        )

        return subarray

    def _generator(self):

        # container for LST data
        array_event = LSTArrayEventContainer()
        array_event.meta['input_url'] = self.input_url
        array_event.meta['max_events'] = self.max_events
        array_event.meta['origin'] = 'LSTCAM'

        # also add service container to the event section
        array_event.lst.tel[self.tel_id].svc = self.lst_service

        # initialize general monitoring container
        self.initialize_mon_container(array_event)

        # loop on events
        for count, zfits_event in enumerate(self.multi_file):
            array_event.count = count
            array_event.index.event_id = zfits_event.event_id
            array_event.index.obs_id = self.obs_ids[0]

            # Skip "empty" events that occur at the end of some runs
            if zfits_event.event_id == 0:
                self.log.warning('Event with event_id=0 found, skipping')
                continue

            self.fill_r0r1_container(array_event, zfits_event)
            self.fill_lst_event_container(array_event, zfits_event)
            if self.trigger_information:
                self.fill_trigger_info(array_event)

            self.fill_mon_container(array_event, zfits_event)

            if self.pointing_information:
                self.fill_pointing_info(array_event)

            # apply low level corrections
            if self.apply_drs4_corrections:
                self.r0_r1_calibrator.apply_drs4_corrections(array_event)

                # flat field tagging is performed on r1 data, so can only
                # be done after the drs4 corrections are applied
                if self.use_flatfield_heuristic:
                    self.tag_flatfield_events(array_event)

            # gain select and calibrate to pe
            if self.r0_r1_calibrator.calibration_path is not None:
                # skip flatfield and pedestal events if asked
                if (self.calibrate_flatfields_and_pedestals
                        or array_event.trigger.event_type
                        not in {EventType.FLATFIELD, EventType.SKY_PEDESTAL}):
                    self.r0_r1_calibrator.calibrate(array_event)

            yield array_event

    @staticmethod
    def is_compatible(file_path):
        from astropy.io import fits
        try:
            # The file contains two tables:
            #  1: CameraConfig
            #  2: Events
            h = fits.open(file_path)[2].header
            ttypes = [h[x] for x in h.keys() if 'TTYPE' in x]
        except OSError:
            # not even a fits file
            return False

        except IndexError:
            # A fits file of a different format
            return False

        is_protobuf_zfits_file = ((h['XTENSION'] == 'BINTABLE')
                                  and (h['EXTNAME'] == 'Events')
                                  and (h['ZTABLE'] is True)
                                  and (h['ORIGIN'] == 'CTA')
                                  and (h['PBFHEAD'] == 'R1.CameraEvent'))

        is_lst_file = 'lstcam_counters' in ttypes
        return is_protobuf_zfits_file & is_lst_file

    @staticmethod
    def fill_lst_service_container(tel_id, camera_config):
        """
        Fill LSTServiceContainer with specific LST service data data
        (from the CameraConfig table of zfit file)

        """
        return LSTServiceContainer(
            telescope_id=tel_id,
            cs_serial=camera_config.cs_serial,
            configuration_id=camera_config.configuration_id,
            date=camera_config.date,
            num_pixels=camera_config.num_pixels,
            num_samples=camera_config.num_samples,
            pixel_ids=camera_config.expected_pixels_id,
            data_model_version=camera_config.data_model_version,
            num_modules=camera_config.lstcam.num_modules,
            module_ids=camera_config.lstcam.expected_modules_id,
            idaq_version=camera_config.lstcam.idaq_version,
            cdhs_version=camera_config.lstcam.cdhs_version,
            algorithms=camera_config.lstcam.algorithms,
            pre_proc_algorithms=camera_config.lstcam.pre_proc_algorithms,
        )

    def fill_lst_event_container(self, array_event, zfits_event):
        """
        Fill LSTEventContainer with specific LST service data
        (from the Event table of zfit file)

        """
        tel_id = self.tel_id

        lst_evt = array_event.lst.tel[tel_id].evt

        lst_evt.configuration_id = zfits_event.configuration_id
        lst_evt.event_id = zfits_event.event_id
        lst_evt.tel_event_id = zfits_event.tel_event_id
        lst_evt.pixel_status = zfits_event.pixel_status
        lst_evt.ped_id = zfits_event.ped_id
        lst_evt.module_status = zfits_event.lstcam.module_status
        lst_evt.extdevices_presence = zfits_event.lstcam.extdevices_presence

        # if TIB data are there
        if lst_evt.extdevices_presence & 1:
            tib = zfits_event.lstcam.tib_data.view(TIB_DTYPE)[0]
            lst_evt.tib_event_counter = tib['event_counter']
            lst_evt.tib_pps_counter = tib['pps_counter']
            lst_evt.tib_tenMHz_counter = tib['tenMHz_counter']
            lst_evt.tib_stereo_pattern = tib['stereo_pattern']
            lst_evt.tib_masked_trigger = tib['masked_trigger']

        # if UCTS data are there
        if lst_evt.extdevices_presence & 2:
            if int(array_event.lst.tel[tel_id].svc.idaq_version) > 37201:
                cdts = zfits_event.lstcam.cdts_data.view(
                    CDTS_AFTER_37201_DTYPE)[0]
                lst_evt.ucts_timestamp = cdts[0]
                lst_evt.ucts_address = cdts[1]  # new
                lst_evt.ucts_event_counter = cdts[2]
                lst_evt.ucts_busy_counter = cdts[3]  # new
                lst_evt.ucts_pps_counter = cdts[4]
                lst_evt.ucts_clock_counter = cdts[5]
                lst_evt.ucts_trigger_type = cdts[6]
                lst_evt.ucts_white_rabbit_status = cdts[7]
                lst_evt.ucts_stereo_pattern = cdts[8]  # new
                lst_evt.ucts_num_in_bunch = cdts[9]  # new
                lst_evt.ucts_cdts_version = cdts[10]  # new

            else:
                # unpack UCTS-CDTS data (old version)
                cdts = zfits_event.lstcam.cdts_data.view(
                    CDTS_BEFORE_37201_DTYPE)[0]
                lst_evt.ucts_event_counter = cdts[0]
                lst_evt.ucts_pps_counter = cdts[1]
                lst_evt.ucts_clock_counter = cdts[2]
                lst_evt.ucts_timestamp = cdts[3]
                lst_evt.ucts_camera_timestamp = cdts[4]
                lst_evt.ucts_trigger_type = cdts[5]
                lst_evt.ucts_white_rabbit_status = cdts[6]

        # if SWAT data are there
        if lst_evt.extdevices_presence & 4:
            # unpack SWAT data
            unpacked_swat = zfits_event.lstcam.swat_data.view(SWAT_DTYPE)[0]
            lst_evt.swat_timestamp = unpacked_swat[0]
            lst_evt.swat_counter1 = unpacked_swat[1]
            lst_evt.swat_counter2 = unpacked_swat[2]
            lst_evt.swat_event_type = unpacked_swat[3]
            lst_evt.swat_camera_flag = unpacked_swat[4]
            lst_evt.swat_camera_event_num = unpacked_swat[5]
            lst_evt.swat_array_flag = unpacked_swat[6]
            lst_evt.swat_array_event_num = unpacked_swat[7]

        # unpack Dragon counters
        counters = zfits_event.lstcam.counters.view(DRAGON_COUNTERS_DTYPE)
        lst_evt.pps_counter = counters['pps_counter']
        lst_evt.tenMHz_counter = counters['tenMHz_counter']
        lst_evt.event_counter = counters['event_counter']
        lst_evt.trigger_counter = counters['trigger_counter']
        lst_evt.local_clock_counter = counters['local_clock_counter']

        lst_evt.chips_flags = zfits_event.lstcam.chips_flags
        lst_evt.first_capacitor_id = zfits_event.lstcam.first_capacitor_id
        lst_evt.drs_tag_status = zfits_event.lstcam.drs_tag_status
        lst_evt.drs_tag = zfits_event.lstcam.drs_tag

        lst_evt.ucts_jump = False

    def fill_trigger_info(self, array_event):
        tel_id = self.tel_id

        trigger = array_event.trigger
        trigger.time = self.time_calculator(tel_id, array_event)
        trigger.tels_with_trigger = [tel_id]
        trigger.tel[tel_id].time = trigger.time

        lst = array_event.lst.tel[tel_id]
        tib_available = lst.evt.extdevices_presence & 1
        ucts_available = lst.evt.extdevices_presence & 2

        # decide which source to use, if both are available,
        # the option decides, if not, fallback to the avilable source
        # if no source available, warn and do not fill trigger info
        if tib_available and ucts_available:
            if self.default_trigger_type == 'ucts':
                trigger_bits = lst.evt.ucts_trigger_type
            else:
                trigger_bits = lst.evt.tib_masked_trigger

        elif tib_available:
            trigger_bits = lst.evt.tib_masked_trigger

        elif ucts_available:
            trigger_bits = lst.evt.ucts_trigger_type

        else:
            self.log.warning('No trigger info available.')
            trigger.event_type = EventType.UNKNOWN
            return

        if (ucts_available and lst.evt.ucts_trigger_type == 42
                and self.default_trigger_type == "ucts"):
            self.log.warning(
                'Event with UCTS trigger_type 42 found.'
                ' Probably means unreliable or shifted UCTS data.'
                ' Consider switching to TIB using `default_trigger_type="tib"`'
            )

        # first bit mono trigger, second stereo.
        # If *only* those two are set, we assume it's a physics event
        # for all other we only check if the flag is present
        if (trigger_bits & TriggerBits.PHYSICS) and not (trigger_bits
                                                         & TriggerBits.OTHER):
            trigger.event_type = EventType.SUBARRAY
        elif trigger_bits & TriggerBits.CALIBRATION:
            trigger.event_type = EventType.FLATFIELD
        elif trigger_bits & TriggerBits.PEDESTAL:
            trigger.event_type = EventType.SKY_PEDESTAL
        elif trigger_bits & TriggerBits.SINGLE_PE:
            trigger.event_type = EventType.SINGLE_PE
        else:
            self.log.warning(
                f'Event {array_event.index.event_id} has unknown event type, trigger: {trigger_bits:08b}'
            )
            trigger.event_type = EventType.UNKNOWN

    def tag_flatfield_events(self, array_event):
        '''
        Use a heuristic based on R1 waveforms to recognize flat field events

        Currently, tagging of flat field events does not work,
        they are reported as physics events, here a heuristic identifies
        those events. Since trigger types might be wrong due to ucts errors,
        we try to identify flat field events in all trigger types.

        DRS4 corrections but not the p.e. calibration must be applied
        '''
        tel_id = self.tel_id
        waveform = array_event.r1.tel[tel_id].waveform

        # needs to work for gain already selected or not
        if waveform.ndim == 3:
            image = waveform[HIGH_GAIN].sum(axis=1)
        else:
            image = waveform.sum(axis=1)

        in_range = (image >= self.min_flatfield_adc) & (image <=
                                                        self.max_flatfield_adc)
        n_in_range = np.count_nonzero(in_range)

        looks_like_ff = n_in_range >= self.min_flatfield_pixel_fraction * image.size
        if looks_like_ff:
            array_event.trigger.event_type = EventType.FLATFIELD
            self.log.debug('Setting event type of event'
                           f' {array_event.index.event_id} to FLATFIELD')
        elif array_event.trigger.event_type == EventType.FLATFIELD:
            self.log.warning(
                'Found FF event that does not fulfill FF criteria: %d',
                array_event.index.event_id,
            )
            array_event.trigger.event_type = EventType.UNKNOWN

    def fill_pointing_info(self, array_event):
        tel_id = self.tel_id
        pointing = self.pointing_source.get_pointing_position_altaz(
            tel_id,
            array_event.trigger.time,
        )
        array_event.pointing.tel[tel_id] = pointing
        array_event.pointing.array_altitude = pointing.altitude
        array_event.pointing.array_azimuth = pointing.azimuth

        ra, dec = self.pointing_source.get_pointing_position_icrs(
            tel_id, array_event.trigger.time)
        array_event.pointing.array_ra = ra
        array_event.pointing.array_dec = dec

    def fill_r0r1_camera_container(self, zfits_event):
        """
        Fill the r0 or r1 container, depending on whether gain
        selection has already happened (r1) or not (r0)

        This will create waveforms of shape (N_GAINS, N_PIXELS, N_SAMPLES),
        or (N_PIXELS, N_SAMPLES) respectively regardless of the n_pixels, n_samples
        in the file.

        Missing or broken pixels are filled using maxval of the waveform dtype.
        """
        n_pixels = self.camera_config.num_pixels
        n_samples = self.camera_config.num_samples
        expected_pixels = self.camera_config.expected_pixels_id

        has_low_gain = (zfits_event.pixel_status
                        & PixelStatus.LOW_GAIN_STORED).astype(bool)
        has_high_gain = (zfits_event.pixel_status
                         & PixelStatus.HIGH_GAIN_STORED).astype(bool)
        not_broken = (has_low_gain | has_high_gain).astype(bool)

        # broken pixels have both false, so gain selected means checking
        # if there are any pixels where exactly one of high or low gain is stored
        gain_selected = np.any(has_low_gain != has_high_gain)

        # fill value for broken pixels
        dtype = zfits_event.waveform.dtype
        fill = np.iinfo(dtype).max
        # we assume that either all pixels are gain selected or none
        # only broken pixels are allowed to be missing completely
        if gain_selected:
            selected_gain = np.where(has_high_gain, 0, 1)
            waveform = np.full((n_pixels, n_samples), fill, dtype=dtype)
            waveform[not_broken] = zfits_event.waveform.reshape(
                (-1, n_samples))

            reordered_waveform = np.full((N_PIXELS, N_SAMPLES),
                                         fill,
                                         dtype=dtype)
            reordered_waveform[expected_pixels] = waveform

            reordered_selected_gain = np.full(N_PIXELS, -1, dtype=np.int8)
            reordered_selected_gain[expected_pixels] = selected_gain

            r0 = R0CameraContainer()
            r1 = R1CameraContainer(
                waveform=reordered_waveform,
                selected_gain_channel=reordered_selected_gain,
            )
        else:
            reshaped_waveform = zfits_event.waveform.reshape(
                N_GAINS, n_pixels, n_samples)
            # re-order the waveform following the expected_pixels_id values
            #  could also just do waveform = reshaped_waveform[np.argsort(expected_ids)]
            reordered_waveform = np.full((N_GAINS, N_PIXELS, N_SAMPLES),
                                         fill,
                                         dtype=dtype)
            reordered_waveform[:, expected_pixels, :] = reshaped_waveform
            r0 = R0CameraContainer(waveform=reordered_waveform)
            r1 = R1CameraContainer()

        return r0, r1

    def fill_r0r1_container(self, array_event, zfits_event):
        """
        Fill with R0Container

        """
        r0, r1 = self.fill_r0r1_camera_container(zfits_event)
        array_event.r0.tel[self.tel_id] = r0
        array_event.r1.tel[self.tel_id] = r1

    def initialize_mon_container(self, array_event):
        """
        Fill with MonitoringContainer.
        For the moment, initialize only the PixelStatusContainer

        """
        container = array_event.mon
        mon_camera_container = container.tel[self.tel_id]

        shape = (N_GAINS, N_PIXELS)
        # all pixels broken by default
        status_container = PixelStatusContainer(
            hardware_failing_pixels=np.ones(shape, dtype=bool),
            pedestal_failing_pixels=np.zeros(shape, dtype=bool),
            flatfield_failing_pixels=np.zeros(shape, dtype=bool),
        )
        mon_camera_container.pixel_status = status_container

    def fill_mon_container(self, array_event, zfits_event):
        """
        Fill with MonitoringContainer.
        For the moment, initialize only the PixelStatusContainer

        """
        status_container = array_event.mon.tel[self.tel_id].pixel_status

        # reorder the array
        expected_pixels_id = self.camera_config.expected_pixels_id

        reordered_pixel_status = np.zeros(N_PIXELS,
                                          dtype=zfits_event.pixel_status.dtype)
        reordered_pixel_status[expected_pixels_id] = zfits_event.pixel_status

        channel_info = get_channel_info(reordered_pixel_status)
        status_container.hardware_failing_pixels[:] = channel_info == 0

    def __exit__(self, exc_type, exc_value, traceback):
        self.close()

    def __len__(self):
        if self.max_events is not None:
            return min(self.max_events, len(self.multi_file))
        return len(self.multi_file)

    def close(self):
        self.multi_file.close()
Esempio n. 15
0
class SimTelEventSource(EventSource):
    skip_calibration_events = Bool(
        True, help="Skip calibration events").tag(config=True)
    back_seekable = Bool(
        False,
        help=("Require the event source to be backwards seekable."
              " This will reduce in slower read speed for gzipped files"
              " and is not possible for zstd compressed files"),
    ).tag(config=True)

    focal_length_choice = CaselessStrEnum(
        ["nominal", "effective"],
        default_value="nominal",
        help=
        ("if both nominal and effective focal lengths are available in the "
         "SimTelArray file, which one to use when constructing the "
         "SubarrayDescription (which will be used in CameraFrame to TelescopeFrame "
         "coordinate transforms. The 'nominal' focal length is the one used during "
         "the simulation, the 'effective' focal length is computed using specialized "
         "ray-tracing from a point light source"),
    ).tag(config=True)

    def __init__(self, config=None, parent=None, gain_selector=None, **kwargs):
        """
        EventSource for simtelarray files using the pyeventio library.

        Parameters
        ----------
        config : traitlets.loader.Config
            Configuration specified by config file or cmdline arguments.
            Used to set traitlet values.
            Set to None if no configuration to pass.
        tool : ctapipe.core.Tool
            Tool executable that is calling this component.
            Passes the correct logger to the component.
            Set to None if no Tool to pass.
        gain_selector : ctapipe.calib.camera.gainselection.GainSelector
            The GainSelector to use. If None, then ThresholdGainSelector will be used.
        kwargs
        """
        super().__init__(config=config, parent=parent, **kwargs)
        self.metadata["is_simulation"] = True
        self._camera_cache = {}

        # traitlets creates an empty set as default,
        # which ctapipe treats as no restriction on the telescopes
        # but eventio treats an emty set as "no telescopes allowed"
        # so we explicitly pass None in that case
        self.file_ = SimTelFile(
            Path(self.input_url).expanduser(),
            allowed_telescopes=set(self.allowed_tels)
            if self.allowed_tels else None,
            skip_calibration=self.skip_calibration_events,
            zcat=not self.back_seekable,
        )
        if self.back_seekable and self.is_stream:
            raise IOError(
                "back seekable was required but not possible for inputfile")

        self._subarray_info = self.prepare_subarray_info(
            self.file_.telescope_descriptions, self.file_.header)
        self.start_pos = self.file_.tell()

        # Waveforms from simtelarray have both gain channels
        # Gain selection is performed by this EventSource to produce R1 waveforms
        if gain_selector is None:
            gain_selector = ThresholdGainSelector(parent=self)
        self.gain_selector = gain_selector

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    def close(self):
        self.file_.close()

    @property
    def is_stream(self):
        return not isinstance(self.file_._filehandle,
                              (BufferedReader, GzipFile))

    def prepare_subarray_info(self, telescope_descriptions, header):
        """
        Constructs a SubarrayDescription object from the
        ``telescope_descriptions`` given by ``SimTelFile``

        Parameters
        ----------
        telescope_descriptions: dict
            telescope descriptions as given by ``SimTelFile.telescope_descriptions``
        header: dict
            header as returned by ``SimTelFile.header``

        Returns
        -------
        SubarrayDescription :
            instrumental information
        """

        tel_descriptions = {}  # tel_id : TelescopeDescription
        tel_positions = {}  # tel_id : TelescopeDescription

        for tel_id, telescope_description in telescope_descriptions.items():
            cam_settings = telescope_description["camera_settings"]
            pixel_settings = telescope_description["pixel_settings"]

            n_pixels = cam_settings["n_pixels"]
            focal_length = u.Quantity(cam_settings["focal_length"], u.m)

            if self.focal_length_choice == "effective":
                try:
                    focal_length = u.Quantity(
                        cam_settings["effective_focal_length"], u.m)
                except KeyError as err:
                    raise RuntimeError(
                        f"the SimTelEventSource option 'focal_length_choice' was set to "
                        f"{self.focal_length_choice}, but the effective focal length "
                        f"was not present in the file. ({err})")

            try:
                telescope = guess_telescope(n_pixels, focal_length)
            except ValueError:
                telescope = UNKNOWN_TELESCOPE

            camera = self._camera_cache.get(telescope.camera_name)
            if camera is None:
                camera = build_camera(cam_settings, pixel_settings, telescope)
                self._camera_cache[telescope.camera_name] = camera

            optics = OpticsDescription(
                name=telescope.name,
                num_mirrors=telescope.n_mirrors,
                equivalent_focal_length=focal_length,
                mirror_area=u.Quantity(cam_settings["mirror_area"], u.m**2),
                num_mirror_tiles=cam_settings["n_mirrors"],
            )

            tel_descriptions[tel_id] = TelescopeDescription(
                name=telescope.name,
                tel_type=telescope.type,
                optics=optics,
                camera=camera,
            )

            tel_idx = np.where(header["tel_id"] == tel_id)[0][0]
            tel_positions[tel_id] = header["tel_pos"][tel_idx] * u.m

        return SubarrayDescription(
            "MonteCarloArray",
            tel_positions=tel_positions,
            tel_descriptions=tel_descriptions,
        )

    @staticmethod
    def is_compatible(file_path):
        return is_eventio(Path(file_path).expanduser())

    @property
    def subarray(self):
        return self._subarray_info

    def _generator(self):
        if self.file_.tell() > self.start_pos:
            self.file_._next_header_pos = 0
            warnings.warn("Backseeking to start of file.")

        try:
            yield from self.__generator()
        except EOFError:
            msg = 'EOFError reading from "{input_url}". Might be truncated'.format(
                input_url=self.input_url)
            self.log.warning(msg)
            warnings.warn(msg)

    def __generator(self):
        data = EventAndMonDataContainer()
        data.meta["origin"] = "hessio"
        data.meta["input_url"] = self.input_url
        data.meta["max_events"] = self.max_events

        for counter, array_event in enumerate(self.file_):
            # next lines are just for debugging
            self.array_event = array_event
            data.event_type = array_event["type"]

            # calibration events do not have an event id
            if data.event_type == "calibration":
                event_id = -1
            else:
                event_id = array_event["event_id"]

            data.inst.subarray = self._subarray_info

            obs_id = self.file_.header["run"]
            tels_with_data = set(array_event["telescope_events"].keys())
            data.count = counter
            data.index.obs_id = obs_id
            data.index.event_id = event_id
            data.r0.obs_id = obs_id  # deprecated
            data.r0.event_id = event_id  # deprecated
            data.r0.tels_with_data = tels_with_data
            data.r1.obs_id = obs_id  # deprecated
            data.r1.event_id = event_id  # deprecated
            data.r1.tels_with_data = tels_with_data
            data.dl0.obs_id = obs_id  # deprecated
            data.dl0.event_id = event_id  # deprecated
            data.dl0.tels_with_data = tels_with_data

            # handle telescope filtering by taking the intersection of
            # tels_with_data and allowed_tels
            if len(self.allowed_tels) > 0:
                selected = tels_with_data & self.allowed_tels
                if len(selected) == 0:
                    continue  # skip event
                data.r0.tels_with_data = selected
                data.r1.tels_with_data = selected
                data.dl0.tels_with_data = selected

            trigger_information = array_event["trigger_information"]

            data.trig.tels_with_trigger = trigger_information[
                "triggered_telescopes"]
            time_s, time_ns = trigger_information["gps_time"]
            data.trig.gps_time = Time(time_s * u.s,
                                      time_ns * u.ns,
                                      format="unix",
                                      scale="utc")

            if data.event_type == "data":
                self.fill_mc_information(data, array_event)

            # this should be done in a nicer way to not re-allocate the
            # data each time (right now it's just deleted and garbage
            # collected)
            data.r0.tel.clear()
            data.r1.tel.clear()
            data.dl0.tel.clear()
            data.dl1.tel.clear()
            data.mc.tel.clear()  # clear the previous telescopes

            telescope_events = array_event["telescope_events"]
            tracking_positions = array_event["tracking_positions"]
            for tel_id, telescope_event in telescope_events.items():
                tel_index = self.file_.header["tel_id"].tolist().index(tel_id)

                adc_samples = telescope_event.get("adc_samples")
                if adc_samples is None:
                    adc_samples = telescope_event["adc_sums"][:, :, np.newaxis]
                _, n_pixels, n_samples = adc_samples.shape

                mc = data.mc.tel[tel_id]
                mc.dc_to_pe = array_event["laser_calibrations"][tel_id][
                    "calib"]
                mc.pedestal = array_event["camera_monitorings"][tel_id][
                    "pedestal"]
                mc.photo_electron_image = (array_event.get(
                    "photoelectrons",
                    {}).get(tel_index,
                            {}).get("photoelectrons",
                                    np.zeros(n_pixels, dtype="float32")))

                tracking_position = tracking_positions[tel_id]
                mc.azimuth_raw = tracking_position["azimuth_raw"]
                mc.altitude_raw = tracking_position["altitude_raw"]
                mc.azimuth_cor = tracking_position.get("azimuth_cor", np.nan)
                mc.altitude_cor = tracking_position.get("altitude_cor", np.nan)
                if np.isnan(mc.azimuth_cor):
                    data.pointing[tel_id].azimuth = u.Quantity(
                        mc.azimuth_raw, u.rad)
                else:
                    data.pointing[tel_id].azimuth = u.Quantity(
                        mc.azimuth_cor, u.rad)
                if np.isnan(mc.altitude_cor):
                    data.pointing[tel_id].altitude = u.Quantity(
                        mc.altitude_raw, u.rad)
                else:
                    data.pointing[tel_id].altitude = u.Quantity(
                        mc.altitude_cor, u.rad)

                r0 = data.r0.tel[tel_id]
                r1 = data.r1.tel[tel_id]
                r0.waveform = adc_samples
                r1.waveform, r1.selected_gain_channel = apply_simtel_r1_calibration(
                    adc_samples, mc.pedestal, mc.dc_to_pe, self.gain_selector)

                pixel_lists = telescope_event["pixel_lists"]
                r0.num_trig_pix = pixel_lists.get(0, {"pixels": 0})["pixels"]
                if r0.num_trig_pix > 0:
                    r0.trig_pix_id = pixel_lists[0]["pixel_list"]

            yield data

    def fill_mc_information(self, data, array_event):
        mc_event = array_event["mc_event"]
        mc_shower = array_event["mc_shower"]

        data.mc.energy = mc_shower["energy"] * u.TeV
        data.mc.alt = Angle(mc_shower["altitude"], u.rad)
        data.mc.az = Angle(mc_shower["azimuth"], u.rad)
        data.mc.core_x = mc_event["xcore"] * u.m
        data.mc.core_y = mc_event["ycore"] * u.m
        first_int = mc_shower["h_first_int"] * u.m
        data.mc.h_first_int = first_int
        data.mc.x_max = mc_shower["xmax"] * u.g / (u.cm**2)
        data.mc.shower_primary_id = mc_shower["primary_id"]

        # mc run header data
        data.mcheader.run_array_direction = Angle(
            self.file_.header["direction"] * u.rad)
        mc_run_head = self.file_.mc_run_headers[-1]
        data.mcheader.corsika_version = mc_run_head["shower_prog_vers"]
        data.mcheader.simtel_version = mc_run_head["detector_prog_vers"]
        data.mcheader.energy_range_min = mc_run_head["E_range"][0] * u.TeV
        data.mcheader.energy_range_max = mc_run_head["E_range"][1] * u.TeV
        data.mcheader.prod_site_B_total = mc_run_head["B_total"] * u.uT
        data.mcheader.prod_site_B_declination = Angle(
            mc_run_head["B_declination"] * u.rad)
        data.mcheader.prod_site_B_inclination = Angle(
            mc_run_head["B_inclination"] * u.rad)
        data.mcheader.prod_site_alt = mc_run_head["obsheight"] * u.m
        data.mcheader.spectral_index = mc_run_head["spectral_index"]
        data.mcheader.shower_prog_start = mc_run_head["shower_prog_start"]
        data.mcheader.shower_prog_id = mc_run_head["shower_prog_id"]
        data.mcheader.detector_prog_start = mc_run_head["detector_prog_start"]
        data.mcheader.detector_prog_id = mc_run_head["detector_prog_id"]
        data.mcheader.num_showers = mc_run_head["n_showers"]
        data.mcheader.shower_reuse = mc_run_head["n_use"]
        data.mcheader.max_alt = mc_run_head["alt_range"][1] * u.rad
        data.mcheader.min_alt = mc_run_head["alt_range"][0] * u.rad
        data.mcheader.max_az = mc_run_head["az_range"][1] * u.rad
        data.mcheader.min_az = mc_run_head["az_range"][0] * u.rad
        data.mcheader.diffuse = mc_run_head["diffuse"]
        data.mcheader.max_viewcone_radius = mc_run_head["viewcone"][1] * u.deg
        data.mcheader.min_viewcone_radius = mc_run_head["viewcone"][0] * u.deg
        data.mcheader.max_scatter_range = mc_run_head["core_range"][1] * u.m
        data.mcheader.min_scatter_range = mc_run_head["core_range"][0] * u.m
        data.mcheader.core_pos_mode = mc_run_head["core_pos_mode"]
        data.mcheader.injection_height = mc_run_head["injection_height"] * u.m
        data.mcheader.atmosphere = mc_run_head["atmosphere"]
        data.mcheader.corsika_iact_options = mc_run_head[
            "corsika_iact_options"]
        data.mcheader.corsika_low_E_model = mc_run_head["corsika_low_E_model"]
        data.mcheader.corsika_high_E_model = mc_run_head[
            "corsika_high_E_model"]
        data.mcheader.corsika_bunchsize = mc_run_head["corsika_bunchsize"]
        data.mcheader.corsika_wlen_min = mc_run_head["corsika_wlen_min"] * u.nm
        data.mcheader.corsika_wlen_max = mc_run_head["corsika_wlen_max"] * u.nm
        data.mcheader.corsika_low_E_detail = mc_run_head[
            "corsika_low_E_detail"]
        data.mcheader.corsika_high_E_detail = mc_run_head[
            "corsika_high_E_detail"]
Esempio n. 16
0
class TimeWaveformFitter(TelescopeComponent):
    """
    Class used to perform event reconstruction by fitting of a model on waveforms.
    """
    sigma_s = FloatTelescopeParameter(
        default_value=1,
        help='Width of the single photo-electron peak distribution.',
        allow_none=False).tag(config=True)
    crosstalk = FloatTelescopeParameter(default_value=0,
                                        help='Average pixel crosstalk.',
                                        allow_none=False).tag(config=True)
    sigma_space = Float(
        4,
        help=
        'Size of the region on which the fit is performed relative to the image extension.',
        allow_none=False).tag(config=True)
    sigma_time = Float(
        3,
        help=
        'Time window on which the fit is performed relative to the image temporal extension.',
        allow_none=False).tag(config=True)
    time_before_shower = FloatTelescopeParameter(
        default_value=10,
        help='Additional time at the start of the fit temporal window.',
        allow_none=False).tag(config=True)
    time_after_shower = FloatTelescopeParameter(
        default_value=20,
        help='Additional time at the end of the fit temporal window.',
        allow_none=False).tag(config=True)
    use_weight = Bool(
        False,
        help=
        'If True, the brightest sample is twice as important as the dimmest pixel in the '
        'likelihood. If false all samples are equivalent.',
        allow_none=False).tag(config=True)
    no_asymmetry = Bool(
        False,
        help='If true, the asymmetry of the spatial model is fixed to 0.',
        allow_none=False).tag(config=True)
    use_interleaved = Path(
        None,
        help=
        'Location of the dl1 file used to estimate the pedestal exploiting interleaved'
        ' events.',
        allow_none=True).tag(config=True)
    n_peaks = Int(
        0,
        help=
        'Maximum brightness (p.e.) for which the full likelihood computation is used. '
        'If the Poisson term for Np.e.>n_peak is more than 1e-6 a Gaussian approximation is used.',
        allow_none=False).tag(config=True)
    bound_charge_factor = FloatTelescopeParameter(
        default_value=4,
        help='Maximum relative change to the fitted charge parameter.',
        allow_none=False).tag(config=True)
    bound_t_cm_value = FloatTelescopeParameter(
        default_value=10,
        help='Maximum change to the t_cm parameter.',
        allow_none=False).tag(config=True)
    bound_centroid_control_parameter = FloatTelescopeParameter(
        default_value=1,
        help='Maximum change of the centroid coordinated in '
        'number of seed length',
        allow_none=False).tag(config=True)
    bound_max_length_factor = FloatTelescopeParameter(
        default_value=2,
        help='Maximum relative increase to the fitted length parameter.',
        allow_none=False).tag(config=True)
    bound_length_asymmetry = FloatTelescopeParameter(
        default_value=9,
        help='Bounds for the fitted rl parameter.',
        allow_none=False).tag(config=True)
    bound_max_v_cm_factor = FloatTelescopeParameter(
        default_value=2,
        help='Maximum relative increase to the fitted v_cm parameter.',
        allow_none=False).tag(config=True)
    default_seed_t_cm = FloatTelescopeParameter(
        default_value=0,
        help='Default starting value of t_cm when the seed extraction failed.',
        allow_none=False).tag(config=True)
    default_seed_v_cm = FloatTelescopeParameter(
        default_value=40,
        help='Default starting value of v_cm when the seed extraction failed.',
        allow_none=False).tag(config=True)
    verbose = Int(
        0,
        help='4 - used for tests: create debug plots\n'
        '3 - create debug plots, wait for input after each event, increase minuit verbose level\n'
        '2 - create debug plots, increase minuit verbose level\n'
        '1 - increase minuit verbose level\n'
        '0 - silent',
        allow_none=False).tag(config=True)

    def __init__(self, subarray, config=None, parent=None, **kwargs):
        super().__init__(subarray=subarray,
                         config=config,
                         parent=parent,
                         **kwargs)
        self.subarray = subarray
        self.template_dict = {}
        self.template_time_of_max_dict = {}
        for tel_id in subarray.tel:
            self.template_dict[
                tel_id] = NormalizedPulseTemplate.load_from_eventsource(
                    subarray.tel[tel_id].camera.readout)
            self.template_time_of_max_dict[tel_id] = self.template_dict[
                tel_id].compute_time_of_max()
        poisson_peaks = np.arange(self.n_peaks + 1, dtype=int)
        poisson_peaks[0] = 1
        self.factorial = np.cumprod(poisson_peaks, dtype='u8')
        # Find the transition charge between full likelihood computation and Gaussian approximation
        # The maximum charge is selected such that each Poisson terms in the full likelihood computation
        # above the n_peaks limit account for less than (1/n_peaks)%
        transition_charges = {}
        for config_crosstalk in self.crosstalk:
            # if n_peaks is set to 0, only the Gaussian approximation is used
            transition_charges[config_crosstalk[2]] = 0.0 if self.n_peaks == 0\
                else self.find_transition_charge(config_crosstalk[2], 1e-2/self.n_peaks)
        self.transition_charges = {}
        for tel_id in subarray.tel:
            self.transition_charges[tel_id] = transition_charges[
                self.crosstalk.tel[tel_id]]

        self.start_parameters = None
        self.names_parameters = None
        self.end_parameters = None
        self.error_parameters = None
        self.bound_parameters = None
        self.fcn = None

    def call_setup(self, event, telescope_id, dl1_container):
        """
        Extract all event dependent quantities used for the fit.

        Parameters
        ----------
        event: ctapipe event container
            Current event container.
        telescope_id: int
            Id of the telescope
        dl1_container: DL1ParametersContainer
            Contains the Hillas parameters used as seed for the fit

        Returns
        -------
        focal_length: astropy.units.Quantity
            Focal length of the telescope
        fit_params: array
            Array containing all the variable needed to compute the likelihood
            during the fir excluding the model parameters
        """

        geometry = self.subarray.tel[telescope_id].camera.geometry
        unit = geometry.pix_x.unit
        pix_x = geometry.pix_x.to_value(unit)
        pix_y = geometry.pix_y.to_value(unit)
        r_max = geometry.guess_radius().to_value(unit)
        pix_radius = np.sqrt(geometry.pix_area[0].to_value(unit**2) /
                             np.pi)  # find linear size of a pixel
        readout = self.subarray.tel[telescope_id].camera.readout
        sampling_rate = readout.sampling_rate.to_value(u.GHz)
        dt = (1.0 / sampling_rate)
        template = self.template_dict[telescope_id]
        image = event.dl1.tel[telescope_id].image
        hillas_signal_pixels = event.dl1.tel[telescope_id].image_mask
        start_x_cm, start_y_cm = init_centroid(dl1_container,
                                               geometry[hillas_signal_pixels],
                                               unit,
                                               image[hillas_signal_pixels],
                                               self.no_asymmetry)

        waveform = event.r1.tel[telescope_id].waveform

        dl1_calib = event.calibration.tel[telescope_id].dl1
        time_shift = dl1_calib.time_shift
        # TODO check if this is correct here or if it is applied to r1 waveform earlier
        if dl1_calib.pedestal_offset is not None:
            waveform = waveform - dl1_calib.pedestal_offset[:, np.newaxis]

        n_pixels, n_samples = waveform.shape
        times = np.arange(0, n_samples) * dt
        selected_gains = event.r1.tel[telescope_id].selected_gain_channel
        is_high_gain = (selected_gains == 0)

        # We assume that the time gradient is given in unit of 'geometry spatial unit'/ns
        v = dl1_container.time_gradient
        psi = dl1_container.psi.to_value(u.rad)
        # We use only positive time gradients and psi is projected in [-pi,pi] from [-pi/2,pi/2]
        if v < 0:
            if psi >= 0:
                psi = psi - np.pi
            else:
                psi = psi + np.pi

        start_length = max(dl1_container.length.to_value(unit), pix_radius)
        # With current likelihood computation, order and type of the parameters are important
        start_parameters = {
            'charge':
            dl1_container.intensity,
            't_cm':
            dl1_container.intercept -
            self.template_time_of_max_dict[telescope_id],
            'x_cm':
            start_x_cm.to_value(unit),
            'y_cm':
            start_y_cm.to_value(unit),
            'length':
            start_length,
            'wl':
            max(dl1_container.wl, 0.01),
            'psi':
            psi,
            'v':
            np.abs(v),
            'rl':
            0.0
        }

        # Temporal parameters extraction fails when cleaning select only 2 pixels, we use defaults values in this case
        if np.isnan(start_parameters['t_cm']):
            start_parameters['t_cm'] = self.default_seed_t_cm.tel[telescope_id]
        if np.isnan(start_parameters['v']):
            start_parameters['v'] = self.default_seed_v_cm.tel[telescope_id]

        t_max = n_samples * dt
        v_min, v_max = 0, max(
            self.bound_max_v_cm_factor.tel[telescope_id] *
            start_parameters['v'], 50)
        rl_min, rl_max = -self.bound_length_asymmetry.tel[
            telescope_id], self.bound_length_asymmetry.tel[telescope_id]
        if self.no_asymmetry:
            rl_min, rl_max = 0.0, 0.0
        bound_centroid = self.bound_centroid_control_parameter.tel[
            telescope_id] * start_length

        bound_parameters = {
            'charge': (dl1_container.intensity /
                       self.bound_charge_factor.tel[telescope_id],
                       dl1_container.intensity *
                       self.bound_charge_factor.tel[telescope_id]),
            't_cm': (-self.bound_t_cm_value.tel[telescope_id],
                     t_max + self.bound_t_cm_value.tel[telescope_id]),
            'x_cm': (start_x_cm.to_value(unit) - bound_centroid,
                     start_x_cm.to_value(unit) + bound_centroid),
            'y_cm': (start_y_cm.to_value(unit) - bound_centroid,
                     start_y_cm.to_value(unit) + bound_centroid),
            'length':
            (pix_radius,
             min(self.bound_max_length_factor.tel[telescope_id] * start_length,
                 r_max)),
            'wl': (0.001, 1.0),
            'psi': (-np.pi * 2.0, np.pi * 2.0),
            'v': (v_min, v_max),
            'rl': (rl_min, rl_max)
        }

        mask_pixel, mask_time = self.clean_data(pix_x, pix_y, pix_radius,
                                                times, start_parameters,
                                                telescope_id)
        spatial_ones = np.ones(np.sum(mask_pixel))

        is_high_gain = is_high_gain[mask_pixel]
        sig_s = spatial_ones * self.sigma_s.tel[telescope_id]
        crosstalks = spatial_ones * self.crosstalk.tel[telescope_id]

        times = (np.arange(0, n_samples) * dt)[mask_time]
        time_shift = time_shift[mask_pixel]

        p_x = pix_x[mask_pixel]
        p_y = pix_y[mask_pixel]
        pix_area = geometry.pix_area[mask_pixel].to_value(unit**2)

        data = waveform
        error = None  # TODO include option to use calibration data

        filter_pixels = np.nonzero(~mask_pixel)
        filter_times = np.nonzero(~mask_time)

        if error is None:
            std = np.std(data[~mask_pixel])
            error = np.full(data.shape[0], std)

        data = np.delete(data, filter_pixels, axis=0)
        data = np.delete(data, filter_times, axis=1)
        error = np.delete(error, filter_pixels, axis=0)

        # Fill the set of non-fitted parameters needed to compute the likelihood. Order and type sensitive.
        fit_params = [
            data, error, is_high_gain, sig_s, crosstalks, times,
            np.float32(time_shift), p_x, p_y,
            np.float64(pix_area), template.dt, template.t0,
            template.amplitude_LG, template.amplitude_HG, self.n_peaks,
            self.transition_charges[telescope_id], self.use_weight,
            self.factorial
        ]

        self.start_parameters = start_parameters
        self.names_parameters = start_parameters.keys()
        self.bound_parameters = bound_parameters

        return unit, fit_params

    def __call__(self, event, telescope_id, dl1_container):
        # setup angle to distance conversion on the camera plane for the current telescope
        focal_length = self.subarray.tel[
            telescope_id].optics.equivalent_focal_length
        angle_dist_eq = [
            (u.rad, u.m, lambda x: np.tan(x) * focal_length.to_value(u.m),
             lambda x: np.arctan(x / focal_length.to_value(u.m))),
            (u.rad**2, u.m**2, lambda x:
             (np.tan(np.sqrt(x)) * focal_length.to_value(u.m))**2, lambda x:
             (np.arctan(np.sqrt(x) / focal_length.to_value(u.m)))**2)
        ]
        with u.set_enabled_equivalencies(angle_dist_eq):
            self.start_parameters = None
            self.names_parameters = None
            unit_cam, fit_params = self.call_setup(event, telescope_id,
                                                   dl1_container)
            self.end_parameters = None
            self.error_parameters = None
            self.fcn = None

            return self.predict(unit_cam, fit_params)

    def clean_data(self, pix_x, pix_y, pix_radius, times, start_parameters,
                   telescope_id):
        """
        Method used to select pixels and time samples used in the fitting procedure.
        The spatial selection takes pixels in an ellipsis obtained from the seed Hillas parameters extended by one pixel
        size and multiplied by a factor sigma_space.
        The temporal selection takes a time window centered on the seed time of center of mass and of duration equal to
        the time of propagation of the signal along the length of the ellipsis times a factor sigma_time.
        An additional fixed duration is also added before and after this time window through the time_before_shower and
        time_after_shower arguments.

        Parameters
        ----------
        pix_x, pix_y: array-like
            Pixels positions
        pix_radius: float
        times: array-like
            Sampling times before timeshift corrections
        start_parameters: dict
            Seed parameters derived from the Hillas parameters
        telescope_id: int

        Returns
        ----------
        mask_pixel, mask_time: array-like
            Mask used to select pixels and times for the fit

        """
        x_cm = start_parameters['x_cm']
        y_cm = start_parameters['y_cm']
        length = start_parameters['length']
        width = start_parameters['wl'] * length
        psi = start_parameters['psi']

        dx = pix_x - x_cm
        dy = pix_y - y_cm

        lon = dx * np.cos(psi) + dy * np.sin(psi)
        lat = dx * np.sin(psi) - dy * np.cos(psi)

        mask_pixel = ((lon / (length + pix_radius))**2 +
                      (lat / (width + pix_radius))**2) < self.sigma_space**2

        v = start_parameters['v']
        t_start = (start_parameters['t_cm'] -
                   (np.abs(v) * length / 2 * self.sigma_time) -
                   self.time_before_shower.tel[telescope_id])
        t_end = (start_parameters['t_cm'] +
                 (np.abs(v) * length / 2 * self.sigma_time) +
                 self.time_after_shower.tel[telescope_id])

        mask_time = (times < t_end) * (times > t_start)

        return mask_pixel, mask_time

    def find_transition_charge(self, crosstalk, poisson_proba_min=1e-2):
        """
        Find the charge below which the full likelihood computation is performed and above which a Gaussian
        approximation is used. For a given pixel crosstalk it finds the maximum charge with a Generalised Poisson term
        below poisson_proba_min for n_peaks photo-electrons. n_peaks here is the configured maximum number of
        photo-electron considered in the full likelihood computation.

        Parameters
        ----------
        crosstalk : float
            Pixels crosstalk
        poisson_proba_min: float

        Returns
        -------
        transition_charge: float32
            Model charge of transition between full and approximated likelihood

        """
        transition_charge = self.n_peaks / (1 + crosstalk)
        step = transition_charge / 100

        def poisson(mu, cross_talk):
            return (mu * pow(mu + self.n_peaks * cross_talk,
                             (self.n_peaks - 1)) /
                    self.factorial[self.n_peaks] *
                    np.exp(-mu - self.n_peaks * cross_talk))

        while poisson(transition_charge, crosstalk) > poisson_proba_min:
            transition_charge -= step
        logger.info(
            f'Transition charge between full and approximated likelihood for camera '
            f'with crosstalk = {crosstalk:.4f} is,  {transition_charge:.4f}, p.e.'
        )
        return np.float32(transition_charge)

    def fit(self, fit_params):
        """
        Performs the fitting procedure.

        Parameters
        ----------
        fit_params: array
            Parameters used to compute the likelihood but not fitted

        """
        def f(*args):
            return -2 * self.log_likelihood(*args, fit_params=fit_params)

        print_level = 2 if self.verbose in [1, 2, 3] else 0
        m = Minuit(f,
                   name=self.names_parameters,
                   *self.start_parameters.values())
        for key, val in self.bound_parameters.items():
            m.limits[key] = val
        m.print_level = print_level
        m.errordef = 0.5
        m.simplex().migrad()
        self.end_parameters = m.values.to_dict()
        self.fcn = m.fval
        self.error_parameters = m.errors.to_dict()

    def predict(self, unit_cam, fit_params):
        """
            Call the fitting procedure and fill the results.

        Parameters
        ----------
        unit_cam: astropy.units.unit
            Unit used for the camera geometry and for spatial variable in the fit
        fit_params: array
            Parameters used to compute the likelihood but not fitted

        Returns
        ----------
        container: DL1LikelihoodParametersContainer
            Filled parameter container

        """
        container = DL1LikelihoodParametersContainer(lhfit_call_status=1)
        try:
            self.fit(fit_params)
            container.lhfit_TS = self.fcn

            container.lhfit_x = (self.end_parameters['x_cm'] * unit_cam).to(
                u.m)
            container.lhfit_x_uncertainty = (self.error_parameters['x_cm'] *
                                             unit_cam).to(u.m)
            container.lhfit_y = (self.end_parameters['y_cm'] * unit_cam).to(
                u.m)
            container.lhfit_y_uncertainty = (self.error_parameters['y_cm'] *
                                             unit_cam).to(u.m)
            container.lhfit_r = np.sqrt(container.lhfit_x**2 +
                                        container.lhfit_y**2)
            container.lhfit_phi = np.arctan2(container.lhfit_y,
                                             container.lhfit_x)
            if self.end_parameters['psi'] > np.pi:
                self.end_parameters['psi'] -= 2 * np.pi
            if self.end_parameters['psi'] < -np.pi:
                self.end_parameters['psi'] += 2 * np.pi
            container.lhfit_psi = self.end_parameters['psi'] * u.rad
            container.lhfit_psi_uncertainty = self.error_parameters[
                'psi'] * u.rad
            length_asy = 1 + self.end_parameters['rl'] if self.end_parameters[
                'rl'] >= 0 else 1 / (1 - self.end_parameters['rl'])
            lhfit_length = ((
                (1.0 + length_asy) * self.end_parameters['length'] / 2.0) *
                            unit_cam).to(u.deg)
            container.lhfit_length = lhfit_length
            lhfit_length_rel_err = self.error_parameters[
                'length'] / self.end_parameters['length']
            # We assume that the relative error is the same in the fitted and saved unit
            container.lhfit_length_uncertainty = lhfit_length_rel_err * container.lhfit_length
            container.lhfit_width = self.end_parameters[
                'wl'] * container.lhfit_length

            container.lhfit_time_gradient = self.end_parameters['v']
            container.lhfit_time_gradient_uncertainty = self.error_parameters[
                'v']
            container.lhfit_ref_time = self.end_parameters['t_cm']
            container.lhfit_ref_time_uncertainty = self.error_parameters[
                't_cm']

            container.lhfit_wl = u.Quantity(self.end_parameters['wl'])
            container.lhfit_wl_uncertainty = u.Quantity(
                self.error_parameters['wl'])
            container.lhfit_intensity = self.end_parameters['charge']
            container.lhfit_intensity_uncertainty = self.error_parameters[
                'charge']
            container.lhfit_log_intensity = np.log10(container.lhfit_intensity)
            container.lhfit_t_68 = container.lhfit_length.value * container.lhfit_time_gradient
            container.lhfit_area = container.lhfit_length * container.lhfit_width
            container.lhfit_length_asymmetry = self.end_parameters['rl']
            container.lhfit_length_asymmetry_uncertainty = self.error_parameters[
                'rl']
        except ZeroDivisionError:
            # TODO Check occurrence rate and solve
            container = DL1LikelihoodParametersContainer(lhfit_call_status=-1)
            logger.error(
                'ZeroDivisionError encounter during the fitting procedure, skipping event.'
            )

        return container

    def __str__(self):
        """
            Define the print format of TimeWaveformFitter objects.

            Returns
            -------
            str: string
                Contains the starting and bound parameters used for the fit,
                and the end results with errors and associated log-likelihood
                in readable format.

        """
        s = 'Event processed\n'
        s += 'Start parameters :\n\t{}\n'.format(self.start_parameters)
        s += 'Bound parameters :\n\t{}\n'.format(self.bound_parameters)
        s += 'End parameters :\n\t{}\n'.format(self.end_parameters)
        s += 'Error parameters :\n\t{}\n'.format(self.error_parameters)
        s += '-2Log-Likelihood :\t{}'.format(self.fcn)

        return s

    @staticmethod
    def log_likelihood(*args, fit_params, **kwargs):
        """Compute the log-likelihood used in the fitting procedure."""
        llh = log_pdf(*args, *fit_params, **kwargs)
        return np.sum(llh)