Example #1
0
class SimTelEventSource(EventSource):
    skip_calibration_events = Bool(True, help="Skip calibration events").tag(
        config=True
    )
    back_seekable = Bool(
        False,
        help=(
            "Require the event source to be backwards seekable."
            " This will reduce in slower read speed for gzipped files"
            " and is not possible for zstd compressed files"
        ),
    ).tag(config=True)

    focal_length_choice = CaselessStrEnum(
        ["nominal", "effective"],
        default_value="nominal",
        help=(
            "if both nominal and effective focal lengths are available in the "
            "SimTelArray file, which one to use when constructing the "
            "SubarrayDescription (which will be used in CameraFrame to TelescopeFrame "
            "coordinate transforms. The 'nominal' focal length is the one used during "
            "the simulation, the 'effective' focal length is computed using specialized "
            "ray-tracing from a point light source"
        ),
    ).tag(config=True)

    def __init__(
        self, input_url, config=None, parent=None, gain_selector=None, **kwargs
    ):
        """
        EventSource for simtelarray files using the pyeventio library.

        Parameters
        ----------
        config : traitlets.loader.Config
            Configuration specified by config file or cmdline arguments.
            Used to set traitlet values.
            Set to None if no configuration to pass.
        tool : ctapipe.core.Tool
            Tool executable that is calling this component.
            Passes the correct logger to the component.
            Set to None if no Tool to pass.
        gain_selector : ctapipe.calib.camera.gainselection.GainSelector
            The GainSelector to use. If None, then ThresholdGainSelector will be used.
        kwargs
        """
        super().__init__(input_url=input_url, config=config, parent=parent, **kwargs)
        self._camera_cache = {}

        self.file_ = SimTelFile(
            self.input_url.expanduser(),
            allowed_telescopes=self.allowed_tels,
            skip_calibration=self.skip_calibration_events,
            zcat=not self.back_seekable,
        )
        if self.back_seekable and self.is_stream:
            raise IOError("back seekable was required but not possible for inputfile")

        self._subarray_info = self.prepare_subarray_info(
            self.file_.telescope_descriptions, self.file_.header
        )
        self._mc_header = self._parse_mc_header()
        self.start_pos = self.file_.tell()

        # Waveforms from simtelarray have both gain channels
        # Gain selection is performed by this EventSource to produce R1 waveforms
        if gain_selector is None:
            gain_selector = ThresholdGainSelector(parent=self)

        self.gain_selector = gain_selector

    @observe("allowed_tels")
    def _observe_allowed_tels(self, change):
        # this can run in __init__ before file_ is created
        if hasattr(self, "file_"):
            allowed_tels = set(self.allowed_tels) if self.allowed_tels else None
            self.file_.allowed_telescopes = allowed_tels

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    def close(self):
        self.file_.close()

    @property
    def is_simulation(self):
        return True

    @property
    def datalevels(self):
        return (DataLevel.R0, DataLevel.R1, DataLevel.DL0)

    @property
    def obs_id(self):
        return self.file_.header["run"]

    @property
    def mc_header(self):
        return self._mc_header

    @property
    def is_stream(self):
        return not isinstance(self.file_._filehandle, (BufferedReader, GzipFile))

    def prepare_subarray_info(self, telescope_descriptions, header):
        """
        Constructs a SubarrayDescription object from the
        ``telescope_descriptions`` given by ``SimTelFile``

        Parameters
        ----------
        telescope_descriptions: dict
            telescope descriptions as given by ``SimTelFile.telescope_descriptions``
        header: dict
            header as returned by ``SimTelFile.header``

        Returns
        -------
        SubarrayDescription :
            instrumental information
        """

        tel_descriptions = {}  # tel_id : TelescopeDescription
        tel_positions = {}  # tel_id : TelescopeDescription

        for tel_id, telescope_description in telescope_descriptions.items():
            cam_settings = telescope_description["camera_settings"]
            pixel_settings = telescope_description["pixel_settings"]

            n_pixels = cam_settings["n_pixels"]
            focal_length = u.Quantity(cam_settings["focal_length"], u.m)

            if self.focal_length_choice == "effective":
                try:
                    focal_length = u.Quantity(
                        cam_settings["effective_focal_length"], u.m
                    )
                except KeyError as err:
                    raise RuntimeError(
                        f"the SimTelEventSource option 'focal_length_choice' was set to "
                        f"{self.focal_length_choice}, but the effective focal length "
                        f"was not present in the file. ({err})"
                    )

            try:
                telescope = guess_telescope(n_pixels, focal_length)
            except ValueError:
                telescope = UNKNOWN_TELESCOPE

            camera = self._camera_cache.get(telescope.camera_name)
            if camera is None:
                camera = build_camera(cam_settings, pixel_settings, telescope)
                self._camera_cache[telescope.camera_name] = camera

            optics = OpticsDescription(
                name=telescope.name,
                num_mirrors=telescope.n_mirrors,
                equivalent_focal_length=focal_length,
                mirror_area=u.Quantity(cam_settings["mirror_area"], u.m ** 2),
                num_mirror_tiles=cam_settings["n_mirrors"],
            )

            tel_descriptions[tel_id] = TelescopeDescription(
                name=telescope.name,
                tel_type=telescope.type,
                optics=optics,
                camera=camera,
            )

            tel_idx = np.where(header["tel_id"] == tel_id)[0][0]
            tel_positions[tel_id] = header["tel_pos"][tel_idx] * u.m

        return SubarrayDescription(
            "MonteCarloArray",
            tel_positions=tel_positions,
            tel_descriptions=tel_descriptions,
        )

    @staticmethod
    def is_compatible(file_path):
        return is_eventio(Path(file_path).expanduser())

    @property
    def subarray(self):
        return self._subarray_info

    def _generator(self):
        if self.file_.tell() > self.start_pos:
            self.file_._next_header_pos = 0
            warnings.warn("Backseeking to start of file.")

        try:
            yield from self._generate_events()
        except EOFError:
            msg = 'EOFError reading from "{input_url}". Might be truncated'.format(
                input_url=self.input_url
            )
            self.log.warning(msg)
            warnings.warn(msg)

    def _generate_events(self):
        data = EventAndMonDataContainer()
        data.meta["origin"] = "hessio"
        data.meta["input_url"] = self.input_url
        data.meta["max_events"] = self.max_events
        data.mcheader = self._mc_header

        self._fill_array_pointing(data)

        for counter, array_event in enumerate(self.file_):

            event_id = array_event.get("event_id", -1)
            obs_id = self.file_.header["run"]
            tels_with_data = set(array_event["telescope_events"].keys())
            data.count = counter
            data.index.obs_id = obs_id
            data.index.event_id = event_id
            data.r0.tels_with_data = tels_with_data
            data.r1.tels_with_data = tels_with_data
            data.dl0.tels_with_data = tels_with_data

            self._fill_trigger_info(data, array_event)

            if data.trigger.event_type == EventType.SUBARRAY:
                self._fill_mc_event_information(data, array_event)

            # this should be done in a nicer way to not re-allocate the
            # data each time (right now it's just deleted and garbage
            # collected)
            data.r0.tel.clear()
            data.r1.tel.clear()
            data.dl0.tel.clear()
            data.dl1.tel.clear()
            data.mc.tel.clear()
            data.pointing.tel.clear()

            telescope_events = array_event["telescope_events"]
            tracking_positions = array_event["tracking_positions"]
            for tel_id, telescope_event in telescope_events.items():
                adc_samples = telescope_event.get("adc_samples")
                if adc_samples is None:
                    adc_samples = telescope_event["adc_sums"][:, :, np.newaxis]
                n_gains, n_pixels, n_samples = adc_samples.shape

                mc = data.mc.tel[tel_id]
                mc.dc_to_pe = array_event["laser_calibrations"][tel_id]["calib"]
                mc.pedestal = array_event["camera_monitorings"][tel_id]["pedestal"]
                mc.true_image = (
                    array_event.get("photoelectrons", {})
                    .get(tel_id - 1, {})
                    .get("photoelectrons", np.zeros(n_pixels, dtype="float32"))
                )

                self._fill_event_pointing(
                    data.pointing.tel[tel_id], mc, tracking_positions[tel_id],
                )

                r0 = data.r0.tel[tel_id]
                r1 = data.r1.tel[tel_id]
                r0.waveform = adc_samples
                r1.waveform, r1.selected_gain_channel = apply_simtel_r1_calibration(
                    adc_samples, mc.pedestal, mc.dc_to_pe, self.gain_selector
                )

                pixel_lists = telescope_event["pixel_lists"]
                r0.num_trig_pix = pixel_lists.get(0, {"pixels": 0})["pixels"]
                if r0.num_trig_pix > 0:
                    r0.trig_pix_id = pixel_lists[0]["pixel_list"]

            yield data

    @staticmethod
    def _fill_event_pointing(pointing, mc, tracking_position):
        mc.azimuth_raw = tracking_position["azimuth_raw"]
        mc.altitude_raw = tracking_position["altitude_raw"]
        mc.azimuth_cor = tracking_position.get("azimuth_cor", np.nan)
        mc.altitude_cor = tracking_position.get("altitude_cor", np.nan)

        # take pointing corrected position if available
        if np.isnan(mc.azimuth_cor):
            pointing.azimuth = u.Quantity(mc.azimuth_raw, u.rad)
        else:
            pointing.azimuth = u.Quantity(mc.azimuth_cor, u.rad)

        # take pointing corrected position if available
        if np.isnan(mc.altitude_cor):
            pointing.altitude = u.Quantity(mc.altitude_raw, u.rad)
        else:
            pointing.altitude = u.Quantity(mc.altitude_cor, u.rad)

    @staticmethod
    def _fill_trigger_info(data, array_event):
        trigger = array_event["trigger_information"]

        if array_event["type"] == "data":
            data.trigger.event_type = EventType.SUBARRAY

        elif array_event["type"] == "calibration":
            # if using eventio >= 1.1.1, we can use the calibration_type
            data.trigger.event_type = SIMTEL_TO_CTA_EVENT_TYPE.get(
                array_event.get("calibration_type", -1), EventType.OTHER_CALIBRATION
            )

        else:
            data.trigger.event_type = EventType.UNKNOWN

        data.trigger.tels_with_trigger = trigger["triggered_telescopes"]
        data.trigger.time = parse_simtel_time(trigger["gps_time"])

        for tel_id, time in zip(
            trigger["triggered_telescopes"], trigger["trigger_times"]
        ):
            # time is relative to central trigger in nano seconds
            data.trigger.tel[tel_id].time = data.trigger.time + u.Quantity(time, u.ns)

    def _fill_array_pointing(self, data):
        if self.file_.header["tracking_mode"] == 0:
            az, alt = self.file_.header["direction"]
            data.pointing.array_altitude = u.Quantity(alt, u.rad)
            data.pointing.array_azimuth = u.Quantity(az, u.rad)
        else:
            ra, dec = self.file_.header["direction"]
            data.pointing.array_ra = u.Quantity(ra, u.rad)
            data.pointing.array_dec = u.Quantity(dec, u.rad)

    def _parse_mc_header(self):
        mc_run_head = self.file_.mc_run_headers[-1]

        return MCHeaderContainer(
            corsika_version=mc_run_head["shower_prog_vers"],
            simtel_version=mc_run_head["detector_prog_vers"],
            energy_range_min=mc_run_head["E_range"][0] * u.TeV,
            energy_range_max=mc_run_head["E_range"][1] * u.TeV,
            prod_site_B_total=mc_run_head["B_total"] * u.uT,
            prod_site_B_declination=Angle(mc_run_head["B_declination"], u.rad,),
            prod_site_B_inclination=Angle(mc_run_head["B_inclination"], u.rad,),
            prod_site_alt=mc_run_head["obsheight"] * u.m,
            spectral_index=mc_run_head["spectral_index"],
            shower_prog_start=mc_run_head["shower_prog_start"],
            shower_prog_id=mc_run_head["shower_prog_id"],
            detector_prog_start=mc_run_head["detector_prog_start"],
            detector_prog_id=mc_run_head["detector_prog_id"],
            num_showers=mc_run_head["n_showers"],
            shower_reuse=mc_run_head["n_use"],
            max_alt=mc_run_head["alt_range"][1] * u.rad,
            min_alt=mc_run_head["alt_range"][0] * u.rad,
            max_az=mc_run_head["az_range"][1] * u.rad,
            min_az=mc_run_head["az_range"][0] * u.rad,
            diffuse=mc_run_head["diffuse"],
            max_viewcone_radius=mc_run_head["viewcone"][1] * u.deg,
            min_viewcone_radius=mc_run_head["viewcone"][0] * u.deg,
            max_scatter_range=mc_run_head["core_range"][1] * u.m,
            min_scatter_range=mc_run_head["core_range"][0] * u.m,
            core_pos_mode=mc_run_head["core_pos_mode"],
            injection_height=mc_run_head["injection_height"] * u.m,
            atmosphere=mc_run_head["atmosphere"],
            corsika_iact_options=mc_run_head["corsika_iact_options"],
            corsika_low_E_model=mc_run_head["corsika_low_E_model"],
            corsika_high_E_model=mc_run_head["corsika_high_E_model"],
            corsika_bunchsize=mc_run_head["corsika_bunchsize"],
            corsika_wlen_min=mc_run_head["corsika_wlen_min"] * u.nm,
            corsika_wlen_max=mc_run_head["corsika_wlen_max"] * u.nm,
            corsika_low_E_detail=mc_run_head["corsika_low_E_detail"],
            corsika_high_E_detail=mc_run_head["corsika_high_E_detail"],
            run_array_direction=Angle(self.file_.header["direction"] * u.rad),
        )

    @staticmethod
    def _fill_mc_event_information(data, array_event):
        mc_event = array_event["mc_event"]
        mc_shower = array_event["mc_shower"]

        data.mc.energy = u.Quantity(mc_shower["energy"], u.TeV)
        data.mc.alt = Angle(mc_shower["altitude"], u.rad)
        data.mc.az = Angle(mc_shower["azimuth"], u.rad)
        data.mc.core_x = u.Quantity(mc_event["xcore"], u.m)
        data.mc.core_y = u.Quantity(mc_event["ycore"], u.m)
        first_int = u.Quantity(mc_shower["h_first_int"], u.m)
        data.mc.h_first_int = first_int
        data.mc.x_max = u.Quantity(mc_shower["xmax"], X_MAX_UNIT)
        data.mc.shower_primary_id = mc_shower["primary_id"]
Example #2
0
class SimTelEventSource(EventSource):
    """ Read events from a SimTelArray data file (in EventIO format)."""

    skip_calibration_events = Bool(True, help="Skip calibration events").tag(
        config=True
    )
    back_seekable = Bool(
        False,
        help=(
            "Require the event source to be backwards seekable."
            " This will reduce in slower read speed for gzipped files"
            " and is not possible for zstd compressed files"
        ),
    ).tag(config=True)

    focal_length_choice = CaselessStrEnum(
        ["nominal", "effective"],
        default_value="nominal",
        help=(
            "if both nominal and effective focal lengths are available in the "
            "SimTelArray file, which one to use when constructing the "
            "SubarrayDescription (which will be used in CameraFrame to TelescopeFrame "
            "coordinate transforms. The 'nominal' focal length is the one used during "
            "the simulation, the 'effective' focal length is computed using specialized "
            "ray-tracing from a point light source"
        ),
    ).tag(config=True)

    gain_selector_type = create_class_enum_trait(
        base_class=GainSelector, default_value="ThresholdGainSelector"
    ).tag(config=True)

    def __init__(self, input_url=None, config=None, parent=None, **kwargs):
        """
        EventSource for simtelarray files using the pyeventio library.

        Parameters
        ----------
        config : traitlets.loader.Config
            Configuration specified by config file or cmdline arguments.
            Used to set traitlet values.
            Set to None if no configuration to pass.
        tool : ctapipe.core.Tool
            Tool executable that is calling this component.
            Passes the correct logger to the component.
            Set to None if no Tool to pass.
        gain_selector : ctapipe.calib.camera.gainselection.GainSelector
            The GainSelector to use. If None, then ThresholdGainSelector will be used.
        kwargs
        """
        super().__init__(input_url=input_url, config=config, parent=parent, **kwargs)

        self._camera_cache = {}

        self.file_ = SimTelFile(
            self.input_url.expanduser(),
            allowed_telescopes=self.allowed_tels,
            skip_calibration=self.skip_calibration_events,
            zcat=not self.back_seekable,
        )
        if self.back_seekable and self.is_stream:
            raise IOError("back seekable was required but not possible for inputfile")

        self._subarray_info = self.prepare_subarray_info(
            self.file_.telescope_descriptions, self.file_.header
        )
        self._simulation_config = self._parse_simulation_header()
        self.start_pos = self.file_.tell()

        self.gain_selector = GainSelector.from_name(
            self.gain_selector_type, parent=self
        )
        self.log.debug(f"Using gain selector {self.gain_selector}")

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    def close(self):
        self.file_.close()

    @property
    def is_simulation(self):
        return True

    @property
    def datalevels(self):
        return (DataLevel.R0, DataLevel.R1)

    @property
    def obs_ids(self):
        # ToDo: This does not support merged simtel files!
        return [self.file_.header["run"]]

    @property
    def simulation_config(self) -> SimulationConfigContainer:
        return self._simulation_config

    @property
    def is_stream(self):
        return not isinstance(self.file_._filehandle, (BufferedReader, GzipFile))

    def prepare_subarray_info(self, telescope_descriptions, header):
        """
        Constructs a SubarrayDescription object from the
        ``telescope_descriptions`` given by ``SimTelFile``

        Parameters
        ----------
        telescope_descriptions: dict
            telescope descriptions as given by ``SimTelFile.telescope_descriptions``
        header: dict
            header as returned by ``SimTelFile.header``

        Returns
        -------
        SubarrayDescription :
            instrumental information
        """

        tel_descriptions = {}  # tel_id : TelescopeDescription
        tel_positions = {}  # tel_id : TelescopeDescription

        for tel_id, telescope_description in telescope_descriptions.items():
            cam_settings = telescope_description["camera_settings"]
            pixel_settings = telescope_description["pixel_settings"]

            n_pixels = cam_settings["n_pixels"]
            focal_length = u.Quantity(cam_settings["focal_length"], u.m)

            if self.focal_length_choice == "effective":
                try:
                    focal_length = u.Quantity(
                        cam_settings["effective_focal_length"], u.m
                    )
                except KeyError as err:
                    raise RuntimeError(
                        f"the SimTelEventSource option 'focal_length_choice' was set to "
                        f"{self.focal_length_choice}, but the effective focal length "
                        f"was not present in the file. ({err})"
                    )

            try:
                telescope = guess_telescope(n_pixels, focal_length)
            except ValueError:
                telescope = UNKNOWN_TELESCOPE

            optics = OpticsDescription(
                name=telescope.name,
                num_mirrors=telescope.n_mirrors,
                equivalent_focal_length=focal_length,
                mirror_area=u.Quantity(cam_settings["mirror_area"], u.m ** 2),
                num_mirror_tiles=cam_settings["n_mirrors"],
            )

            camera = self._camera_cache.get(telescope.camera_name)
            if camera is None:
                camera = build_camera(
                    cam_settings,
                    pixel_settings,
                    telescope,
                    frame=CameraFrame(focal_length=optics.equivalent_focal_length),
                )
                self._camera_cache[telescope.camera_name] = camera

            tel_descriptions[tel_id] = TelescopeDescription(
                name=telescope.name,
                tel_type=telescope.type,
                optics=optics,
                camera=camera,
            )

            tel_idx = np.where(header["tel_id"] == tel_id)[0][0]
            tel_positions[tel_id] = header["tel_pos"][tel_idx] * u.m

        subarray = SubarrayDescription(
            name="MonteCarloArray",
            tel_positions=tel_positions,
            tel_descriptions=tel_descriptions,
        )

        if self.allowed_tels:
            subarray = subarray.select_subarray(self.allowed_tels)
        return subarray

    @staticmethod
    def is_compatible(file_path):
        return is_eventio(Path(file_path).expanduser())

    @property
    def subarray(self):
        return self._subarray_info

    def _generator(self):
        if self.file_.tell() > self.start_pos:
            self.file_._next_header_pos = 0
            warnings.warn("Backseeking to start of file.")

        try:
            yield from self._generate_events()
        except EOFError:
            msg = 'EOFError reading from "{input_url}". Might be truncated'.format(
                input_url=self.input_url
            )
            self.log.warning(msg)
            warnings.warn(msg)

    def _generate_events(self):
        data = ArrayEventContainer()
        data.simulation = SimulatedEventContainer()
        data.meta["origin"] = "hessio"
        data.meta["input_url"] = self.input_url
        data.meta["max_events"] = self.max_events

        self._fill_array_pointing(data)

        for counter, array_event in enumerate(self.file_):

            event_id = array_event.get("event_id", -1)
            obs_id = self.file_.header["run"]
            data.count = counter
            data.index.obs_id = obs_id
            data.index.event_id = event_id

            self._fill_trigger_info(data, array_event)

            if data.trigger.event_type == EventType.SUBARRAY:
                self._fill_simulated_event_information(data, array_event)

            # this should be done in a nicer way to not re-allocate the
            # data each time (right now it's just deleted and garbage
            # collected)
            data.r0.tel.clear()
            data.r1.tel.clear()
            data.dl0.tel.clear()
            data.dl1.tel.clear()
            data.pointing.tel.clear()
            data.simulation.tel.clear()

            telescope_events = array_event["telescope_events"]
            tracking_positions = array_event["tracking_positions"]

            for tel_id, telescope_event in telescope_events.items():
                adc_samples = telescope_event.get("adc_samples")
                if adc_samples is None:
                    adc_samples = telescope_event["adc_sums"][:, :, np.newaxis]

                n_gains, n_pixels, n_samples = adc_samples.shape
                true_image = (
                    array_event.get("photoelectrons", {})
                    .get(tel_id - 1, {})
                    .get("photoelectrons", None)
                )

                data.simulation.tel[tel_id] = SimulatedCameraContainer(
                    true_image=true_image
                )

                data.pointing.tel[tel_id] = self._fill_event_pointing(
                    tracking_positions[tel_id]
                )

                r0 = data.r0.tel[tel_id]
                r1 = data.r1.tel[tel_id]
                r0.waveform = adc_samples

                cam_mon = array_event["camera_monitorings"][tel_id]
                pedestal = cam_mon["pedestal"] / cam_mon["n_ped_slices"]
                dc_to_pe = array_event["laser_calibrations"][tel_id]["calib"]

                # fill dc_to_pe and pedestal_per_sample info into monitoring
                # container
                mon = data.mon.tel[tel_id]
                mon.calibration.dc_to_pe = dc_to_pe
                mon.calibration.pedestal_per_sample = pedestal

                r1.waveform, r1.selected_gain_channel = apply_simtel_r1_calibration(
                    adc_samples, pedestal, dc_to_pe, self.gain_selector
                )

                # get time_shift from laser calibration
                time_calib = array_event["laser_calibrations"][tel_id]["tm_calib"]
                pix_index = np.arange(n_pixels)

                dl1_calib = data.calibration.tel[tel_id].dl1
                dl1_calib.time_shift = time_calib[r1.selected_gain_channel, pix_index]

            yield data

    @staticmethod
    def _fill_event_pointing(tracking_position):
        azimuth_raw = tracking_position["azimuth_raw"]
        altitude_raw = tracking_position["altitude_raw"]
        azimuth_cor = tracking_position.get("azimuth_cor", np.nan)
        altitude_cor = tracking_position.get("altitude_cor", np.nan)

        # take pointing corrected position if available
        if np.isnan(azimuth_cor):
            azimuth = u.Quantity(azimuth_raw, u.rad, copy=False)
        else:
            azimuth = u.Quantity(azimuth_cor, u.rad, copy=False)

        # take pointing corrected position if available
        if np.isnan(altitude_cor):
            altitude = u.Quantity(altitude_raw, u.rad, copy=False)
        else:
            altitude = u.Quantity(altitude_cor, u.rad, copy=False)

        return TelescopePointingContainer(azimuth=azimuth, altitude=altitude)

    def _fill_trigger_info(self, data, array_event):
        trigger = array_event["trigger_information"]

        if array_event["type"] == "data":
            data.trigger.event_type = EventType.SUBARRAY

        elif array_event["type"] == "calibration":
            # if using eventio >= 1.1.1, we can use the calibration_type
            data.trigger.event_type = SIMTEL_TO_CTA_EVENT_TYPE.get(
                array_event.get("calibration_type", -1), EventType.OTHER_CALIBRATION
            )

        else:
            data.trigger.event_type = EventType.UNKNOWN

        data.trigger.tels_with_trigger = trigger["triggered_telescopes"]
        if self.allowed_tels:
            data.trigger.tels_with_trigger = np.intersect1d(
                data.trigger.tels_with_trigger,
                self.subarray.tel_ids,
                assume_unique=True,
            )
        central_time = parse_simtel_time(trigger["gps_time"])
        data.trigger.time = central_time

        for tel_id, time in zip(
            trigger["triggered_telescopes"], trigger["trigger_times"]
        ):
            if self.allowed_tels and tel_id not in self.allowed_tels:
                continue
            # telescope time is relative to central trigger in ns
            time = Time(
                central_time.jd1,
                central_time.jd2 + time / NANOSECONDS_PER_DAY,
                scale=central_time.scale,
                format="jd",
            )

            # triggered pixel info
            n_trigger_pixels = -1
            trigger_pixels = None

            tel_event = array_event["telescope_events"].get(tel_id)
            if tel_event:
                # code 0 = trigger pixels
                pixel_list = tel_event["pixel_lists"].get(0)
                if pixel_list:
                    n_trigger_pixels = pixel_list["pixels"]
                    trigger_pixels = pixel_list["pixel_list"]

            trigger = data.trigger.tel[tel_id] = TelescopeTriggerContainer(
                time=time,
                n_trigger_pixels=n_trigger_pixels,
                trigger_pixels=trigger_pixels,
            )

    def _fill_array_pointing(self, data):
        if self.file_.header["tracking_mode"] == 0:
            az, alt = self.file_.header["direction"]
            data.pointing.array_altitude = u.Quantity(alt, u.rad)
            data.pointing.array_azimuth = u.Quantity(az, u.rad)
        else:
            ra, dec = self.file_.header["direction"]
            data.pointing.array_ra = u.Quantity(ra, u.rad)
            data.pointing.array_dec = u.Quantity(dec, u.rad)

    def _parse_simulation_header(self):
        mc_run_head = self.file_.mc_run_headers[-1]

        return SimulationConfigContainer(
            corsika_version=mc_run_head["shower_prog_vers"],
            simtel_version=mc_run_head["detector_prog_vers"],
            energy_range_min=mc_run_head["E_range"][0] * u.TeV,
            energy_range_max=mc_run_head["E_range"][1] * u.TeV,
            prod_site_B_total=mc_run_head["B_total"] * u.uT,
            prod_site_B_declination=Angle(mc_run_head["B_declination"], u.rad),
            prod_site_B_inclination=Angle(mc_run_head["B_inclination"], u.rad),
            prod_site_alt=mc_run_head["obsheight"] * u.m,
            spectral_index=mc_run_head["spectral_index"],
            shower_prog_start=mc_run_head["shower_prog_start"],
            shower_prog_id=mc_run_head["shower_prog_id"],
            detector_prog_start=mc_run_head["detector_prog_start"],
            detector_prog_id=mc_run_head["detector_prog_id"],
            num_showers=mc_run_head["n_showers"],
            shower_reuse=mc_run_head["n_use"],
            max_alt=mc_run_head["alt_range"][1] * u.rad,
            min_alt=mc_run_head["alt_range"][0] * u.rad,
            max_az=mc_run_head["az_range"][1] * u.rad,
            min_az=mc_run_head["az_range"][0] * u.rad,
            diffuse=mc_run_head["diffuse"],
            max_viewcone_radius=mc_run_head["viewcone"][1] * u.deg,
            min_viewcone_radius=mc_run_head["viewcone"][0] * u.deg,
            max_scatter_range=mc_run_head["core_range"][1] * u.m,
            min_scatter_range=mc_run_head["core_range"][0] * u.m,
            core_pos_mode=mc_run_head["core_pos_mode"],
            injection_height=mc_run_head["injection_height"] * u.m,
            atmosphere=mc_run_head["atmosphere"],
            corsika_iact_options=mc_run_head["corsika_iact_options"],
            corsika_low_E_model=mc_run_head["corsika_low_E_model"],
            corsika_high_E_model=mc_run_head["corsika_high_E_model"],
            corsika_bunchsize=mc_run_head["corsika_bunchsize"],
            corsika_wlen_min=mc_run_head["corsika_wlen_min"] * u.nm,
            corsika_wlen_max=mc_run_head["corsika_wlen_max"] * u.nm,
            corsika_low_E_detail=mc_run_head["corsika_low_E_detail"],
            corsika_high_E_detail=mc_run_head["corsika_high_E_detail"],
        )

    @staticmethod
    def _fill_simulated_event_information(data, array_event):
        mc_event = array_event["mc_event"]
        mc_shower = array_event["mc_shower"]

        data.simulation.shower = SimulatedShowerContainer(
            energy=u.Quantity(mc_shower["energy"], u.TeV),
            alt=Angle(mc_shower["altitude"], u.rad),
            az=Angle(mc_shower["azimuth"], u.rad),
            core_x=u.Quantity(mc_event["xcore"], u.m),
            core_y=u.Quantity(mc_event["ycore"], u.m),
            h_first_int=u.Quantity(mc_shower["h_first_int"], u.m),
            x_max=u.Quantity(mc_shower["xmax"], X_MAX_UNIT),
            shower_primary_id=mc_shower["primary_id"],
        )
Example #3
0
class SimTelEventSource(EventSource):
    skip_calibration_events = Bool(
        True, help='Skip calibration events').tag(config=True)
    back_seekable = Bool(
        False,
        help=('Require the event source to be backwards seekable.'
              ' This will reduce in slower read speed for gzipped files'
              ' and is not possible for zstd compressed files')).tag(
                  config=True)

    def __init__(self, config=None, parent=None, gain_selector=None, **kwargs):
        """
        EventSource for simtelarray files using the pyeventio library.

        Parameters
        ----------
        config : traitlets.loader.Config
            Configuration specified by config file or cmdline arguments.
            Used to set traitlet values.
            Set to None if no configuration to pass.
        tool : ctapipe.core.Tool
            Tool executable that is calling this component.
            Passes the correct logger to the component.
            Set to None if no Tool to pass.
        gain_selector : ctapipe.calib.camera.gainselection.GainSelector
            The GainSelector to use. If None, then ThresholdGainSelector will be used.
        kwargs
        """
        super().__init__(config=config, parent=parent, **kwargs)
        self.metadata['is_simulation'] = True
        self._camera_cache = {}

        # traitlets creates an empty set as default,
        # which ctapipe treats as no restriction on the telescopes
        # but eventio treats an emty set as "no telescopes allowed"
        # so we explicitly pass None in that case
        self.file_ = SimTelFile(
            Path(self.input_url).expanduser(),
            allowed_telescopes=set(self.allowed_tels)
            if self.allowed_tels else None,
            skip_calibration=self.skip_calibration_events,
            zcat=not self.back_seekable,
        )
        if self.back_seekable and self.is_stream:
            raise IOError(
                'back seekable was required but not possible for inputfile')

        self._subarray_info = self.prepare_subarray_info(
            self.file_.telescope_descriptions, self.file_.header)
        self.start_pos = self.file_.tell()

        # Waveforms from simtelarray have both gain channels
        # Gain selection is performed by this EventSource to produce R1 waveforms
        if gain_selector is None:
            gain_selector = ThresholdGainSelector(parent=self)
        self.gain_selector = gain_selector

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    def close(self):
        self.file_.close()

    @property
    def is_stream(self):
        return not isinstance(self.file_._filehandle,
                              (BufferedReader, GzipFile))

    def prepare_subarray_info(self, telescope_descriptions, header):
        """
        Constructs a SubarrayDescription object from the
        ``telescope_descriptions`` given by ``SimTelFile``

        Parameters
        ----------
        telescope_descriptions: dict
            telescope descriptions as given by ``SimTelFile.telescope_descriptions``
        header: dict
            header as returned by ``SimTelFile.header``

        Returns
        -------
        SubarrayDescription :
            instrumental information
        """

        tel_descriptions = {}  # tel_id : TelescopeDescription
        tel_positions = {}  # tel_id : TelescopeDescription

        for tel_id, telescope_description in telescope_descriptions.items():
            cam_settings = telescope_description['camera_settings']

            n_pixels = cam_settings['n_pixels']
            focal_length = u.Quantity(cam_settings['focal_length'], u.m)

            try:
                telescope = guess_telescope(n_pixels, focal_length)
            except ValueError:
                telescope = UNKNOWN_TELESCOPE

            camera = self._camera_cache.get(telescope.camera_name)
            if camera is None:
                camera = build_camera_geometry(cam_settings, telescope)
                self._camera_cache[telescope.camera_name] = camera

            optics = OpticsDescription(
                name=telescope.name,
                num_mirrors=telescope.n_mirrors,
                equivalent_focal_length=focal_length,
                mirror_area=u.Quantity(cam_settings['mirror_area'], u.m**2),
                num_mirror_tiles=cam_settings['n_mirrors'],
            )

            tel_descriptions[tel_id] = TelescopeDescription(
                name=telescope.name,
                tel_type=telescope.type,
                optics=optics,
                camera=camera)

            tel_idx = np.where(header['tel_id'] == tel_id)[0][0]
            tel_positions[tel_id] = header['tel_pos'][tel_idx] * u.m

        return SubarrayDescription(
            "MonteCarloArray",
            tel_positions=tel_positions,
            tel_descriptions=tel_descriptions,
        )

    @staticmethod
    def is_compatible(file_path):
        return is_eventio(Path(file_path).expanduser())

    def _generator(self):
        if self.file_.tell() > self.start_pos:
            self.file_._next_header_pos = 0
            warnings.warn('Backseeking to start of file.')

        try:
            yield from self.__generator()
        except EOFError:
            msg = 'EOFError reading from "{input_url}". Might be truncated'.format(
                input_url=self.input_url)
            self.log.warning(msg)
            warnings.warn(msg)

    def __generator(self):
        data = DataContainer()
        data.meta['origin'] = 'hessio'
        data.meta['input_url'] = self.input_url
        data.meta['max_events'] = self.max_events

        for counter, array_event in enumerate(self.file_):
            # next lines are just for debugging
            self.array_event = array_event
            data.event_type = array_event['type']

            # calibration events do not have an event id
            if data.event_type == 'calibration':
                event_id = -1
            else:
                event_id = array_event['event_id']

            data.inst.subarray = self._subarray_info

            obs_id = self.file_.header['run']
            tels_with_data = set(array_event['telescope_events'].keys())
            data.count = counter
            data.index.obs_id = obs_id
            data.index.event_id = event_id
            data.r0.obs_id = obs_id  # deprecated
            data.r0.event_id = event_id  # deprecated
            data.r0.tels_with_data = tels_with_data
            data.r1.obs_id = obs_id  # deprecated
            data.r1.event_id = event_id  # deprecated
            data.r1.tels_with_data = tels_with_data
            data.dl0.obs_id = obs_id  # deprecated
            data.dl0.event_id = event_id  # deprecated
            data.dl0.tels_with_data = tels_with_data

            # handle telescope filtering by taking the intersection of
            # tels_with_data and allowed_tels
            if len(self.allowed_tels) > 0:
                selected = tels_with_data & self.allowed_tels
                if len(selected) == 0:
                    continue  # skip event
                data.r0.tels_with_data = selected
                data.r1.tels_with_data = selected
                data.dl0.tels_with_data = selected

            trigger_information = array_event['trigger_information']

            data.trig.tels_with_trigger = trigger_information[
                'triggered_telescopes']
            time_s, time_ns = trigger_information['gps_time']
            data.trig.gps_time = Time(time_s * u.s,
                                      time_ns * u.ns,
                                      format='unix',
                                      scale='utc')

            if data.event_type == 'data':
                self.fill_mc_information(data, array_event)

            # this should be done in a nicer way to not re-allocate the
            # data each time (right now it's just deleted and garbage
            # collected)
            data.r0.tel.clear()
            data.r1.tel.clear()
            data.dl0.tel.clear()
            data.dl1.tel.clear()
            data.mc.tel.clear()  # clear the previous telescopes

            telescope_events = array_event['telescope_events']
            tracking_positions = array_event['tracking_positions']
            for tel_id, telescope_event in telescope_events.items():
                tel_index = self.file_.header['tel_id'].tolist().index(tel_id)
                telescope_description = self.file_.telescope_descriptions[
                    tel_id]

                adc_samples = telescope_event.get('adc_samples')
                if adc_samples is None:
                    adc_samples = telescope_event['adc_sums'][:, :, np.newaxis]
                _, n_pixels, n_samples = adc_samples.shape
                pixel_settings = telescope_description['pixel_settings']

                mc = data.mc.tel[tel_id]
                mc.dc_to_pe = array_event['laser_calibrations'][tel_id][
                    'calib']
                mc.pedestal = array_event['camera_monitorings'][tel_id][
                    'pedestal']
                mc.reference_pulse_shape = pixel_settings['ref_shape'].astype(
                    'float64')
                mc.meta['refstep'] = float(pixel_settings['ref_step'])
                mc.time_slice = float(pixel_settings['time_slice'])
                mc.photo_electron_image = (array_event.get(
                    'photoelectrons',
                    {}).get(tel_index,
                            {}).get('photoelectrons',
                                    np.zeros(n_pixels, dtype='float32')))

                tracking_position = tracking_positions[tel_id]
                mc.azimuth_raw = tracking_position['azimuth_raw']
                mc.altitude_raw = tracking_position['altitude_raw']
                mc.azimuth_cor = tracking_position.get('azimuth_cor', np.nan)
                mc.altitude_cor = tracking_position.get('altitude_cor', np.nan)
                if np.isnan(mc.azimuth_cor):
                    data.pointing[tel_id].azimuth = u.Quantity(
                        mc.azimuth_raw, u.rad)
                else:
                    data.pointing[tel_id].azimuth = u.Quantity(
                        mc.azimuth_cor, u.rad)
                if np.isnan(mc.altitude_cor):
                    data.pointing[tel_id].altitude = u.Quantity(
                        mc.altitude_raw, u.rad)
                else:
                    data.pointing[tel_id].altitude = u.Quantity(
                        mc.altitude_cor, u.rad)

                r0 = data.r0.tel[tel_id]
                r1 = data.r1.tel[tel_id]
                r0.waveform = adc_samples
                r1.waveform, r1.selected_gain_channel = apply_simtel_r1_calibration(
                    adc_samples, mc.pedestal, mc.dc_to_pe, self.gain_selector)

                pixel_lists = telescope_event['pixel_lists']
                r0.num_trig_pix = pixel_lists.get(0, {'pixels': 0})['pixels']
                if r0.num_trig_pix > 0:
                    r0.trig_pix_id = pixel_lists[0]['pixel_list']

            yield data

    def fill_mc_information(self, data, array_event):
        mc_event = array_event['mc_event']
        mc_shower = array_event['mc_shower']

        data.mc.energy = mc_shower['energy'] * u.TeV
        data.mc.alt = Angle(mc_shower['altitude'], u.rad)
        data.mc.az = Angle(mc_shower['azimuth'], u.rad)
        data.mc.core_x = mc_event['xcore'] * u.m
        data.mc.core_y = mc_event['ycore'] * u.m
        first_int = mc_shower['h_first_int'] * u.m
        data.mc.h_first_int = first_int
        data.mc.x_max = mc_shower['xmax'] * u.g / (u.cm**2)
        data.mc.shower_primary_id = mc_shower['primary_id']

        # mc run header data
        data.mcheader.run_array_direction = Angle(
            self.file_.header['direction'] * u.rad)
        mc_run_head = self.file_.mc_run_headers[-1]
        data.mcheader.corsika_version = mc_run_head['shower_prog_vers']
        data.mcheader.simtel_version = mc_run_head['detector_prog_vers']
        data.mcheader.energy_range_min = mc_run_head['E_range'][0] * u.TeV
        data.mcheader.energy_range_max = mc_run_head['E_range'][1] * u.TeV
        data.mcheader.prod_site_B_total = mc_run_head['B_total'] * u.uT
        data.mcheader.prod_site_B_declination = Angle(
            mc_run_head['B_declination'] * u.rad)
        data.mcheader.prod_site_B_inclination = Angle(
            mc_run_head['B_inclination'] * u.rad)
        data.mcheader.prod_site_alt = mc_run_head['obsheight'] * u.m
        data.mcheader.spectral_index = mc_run_head['spectral_index']
        data.mcheader.shower_prog_start = mc_run_head['shower_prog_start']
        data.mcheader.shower_prog_id = mc_run_head['shower_prog_id']
        data.mcheader.detector_prog_start = mc_run_head['detector_prog_start']
        data.mcheader.detector_prog_id = mc_run_head['detector_prog_id']
        data.mcheader.num_showers = mc_run_head['n_showers']
        data.mcheader.shower_reuse = mc_run_head['n_use']
        data.mcheader.max_alt = mc_run_head['alt_range'][1] * u.rad
        data.mcheader.min_alt = mc_run_head['alt_range'][0] * u.rad
        data.mcheader.max_az = mc_run_head['az_range'][1] * u.rad
        data.mcheader.min_az = mc_run_head['az_range'][0] * u.rad
        data.mcheader.diffuse = mc_run_head['diffuse']
        data.mcheader.max_viewcone_radius = mc_run_head['viewcone'][1] * u.deg
        data.mcheader.min_viewcone_radius = mc_run_head['viewcone'][0] * u.deg
        data.mcheader.max_scatter_range = mc_run_head['core_range'][1] * u.m
        data.mcheader.min_scatter_range = mc_run_head['core_range'][0] * u.m
        data.mcheader.core_pos_mode = mc_run_head['core_pos_mode']
        data.mcheader.injection_height = mc_run_head['injection_height'] * u.m
        data.mcheader.atmosphere = mc_run_head['atmosphere']
        data.mcheader.corsika_iact_options = mc_run_head[
            'corsika_iact_options']
        data.mcheader.corsika_low_E_model = mc_run_head['corsika_low_E_model']
        data.mcheader.corsika_high_E_model = mc_run_head[
            'corsika_high_E_model']
        data.mcheader.corsika_bunchsize = mc_run_head['corsika_bunchsize']
        data.mcheader.corsika_wlen_min = mc_run_head['corsika_wlen_min'] * u.nm
        data.mcheader.corsika_wlen_max = mc_run_head['corsika_wlen_max'] * u.nm
        data.mcheader.corsika_low_E_detail = mc_run_head[
            'corsika_low_E_detail']
        data.mcheader.corsika_high_E_detail = mc_run_head[
            'corsika_high_E_detail']
Example #4
0
class SimTelEventSource(EventSource):
    skip_calibration_events = Bool(
        True, help="Skip calibration events").tag(config=True)
    back_seekable = Bool(
        False,
        help=("Require the event source to be backwards seekable."
              " This will reduce in slower read speed for gzipped files"
              " and is not possible for zstd compressed files"),
    ).tag(config=True)

    focal_length_choice = CaselessStrEnum(
        ["nominal", "effective"],
        default_value="nominal",
        help=
        ("if both nominal and effective focal lengths are available in the "
         "SimTelArray file, which one to use when constructing the "
         "SubarrayDescription (which will be used in CameraFrame to TelescopeFrame "
         "coordinate transforms. The 'nominal' focal length is the one used during "
         "the simulation, the 'effective' focal length is computed using specialized "
         "ray-tracing from a point light source"),
    ).tag(config=True)

    def __init__(self, config=None, parent=None, gain_selector=None, **kwargs):
        """
        EventSource for simtelarray files using the pyeventio library.

        Parameters
        ----------
        config : traitlets.loader.Config
            Configuration specified by config file or cmdline arguments.
            Used to set traitlet values.
            Set to None if no configuration to pass.
        tool : ctapipe.core.Tool
            Tool executable that is calling this component.
            Passes the correct logger to the component.
            Set to None if no Tool to pass.
        gain_selector : ctapipe.calib.camera.gainselection.GainSelector
            The GainSelector to use. If None, then ThresholdGainSelector will be used.
        kwargs
        """
        super().__init__(config=config, parent=parent, **kwargs)
        self.metadata["is_simulation"] = True
        self._camera_cache = {}

        # traitlets creates an empty set as default,
        # which ctapipe treats as no restriction on the telescopes
        # but eventio treats an emty set as "no telescopes allowed"
        # so we explicitly pass None in that case
        self.file_ = SimTelFile(
            Path(self.input_url).expanduser(),
            allowed_telescopes=set(self.allowed_tels)
            if self.allowed_tels else None,
            skip_calibration=self.skip_calibration_events,
            zcat=not self.back_seekable,
        )
        if self.back_seekable and self.is_stream:
            raise IOError(
                "back seekable was required but not possible for inputfile")

        self._subarray_info = self.prepare_subarray_info(
            self.file_.telescope_descriptions, self.file_.header)
        self.start_pos = self.file_.tell()

        # Waveforms from simtelarray have both gain channels
        # Gain selection is performed by this EventSource to produce R1 waveforms
        if gain_selector is None:
            gain_selector = ThresholdGainSelector(parent=self)
        self.gain_selector = gain_selector

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    def close(self):
        self.file_.close()

    @property
    def is_stream(self):
        return not isinstance(self.file_._filehandle,
                              (BufferedReader, GzipFile))

    def prepare_subarray_info(self, telescope_descriptions, header):
        """
        Constructs a SubarrayDescription object from the
        ``telescope_descriptions`` given by ``SimTelFile``

        Parameters
        ----------
        telescope_descriptions: dict
            telescope descriptions as given by ``SimTelFile.telescope_descriptions``
        header: dict
            header as returned by ``SimTelFile.header``

        Returns
        -------
        SubarrayDescription :
            instrumental information
        """

        tel_descriptions = {}  # tel_id : TelescopeDescription
        tel_positions = {}  # tel_id : TelescopeDescription

        for tel_id, telescope_description in telescope_descriptions.items():
            cam_settings = telescope_description["camera_settings"]
            pixel_settings = telescope_description["pixel_settings"]

            n_pixels = cam_settings["n_pixels"]
            focal_length = u.Quantity(cam_settings["focal_length"], u.m)

            if self.focal_length_choice == "effective":
                try:
                    focal_length = u.Quantity(
                        cam_settings["effective_focal_length"], u.m)
                except KeyError as err:
                    raise RuntimeError(
                        f"the SimTelEventSource option 'focal_length_choice' was set to "
                        f"{self.focal_length_choice}, but the effective focal length "
                        f"was not present in the file. ({err})")

            try:
                telescope = guess_telescope(n_pixels, focal_length)
            except ValueError:
                telescope = UNKNOWN_TELESCOPE

            camera = self._camera_cache.get(telescope.camera_name)
            if camera is None:
                camera = build_camera(cam_settings, pixel_settings, telescope)
                self._camera_cache[telescope.camera_name] = camera

            optics = OpticsDescription(
                name=telescope.name,
                num_mirrors=telescope.n_mirrors,
                equivalent_focal_length=focal_length,
                mirror_area=u.Quantity(cam_settings["mirror_area"], u.m**2),
                num_mirror_tiles=cam_settings["n_mirrors"],
            )

            tel_descriptions[tel_id] = TelescopeDescription(
                name=telescope.name,
                tel_type=telescope.type,
                optics=optics,
                camera=camera,
            )

            tel_idx = np.where(header["tel_id"] == tel_id)[0][0]
            tel_positions[tel_id] = header["tel_pos"][tel_idx] * u.m

        return SubarrayDescription(
            "MonteCarloArray",
            tel_positions=tel_positions,
            tel_descriptions=tel_descriptions,
        )

    @staticmethod
    def is_compatible(file_path):
        return is_eventio(Path(file_path).expanduser())

    @property
    def subarray(self):
        return self._subarray_info

    def _generator(self):
        if self.file_.tell() > self.start_pos:
            self.file_._next_header_pos = 0
            warnings.warn("Backseeking to start of file.")

        try:
            yield from self.__generator()
        except EOFError:
            msg = 'EOFError reading from "{input_url}". Might be truncated'.format(
                input_url=self.input_url)
            self.log.warning(msg)
            warnings.warn(msg)

    def __generator(self):
        data = EventAndMonDataContainer()
        data.meta["origin"] = "hessio"
        data.meta["input_url"] = self.input_url
        data.meta["max_events"] = self.max_events

        for counter, array_event in enumerate(self.file_):
            # next lines are just for debugging
            self.array_event = array_event
            data.event_type = array_event["type"]

            # calibration events do not have an event id
            if data.event_type == "calibration":
                event_id = -1
            else:
                event_id = array_event["event_id"]

            data.inst.subarray = self._subarray_info

            obs_id = self.file_.header["run"]
            tels_with_data = set(array_event["telescope_events"].keys())
            data.count = counter
            data.index.obs_id = obs_id
            data.index.event_id = event_id
            data.r0.obs_id = obs_id  # deprecated
            data.r0.event_id = event_id  # deprecated
            data.r0.tels_with_data = tels_with_data
            data.r1.obs_id = obs_id  # deprecated
            data.r1.event_id = event_id  # deprecated
            data.r1.tels_with_data = tels_with_data
            data.dl0.obs_id = obs_id  # deprecated
            data.dl0.event_id = event_id  # deprecated
            data.dl0.tels_with_data = tels_with_data

            # handle telescope filtering by taking the intersection of
            # tels_with_data and allowed_tels
            if len(self.allowed_tels) > 0:
                selected = tels_with_data & self.allowed_tels
                if len(selected) == 0:
                    continue  # skip event
                data.r0.tels_with_data = selected
                data.r1.tels_with_data = selected
                data.dl0.tels_with_data = selected

            trigger_information = array_event["trigger_information"]

            data.trig.tels_with_trigger = trigger_information[
                "triggered_telescopes"]
            time_s, time_ns = trigger_information["gps_time"]
            data.trig.gps_time = Time(time_s * u.s,
                                      time_ns * u.ns,
                                      format="unix",
                                      scale="utc")

            if data.event_type == "data":
                self.fill_mc_information(data, array_event)

            # this should be done in a nicer way to not re-allocate the
            # data each time (right now it's just deleted and garbage
            # collected)
            data.r0.tel.clear()
            data.r1.tel.clear()
            data.dl0.tel.clear()
            data.dl1.tel.clear()
            data.mc.tel.clear()  # clear the previous telescopes

            telescope_events = array_event["telescope_events"]
            tracking_positions = array_event["tracking_positions"]
            for tel_id, telescope_event in telescope_events.items():
                tel_index = self.file_.header["tel_id"].tolist().index(tel_id)

                adc_samples = telescope_event.get("adc_samples")
                if adc_samples is None:
                    adc_samples = telescope_event["adc_sums"][:, :, np.newaxis]
                _, n_pixels, n_samples = adc_samples.shape

                mc = data.mc.tel[tel_id]
                mc.dc_to_pe = array_event["laser_calibrations"][tel_id][
                    "calib"]
                mc.pedestal = array_event["camera_monitorings"][tel_id][
                    "pedestal"]
                mc.photo_electron_image = (array_event.get(
                    "photoelectrons",
                    {}).get(tel_index,
                            {}).get("photoelectrons",
                                    np.zeros(n_pixels, dtype="float32")))

                tracking_position = tracking_positions[tel_id]
                mc.azimuth_raw = tracking_position["azimuth_raw"]
                mc.altitude_raw = tracking_position["altitude_raw"]
                mc.azimuth_cor = tracking_position.get("azimuth_cor", np.nan)
                mc.altitude_cor = tracking_position.get("altitude_cor", np.nan)
                if np.isnan(mc.azimuth_cor):
                    data.pointing[tel_id].azimuth = u.Quantity(
                        mc.azimuth_raw, u.rad)
                else:
                    data.pointing[tel_id].azimuth = u.Quantity(
                        mc.azimuth_cor, u.rad)
                if np.isnan(mc.altitude_cor):
                    data.pointing[tel_id].altitude = u.Quantity(
                        mc.altitude_raw, u.rad)
                else:
                    data.pointing[tel_id].altitude = u.Quantity(
                        mc.altitude_cor, u.rad)

                r0 = data.r0.tel[tel_id]
                r1 = data.r1.tel[tel_id]
                r0.waveform = adc_samples
                r1.waveform, r1.selected_gain_channel = apply_simtel_r1_calibration(
                    adc_samples, mc.pedestal, mc.dc_to_pe, self.gain_selector)

                pixel_lists = telescope_event["pixel_lists"]
                r0.num_trig_pix = pixel_lists.get(0, {"pixels": 0})["pixels"]
                if r0.num_trig_pix > 0:
                    r0.trig_pix_id = pixel_lists[0]["pixel_list"]

            yield data

    def fill_mc_information(self, data, array_event):
        mc_event = array_event["mc_event"]
        mc_shower = array_event["mc_shower"]

        data.mc.energy = mc_shower["energy"] * u.TeV
        data.mc.alt = Angle(mc_shower["altitude"], u.rad)
        data.mc.az = Angle(mc_shower["azimuth"], u.rad)
        data.mc.core_x = mc_event["xcore"] * u.m
        data.mc.core_y = mc_event["ycore"] * u.m
        first_int = mc_shower["h_first_int"] * u.m
        data.mc.h_first_int = first_int
        data.mc.x_max = mc_shower["xmax"] * u.g / (u.cm**2)
        data.mc.shower_primary_id = mc_shower["primary_id"]

        # mc run header data
        data.mcheader.run_array_direction = Angle(
            self.file_.header["direction"] * u.rad)
        mc_run_head = self.file_.mc_run_headers[-1]
        data.mcheader.corsika_version = mc_run_head["shower_prog_vers"]
        data.mcheader.simtel_version = mc_run_head["detector_prog_vers"]
        data.mcheader.energy_range_min = mc_run_head["E_range"][0] * u.TeV
        data.mcheader.energy_range_max = mc_run_head["E_range"][1] * u.TeV
        data.mcheader.prod_site_B_total = mc_run_head["B_total"] * u.uT
        data.mcheader.prod_site_B_declination = Angle(
            mc_run_head["B_declination"] * u.rad)
        data.mcheader.prod_site_B_inclination = Angle(
            mc_run_head["B_inclination"] * u.rad)
        data.mcheader.prod_site_alt = mc_run_head["obsheight"] * u.m
        data.mcheader.spectral_index = mc_run_head["spectral_index"]
        data.mcheader.shower_prog_start = mc_run_head["shower_prog_start"]
        data.mcheader.shower_prog_id = mc_run_head["shower_prog_id"]
        data.mcheader.detector_prog_start = mc_run_head["detector_prog_start"]
        data.mcheader.detector_prog_id = mc_run_head["detector_prog_id"]
        data.mcheader.num_showers = mc_run_head["n_showers"]
        data.mcheader.shower_reuse = mc_run_head["n_use"]
        data.mcheader.max_alt = mc_run_head["alt_range"][1] * u.rad
        data.mcheader.min_alt = mc_run_head["alt_range"][0] * u.rad
        data.mcheader.max_az = mc_run_head["az_range"][1] * u.rad
        data.mcheader.min_az = mc_run_head["az_range"][0] * u.rad
        data.mcheader.diffuse = mc_run_head["diffuse"]
        data.mcheader.max_viewcone_radius = mc_run_head["viewcone"][1] * u.deg
        data.mcheader.min_viewcone_radius = mc_run_head["viewcone"][0] * u.deg
        data.mcheader.max_scatter_range = mc_run_head["core_range"][1] * u.m
        data.mcheader.min_scatter_range = mc_run_head["core_range"][0] * u.m
        data.mcheader.core_pos_mode = mc_run_head["core_pos_mode"]
        data.mcheader.injection_height = mc_run_head["injection_height"] * u.m
        data.mcheader.atmosphere = mc_run_head["atmosphere"]
        data.mcheader.corsika_iact_options = mc_run_head[
            "corsika_iact_options"]
        data.mcheader.corsika_low_E_model = mc_run_head["corsika_low_E_model"]
        data.mcheader.corsika_high_E_model = mc_run_head[
            "corsika_high_E_model"]
        data.mcheader.corsika_bunchsize = mc_run_head["corsika_bunchsize"]
        data.mcheader.corsika_wlen_min = mc_run_head["corsika_wlen_min"] * u.nm
        data.mcheader.corsika_wlen_max = mc_run_head["corsika_wlen_max"] * u.nm
        data.mcheader.corsika_low_E_detail = mc_run_head[
            "corsika_low_E_detail"]
        data.mcheader.corsika_high_E_detail = mc_run_head[
            "corsika_high_E_detail"]