Example #1
0
def test_writing_nan_defaults():
    from ctapipe.containers import ImageParametersContainer

    params = ImageParametersContainer()

    with tempfile.NamedTemporaryFile(suffix=".hdf5") as f:
        with HDF5TableWriter(f.name, mode="w") as writer:
            writer.write("params", params.values())
Example #2
0
def test_writing_nan_defaults(tmp_path):
    from ctapipe.containers import ImageParametersContainer

    path = tmp_path / "test.h5"

    params = ImageParametersContainer()

    with HDF5TableWriter(path, mode="w") as writer:
        writer.write("params", params.values())
Example #3
0
def test_reconstructors(reconstructors):
    """
    a test of the complete fit procedure on one event including:
    • tailcut cleaning
    • hillas parametrisation
    • HillasPlane creation
    • direction fit
    • position fit

    in the end, proper units in the output are asserted"""

    filename = get_dataset_path(
        "gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz")

    source = EventSource(filename, max_events=10)
    subarray = source.subarray
    calib = CameraCalibrator(source.subarray)
    horizon_frame = AltAz()

    for event in source:
        calib(event)
        sim_shower = event.simulation.shower
        array_pointing = SkyCoord(az=sim_shower.az,
                                  alt=sim_shower.alt,
                                  frame=horizon_frame)

        telescope_pointings = {}

        for tel_id, dl1 in event.dl1.tel.items():

            geom = source.subarray.tel[tel_id].camera.geometry

            telescope_pointings[tel_id] = SkyCoord(
                alt=event.pointing.tel[tel_id].altitude,
                az=event.pointing.tel[tel_id].azimuth,
                frame=horizon_frame,
            )
            mask = tailcuts_clean(geom,
                                  dl1.image,
                                  picture_thresh=10.0,
                                  boundary_thresh=5.0)

            dl1.parameters = ImageParametersContainer()

            try:
                moments = hillas_parameters(geom[mask], dl1.image[mask])
            except HillasParameterizationError:
                dl1.parameters.hillas = HillasParametersContainer()
                continue

            # Make sure we provide only good images for the test
            if np.isnan(moments.width.value) or (moments.width.value == 0):
                dl1.parameters.hillas = HillasParametersContainer()
            else:
                dl1.parameters.hillas = moments

        hillas_dict = {
            tel_id: dl1.parameters.hillas
            for tel_id, dl1 in event.dl1.tel.items()
            if np.isfinite(dl1.parameters.hillas.intensity)
        }
        if len(hillas_dict) < 2:
            continue

        for count, reco_method in enumerate(reconstructors):
            if reco_method is HillasReconstructor:
                reconstructor = HillasReconstructor(subarray)

                reconstructor(event)

                event.dl2.stereo.geometry["HillasReconstructor"].alt.to(u.deg)
                event.dl2.stereo.geometry["HillasReconstructor"].az.to(u.deg)
                event.dl2.stereo.geometry["HillasReconstructor"].core_x.to(u.m)
                assert event.dl2.stereo.geometry[
                    "HillasReconstructor"].is_valid

            else:
                reconstructor = reco_method()

                try:
                    reconstructor_out = reconstructor.predict(
                        hillas_dict,
                        source.subarray,
                        array_pointing,
                        telescope_pointings,
                    )
                except InvalidWidthException:
                    continue

                reconstructor_out.alt.to(u.deg)
                reconstructor_out.az.to(u.deg)
                reconstructor_out.core_x.to(u.m)
                assert reconstructor_out.is_valid
Example #4
0
def test_array_display():
    """ check that we can do basic array display functionality """
    from ctapipe.visualization.mpl_array import ArrayDisplay
    from ctapipe.image import timing_parameters

    from ctapipe.containers import (
        ArrayEventContainer,
        DL1Container,
        DL1CameraContainer,
        ImageParametersContainer,
        CoreParametersContainer,
    )

    # build a test subarray:
    tels = dict()
    tel_pos = dict()
    for ii, pos in enumerate([[0, 0, 0], [100, 0, 0], [-100, 0, 0]] * u.m):
        tels[ii + 1] = TelescopeDescription.from_name("MST", "NectarCam")
        tel_pos[ii + 1] = pos

    sub = SubarrayDescription(name="TestSubarray",
                              tel_positions=tel_pos,
                              tel_descriptions=tels)

    # Create a fake event containing telescope-wise information about
    # the image directions projected on the ground
    event = ArrayEventContainer()
    event.dl1 = DL1Container()
    event.dl1.tel = {1: DL1CameraContainer(), 2: DL1CameraContainer()}
    event.dl1.tel[1].parameters = ImageParametersContainer()
    event.dl1.tel[2].parameters = ImageParametersContainer()
    event.dl1.tel[2].parameters.core = CoreParametersContainer()
    event.dl1.tel[1].parameters.core = CoreParametersContainer()
    event.dl1.tel[1].parameters.core.psi = u.Quantity(2.0, unit=u.deg)
    event.dl1.tel[2].parameters.core.psi = u.Quantity(1.0, unit=u.deg)

    ad = ArrayDisplay(subarray=sub)
    ad.set_vector_rho_phi(1 * u.m, 90 * u.deg)

    # try setting a value
    vals = np.ones(sub.num_tels)
    ad.values = vals

    assert (vals == ad.values).all()

    # test UV field ...

    # ...with colors by telescope type
    ad.set_vector_uv(np.array([1, 2, 3]) * u.m, np.array([1, 2, 3]) * u.m)
    # ...with scalar color
    ad.set_vector_uv(np.array([1, 2, 3]) * u.m, np.array([1, 2, 3]) * u.m, c=3)

    geom = CameraGeometry.from_name("LSTCam")
    rot_angle = 20 * u.deg
    hillas = CameraHillasParametersContainer(x=0 * u.m,
                                             y=0 * u.m,
                                             psi=rot_angle)

    # test using hillas params CameraFrame:
    hillas_dict = {
        1: CameraHillasParametersContainer(length=100.0 * u.m, psi=90 * u.deg),
        2: CameraHillasParametersContainer(length=20000 * u.cm, psi="95deg"),
    }

    grad = 2
    intercept = 1

    timing_rot20 = timing_parameters(
        geom,
        image=np.ones(geom.n_pixels),
        peak_time=intercept + grad * geom.pix_x.value,
        hillas_parameters=hillas,
        cleaning_mask=np.ones(geom.n_pixels, dtype=bool),
    )
    gradient_dict = {1: timing_rot20.slope.value, 2: timing_rot20.slope.value}
    core_dict = {
        tel_id: dl1.parameters.core.psi
        for tel_id, dl1 in event.dl1.tel.items()
    }
    ad.set_vector_hillas(
        hillas_dict=hillas_dict,
        core_dict=core_dict,
        length=500,
        time_gradient=gradient_dict,
        angle_offset=0 * u.deg,
    )
    ad.set_line_hillas(hillas_dict=hillas_dict, core_dict=core_dict, range=300)

    # test using hillas params for divergent pointing in telescopeframe:
    hillas_dict = {
        1:
        HillasParametersContainer(fov_lon=1.0 * u.deg,
                                  fov_lat=1.0 * u.deg,
                                  length=1.0 * u.deg,
                                  psi=90 * u.deg),
        2:
        HillasParametersContainer(fov_lon=1.0 * u.deg,
                                  fov_lat=1.0 * u.deg,
                                  length=1.0 * u.deg,
                                  psi=95 * u.deg),
    }
    ad.set_vector_hillas(
        hillas_dict=hillas_dict,
        core_dict=core_dict,
        length=500,
        time_gradient=gradient_dict,
        angle_offset=0 * u.deg,
    )
    ad.set_line_hillas(hillas_dict=hillas_dict, core_dict=core_dict, range=300)

    # test using hillas params for parallel pointing in telescopeframe:
    hillas_dict = {
        1:
        HillasParametersContainer(fov_lon=1.0 * u.deg,
                                  fov_lat=1.0 * u.deg,
                                  length=1.0 * u.deg,
                                  psi=90 * u.deg),
        2:
        HillasParametersContainer(fov_lon=1.0 * u.deg,
                                  fov_lat=1.0 * u.deg,
                                  length=1.0 * u.deg,
                                  psi=95 * u.deg),
    }
    ad.set_vector_hillas(
        hillas_dict=hillas_dict,
        core_dict=core_dict,
        length=500,
        time_gradient=gradient_dict,
        angle_offset=0 * u.deg,
    )

    # test negative time_gradients
    gradient_dict = {1: -0.03, 2: -0.02}
    ad.set_vector_hillas(
        hillas_dict=hillas_dict,
        core_dict=core_dict,
        length=500,
        time_gradient=gradient_dict,
        angle_offset=0 * u.deg,
    )
    # and very small
    gradient_dict = {1: 0.003, 2: 0.002}
    ad.set_vector_hillas(
        hillas_dict=hillas_dict,
        core_dict=core_dict,
        length=500,
        time_gradient=gradient_dict,
        angle_offset=0 * u.deg,
    )

    # Test background contour
    ad.background_contour(
        x=np.array([0, 1, 2]),
        y=np.array([0, 1, 2]),
        background=np.array([[0, 1, 2], [0, 1, 2], [0, 1, 2]]),
    )

    ad.set_line_hillas(hillas_dict=hillas_dict, core_dict=core_dict, range=300)
    ad.add_labels()
    ad.remove_labels()
Example #5
0
    def _generate_events(self):
        """
        Yield ArrayEventContainer to iterate through events.
        """
        data = ArrayEventContainer()
        # Maybe take some other metadata, but there are still some 'unknown'
        # written out by the stage1 tool
        data.meta["origin"] = self.file_.root._v_attrs["CTA PROCESS TYPE"]
        data.meta["input_url"] = self.input_url
        data.meta["max_events"] = self.max_events

        if DataLevel.DL1_IMAGES in self.datalevels:
            image_iterators = {
                tel.name: self.file_.root.dl1.event.telescope.images[
                    tel.name].iterrows()
                for tel in self.file_.root.dl1.event.telescope.images
            }
            if self.has_simulated_dl1:
                simulated_image_iterators = {
                    tel.name:
                    self.file_.root.simulation.event.telescope.images[
                        tel.name].iterrows()
                    for tel in
                    self.file_.root.simulation.event.telescope.images
                }

        if DataLevel.DL1_PARAMETERS in self.datalevels:
            param_readers = {
                tel.name: HDF5TableReader(self.file_).read(
                    f"/dl1/event/telescope/parameters/{tel.name}",
                    containers=[
                        HillasParametersContainer(),
                        TimingParametersContainer(),
                        LeakageContainer(),
                        ConcentrationContainer(),
                        MorphologyContainer(),
                        IntensityStatisticsContainer(),
                        PeakTimeStatisticsContainer(),
                    ],
                    prefixes=True,
                )
                for tel in self.file_.root.dl1.event.telescope.parameters
            }
            if self.has_simulated_dl1:
                simulated_param_readers = {
                    tel.name: HDF5TableReader(self.file_).read(
                        f"/simulation/event/telescope/parameters/{tel.name}",
                        containers=[
                            HillasParametersContainer(),
                            LeakageContainer(),
                            ConcentrationContainer(),
                            MorphologyContainer(),
                            IntensityStatisticsContainer(),
                        ],
                        prefixes=True,
                    )
                    for tel in self.file_.root.dl1.event.telescope.parameters
                }

        if self.is_simulation:
            # simulated shower wide information
            mc_shower_reader = HDF5TableReader(self.file_).read(
                "/simulation/event/subarray/shower",
                SimulatedShowerContainer(),
                prefixes="true",
            )

        # Setup iterators for the array events
        events = HDF5TableReader(self.file_).read(
            "/dl1/event/subarray/trigger",
            [TriggerContainer(), EventIndexContainer()])

        array_pointing_finder = IndexFinder(
            self.file_.root.dl1.monitoring.subarray.pointing.col("time"))

        tel_pointing_finder = {
            tel.name: IndexFinder(tel.col("time"))
            for tel in self.file_.root.dl1.monitoring.telescope.pointing
        }

        for counter, (trigger, index) in enumerate(events):
            data.dl1.tel.clear()
            data.simulation.tel.clear()
            data.pointing.tel.clear()
            data.trigger.tel.clear()

            data.count = counter
            data.trigger = trigger
            data.index = index
            data.trigger.tels_with_trigger = self.subarray.tel_mask_to_tel_ids(
                data.trigger.tels_with_trigger)

            # Maybe there is a simpler way  to do this
            # Beware: tels_with_trigger contains all triggered telescopes whereas
            # the telescope trigger table contains only the subset of
            # allowed_tels given during the creation of the dl1 file
            for i in self.file_.root.dl1.event.telescope.trigger.where(
                    f"(obs_id=={data.index.obs_id}) & (event_id=={data.index.event_id})"
            ):
                if self.allowed_tels and i["tel_id"] not in self.allowed_tels:
                    continue
                if self.datamodel_version == "v1.0.0":
                    data.trigger.tel[
                        i["tel_id"]].time = i["telescopetrigger_time"]
                else:
                    data.trigger.tel[i["tel_id"]].time = i["time"]

            self._fill_array_pointing(data, array_pointing_finder)
            self._fill_telescope_pointing(data, tel_pointing_finder)

            if self.is_simulation:
                data.simulation.shower = next(mc_shower_reader)

            for tel in data.trigger.tel.keys():
                if self.allowed_tels and tel not in self.allowed_tels:
                    continue
                if self.has_simulated_dl1:
                    simulated = data.simulation.tel[tel]
                dl1 = data.dl1.tel[tel]
                if DataLevel.DL1_IMAGES in self.datalevels:
                    if f"tel_{tel:03d}" not in image_iterators.keys():
                        logger.debug(f"Triggered telescope {tel} is missing "
                                     "from the image table.")
                        continue
                    image_row = next(image_iterators[f"tel_{tel:03d}"])
                    dl1.image = image_row["image"]
                    dl1.peak_time = image_row["peak_time"]
                    dl1.image_mask = image_row["image_mask"]

                    if self.has_simulated_dl1:
                        if f"tel_{tel:03d}" not in simulated_image_iterators.keys(
                        ):
                            logger.warning(
                                f"Triggered telescope {tel} is missing "
                                "from the simulated image table, but was present at the "
                                "reconstructed image table.")
                            continue
                        simulated_image_row = next(
                            simulated_image_iterators[f"tel_{tel:03d}"])
                        simulated.true_image = simulated_image_row[
                            "true_image"]

                if DataLevel.DL1_PARAMETERS in self.datalevels:
                    if f"tel_{tel:03d}" not in param_readers.keys():
                        logger.debug(f"Triggered telescope {tel} is missing "
                                     "from the parameters table.")
                        continue
                    # Is there a smarter way to unpack this?
                    # Best would probbaly be if we could directly read
                    # into the ImageParametersContainer
                    params = next(param_readers[f"tel_{tel:03d}"])
                    dl1.parameters = ImageParametersContainer(
                        hillas=params[0],
                        timing=params[1],
                        leakage=params[2],
                        concentration=params[3],
                        morphology=params[4],
                        intensity_statistics=params[5],
                        peak_time_statistics=params[6],
                    )

                    if self.has_simulated_dl1:
                        if f"tel_{tel:03d}" not in param_readers.keys():
                            logger.debug(
                                f"Triggered telescope {tel} is missing "
                                "from the simulated parameters table, but was "
                                "present at the reconstructed parameters table."
                            )
                            continue
                        simulated_params = next(
                            simulated_param_readers[f"tel_{tel:03d}"])
                        simulated.true_parameters = ImageParametersContainer(
                            hillas=simulated_params[0],
                            leakage=simulated_params[1],
                            concentration=simulated_params[2],
                            morphology=simulated_params[3],
                            intensity_statistics=simulated_params[4],
                        )

            yield data
Example #6
0
def test_CameraFrame_against_TelescopeFrame(filename):

    input_file = get_dataset_path(
        "gamma_divergent_LaPalma_baseline_20Zd_180Az_prod3_test.simtel.gz")

    source = SimTelEventSource(input_file, max_events=10)

    calib = CameraCalibrator(subarray=source.subarray)
    reconstructor = HillasReconstructor(source.subarray)

    reconstructed_events = 0

    for event in source:

        calib(event)
        # make a copy of the calibrated event for the camera frame case
        # later we clean and paramretrize the 2 events in the same way
        # but in 2 different frames to check they return compatible results
        event_camera_frame = deepcopy(event)

        telescope_pointings = {}
        hillas_dict_camera_frame = {}
        hillas_dict_telescope_frame = {}

        for tel_id, dl1 in event.dl1.tel.items():

            event_camera_frame.dl1.tel[
                tel_id].parameters = ImageParametersContainer()
            event.dl1.tel[tel_id].parameters = ImageParametersContainer()

            # this is needed only here to transform the camera geometries
            telescope_pointings[tel_id] = SkyCoord(
                alt=event.pointing.tel[tel_id].altitude,
                az=event.pointing.tel[tel_id].azimuth,
                frame=AltAz(),
            )

            geom_camera_frame = source.subarray.tel[tel_id].camera.geometry

            # this could be done also out of this loop,
            # but in case of real data each telescope would have a
            # different telescope_pointing
            geom_telescope_frame = geom_camera_frame.transform_to(
                TelescopeFrame(telescope_pointing=telescope_pointings[tel_id]))

            mask = tailcuts_clean(geom_telescope_frame,
                                  dl1.image,
                                  picture_thresh=10.0,
                                  boundary_thresh=5.0)

            try:
                moments_camera_frame = hillas_parameters(
                    geom_camera_frame[mask], dl1.image[mask])
                moments_telescope_frame = hillas_parameters(
                    geom_telescope_frame[mask], dl1.image[mask])

                if (moments_camera_frame.width.value >
                        0) and (moments_telescope_frame.width.value > 0):
                    event_camera_frame.dl1.tel[
                        tel_id].parameters.hillas = moments_camera_frame
                    dl1.parameters.hillas = moments_telescope_frame

                    hillas_dict_camera_frame[tel_id] = moments_camera_frame
                    hillas_dict_telescope_frame[
                        tel_id] = moments_telescope_frame
                else:
                    continue

            except HillasParameterizationError as e:
                print(e)
                continue

        if (len(hillas_dict_camera_frame) >
                2) and (len(hillas_dict_telescope_frame) > 2):
            reconstructor(event_camera_frame)
            reconstructor(event)
            reconstructed_events += 1
        else:  # this event was not good enough to be tested on
            continue

        # Compare old approach with new approach
        result_camera_frame = event_camera_frame.dl2.stereo.geometry[
            "HillasReconstructor"]
        result_telescope_frame = event.dl2.stereo.geometry[
            "HillasReconstructor"]

        assert result_camera_frame.is_valid
        assert result_telescope_frame.is_valid

        for field in event.dl2.stereo.geometry["HillasReconstructor"].as_dict(
        ):
            C = np.asarray(result_camera_frame.as_dict()[field])
            T = np.asarray(result_telescope_frame.as_dict()[field])
            assert (np.isclose(C, T, rtol=1e-03, atol=1e-03,
                               equal_nan=True)).all()

    assert reconstructed_events > 0  # check that we reconstruct at least 1 event
Example #7
0
def test_reconstruction_against_simulation(
        subarray_and_event_gamma_off_axis_500_gev):
    """Reconstruction is here done only in the TelescopeFrame,
    since the previous tests test already for the compatibility between
    frames"""

    # 4-LST bright event already calibrated
    # we'll clean it and parametrize it again in the TelescopeFrame
    subarray, event = subarray_and_event_gamma_off_axis_500_gev

    # define reconstructor
    reconstructor = HillasReconstructor(subarray)

    hillas_dict = {}
    telescope_pointings = {}

    for tel_id, dl1 in event.dl1.tel.items():

        telescope_pointings[tel_id] = SkyCoord(
            alt=event.pointing.tel[tel_id].altitude,
            az=event.pointing.tel[tel_id].azimuth,
            frame=AltAz(),
        )

        geom_CameraFrame = subarray.tel[tel_id].camera.geometry

        # this could be done also out of this loop,
        # but in case of real data each telescope would have a
        # different telescope_pointing
        geom_TelescopeFrame = geom_CameraFrame.transform_to(
            TelescopeFrame(telescope_pointing=telescope_pointings[tel_id]))

        mask = tailcuts_clean(
            geom_TelescopeFrame,
            dl1.image,
            picture_thresh=5.0,
            boundary_thresh=2.5,
            keep_isolated_pixels=False,
            min_number_picture_neighbors=2,
        )

        try:
            hillas_dict[tel_id] = hillas_parameters(geom_TelescopeFrame[mask],
                                                    dl1.image[mask])

            # the original event is created from a
            # pytest fixture with "session" scope, so it's always the same
            # and if we used the same event we would overwrite the image
            # parameters for the next tests, thus causing their failure
            test_event = deepcopy(event)
            test_event.dl1.tel[tel_id].parameters = ImageParametersContainer()
            test_event.dl1.tel[tel_id].parameters.hillas = hillas_dict[tel_id]

        except HillasParameterizationError as e:
            print(e)
            continue

    # Get shower geometry
    reconstructor(event)
    # get the result from the correct DL2 container
    result = event.dl2.stereo.geometry["HillasReconstructor"]

    # get the reconstructed coordinates in the sky
    reco_coord = SkyCoord(alt=result.alt, az=result.az, frame=AltAz())
    # get the simulated coordinates in the sky
    true_coord = SkyCoord(alt=event.simulation.shower.alt,
                          az=event.simulation.shower.az,
                          frame=AltAz())

    # check that we are not more far than 0.1 degrees
    assert reco_coord.separation(true_coord) < 0.1 * u.deg
Example #8
0
def test_invalid_events(subarray_and_event_gamma_off_axis_500_gev):
    """
    The HillasReconstructor is supposed to fail
    in these cases:
    - less than two teleskopes
    - any width is NaN
    - any width is 0

    This test takes 1 shower from a test simtel file and modifies a-posteriori
    some hillas dictionaries to make it non-reconstructable.
    It is supposed to fail if no Exception or another Exception gets thrown.
    """

    # 4-LST bright event already calibrated
    # we'll clean it and parametrize it again in the TelescopeFrame
    subarray, event = subarray_and_event_gamma_off_axis_500_gev

    tel_azimuth = {}
    tel_altitude = {}

    #source = EventSource(filename, max_events=1)
    #subarray = source.subarray
    calib = CameraCalibrator(subarray)
    fit = HillasReconstructor(subarray)

    #for event in source:

    calib(event)

    hillas_dict = {}
    for tel_id, dl1 in event.dl1.tel.items():

        geom = subarray.tel[tel_id].camera.geometry
        tel_azimuth[tel_id] = event.pointing.tel[tel_id].azimuth
        tel_altitude[tel_id] = event.pointing.tel[tel_id].altitude

        mask = tailcuts_clean(geom,
                              dl1.image,
                              picture_thresh=10.0,
                              boundary_thresh=5.0)

        dl1.parameters = ImageParametersContainer()

        try:
            moments = hillas_parameters(geom[mask], dl1.image[mask])
            hillas_dict[tel_id] = moments
            dl1.parameters.hillas = moments
        except HillasParameterizationError:
            dl1.parameters.hillas = HillasParametersContainer()
            continue

    # copy event container to modify it
    event_copy = deepcopy(event)
    # overwrite all image parameters but the last one with dummy ones
    for tel_id in list(event_copy.dl1.tel.keys())[:-1]:
        event_copy.dl1.tel[
            tel_id].parameters.hillas = HillasParametersContainer()
    fit(event_copy)
    assert event_copy.dl2.stereo.geometry[
        "HillasReconstructor"].is_valid is False

    # Now use the original event, but overwrite the last width to 0
    event.dl1.tel[tel_id].parameters.hillas.width = 0 * u.m
    fit(event)
    assert event.dl2.stereo.geometry["HillasReconstructor"].is_valid is False

    # Now use the original event, but overwrite the last width to NaN
    event.dl1.tel[tel_id].parameters.hillas.width = np.nan * u.m
    fit(event)
    assert event.dl2.stereo.geometry["HillasReconstructor"].is_valid is False