def test_writing_nan_defaults(): from ctapipe.containers import ImageParametersContainer params = ImageParametersContainer() with tempfile.NamedTemporaryFile(suffix=".hdf5") as f: with HDF5TableWriter(f.name, mode="w") as writer: writer.write("params", params.values())
def test_writing_nan_defaults(tmp_path): from ctapipe.containers import ImageParametersContainer path = tmp_path / "test.h5" params = ImageParametersContainer() with HDF5TableWriter(path, mode="w") as writer: writer.write("params", params.values())
def test_reconstructors(reconstructors): """ a test of the complete fit procedure on one event including: • tailcut cleaning • hillas parametrisation • HillasPlane creation • direction fit • position fit in the end, proper units in the output are asserted""" filename = get_dataset_path( "gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz") source = EventSource(filename, max_events=10) subarray = source.subarray calib = CameraCalibrator(source.subarray) horizon_frame = AltAz() for event in source: calib(event) sim_shower = event.simulation.shower array_pointing = SkyCoord(az=sim_shower.az, alt=sim_shower.alt, frame=horizon_frame) telescope_pointings = {} for tel_id, dl1 in event.dl1.tel.items(): geom = source.subarray.tel[tel_id].camera.geometry telescope_pointings[tel_id] = SkyCoord( alt=event.pointing.tel[tel_id].altitude, az=event.pointing.tel[tel_id].azimuth, frame=horizon_frame, ) mask = tailcuts_clean(geom, dl1.image, picture_thresh=10.0, boundary_thresh=5.0) dl1.parameters = ImageParametersContainer() try: moments = hillas_parameters(geom[mask], dl1.image[mask]) except HillasParameterizationError: dl1.parameters.hillas = HillasParametersContainer() continue # Make sure we provide only good images for the test if np.isnan(moments.width.value) or (moments.width.value == 0): dl1.parameters.hillas = HillasParametersContainer() else: dl1.parameters.hillas = moments hillas_dict = { tel_id: dl1.parameters.hillas for tel_id, dl1 in event.dl1.tel.items() if np.isfinite(dl1.parameters.hillas.intensity) } if len(hillas_dict) < 2: continue for count, reco_method in enumerate(reconstructors): if reco_method is HillasReconstructor: reconstructor = HillasReconstructor(subarray) reconstructor(event) event.dl2.stereo.geometry["HillasReconstructor"].alt.to(u.deg) event.dl2.stereo.geometry["HillasReconstructor"].az.to(u.deg) event.dl2.stereo.geometry["HillasReconstructor"].core_x.to(u.m) assert event.dl2.stereo.geometry[ "HillasReconstructor"].is_valid else: reconstructor = reco_method() try: reconstructor_out = reconstructor.predict( hillas_dict, source.subarray, array_pointing, telescope_pointings, ) except InvalidWidthException: continue reconstructor_out.alt.to(u.deg) reconstructor_out.az.to(u.deg) reconstructor_out.core_x.to(u.m) assert reconstructor_out.is_valid
def test_array_display(): """ check that we can do basic array display functionality """ from ctapipe.visualization.mpl_array import ArrayDisplay from ctapipe.image import timing_parameters from ctapipe.containers import ( ArrayEventContainer, DL1Container, DL1CameraContainer, ImageParametersContainer, CoreParametersContainer, ) # build a test subarray: tels = dict() tel_pos = dict() for ii, pos in enumerate([[0, 0, 0], [100, 0, 0], [-100, 0, 0]] * u.m): tels[ii + 1] = TelescopeDescription.from_name("MST", "NectarCam") tel_pos[ii + 1] = pos sub = SubarrayDescription(name="TestSubarray", tel_positions=tel_pos, tel_descriptions=tels) # Create a fake event containing telescope-wise information about # the image directions projected on the ground event = ArrayEventContainer() event.dl1 = DL1Container() event.dl1.tel = {1: DL1CameraContainer(), 2: DL1CameraContainer()} event.dl1.tel[1].parameters = ImageParametersContainer() event.dl1.tel[2].parameters = ImageParametersContainer() event.dl1.tel[2].parameters.core = CoreParametersContainer() event.dl1.tel[1].parameters.core = CoreParametersContainer() event.dl1.tel[1].parameters.core.psi = u.Quantity(2.0, unit=u.deg) event.dl1.tel[2].parameters.core.psi = u.Quantity(1.0, unit=u.deg) ad = ArrayDisplay(subarray=sub) ad.set_vector_rho_phi(1 * u.m, 90 * u.deg) # try setting a value vals = np.ones(sub.num_tels) ad.values = vals assert (vals == ad.values).all() # test UV field ... # ...with colors by telescope type ad.set_vector_uv(np.array([1, 2, 3]) * u.m, np.array([1, 2, 3]) * u.m) # ...with scalar color ad.set_vector_uv(np.array([1, 2, 3]) * u.m, np.array([1, 2, 3]) * u.m, c=3) geom = CameraGeometry.from_name("LSTCam") rot_angle = 20 * u.deg hillas = CameraHillasParametersContainer(x=0 * u.m, y=0 * u.m, psi=rot_angle) # test using hillas params CameraFrame: hillas_dict = { 1: CameraHillasParametersContainer(length=100.0 * u.m, psi=90 * u.deg), 2: CameraHillasParametersContainer(length=20000 * u.cm, psi="95deg"), } grad = 2 intercept = 1 timing_rot20 = timing_parameters( geom, image=np.ones(geom.n_pixels), peak_time=intercept + grad * geom.pix_x.value, hillas_parameters=hillas, cleaning_mask=np.ones(geom.n_pixels, dtype=bool), ) gradient_dict = {1: timing_rot20.slope.value, 2: timing_rot20.slope.value} core_dict = { tel_id: dl1.parameters.core.psi for tel_id, dl1 in event.dl1.tel.items() } ad.set_vector_hillas( hillas_dict=hillas_dict, core_dict=core_dict, length=500, time_gradient=gradient_dict, angle_offset=0 * u.deg, ) ad.set_line_hillas(hillas_dict=hillas_dict, core_dict=core_dict, range=300) # test using hillas params for divergent pointing in telescopeframe: hillas_dict = { 1: HillasParametersContainer(fov_lon=1.0 * u.deg, fov_lat=1.0 * u.deg, length=1.0 * u.deg, psi=90 * u.deg), 2: HillasParametersContainer(fov_lon=1.0 * u.deg, fov_lat=1.0 * u.deg, length=1.0 * u.deg, psi=95 * u.deg), } ad.set_vector_hillas( hillas_dict=hillas_dict, core_dict=core_dict, length=500, time_gradient=gradient_dict, angle_offset=0 * u.deg, ) ad.set_line_hillas(hillas_dict=hillas_dict, core_dict=core_dict, range=300) # test using hillas params for parallel pointing in telescopeframe: hillas_dict = { 1: HillasParametersContainer(fov_lon=1.0 * u.deg, fov_lat=1.0 * u.deg, length=1.0 * u.deg, psi=90 * u.deg), 2: HillasParametersContainer(fov_lon=1.0 * u.deg, fov_lat=1.0 * u.deg, length=1.0 * u.deg, psi=95 * u.deg), } ad.set_vector_hillas( hillas_dict=hillas_dict, core_dict=core_dict, length=500, time_gradient=gradient_dict, angle_offset=0 * u.deg, ) # test negative time_gradients gradient_dict = {1: -0.03, 2: -0.02} ad.set_vector_hillas( hillas_dict=hillas_dict, core_dict=core_dict, length=500, time_gradient=gradient_dict, angle_offset=0 * u.deg, ) # and very small gradient_dict = {1: 0.003, 2: 0.002} ad.set_vector_hillas( hillas_dict=hillas_dict, core_dict=core_dict, length=500, time_gradient=gradient_dict, angle_offset=0 * u.deg, ) # Test background contour ad.background_contour( x=np.array([0, 1, 2]), y=np.array([0, 1, 2]), background=np.array([[0, 1, 2], [0, 1, 2], [0, 1, 2]]), ) ad.set_line_hillas(hillas_dict=hillas_dict, core_dict=core_dict, range=300) ad.add_labels() ad.remove_labels()
def _generate_events(self): """ Yield ArrayEventContainer to iterate through events. """ data = ArrayEventContainer() # Maybe take some other metadata, but there are still some 'unknown' # written out by the stage1 tool data.meta["origin"] = self.file_.root._v_attrs["CTA PROCESS TYPE"] data.meta["input_url"] = self.input_url data.meta["max_events"] = self.max_events if DataLevel.DL1_IMAGES in self.datalevels: image_iterators = { tel.name: self.file_.root.dl1.event.telescope.images[ tel.name].iterrows() for tel in self.file_.root.dl1.event.telescope.images } if self.has_simulated_dl1: simulated_image_iterators = { tel.name: self.file_.root.simulation.event.telescope.images[ tel.name].iterrows() for tel in self.file_.root.simulation.event.telescope.images } if DataLevel.DL1_PARAMETERS in self.datalevels: param_readers = { tel.name: HDF5TableReader(self.file_).read( f"/dl1/event/telescope/parameters/{tel.name}", containers=[ HillasParametersContainer(), TimingParametersContainer(), LeakageContainer(), ConcentrationContainer(), MorphologyContainer(), IntensityStatisticsContainer(), PeakTimeStatisticsContainer(), ], prefixes=True, ) for tel in self.file_.root.dl1.event.telescope.parameters } if self.has_simulated_dl1: simulated_param_readers = { tel.name: HDF5TableReader(self.file_).read( f"/simulation/event/telescope/parameters/{tel.name}", containers=[ HillasParametersContainer(), LeakageContainer(), ConcentrationContainer(), MorphologyContainer(), IntensityStatisticsContainer(), ], prefixes=True, ) for tel in self.file_.root.dl1.event.telescope.parameters } if self.is_simulation: # simulated shower wide information mc_shower_reader = HDF5TableReader(self.file_).read( "/simulation/event/subarray/shower", SimulatedShowerContainer(), prefixes="true", ) # Setup iterators for the array events events = HDF5TableReader(self.file_).read( "/dl1/event/subarray/trigger", [TriggerContainer(), EventIndexContainer()]) array_pointing_finder = IndexFinder( self.file_.root.dl1.monitoring.subarray.pointing.col("time")) tel_pointing_finder = { tel.name: IndexFinder(tel.col("time")) for tel in self.file_.root.dl1.monitoring.telescope.pointing } for counter, (trigger, index) in enumerate(events): data.dl1.tel.clear() data.simulation.tel.clear() data.pointing.tel.clear() data.trigger.tel.clear() data.count = counter data.trigger = trigger data.index = index data.trigger.tels_with_trigger = self.subarray.tel_mask_to_tel_ids( data.trigger.tels_with_trigger) # Maybe there is a simpler way to do this # Beware: tels_with_trigger contains all triggered telescopes whereas # the telescope trigger table contains only the subset of # allowed_tels given during the creation of the dl1 file for i in self.file_.root.dl1.event.telescope.trigger.where( f"(obs_id=={data.index.obs_id}) & (event_id=={data.index.event_id})" ): if self.allowed_tels and i["tel_id"] not in self.allowed_tels: continue if self.datamodel_version == "v1.0.0": data.trigger.tel[ i["tel_id"]].time = i["telescopetrigger_time"] else: data.trigger.tel[i["tel_id"]].time = i["time"] self._fill_array_pointing(data, array_pointing_finder) self._fill_telescope_pointing(data, tel_pointing_finder) if self.is_simulation: data.simulation.shower = next(mc_shower_reader) for tel in data.trigger.tel.keys(): if self.allowed_tels and tel not in self.allowed_tels: continue if self.has_simulated_dl1: simulated = data.simulation.tel[tel] dl1 = data.dl1.tel[tel] if DataLevel.DL1_IMAGES in self.datalevels: if f"tel_{tel:03d}" not in image_iterators.keys(): logger.debug(f"Triggered telescope {tel} is missing " "from the image table.") continue image_row = next(image_iterators[f"tel_{tel:03d}"]) dl1.image = image_row["image"] dl1.peak_time = image_row["peak_time"] dl1.image_mask = image_row["image_mask"] if self.has_simulated_dl1: if f"tel_{tel:03d}" not in simulated_image_iterators.keys( ): logger.warning( f"Triggered telescope {tel} is missing " "from the simulated image table, but was present at the " "reconstructed image table.") continue simulated_image_row = next( simulated_image_iterators[f"tel_{tel:03d}"]) simulated.true_image = simulated_image_row[ "true_image"] if DataLevel.DL1_PARAMETERS in self.datalevels: if f"tel_{tel:03d}" not in param_readers.keys(): logger.debug(f"Triggered telescope {tel} is missing " "from the parameters table.") continue # Is there a smarter way to unpack this? # Best would probbaly be if we could directly read # into the ImageParametersContainer params = next(param_readers[f"tel_{tel:03d}"]) dl1.parameters = ImageParametersContainer( hillas=params[0], timing=params[1], leakage=params[2], concentration=params[3], morphology=params[4], intensity_statistics=params[5], peak_time_statistics=params[6], ) if self.has_simulated_dl1: if f"tel_{tel:03d}" not in param_readers.keys(): logger.debug( f"Triggered telescope {tel} is missing " "from the simulated parameters table, but was " "present at the reconstructed parameters table." ) continue simulated_params = next( simulated_param_readers[f"tel_{tel:03d}"]) simulated.true_parameters = ImageParametersContainer( hillas=simulated_params[0], leakage=simulated_params[1], concentration=simulated_params[2], morphology=simulated_params[3], intensity_statistics=simulated_params[4], ) yield data
def test_CameraFrame_against_TelescopeFrame(filename): input_file = get_dataset_path( "gamma_divergent_LaPalma_baseline_20Zd_180Az_prod3_test.simtel.gz") source = SimTelEventSource(input_file, max_events=10) calib = CameraCalibrator(subarray=source.subarray) reconstructor = HillasReconstructor(source.subarray) reconstructed_events = 0 for event in source: calib(event) # make a copy of the calibrated event for the camera frame case # later we clean and paramretrize the 2 events in the same way # but in 2 different frames to check they return compatible results event_camera_frame = deepcopy(event) telescope_pointings = {} hillas_dict_camera_frame = {} hillas_dict_telescope_frame = {} for tel_id, dl1 in event.dl1.tel.items(): event_camera_frame.dl1.tel[ tel_id].parameters = ImageParametersContainer() event.dl1.tel[tel_id].parameters = ImageParametersContainer() # this is needed only here to transform the camera geometries telescope_pointings[tel_id] = SkyCoord( alt=event.pointing.tel[tel_id].altitude, az=event.pointing.tel[tel_id].azimuth, frame=AltAz(), ) geom_camera_frame = source.subarray.tel[tel_id].camera.geometry # this could be done also out of this loop, # but in case of real data each telescope would have a # different telescope_pointing geom_telescope_frame = geom_camera_frame.transform_to( TelescopeFrame(telescope_pointing=telescope_pointings[tel_id])) mask = tailcuts_clean(geom_telescope_frame, dl1.image, picture_thresh=10.0, boundary_thresh=5.0) try: moments_camera_frame = hillas_parameters( geom_camera_frame[mask], dl1.image[mask]) moments_telescope_frame = hillas_parameters( geom_telescope_frame[mask], dl1.image[mask]) if (moments_camera_frame.width.value > 0) and (moments_telescope_frame.width.value > 0): event_camera_frame.dl1.tel[ tel_id].parameters.hillas = moments_camera_frame dl1.parameters.hillas = moments_telescope_frame hillas_dict_camera_frame[tel_id] = moments_camera_frame hillas_dict_telescope_frame[ tel_id] = moments_telescope_frame else: continue except HillasParameterizationError as e: print(e) continue if (len(hillas_dict_camera_frame) > 2) and (len(hillas_dict_telescope_frame) > 2): reconstructor(event_camera_frame) reconstructor(event) reconstructed_events += 1 else: # this event was not good enough to be tested on continue # Compare old approach with new approach result_camera_frame = event_camera_frame.dl2.stereo.geometry[ "HillasReconstructor"] result_telescope_frame = event.dl2.stereo.geometry[ "HillasReconstructor"] assert result_camera_frame.is_valid assert result_telescope_frame.is_valid for field in event.dl2.stereo.geometry["HillasReconstructor"].as_dict( ): C = np.asarray(result_camera_frame.as_dict()[field]) T = np.asarray(result_telescope_frame.as_dict()[field]) assert (np.isclose(C, T, rtol=1e-03, atol=1e-03, equal_nan=True)).all() assert reconstructed_events > 0 # check that we reconstruct at least 1 event
def test_reconstruction_against_simulation( subarray_and_event_gamma_off_axis_500_gev): """Reconstruction is here done only in the TelescopeFrame, since the previous tests test already for the compatibility between frames""" # 4-LST bright event already calibrated # we'll clean it and parametrize it again in the TelescopeFrame subarray, event = subarray_and_event_gamma_off_axis_500_gev # define reconstructor reconstructor = HillasReconstructor(subarray) hillas_dict = {} telescope_pointings = {} for tel_id, dl1 in event.dl1.tel.items(): telescope_pointings[tel_id] = SkyCoord( alt=event.pointing.tel[tel_id].altitude, az=event.pointing.tel[tel_id].azimuth, frame=AltAz(), ) geom_CameraFrame = subarray.tel[tel_id].camera.geometry # this could be done also out of this loop, # but in case of real data each telescope would have a # different telescope_pointing geom_TelescopeFrame = geom_CameraFrame.transform_to( TelescopeFrame(telescope_pointing=telescope_pointings[tel_id])) mask = tailcuts_clean( geom_TelescopeFrame, dl1.image, picture_thresh=5.0, boundary_thresh=2.5, keep_isolated_pixels=False, min_number_picture_neighbors=2, ) try: hillas_dict[tel_id] = hillas_parameters(geom_TelescopeFrame[mask], dl1.image[mask]) # the original event is created from a # pytest fixture with "session" scope, so it's always the same # and if we used the same event we would overwrite the image # parameters for the next tests, thus causing their failure test_event = deepcopy(event) test_event.dl1.tel[tel_id].parameters = ImageParametersContainer() test_event.dl1.tel[tel_id].parameters.hillas = hillas_dict[tel_id] except HillasParameterizationError as e: print(e) continue # Get shower geometry reconstructor(event) # get the result from the correct DL2 container result = event.dl2.stereo.geometry["HillasReconstructor"] # get the reconstructed coordinates in the sky reco_coord = SkyCoord(alt=result.alt, az=result.az, frame=AltAz()) # get the simulated coordinates in the sky true_coord = SkyCoord(alt=event.simulation.shower.alt, az=event.simulation.shower.az, frame=AltAz()) # check that we are not more far than 0.1 degrees assert reco_coord.separation(true_coord) < 0.1 * u.deg
def test_invalid_events(subarray_and_event_gamma_off_axis_500_gev): """ The HillasReconstructor is supposed to fail in these cases: - less than two teleskopes - any width is NaN - any width is 0 This test takes 1 shower from a test simtel file and modifies a-posteriori some hillas dictionaries to make it non-reconstructable. It is supposed to fail if no Exception or another Exception gets thrown. """ # 4-LST bright event already calibrated # we'll clean it and parametrize it again in the TelescopeFrame subarray, event = subarray_and_event_gamma_off_axis_500_gev tel_azimuth = {} tel_altitude = {} #source = EventSource(filename, max_events=1) #subarray = source.subarray calib = CameraCalibrator(subarray) fit = HillasReconstructor(subarray) #for event in source: calib(event) hillas_dict = {} for tel_id, dl1 in event.dl1.tel.items(): geom = subarray.tel[tel_id].camera.geometry tel_azimuth[tel_id] = event.pointing.tel[tel_id].azimuth tel_altitude[tel_id] = event.pointing.tel[tel_id].altitude mask = tailcuts_clean(geom, dl1.image, picture_thresh=10.0, boundary_thresh=5.0) dl1.parameters = ImageParametersContainer() try: moments = hillas_parameters(geom[mask], dl1.image[mask]) hillas_dict[tel_id] = moments dl1.parameters.hillas = moments except HillasParameterizationError: dl1.parameters.hillas = HillasParametersContainer() continue # copy event container to modify it event_copy = deepcopy(event) # overwrite all image parameters but the last one with dummy ones for tel_id in list(event_copy.dl1.tel.keys())[:-1]: event_copy.dl1.tel[ tel_id].parameters.hillas = HillasParametersContainer() fit(event_copy) assert event_copy.dl2.stereo.geometry[ "HillasReconstructor"].is_valid is False # Now use the original event, but overwrite the last width to 0 event.dl1.tel[tel_id].parameters.hillas.width = 0 * u.m fit(event) assert event.dl2.stereo.geometry["HillasReconstructor"].is_valid is False # Now use the original event, but overwrite the last width to NaN event.dl1.tel[tel_id].parameters.hillas.width = np.nan * u.m fit(event) assert event.dl2.stereo.geometry["HillasReconstructor"].is_valid is False