def test_estimator_results(): """ creating some planes pointing in different directions (two north-south, two east-west) and that have a slight position errors (+- 0.1 m in one of the four cardinal directions """ horizon_frame = HorizonFrame() p1 = SkyCoord(alt=43 * u.deg, az=45 * u.deg, frame=horizon_frame) p2 = SkyCoord(alt=47 * u.deg, az=45 * u.deg, frame=horizon_frame) circle1 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, 1, 0] * u.m) p1 = SkyCoord(alt=44 * u.deg, az=90 * u.deg, frame=horizon_frame) p2 = SkyCoord(alt=46 * u.deg, az=90 * u.deg, frame=horizon_frame) circle2 = HillasPlane(p1=p1, p2=p2, telescope_position=[1, 0, 0] * u.m) p1 = SkyCoord(alt=44.5 * u.deg, az=45 * u.deg, frame=horizon_frame) p2 = SkyCoord(alt=46.5 * u.deg, az=45 * u.deg, frame=horizon_frame) circle3 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, -1, 0] * u.m) p1 = SkyCoord(alt=43.5 * u.deg, az=90 * u.deg, frame=horizon_frame) p2 = SkyCoord(alt=45.5 * u.deg, az=90 * u.deg, frame=horizon_frame) circle4 = HillasPlane(p1=p1, p2=p2, telescope_position=[-1, 0, 0] * u.m) # creating the fit class and setting the the great circle member fit = HillasReconstructor() fit.hillas_planes = {1: circle1, 2: circle2, 3: circle3, 4: circle4} # performing the direction fit with the minimisation algorithm # and a seed that is perpendicular to the up direction dir_fit_minimise, _ = fit.estimate_direction() print("direction fit test minimise:", dir_fit_minimise) print()
def test_FitGammaHillas(): ''' a test of the complete fit procedure on one event including: • tailcut cleaning • hillas parametrisation • GreatCircle creation • direction fit • position fit in the end, proper units in the output are asserted ''' filename = get_dataset("gamma_test.simtel.gz") fit = HillasReconstructor() cam_geom = {} tel_phi = {} tel_theta = {} source = hessio_event_source(filename) for event in source: hillas_dict = {} for tel_id in event.dl0.tels_with_data: if tel_id not in cam_geom: cam_geom[tel_id] = CameraGeometry.guess( event.inst.pixel_pos[tel_id][0], event.inst.pixel_pos[tel_id][1], event.inst.optical_foclen[tel_id]) tel_phi[tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad tel_theta[tel_id] = (np.pi/2-event.mc.tel[tel_id].altitude_raw)*u.rad pmt_signal = event.r0.tel[tel_id].adc_sums[0] mask = tailcuts_clean(cam_geom[tel_id], pmt_signal, picture_thresh=10., boundary_thresh=5.) pmt_signal[mask == 0] = 0 try: moments = hillas_parameters(event.inst.pixel_pos[tel_id][0], event.inst.pixel_pos[tel_id][1], pmt_signal) hillas_dict[tel_id] = moments except HillasParameterizationError as e: print(e) continue if len(hillas_dict) < 2: continue fit_result = fit.predict(hillas_dict, event.inst, tel_phi, tel_theta) print(fit_result) fit_result.alt.to(u.deg) fit_result.az.to(u.deg) fit_result.core_x.to(u.m) assert fit_result.is_valid return
def test_estimator_results(): """ creating some planes pointing in different directions (two north-south, two east-west) and that have a slight position errors (+- 0.1 m in one of the four cardinal directions """ p1 = SkyCoord(alt=43 * u.deg, az=45 * u.deg, frame='altaz') p2 = SkyCoord(alt=47 * u.deg, az=45 * u.deg, frame='altaz') circle1 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, 1, 0] * u.m) p1 = SkyCoord(alt=44 * u.deg, az=90 * u.deg, frame='altaz') p2 = SkyCoord(alt=46 * u.deg, az=90 * u.deg, frame='altaz') circle2 = HillasPlane(p1=p1, p2=p2, telescope_position=[1, 0, 0] * u.m) p1 = SkyCoord(alt=44.5 * u.deg, az=45 * u.deg, frame='altaz') p2 = SkyCoord(alt=46.5 * u.deg, az=45 * u.deg, frame='altaz') circle3 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, -1, 0] * u.m) p1 = SkyCoord(alt=43.5 * u.deg, az=90 * u.deg, frame='altaz') p2 = SkyCoord(alt=45.5 * u.deg, az=90 * u.deg, frame='altaz') circle4 = HillasPlane(p1=p1, p2=p2, telescope_position=[-1, 0, 0] * u.m) # creating the fit class and setting the the great circle member fit = HillasReconstructor() fit.hillas_planes = {1: circle1, 2: circle2, 3: circle3, 4: circle4} # performing the direction fit with the minimisation algorithm # and a seed that is perpendicular to the up direction dir_fit_minimise, _ = fit.estimate_direction() print("direction fit test minimise:", dir_fit_minimise) print()
def test_h_max_results(): """ creating some planes pointing in different directions (two north-south, two east-west) and that have a slight position errors (+- 0.1 m in one of the four cardinal directions """ p1 = SkyCoord(alt=0 * u.deg, az=45 * u.deg, frame='altaz') p2 = SkyCoord(alt=0 * u.deg, az=45 * u.deg, frame='altaz') circle1 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, 1, 0] * u.m) p1 = SkyCoord(alt=0 * u.deg, az=90 * u.deg, frame='altaz') p2 = SkyCoord(alt=0 * u.deg, az=90 * u.deg, frame='altaz') circle2 = HillasPlane(p1=p1, p2=p2, telescope_position=[1, 0, 0] * u.m) p1 = SkyCoord(alt=0 * u.deg, az=45 * u.deg, frame='altaz') p2 = SkyCoord(alt=0 * u.deg, az=45 * u.deg, frame='altaz') circle3 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, -1, 0] * u.m) p1 = SkyCoord(alt=0 * u.deg, az=90 * u.deg, frame='altaz') p2 = SkyCoord(alt=0 * u.deg, az=90 * u.deg, frame='altaz') circle4 = HillasPlane(p1=p1, p2=p2, telescope_position=[-1, 0, 0] * u.m) # creating the fit class and setting the the great circle member fit = HillasReconstructor() fit.hillas_planes = {1: circle1, 2: circle2, 3: circle3, 4: circle4} # performing the direction fit with the minimisation algorithm # and a seed that is perpendicular to the up direction h_max_reco = fit.estimate_h_max() print("h max fit test minimise:", h_max_reco) # the results should be close to the direction straight up np.testing.assert_allclose(h_max_reco, 0, atol=1e-8)
def test_h_max_results(): """ creating some planes pointing in different directions (two north-south, two east-west) and that have a slight position errors (+- 0.1 m in one of the four cardinal directions """ horizon_frame = HorizonFrame() p1 = SkyCoord(alt=0 * u.deg, az=45 * u.deg, frame=horizon_frame) p2 = SkyCoord(alt=0 * u.deg, az=45 * u.deg, frame=horizon_frame) circle1 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, 1, 0] * u.m) p1 = SkyCoord(alt=0 * u.deg, az=90 * u.deg, frame=horizon_frame) p2 = SkyCoord(alt=0 * u.deg, az=90 * u.deg, frame=horizon_frame) circle2 = HillasPlane(p1=p1, p2=p2, telescope_position=[1, 0, 0] * u.m) p1 = SkyCoord(alt=0 * u.deg, az=45 * u.deg, frame=horizon_frame) p2 = SkyCoord(alt=0 * u.deg, az=45 * u.deg, frame=horizon_frame) circle3 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, -1, 0] * u.m) p1 = SkyCoord(alt=0 * u.deg, az=90 * u.deg, frame=horizon_frame) p2 = SkyCoord(alt=0 * u.deg, az=90 * u.deg, frame=horizon_frame) circle4 = HillasPlane(p1=p1, p2=p2, telescope_position=[-1, 0, 0] * u.m) # creating the fit class and setting the the great circle member fit = HillasReconstructor() fit.hillas_planes = {1: circle1, 2: circle2, 3: circle3, 4: circle4} # performing the direction fit with the minimisation algorithm # and a seed that is perpendicular to the up direction h_max_reco = fit.estimate_h_max() print("h max fit test minimise:", h_max_reco) # the results should be close to the direction straight up np.testing.assert_allclose(h_max_reco.value, 0, atol=1e-8)
def test_reconstruction(): """ a test of the complete fit procedure on one event including: • tailcut cleaning • hillas parametrisation • HillasPlane creation • direction fit • position fit in the end, proper units in the output are asserted """ filename = get_dataset_path("gamma_test.simtel.gz") fit = HillasReconstructor() tel_azimuth = {} tel_altitude = {} source = EventSourceFactory.produce( input_url=filename, product='HESSIOEventSource', ) for event in source: hillas_dict = {} for tel_id in event.dl0.tels_with_data: geom = event.inst.subarray.tel[tel_id].camera tel_azimuth[tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad tel_altitude[tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad pmt_signal = event.r0.tel[tel_id].image[0] mask = tailcuts_clean(geom, pmt_signal, picture_thresh=10., boundary_thresh=5.) pmt_signal[mask == 0] = 0 try: moments = hillas_parameters(geom, pmt_signal) hillas_dict[tel_id] = moments except HillasParameterizationError as e: print(e) continue if len(hillas_dict) < 2: continue fit_result = fit.predict(hillas_dict, event.inst, tel_azimuth, tel_altitude) print(fit_result) fit_result.alt.to(u.deg) fit_result.az.to(u.deg) fit_result.core_x.to(u.m) assert fit_result.is_valid return
def __init__(self, event, telescope_list, camera_types, ChargeExtration, pe_thresh, min_neighbors, tail_thresholds, DirReco, quality_cuts, LUT=None): super().__init__() ''' Parmeters --------- event : ctapipe event container calibrator : ctapipe camera calibrator reconstructor : ctapipe hillas reconstructor telescope_list : list with telescope configuration or "all" pe_thresh : dict with thresholds for gain selection tail_thresholds : dict with thresholds for image cleaning quality_cuts : dict containing quality cuts canera_types : list with camera types to analyze ''' self.event = event self.telescope_list = telescope_list self.pe_thresh = pe_thresh self.min_neighbors = min_neighbors self.tail_thresholds = tail_thresholds self.quality_cuts = quality_cuts self.camera_types = camera_types self.dirreco = DirReco self.LUTgenerator = LUT if (self.dirreco["weights"] == "LUT") | (self.dirreco["weights"] == "doublepass"): self.weights = {} else: self.weights = None self.hillas_dict = {} self.camera_dict = {} self.edge_pixels = {} # configurations for calibrator cfg = Config() cfg["ChargeExtractorFactory"]["product"] = \ ChargeExtration["ChargeExtractorProduct"] cfg['WaveformCleanerFactory']['product'] = \ ChargeExtration["WaveformCleanerProduct"] self.calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator", config=cfg) # calibration self.reconstructor = HillasReconstructor() # direction
def test_FitGammaHillas(): ''' a test of the complete fit procedure on one event including: • tailcut cleaning • hillas parametrisation • GreatCircle creation • direction fit • position fit in the end, proper units in the output are asserted ''' filename = get_dataset("gamma_test.simtel.gz") fit = HillasReconstructor() tel_phi = {} tel_theta = {} source = hessio_event_source(filename) for event in source: hillas_dict = {} for tel_id in event.dl0.tels_with_data: geom = event.inst.subarray.tel[tel_id].camera tel_phi[tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad tel_theta[tel_id] = (np.pi / 2 - event.mc.tel[tel_id].altitude_raw) * u.rad pmt_signal = event.r0.tel[tel_id].image[0] mask = tailcuts_clean(geom, pmt_signal, picture_thresh=10., boundary_thresh=5.) pmt_signal[mask == 0] = 0 try: moments = hillas_parameters(geom, pmt_signal) hillas_dict[tel_id] = moments except HillasParameterizationError as e: print(e) continue if len(hillas_dict) < 2: continue fit_result = fit.predict(hillas_dict, event.inst, tel_phi, tel_theta) print(fit_result) fit_result.alt.to(u.deg) fit_result.az.to(u.deg) fit_result.core_x.to(u.m) assert fit_result.is_valid return
def test_reconstruction(): """ a test of the complete fit procedure on one event including: • tailcut cleaning • hillas parametrisation • HillasPlane creation • direction fit • position fit in the end, proper units in the output are asserted """ filename = get_dataset_path("gamma_test.simtel.gz") fit = HillasReconstructor() tel_azimuth = {} tel_altitude = {} source = EventSourceFactory.produce(input_url=filename) for event in source: hillas_dict = {} for tel_id in event.dl0.tels_with_data: geom = event.inst.subarray.tel[tel_id].camera tel_azimuth[tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad tel_altitude[tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad pmt_signal = event.r0.tel[tel_id].image[0] mask = tailcuts_clean(geom, pmt_signal, picture_thresh=10., boundary_thresh=5.) pmt_signal[mask == 0] = 0 try: moments = hillas_parameters(geom, pmt_signal) hillas_dict[tel_id] = moments except HillasParameterizationError as e: print(e) continue if len(hillas_dict) < 2: continue fit_result = fit.predict(hillas_dict, event.inst, tel_azimuth, tel_altitude) print(fit_result) fit_result.alt.to(u.deg) fit_result.az.to(u.deg) fit_result.core_x.to(u.m) assert fit_result.is_valid
def test_fit_core(): ''' creating some great circles pointing in different directions (two north-south, two east-west) and that have a slight position errors (+- 0.1 m in one of the four cardinal directions ''' circle1 = GreatCircle([[1, 0, 0], [0, 0, 1]]) circle1.pos = [0, 0.1] * u.m circle1.trace = [1, 0, 0] circle2 = GreatCircle([[0, 1, 0], [0, 0, 1]]) circle2.pos = [0.1, 0] * u.m circle2.trace = [0, 1, 0] circle3 = GreatCircle([[1, 0, 0], [0, 0, 1]]) circle3.pos = [0, -.1] * u.m circle3.trace = [1, 0, 0] circle4 = GreatCircle([[0, 1, 0], [0, 0, 1]]) circle4.pos = [-.1, 0] * u.m circle4.trace = [0, 1, 0] # creating the fit class and setting the the great circle member fit = HillasReconstructor() fit.circles = {1: circle1, 2: circle2, 3: circle3, 4: circle4} # performing the position fit with the minimisation algorithm # and a seed that is quite far away pos_fit_minimise = fit.fit_core_minimise([100, 1000] * u.m) print("position fit test minimise:", pos_fit_minimise) print() # performing the position fit with the geometric algorithm pos_fit_crosses, err_est_pos_fit_crosses = fit.fit_core_crosses() print("position fit test crosses:", pos_fit_crosses) print("error estimate:", err_est_pos_fit_crosses) print() # the results should be close to the origin of the coordinate system np.testing.assert_allclose(pos_fit_minimise / u.m, [0, 0], atol=1e-3) np.testing.assert_allclose(pos_fit_crosses / u.m, [0, 0], atol=1e-3)
def test_fit_origin(): ''' creating some great circles pointing in different directions (two north-south, two east-west) and that have a slight position errors (+- 0.1 m in one of the four cardinal directions ''' circle1 = GreatCircle([[1, 0, 0], [0, 0, 1]]) circle1.pos = [0, 0.1] * u.m circle1.trace = [1, 0, 0] circle2 = GreatCircle([[0, 1, 0], [0, 0, 1]]) circle2.pos = [0.1, 0] * u.m circle2.trace = [0, 1, 0] circle3 = GreatCircle([[1, 0, 0], [0, 0, 1]]) circle3.pos = [0, -.1] * u.m circle3.trace = [1, 0, 0] circle4 = GreatCircle([[0, 1, 0], [0, 0, 1]]) circle4.pos = [-.1, 0] * u.m circle4.trace = [0, 1, 0] # creating the fit class and setting the the great circle member fit = HillasReconstructor() fit.circles = {1: circle1, 2: circle2, 3: circle3, 4: circle4} # performing the direction fit with the minimisation algorithm # and a seed that is perpendicular to the up direction dir_fit_minimise = fit.fit_origin_minimise((0.1, 0.1, 1)) print("direction fit test minimise:", dir_fit_minimise) print() # performing the direction fit with the geometric algorithm dir_fit_crosses = fit.fit_origin_crosses()[0] print("direction fit test crosses:", dir_fit_crosses) print() # the results should be close to the direction straight up # np.testing.assert_allclose(dir_fit_minimise, [0, 0, 1], atol=1e-1) np.testing.assert_allclose(dir_fit_crosses, [0, 0, 1], atol=1e-3)
class EventPreparer: """ Class which loop on events and returns results stored in container The Class has several purposes. First of all, it prepares the images of the event that will be further use for reconstruction by applying calibration, cleaning and selection. Then, it reconstructs the geometry of the event and then returns image (e.g. Hillas parameters)and event information (e.g. reults od the reconstruction). Parameters ---------- config: dict Configuration with analysis parameters mode: str Mode of the reconstruction, e.g. tail or wave event_cutflow: ctapipe.utils.CutFlow Statistic of events processed image_cutflow: ctapipe.utils.CutFlow Statistic of images processed Returns: dict Dictionnary of results """ def __init__(self, config, mode, event_cutflow=None, image_cutflow=None): # Cleaning for reconstruction self.cleaner_reco = ImageCleaner( # for reconstruction config=config["ImageCleaning"]["biggest"], mode=mode) # Cleaning for energy/score estimation # Add possibility to force energy/score cleaning with tailcut analysis force_mode = mode try: if config["General"]["force_tailcut_for_extended_cleaning"] is True: force_mode = config["General"]["force_mode"] print("> Activate force-mode for cleaning!!!!") except: pass # force_mode = mode self.cleaner_extended = ImageCleaner( # for energy/score estimation config=config["ImageCleaning"]["extended"], mode=force_mode) # Image book keeping self.image_cutflow = image_cutflow or CutFlow("ImageCutFlow") # Add quality cuts on images charge_bounds = config["ImageSelection"]["charge"] npix_bounds = config["ImageSelection"]["pixel"] ellipticity_bounds = config["ImageSelection"]["ellipticity"] nominal_distance_bounds = config["ImageSelection"]["nominal_distance"] self.camera_radius = { "LSTCam": 1.126, "NectarCam": 1.126, } # Average between max(xpix) and max(ypix), in meter self.image_cutflow.set_cuts( OrderedDict([ ("noCuts", None), ("min pixel", lambda s: np.count_nonzero(s) < npix_bounds[0]), ("min charge", lambda x: x < charge_bounds[0]), # ("poor moments", lambda m: m.width <= 0 or m.length <= 0 or np.isnan(m.width) or np.isnan(m.length)), # TBC, maybe we loose events without nan conditions ("poor moments", lambda m: m.width <= 0 or m.length <= 0), ( "bad ellipticity", lambda m: (m.width / m.length) < ellipticity_bounds[0] or (m.width / m.length) > ellipticity_bounds[-1], ), # ("close to the edge", lambda m, cam_id: m.r.value > (nominal_distance_bounds[-1] * 1.12949101073069946)) # in meter ( "close to the edge", lambda m, cam_id: m.r.value > (nominal_distance_bounds[-1] * self.camera_radius[cam_id]), ), # in meter ])) # Reconstruction self.shower_reco = HillasReconstructor() # Event book keeping self.event_cutflow = event_cutflow or CutFlow("EventCutFlow") # Add cuts on events min_ntel = config["Reconstruction"]["min_tel"] self.event_cutflow.set_cuts( OrderedDict([ ("noCuts", None), ("min2Tels trig", lambda x: x < min_ntel), ("min2Tels reco", lambda x: x < min_ntel), ("direction nan", lambda x: x.is_valid == False), ])) def prepare_event(self, source, return_stub=False): # configuration for the camera calibrator # modifies the integration window to be more like in MARS # JLK, only for LST!!!! # Option for integration correction is done above cfg = Config() cfg["ChargeExtractorFactory"]["window_width"] = 5 cfg["ChargeExtractorFactory"]["window_shift"] = 2 self.calib = CameraCalibrator( config=cfg, extractor_product="LocalPeakIntegrator", eventsource=source, tool=None, ) for event in source: self.event_cutflow.count("noCuts") if self.event_cutflow.cut("min2Tels trig", len(event.dl0.tels_with_data)): if return_stub: yield stub(event) else: continue # calibrate the event self.calib.calibrate(event) # telescope loop tot_signal = 0 max_signals = {} n_pixel_dict = {} hillas_dict_reco = {} # for geometry hillas_dict = {} # for discrimination n_tels = { "tot": len(event.dl0.tels_with_data), "LST": 0, "MST": 0, "SST": 0, } n_cluster_dict = {} impact_dict_reco = {} # impact distance measured in tilt system point_azimuth_dict = {} point_altitude_dict = {} # To compute impact parameter in tilt system run_array_direction = event.mcheader.run_array_direction az, alt = run_array_direction[0], run_array_direction[1] ground_frame = GroundFrame() for tel_id in event.dl0.tels_with_data: self.image_cutflow.count("noCuts") camera = event.inst.subarray.tel[tel_id].camera # count the current telescope according to its size tel_type = event.inst.subarray.tel[tel_id].optics.tel_type # JLK, N telescopes before cut selection are not really interesting for # discrimination, too much fluctuations # n_tels[tel_type] += 1 # the camera image as a 1D array and stuff needed for calibration # Choose gain according to pywicta's procedure image_1d = simtel_event_to_images(event=event, tel_id=tel_id, ctapipe_format=True) pmt_signal = image_1d.input_image # calibrated image # clean the image try: with warnings.catch_warnings(): # Image with biggest cluster (reco cleaning) image_biggest = self.cleaner_reco.clean_image( pmt_signal, camera) image_biggest2d = geometry_converter.image_1d_to_2d( image_biggest, camera.cam_id) image_biggest2d = filter_pixels_clusters( image_biggest2d) image_biggest = geometry_converter.image_2d_to_1d( image_biggest2d, camera.cam_id) # Image for score/energy estimation (with clusters) image_extended = self.cleaner_extended.clean_image( pmt_signal, camera) except FileNotFoundError as e: # JLK, WHAT? print(e) continue # Apply some selection if self.image_cutflow.cut("min pixel", image_biggest): continue if self.image_cutflow.cut("min charge", np.sum(image_biggest)): continue # For cluster counts image_2d = geometry_converter.image_1d_to_2d( image_extended, camera.cam_id) n_cluster_dict[ tel_id] = pixel_clusters.number_of_pixels_clusters( array=image_2d, threshold=0) # could this go into `hillas_parameters` ...? max_signals[tel_id] = np.max(image_extended) # do the hillas reconstruction of the images # QUESTION should this change in numpy behaviour be done here # or within `hillas_parameters` itself? # JLK: make selection on biggest cluster with np.errstate(invalid="raise", divide="raise"): try: moments_reco = hillas_parameters( camera, image_biggest) # for geometry (eg direction) moments = hillas_parameters( camera, image_extended ) # for discrimination and energy reconstruction # if width and/or length are zero (e.g. when there is only only one # pixel or when all pixel are exactly in one row), the # parametrisation won't be very useful: skip if self.image_cutflow.cut("poor moments", moments_reco): # print('poor moments') continue if self.image_cutflow.cut("close to the edge", moments_reco, camera.cam_id): # print('close to the edge') continue if self.image_cutflow.cut("bad ellipticity", moments_reco): # print('bad ellipticity: w={}, l={}'.format(moments_reco.width, moments_reco.length)) continue except (FloatingPointError, hillas.HillasParameterizationError): continue point_azimuth_dict[ tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad point_altitude_dict[ tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad n_tels[tel_type] += 1 hillas_dict[tel_id] = moments hillas_dict_reco[tel_id] = moments_reco n_pixel_dict[tel_id] = len(np.where(image_extended > 0)[0]) tot_signal += moments.intensity n_tels["reco"] = len(hillas_dict_reco) n_tels["discri"] = len(hillas_dict) if self.event_cutflow.cut("min2Tels reco", n_tels["reco"]): if return_stub: yield stub(event) else: continue try: with warnings.catch_warnings(): warnings.simplefilter("ignore") # Reconstruction results reco_result = self.shower_reco.predict( hillas_dict_reco, event.inst, point_altitude_dict, point_azimuth_dict, ) # shower_sys = TiltedGroundFrame(pointing_direction=HorizonFrame( # az=reco_result.az, # alt=reco_result.alt # )) # Impact parameter for energy estimation (/ tel) subarray = event.inst.subarray for tel_id in hillas_dict.keys(): pos = subarray.positions[tel_id] tel_ground = SkyCoord(pos[0], pos[1], pos[2], frame=ground_frame) # tel_tilt = tel_ground.transform_to(shower_sys) core_ground = SkyCoord( reco_result.core_x, reco_result.core_y, 0 * u.m, frame=ground_frame, ) # core_tilt = core_ground.transform_to(shower_sys) # Should be better handled (tilted frame) impact_dict_reco[tel_id] = np.sqrt( (core_ground.x - tel_ground.x)**2 + (core_ground.y - tel_ground.y)**2) except Exception as e: print("exception in reconstruction:", e) raise if return_stub: yield stub(event) else: continue if self.event_cutflow.cut("direction nan", reco_result): if return_stub: yield stub(event) else: continue yield PreparedEvent( event=event, n_pixel_dict=n_pixel_dict, hillas_dict=hillas_dict, hillas_dict_reco=hillas_dict_reco, n_tels=n_tels, tot_signal=tot_signal, max_signals=max_signals, n_cluster_dict=n_cluster_dict, reco_result=reco_result, impact_dict=impact_dict_reco, )
class PrepareList(Cutter): ''' Prepare a feature list to save to table. It takes an event, does the calibration, image cleaning, parametrization and reconstruction. From this some basic features will be extracted and written to the file which later on can be used for training of the classifiers or energy regressors. test ''' true_az = {} true_alt = {} max_signal = {} tot_signal = 0 impact = {} def __init__(self, event, telescope_list, camera_types, ChargeExtration, pe_thresh, min_neighbors, tail_thresholds, DirReco, quality_cuts, LUT=None): super().__init__() ''' Parmeters --------- event : ctapipe event container calibrator : ctapipe camera calibrator reconstructor : ctapipe hillas reconstructor telescope_list : list with telescope configuration or "all" pe_thresh : dict with thresholds for gain selection tail_thresholds : dict with thresholds for image cleaning quality_cuts : dict containing quality cuts canera_types : list with camera types to analyze ''' self.event = event self.telescope_list = telescope_list self.pe_thresh = pe_thresh self.min_neighbors = min_neighbors self.tail_thresholds = tail_thresholds self.quality_cuts = quality_cuts self.camera_types = camera_types self.dirreco = DirReco self.LUTgenerator = LUT if (self.dirreco["weights"] == "LUT") | (self.dirreco["weights"] == "doublepass"): self.weights = {} else: self.weights = None self.hillas_dict = {} self.camera_dict = {} self.edge_pixels = {} # configurations for calibrator cfg = Config() cfg["ChargeExtractorFactory"]["product"] = \ ChargeExtration["ChargeExtractorProduct"] cfg['WaveformCleanerFactory']['product'] = \ ChargeExtration["WaveformCleanerProduct"] self.calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator", config=cfg) # calibration self.reconstructor = HillasReconstructor() # direction def get_impact(self, hillas_dict): ''' calculate impact parameters for all telescopes that were used for parametrization. Paremeters ---------- hillas_dict : dict with hillas HillasParameterContainers Returnes -------- impact : impact parameter or NaN if calculation failed ''' # check if event was prepared before try: assert self.reco_result except AssertionError: self.prepare() impact = {} for tel_id in hillas_dict.keys(): try: pred_core = np.array([ self.reco_result.core_x.value, self.reco_result.core_y.value ]) * u.m # tel_coords start at 0 instead of 1... tel_position = np.array([ self.event.inst.subarray.tel_coords[tel_id - 1].x.value, self.event.inst.subarray.tel_coords[tel_id - 1].y.value ]) * u.m impact[tel_id] = linalg.length(pred_core - tel_position) except AttributeError: impact[tel_id] = np.nan return impact def get_offangle(self, tel_id, direction="reco"): ''' Get the angular offset between the reconstructed direction and the pointing direction of the telescope. Parameters ---------- tel_id : integer Telecope ID true_off : string if "mc", the true MC direction is taken for calculation. Otherwise, if "reco" the reconstructed value will be taken Returns ------- off_angles : dictionary dictionary with tel_ids as keys and the offangle as entries. ''' if direction == "reco": off_angle = angular_separation( self.event.mc.tel[tel_id].azimuth_raw * u.rad, self.event.mc.tel[tel_id].altitude_raw * u.rad, self.reco_result.az, self.reco_result.alt) elif direction == "mc": off_angle = angular_separation( self.event.mc.tel[tel_id].azimuth_raw * u.rad, self.event.mc.tel[tel_id].altitude_raw * u.rad, self.event.mc.az, self.event.mc.alt) return off_angle def get_weight(self, method, camera, tel_id, hillas_par, offangle=None): """ Get the weighting for HillasReconustructor. Possible methods are 'default', which will fall back to the standard weighting applied in capipe, 'LUT' which will take the weights from a LUT and 'doublepass' which might be used for diffuse simulations. In this case it returns the weights for the first pass. method : sting method to get the weighting. camera: CameraDescription tel_id: integer hillas_par: HillasParameterContainer """ if method == "default": pass elif method == "LUT": if np.isnan(hillas_par.width) & (not np.isnan(hillas_par.length)): hillas_par.width = 0 * u.m self.weights[tel_id] = self.LUTgenerator.get_weight_from_LUT( hillas_par, camera.cam_id, min_stat=self.dirreco["min_stat"], ratio_cut=self.dirreco["wl_ratio_cut"][camera.cam_id]) elif method == "doublepass": # first pass with default weighting if np.isnan(hillas_par.width) & (not np.isnan(hillas_par.length)): hillas_par.width = 0 * u.m self.weights[tel_id] = hillas_par.intensity * ( 1 * u.m + hillas_par.length) / (1 * u.m + hillas_par.width) elif method == "second_pass": # weights for second pass self.weights[ tel_id] = self.LUTgenerator.get_weight_from_diffuse_LUT( self.hillas_dict[tel_id], offangle, camera.cam_id, min_stat=self.dirreco["min_stat"], ratio_cut=self.dirreco["wl_ratio_cut"][camera.cam_id]) else: raise KeyError("Weighting method {} not known.".format(method)) def prepare(self): ''' Prepare event performimng calibration, image cleaning, hillas parametrization, hillas intersection for the single event. Additionally, the impact distance will be calculated. ''' tels_per_type = {} no_weight = [] # calibrate event self.calibrator.calibrate(self.event) # loop over all telescopeswith data in it for tel_id in self.event.r0.tels_with_data: # check if telescope is selected for analysis # This also could be done already in event_source when reading th data if (tel_id in self.telescope_list) | (self.telescope_list == "all"): pass else: continue # get camera information camera = self.event.inst.subarray.tel[tel_id].camera self.camera_dict[tel_id] = camera image = self.event.dl1.tel[tel_id].image if camera.cam_id in self.pe_thresh.keys(): image, select = pick_gain_channel( image, self.pe_thresh[camera.cam_id], True) else: image = np.squeeze(image) # image cleaning mask = tailcuts_clean( camera, image, picture_thresh=self.tail_thresholds[camera.cam_id][1], boundary_thresh=self.tail_thresholds[camera.cam_id][0], min_number_picture_neighbors=self.min_neighbors) # go to next telescope if no pixels survived cleaning if not any(mask): continue cleaned_image = np.copy(image) cleaned_image[~mask] = 0 # calculate the hillas parameters hillas_par = hillas_parameters(camera, cleaned_image) # quality cuts leakage = leakage = self.leakage_cut( camera=camera, hillas_parameters=hillas_par, radius=self.quality_cuts["leakage_cut"]["radius"], max_dist=self.quality_cuts["leakage_cut"]["dist"], image=cleaned_image, rows=self.quality_cuts["leakage_cut"]["rows"], fraction=self.quality_cuts["leakage_cut"]["frac"], method=self.quality_cuts["leakage_cut"]["method"], ) size = self.size_cut(hillas_par, self.quality_cuts["size"]) if not (leakage & size): # size or leakage cuts not passed continue # get the weighting for HillasReconstructor try: self.get_weight(self.dirreco["weights"], camera, tel_id, hillas_par) except LookupFailedError: # this telescope will be ignored, should only happen for method LUT here no_weight.append(tel_id) continue self.hillas_dict[tel_id] = hillas_par self.max_signal[tel_id] = np.max(cleaned_image) # brightest pix try: tels_per_type[camera.cam_id].append(tel_id) except KeyError: tels_per_type[camera.cam_id] = [tel_id] try: assert tels_per_type except AssertionError: raise TooFewTelescopesException("No image survived the leakage " "or size cuts.") # collect some additional information for tel_id in self.hillas_dict: self.tot_signal += self.hillas_dict[tel_id].intensity # total size self.true_az[ tel_id] = self.event.mc.tel[tel_id].azimuth_raw * u.rad self.true_alt[ tel_id] = self.event.mc.tel[tel_id].altitude_raw * u.rad # wil raise exception if cut was not passed # self.multiplicity_cut(self.quality_cuts["multiplicity"]["cuts"], # tels_per_type, method=self.quality_cuts["multiplicity"]["method"]) if self.dirreco["weights"] == "LUT": # remove telescopes withough weights print("Removed {} of {} telescopes due LUT problems".format( len(no_weight), len(self.hillas_dict) + len(no_weight))) # do Hillas reconstruction self.reco_result = self.reconstructor.predict(self.hillas_dict, self.event.inst, self.true_alt, self.true_az, ext_weight=self.weights) if self.dirreco["weights"] == "doublepass": # take the reconstructed direction to get an estimate of the offangle and # get weights from the second pass from the diffuse LUT. self.weights = {} # reset the weights from earlier no_weight = [] for tel_id in self.hillas_dict: predicted_offangle = self.get_offangle(tel_id, direction="reco") predicted_offangle = predicted_offangle.to(u.deg).value camera = self.camera_dict[tel_id] # reload camera_information # get the weighting for HillasReconstructor try: self.get_weight("second_pass", camera, tel_id, self.hillas_dict[tel_id], predicted_offangle) except LookupFailedError: no_weight.append(tel_id) print("Removed {} of {} telescopes due LUT problems".format( len(no_weight), len(self.hillas_dict))) # remove those types from tels_per_type for tel_id in no_weight: del self.hillas_dict[tel_id] for cam_id in tels_per_type: if tel_id in tels_per_type[cam_id]: index = np.where( np.array(tels_per_type[cam_id]) == tel_id) tels_per_type[cam_id].pop(int( index[0])) # remove from list # redo the multiplicity cut to check if it still fulfilled # self.multiplicity_cut(self.quality_cuts["multiplicity"]["cuts"], tels_per_type, # method=self.quality_cuts["multiplicity"]["method"]) # do the second pass with new weights self.reco_result = self.reconstructor.predict( self.hillas_dict, self.event.inst, self.true_alt, self.true_az, ext_weight=self.weights) # Number of telescopes triggered per type self.n_tels_per_type = { tel: len(tels_per_type[tel]) for tel in tels_per_type } for tel_id in self.hillas_dict: try: agree = self.mc_offset == self.get_offangle(tel_id, direction="mc") except AttributeError: self.mc_offset = self.get_offangle(tel_id, direction="mc") continue if not agree: raise ValueError( "The pointing of the telescopes seems to be different.") self.impact = self.get_impact(self.hillas_dict) # impact parameter def get_reconstructed_parameters(self): ''' Return the parameters for writing to table. Returns ------- prepared parameters : impact max_signal tot_signal n_tels_per_type hillas_dict mc_offangle reco_result ''' # check if event was prepared before try: assert self.impact except AssertionError: self.prepare() return (self.impact, self.max_signal, self.tot_signal, self.n_tels_per_type, self.hillas_dict, self.mc_offset, self.reco_result)
def __init__(self, config, mode, event_cutflow=None, image_cutflow=None): """Initiliaze an EventPreparer object.""" # Cleaning for reconstruction self.cleaner_reco = ImageCleaner( # for reconstruction config=config["ImageCleaning"]["biggest"], mode=mode) # Cleaning for energy/score estimation # Add possibility to force energy/score cleaning with tailcut analysis force_mode = mode try: if config["General"]["force_tailcut_for_extended_cleaning"] is True: force_mode = config["General"]["force_mode"] print("> Activate force-mode for cleaning!!!!") except: pass # force_mode = mode self.cleaner_extended = ImageCleaner( # for energy/score estimation config=config["ImageCleaning"]["extended"], mode=force_mode) # Image book keeping self.image_cutflow = image_cutflow or CutFlow("ImageCutFlow") # Add quality cuts on images charge_bounds = config["ImageSelection"]["charge"] npix_bounds = config["ImageSelection"]["pixel"] ellipticity_bounds = config["ImageSelection"]["ellipticity"] nominal_distance_bounds = config["ImageSelection"]["nominal_distance"] self.camera_radius = { "LSTCam": 1.126, "NectarCam": 1.126, } # Average between max(xpix) and max(ypix), in meters self.image_cutflow.set_cuts( OrderedDict([ ("noCuts", None), ("min pixel", lambda s: np.count_nonzero(s) < npix_bounds[0]), ("min charge", lambda x: x < charge_bounds[0]), # ("poor moments", lambda m: m.width <= 0 or m.length <= 0 or np.isnan(m.width) or np.isnan(m.length)), # TBC, maybe we loose events without nan conditions ("poor moments", lambda m: m.width <= 0 or m.length <= 0), ( "bad ellipticity", lambda m: (m.width / m.length) < ellipticity_bounds[0] or (m.width / m.length) > ellipticity_bounds[-1], ), # ("close to the edge", lambda m, cam_id: m.r.value > (nominal_distance_bounds[-1] * 1.12949101073069946)) # in meter ( "close to the edge", lambda m, cam_id: m.r.value > (nominal_distance_bounds[-1] * self.camera_radius[cam_id]), ), # in meter ])) # configuration for the camera calibrator # modifies the integration window to be more like in MARS # JLK, only for LST!!!! cfg = Config() cfg["ChargeExtractorFactory"]["window_width"] = 5 cfg["ChargeExtractorFactory"]["window_shift"] = 2 extractor = LocalPeakWindowSum(config=cfg) self.calib = CameraCalibrator(config=cfg, image_extractor=extractor) # Reconstruction self.shower_reco = HillasReconstructor() # Event book keeping self.event_cutflow = event_cutflow or CutFlow("EventCutFlow") # Add cuts on events min_ntel = config["Reconstruction"]["min_tel"] self.event_cutflow.set_cuts( OrderedDict([ ("noCuts", None), ("min2Tels trig", lambda x: x < min_ntel), ("min2Tels reco", lambda x: x < min_ntel), ("direction nan", lambda x: x.is_valid == False), ]))
class EventPreparer: """Class which loop on events and returns results stored in container. The Class has several purposes. First of all, it prepares the images of the event that will be further use for reconstruction by applying calibration, cleaning and selection. Then, it reconstructs the geometry of the event and then returns image (e.g. Hillas parameters)and event information (e.g. results of the reconstruction). Parameters ---------- config: dict Configuration with analysis parameters mode: str Mode of the reconstruction, e.g. tail or wave event_cutflow: ctapipe.utils.CutFlow Statistic of events processed image_cutflow: ctapipe.utils.CutFlow Statistic of images processed Returns: dict Dictionnary of results """ def __init__(self, config, mode, event_cutflow=None, image_cutflow=None): """Initiliaze an EventPreparer object.""" # Cleaning for reconstruction self.cleaner_reco = ImageCleaner( # for reconstruction config=config["ImageCleaning"]["biggest"], mode=mode) # Cleaning for energy/score estimation # Add possibility to force energy/score cleaning with tailcut analysis force_mode = mode try: if config["General"]["force_tailcut_for_extended_cleaning"] is True: force_mode = config["General"]["force_mode"] print("> Activate force-mode for cleaning!!!!") except: pass # force_mode = mode self.cleaner_extended = ImageCleaner( # for energy/score estimation config=config["ImageCleaning"]["extended"], mode=force_mode) # Image book keeping self.image_cutflow = image_cutflow or CutFlow("ImageCutFlow") # Add quality cuts on images charge_bounds = config["ImageSelection"]["charge"] npix_bounds = config["ImageSelection"]["pixel"] ellipticity_bounds = config["ImageSelection"]["ellipticity"] nominal_distance_bounds = config["ImageSelection"]["nominal_distance"] self.camera_radius = { "LSTCam": 1.126, "NectarCam": 1.126, } # Average between max(xpix) and max(ypix), in meters self.image_cutflow.set_cuts( OrderedDict([ ("noCuts", None), ("min pixel", lambda s: np.count_nonzero(s) < npix_bounds[0]), ("min charge", lambda x: x < charge_bounds[0]), # ("poor moments", lambda m: m.width <= 0 or m.length <= 0 or np.isnan(m.width) or np.isnan(m.length)), # TBC, maybe we loose events without nan conditions ("poor moments", lambda m: m.width <= 0 or m.length <= 0), ( "bad ellipticity", lambda m: (m.width / m.length) < ellipticity_bounds[0] or (m.width / m.length) > ellipticity_bounds[-1], ), # ("close to the edge", lambda m, cam_id: m.r.value > (nominal_distance_bounds[-1] * 1.12949101073069946)) # in meter ( "close to the edge", lambda m, cam_id: m.r.value > (nominal_distance_bounds[-1] * self.camera_radius[cam_id]), ), # in meter ])) # configuration for the camera calibrator # modifies the integration window to be more like in MARS # JLK, only for LST!!!! cfg = Config() cfg["ChargeExtractorFactory"]["window_width"] = 5 cfg["ChargeExtractorFactory"]["window_shift"] = 2 extractor = LocalPeakWindowSum(config=cfg) self.calib = CameraCalibrator(config=cfg, image_extractor=extractor) # Reconstruction self.shower_reco = HillasReconstructor() # Event book keeping self.event_cutflow = event_cutflow or CutFlow("EventCutFlow") # Add cuts on events min_ntel = config["Reconstruction"]["min_tel"] self.event_cutflow.set_cuts( OrderedDict([ ("noCuts", None), ("min2Tels trig", lambda x: x < min_ntel), ("min2Tels reco", lambda x: x < min_ntel), ("direction nan", lambda x: x.is_valid == False), ])) def prepare_event(self, source, return_stub=False, save_images=False): for event in source: self.event_cutflow.count("noCuts") if self.event_cutflow.cut("min2Tels trig", len(event.dl0.tels_with_data)): if return_stub: yield stub(event) else: continue self.calib(event) # telescope loop tot_signal = 0 dl1_phe_image = None mc_phe_image = None max_signals = {} n_pixel_dict = {} hillas_dict_reco = {} # for direction reconstruction hillas_dict = {} # for discrimination n_tels = { "tot": len(event.dl0.tels_with_data), "LST_LST_LSTCam": 0, "MST_MST_NectarCam": 0, "SST": 0, # add later correct names when testing on Paranal } n_cluster_dict = {} impact_dict_reco = {} # impact distance measured in tilt system point_azimuth_dict = {} point_altitude_dict = {} # Compute impact parameter in tilt system run_array_direction = event.mcheader.run_array_direction az, alt = run_array_direction[0], run_array_direction[1] ground_frame = GroundFrame() for tel_id in event.dl0.tels_with_data: self.image_cutflow.count("noCuts") camera = event.inst.subarray.tel[tel_id].camera # count the current telescope according to its size tel_type = str(event.inst.subarray.tel[tel_id]) # use ctapipe's functionality to get the calibrated image pmt_signal = event.dl1.tel[tel_id].image # Save the calibrated image after the gain has been chosen # automatically by ctapipe, together with the simulated one if save_images is True: dl1_phe_image = pmt_signal mc_phe_image = event.mc.tel[tel_id].photo_electron_image if self.cleaner_reco.mode == "tail": # tail uses only ctapipe # Cleaning used for direction reconstruction image_biggest, mask_reco = self.cleaner_reco.clean_image( pmt_signal, camera) # find all islands using this cleaning num_islands, labels = number_of_islands(camera, mask_reco) if num_islands == 1: # if only ONE islands is left ... # ...use directly the old mask and reduce dimensions # to make Hillas parametrization faster camera_biggest = camera[mask_reco] image_biggest = image_biggest[mask_reco] elif num_islands > 1: # if more islands survived.. # ...find the biggest one mask_biggest = largest_island(labels) # and also reduce dimensions camera_biggest = camera[mask_biggest] image_biggest = image_biggest[mask_biggest] else: # if no islands survived use old camera and image camera_biggest = camera # Cleaning used for score/energy estimation image_extended, mask_extended = self.cleaner_extended.clean_image( pmt_signal, camera) # find all islands with this cleaning # we will also register how many have been found n_cluster_dict[tel_id], labels = number_of_islands( camera, mask_extended) # NOTE: the next check shouldn't be necessary if we keep # all the isolated pixel clusters, but for now the # two cleanings are set the same in analysis.yml because # the performance of the extended one has never been really # studied in model estimation. # (This is a nice way to ask for volunteers :P) # if some islands survived if num_islands > 0: # keep all of them and reduce dimensions camera_extended = camera[mask_extended] image_extended = image_extended[mask_extended] else: # otherwise continue with the old camera and image camera_extended = camera # could this go into `hillas_parameters` ...? # this is basically the charge of ALL islands # not calculated later by the Hillas parametrization! max_signals[tel_id] = np.max(image_extended) else: # for wavelets we stick to old pywi-cta code try: # "try except FileNotFoundError" not clear to me, but for now it stays... with warnings.catch_warnings(): # Image with biggest cluster (reco cleaning) image_biggest, mask_reco = self.cleaner_reco.clean_image( pmt_signal, camera) image_biggest2d = geometry_converter.image_1d_to_2d( image_biggest, camera.cam_id) image_biggest2d = filter_pixels_clusters( image_biggest2d) image_biggest = geometry_converter.image_2d_to_1d( image_biggest2d, camera.cam_id) # Image for score/energy estimation (with clusters) image_extended, mask_extended = self.cleaner_extended.clean_image( pmt_signal, camera) # This last part was outside the pywi-cta block # before, but is indeed part of it because it uses # pywi-cta functions in the "extended" case # For cluster counts image_2d = geometry_converter.image_1d_to_2d( image_extended, camera.cam_id) n_cluster_dict[ tel_id] = pixel_clusters.number_of_pixels_clusters( array=image_2d, threshold=0) # could this go into `hillas_parameters` ...? max_signals[tel_id] = np.max(image_extended) except FileNotFoundError as e: # JLK, WHAT? print(e) continue # ============================================================== # Apply some selection if self.image_cutflow.cut("min pixel", image_biggest): continue if self.image_cutflow.cut("min charge", np.sum(image_biggest)): continue # do the hillas reconstruction of the images # QUESTION should this change in numpy behaviour be done here # or within `hillas_parameters` itself? # JLK: make selection on biggest cluster with np.errstate(invalid="raise", divide="raise"): try: moments_reco = hillas_parameters( camera_biggest, image_biggest) # for geometry (eg direction) moments = hillas_parameters( camera_extended, image_extended ) # for discrimination and energy reconstruction # if width and/or length are zero (e.g. when there is # only only one pixel or when all pixel are exactly # in one row), the parametrisation # won't be very useful: skip if self.image_cutflow.cut("poor moments", moments_reco): continue if self.image_cutflow.cut("close to the edge", moments_reco, camera.cam_id): continue if self.image_cutflow.cut("bad ellipticity", moments_reco): continue except (FloatingPointError, hillas.HillasParameterizationError): continue point_azimuth_dict[ tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad point_altitude_dict[ tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad n_tels[tel_type] += 1 hillas_dict[tel_id] = moments hillas_dict_reco[tel_id] = moments_reco n_pixel_dict[tel_id] = len(np.where(image_extended > 0)[0]) tot_signal += moments.intensity n_tels["reco"] = len(hillas_dict_reco) n_tels["discri"] = len(hillas_dict) if self.event_cutflow.cut("min2Tels reco", n_tels["reco"]): if return_stub: yield stub(event) else: continue try: with warnings.catch_warnings(): warnings.simplefilter("ignore") # Reconstruction results reco_result = self.shower_reco.predict( hillas_dict_reco, event.inst, SkyCoord(alt=alt, az=az, frame="altaz"), { tel_id: SkyCoord( alt=point_altitude_dict[tel_id], az=point_azimuth_dict[tel_id], frame="altaz", ) # cycle only on tels which still have an image for tel_id in point_altitude_dict.keys() }, ) # Impact parameter for energy estimation (/ tel) subarray = event.inst.subarray for tel_id in hillas_dict.keys(): pos = subarray.positions[tel_id] tel_ground = SkyCoord(pos[0], pos[1], pos[2], frame=ground_frame) core_ground = SkyCoord( reco_result.core_x, reco_result.core_y, 0 * u.m, frame=ground_frame, ) # Should be better handled (tilted frame) impact_dict_reco[tel_id] = np.sqrt( (core_ground.x - tel_ground.x)**2 + (core_ground.y - tel_ground.y)**2) except Exception as e: print("exception in reconstruction:", e) raise if return_stub: yield stub(event) else: continue if self.event_cutflow.cut("direction nan", reco_result): if return_stub: yield stub(event) else: continue yield PreparedEvent( event=event, dl1_phe_image=dl1_phe_image, mc_phe_image=mc_phe_image, n_pixel_dict=n_pixel_dict, hillas_dict=hillas_dict, hillas_dict_reco=hillas_dict_reco, n_tels=n_tels, tot_signal=tot_signal, max_signals=max_signals, n_cluster_dict=n_cluster_dict, reco_result=reco_result, impact_dict=impact_dict_reco, )
def main(): # your favourite units here energy_unit = u.TeV angle_unit = u.deg dist_unit = u.m agree_threshold = .5 min_tel = 3 parser = make_argparser() parser.add_argument('--classifier', type=str, default=expandvars( "$CTA_SOFT/tino_cta/data/classifier_pickle/" "classifier_{mode}_{cam_id}_{classifier}.pkl")) parser.add_argument('--regressor', type=str, default=expandvars( "$CTA_SOFT/tino_cta/data/classifier_pickle/" "regressor_{mode}_{cam_id}_{regressor}.pkl")) parser.add_argument('-o', '--outfile', type=str, default="", help="location to write the classified events to.") parser.add_argument('--wave_dir', type=str, default=None, help="directory where to find mr_filter. " "if not set look in $PATH") parser.add_argument( '--wave_temp_dir', type=str, default='/dev/shm/', help="directory where mr_filter to store the temporary fits " "files") group = parser.add_mutually_exclusive_group() group.add_argument('--proton', action='store_true', help="do protons instead of gammas") group.add_argument('--electron', action='store_true', help="do electrons instead of gammas") args = parser.parse_args() if args.infile_list: filenamelist = [] for f in args.infile_list: filenamelist += glob("{}/{}".format(args.indir, f)) filenamelist.sort() elif args.proton: filenamelist = sorted(glob("{}/proton/*gz".format(args.indir))) elif args.electron: filenamelist = glob("{}/electron/*gz".format(args.indir)) channel = "electron" else: filenamelist = sorted(glob("{}/gamma/*gz".format(args.indir))) if not filenamelist: print("no files found; check indir: {}".format(args.indir)) exit(-1) # keeping track of events and where they were rejected Eventcutflow = CutFlow("EventCutFlow") Imagecutflow = CutFlow("ImageCutFlow") # takes care of image cleaning cleaner = ImageCleaner(mode=args.mode, cutflow=Imagecutflow, wavelet_options=args.raw, tmp_files_directory=args.wave_temp_dir, skip_edge_events=False, island_cleaning=True) # the class that does the shower reconstruction shower_reco = HillasReconstructor() preper = EventPreparer( cleaner=cleaner, hillas_parameters=hillas_parameters, shower_reco=shower_reco, event_cutflow=Eventcutflow, image_cutflow=Imagecutflow, # event/image cuts: allowed_cam_ids=[], min_ntel=2, min_charge=args.min_charge, min_pixel=3) # wrapper for the scikit-learn classifier classifier = EventClassifier.load(args.classifier.format( **{ "mode": args.mode, "wave_args": "mixed", "classifier": 'RandomForestClassifier', "cam_id": "{cam_id}" }), cam_id_list=args.cam_ids) # wrapper for the scikit-learn regressor regressor = EnergyRegressor.load(args.regressor.format( **{ "mode": args.mode, "wave_args": "mixed", "regressor": "RandomForestRegressor", "cam_id": "{cam_id}" }), cam_id_list=args.cam_ids) ClassifierFeatures = namedtuple( "ClassifierFeatures", ("impact_dist", "sum_signal_evt", "max_signal_cam", "sum_signal_cam", "N_LST", "N_MST", "N_SST", "width", "length", "skewness", "kurtosis", "h_max", "err_est_pos", "err_est_dir")) EnergyFeatures = namedtuple( "EnergyFeatures", ("impact_dist", "sum_signal_evt", "max_signal_cam", "sum_signal_cam", "N_LST", "N_MST", "N_SST", "width", "length", "skewness", "kurtosis", "h_max", "err_est_pos", "err_est_dir")) # catch ctr-c signal to exit current loop and still display results signal_handler = SignalHandler() signal.signal(signal.SIGINT, signal_handler) # this class defines the reconstruction parameters to keep track of class RecoEvent(tb.IsDescription): Run_ID = tb.Int16Col(dflt=-1, pos=0) Event_ID = tb.Int16Col(dflt=-1, pos=1) NTels_trig = tb.Int16Col(dflt=0, pos=0) NTels_reco = tb.Int16Col(dflt=0, pos=1) NTels_reco_lst = tb.Int16Col(dflt=0, pos=2) NTels_reco_mst = tb.Int16Col(dflt=0, pos=3) NTels_reco_sst = tb.Int16Col(dflt=0, pos=4) MC_Energy = tb.Float32Col(dflt=np.nan, pos=5) reco_Energy = tb.Float32Col(dflt=np.nan, pos=6) reco_phi = tb.Float32Col(dflt=np.nan, pos=7) reco_theta = tb.Float32Col(dflt=np.nan, pos=8) off_angle = tb.Float32Col(dflt=np.nan, pos=9) xi = tb.Float32Col(dflt=np.nan, pos=10) DeltaR = tb.Float32Col(dflt=np.nan, pos=11) ErrEstPos = tb.Float32Col(dflt=np.nan, pos=12) ErrEstDir = tb.Float32Col(dflt=np.nan, pos=13) gammaness = tb.Float32Col(dflt=np.nan, pos=14) success = tb.BoolCol(dflt=False, pos=15) channel = "gamma" if "gamma" in " ".join(filenamelist) else "proton" reco_outfile = tb.open_file( mode="w", # if no outfile name is given (i.e. don't to write the event list to disk), # need specify two "driver" arguments **({ "filename": args.outfile } if args.outfile else { "filename": "no_outfile.h5", "driver": "H5FD_CORE", "driver_core_backing_store": False })) reco_table = reco_outfile.create_table("/", "reco_events", RecoEvent) reco_event = reco_table.row allowed_tels = None # all telescopes allowed_tels = prod3b_tel_ids("L+N+D") for i, filename in enumerate(filenamelist[:args.last]): # print(f"file: {i} filename = {filename}") source = hessio_event_source(filename, allowed_tels=allowed_tels, max_events=args.max_events) # loop that cleans and parametrises the images and performs the reconstruction for (event, hillas_dict, n_tels, tot_signal, max_signals, pos_fit, dir_fit, h_max, err_est_pos, err_est_dir) in preper.prepare_event(source, True): # now prepare the features for the classifier cls_features_evt = {} reg_features_evt = {} if hillas_dict is not None: for tel_id in hillas_dict.keys(): Imagecutflow.count("pre-features") tel_pos = np.array(event.inst.tel_pos[tel_id][:2]) * u.m moments = hillas_dict[tel_id] impact_dist = linalg.length(tel_pos - pos_fit) cls_features_tel = ClassifierFeatures( impact_dist=impact_dist / u.m, sum_signal_evt=tot_signal, max_signal_cam=max_signals[tel_id], sum_signal_cam=moments.size, N_LST=n_tels["LST"], N_MST=n_tels["MST"], N_SST=n_tels["SST"], width=moments.width / u.m, length=moments.length / u.m, skewness=moments.skewness, kurtosis=moments.kurtosis, h_max=h_max / u.m, err_est_pos=err_est_pos / u.m, err_est_dir=err_est_dir / u.deg) reg_features_tel = EnergyFeatures( impact_dist=impact_dist / u.m, sum_signal_evt=tot_signal, max_signal_cam=max_signals[tel_id], sum_signal_cam=moments.size, N_LST=n_tels["LST"], N_MST=n_tels["MST"], N_SST=n_tels["SST"], width=moments.width / u.m, length=moments.length / u.m, skewness=moments.skewness, kurtosis=moments.kurtosis, h_max=h_max / u.m, err_est_pos=err_est_pos / u.m, err_est_dir=err_est_dir / u.deg) if np.isnan(cls_features_tel).any() or np.isnan( reg_features_tel).any(): continue Imagecutflow.count("features nan") cam_id = event.inst.subarray.tel[tel_id].camera.cam_id try: reg_features_evt[cam_id] += [reg_features_tel] cls_features_evt[cam_id] += [cls_features_tel] except KeyError: reg_features_evt[cam_id] = [reg_features_tel] cls_features_evt[cam_id] = [cls_features_tel] if cls_features_evt and reg_features_evt: predict_energ = regressor.predict_by_event([reg_features_evt ])["mean"][0] predict_proba = classifier.predict_proba_by_event( [cls_features_evt]) gammaness = predict_proba[0, 0] try: # the MC direction of origin of the simulated particle shower = event.mc shower_core = np.array( [shower.core_x / u.m, shower.core_y / u.m]) * u.m shower_org = linalg.set_phi_theta(az_to_phi(shower.az), alt_to_theta(shower.alt)) # and how the reconstructed direction compares to that xi = linalg.angle(dir_fit, shower_org) DeltaR = linalg.length(pos_fit[:2] - shower_core) except Exception: # naked exception catch, because I'm not sure where # it would break in non-MC files xi = np.nan DeltaR = np.nan phi, theta = linalg.get_phi_theta(dir_fit) phi = (phi if phi > 0 else phi + 360 * u.deg) # TODO: replace with actual array pointing direction array_pointing = linalg.set_phi_theta(0 * u.deg, 20. * u.deg) # angular offset between the reconstructed direction and the array # pointing off_angle = linalg.angle(dir_fit, array_pointing) reco_event["NTels_trig"] = len(event.dl0.tels_with_data) reco_event["NTels_reco"] = len(hillas_dict) reco_event["NTels_reco_lst"] = n_tels["LST"] reco_event["NTels_reco_mst"] = n_tels["MST"] reco_event["NTels_reco_sst"] = n_tels["SST"] reco_event["reco_Energy"] = predict_energ.to(energy_unit).value reco_event["reco_phi"] = phi / angle_unit reco_event["reco_theta"] = theta / angle_unit reco_event["off_angle"] = off_angle / angle_unit reco_event["xi"] = xi / angle_unit reco_event["DeltaR"] = DeltaR / dist_unit reco_event["ErrEstPos"] = err_est_pos / dist_unit reco_event["ErrEstDir"] = err_est_dir / angle_unit reco_event["gammaness"] = gammaness reco_event["success"] = True else: reco_event["success"] = False # save basic event infos reco_event["MC_Energy"] = event.mc.energy.to(energy_unit).value reco_event["Event_ID"] = event.r1.event_id reco_event["Run_ID"] = event.r1.run_id reco_table.flush() reco_event.append() if signal_handler.stop: break if signal_handler.stop: break # make sure everything gets written out nicely reco_table.flush() try: print() Eventcutflow() print() Imagecutflow() # do some simple event selection # and print the corresponding selection efficiency N_selected = len([ x for x in reco_table.where( """(NTels_reco > min_tel) & (gammaness > agree_threshold)""") ]) N_total = len(reco_table) print("\nfraction selected events:") print("{} / {} = {} %".format(N_selected, N_total, N_selected / N_total * 100)) except ZeroDivisionError: pass print("\nlength filenamelist:", len(filenamelist[:args.last])) # do some plotting if so desired if args.plot: gammaness = [x['gammaness'] for x in reco_table] NTels_rec = [x['NTels_reco'] for x in reco_table] NTel_bins = np.arange(np.min(NTels_rec), np.max(NTels_rec) + 2) - .5 NTels_rec_lst = [x['NTels_reco_lst'] for x in reco_table] NTels_rec_mst = [x['NTels_reco_mst'] for x in reco_table] NTels_rec_sst = [x['NTels_reco_sst'] for x in reco_table] reco_energy = np.array([x['reco_Energy'] for x in reco_table]) mc_energy = np.array([x['MC_Energy'] for x in reco_table]) fig = plt.figure(figsize=(15, 5)) plt.suptitle(" ** ".join( [args.mode, "protons" if args.proton else "gamma"])) plt.subplots_adjust(left=0.05, right=0.97, hspace=0.39, wspace=0.2) ax = plt.subplot(131) histo = np.histogram2d(NTels_rec, gammaness, bins=(NTel_bins, np.linspace(0, 1, 11)))[0].T histo_normed = histo / histo.max(axis=0) im = ax.imshow( histo_normed, interpolation='none', origin='lower', aspect='auto', # extent=(*NTel_bins[[0, -1]], 0, 1), cmap=plt.cm.inferno) ax.set_xlabel("NTels") ax.set_ylabel("drifted gammaness") plt.title("Total Number of Telescopes") # next subplot ax = plt.subplot(132) histo = np.histogram2d(NTels_rec_sst, gammaness, bins=(NTel_bins, np.linspace(0, 1, 11)))[0].T histo_normed = histo / histo.max(axis=0) im = ax.imshow( histo_normed, interpolation='none', origin='lower', aspect='auto', # extent=(*NTel_bins[[0, -1]], 0, 1), cmap=plt.cm.inferno) ax.set_xlabel("NTels") plt.setp(ax.get_yticklabels(), visible=False) plt.title("Number of SSTs") # next subplot ax = plt.subplot(133) histo = np.histogram2d(NTels_rec_mst, gammaness, bins=(NTel_bins, np.linspace(0, 1, 11)))[0].T histo_normed = histo / histo.max(axis=0) im = ax.imshow( histo_normed, interpolation='none', origin='lower', aspect='auto', # extent=(*NTel_bins[[0, -1]], 0, 1), cmap=plt.cm.inferno) cb = fig.colorbar(im, ax=ax) ax.set_xlabel("NTels") plt.setp(ax.get_yticklabels(), visible=False) plt.title("Number of MSTs") plt.subplots_adjust(wspace=0.05) # plot the energy migration matrix plt.figure() plt.hist2d(np.log10(reco_energy), np.log10(mc_energy), bins=20, cmap=plt.cm.inferno) plt.xlabel("E_MC / TeV") plt.ylabel("E_rec / TeV") plt.colorbar() plt.show()
Cleaner = { "w": ImageCleaner(mode="wave", cutflow=Imagecutflow, skip_edge_events=skip_edge_events, island_cleaning=island_cleaning, wavelet_options=args.raw), "t": ImageCleaner(mode="tail", cutflow=Imagecutflow, skip_edge_events=skip_edge_events, island_cleaning=island_cleaning) } # simple hillas-based shower reco fit = HillasReconstructor() signal_handler = SignalHandler() if args.plot_c: signal.signal(signal.SIGINT, signal_handler.stop_drawing) else: signal.signal(signal.SIGINT, signal_handler) # keeping track of the hit distribution transverse to the shower axis on the camera # for different energy bins from modules.Histogram import nDHistogram pe_vs_dp = {'p': {}, 'w': {}, 't': {}} for k in pe_vs_dp.keys(): pe_vs_dp[k] = nDHistogram( bin_edges=[np.arange(6), np.linspace(-.1, .1, 42) * u.m],
def test_reconstruction(): """ a test of the complete fit procedure on one event including: • tailcut cleaning • hillas parametrisation • HillasPlane creation • direction fit • position fit in the end, proper units in the output are asserted """ filename = get_dataset_path("gamma_test_large.simtel.gz") source = EventSource(filename, max_events=10) calib = CameraCalibrator(subarray=source.subarray) horizon_frame = AltAz() reconstructed_events = 0 for event in source: calib(event) mc = event.simulation.shower array_pointing = SkyCoord(az=mc.az, alt=mc.alt, frame=horizon_frame) hillas_dict = {} telescope_pointings = {} for tel_id, dl1 in event.dl1.tel.items(): geom = source.subarray.tel[tel_id].camera.geometry telescope_pointings[tel_id] = SkyCoord( alt=event.pointing.tel[tel_id].altitude, az=event.pointing.tel[tel_id].azimuth, frame=horizon_frame, ) mask = tailcuts_clean(geom, dl1.image, picture_thresh=10.0, boundary_thresh=5.0) try: moments = hillas_parameters(geom[mask], dl1.image[mask]) hillas_dict[tel_id] = moments except HillasParameterizationError as e: print(e) continue if len(hillas_dict) < 2: continue else: reconstructed_events += 1 # The three reconstructions below gives the same results fit = HillasReconstructor() fit_result_parall = fit.predict(hillas_dict, source.subarray, array_pointing) fit = HillasReconstructor() fit_result_tel_point = fit.predict(hillas_dict, source.subarray, array_pointing, telescope_pointings) for key in fit_result_parall.keys(): print(key, fit_result_parall[key], fit_result_tel_point[key]) fit_result_parall.alt.to(u.deg) fit_result_parall.az.to(u.deg) fit_result_parall.core_x.to(u.m) assert fit_result_parall.is_valid assert reconstructed_events > 0
# do some cross-validation now if args.check: # keeping track of events and where they were rejected Eventcutflow = CutFlow("EventCutFlow") Imagecutflow = CutFlow("ImageCutFlow") # takes care of image cleaning cleaner = ImageCleaner(mode=args.mode, cutflow=Imagecutflow, wavelet_options=args.raw, skip_edge_events=False, island_cleaning=True) # the class that does the shower reconstruction shower_reco = HillasReconstructor() preper = EventPreparer( cleaner=cleaner, shower_reco=shower_reco, event_cutflow=Eventcutflow, image_cutflow=Imagecutflow, # event/image cuts: allowed_cam_ids=[], # [] or None means: all min_ntel=2, min_charge=args.min_charge, min_pixel=3) Imagecutflow.add_cut("features nan", lambda x: np.isnan(x).any()) energy_mc = [] energy_rec = []
def test_invalid_events(): """ The HillasReconstructor is supposed to fail in these cases: - less than two teleskopes - any width is NaN - any width is 0 This test uses the same sample simtel file as test_reconstruction(). As there are no invalid events in this file, multiple hillas_dicts are constructed to make sure Exceptions get thrown in the mentioned edge cases. Test will fail if no Exception or another Exception gets thrown.""" filename = get_dataset_path("gamma_test_large.simtel.gz") fit = HillasReconstructor() tel_azimuth = {} tel_altitude = {} source = EventSource(filename, max_events=10) subarray = source.subarray calib = CameraCalibrator(subarray) for event in source: calib(event) hillas_dict = {} for tel_id, dl1 in event.dl1.tel.items(): geom = source.subarray.tel[tel_id].camera.geometry tel_azimuth[tel_id] = event.pointing.tel[tel_id].azimuth tel_altitude[tel_id] = event.pointing.tel[tel_id].altitude mask = tailcuts_clean(geom, dl1.image, picture_thresh=10.0, boundary_thresh=5.0) try: moments = hillas_parameters(geom[mask], dl1.image[mask]) hillas_dict[tel_id] = moments except HillasParameterizationError as e: continue # construct a dict only containing the last telescope events # (#telescopes < 2) hillas_dict_only_one_tel = dict() hillas_dict_only_one_tel[tel_id] = hillas_dict[tel_id] with pytest.raises(TooFewTelescopesException): fit.predict(hillas_dict_only_one_tel, subarray, tel_azimuth, tel_altitude) # construct a hillas dict with the width of the last event set to 0 # (any width == 0) hillas_dict_zero_width = hillas_dict.copy() hillas_dict_zero_width[tel_id]["width"] = 0 * u.m with pytest.raises(InvalidWidthException): fit.predict(hillas_dict_zero_width, subarray, tel_azimuth, tel_altitude) # construct a hillas dict with the width of the last event set to np.nan # (any width == nan) hillas_dict_nan_width = hillas_dict.copy() hillas_dict_zero_width[tel_id]["width"] = np.nan * u.m with pytest.raises(InvalidWidthException): fit.predict(hillas_dict_nan_width, subarray, tel_azimuth, tel_altitude)
def main(): # your favourite units here energy_unit = u.TeV angle_unit = u.deg dist_unit = u.m parser = make_argparser() parser.add_argument( '-o', '--outfile', type=str, help="if given, write output file with reconstruction results") parser.add_argument('--plot_c', action='store_true', help="plot camera-wise displays") group = parser.add_mutually_exclusive_group() group.add_argument('--proton', action='store_true', help="do protons instead of gammas") group.add_argument('--electron', action='store_true', help="do electrons instead of gammas") args = parser.parse_args() if args.infile_list: filenamelist = [] for f in args.infile_list: filenamelist += glob("{}/{}".format(args.indir, f)) elif args.proton: filenamelist = glob("{}/proton/*gz".format(args.indir)) channel = "proton" elif args.electron: filenamelist = glob("{}/electron/*gz".format(args.indir)) channel = "electron" elif args.gamma: filenamelist = glob("{}/gamma/*gz".format(args.indir)) channel = "gamma" else: raise ValueError("don't know which input to use...") filenamelist.sort() if not filenamelist: print("no files found; check indir: {}".format(args.indir)) exit(-1) else: print("found {} files".format(len(filenamelist))) tel_phi = {} tel_theta = {} # keeping track of events and where they were rejected Eventcutflow = CutFlow("EventCutFlow") Imagecutflow = CutFlow("ImageCutFlow") # takes care of image cleaning cleaner = ImageCleaner(mode=args.mode, cutflow=Imagecutflow, wavelet_options=args.raw, skip_edge_events=args.skip_edge_events, island_cleaning=True) # the class that does the shower reconstruction shower_reco = HillasReconstructor() shower_max_estimator = ShowerMaxEstimator("paranal") preper = EventPreparer( cleaner=cleaner, hillas_parameters=hillas_parameters, shower_reco=shower_reco, event_cutflow=Eventcutflow, image_cutflow=Imagecutflow, # event/image cuts: allowed_cam_ids=[], # means: all min_ntel=3, min_charge=args.min_charge, min_pixel=3) # a signal handler to abort the event loop but still do the post-processing signal_handler = SignalHandler() signal.signal(signal.SIGINT, signal_handler) try: # this class defines the reconstruction parameters to keep track of class RecoEvent(tb.IsDescription): NTels_trigg = tb.Int16Col(dflt=1, pos=0) NTels_clean = tb.Int16Col(dflt=1, pos=1) EnMC = tb.Float32Col(dflt=1, pos=2) xi = tb.Float32Col(dflt=1, pos=3) DeltaR = tb.Float32Col(dflt=1, pos=4) ErrEstPos = tb.Float32Col(dflt=1, pos=5) ErrEstDir = tb.Float32Col(dflt=1, pos=6) h_max = tb.Float32Col(dflt=1, pos=7) reco_outfile = tb.open_file( args.outfile, mode="w", # if we don't want to write the event list to disk, need to add more arguments **({} if args.store else { "driver": "H5FD_CORE", "driver_core_backing_store": False })) reco_table = reco_outfile.create_table("/", "reco_event", RecoEvent) reco_event = reco_table.row except: reco_event = RecoEvent() print("no pytables installed?") # ## ####### ####### ######## # ## ## ## ## ## ## ## # ## ## ## ## ## ## ## # ## ## ## ## ## ######## # ## ## ## ## ## ## # ## ## ## ## ## ## # ######## ####### ####### ## cam_id_map = {} # define here which telescopes to loop over allowed_tels = None # allowed_tels = prod3b_tel_ids("L+F+D") for i, filename in enumerate(filenamelist[:args.last]): print("file: {i} filename = {filename}".format(i=i, filename=filename)) source = hessio_event_source(filename, allowed_tels=allowed_tels, max_events=args.max_events) # loop that cleans and parametrises the images and performs the reconstruction for (event, hillas_dict, n_tels, tot_signal, max_signal, pos_fit, dir_fit, h_max, err_est_pos, err_est_dir) in preper.prepare_event(source): shower = event.mc org_alt = u.Quantity(shower.alt).to(u.deg) org_az = u.Quantity(shower.az).to(u.deg) if org_az > 180 * u.deg: org_az -= 360 * u.deg org_the = alt_to_theta(org_alt) org_phi = az_to_phi(org_az) if org_phi > 180 * u.deg: org_phi -= 360 * u.deg if org_phi < -180 * u.deg: org_phi += 360 * u.deg shower_org = linalg.set_phi_theta(org_phi, org_the) shower_core = convert_astropy_array([shower.core_x, shower.core_y]) xi = linalg.angle(dir_fit, shower_org).to(angle_unit) diff = linalg.length(pos_fit[:2] - shower_core) # print some performance print() print("xi = {:4.3f}".format(xi)) print("pos = {:4.3f}".format(diff)) print("h_max reco: {:4.3f}".format(h_max.to(u.km))) print("err_est_dir: {:4.3f}".format(err_est_dir.to(angle_unit))) print("err_est_pos: {:4.3f}".format(err_est_pos)) try: # store the reconstruction data in the PyTable reco_event["NTels_trigg"] = n_tels["tot"] reco_event["NTels_clean"] = len(shower_reco.circles) reco_event["EnMC"] = event.mc.energy / energy_unit reco_event["xi"] = xi / angle_unit reco_event["DeltaR"] = diff / dist_unit reco_event["ErrEstPos"] = err_est_pos / dist_unit reco_event["ErrEstDir"] = err_est_dir / angle_unit reco_event["h_max"] = h_max / dist_unit reco_event.append() reco_table.flush() print() print("xi res (68-percentile) = {:4.3f} {}".format( np.percentile(reco_table.cols.xi, 68), angle_unit)) print("core res (68-percentile) = {:4.3f} {}".format( np.percentile(reco_table.cols.DeltaR, 68), dist_unit)) print("h_max (median) = {:4.3f} {}".format( np.percentile(reco_table.cols.h_max, 50), dist_unit)) except NoPyTables: pass if args.plot_c: from mpl_toolkits.mplot3d import Axes3D fig = plt.figure() ax = fig.gca(projection='3d') for c in shower_reco.circles.values(): points = [ c.pos + t * c.a * u.km for t in np.linspace(0, 15, 3) ] ax.plot(*np.array(points).T, linewidth=np.sqrt(c.weight) / 10) ax.scatter(*c.pos[:, None].value, s=np.sqrt(c.weight)) plt.xlabel("x") plt.ylabel("y") plt.pause(.1) # this plots # • the MC shower core # • the reconstructed shower core # • the used telescopes # • and the trace of the Hillas plane on the ground plt.figure() for tel_id, c in shower_reco.circles.items(): plt.scatter(c.pos[0], c.pos[1], s=np.sqrt(c.weight)) plt.gca().annotate(tel_id, (c.pos[0].value, c.pos[1].value)) plt.plot([ c.pos[0].value - 500 * c.norm[1], c.pos[0].value + 500 * c.norm[1] ], [ c.pos[1].value + 500 * c.norm[0], c.pos[1].value - 500 * c.norm[0] ], linewidth=np.sqrt(c.weight) / 10) plt.scatter(*pos_fit[:2], c="black", marker="*", label="fitted") plt.scatter(*shower_core[:2], c="black", marker="P", label="MC") plt.legend() plt.xlabel("x") plt.ylabel("y") plt.xlim(-1400, 1400) plt.ylim(-1400, 1400) plt.show() if signal_handler.stop: break if signal_handler.stop: break print("\n" + "=" * 35 + "\n") print("xi res (68-percentile) = {:4.3f} {}".format( np.percentile(reco_table.cols.xi, 68), angle_unit)) print("core res (68-percentile) = {:4.3f} {}".format( np.percentile(reco_table.cols.DeltaR, 68), dist_unit)) print("h_max (median) = {:4.3f} {}".format( np.percentile(reco_table.cols.h_max, 50), dist_unit)) # print the cutflows for telescopes and camera images print("\n\n") Eventcutflow("min2Tels trig") print() Imagecutflow(sort_column=1) # if we don't want to plot anything, we can exit now if not args.plot: return # ######## ## ####### ######## ###### # ## ## ## ## ## ## ## ## # ## ## ## ## ## ## ## # ######## ## ## ## ## ###### # ## ## ## ## ## ## # ## ## ## ## ## ## ## # ## ######## ####### ## ###### plt.figure() plt.hist(reco_table.cols.h_max, bins=np.linspace(000, 15000, 51, True)) plt.title(channel) plt.xlabel("h_max reco") plt.pause(.1) figure = plt.figure() xi_edges = np.linspace(0, 5, 20) plt.hist(reco_table.cols.xi, bins=xi_edges, log=True) plt.xlabel(r"$\xi$ / deg") if args.write: save_fig('{}/reco_xi_{}'.format(args.plots_dir, args.mode)) plt.pause(.1) plt.figure() plt.hist(reco_table.cols.ErrEstDir[:], bins=np.linspace(0, 20, 50)) plt.title(channel) plt.xlabel("beta") plt.pause(.1) plt.figure() plt.hist(np.log10(reco_table.cols.xi[:] / reco_table.cols.ErrEstDir[:]), bins=50) plt.title(channel) plt.xlabel("log_10(xi / beta)") plt.pause(.1) # convert the xi-list into a dict with the number of used telescopes as keys xi_vs_tel = {} for xi, ntel in zip(reco_table.cols.xi, reco_table.cols.NTels_clean): if ntel not in xi_vs_tel: xi_vs_tel[ntel] = [xi] else: xi_vs_tel[ntel].append(xi) print(args.mode) for ntel, xis in sorted(xi_vs_tel.items()): print("NTel: {} -- median xi: {}".format(ntel, np.median(xis))) # print("histogram:", np.histogram(xis, bins=xi_edges)) # create a list of energy bin-edges and -centres for violin plots Energy_edges = np.linspace(2, 8, 13) Energy_centres = (Energy_edges[1:] + Energy_edges[:-1]) / 2. # convert the xi-list in to an energy-binned dict with the bin centre as keys xi_vs_energy = {} for en, xi in zip(reco_table.cols.EnMC, reco_table.cols.xi): # get the bin number this event belongs into sbin = np.digitize(np.log10(en), Energy_edges) - 1 # the central value of the bin is the key for the dictionary if Energy_centres[sbin] not in xi_vs_energy: xi_vs_energy[Energy_centres[sbin]] = [xi] else: xi_vs_energy[Energy_centres[sbin]] += [xi] # plotting the angular error as violin plots with binning in # number of telescopes and shower energy figure = plt.figure() plt.subplot(211) plt.violinplot([np.log10(a) for a in xi_vs_tel.values()], [a for a in xi_vs_tel.keys()], points=60, widths=.75, showextrema=False, showmedians=True) plt.xlabel("Number of Telescopes") plt.ylabel(r"log($\xi$ / deg)") plt.ylim(-3, 2) plt.grid() plt.subplot(212) plt.violinplot([np.log10(a) for a in xi_vs_energy.values()], [a for a in xi_vs_energy.keys()], points=60, widths=(Energy_edges[1] - Energy_edges[0]) / 1.5, showextrema=False, showmedians=True) plt.xlabel(r"log(Energy / GeV)") plt.ylabel(r"log($\xi$ / deg)") plt.ylim(-3, 2) plt.grid() plt.tight_layout() if args.write: save_fig('{}/reco_xi_vs_E_NTel_{}'.format(args.plots_dir, args.mode)) plt.pause(.1) # convert the diffs-list into a dict with the number of used telescopes as keys diff_vs_tel = {} for diff, ntel in zip(reco_table.cols.DeltaR, reco_table.cols.NTels_clean): if ntel not in diff_vs_tel: diff_vs_tel[ntel] = [diff] else: diff_vs_tel[ntel].append(diff) # convert the diffs-list in to an energy-binned dict with the bin centre as keys diff_vs_energy = {} for en, diff in zip(reco_table.cols.EnMC, reco_table.cols.DeltaR): # get the bin number this event belongs into sbin = np.digitize(np.log10(en), Energy_edges) - 1 # the central value of the bin is the key for the dictionary if Energy_centres[sbin] not in diff_vs_energy: diff_vs_energy[Energy_centres[sbin]] = [diff] else: diff_vs_energy[Energy_centres[sbin]] += [diff] # plotting the core position error as violin plots with binning in # number of telescopes an shower energy plt.figure() plt.subplot(211) plt.violinplot([np.log10(a) for a in diff_vs_tel.values()], [a for a in diff_vs_tel.keys()], points=60, widths=.75, showextrema=False, showmedians=True) plt.xlabel("Number of Telescopes") plt.ylabel(r"log($\Delta R$ / m)") plt.grid() plt.subplot(212) plt.violinplot([np.log10(a) for a in diff_vs_energy.values()], [a for a in diff_vs_energy.keys()], points=60, widths=(Energy_edges[1] - Energy_edges[0]) / 1.5, showextrema=False, showmedians=True) plt.xlabel(r"log(Energy / GeV)") plt.ylabel(r"log($\Delta R$ / m)") plt.grid() plt.tight_layout() if args.write: save_fig('{}/reco_dist_vs_E_NTel_{}'.format(args.plots_dir, args.mode)) plt.show()
def test_invalid_events(): """ The HillasReconstructor is supposed to fail in these cases: - less than two teleskopes - any width is NaN - any width is 0 This test uses the same sample simtel file as test_reconstruction(). As there are no invalid events in this file, multiple hillas_dicts are constructed to make sure Exceptions get thrown in the mentioned edge cases. Test will fail if no Exception or another Exception gets thrown.""" filename = get_dataset_path("gamma_test_large.simtel.gz") fit = HillasReconstructor() tel_azimuth = {} tel_altitude = {} source = event_source(filename, max_events=10) for event in source: hillas_dict = {} for tel_id in event.dl0.tels_with_data: geom = event.inst.subarray.tel[tel_id].camera tel_azimuth[tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad tel_altitude[tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad pmt_signal = event.r0.tel[tel_id].waveform[0].sum(axis=1) mask = tailcuts_clean(geom, pmt_signal, picture_thresh=10., boundary_thresh=5.) pmt_signal[mask == 0] = 0 try: moments = hillas_parameters(geom, pmt_signal) hillas_dict[tel_id] = moments except HillasParameterizationError as e: continue # construct a dict only containing the last telescope events # (#telescopes < 2) hillas_dict_only_one_tel = dict() hillas_dict_only_one_tel[tel_id] = hillas_dict[tel_id] with pytest.raises(TooFewTelescopesException): fit.predict(hillas_dict_only_one_tel, event.inst, tel_azimuth, tel_altitude) # construct a hillas dict with the width of the last event set to 0 # (any width == 0) hillas_dict_zero_width = hillas_dict.copy() hillas_dict_zero_width[tel_id]['width'] = 0 * u.m with pytest.raises(InvalidWidthException): fit.predict(hillas_dict_zero_width, event.inst, tel_azimuth, tel_altitude) # construct a hillas dict with the width of the last event set to np.nan # (any width == nan) hillas_dict_nan_width = hillas_dict.copy() hillas_dict_zero_width[tel_id]['width'] = np.nan * u.m with pytest.raises(InvalidWidthException): fit.predict(hillas_dict_nan_width, event.inst, tel_azimuth, tel_altitude)
# importing data from avaiable datasets in ctapipe filename = datasets.get_dataset("gamma_test_large.simtel.gz") # filename # reading the Monte Carlo file for LST source = event_source(filename, allowed_tels={1, 2, 3, 4}) # pointing direction of the telescopes point_azimuth = {} point_altitude = {} reco = HillasReconstructor() calib = CameraCalibrator(r1_product="HESSIOR1Calibrator") off_angles = [] for event in source: # The direction the incident particle. # Converting Monte Carlo Shower parameter theta and phi to # corresponding to 3 components (x,y,z) of a vector shower_azimuth = event.mc.az # same as in Monte Carlo file i.e. phi shower_altitude = np.pi * u.rad / 2 - event.mc.alt # altitude = 90 - theta shower_direction = linalg.set_phi_theta(shower_azimuth, shower_altitude) # calibrating the event calib.calibrate(event) hillas_params = {} subarray = event.inst.subarray
def test_reconstruction(): """ a test of the complete fit procedure on one event including: • tailcut cleaning • hillas parametrisation • HillasPlane creation • direction fit • position fit in the end, proper units in the output are asserted """ filename = get_dataset_path("gamma_test_large.simtel.gz") source = event_source(filename, max_events=10) horizon_frame = AltAz() reconstructed_events = 0 for event in source: array_pointing = SkyCoord( az=event.mc.az, alt=event.mc.alt, frame=horizon_frame ) hillas_dict = {} telescope_pointings = {} for tel_id in event.dl0.tels_with_data: geom = event.inst.subarray.tel[tel_id].camera telescope_pointings[tel_id] = SkyCoord(alt=event.mc.tel[tel_id].altitude_raw * u.rad, az=event.mc.tel[tel_id].azimuth_raw * u.rad, frame=horizon_frame) pmt_signal = event.r0.tel[tel_id].waveform[0].sum(axis=1) mask = tailcuts_clean(geom, pmt_signal, picture_thresh=10., boundary_thresh=5.) pmt_signal[mask == 0] = 0 try: moments = hillas_parameters(geom, pmt_signal) hillas_dict[tel_id] = moments except HillasParameterizationError as e: print(e) continue if len(hillas_dict) < 2: continue else: reconstructed_events += 1 # The three reconstructions below gives the same results fit = HillasReconstructor() fit_result_parall = fit.predict(hillas_dict, event.inst, array_pointing) fit = HillasReconstructor() fit_result_tel_point = fit.predict(hillas_dict, event.inst, array_pointing, telescope_pointings) for key in fit_result_parall.keys(): print(key, fit_result_parall[key], fit_result_tel_point[key]) fit_result_parall.alt.to(u.deg) fit_result_parall.az.to(u.deg) fit_result_parall.core_x.to(u.m) assert fit_result_parall.is_valid assert reconstructed_events > 0