def test_camera_calibrator(example_event): telid = list(example_event.r0.tel)[0] calibrator = CameraCalibrator() calibrator.calibrate(example_event) image = example_event.dl1.tel[telid].image assert image is not None
def test_camera_calibrator(example_event): telid = 11 calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator") calibrator.calibrate(example_event) image = example_event.dl1.tel[telid].image assert_allclose(image[0, 0], -2.216, 1e-3)
def test_camera_calibrator(test_event): event = deepcopy(test_event) # so we don't modify the test event telid = 11 calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator") calibrator.calibrate(event) image = event.dl1.tel[telid].image assert_allclose(image[0, 0], -2.216, 1e-3)
def test_camera_calibrator(): event = get_test_event() telid = 11 calibrator = CameraCalibrator(None, None) calibrator.calibrate(event) image = event.dl1.tel[telid].image assert_allclose(image[0, 0], -2.216, 1e-3)
def analyze_ctapipe(): print("starting ctapipe.") gamma = FileReader( "/lustre/fs21/group/cta/prod3b/prod3b-paranal20deg/gamma_onSource/gamma_20deg_0deg_run624___cta-prod3_desert-2150m-Paranal-merged.simtel.gz", "gamma") gamma.read_files() trace_width = { "ASTRICam": (10, 5), "FlashCam": (4, 2), "LSTCam": (4, 2), "NectarCam": (6, 3), "DigiCam": (4, 2), "CHEC": (8, 4), "SCTCam": (6, 3) } # from Tino pe_thresh = {"ASTRICam": 14, "LSTCam": 100, "NectarCam": 190} #integrators = ['GlobalPeakIntegrator', 'LocalPeakIntegrator', 'NeighbourPeakIntegrator', 'AverageWfPeakIntegrator'] #cleaners = ['NullWaveformCleaner', 'CHECMWaveformCleanerAverage', 'CHECMWaveformCleanerLocal'] for integrator in ['NeighbourPeakIntegrator']: i = 0 for event in gamma.source: i += 1 if not event.r0.event_id in event_IDs: # continue with next event continue else: for tel in event.r0.tels_with_data: camera = event.inst.subarray.tel[tel].camera cfg = Config() cfg["ChargeExtractorFactory"]["product"] = integrator cfg["ChargeExtractorFactory"][ "window_width"] = trace_width[camera.cam_id][0] cfg["ChargeExtractorFactory"][ "window_shift"] = trace_width[camera.cam_id][1] event.dl1.tel[tel].reset() # set up calibrator. calibrator = CameraCalibrator( r1_product="HESSIOR1Calibrator", config=cfg) calibrator.calibrate(event) ##################################### # Tino's solution for gain selection if (camera.cam_id == np.array(list( pe_thresh.keys()))).any(): image = event.dl1.tel[tel].image image = pick_gain_channel(image, camera.cam_id, pe_thresh) else: image = event.dl1.tel[tel].image image = np.reshape(image[0], np.shape(image)[1]) # image = np.reshape(image[0], np.shape(image)[1]) EventID = [int(event.r0.event_id)] * len(image) TelescopeID = [int(tel)] * len(image) cameras = [event.inst.subarray.tel[tel].camera.cam_id ] * len(image) PixelID = np.arange(len(image)) charge = image new_df = pd.DataFrame(np.transpose( [EventID, TelescopeID, PixelID, charge, cameras]), columns=[ "EventID", "TelescopeID", "PixelID", "charge", "cameras" ]) try: ctapipe_charge = ctapipe_charge.append( new_df, ignore_index=True) except (TypeError, NameError): ctapipe_charge = new_df print("{:.2f}% finished".format((i / len(event_IDs)) * 100)) ctapipe_charge.to_csv('ctapipe_charge.csv', sep=",") return ctapipe_charge
class DisplayDL1Calib(Tool): name = "DisplayDL1Calib" description = "Calibrate dl0 data to dl1, and plot the photoelectron " \ "images." telescope = Int(None, allow_none=True, help='Telescope to view. Set to None to display all ' 'telescopes.').tag(config=True) aliases = Dict( dict(f='EventFileReaderFactory.input_path', r='EventFileReaderFactory.reader', max_events='EventFileReaderFactory.max_events', extractor='ChargeExtractorFactory.extractor', window_width='ChargeExtractorFactory.window_width', t0='ChargeExtractorFactory.t0', window_shift='ChargeExtractorFactory.window_shift', sig_amp_cut_HG='ChargeExtractorFactory.sig_amp_cut_HG', sig_amp_cut_LG='ChargeExtractorFactory.sig_amp_cut_LG', lwt='ChargeExtractorFactory.lwt', clip_amplitude='CameraDL1Calibrator.clip_amplitude', T='DisplayDL1Calib.telescope', O='ImagePlotter.output_path')) flags = Dict( dict(D=({ 'ImagePlotter': { 'display': True } }, "Display the photoelectron images on-screen as they " "are produced."))) classes = List([ EventFileReaderFactory, ChargeExtractorFactory, CameraDL1Calibrator, ImagePlotter ]) def __init__(self, **kwargs): super().__init__(**kwargs) self.reader = None self.calibrator = None self.plotter = None def setup(self): self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]" kwargs = dict(config=self.config, tool=self) reader_factory = EventFileReaderFactory(**kwargs) reader_class = reader_factory.get_class() self.reader = reader_class(**kwargs) self.calibrator = CameraCalibrator(origin=self.reader.origin, **kwargs) self.plotter = ImagePlotter(**kwargs) def start(self): source = self.reader.read() for event in source: self.calibrator.calibrate(event) tel_list = event.r0.tels_with_data if self.telescope: if self.telescope not in tel_list: continue tel_list = [self.telescope] for telid in tel_list: self.plotter.plot(event, telid) def finish(self): self.plotter.finish()
class DisplayDL1Calib(Tool): name = "DisplayDL1Calib" description = "Calibrate dl0 data to dl1, and plot the photoelectron " \ "images." telescope = Int(None, allow_none=True, help='Telescope to view. Set to None to display all ' 'telescopes.').tag(config=True) aliases = Dict(dict(f='EventFileReaderFactory.input_path', r='EventFileReaderFactory.reader', max_events='EventFileReaderFactory.max_events', extractor='ChargeExtractorFactory.extractor', window_width='ChargeExtractorFactory.window_width', t0='ChargeExtractorFactory.t0', window_shift='ChargeExtractorFactory.window_shift', sig_amp_cut_HG='ChargeExtractorFactory.sig_amp_cut_HG', sig_amp_cut_LG='ChargeExtractorFactory.sig_amp_cut_LG', lwt='ChargeExtractorFactory.lwt', clip_amplitude='CameraDL1Calibrator.clip_amplitude', T='DisplayDL1Calib.telescope', O='ImagePlotter.output_path' )) flags = Dict(dict(D=({'ImagePlotter': {'display': True}}, "Display the photoelectron images on-screen as they " "are produced.") )) classes = List([EventFileReaderFactory, ChargeExtractorFactory, CameraDL1Calibrator, ImagePlotter ]) def __init__(self, **kwargs): super().__init__(**kwargs) self.reader = None self.calibrator = None self.plotter = None def setup(self): self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]" kwargs = dict(config=self.config, tool=self) reader_factory = EventFileReaderFactory(**kwargs) reader_class = reader_factory.get_class() self.reader = reader_class(**kwargs) self.calibrator = CameraCalibrator(origin=self.reader.origin, **kwargs) self.plotter = ImagePlotter(**kwargs) def start(self): source = self.reader.read() for event in source: self.calibrator.calibrate(event) tel_list = event.r0.tels_with_data if self.telescope: if self.telescope not in tel_list: continue tel_list = [self.telescope] for telid in tel_list: self.plotter.plot(event, telid) def finish(self): self.plotter.finish()
def prepare(self, particles, hillas=True, subarray=np.arange(600)): ''' Performs the processing of the images: calibration perfoming conversion from r0 to dl1 image cleaning using tailcut_cleaning image parametrization with Hillas parameters Fills the dicts for clean_images and hillas moments ''' # Input ####### # particles - array of konsta_cta.readdata.FileReader # self.particles = particles self.geoms, self.geoms_unique = self.get_camera_geoms() # loop through all particle types for particle in self.particles: dtype = particle.datatype # count number of images and number after cleaning self.sum_images[dtype] = 0 self.sum_clean_images[dtype] = 0 # dict to store the hisograms self.histograms = {} # loop through all events for event in particle.source: event_id = event.dl0.event_id n_images = 0 n_clean_images = 0 # count for each camera individually images_cams = {} clean_images_cams = {} self.mc_energy[dtype, event_id] = event.mc.energy # loop through all telescopes with data for tel_id in event.r0.tels_with_data: if type(subarray[0]) != str: subarray = [str(x) for x in subarray] if str(tel_id) in subarray: pass else: continue n_images += 1 # get camera information camera = event.inst.subarray.tel[tel_id].camera try: # check if key already exists. if not initialize it with value 0 _im = images_cams[camera.cam_id] _im = clean_images_cams[camera.cam_id] except KeyError: images_cams[camera.cam_id] = 0 clean_images_cams[camera.cam_id] = 0 # count for each camera individually images_cams[camera.cam_id] += 1 cfg = Config() cfg["ChargeExtractorFactory"]["product"] = self.integrator cfg["ChargeExtractorFactory"][ "window_width"] = self.trace_width[camera.cam_id][0] cfg["ChargeExtractorFactory"][ "window_shift"] = self.trace_width[camera.cam_id][1] cfg['WaveformCleanerFactory']['product'] = self.cleaner # usually all cameras are calibrated at once with the same camera. # In order to force a racalculation to use the differen width and # shift of the camera of this telescope, dl1 container will be reset # to default at each iteration. # #Probably it is not even needed to reset this, as the container #is refilled anyway. event.dl1.tel[tel_id].reset() # danger: The resulting calibrated event doesn't contain the right # extracted charges for all cameras in the end as it seems like # the dl1 images are overwirtten each time so that, the charges, # extracted at the last telescope iteration will be contained in the # dl1 container. # As I'm wirting out all the required information for the telescope # within this loop, this should not be much of a problem for now, but # in the future a appropriate way to work arround this is required. # set up calibrator. calibrator = CameraCalibrator(r1_product=self.r1, config=cfg) # calibrate event calibrator.calibrate(event) ###################################### # create output for comparison plots # ###################################### cam_id = camera.cam_id number_gains = event.dl1.tel[tel_id].image.shape[0] # create dict for the camera on first appearance try: _hist = self.histograms[cam_id] except KeyError: self.histograms[cam_id] = {} # fill histograms and merge for all events dependent on gain for gain, label in zip(range(number_gains), ["gain1", "gain2"]): image = np.array(event.dl1.tel[tel_id].image[gain]) # only positive charges hist_lower = len(image[image > np.power(10., -1.)]) hist_higher = len(image[image > np.power(10., 4.)]) image = image[image > 0] logimage = np.log10(image) hist = np.histogram(logimage, range=[-1, 4], bins=100) # store the values outside of range for sanity check _hist = np.append(hist_lower, hist[0]) _hist = np.append(_hist, hist_higher) _bins = np.append(-1000, hist[1]) _bins = np.append(_bins, 1000) hist = (_hist, _bins) try: self.histograms[cam_id][label] = ( self.histograms[cam_id][label][0] + hist[0], self.histograms[cam_id][label][1]) except KeyError: self.histograms[cam_id][label] = hist ##################################### # Tino's solution for gain selection if (camera.cam_id == np.array(list( self.pe_thresh.keys()))).any(): image = event.dl1.tel[tel_id].image image = self.pick_gain_channel(image, camera.cam_id) else: image = event.dl1.tel[tel_id].image image = np.reshape(image[0], np.shape(image)[1]) ###################### # image cleaning # Threshold values adapted from Tino's repository mask = tailcuts_clean( self.geoms[tel_id], image, picture_thresh=self.tail_thresholds[camera.cam_id][1], boundary_thresh=self.tail_thresholds[camera.cam_id][0], min_number_picture_neighbors=0) try: temp_list except NameError: temp_list = [] if not (camera.cam_id in temp_list): temp_list.append(camera.cam_id) print("Threshold {camera}: {threshold}".format( camera=camera.cam_id, threshold=self.tail_thresholds[camera.cam_id])) number_pixels = np.count_nonzero(mask) # drop images that didn't survive image cleaning if any(mask == True): n_clean_images += 1 self.clean_images[dtype, event_id, tel_id] = np.copy(image) # set rejected pixels to zero self.clean_images[dtype, event_id, tel_id][~mask] = 0 # count for each camera individually try: clean_images_cams[camera.cam_id] += 1 except KeyError: clean_images_cams[camera.cam_id] = 1 ### hillas parametrization if hillas: hillas_moments = hillas_parameters( self.geoms[tel_id], self.clean_images[dtype, event_id, tel_id], True) self.number_pixels.append(number_pixels) self.energy.append(event.mc.energy.base) self.size.append(hillas_moments.intensity) self.length.append(hillas_moments.length) self.width.append(hillas_moments.width) self.skewness.append(hillas_moments.skewness) self.camera.append(camera.cam_id) self.core_x.append(event.mc.core_x.base) self.core_y.append(event.mc.core_y.base) else: pass # count number of images at trigge level and after cleaning # summary: self.sum_images[dtype] += n_images self.sum_clean_images[dtype] += n_clean_images # per event: self.core_x2.append(event.mc.core_x.base) self.core_y2.append(event.mc.core_y.base) self.energy2.append(event.mc.energy.base) self.n_images.append(float(n_images)) self.n_clean_images.append(float(n_clean_images)) for cam in images_cams.keys(): try: self.images_cams[cam].append(float(images_cams[cam])) self.clean_images_cams[cam].append( float(clean_images_cams[cam])) except: self.images_cams[cam] = [float(images_cams[cam])] self.clean_images_cams[cam] = [ float(clean_images_cams[cam]) ] print("Processed {} images for datatype {}. Images " "that didn't survive cleaning: {}".format( self.sum_images[dtype], dtype, self.sum_images[dtype] - self.sum_clean_images[dtype])) self.get_keys()
class PrepareList(Cutter): ''' Prepare a feature list to save to table. It takes an event, does the calibration, image cleaning, parametrization and reconstruction. From this some basic features will be extracted and written to the file which later on can be used for training of the classifiers or energy regressors. test ''' true_az = {} true_alt = {} max_signal = {} tot_signal = 0 impact = {} def __init__(self, event, telescope_list, camera_types, ChargeExtration, pe_thresh, min_neighbors, tail_thresholds, DirReco, quality_cuts, LUT=None): super().__init__() ''' Parmeters --------- event : ctapipe event container calibrator : ctapipe camera calibrator reconstructor : ctapipe hillas reconstructor telescope_list : list with telescope configuration or "all" pe_thresh : dict with thresholds for gain selection tail_thresholds : dict with thresholds for image cleaning quality_cuts : dict containing quality cuts canera_types : list with camera types to analyze ''' self.event = event self.telescope_list = telescope_list self.pe_thresh = pe_thresh self.min_neighbors = min_neighbors self.tail_thresholds = tail_thresholds self.quality_cuts = quality_cuts self.camera_types = camera_types self.dirreco = DirReco self.LUTgenerator = LUT if (self.dirreco["weights"] == "LUT") | (self.dirreco["weights"] == "doublepass"): self.weights = {} else: self.weights = None self.hillas_dict = {} self.camera_dict = {} self.edge_pixels = {} # configurations for calibrator cfg = Config() cfg["ChargeExtractorFactory"]["product"] = \ ChargeExtration["ChargeExtractorProduct"] cfg['WaveformCleanerFactory']['product'] = \ ChargeExtration["WaveformCleanerProduct"] self.calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator", config=cfg) # calibration self.reconstructor = HillasReconstructor() # direction def get_impact(self, hillas_dict): ''' calculate impact parameters for all telescopes that were used for parametrization. Paremeters ---------- hillas_dict : dict with hillas HillasParameterContainers Returnes -------- impact : impact parameter or NaN if calculation failed ''' # check if event was prepared before try: assert self.reco_result except AssertionError: self.prepare() impact = {} for tel_id in hillas_dict.keys(): try: pred_core = np.array([ self.reco_result.core_x.value, self.reco_result.core_y.value ]) * u.m # tel_coords start at 0 instead of 1... tel_position = np.array([ self.event.inst.subarray.tel_coords[tel_id - 1].x.value, self.event.inst.subarray.tel_coords[tel_id - 1].y.value ]) * u.m impact[tel_id] = linalg.length(pred_core - tel_position) except AttributeError: impact[tel_id] = np.nan return impact def get_offangle(self, tel_id, direction="reco"): ''' Get the angular offset between the reconstructed direction and the pointing direction of the telescope. Parameters ---------- tel_id : integer Telecope ID true_off : string if "mc", the true MC direction is taken for calculation. Otherwise, if "reco" the reconstructed value will be taken Returns ------- off_angles : dictionary dictionary with tel_ids as keys and the offangle as entries. ''' if direction == "reco": off_angle = angular_separation( self.event.mc.tel[tel_id].azimuth_raw * u.rad, self.event.mc.tel[tel_id].altitude_raw * u.rad, self.reco_result.az, self.reco_result.alt) elif direction == "mc": off_angle = angular_separation( self.event.mc.tel[tel_id].azimuth_raw * u.rad, self.event.mc.tel[tel_id].altitude_raw * u.rad, self.event.mc.az, self.event.mc.alt) return off_angle def get_weight(self, method, camera, tel_id, hillas_par, offangle=None): """ Get the weighting for HillasReconustructor. Possible methods are 'default', which will fall back to the standard weighting applied in capipe, 'LUT' which will take the weights from a LUT and 'doublepass' which might be used for diffuse simulations. In this case it returns the weights for the first pass. method : sting method to get the weighting. camera: CameraDescription tel_id: integer hillas_par: HillasParameterContainer """ if method == "default": pass elif method == "LUT": if np.isnan(hillas_par.width) & (not np.isnan(hillas_par.length)): hillas_par.width = 0 * u.m self.weights[tel_id] = self.LUTgenerator.get_weight_from_LUT( hillas_par, camera.cam_id, min_stat=self.dirreco["min_stat"], ratio_cut=self.dirreco["wl_ratio_cut"][camera.cam_id]) elif method == "doublepass": # first pass with default weighting if np.isnan(hillas_par.width) & (not np.isnan(hillas_par.length)): hillas_par.width = 0 * u.m self.weights[tel_id] = hillas_par.intensity * ( 1 * u.m + hillas_par.length) / (1 * u.m + hillas_par.width) elif method == "second_pass": # weights for second pass self.weights[ tel_id] = self.LUTgenerator.get_weight_from_diffuse_LUT( self.hillas_dict[tel_id], offangle, camera.cam_id, min_stat=self.dirreco["min_stat"], ratio_cut=self.dirreco["wl_ratio_cut"][camera.cam_id]) else: raise KeyError("Weighting method {} not known.".format(method)) def prepare(self): ''' Prepare event performimng calibration, image cleaning, hillas parametrization, hillas intersection for the single event. Additionally, the impact distance will be calculated. ''' tels_per_type = {} no_weight = [] # calibrate event self.calibrator.calibrate(self.event) # loop over all telescopeswith data in it for tel_id in self.event.r0.tels_with_data: # check if telescope is selected for analysis # This also could be done already in event_source when reading th data if (tel_id in self.telescope_list) | (self.telescope_list == "all"): pass else: continue # get camera information camera = self.event.inst.subarray.tel[tel_id].camera self.camera_dict[tel_id] = camera image = self.event.dl1.tel[tel_id].image if camera.cam_id in self.pe_thresh.keys(): image, select = pick_gain_channel( image, self.pe_thresh[camera.cam_id], True) else: image = np.squeeze(image) # image cleaning mask = tailcuts_clean( camera, image, picture_thresh=self.tail_thresholds[camera.cam_id][1], boundary_thresh=self.tail_thresholds[camera.cam_id][0], min_number_picture_neighbors=self.min_neighbors) # go to next telescope if no pixels survived cleaning if not any(mask): continue cleaned_image = np.copy(image) cleaned_image[~mask] = 0 # calculate the hillas parameters hillas_par = hillas_parameters(camera, cleaned_image) # quality cuts leakage = leakage = self.leakage_cut( camera=camera, hillas_parameters=hillas_par, radius=self.quality_cuts["leakage_cut"]["radius"], max_dist=self.quality_cuts["leakage_cut"]["dist"], image=cleaned_image, rows=self.quality_cuts["leakage_cut"]["rows"], fraction=self.quality_cuts["leakage_cut"]["frac"], method=self.quality_cuts["leakage_cut"]["method"], ) size = self.size_cut(hillas_par, self.quality_cuts["size"]) if not (leakage & size): # size or leakage cuts not passed continue # get the weighting for HillasReconstructor try: self.get_weight(self.dirreco["weights"], camera, tel_id, hillas_par) except LookupFailedError: # this telescope will be ignored, should only happen for method LUT here no_weight.append(tel_id) continue self.hillas_dict[tel_id] = hillas_par self.max_signal[tel_id] = np.max(cleaned_image) # brightest pix try: tels_per_type[camera.cam_id].append(tel_id) except KeyError: tels_per_type[camera.cam_id] = [tel_id] try: assert tels_per_type except AssertionError: raise TooFewTelescopesException("No image survived the leakage " "or size cuts.") # collect some additional information for tel_id in self.hillas_dict: self.tot_signal += self.hillas_dict[tel_id].intensity # total size self.true_az[ tel_id] = self.event.mc.tel[tel_id].azimuth_raw * u.rad self.true_alt[ tel_id] = self.event.mc.tel[tel_id].altitude_raw * u.rad # wil raise exception if cut was not passed # self.multiplicity_cut(self.quality_cuts["multiplicity"]["cuts"], # tels_per_type, method=self.quality_cuts["multiplicity"]["method"]) if self.dirreco["weights"] == "LUT": # remove telescopes withough weights print("Removed {} of {} telescopes due LUT problems".format( len(no_weight), len(self.hillas_dict) + len(no_weight))) # do Hillas reconstruction self.reco_result = self.reconstructor.predict(self.hillas_dict, self.event.inst, self.true_alt, self.true_az, ext_weight=self.weights) if self.dirreco["weights"] == "doublepass": # take the reconstructed direction to get an estimate of the offangle and # get weights from the second pass from the diffuse LUT. self.weights = {} # reset the weights from earlier no_weight = [] for tel_id in self.hillas_dict: predicted_offangle = self.get_offangle(tel_id, direction="reco") predicted_offangle = predicted_offangle.to(u.deg).value camera = self.camera_dict[tel_id] # reload camera_information # get the weighting for HillasReconstructor try: self.get_weight("second_pass", camera, tel_id, self.hillas_dict[tel_id], predicted_offangle) except LookupFailedError: no_weight.append(tel_id) print("Removed {} of {} telescopes due LUT problems".format( len(no_weight), len(self.hillas_dict))) # remove those types from tels_per_type for tel_id in no_weight: del self.hillas_dict[tel_id] for cam_id in tels_per_type: if tel_id in tels_per_type[cam_id]: index = np.where( np.array(tels_per_type[cam_id]) == tel_id) tels_per_type[cam_id].pop(int( index[0])) # remove from list # redo the multiplicity cut to check if it still fulfilled # self.multiplicity_cut(self.quality_cuts["multiplicity"]["cuts"], tels_per_type, # method=self.quality_cuts["multiplicity"]["method"]) # do the second pass with new weights self.reco_result = self.reconstructor.predict( self.hillas_dict, self.event.inst, self.true_alt, self.true_az, ext_weight=self.weights) # Number of telescopes triggered per type self.n_tels_per_type = { tel: len(tels_per_type[tel]) for tel in tels_per_type } for tel_id in self.hillas_dict: try: agree = self.mc_offset == self.get_offangle(tel_id, direction="mc") except AttributeError: self.mc_offset = self.get_offangle(tel_id, direction="mc") continue if not agree: raise ValueError( "The pointing of the telescopes seems to be different.") self.impact = self.get_impact(self.hillas_dict) # impact parameter def get_reconstructed_parameters(self): ''' Return the parameters for writing to table. Returns ------- prepared parameters : impact max_signal tot_signal n_tels_per_type hillas_dict mc_offangle reco_result ''' # check if event was prepared before try: assert self.impact except AssertionError: self.prepare() return (self.impact, self.max_signal, self.tot_signal, self.n_tels_per_type, self.hillas_dict, self.mc_offset, self.reco_result)