def test_camera_calibrator(example_event): telid = list(example_event.r0.tel)[0] calibrator = CameraCalibrator() calibrator.calibrate(example_event) image = example_event.dl1.tel[telid].image assert image is not None
def test_camera_calibrator(example_event): telid = 11 calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator") calibrator.calibrate(example_event) image = example_event.dl1.tel[telid].image assert_allclose(image[0, 0], -2.216, 1e-3)
def test_camera_calibrator(): event = get_test_event() telid = 11 calibrator = CameraCalibrator(None, None) calibrator.calibrate(event) image = event.dl1.tel[telid].image assert_allclose(image[0, 0], -2.216, 1e-3)
def test_camera_calibrator(test_event): event = deepcopy(test_event) # so we don't modify the test event telid = 11 calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator") calibrator.calibrate(event) image = event.dl1.tel[telid].image assert_allclose(image[0, 0], -2.216, 1e-3)
def __init__(self, event, telescope_list, camera_types, ChargeExtration, pe_thresh, min_neighbors, tail_thresholds, DirReco, quality_cuts, LUT=None): super().__init__() ''' Parmeters --------- event : ctapipe event container calibrator : ctapipe camera calibrator reconstructor : ctapipe hillas reconstructor telescope_list : list with telescope configuration or "all" pe_thresh : dict with thresholds for gain selection tail_thresholds : dict with thresholds for image cleaning quality_cuts : dict containing quality cuts canera_types : list with camera types to analyze ''' self.event = event self.telescope_list = telescope_list self.pe_thresh = pe_thresh self.min_neighbors = min_neighbors self.tail_thresholds = tail_thresholds self.quality_cuts = quality_cuts self.camera_types = camera_types self.dirreco = DirReco self.LUTgenerator = LUT if (self.dirreco["weights"] == "LUT") | (self.dirreco["weights"] == "doublepass"): self.weights = {} else: self.weights = None self.hillas_dict = {} self.camera_dict = {} self.edge_pixels = {} # configurations for calibrator cfg = Config() cfg["ChargeExtractorFactory"]["product"] = \ ChargeExtration["ChargeExtractorProduct"] cfg['WaveformCleanerFactory']['product'] = \ ChargeExtration["WaveformCleanerProduct"] self.calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator", config=cfg) # calibration self.reconstructor = HillasReconstructor() # direction
def setup(self): self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]" kwargs = dict(config=self.config, tool=self) reader_factory = EventFileReaderFactory(**kwargs) reader_class = reader_factory.get_class() self.reader = reader_class(**kwargs) self.calibrator = CameraCalibrator(origin=self.reader.origin, **kwargs) self.plotter = ImagePlotter(**kwargs)
def test_eventsource_override_r1(): dataset = get_dataset("gamma_test.simtel.gz") eventsource = HESSIOEventSource(input_url=dataset) calibrator = CameraCalibrator( eventsource=eventsource, r1_product="NullR1Calibrator" ) assert isinstance(calibrator.r1, NullR1Calibrator)
def test_config(): window_shift = 3 window_width = 9 config = Config({"LocalPeakWindowSum": { "window_shift": window_shift, "window_width": window_width, }}) calibrator = CameraCalibrator( extractor_name='LocalPeakWindowSum', config=config ) assert calibrator.dl1.extractor.window_shift == window_shift assert calibrator.dl1.extractor.window_width == window_width
def test_eventsource_r1(): dataset = get_dataset_path("gamma_test.simtel.gz") eventsource = HESSIOEventSource(input_url=dataset) calibrator = CameraCalibrator(eventsource=eventsource) assert isinstance(calibrator.r1, HESSIOR1Calibrator)
def test_manual_extractor(): calibrator = CameraCalibrator(extractor_product="LocalPeakIntegrator") assert isinstance(calibrator.dl1.extractor, LocalPeakIntegrator)
def test_manual_r1(): calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator") assert isinstance(calibrator.r1, HESSIOR1Calibrator)
def test_manual_extractor(): calibrator = CameraCalibrator(extractor_name="LocalPeakWindowSum") assert isinstance(calibrator.dl1.extractor, LocalPeakWindowSum)
def analyze_ctapipe(): print("starting ctapipe.") gamma = FileReader( "/lustre/fs21/group/cta/prod3b/prod3b-paranal20deg/gamma_onSource/gamma_20deg_0deg_run624___cta-prod3_desert-2150m-Paranal-merged.simtel.gz", "gamma") gamma.read_files() trace_width = { "ASTRICam": (10, 5), "FlashCam": (4, 2), "LSTCam": (4, 2), "NectarCam": (6, 3), "DigiCam": (4, 2), "CHEC": (8, 4), "SCTCam": (6, 3) } # from Tino pe_thresh = {"ASTRICam": 14, "LSTCam": 100, "NectarCam": 190} #integrators = ['GlobalPeakIntegrator', 'LocalPeakIntegrator', 'NeighbourPeakIntegrator', 'AverageWfPeakIntegrator'] #cleaners = ['NullWaveformCleaner', 'CHECMWaveformCleanerAverage', 'CHECMWaveformCleanerLocal'] for integrator in ['NeighbourPeakIntegrator']: i = 0 for event in gamma.source: i += 1 if not event.r0.event_id in event_IDs: # continue with next event continue else: for tel in event.r0.tels_with_data: camera = event.inst.subarray.tel[tel].camera cfg = Config() cfg["ChargeExtractorFactory"]["product"] = integrator cfg["ChargeExtractorFactory"][ "window_width"] = trace_width[camera.cam_id][0] cfg["ChargeExtractorFactory"][ "window_shift"] = trace_width[camera.cam_id][1] event.dl1.tel[tel].reset() # set up calibrator. calibrator = CameraCalibrator( r1_product="HESSIOR1Calibrator", config=cfg) calibrator.calibrate(event) ##################################### # Tino's solution for gain selection if (camera.cam_id == np.array(list( pe_thresh.keys()))).any(): image = event.dl1.tel[tel].image image = pick_gain_channel(image, camera.cam_id, pe_thresh) else: image = event.dl1.tel[tel].image image = np.reshape(image[0], np.shape(image)[1]) # image = np.reshape(image[0], np.shape(image)[1]) EventID = [int(event.r0.event_id)] * len(image) TelescopeID = [int(tel)] * len(image) cameras = [event.inst.subarray.tel[tel].camera.cam_id ] * len(image) PixelID = np.arange(len(image)) charge = image new_df = pd.DataFrame(np.transpose( [EventID, TelescopeID, PixelID, charge, cameras]), columns=[ "EventID", "TelescopeID", "PixelID", "charge", "cameras" ]) try: ctapipe_charge = ctapipe_charge.append( new_df, ignore_index=True) except (TypeError, NameError): ctapipe_charge = new_df print("{:.2f}% finished".format((i / len(event_IDs)) * 100)) ctapipe_charge.to_csv('ctapipe_charge.csv', sep=",") return ctapipe_charge
def calculate_noise_parameters(simtel_filename, data_dl1_filename, config_filename=None): """ Calculates the parameters needed to increase the noise in an MC DL1 file to match the noise in a real data DL1 file, using add_noise_in_pixels The returned parameters are those needed by the function add_noise_in_pixels (see description in its documentation above). Parameters ---------- simtel_filename: `str` a simtel file containing showers, from the same production (same NSB and telescope settings) as the MC DL1 file below. It must contain pixel-wise info on true number of p.e.'s from C-photons ( will be used to identify pixels which only contain noise). data_dl1_filename: `str` a real data DL1 file (processed with calibration settings corresponding to those with which the MC is to be processed). It must contain calibrated images, i.e. "DL1a" data. This file has the "target" noise which we want to have in the MC files, for better agreement of data and simulations. config_filename: `str` configuration file containing the calibration settings used for processing both the data and the MC files above Returns ------- extra_noise_in_dim_pixels: `float` Extra noise of dim pixels. extra_bias_in_dim_pixels: `float` Extra bias of dim pixels. extra_noise_in_bright_pixels: `float` Extra noise of bright pixels """ log.setLevel(logging.INFO) if config_filename is None: config = standard_config else: config = read_configuration_file(config_filename) # Real data DL1 tables: data_dl1_calibration = read_table( data_dl1_filename, '/dl1/event/telescope/monitoring/calibration') data_dl1_pedestal = read_table(data_dl1_filename, '/dl1/event/telescope/monitoring/pedestal') data_dl1_parameters = read_table( data_dl1_filename, '/dl1/event/telescope/parameters/LST_LSTCam') data_dl1_image = read_table(data_dl1_filename, '/dl1/event/telescope/image/LST_LSTCam') unusable = data_dl1_calibration['unusable_pixels'] # Locate pixels with HG declared unusable either in original calibration or # in interleaved events: bad_pixels = unusable[0][0] # original calibration for tf in unusable[1:][0]: # calibrations with interleaveds bad_pixels = np.logical_or(bad_pixels, tf) good_pixels = ~bad_pixels # First index: 1,2,... = values from interleaveds (0 is for original # calibration run) # Second index: 0 = high gain # Third index: pixels # HG adc to pe conversion factors from interleaved calibrations: data_HG_dc_to_pe = data_dl1_calibration['dc_to_pe'][:, 0, :] # Pixel-wise pedestal standard deviation (for an unbiased extractor), # in adc counts: data_HG_ped_std = data_dl1_pedestal['charge_std'][1:, 0, :] # indices which connect each pedestal calculation to a given calibration: calibration_id = data_dl1_pedestal['calibration_id'][1:] # convert pedestal st deviations to p.e. dummy = [] for i, x in enumerate(data_HG_ped_std[:, ]): dummy.append(x * data_HG_dc_to_pe[calibration_id[i], ]) dummy = np.array(dummy) # Average for all interleaved calibrations (in case there are more than one) data_HG_ped_std_pe = np.mean(dummy, axis=0) # one value per pixel # Identify noisy pixels, likely containing stars - we want to adjust MC to # the average diffuse NSB across the camera data_median_std_ped_pe = np.median(data_HG_ped_std_pe) data_std_std_ped_pe = np.std(data_HG_ped_std_pe) log.info(f'Real data: median across camera of good pixels\' pedestal std ' f'{data_median_std_ped_pe:.3f} p.e.') brightness_limit = data_median_std_ped_pe + 3 * data_std_std_ped_pe too_bright_pixels = (data_HG_ped_std_pe > brightness_limit) log.info(f'Number of pixels beyond 3 std dev of median: ' f'{too_bright_pixels.sum()}, (above {brightness_limit:.2f} p.e.)') ped_mask = data_dl1_parameters['event_type'] == 2 # The charges in the images below are obtained with the extractor for # showers, usually a biased one, like e.g. LocalPeakWindowSum data_ped_charges = data_dl1_image['image'][ped_mask] # Exclude too bright pixels, besides those with unusable calibration: good_pixels &= ~too_bright_pixels # recalculate the median of the pixels' std dev, with good_pixels: data_median_std_ped_pe = np.median(data_HG_ped_std_pe[good_pixels]) log.info(f'Good and not too bright pixels: {good_pixels.sum()}') # all_good is an events*pixels boolean array of valid signals: all_good = np.reshape(np.tile(good_pixels, data_ped_charges.shape[0]), data_ped_charges.shape) # histogram of pedestal charges (biased extractor) from good and not noisy # pixels: qbins = 100 qrange = (-10, 15) dataq = np.histogram(data_ped_charges[all_good].flatten(), bins=qbins, range=qrange, density=True) # Find the peak of the pedestal biased charge distribution of real data. # Use an interpolated version of the histogram, for robustness: func = interp1d(0.5 * (dataq[1][1:] + dataq[1][:-1]), dataq[0], kind='quadratic', fill_value='extrapolate') xx = np.linspace(qrange[0], qrange[1], 100 * qbins) mode_data = xx[np.argmax(func(xx))] # Event reader for simtel file: mc_reader = EventSource(input_url=simtel_filename, config=Config(config)) # Obtain the configuration with which the pedestal calculations were # performed: ped_config = config['LSTCalibrationCalculator']['PedestalIntegrator'] tel_id = ped_config['tel_id'] # Obtain the (unbiased) extractor used for pedestal calculations: pedestal_extractor_type = ped_config['charge_product'] pedestal_calibrator = CameraCalibrator( image_extractor_type=pedestal_extractor_type, config=Config(ped_config), subarray=mc_reader.subarray) # Obtain the (usually biased) extractor used for shower images: shower_extractor_type = config['image_extractor'] shower_calibrator = CameraCalibrator( image_extractor_type=shower_extractor_type, config=Config(config), subarray=mc_reader.subarray) # Since these extractors are now for use on MC, we have to apply the pulse # integration correction (in data that is currently, as of # lstchain v0.7.5, replaced by an empirical (hard-coded) correction of the # adc to pe conversion factors ) pedestal_calibrator.image_extractors[ ped_config['charge_product']].apply_integration_correction = True shower_calibrator.image_extractors[ shower_extractor_type].apply_integration_correction = True # Pulse integration window width of the (biased) extractor for showers: shower_extractor_window_width = config[ config['image_extractor']]['window_width'] # Pulse integration window width for the pedestal estimation: pedestal_extractor_config = ped_config[pedestal_extractor_type] pedestal_extractor_window_width = pedestal_extractor_config['window_width'] # MC pedestals integrated with the unbiased pedestal extractor mc_ped_charges = [] # MC pedestals integrated with the biased shower extractor mc_ped_charges_biased = [] for event in mc_reader: if tel_id not in event.trigger.tels_with_trigger: continue # Extract the signals as we do for pedestals (unbiased fixed window # extractor): pedestal_calibrator(event) charges = event.dl1.tel[tel_id].image # True number of pe's from Cherenkov photons (to identify noise-only pixels) true_image = event.simulation.tel[tel_id].true_image mc_ped_charges.append(charges[true_image == 0]) # Now extract the signal as we would do for shower events (usually # with a biased extractor, e.g. LocalPeakWindowSum): shower_calibrator(event) charges_biased = event.dl1.tel[tel_id].image mc_ped_charges_biased.append(charges_biased[true_image == 0]) # All pixels behave (for now) in the same way in MC, just put them together mc_ped_charges = np.concatenate(mc_ped_charges) mc_ped_charges_biased = np.concatenate(mc_ped_charges_biased) mcq = np.histogram(mc_ped_charges_biased, bins=qbins, range=qrange, density=True) # Find the peak of the pedestal biased charge distribution of MC. Use # an interpolated version of the histogram, for robustness: func = interp1d(0.5 * (mcq[1][1:] + mcq[1][:-1]), mcq[0], kind='quadratic', fill_value='extrapolate') xx = np.linspace(qrange[0], qrange[1], 100 * qbins) mode_mc = xx[np.argmax(func(xx))] mc_unbiased_std_ped_pe = np.std(mc_ped_charges) # Find the additional noise (in data w.r.t. MC) for the unbiased extractor, # and scale it to the width of the window for integration of shower images. # The idea is that when a strong signal is present, the biased extractor # will integrate around it, and the additional noise is unbiased because # it won't modify the integration range. extra_noise_in_bright_pixels = \ ((data_median_std_ped_pe**2 - mc_unbiased_std_ped_pe**2) * shower_extractor_window_width / pedestal_extractor_window_width) # Just in case, makes sure we just add noise if the MC noise is smaller # than the real data's: extra_noise_in_bright_pixels = max(0., extra_noise_in_bright_pixels) bias = mode_data - mode_mc extra_bias_in_dim_pixels = max(bias, 0) # differences of values to peak charge: dq = data_ped_charges[all_good].flatten() - mode_data dqmc = mc_ped_charges_biased - mode_mc # maximum distance (in pe) from peak, to avoid strong impact of outliers: maxq = 10 # calculate widening of the noise bump: added_noise = (np.sum(dq[dq < maxq]**2) / len(dq[dq < maxq]) - np.sum(dqmc[dqmc < maxq]**2) / len(dqmc[dqmc < maxq])) added_noise = (max(0, added_noise))**0.5 extra_noise_in_dim_pixels = added_noise return extra_noise_in_dim_pixels, extra_bias_in_dim_pixels, \ extra_noise_in_bright_pixels
class PrepareList(Cutter): ''' Prepare a feature list to save to table. It takes an event, does the calibration, image cleaning, parametrization and reconstruction. From this some basic features will be extracted and written to the file which later on can be used for training of the classifiers or energy regressors. test ''' true_az = {} true_alt = {} max_signal = {} tot_signal = 0 impact = {} def __init__(self, event, telescope_list, camera_types, ChargeExtration, pe_thresh, min_neighbors, tail_thresholds, DirReco, quality_cuts, LUT=None): super().__init__() ''' Parmeters --------- event : ctapipe event container calibrator : ctapipe camera calibrator reconstructor : ctapipe hillas reconstructor telescope_list : list with telescope configuration or "all" pe_thresh : dict with thresholds for gain selection tail_thresholds : dict with thresholds for image cleaning quality_cuts : dict containing quality cuts canera_types : list with camera types to analyze ''' self.event = event self.telescope_list = telescope_list self.pe_thresh = pe_thresh self.min_neighbors = min_neighbors self.tail_thresholds = tail_thresholds self.quality_cuts = quality_cuts self.camera_types = camera_types self.dirreco = DirReco self.LUTgenerator = LUT if (self.dirreco["weights"] == "LUT") | (self.dirreco["weights"] == "doublepass"): self.weights = {} else: self.weights = None self.hillas_dict = {} self.camera_dict = {} self.edge_pixels = {} # configurations for calibrator cfg = Config() cfg["ChargeExtractorFactory"]["product"] = \ ChargeExtration["ChargeExtractorProduct"] cfg['WaveformCleanerFactory']['product'] = \ ChargeExtration["WaveformCleanerProduct"] self.calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator", config=cfg) # calibration self.reconstructor = HillasReconstructor() # direction def get_impact(self, hillas_dict): ''' calculate impact parameters for all telescopes that were used for parametrization. Paremeters ---------- hillas_dict : dict with hillas HillasParameterContainers Returnes -------- impact : impact parameter or NaN if calculation failed ''' # check if event was prepared before try: assert self.reco_result except AssertionError: self.prepare() impact = {} for tel_id in hillas_dict.keys(): try: pred_core = np.array([ self.reco_result.core_x.value, self.reco_result.core_y.value ]) * u.m # tel_coords start at 0 instead of 1... tel_position = np.array([ self.event.inst.subarray.tel_coords[tel_id - 1].x.value, self.event.inst.subarray.tel_coords[tel_id - 1].y.value ]) * u.m impact[tel_id] = linalg.length(pred_core - tel_position) except AttributeError: impact[tel_id] = np.nan return impact def get_offangle(self, tel_id, direction="reco"): ''' Get the angular offset between the reconstructed direction and the pointing direction of the telescope. Parameters ---------- tel_id : integer Telecope ID true_off : string if "mc", the true MC direction is taken for calculation. Otherwise, if "reco" the reconstructed value will be taken Returns ------- off_angles : dictionary dictionary with tel_ids as keys and the offangle as entries. ''' if direction == "reco": off_angle = angular_separation( self.event.mc.tel[tel_id].azimuth_raw * u.rad, self.event.mc.tel[tel_id].altitude_raw * u.rad, self.reco_result.az, self.reco_result.alt) elif direction == "mc": off_angle = angular_separation( self.event.mc.tel[tel_id].azimuth_raw * u.rad, self.event.mc.tel[tel_id].altitude_raw * u.rad, self.event.mc.az, self.event.mc.alt) return off_angle def get_weight(self, method, camera, tel_id, hillas_par, offangle=None): """ Get the weighting for HillasReconustructor. Possible methods are 'default', which will fall back to the standard weighting applied in capipe, 'LUT' which will take the weights from a LUT and 'doublepass' which might be used for diffuse simulations. In this case it returns the weights for the first pass. method : sting method to get the weighting. camera: CameraDescription tel_id: integer hillas_par: HillasParameterContainer """ if method == "default": pass elif method == "LUT": if np.isnan(hillas_par.width) & (not np.isnan(hillas_par.length)): hillas_par.width = 0 * u.m self.weights[tel_id] = self.LUTgenerator.get_weight_from_LUT( hillas_par, camera.cam_id, min_stat=self.dirreco["min_stat"], ratio_cut=self.dirreco["wl_ratio_cut"][camera.cam_id]) elif method == "doublepass": # first pass with default weighting if np.isnan(hillas_par.width) & (not np.isnan(hillas_par.length)): hillas_par.width = 0 * u.m self.weights[tel_id] = hillas_par.intensity * ( 1 * u.m + hillas_par.length) / (1 * u.m + hillas_par.width) elif method == "second_pass": # weights for second pass self.weights[ tel_id] = self.LUTgenerator.get_weight_from_diffuse_LUT( self.hillas_dict[tel_id], offangle, camera.cam_id, min_stat=self.dirreco["min_stat"], ratio_cut=self.dirreco["wl_ratio_cut"][camera.cam_id]) else: raise KeyError("Weighting method {} not known.".format(method)) def prepare(self): ''' Prepare event performimng calibration, image cleaning, hillas parametrization, hillas intersection for the single event. Additionally, the impact distance will be calculated. ''' tels_per_type = {} no_weight = [] # calibrate event self.calibrator.calibrate(self.event) # loop over all telescopeswith data in it for tel_id in self.event.r0.tels_with_data: # check if telescope is selected for analysis # This also could be done already in event_source when reading th data if (tel_id in self.telescope_list) | (self.telescope_list == "all"): pass else: continue # get camera information camera = self.event.inst.subarray.tel[tel_id].camera self.camera_dict[tel_id] = camera image = self.event.dl1.tel[tel_id].image if camera.cam_id in self.pe_thresh.keys(): image, select = pick_gain_channel( image, self.pe_thresh[camera.cam_id], True) else: image = np.squeeze(image) # image cleaning mask = tailcuts_clean( camera, image, picture_thresh=self.tail_thresholds[camera.cam_id][1], boundary_thresh=self.tail_thresholds[camera.cam_id][0], min_number_picture_neighbors=self.min_neighbors) # go to next telescope if no pixels survived cleaning if not any(mask): continue cleaned_image = np.copy(image) cleaned_image[~mask] = 0 # calculate the hillas parameters hillas_par = hillas_parameters(camera, cleaned_image) # quality cuts leakage = leakage = self.leakage_cut( camera=camera, hillas_parameters=hillas_par, radius=self.quality_cuts["leakage_cut"]["radius"], max_dist=self.quality_cuts["leakage_cut"]["dist"], image=cleaned_image, rows=self.quality_cuts["leakage_cut"]["rows"], fraction=self.quality_cuts["leakage_cut"]["frac"], method=self.quality_cuts["leakage_cut"]["method"], ) size = self.size_cut(hillas_par, self.quality_cuts["size"]) if not (leakage & size): # size or leakage cuts not passed continue # get the weighting for HillasReconstructor try: self.get_weight(self.dirreco["weights"], camera, tel_id, hillas_par) except LookupFailedError: # this telescope will be ignored, should only happen for method LUT here no_weight.append(tel_id) continue self.hillas_dict[tel_id] = hillas_par self.max_signal[tel_id] = np.max(cleaned_image) # brightest pix try: tels_per_type[camera.cam_id].append(tel_id) except KeyError: tels_per_type[camera.cam_id] = [tel_id] try: assert tels_per_type except AssertionError: raise TooFewTelescopesException("No image survived the leakage " "or size cuts.") # collect some additional information for tel_id in self.hillas_dict: self.tot_signal += self.hillas_dict[tel_id].intensity # total size self.true_az[ tel_id] = self.event.mc.tel[tel_id].azimuth_raw * u.rad self.true_alt[ tel_id] = self.event.mc.tel[tel_id].altitude_raw * u.rad # wil raise exception if cut was not passed # self.multiplicity_cut(self.quality_cuts["multiplicity"]["cuts"], # tels_per_type, method=self.quality_cuts["multiplicity"]["method"]) if self.dirreco["weights"] == "LUT": # remove telescopes withough weights print("Removed {} of {} telescopes due LUT problems".format( len(no_weight), len(self.hillas_dict) + len(no_weight))) # do Hillas reconstruction self.reco_result = self.reconstructor.predict(self.hillas_dict, self.event.inst, self.true_alt, self.true_az, ext_weight=self.weights) if self.dirreco["weights"] == "doublepass": # take the reconstructed direction to get an estimate of the offangle and # get weights from the second pass from the diffuse LUT. self.weights = {} # reset the weights from earlier no_weight = [] for tel_id in self.hillas_dict: predicted_offangle = self.get_offangle(tel_id, direction="reco") predicted_offangle = predicted_offangle.to(u.deg).value camera = self.camera_dict[tel_id] # reload camera_information # get the weighting for HillasReconstructor try: self.get_weight("second_pass", camera, tel_id, self.hillas_dict[tel_id], predicted_offangle) except LookupFailedError: no_weight.append(tel_id) print("Removed {} of {} telescopes due LUT problems".format( len(no_weight), len(self.hillas_dict))) # remove those types from tels_per_type for tel_id in no_weight: del self.hillas_dict[tel_id] for cam_id in tels_per_type: if tel_id in tels_per_type[cam_id]: index = np.where( np.array(tels_per_type[cam_id]) == tel_id) tels_per_type[cam_id].pop(int( index[0])) # remove from list # redo the multiplicity cut to check if it still fulfilled # self.multiplicity_cut(self.quality_cuts["multiplicity"]["cuts"], tels_per_type, # method=self.quality_cuts["multiplicity"]["method"]) # do the second pass with new weights self.reco_result = self.reconstructor.predict( self.hillas_dict, self.event.inst, self.true_alt, self.true_az, ext_weight=self.weights) # Number of telescopes triggered per type self.n_tels_per_type = { tel: len(tels_per_type[tel]) for tel in tels_per_type } for tel_id in self.hillas_dict: try: agree = self.mc_offset == self.get_offangle(tel_id, direction="mc") except AttributeError: self.mc_offset = self.get_offangle(tel_id, direction="mc") continue if not agree: raise ValueError( "The pointing of the telescopes seems to be different.") self.impact = self.get_impact(self.hillas_dict) # impact parameter def get_reconstructed_parameters(self): ''' Return the parameters for writing to table. Returns ------- prepared parameters : impact max_signal tot_signal n_tels_per_type hillas_dict mc_offangle reco_result ''' # check if event was prepared before try: assert self.impact except AssertionError: self.prepare() return (self.impact, self.max_signal, self.tot_signal, self.n_tels_per_type, self.hillas_dict, self.mc_offset, self.reco_result)
def r0_to_dl1( input_filename=get_dataset_path('gamma_test_large.simtel.gz'), output_filename=None, custom_config={}, ): """ Chain r0 to dl1 Save the extracted dl1 parameters in output_filename Parameters ---------- input_filename: str path to input file, default: `gamma_test_large.simtel.gz` output_filename: str or None path to output file, defaults to writing dl1 into the current directory custom_config: path to a configuration file Returns ------- """ if output_filename is None: try: run = parse_r0_filename(input_filename) output_filename = run_to_dl1_filename(run.tel_id, run.run, run.subrun) except ValueError: output_filename = r0_to_dl1_filename(Path(input_filename).name) if os.path.exists(output_filename): raise IOError(str(output_filename) + ' exists, exiting.') config = replace_config(standard_config, custom_config) custom_calibration = config["custom_calibration"] source = EventSource(input_url=input_filename, config=Config(config["source_config"])) subarray = source.subarray is_simu = source.is_simulation metadata = global_metadata(source) write_metadata(metadata, output_filename) cal_mc = load_calibrator_from_config(config, subarray) # minimum number of pe in a pixel to include it # in calculation of muon ring time (peak sample): min_pe_for_muon_t_calc = 10. # Dictionary to store muon ring parameters muon_parameters = create_muon_table() # all this will be cleaned up in a next PR related to the configuration files r1_dl1_calibrator = CameraCalibrator( image_extractor_type=config['image_extractor'], config=Config(config), subarray=subarray) if not is_simu: # Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but # using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous r1_dl1_calibrator_for_muon_rings = CameraCalibrator( image_extractor_type=config['image_extractor_for_muons'], config=Config(config), subarray=subarray) # Component to process interleaved pedestal and flat-fields calib_config = Config(config[config['calibration_product']]) calibration_calculator = CalibrationCalculator.from_name( config['calibration_product'], config=calib_config, subarray=source.subarray) calibration_index = DL1MonitoringEventIndexContainer() dl1_container = DL1ParametersContainer() extra_im = ExtraImageInfo() extra_im.prefix = '' # get rid of the prefix # Write extra information to the DL1 file write_array_info(subarray, output_filename) write_array_info_08(subarray, output_filename) if is_simu: write_mcheader( source.simulation_config, output_filename, obs_id=source.obs_ids[0], filters=HDF5_ZSTD_FILTERS, metadata=metadata, ) with HDF5TableWriter( filename=output_filename, group_name='dl1/event', mode='a', filters=HDF5_ZSTD_FILTERS, add_prefix=True, # overwrite=True, ) as writer: if is_simu: subarray = subarray # build a mapping of tel_id back to tel_index: # (note this should be part of SubarrayDescription) idx = np.zeros(max(subarray.tel_indices) + 1) for key, val in subarray.tel_indices.items(): idx[key] = val # the final transform then needs the mapping and the number of telescopes tel_list_transform = partial( utils.expand_tel_list, max_tels=max(subarray.tel) + 1, ) writer.add_column_transform(table_name='subarray/trigger', col_name='tels_with_trigger', transform=tel_list_transform) # Forcing filters for the dl1 dataset that are currently read from the pre-existing files # This should be fixed in ctapipe and then corrected here writer._h5file.filters = HDF5_ZSTD_FILTERS logger.info(f"USING FILTERS: {writer._h5file.filters}") for i, event in enumerate(source): if i % 100 == 0: logger.info(i) event.dl0.prefix = '' event.trigger.prefix = '' if event.simulation is not None: event.simulation.prefix = 'mc' dl1_container.reset() # write sub tables if is_simu: write_subarray_tables(writer, event, metadata) if not custom_calibration: cal_mc(event) if config['mc_image_scaling_factor'] != 1: rescale_dl1_charge(event, config['mc_image_scaling_factor']) else: if i == 0: # initialize the telescope # FIXME? LST calibrator is only for one telescope # it should be inside the telescope loop (?) tel_id = calibration_calculator.tel_id #initialize the event monitoring data event.mon = source.r0_r1_calibrator.mon_data # write the first calibration event (initialized from calibration h5 file) write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=True, new_ff=True) # flat-field or pedestal: if (event.trigger.event_type == EventType.FLATFIELD or event.trigger.event_type == EventType.SKY_PEDESTAL): # process interleaved events (pedestals, ff, calibration) new_ped_event, new_ff_event = calibration_calculator.process_interleaved( event) # write monitoring containers if updated if new_ped_event or new_ff_event: write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=new_ped_event, new_ff=new_ff_event) # calibrate and gain select the event by hand for DL1 source.r0_r1_calibrator.calibrate(event) # create image for all events r1_dl1_calibrator(event) # Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten. # When the last version of the method is implemented, vol. reduction will be done at dl0 apply_volume_reduction(event, subarray, config) # FIXME? This should be eventually done after we evaluate whether the image is # a candidate muon ring. In that case the full image could be kept, or reduced # only after the ring analysis is complete. for ii, telescope_id in enumerate(event.dl1.tel.keys()): dl1_container.reset() # update the calibration index in the dl1 event container dl1_container.calibration_id = calibration_index.calibration_id dl1_container.fill_event_info(event) tel = event.dl1.tel[telescope_id] tel.prefix = '' # don't really need one # remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST' tel_name = str(subarray.tel[telescope_id])[4:] if custom_calibration: lst_calibration(event, telescope_id) write_event = True # Will determine whether this event has to be written to the # DL1 output or not. if is_simu: dl1_container.fill_mc(event, subarray.positions[telescope_id]) assert event.dl1.tel[telescope_id].image is not None try: get_dl1( event, subarray, telescope_id, dl1_container=dl1_container, custom_config=config, ) except HillasParameterizationError: logging.exception( 'HillasParameterizationError in get_dl1()') if not is_simu: dl1_container.ucts_time = 0 # convert Time to unix timestamp in (UTC) to keep compatibility # with older lstchain # FIXME: just keep it as time, table writer and reader handle it dl1_container.dragon_time = event.trigger.time.unix dl1_container.tib_time = 0 dl1_container.ucts_trigger_type = event.lst.tel[ telescope_id].evt.ucts_trigger_type dl1_container.trigger_type = event.lst.tel[ telescope_id].evt.tib_masked_trigger else: dl1_container.trigger_type = event.trigger.event_type dl1_container.az_tel = event.pointing.tel[telescope_id].azimuth dl1_container.alt_tel = event.pointing.tel[ telescope_id].altitude dl1_container.trigger_time = event.trigger.time.unix dl1_container.event_type = event.trigger.event_type # FIXME: no need to read telescope characteristics like foclen for every event! foclen = subarray.tel[ telescope_id].optics.equivalent_focal_length mirror_area = u.Quantity( subarray.tel[telescope_id].optics.mirror_area, u.m**2) dl1_container.prefix = tel.prefix # extra info for the image table extra_im.tel_id = telescope_id extra_im.selected_gain_channel = event.r1.tel[ telescope_id].selected_gain_channel for container in [extra_im, dl1_container, event.r0, tel]: add_global_metadata(container, metadata) event.r0.prefix = '' writer.write(table_name=f'telescope/image/{tel_name}', containers=[event.index, tel, extra_im]) writer.write(table_name=f'telescope/parameters/{tel_name}', containers=[event.index, dl1_container]) # Muon ring analysis, for real data only (MC is done starting from DL1 files) if not is_simu: bad_pixels = event.mon.tel[ telescope_id].calibration.unusable_pixels[0] # Set to 0 unreliable pixels: image = tel.image * (~bad_pixels) # process only promising events, in terms of # of pixels with large signals: if tag_pix_thr(image): # re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings numsamples = event.r1.tel[telescope_id].waveform.shape[ 1] # not necessarily the same as in r0! bad_pixels_hg = event.mon.tel[ telescope_id].calibration.unusable_pixels[0] bad_pixels_lg = event.mon.tel[ telescope_id].calibration.unusable_pixels[1] # Now set to 0 all samples in unreliable pixels. Important for global peak # integrator in case of crazy pixels! TBD: can this be done in a simpler # way? bad_pixels = bad_pixels_hg | bad_pixels_lg bad_waveform = np.transpose( np.array(numsamples * [bad_pixels])) # print('hg bad pixels:',np.where(bad_pixels_hg)) # print('lg bad pixels:',np.where(bad_pixels_lg)) event.r1.tel[telescope_id].waveform *= ~bad_waveform r1_dl1_calibrator_for_muon_rings(event) tel = event.dl1.tel[telescope_id] image = tel.image * (~bad_pixels) # Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum) # perhaps the event is no longer promising (e.g. if it has a large time evolution) if not tag_pix_thr(image): good_ring = False else: # read geometry from event.inst. But not needed for every event. FIXME? geom = subarray.tel[telescope_id].\ camera.geometry muonintensityparam, dist_mask, \ ring_size, size_outside_ring, muonringparam, \ good_ring, radial_distribution, \ mean_pixel_charge_around_ring,\ muonpars = \ analyze_muon_event(subarray, event.index.event_id, image, geom, foclen, mirror_area, False, '') # mirror_area, True, './') # (test) plot muon rings as png files # Now we want to obtain the waveform sample (in HG & LG) at which the ring light peaks: bright_pixels = image > min_pe_for_muon_t_calc selected_gain = event.r1.tel[ telescope_id].selected_gain_channel mask_hg = bright_pixels & (selected_gain == 0) mask_lg = bright_pixels & (selected_gain == 1) bright_pixels_waveforms_hg = event.r1.tel[ telescope_id].waveform[mask_hg, :] bright_pixels_waveforms_lg = event.r1.tel[ telescope_id].waveform[mask_lg, :] stacked_waveforms_hg = np.sum( bright_pixels_waveforms_hg, axis=0) stacked_waveforms_lg = np.sum( bright_pixels_waveforms_lg, axis=0) # stacked waveforms from all bright pixels; shape (ngains, nsamples) hg_peak_sample = np.argmax(stacked_waveforms_hg, axis=-1) lg_peak_sample = np.argmax(stacked_waveforms_lg, axis=-1) if good_ring: fill_muon_event(-1, muon_parameters, good_ring, event.index.event_id, dl1_container.dragon_time, muonintensityparam, dist_mask, muonringparam, radial_distribution, ring_size, size_outside_ring, mean_pixel_charge_around_ring, muonpars, hg_peak_sample, lg_peak_sample) # writes mc information per telescope, including photo electron image if (is_simu and config['write_pe_image'] and event.simulation.tel[telescope_id].true_image is not None and event.simulation.tel[telescope_id].true_image.any()): event.simulation.tel[telescope_id].prefix = '' writer.write(table_name=f'simulation/{tel_name}', containers=[ event.simulation.tel[telescope_id], extra_im ]) if not is_simu: # at the end of event loop ask calculation of remaining interleaved statistics new_ped, new_ff = calibration_calculator.output_interleaved_results( event) # write monitoring events write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=new_ped, new_ff=new_ff) if is_simu: # Reconstruct source position from disp for all events and write the result in the output file for tel_name in ['LST_LSTCam']: focal = OpticsDescription.from_name( tel_name.split('_')[0]).equivalent_focal_length add_disp_to_parameters_table(output_filename, dl1_params_lstcam_key, focal) # Write energy histogram from simtel file and extra metadata # ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever if source.max_events is None: write_simtel_energy_histogram(source, output_filename, obs_id=event.index.obs_id, metadata=metadata) else: dir, name = os.path.split(output_filename) name = name.replace('dl1', 'muons').replace('LST-1.1', 'LST-1') # Consider the possibilities of DL1 files with .fits.h5 & .h5 ending: name = name.replace('.fits.h5', '.fits').replace('.h5', '.fits') muon_output_filename = Path(dir, name) table = Table(muon_parameters) table.write(muon_output_filename, format='fits', overwrite=True)
class DisplayDL1Calib(Tool): name = "DisplayDL1Calib" description = "Calibrate dl0 data to dl1, and plot the photoelectron " \ "images." telescope = Int(None, allow_none=True, help='Telescope to view. Set to None to display all ' 'telescopes.').tag(config=True) aliases = Dict(dict(f='EventFileReaderFactory.input_path', r='EventFileReaderFactory.reader', max_events='EventFileReaderFactory.max_events', extractor='ChargeExtractorFactory.extractor', window_width='ChargeExtractorFactory.window_width', t0='ChargeExtractorFactory.t0', window_shift='ChargeExtractorFactory.window_shift', sig_amp_cut_HG='ChargeExtractorFactory.sig_amp_cut_HG', sig_amp_cut_LG='ChargeExtractorFactory.sig_amp_cut_LG', lwt='ChargeExtractorFactory.lwt', clip_amplitude='CameraDL1Calibrator.clip_amplitude', T='DisplayDL1Calib.telescope', O='ImagePlotter.output_path' )) flags = Dict(dict(D=({'ImagePlotter': {'display': True}}, "Display the photoelectron images on-screen as they " "are produced.") )) classes = List([EventFileReaderFactory, ChargeExtractorFactory, CameraDL1Calibrator, ImagePlotter ]) def __init__(self, **kwargs): super().__init__(**kwargs) self.reader = None self.calibrator = None self.plotter = None def setup(self): self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]" kwargs = dict(config=self.config, tool=self) reader_factory = EventFileReaderFactory(**kwargs) reader_class = reader_factory.get_class() self.reader = reader_class(**kwargs) self.calibrator = CameraCalibrator(origin=self.reader.origin, **kwargs) self.plotter = ImagePlotter(**kwargs) def start(self): source = self.reader.read() for event in source: self.calibrator.calibrate(event) tel_list = event.r0.tels_with_data if self.telescope: if self.telescope not in tel_list: continue tel_list = [self.telescope] for telid in tel_list: self.plotter.plot(event, telid) def finish(self): self.plotter.finish()
def calculate_required_additional_nsb(simtel_filename, data_dl1_filename, config=None): # TODO check if good estimation # TODO reduce duplicated code with 'calculate_noise_parameters' """ Calculates the additional NSB needed in the MC waveforms to match a real data DL1 file Parameters ---------- simtel_filename: a simtel file containing showers, from the production (same NSB and telescope settings) as the one on which the correction will be applied. It must contain pixel-wise info on true number of p.e.'s from C-photons (will be used to identify pixels which only contain noise). data_dl1_filename: a real data DL1 file (processed with calibration settings corresponding to those with which the MC is to be processed). It must contain calibrated images, i.e. "DL1a" data. This file has the "target" NSB which we want to have in the MC files, for better agreement of data and simulations. config: configuration containing the calibration settings used for processing both the data and the MC files above Returns ------- extra_nsb: Fraction of the additional NSB in data compared to MC. data_ped_variance: Pedestal variance from data mc_ped_variance: Pedestal variance from MC """ log.setLevel(logging.INFO) if config is None: config = standard_config # Real data DL1 tables: data_dl1_calibration = read_table( data_dl1_filename, '/dl1/event/telescope/monitoring/calibration') data_dl1_pedestal = read_table(data_dl1_filename, '/dl1/event/telescope/monitoring/pedestal') unusable = data_dl1_calibration['unusable_pixels'] # Locate pixels with HG declared unusable either in original calibration or # in interleaved events: bad_pixels = unusable[0][0] # original calibration for tf in unusable[1:][0]: # calibrations with interleaved bad_pixels = np.logical_or(bad_pixels, tf) good_pixels = ~bad_pixels # First index: 1,2,... = values from interleaved (0 is for original # calibration run) # Second index: 0 = high gain # Third index: pixels # HG adc to pe conversion factors from interleaved calibrations: data_HG_dc_to_pe = data_dl1_calibration['dc_to_pe'][:, 0, :] # Pixel-wise pedestal standard deviation (for an unbiased extractor), # in adc counts: data_HG_ped_std = data_dl1_pedestal['charge_std'][1:, 0, :] # indices which connect each pedestal calculation to a given calibration: calibration_id = data_dl1_pedestal['calibration_id'][1:] # convert pedestal st deviations to p.e. dummy = [] for i, x in enumerate(data_HG_ped_std[:, ]): dummy.append(x * data_HG_dc_to_pe[calibration_id[i], ]) dummy = np.array(dummy) # Average for all interleaved calibrations (in case there are more than one) data_HG_ped_std_pe = np.mean(dummy, axis=0) # one value per pixel # Identify noisy pixels, likely containing stars - we want to adjust MC to # the average diffuse NSB across the camera data_median_std_ped_pe = np.median(data_HG_ped_std_pe) data_std_std_ped_pe = np.std(data_HG_ped_std_pe) log.info(f'Real data: median across camera of good pixels\' pedestal std ' f'{data_median_std_ped_pe:.3f} p.e.') brightness_limit = data_median_std_ped_pe + 3 * data_std_std_ped_pe too_bright_pixels = (data_HG_ped_std_pe > brightness_limit) log.info(f'Number of pixels beyond 3 std dev of median: ' f'{too_bright_pixels.sum()}, (above {brightness_limit:.2f} p.e.)') # Exclude too bright pixels, besides those with unusable calibration: good_pixels &= ~too_bright_pixels # recalculate the median of the pixels' std dev, with good_pixels: data_median_std_ped_pe = np.median(data_HG_ped_std_pe[good_pixels]) log.info(f'Good and not too bright pixels: {good_pixels.sum()}') # Event reader for simtel file: mc_reader = EventSource(input_url=simtel_filename, config=Config(config)) # Obtain the configuration with which the pedestal calculations were # performed: ped_config = config['LSTCalibrationCalculator']['PedestalIntegrator'] tel_id = ped_config['tel_id'] # Obtain the (unbiased) extractor used for pedestal calculations: pedestal_calibrator = CameraCalibrator( image_extractor_type=ped_config['charge_product'], config=Config(config['LSTCalibrationCalculator']), subarray=mc_reader.subarray) # Since these extractors are now for use on MC, we have to apply the pulse # integration correction (in data that is currently, as of # lstchain v0.7.5, replaced by an empirical (hard-coded) correction of the # adc to pe conversion factors ) pedestal_calibrator.image_extractors[ ped_config['charge_product']].apply_integration_correction = True # MC pedestals integrated with the unbiased pedestal extractor mc_ped_charges = [] for event in mc_reader: if tel_id not in event.trigger.tels_with_trigger: continue # Extract the signals as we do for pedestals (unbiased fixed window # extractor): pedestal_calibrator(event) charges = event.dl1.tel[tel_id].image # True number of pe's from Cherenkov photons (to identify noise-only pixels) true_image = event.simulation.tel[tel_id].true_image mc_ped_charges.append(charges[true_image == 0]) # All pixels behave (for now) in the same way in MC, just put them together mc_ped_charges = np.concatenate(mc_ped_charges) mc_unbiased_std_ped_pe = np.std(mc_ped_charges) # Find the additional noise (in data w.r.t. MC) for the unbiased extractor # The idea is that pedestal variance scales with NSB data_ped_variance = data_median_std_ped_pe**2 mc_ped_variance = mc_unbiased_std_ped_pe**2 extra_nsb = ((data_ped_variance - mc_ped_variance) / mc_ped_variance) return extra_nsb, data_ped_variance, mc_ped_variance
def r0_to_dl1( input_filename=None, output_filename=None, custom_config={}, ): """ Chain r0 to dl1 Save the extracted dl1 parameters in output_filename Parameters ---------- input_filename: str path to input file, default: `gamma_test_large.simtel.gz` output_filename: str or None path to output file, defaults to writing dl1 into the current directory custom_config: path to a configuration file Returns ------- """ # using None as default and using `get_dataset_path` only inside the function # prevents downloading at import time. if input_filename is None: get_dataset_path('gamma_test_large.simtel.gz') if output_filename is None: try: run = parse_r0_filename(input_filename) output_filename = run_to_dl1_filename(run.tel_id, run.run, run.subrun) except ValueError: output_filename = r0_to_dl1_filename(Path(input_filename).name) if os.path.exists(output_filename): raise IOError(str(output_filename) + ' exists, exiting.') config = replace_config(standard_config, custom_config) source = EventSource(input_url=input_filename, config=Config(config["source_config"])) subarray = source.subarray is_simu = source.is_simulation metadata = global_metadata() write_metadata(metadata, output_filename) cal_mc = load_calibrator_from_config(config, subarray) # minimum number of pe in a pixel to include it # in calculation of muon ring time (peak sample): min_pe_for_muon_t_calc = 10. # Dictionary to store muon ring parameters muon_parameters = create_muon_table() # all this will be cleaned up in a next PR related to the configuration files r1_dl1_calibrator = CameraCalibrator( image_extractor_type=config['image_extractor'], config=Config(config), subarray=subarray) if not is_simu: # Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but # using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous r1_dl1_calibrator_for_muon_rings = CameraCalibrator( image_extractor_type=config['image_extractor_for_muons'], config=Config(config), subarray=subarray) # Component to process interleaved pedestal and flat-fields calib_config = Config({ config['calibration_product']: config[config['calibration_product']] }) calibration_calculator = CalibrationCalculator.from_name( config['calibration_product'], config=calib_config, subarray=source.subarray) calibration_index = DL1MonitoringEventIndexContainer() dl1_container = DL1ParametersContainer() extra_im = ExtraImageInfo() extra_im.prefix = '' # get rid of the prefix # Write extra information to the DL1 file subarray.to_hdf(output_filename) if is_simu: write_mcheader( source.simulation_config, output_filename, obs_id=source.obs_ids[0], filters=HDF5_ZSTD_FILTERS, metadata=metadata, ) nsb_tuning = False if 'waveform_nsb_tuning' in config.keys(): nsb_tuning = config['waveform_nsb_tuning']['nsb_tuning'] if nsb_tuning: if is_simu: nsb_original = extract_simulation_nsb(input_filename) pulse_template = NormalizedPulseTemplate.load_from_eventsource( subarray.tel[1].camera.readout) if 'nsb_tuning_ratio' in config['waveform_nsb_tuning'].keys(): # get value from config to possibly extract it beforehand on multiple files for averaging purposes # or gain time nsb_tuning_ratio = config['waveform_nsb_tuning'][ 'nsb_tuning_ratio'] else: # extract the pedestal variance difference between the current MC file and the target data # FIXME? fails for multiple telescopes nsb_tuning_ratio = calculate_required_additional_nsb( input_filename, config['waveform_nsb_tuning']['target_data'], config=config)[0] spe = np.loadtxt( config['waveform_nsb_tuning']['spe_location']).T spe_integral = np.cumsum(spe[1]) charge_spe_cumulative_pdf = interp1d(spe_integral, spe[0], kind='cubic', bounds_error=False, fill_value=0., assume_sorted=True) allowed_tel = np.zeros(len(nsb_original), dtype=bool) allowed_tel[np.array(config['source_config']['LSTEventSource'] ['allowed_tels'])] = True logger.info('Tuning NSB on MC waveform from ' + str(np.asarray(nsb_original)[allowed_tel]) + 'GHz to {0:d}%'.format( int(nsb_tuning_ratio * 100 + 100.5)) + ' for telescopes ids ' + str(config['source_config']['LSTEventSource'] ['allowed_tels'])) else: logger.warning( 'NSB tuning on waveform active in config but file is real data, option will be ignored' ) nsb_tuning = False with HDF5TableWriter( filename=output_filename, group_name='dl1/event', mode='a', filters=HDF5_ZSTD_FILTERS, add_prefix=True, # overwrite=True, ) as writer: setup_writer(writer, source.subarray, is_simulation=is_simu) event = None for i, event in enumerate(source): if i % 100 == 0: logger.info(i) event.dl0.prefix = '' event.trigger.prefix = '' if event.simulation is not None: event.simulation.prefix = 'mc' dl1_container.reset() # write sub tables if is_simu: write_subarray_tables(writer, event, metadata) cal_mc(event) if config['mc_image_scaling_factor'] != 1: rescale_dl1_charge(event, config['mc_image_scaling_factor']) else: if i == 0: # initialize the telescope # FIXME? LST calibrator is only for one telescope # it should be inside the telescope loop (?) tel_id = calibration_calculator.tel_id #initialize the event monitoring data event.mon = deepcopy(source.r0_r1_calibrator.mon_data) for container in [ event.mon.tel[tel_id].pedestal, event.mon.tel[tel_id].flatfield, event.mon.tel[tel_id].calibration ]: add_global_metadata(container, metadata) add_config_metadata(container, config) # write the first calibration event (initialized from calibration h5 file) write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=True, new_ff=True) # flat-field or pedestal: if (event.trigger.event_type == EventType.FLATFIELD or event.trigger.event_type == EventType.SKY_PEDESTAL): # process interleaved events (pedestals, ff, calibration) new_ped_event, new_ff_event = calibration_calculator.process_interleaved( event) # write monitoring containers if updated if new_ped_event or new_ff_event: write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=new_ped_event, new_ff=new_ff_event) # calibrate and gain select the event by hand for DL1 source.r0_r1_calibrator.calibrate(event) # Option to add nsb in waveforms if nsb_tuning: # FIXME? assumes same correction ratio for all telescopes for tel_id in config['source_config']['LSTEventSource'][ 'allowed_tels']: waveform = event.r1.tel[tel_id].waveform readout = subarray.tel[tel_id].camera.readout sampling_rate = readout.sampling_rate.to_value(u.GHz) dt = (1.0 / sampling_rate) selected_gains = event.r1.tel[tel_id].selected_gain_channel mask_high = (selected_gains == 0) tune_nsb_on_waveform(waveform, nsb_tuning_ratio, nsb_original[tel_id] * u.GHz, dt * u.ns, pulse_template, mask_high, charge_spe_cumulative_pdf) # create image for all events r1_dl1_calibrator(event) # Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten. # When the last version of the method is implemented, vol. reduction will be done at dl0 apply_volume_reduction(event, subarray, config) # FIXME? This should be eventually done after we evaluate whether the image is # a candidate muon ring. In that case the full image could be kept, or reduced # only after the ring analysis is complete. for telescope_id, dl1_tel in event.dl1.tel.items(): dl1_tel.prefix = '' # don't really need one tel_name = str(subarray.tel[telescope_id])[4:] # extra info for the image table extra_im.tel_id = telescope_id extra_im.selected_gain_channel = event.r1.tel[ telescope_id].selected_gain_channel add_global_metadata(extra_im, metadata) add_config_metadata(extra_im, config) focal_length = subarray.tel[ telescope_id].optics.equivalent_focal_length mirror_area = subarray.tel[telescope_id].optics.mirror_area dl1_container.reset() # update the calibration index in the dl1 event container dl1_container.calibration_id = calibration_index.calibration_id dl1_container.fill_event_info(event) # Will determine whether this event has to be written to the # DL1 output or not. if is_simu: dl1_container.fill_mc(event, subarray.positions[telescope_id]) assert event.dl1.tel[telescope_id].image is not None try: get_dl1( event, subarray, telescope_id, dl1_container=dl1_container, custom_config=config, ) except HillasParameterizationError: logging.exception( 'HillasParameterizationError in get_dl1()') if not is_simu: dl1_container.ucts_time = 0 # convert Time to unix timestamp in (UTC) to keep compatibility # with older lstchain # FIXME: just keep it as time, table writer and reader handle it dl1_container.dragon_time = event.trigger.time.unix dl1_container.tib_time = 0 if 'ucts_jump' in vars( event.lst.tel[telescope_id].evt.__class__): dl1_container.ucts_jump = event.lst.tel[ telescope_id].evt.ucts_jump dl1_container.ucts_trigger_type = event.lst.tel[ telescope_id].evt.ucts_trigger_type dl1_container.trigger_type = event.lst.tel[ telescope_id].evt.tib_masked_trigger else: dl1_container.trigger_type = event.trigger.event_type dl1_container.az_tel = event.pointing.tel[telescope_id].azimuth dl1_container.alt_tel = event.pointing.tel[ telescope_id].altitude dl1_container.trigger_time = event.trigger.time.unix dl1_container.event_type = event.trigger.event_type dl1_container.prefix = dl1_tel.prefix for container in [extra_im, dl1_container, event.r0, dl1_tel]: add_global_metadata(container, metadata) add_config_metadata(container, config) writer.write(table_name=f'telescope/parameters/{tel_name}', containers=[event.index, dl1_container]) writer.write(table_name=f'telescope/image/{tel_name}', containers=[event.index, dl1_tel, extra_im]) # Muon ring analysis, for real data only (MC is done starting from DL1 files) if not is_simu: bad_pixels = event.mon.tel[ telescope_id].calibration.unusable_pixels[0] # Set to 0 unreliable pixels: image = dl1_tel.image * (~bad_pixels) # process only promising events, in terms of # of pixels with large signals: if tag_pix_thr(image): # re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings numsamples = event.r1.tel[telescope_id].waveform.shape[ 1] # not necessarily the same as in r0! bad_pixels_hg = event.mon.tel[ telescope_id].calibration.unusable_pixels[0] bad_pixels_lg = event.mon.tel[ telescope_id].calibration.unusable_pixels[1] # Now set to 0 all samples in unreliable pixels. Important for global peak # integrator in case of crazy pixels! TBD: can this be done in a simpler # way? bad_pixels = bad_pixels_hg | bad_pixels_lg bad_waveform = np.transpose( np.array(numsamples * [bad_pixels])) # print('hg bad pixels:',np.where(bad_pixels_hg)) # print('lg bad pixels:',np.where(bad_pixels_lg)) event.r1.tel[telescope_id].waveform *= ~bad_waveform r1_dl1_calibrator_for_muon_rings(event) image = dl1_tel.image * (~bad_pixels) # Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum) # perhaps the event is no longer promising (e.g. if it has a large time evolution) if not tag_pix_thr(image): good_ring = False else: # read geometry from event.inst. But not needed for every event. FIXME? geom = subarray.tel[telescope_id].\ camera.geometry muonintensityparam, dist_mask, \ ring_size, size_outside_ring, muonringparam, \ good_ring, radial_distribution, \ mean_pixel_charge_around_ring,\ muonpars = \ analyze_muon_event(subarray, event.index.event_id, image, geom, focal_length, mirror_area, False, '') # mirror_area, True, './') # (test) plot muon rings as png files # Now we want to obtain the waveform sample (in HG & LG) at which the ring light peaks: bright_pixels = image > min_pe_for_muon_t_calc selected_gain = event.r1.tel[ telescope_id].selected_gain_channel mask_hg = bright_pixels & (selected_gain == 0) mask_lg = bright_pixels & (selected_gain == 1) bright_pixels_waveforms_hg = event.r1.tel[ telescope_id].waveform[mask_hg, :] bright_pixels_waveforms_lg = event.r1.tel[ telescope_id].waveform[mask_lg, :] stacked_waveforms_hg = np.sum( bright_pixels_waveforms_hg, axis=0) stacked_waveforms_lg = np.sum( bright_pixels_waveforms_lg, axis=0) # stacked waveforms from all bright pixels; shape (ngains, nsamples) hg_peak_sample = np.argmax(stacked_waveforms_hg, axis=-1) lg_peak_sample = np.argmax(stacked_waveforms_lg, axis=-1) if good_ring: fill_muon_event(-1, muon_parameters, good_ring, event.index.event_id, dl1_container.dragon_time, muonintensityparam, dist_mask, muonringparam, radial_distribution, ring_size, size_outside_ring, mean_pixel_charge_around_ring, muonpars, hg_peak_sample, lg_peak_sample) # writes mc information per telescope, including photo electron image if (is_simu and config['write_pe_image'] and event.simulation.tel[telescope_id].true_image is not None and event.simulation.tel[telescope_id].true_image.any()): event.simulation.tel[telescope_id].prefix = '' writer.write(table_name=f'simulation/{tel_name}', containers=[ event.simulation.tel[telescope_id], extra_im ]) if event is None: logger.warning('No events in file') if not is_simu and event is not None: # at the end of event loop ask calculation of remaining interleaved statistics new_ped, new_ff = calibration_calculator.output_interleaved_results( event) # write monitoring events write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=new_ped, new_ff=new_ff) if is_simu and event is not None: # Reconstruct source position from disp for all events and write the result in the output file add_disp_to_parameters_table(output_filename, dl1_params_lstcam_key, focal_length) # Write energy histogram from simtel file and extra metadata # ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever if source.max_events is None: write_simtel_energy_histogram(source, output_filename, obs_id=event.index.obs_id, metadata=metadata) if not is_simu: dir, name = os.path.split(output_filename) name = name.replace('dl1', 'muons').replace('LST-1.1', 'LST-1') # Consider the possibilities of DL1 files with .fits.h5 & .h5 ending: name = name.replace('.fits.h5', '.fits').replace('.h5', '.fits') muon_output_filename = Path(dir, name) table = Table(muon_parameters) table.write(muon_output_filename, format='fits', overwrite=True)
def prepare(self, particles, hillas=True, subarray=np.arange(600)): ''' Performs the processing of the images: calibration perfoming conversion from r0 to dl1 image cleaning using tailcut_cleaning image parametrization with Hillas parameters Fills the dicts for clean_images and hillas moments ''' # Input ####### # particles - array of konsta_cta.readdata.FileReader # self.particles = particles self.geoms, self.geoms_unique = self.get_camera_geoms() # loop through all particle types for particle in self.particles: dtype = particle.datatype # count number of images and number after cleaning self.sum_images[dtype] = 0 self.sum_clean_images[dtype] = 0 # dict to store the hisograms self.histograms = {} # loop through all events for event in particle.source: event_id = event.dl0.event_id n_images = 0 n_clean_images = 0 # count for each camera individually images_cams = {} clean_images_cams = {} self.mc_energy[dtype, event_id] = event.mc.energy # loop through all telescopes with data for tel_id in event.r0.tels_with_data: if type(subarray[0]) != str: subarray = [str(x) for x in subarray] if str(tel_id) in subarray: pass else: continue n_images += 1 # get camera information camera = event.inst.subarray.tel[tel_id].camera try: # check if key already exists. if not initialize it with value 0 _im = images_cams[camera.cam_id] _im = clean_images_cams[camera.cam_id] except KeyError: images_cams[camera.cam_id] = 0 clean_images_cams[camera.cam_id] = 0 # count for each camera individually images_cams[camera.cam_id] += 1 cfg = Config() cfg["ChargeExtractorFactory"]["product"] = self.integrator cfg["ChargeExtractorFactory"][ "window_width"] = self.trace_width[camera.cam_id][0] cfg["ChargeExtractorFactory"][ "window_shift"] = self.trace_width[camera.cam_id][1] cfg['WaveformCleanerFactory']['product'] = self.cleaner # usually all cameras are calibrated at once with the same camera. # In order to force a racalculation to use the differen width and # shift of the camera of this telescope, dl1 container will be reset # to default at each iteration. # #Probably it is not even needed to reset this, as the container #is refilled anyway. event.dl1.tel[tel_id].reset() # danger: The resulting calibrated event doesn't contain the right # extracted charges for all cameras in the end as it seems like # the dl1 images are overwirtten each time so that, the charges, # extracted at the last telescope iteration will be contained in the # dl1 container. # As I'm wirting out all the required information for the telescope # within this loop, this should not be much of a problem for now, but # in the future a appropriate way to work arround this is required. # set up calibrator. calibrator = CameraCalibrator(r1_product=self.r1, config=cfg) # calibrate event calibrator.calibrate(event) ###################################### # create output for comparison plots # ###################################### cam_id = camera.cam_id number_gains = event.dl1.tel[tel_id].image.shape[0] # create dict for the camera on first appearance try: _hist = self.histograms[cam_id] except KeyError: self.histograms[cam_id] = {} # fill histograms and merge for all events dependent on gain for gain, label in zip(range(number_gains), ["gain1", "gain2"]): image = np.array(event.dl1.tel[tel_id].image[gain]) # only positive charges hist_lower = len(image[image > np.power(10., -1.)]) hist_higher = len(image[image > np.power(10., 4.)]) image = image[image > 0] logimage = np.log10(image) hist = np.histogram(logimage, range=[-1, 4], bins=100) # store the values outside of range for sanity check _hist = np.append(hist_lower, hist[0]) _hist = np.append(_hist, hist_higher) _bins = np.append(-1000, hist[1]) _bins = np.append(_bins, 1000) hist = (_hist, _bins) try: self.histograms[cam_id][label] = ( self.histograms[cam_id][label][0] + hist[0], self.histograms[cam_id][label][1]) except KeyError: self.histograms[cam_id][label] = hist ##################################### # Tino's solution for gain selection if (camera.cam_id == np.array(list( self.pe_thresh.keys()))).any(): image = event.dl1.tel[tel_id].image image = self.pick_gain_channel(image, camera.cam_id) else: image = event.dl1.tel[tel_id].image image = np.reshape(image[0], np.shape(image)[1]) ###################### # image cleaning # Threshold values adapted from Tino's repository mask = tailcuts_clean( self.geoms[tel_id], image, picture_thresh=self.tail_thresholds[camera.cam_id][1], boundary_thresh=self.tail_thresholds[camera.cam_id][0], min_number_picture_neighbors=0) try: temp_list except NameError: temp_list = [] if not (camera.cam_id in temp_list): temp_list.append(camera.cam_id) print("Threshold {camera}: {threshold}".format( camera=camera.cam_id, threshold=self.tail_thresholds[camera.cam_id])) number_pixels = np.count_nonzero(mask) # drop images that didn't survive image cleaning if any(mask == True): n_clean_images += 1 self.clean_images[dtype, event_id, tel_id] = np.copy(image) # set rejected pixels to zero self.clean_images[dtype, event_id, tel_id][~mask] = 0 # count for each camera individually try: clean_images_cams[camera.cam_id] += 1 except KeyError: clean_images_cams[camera.cam_id] = 1 ### hillas parametrization if hillas: hillas_moments = hillas_parameters( self.geoms[tel_id], self.clean_images[dtype, event_id, tel_id], True) self.number_pixels.append(number_pixels) self.energy.append(event.mc.energy.base) self.size.append(hillas_moments.intensity) self.length.append(hillas_moments.length) self.width.append(hillas_moments.width) self.skewness.append(hillas_moments.skewness) self.camera.append(camera.cam_id) self.core_x.append(event.mc.core_x.base) self.core_y.append(event.mc.core_y.base) else: pass # count number of images at trigge level and after cleaning # summary: self.sum_images[dtype] += n_images self.sum_clean_images[dtype] += n_clean_images # per event: self.core_x2.append(event.mc.core_x.base) self.core_y2.append(event.mc.core_y.base) self.energy2.append(event.mc.energy.base) self.n_images.append(float(n_images)) self.n_clean_images.append(float(n_clean_images)) for cam in images_cams.keys(): try: self.images_cams[cam].append(float(images_cams[cam])) self.clean_images_cams[cam].append( float(clean_images_cams[cam])) except: self.images_cams[cam] = [float(images_cams[cam])] self.clean_images_cams[cam] = [ float(clean_images_cams[cam]) ] print("Processed {} images for datatype {}. Images " "that didn't survive cleaning: {}".format( self.sum_images[dtype], dtype, self.sum_images[dtype] - self.sum_clean_images[dtype])) self.get_keys()
def setup(self): self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]" data_config = self.config.copy() data_config['WaveformCleanerFactory'] = Config(cleaner='CHECMWaveformCleanerLocal') mc_config = self.config.copy() data_kwargs = dict(config=data_config, tool=self) mc_kwargs = dict(config=mc_config, tool=self) filepath = '/Volumes/gct-jason/data/170330/onsky-mrk501/Run05477_r1.tio' reader = TargetioFileReader(input_path=filepath, **data_kwargs) filepath = '/Users/Jason/Software/outputs/sim_telarray/meudon_cr/simtel_proton_nsb50_thrs30_1petal_rndm015_heide.gz' # filepath = '/Users/Jason/Software/outputs/sim_telarray/meudon_cr/simtel_proton_nsb50_thrs30.gz' reader_mc = HessioFileReader(input_path=filepath, **mc_kwargs) calibrator = CameraCalibrator(origin=reader.origin, **data_kwargs) calibrator_mc = CameraCalibrator(origin=reader_mc.origin, **mc_kwargs) first_event = reader.get_event(0) telid = list(first_event.r0.tels_with_data)[0] pos = first_event.inst.pixel_pos[telid] foclen = first_event.inst.optical_foclen[telid] geom = CameraGeometry.guess(*pos, foclen) first_event = reader_mc.get_event(0) telid = list(first_event.r0.tels_with_data)[0] pos_mc = first_event.inst.pixel_pos[telid] foclen = first_event.inst.optical_foclen[telid] geom_mc = CameraGeometry.guess(*pos_mc, foclen) d1 = dict(type='Data', reader=reader, calibrator=calibrator, pos=pos, geom=geom, t1=20, t2=10) d2 = dict(type='MC', reader=reader_mc, calibrator=calibrator_mc, pos=pos_mc, geom=geom_mc, t1=20, t2=10) self.reader_df = pd.DataFrame([d1, d2]) p_kwargs = data_kwargs p_kwargs['script'] = "checm_paper_hillas" p_kwargs['figure_name'] = "all_images" self.p_allimage = AllImagePlotter(**p_kwargs) p_kwargs['figure_name'] = "all_peak_time_images" self.p_alltimeimage = PeakTimePlotter(**p_kwargs) p_kwargs['figure_name'] = "all_mc_images" self.p_allmcimage = AllImagePlotter(**p_kwargs) p_kwargs['figure_name'] = "zero_width_images" self.p_zwimage = ZeroWidthImagePlotter(**p_kwargs) p_kwargs['figure_name'] = "zero_width_mc_images" self.p_zwmcimage = ZeroWidthImagePlotter(**p_kwargs) p_kwargs['figure_name'] = "muon_images" self.p_muonimage = MuonImagePlotter(**p_kwargs) p_kwargs['figure_name'] = "bright_images" self.p_brightimage = BrightImagePlotter(**p_kwargs) p_kwargs['figure_name'] = "count_image" self.p_countimage = CountPlotter(**p_kwargs) p_kwargs['figure_name'] = "whole_distribution" self.p_whole_dist = WholeDist(**p_kwargs, shape='wide') p_kwargs['figure_name'] = "width_vs_length" self.p_widthvslength = WidthVsLength(**p_kwargs, shape='wide') p_kwargs['figure_name'] = "size_vs_length" self.p_sizevslength = SizeVsLength(**p_kwargs, shape='wide') p_kwargs['figure_name'] = "width_div_length" self.p_widthdivlength = WidthDivLength(**p_kwargs, shape='wide') p_kwargs['figure_name'] = "length_div_size" self.p_lengthdivsize = LengthDivSize(**p_kwargs, shape='wide') p_kwargs['figure_name'] = "pair_plot" self.p_pair = PairPlotter(**p_kwargs, shape='wide') p_kwargs['figure_name'] = "pair_mc_plot" self.p_mc_pair = PairPlotter(**p_kwargs, shape='wide') p_kwargs['figure_name'] = "length" self.p_length = LengthPlotter(**p_kwargs, shape='wide') p_kwargs['figure_name'] = "width" self.p_width = WidthPlotter(**p_kwargs, shape='wide')
class DisplayDL1Calib(Tool): name = "DisplayDL1Calib" description = "Calibrate dl0 data to dl1, and plot the photoelectron " \ "images." telescope = Int(None, allow_none=True, help='Telescope to view. Set to None to display all ' 'telescopes.').tag(config=True) aliases = Dict( dict(f='EventFileReaderFactory.input_path', r='EventFileReaderFactory.reader', max_events='EventFileReaderFactory.max_events', extractor='ChargeExtractorFactory.extractor', window_width='ChargeExtractorFactory.window_width', t0='ChargeExtractorFactory.t0', window_shift='ChargeExtractorFactory.window_shift', sig_amp_cut_HG='ChargeExtractorFactory.sig_amp_cut_HG', sig_amp_cut_LG='ChargeExtractorFactory.sig_amp_cut_LG', lwt='ChargeExtractorFactory.lwt', clip_amplitude='CameraDL1Calibrator.clip_amplitude', T='DisplayDL1Calib.telescope', O='ImagePlotter.output_path')) flags = Dict( dict(D=({ 'ImagePlotter': { 'display': True } }, "Display the photoelectron images on-screen as they " "are produced."))) classes = List([ EventFileReaderFactory, ChargeExtractorFactory, CameraDL1Calibrator, ImagePlotter ]) def __init__(self, **kwargs): super().__init__(**kwargs) self.reader = None self.calibrator = None self.plotter = None def setup(self): self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]" kwargs = dict(config=self.config, tool=self) reader_factory = EventFileReaderFactory(**kwargs) reader_class = reader_factory.get_class() self.reader = reader_class(**kwargs) self.calibrator = CameraCalibrator(origin=self.reader.origin, **kwargs) self.plotter = ImagePlotter(**kwargs) def start(self): source = self.reader.read() for event in source: self.calibrator.calibrate(event) tel_list = event.r0.tels_with_data if self.telescope: if self.telescope not in tel_list: continue tel_list = [self.telescope] for telid in tel_list: self.plotter.plot(event, telid) def finish(self): self.plotter.finish()