def ctapipe_subarray(self): from ctapipe.instrument import TelescopeDescription, SubarrayDescription, \ CameraGeometry, CameraReadout, CameraDescription, OpticsDescription import astropy.units as u geom = CameraGeometry("sstcam", self.mapping.pixel.i, u.Quantity(self.mapping.pixel.x, 'm'), u.Quantity(self.mapping.pixel.y, 'm'), u.Quantity(self.mapping.pixel.size, 'm')**2, 'square') readout = CameraReadout( "sstcam", u.Quantity(1 / self.waveform_sample_width, "GHz"), self.photoelectron_pulse.amplitude[None, :], u.Quantity(self.photoelectron_pulse.sample_width, "ns")) camera = CameraDescription("sstcam", geom, readout) optics = OpticsDescription.from_name('SST-ASTRI') telescope = TelescopeDescription("SST", "SST", optics, camera) subarray = SubarrayDescription( 'toy', tel_positions={1: [0, 0, 0] * u.m}, tel_descriptions={1: telescope}, ) return subarray
def main(): std_config = get_standard_config() if args.config_file is not None: config = replace_config(std_config, read_configuration_file(args.config_file)) else: config = std_config print(config['tailcut']) geom = CameraGeometry.from_name('LSTCam-002') foclen = OpticsDescription.from_name('LST').equivalent_focal_length dl1_container = DL1ParametersContainer() parameters_to_update = list(HillasParametersContainer().keys()) parameters_to_update.extend(['wl', 'r', 'leakage', 'n_islands', 'intercept', 'time_gradient']) nodes_keys = get_dataset_keys(args.input_file) if args.noimage: nodes_keys.remove(dl1_images_lstcam_key) auto_merge_h5files([args.input_file], args.output_file, nodes_keys=nodes_keys) with tables.open_file(args.input_file, mode='r') as input: image_table = input.root[dl1_images_lstcam_key] with tables.open_file(args.output_file, mode='a') as output: params = output.root[dl1_params_lstcam_key].read() for ii, row in enumerate(image_table): if ii%10000 == 0: print(ii) image = row['image'] pulse_time = row['pulse_time'] signal_pixels = tailcuts_clean(geom, image, **config['tailcut']) if image[signal_pixels].shape[0] > 0: num_islands, island_labels = number_of_islands(geom, signal_pixels) hillas = hillas_parameters(geom[signal_pixels], image[signal_pixels]) dl1_container.fill_hillas(hillas) dl1_container.set_timing_features(geom[signal_pixels], image[signal_pixels], pulse_time[signal_pixels], hillas) dl1_container.set_leakage(geom, image, signal_pixels) dl1_container.n_islands = num_islands dl1_container.wl = dl1_container.width / dl1_container.length width = np.rad2deg(np.arctan2(dl1_container.width, foclen)) length = np.rad2deg(np.arctan2(dl1_container.length, foclen)) dl1_container.width = width.value dl1_container.length = length.value dl1_container.r = np.sqrt(dl1_container.x**2 + dl1_container.y**2) for p in parameters_to_update: params[ii][p] = Quantity(dl1_container[p]).value else: for p in parameters_to_update: params[ii][p] = 0 output.root[dl1_params_lstcam_key][:] = params
def _generator(self): # container for LST data self.data = LSTDataContainer() self.data.meta['input_url'] = self.input_url self.data.meta['max_events'] = self.max_events self.data.meta['origin'] = 'LSTCAM' # fill LST data from the CameraConfig table self.fill_lst_service_container_from_zfile() # Instrument information for tel_id in self.data.lst.tels_with_data: assert (tel_id == 0 or tel_id == 1) # only LST1 (for the moment id = 0) # optics info from standard optics.fits.gz file optics = OpticsDescription.from_name("LST") # camera info from LSTCam-[geometry_version].camgeom.fits.gz file geometry_version = 2 camera = CameraGeometry.from_name("LSTCam", geometry_version) tel_descr = TelescopeDescription(name='LST', tel_type='LST', optics=optics, camera=camera) self.n_camera_pixels = tel_descr.camera.n_pixels tels = {tel_id: tel_descr} # LSTs telescope position taken from MC from the moment tel_pos = {tel_id: [50., 50., 16] * u.m} subarray = SubarrayDescription("LST1 subarray") subarray.tels = tels subarray.positions = tel_pos self.data.inst.subarray = subarray # initialize general monitoring container self.initialize_mon_container() # loop on events for count, event in enumerate(self.multi_file): self.data.count = count # fill specific LST event data self.fill_lst_event_container_from_zfile(event) # fill general monitoring data self.fill_mon_container_from_zfile(event) # fill general R0 data self.fill_r0_container_from_zfile(event) yield self.data
def __init__(self, config=None, tool=None, **kwargs): """ Constructor Parameters ---------- config: traitlets.loader.Config Configuration specified by config file or cmdline arguments. Used to set traitlet values. Set to None if no configuration to pass. tool: ctapipe.core.Tool Tool executable that is calling this component. Passes the correct logger to the component. Set to None if no Tool to pass. kwargs: dict Additional parameters to be passed. NOTE: The file mask of the data to read can be passed with the 'input_url' parameter. """ file_list = glob.glob(kwargs['input_url']) file_list.sort() # EventSource can not handle file wild cards as input_url # To overcome this we substitute the input_url with first file matching # the specified file mask. del kwargs['input_url'] super().__init__(config=config, tool=tool, input_url=file_list[0], **kwargs) try: import uproot except ImportError: msg = "The `uproot` python module is required to access the MAGIC data" self.log.error(msg) raise # Retrieving the list of run numbers corresponding to the data files run_numbers = list(map(self._get_run_number, file_list)) self.run_numbers = np.unique(run_numbers) # # Setting up the current run with the first run present in the data # self.current_run = self._set_active_run(run_number=0) self.current_run = None # MAGIC telescope positions in m wrt. to the center of CTA simulations self.magic_tel_positions = { 1: [-27.24, -146.66, 50.00] * u.m, 2: [-96.44, -96.77, 51.00] * u.m } # MAGIC telescope description optics = OpticsDescription.from_name('MAGIC') geom = CameraGeometry.from_name('MAGICCam') self.magic_tel_description = TelescopeDescription(optics=optics, camera=geom) self.magic_tel_descriptions = {1: self.magic_tel_description, 2: self.magic_tel_description} self.magic_subarray = SubarrayDescription('MAGIC', self.magic_tel_positions, self.magic_tel_descriptions)
def _generator(self): # container for NectarCAM data self.data = NectarCAMDataContainer() self.data.meta['input_url'] = self.input_url # fill data from the CameraConfig table self.fill_nectarcam_service_container_from_zfile() # Instrument information for tel_id in self.data.nectarcam.tels_with_data: assert (tel_id == 0) # only one telescope for the moment (id = 0) # optics info from standard optics.fits.gz file optics = OpticsDescription.from_name("MST") optics.tel_subtype = '' # to correct bug in reading # camera info from NectarCam-[geometry_version].camgeom.fits.gz file geometry_version = 2 camera = CameraGeometry.from_name("NectarCam", geometry_version) tel_descr = TelescopeDescription(optics, camera) tel_descr.optics.tel_subtype = '' # to correct bug in reading self.n_camera_pixels = tel_descr.camera.n_pixels tels = {tel_id: tel_descr} # LSTs telescope position tel_pos = {tel_id: [0., 0., 0] * u.m} self.subarray = SubarrayDescription("MST prototype subarray") self.subarray.tels = tels self.subarray.positions = tel_pos self.data.inst.subarray = self.subarray # loop on events for count, event in enumerate(self.multi_file): self.data.count = count # fill specific NectarCAM event data self.fill_nectarcam_event_container_from_zfile(event) # fill general R0 data self.fill_r0_container_from_zfile(event) yield self.data
def _generator(self): # container for LST data self.data = LSTDataContainer() self.data.meta['input_url'] = self.input_url self.data.meta['max_events'] = self.max_events # fill LST data from the CameraConfig table self.fill_lst_service_container_from_zfile() # Instrument information for tel_id in self.data.lst.tels_with_data: assert (tel_id == 0) # only LST1 for the moment (id = 0) # optics info from standard optics.fits.gz file optics = OpticsDescription.from_name("LST") optics.tel_subtype = '' # to correct bug in reading # camera info from LSTCam-[geometry_version].camgeom.fits.gz file geometry_version = 2 camera = CameraGeometry.from_name("LSTCam", geometry_version) tel_descr = TelescopeDescription(optics, camera) self.n_camera_pixels = tel_descr.camera.n_pixels tels = {tel_id: tel_descr} # LSTs telescope position taken from MC from the moment tel_pos = {tel_id: [50., 50., 16] * u.m} subarray = SubarrayDescription("LST1 subarray") subarray.tels = tels subarray.positions = tel_pos self.data.inst.subarray = subarray # loop on events for count, event in enumerate(self.multi_file): self.data.count = count # fill specific LST event data self.fill_lst_event_container_from_zfile(event) # fill general R0 data self.fill_r0_container_from_zfile(event) yield self.data
def _generator(self): # container for NectarCAM data self.data = NectarCAMDataContainer() self.data.meta['input_url'] = self.input_url # fill data from the CameraConfig table self.fill_nectarcam_service_container_from_zfile() # Instrument information for tel_id in self.data.nectarcam.tels_with_data: assert (tel_id == 0) # only one telescope for the moment (id = 0) # optics info from standard optics.fits.gz file optics = OpticsDescription.from_name("MST") optics.tel_subtype = '' # to correct bug in reading # camera info from NectarCam-[geometry_version].camgeom.fits.gz file geometry_version = 2 camera = CameraGeometry.from_name("NectarCam", geometry_version) tel_descr = TelescopeDescription(optics, camera) tel_descr.optics.tel_subtype = '' # to correct bug in reading self.n_camera_pixels = tel_descr.camera.n_pixels tels = {tel_id: tel_descr} # LSTs telescope position tel_pos = {tel_id: [0., 0., 0] * u.m} self.subarray = SubarrayDescription("MST prototype subarray") self.subarray.tels = tels self.subarray.positions = tel_pos self.data.inst.subarray = self.subarray # loop on events for count, event in enumerate(self.multi_file): self.data.count = count # fill specific NectarCAM event data self.fill_nectarcam_event_container_from_zfile(event) # fill general R0 data self.fill_r0_container_from_zfile(event) yield self.data
def get_expected_source_pos(data, data_type, config): """Get expected source position for source-dependent analysis . Parameters: ----------- data: Pandas DataFrame data_type: 'mc_gamma','mc_proton','real_data' config: dictionnary containing configuration """ #For gamma MC, expected source position is actual one for each event if data_type == 'mc_gamma': expected_src_pos_x_m = data['src_x'].values expected_src_pos_y_m = data['src_y'].values #For proton MC, nominal source position is one written in config file if data_type == 'mc_proton': focal_length = OpticsDescription.from_name( 'LST').equivalent_focal_length expected_src_pos = utils.sky_to_camera( u.Quantity(data['mc_alt_tel'].values + config['mc_nominal_source_x_deg'], u.deg, copy=False), u.Quantity(data['mc_az_tel'].values + config['mc_nominal_source_y_deg'], u.deg, copy=False), focal_length, u.Quantity(data['mc_alt_tel'].values, u.deg, copy=False), u.Quantity(data['mc_az_tel'].values, u.deg, copy=False)) expected_src_pos_x_m = expected_src_pos.x.to_value() expected_src_pos_y_m = expected_src_pos.y.to_value() # For real data # TODO: expected source position for real data should be obtained by using tel_alt,az and source RA,Dec # For the moment, expected source position is defined as camera center (ON mode) if data_type == 'real_data': expected_src_pos_x_m = np.zeros(len(data)) expected_src_pos_y_m = np.zeros(len(data)) return expected_src_pos_x_m, expected_src_pos_y_m
def prepare_subarray_info(self, tel_id=0): """ Constructs a SubarrayDescription object. Parameters ---------- tel_id: int Telescope identifier. Returns ------- SubarrayDescription : instrumental information """ tel_descriptions = {} # tel_id : TelescopeDescription tel_positions = {} # tel_id : TelescopeDescription # optics info from standard optics.fits.gz file optics = OpticsDescription.from_name("MST") optics.tel_subtype = '' # to correct bug in reading # camera info from NectarCam-[geometry_version].camgeom.fits.gz file camera = CameraGeometry.from_name("NectarCam", self.geometry_version) tel_descr = TelescopeDescription(name='MST', tel_type='NectarCam', optics=optics, camera=camera) tel_descr.optics.tel_subtype = '' # to correct bug in reading self.n_camera_pixels = tel_descr.camera.n_pixels # MST telescope position tel_positions[tel_id] = [0., 0., 0] * u.m tel_descriptions[tel_id] = tel_descr return SubarrayDescription( "Adlershof", tel_positions=tel_positions, tel_descriptions=tel_descriptions, )
import pandas as pd from astropy.table import Table import RFGHsep_fromfile as rf #Read data into pandas DataFrame filetype = 'hdf5' gammafile = "/scratch/bernardos/LST1/Events/gamma_events_diff.hdf5" #File with events protonfile = "/scratch/bernardos/LST1/Events/proton_events.hdf5" #File with events dat_gamma = Table.read(gammafile, format=filetype, path='gamma') dat_proton = Table.read(protonfile, format=filetype, path='proton') df_gamma = dat_gamma.to_pandas() df_proton = dat_proton.to_pandas() #Get some telescope parameters tel = OpticsDescription.from_name('LST') #Telescope description focal_length = tel.equivalent_focal_length.value #Telescope focal length #Calculate source position and Disp distance: sourcepos_gamma = Disp.calc_CamSourcePos(df_gamma['mcAlt'].get_values(), df_gamma['mcAz'].get_values(), df_gamma['mcAlttel'].get_values(), df_gamma['mcAztel'].get_values(), focal_length) disp_gamma = Disp.calc_DISP(sourcepos_gamma[0], sourcepos_gamma[1], df_gamma['x'].get_values(), df_gamma['y'].get_values()) sourcepos_proton = Disp.calc_CamSourcePos(df_proton['mcAlt'].get_values(), df_proton['mcAz'].get_values(), df_proton['mcAlttel'].get_values(),
def r0_to_dl1(input_filename=get_dataset_path('gamma_test_large.simtel.gz'), output_filename=None, custom_config={}, pedestal_path=None, calibration_path=None, time_calibration_path=None, pointing_file_path=None): """ Chain r0 to dl1 Save the extracted dl1 parameters in output_filename Parameters ---------- input_filename: str path to input file, default: `gamma_test_large.simtel.gz` output_filename: str path to output file, default: `./` + basename(input_filename) config_file: path to a configuration file pointing_file_path: path to the Drive log with the pointing information Returns ------- """ if output_filename is None: output_filename = ('dl1_' + os.path.basename(input_filename).split('.')[0] + '.h5') config = replace_config(standard_config, custom_config) custom_calibration = config["custom_calibration"] try: source = event_source(input_filename, back_seekable=True) except: # back_seekable might not be available for other sources that eventio # TODO for real data: source with calibration file and pointing file source = event_source(input_filename) is_simu = source.metadata['is_simulation'] source.allowed_tels = config["allowed_tels"] source.max_events = config["max_events"] metadata = global_metadata(source) write_metadata(metadata, output_filename) cal = load_calibrator_from_config(config) if not is_simu: # TODO : add calibration config in config file, read it and pass it here r0_r1_calibrator = LSTR0Corrections(pedestal_path=pedestal_path, tel_id=1) # all this will be cleaned up in a next PR related to the configuration files r1_dl1_calibrator = LSTCameraCalibrator( calibration_path=calibration_path, time_calibration_path=time_calibration_path, image_extractor=config['image_extractor'], gain_threshold=Config(config).gain_selector_config['threshold'], config=Config(config), allowed_tels=[1], ) dl1_container = DL1ParametersContainer() if pointing_file_path: # Open drive report pointings = PointingPosition() pointings.drive_path = pointing_file_path drive_data = pointings._read_drive_report() extra_im = ExtraImageInfo() extra_im.prefix = '' # get rid of the prefix event = next(iter(source)) write_array_info(event, output_filename) ### Write extra information to the DL1 file if is_simu: write_mcheader(event.mcheader, output_filename, obs_id=event.r0.obs_id, filters=filters, metadata=metadata) subarray = event.inst.subarray with HDF5TableWriter( filename=output_filename, group_name='dl1/event', mode='a', filters=filters, add_prefix=True, # overwrite=True, ) as writer: print("USING FILTERS: ", writer._h5file.filters) if is_simu: # build a mapping of tel_id back to tel_index: # (note this should be part of SubarrayDescription) idx = np.zeros(max(subarray.tel_indices) + 1) for key, val in subarray.tel_indices.items(): idx[key] = val # the final transform then needs the mapping and the number of telescopes tel_list_transform = partial( utils.expand_tel_list, max_tels=len(event.inst.subarray.tel) + 1, ) writer.add_column_transform(table_name='subarray/trigger', col_name='tels_with_trigger', transform=tel_list_transform) ### EVENT LOOP ### for i, event in enumerate(source): if i % 100 == 0: print(i) event.dl0.prefix = '' event.mc.prefix = 'mc' event.trig.prefix = '' # write sub tables if is_simu: write_subarray_tables(writer, event, metadata) if not custom_calibration and is_simu: cal(event) if not is_simu: r0_r1_calibrator.calibrate(event) r1_dl1_calibrator(event) for ii, telescope_id in enumerate(event.r0.tels_with_data): tel = event.dl1.tel[telescope_id] tel.prefix = '' # don't really need one # remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST' tel_name = str(event.inst.subarray.tel[telescope_id])[4:] tel_name = tel_name.replace('-003', '') if custom_calibration: lst_calibration(event, telescope_id) try: dl1_filled = get_dl1(event, telescope_id, dl1_container=dl1_container, custom_config=config, use_main_island=True) except HillasParameterizationError: logging.exception( 'HillasParameterizationError in get_dl1()') continue if dl1_filled is not None: # Some custom def dl1_container.wl = dl1_container.width / dl1_container.length # Log10(Energy) in GeV if is_simu: dl1_container.mc_energy = event.mc.energy.value dl1_container.log_mc_energy = np.log10( event.mc.energy.value * 1e3) dl1_container.fill_mc(event) dl1_container.log_intensity = np.log10( dl1_container.intensity) dl1_container.gps_time = event.trig.gps_time.value if not is_simu: # GPS time is not available for the time being. Meanwhile, # timestamps can be extracted from the UCTS and alternatively # calculated from the TIB/Dragon modules counters + NTP time # corresponding to the start of the run. For the time being, # we will store the three of them in the dl1 files. # This will be deprecated and modified back to directly use # gps_time whenever the GPS starts working reliably. # gps_time = event.r0.tel[telescope_id].trigger_time ucts_time = event.lst.tel[ telescope_id].evt.ucts_timestamp * 1e-9 # nsecs # Get counters from the central Dragon module module_id = 82 dragon_time = (event.lst.tel[telescope_id].svc.date + event.lst.tel[telescope_id].evt. pps_counter[module_id] + event.lst.tel[telescope_id].evt. tenMHz_counter[module_id] * 10**(-7)) tib_time = ( event.lst.tel[telescope_id].svc.date + event.lst.tel[telescope_id].evt.tib_pps_counter + event.lst.tel[telescope_id].evt.tib_tenMHz_counter * 10**(-7)) #dl1_container.gps_time = gps_time dl1_container.tib_time = tib_time dl1_container.ucts_time = ucts_time dl1_container.dragon_time = dragon_time # Select the timestamps to be used for pointing interpolation if config['timestamps_pointing'] == "ucts": event_timestamps = ucts_time elif config['timestamps_pointing'] == "dragon": event_timestamps = dragon_time elif config['timestamps_pointing'] == "tib": event_timestamps = tib_time else: raise ValueError( "The timestamps_pointing option is not a valid one. \ Try ucts (default), dragon or tib.") if pointing_file_path and event_timestamps > 0: azimuth, altitude = pointings.cal_pointingposition( event_timestamps, drive_data) event.pointing[telescope_id].azimuth = azimuth event.pointing[telescope_id].altitude = altitude dl1_container.az_tel = azimuth dl1_container.alt_tel = altitude foclen = event.inst.subarray.tel[ telescope_id].optics.equivalent_focal_length width = np.rad2deg(np.arctan2(dl1_container.width, foclen)) length = np.rad2deg( np.arctan2(dl1_container.length, foclen)) dl1_container.width = width.value dl1_container.length = length.value dl1_container.prefix = tel.prefix extra_im.tel_id = telescope_id extra_im.num_trig_pix = event.r0.tel[ telescope_id].num_trig_pix extra_im.trigger_time = event.r0.tel[ telescope_id].trigger_time extra_im.trigger_type = event.r0.tel[ telescope_id].trigger_type extra_im.trig_pix_id = event.r0.tel[ telescope_id].trig_pix_id for container in [extra_im, dl1_container, event.r0, tel]: add_global_metadata(container, metadata) event.r0.prefix = '' writer.write(table_name=f'telescope/image/{tel_name}', containers=[event.r0, tel, extra_im]) writer.write(table_name=f'telescope/parameters/{tel_name}', containers=[dl1_container, extra_im]) # writes mc information per telescope, including photo electron image if is_simu \ and (event.mc.tel[telescope_id].photo_electron_image > 0).any() \ and config['write_pe_image']: event.mc.tel[telescope_id].prefix = '' writer.write( table_name=f'simulation/{tel_name}', containers=[event.mc.tel[telescope_id], extra_im]) if is_simu: ### Reconstruct source position from disp for all events and write the result in the output file for tel_name in ['LST_LSTCam']: focal = OpticsDescription.from_name( tel_name.split('_')[0]).equivalent_focal_length dl1_params_key = f'dl1/event/telescope/parameters/{tel_name}' add_disp_to_parameters_table(output_filename, dl1_params_key, focal) # Write energy histogram from simtel file and extra metadata if is_simu: write_simtel_energy_histogram(source, output_filename, obs_id=event.dl0.obs_id, metadata=metadata)
def apply_models(dl1, classifier, reg_energy, reg_disp_vector, custom_config={}): """Apply previously trained Random Forests to a set of data depending on a set of features. Parameters: ----------- data: Pandas DataFrame features: list classifier: Random Forest Classifier RF for Gamma/Hadron separation reg_energy: Random Forest Regressor RF for Energy reconstruction reg_disp: Random Forest Regressor RF for disp_norm reconstruction """ config = replace_config(standard_config, custom_config) regression_features = config["regression_features"] classification_features = config["classification_features"] dl2 = dl1.copy() #Reconstruction of Energy and disp_norm distance dl2['log_reco_energy'] = reg_energy.predict(dl2[regression_features]) dl2['reco_energy'] = 10**(dl2['log_reco_energy']-3) disp_vector = reg_disp_vector.predict(dl2[regression_features]) dl2['reco_disp_dx'] = disp_vector[:, 0] dl2['reco_disp_dy'] = disp_vector[:, 1] #Construction of Source position in camera coordinates from disp_norm distance. dl2['reco_src_x'], dl2['reco_src_y'] = disp.disp_to_pos(dl2.reco_disp_dx, dl2.reco_disp_dy, dl2.x, dl2.y, ) focal_length = OpticsDescription.from_name('LST').equivalent_focal_length if 'mc_alt_tel' in dl2.columns: alt_tel = dl2['mc_alt_tel'].values az_tel = dl2['mc_az_tel'].values elif 'alt_tel' in dl2.columns: alt_tel = dl2['alt_tel'].values az_tel = dl2['az_tel'].values else: alt_tel = - np.pi/2. * np.ones(len(dl2)) az_tel = - np.pi/2. * np.ones(len(dl2)) src_pos_reco = utils.reco_source_position_sky(dl2.x.values * u.m, dl2.y.values * u.m, dl2.reco_disp_dx.values * u.m, dl2.reco_disp_dy.values * u.m, focal_length, alt_tel * u.rad, az_tel * u.rad) dl2['reco_alt'] = src_pos_reco.alt.rad dl2['reco_az'] = src_pos_reco.az.rad dl2['reco_type'] = classifier.predict(dl2[classification_features]).astype(int) probs = classifier.predict_proba(dl2[classification_features])[0:, 0] dl2['gammaness'] = probs return dl2
def r0_to_dl1( input_filename=get_dataset_path('gamma_test_large.simtel.gz'), output_filename=None, custom_config={}, pedestal_path=None, calibration_path=None, pointing_file_path=None, ): """ Chain r0 to dl1 Save the extracted dl1 parameters in output_filename Parameters ---------- input_filename: str path to input file, default: `gamma_test_large.simtel.gz` output_filename: str path to output file, default: `./` + basename(input_filename) config_file: path to a configuration file pointing_file_path: path to the Drive log with the pointing information Returns ------- """ if output_filename is None: output_filename = ('dl1_' + os.path.basename(input_filename).split('.')[0] + '.h5') config = replace_config(standard_config, custom_config) custom_calibration = config["custom_calibration"] try: source = event_source(input_filename, back_seekable=True) except: # back_seekable might not be available for other sources that eventio # TODO for real data: source with calibration file and pointing file source = event_source(input_filename) is_simu = source.metadata['is_simulation'] source.allowed_tels = config["allowed_tels"] source.max_events = config["max_events"] metadata = global_metadata(source) write_metadata(metadata, output_filename) cal = load_calibrator_from_config(config) if not is_simu: # TODO : add calibration config in config file, read it and pass it here charge_config = Config( {"LocalPeakWindowSum": { "window_shift": 5, "window_width": 12 }}) r0_r1_calibrator = LSTR0Corrections( pedestal_path=pedestal_path, r1_sample_start=2, # numbers in config ? r1_sample_end=38, ) r1_dl1_calibrator = LSTCameraCalibrator( calibration_path=calibration_path, image_extractor=config['image_extractor'], config=charge_config, allowed_tels=[1], ) dl1_container = DL1ParametersContainer() if pointing_file_path: # Open drive report pointings = PointingPosition() pointings.drive_path = pointing_file_path drive_data = pointings._read_drive_report() extra_im = ExtraImageInfo() extra_im.prefix = '' # get rid of the prefix event = next(iter(source)) write_array_info(event, output_filename) ### Write extra information to the DL1 file if is_simu: write_mcheader(event.mcheader, output_filename, obs_id=event.r0.obs_id, filters=filters, metadata=metadata) subarray = event.inst.subarray with HDF5TableWriter( filename=output_filename, group_name='dl1/event', mode='a', filters=filters, add_prefix=True, # overwrite=True, ) as writer: print("USING FILTERS: ", writer._h5file.filters) if is_simu: # build a mapping of tel_id back to tel_index: # (note this should be part of SubarrayDescription) idx = np.zeros(max(subarray.tel_indices) + 1) for key, val in subarray.tel_indices.items(): idx[key] = val # the final transform then needs the mapping and the number of telescopes tel_list_transform = partial( utils.expand_tel_list, max_tels=len(event.inst.subarray.tel) + 1, ) writer.add_column_transform(table_name='subarray/trigger', col_name='tels_with_trigger', transform=tel_list_transform) ### EVENT LOOP ### for i, event in enumerate(source): if i % 100 == 0: print(i) event.dl0.prefix = '' event.mc.prefix = 'mc' event.trig.prefix = '' # write sub tables if is_simu: write_subarray_tables(writer, event, metadata) if not custom_calibration and is_simu: cal(event) if not is_simu: r0_r1_calibrator.calibrate(event) r1_dl1_calibrator(event) for ii, telescope_id in enumerate(event.r0.tels_with_data): if not is_simu: combine_channels(event, telescope_id, 4095) tel = event.dl1.tel[telescope_id] tel.prefix = '' # don't really need one # remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST' tel_name = str(event.inst.subarray.tel[telescope_id])[4:] tel_name = tel_name.replace('-002', '') if custom_calibration: lst_calibration(event, telescope_id) try: dl1_filled = get_dl1(event, telescope_id, dl1_container=dl1_container, custom_config=config, use_main_island=True) except HillasParameterizationError: logging.exception( 'HillasParameterizationError in get_dl1()') continue if dl1_filled is not None: # Some custom def dl1_container.wl = dl1_container.width / dl1_container.length # Log10(Energy) in GeV if is_simu: dl1_container.mc_energy = event.mc.energy.value dl1_container.log_mc_energy = np.log10( event.mc.energy.value * 1e3) dl1_container.fill_mc(event) dl1_container.log_intensity = np.log10( dl1_container.intensity) dl1_container.gps_time = event.trig.gps_time.value if not is_simu: # For real data, GPS time is not available for the time being. # In the mean time, it is taken from TIB pps and 10 MHz counters # since UCTS timestamps do not seem to be trustable. This will be # deprecated and modified back to directly use gps_time whenever # the GPS starts working. # TAI time in s taken from TIB tai_time = event.r0.tel[telescope_id].trigger_time utc_time = Time(datetime.utcfromtimestamp(tai_time)) gps_time = utc_time.gps dl1_container.gps_time = gps_time if pointing_file_path: azimuth, altitude = pointings.cal_pointingposition( utc_time.unix, drive_data) event.pointing[telescope_id].azimuth = azimuth event.pointing[telescope_id].altitude = altitude dl1_container.az_tel = azimuth dl1_container.alt_tel = altitude foclen = event.inst.subarray.tel[ telescope_id].optics.equivalent_focal_length width = np.rad2deg(np.arctan2(dl1_container.width, foclen)) length = np.rad2deg( np.arctan2(dl1_container.length, foclen)) dl1_container.width = width.value dl1_container.length = length.value dl1_container.prefix = tel.prefix extra_im.tel_id = telescope_id for container in [extra_im, dl1_container, event.r0, tel]: add_global_metadata(container, metadata) event.r0.prefix = '' writer.write(table_name=f'telescope/image/{tel_name}', containers=[event.r0, tel, extra_im]) writer.write(table_name=f'telescope/parameters/{tel_name}', containers=[dl1_container]) # writes mc information per telescope, including photo electron image if is_simu \ and (event.mc.tel[telescope_id].photo_electron_image > 0).any() \ and config['write_pe_image']: event.mc.tel[telescope_id].prefix = '' writer.write( table_name=f'simulation/{tel_name}', containers=[event.mc.tel[telescope_id], extra_im]) if is_simu: ### Reconstruct source position from disp for all events and write the result in the output file for tel_name in ['LST_LSTCam']: focal = OpticsDescription.from_name( tel_name.split('_')[0]).equivalent_focal_length dl1_params_key = f'dl1/event/telescope/parameters/{tel_name}' add_disp_to_parameters_table(output_filename, dl1_params_key, focal) # Write energy histogram from simtel file and extra metadata if is_simu: write_simtel_energy_histogram(source, output_filename, obs_id=event.dl0.obs_id, metadata=metadata)
def __init__(self, **kwargs): """ Constructor Parameters ---------- kwargs: dict Parameters to be passed. NOTE: The file mask of the data to read can be passed with the 'input_url' parameter. """ try: import uproot except ImportError: raise ImportError( "The 'uproot' package is required for the DLMAGICEventSource class." ) self.file_list = glob.glob(kwargs["input_url"]) self.file_list.sort() # Since EventSource can not handle file wild cards as input_url # We substitute the input_url with first file matching # the specified file mask. del kwargs["input_url"] super().__init__(input_url=self.file_list[0], **kwargs) # Translate MAGIC shower primary id to CTA convention self.magic_to_cta_shower_primary_id = { 1: 0, # gamma 14: 101, # MAGIC proton 3: 1, # MAGIC electron } # MAGIC telescope positions in m wrt. to the center of CTA simulations self.magic_tel_positions = { 1: [-27.24, -146.66, 50.00] * u.m, 2: [-96.44, -96.77, 51.00] * u.m, } self.magic_tel_positions = self.magic_tel_positions # MAGIC telescope description optics = OpticsDescription.from_name("MAGIC") geom = CameraGeometry.from_name("MAGICCam") # Camera Readout for NectarCam used as a placeholder readout = CameraReadout( "MAGICCam", sampling_rate=u.Quantity(1, u.GHz), reference_pulse_shape=np.array([norm.pdf(np.arange(96), 48, 6)]), reference_pulse_sample_width=u.Quantity(1, u.ns), ) camera = CameraDescription("MAGICCam", geom, readout) self.magic_tel_description = TelescopeDescription(name="MAGIC", tel_type="LST", optics=optics, camera=camera) self.magic_tel_descriptions = { 1: self.magic_tel_description, 2: self.magic_tel_description, } self.magic_subarray = SubarrayDescription("MAGIC", self.magic_tel_positions, self.magic_tel_descriptions) # Open ROOT files self.calib_M1, self.calib_M2, self.star_M1, self.star_M2, self.superstar = ( None, None, None, None, None, ) for file in self.file_list: uproot_file = uproot.open(file) if "_Y_" in file: if "_M1_" in file: self.calib_M1 = uproot_file["Events"] self.meta = uproot_file["RunHeaders"] elif "_M2_" in file: self.calib_M2 = uproot_file["Events"] if "_I_" in file: if "_M1_" in file: self.star_M1 = uproot_file["Events"] elif "_M2_" in file: self.star_M2 = uproot_file["Events"] if "_S_" in file: self.superstar = uproot_file["Events"] self.meta = uproot_file["RunHeaders"] # figure out if MC or Data run self.mc = "MMcCorsikaRunHeader." in self.meta.keys() # get the run number directly from the root file if self.mc: self.run_number = int( uproot_file["RunHeaders"]["MMcCorsikaRunHeader."] ["MMcCorsikaRunHeader.fRunNumber"].array()[0]) else: self.run_number = int(uproot_file["RunHeaders"]["MRawRunHeader_1."] ["MRawRunHeader_1.fRunNumber"].array()[0]) self._header = self._parse_header()
def apply_models(dl1, classifier, reg_energy, reg_disp_vector, custom_config={}): """Apply previously trained Random Forests to a set of data depending on a set of features. Parameters: ----------- data: Pandas DataFrame features: list classifier: Random Forest Classifier RF for Gamma/Hadron separation reg_energy: Random Forest Regressor RF for Energy reconstruction reg_disp: Random Forest Regressor RF for disp_norm reconstruction """ config = replace_config(standard_config, custom_config) dl2 = dl1.copy() regression_features = config["regression_features"] classification_features = config["classification_features"] #Reconstruction of Energy and disp_norm distance dl2['log_reco_energy'] = reg_energy.predict(dl2[regression_features]) dl2['reco_energy'] = 10**(dl2['log_reco_energy']) disp_vector = reg_disp_vector.predict(dl2[regression_features]) dl2['reco_disp_dx'] = disp_vector[:, 0] dl2['reco_disp_dy'] = disp_vector[:, 1] #Construction of Source position in camera coordinates from disp_norm distance. dl2['reco_src_x'], dl2['reco_src_y'] = disp.disp_to_pos( dl2.reco_disp_dx, dl2.reco_disp_dy, dl2.x, dl2.y, ) focal_length = OpticsDescription.from_name('LST').equivalent_focal_length if 'mc_alt_tel' in dl2.columns: alt_tel = dl2['mc_alt_tel'].values az_tel = dl2['mc_az_tel'].values elif 'alt_tel' in dl2.columns: alt_tel = dl2['alt_tel'].values az_tel = dl2['az_tel'].values else: alt_tel = -np.pi / 2. * np.ones(len(dl2)) az_tel = -np.pi / 2. * np.ones(len(dl2)) src_pos_reco = utils.reco_source_position_sky( dl2.x.values * u.m, dl2.y.values * u.m, dl2.reco_disp_dx.values * u.m, dl2.reco_disp_dy.values * u.m, focal_length, alt_tel * u.rad, az_tel * u.rad) dl2['reco_alt'] = src_pos_reco.alt.rad dl2['reco_az'] = src_pos_reco.az.rad dl2['reco_type'] = classifier.predict( dl2[classification_features]).astype(int) probs = classifier.predict_proba(dl2[classification_features]) # This check is valid as long as we train on only two classes (gammas and protons) if probs.shape[1] > 2: raise ValueError( "The classifier is predicting more than two classes, " "the predicted probabilty to assign as gammaness is unclear." "Please check training data") # gammaness is the prediction probability for the first class (0) dl2['gammaness'] = probs[:, 0] return dl2
def main(): std_config = get_standard_config() log.setLevel(logging.INFO) handler = logging.StreamHandler() logging.getLogger().addHandler(handler) if args.config_file is not None: config = replace_config(std_config, read_configuration_file(args.config_file)) else: config = std_config log.info(f"Tailcut config used: {config['tailcut']}") foclen = OpticsDescription.from_name('LST').equivalent_focal_length cam_table = Table.read(args.input_file, path="instrument/telescope/camera/LSTCam") camera_geom = CameraGeometry.from_table(cam_table) dl1_container = DL1ParametersContainer() parameters_to_update = list(HillasParametersContainer().keys()) parameters_to_update.extend([ 'concentration_cog', 'concentration_core', 'concentration_pixel', 'leakage_intensity_width_1', 'leakage_intensity_width_2', 'leakage_pixels_width_1', 'leakage_pixels_width_2', 'n_islands', 'intercept', 'time_gradient', 'n_pixels', 'wl', 'log_intensity' ]) nodes_keys = get_dataset_keys(args.input_file) if args.noimage: nodes_keys.remove(dl1_images_lstcam_key) auto_merge_h5files([args.input_file], args.output_file, nodes_keys=nodes_keys) with tables.open_file(args.input_file, mode='r') as input: image_table = input.root[dl1_images_lstcam_key] dl1_params_input = input.root[dl1_params_lstcam_key].colnames disp_params = {'disp_dx', 'disp_dy', 'disp_norm', 'disp_angle', 'disp_sign'} if set(dl1_params_input).intersection(disp_params): parameters_to_update.extend(disp_params) with tables.open_file(args.output_file, mode='a') as output: params = output.root[dl1_params_lstcam_key].read() for ii, row in enumerate(image_table): dl1_container.reset() image = row['image'] peak_time = row['peak_time'] signal_pixels = tailcuts_clean(camera_geom, image, **config['tailcut']) n_pixels = np.count_nonzero(signal_pixels) if n_pixels > 0: num_islands, island_labels = number_of_islands(camera_geom, signal_pixels) n_pixels_on_island = np.bincount(island_labels.astype(np.int)) n_pixels_on_island[0] = 0 # first island is no-island and should not be considered max_island_label = np.argmax(n_pixels_on_island) signal_pixels[island_labels != max_island_label] = False hillas = hillas_parameters(camera_geom[signal_pixels], image[signal_pixels]) dl1_container.fill_hillas(hillas) dl1_container.set_timing_features(camera_geom[signal_pixels], image[signal_pixels], peak_time[signal_pixels], hillas) dl1_container.set_leakage(camera_geom, image, signal_pixels) dl1_container.set_concentration(camera_geom, image, hillas) dl1_container.n_islands = num_islands dl1_container.wl = dl1_container.width / dl1_container.length dl1_container.n_pixels = n_pixels width = np.rad2deg(np.arctan2(dl1_container.width, foclen)) length = np.rad2deg(np.arctan2(dl1_container.length, foclen)) dl1_container.width = width dl1_container.length = length dl1_container.log_intensity = np.log10(dl1_container.intensity) if set(dl1_params_input).intersection(disp_params): disp_dx, disp_dy, disp_norm, disp_angle, disp_sign = disp( dl1_container['x'].to_value(u.m), dl1_container['y'].to_value(u.m), params['src_x'][ii], params['src_y'][ii] ) dl1_container['disp_dx'] = disp_dx dl1_container['disp_dy'] = disp_dy dl1_container['disp_norm'] = disp_norm dl1_container['disp_angle'] = disp_angle dl1_container['disp_sign'] = disp_sign for p in parameters_to_update: params[ii][p] = u.Quantity(dl1_container[p]).value output.root[dl1_params_lstcam_key][:] = params
def r0_to_dl1( input_filename=get_dataset_path('gamma_test_large.simtel.gz'), output_filename=None, custom_config={}, ): """ Chain r0 to dl1 Save the extracted dl1 parameters in output_filename Parameters ---------- input_filename: str path to input file, default: `gamma_test_large.simtel.gz` output_filename: str or None path to output file, defaults to writing dl1 into the current directory custom_config: path to a configuration file Returns ------- """ if output_filename is None: try: run = parse_r0_filename(input_filename) output_filename = run_to_dl1_filename(run.tel_id, run.run, run.subrun) except ValueError: output_filename = r0_to_dl1_filename(Path(input_filename).name) if os.path.exists(output_filename): raise IOError(str(output_filename) + ' exists, exiting.') config = replace_config(standard_config, custom_config) custom_calibration = config["custom_calibration"] source = EventSource(input_url=input_filename, config=Config(config["source_config"])) subarray = source.subarray is_simu = source.is_simulation metadata = global_metadata(source) write_metadata(metadata, output_filename) cal_mc = load_calibrator_from_config(config, subarray) # minimum number of pe in a pixel to include it # in calculation of muon ring time (peak sample): min_pe_for_muon_t_calc = 10. # Dictionary to store muon ring parameters muon_parameters = create_muon_table() # all this will be cleaned up in a next PR related to the configuration files r1_dl1_calibrator = CameraCalibrator( image_extractor_type=config['image_extractor'], config=Config(config), subarray=subarray) if not is_simu: # Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but # using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous r1_dl1_calibrator_for_muon_rings = CameraCalibrator( image_extractor_type=config['image_extractor_for_muons'], config=Config(config), subarray=subarray) # Component to process interleaved pedestal and flat-fields calib_config = Config(config[config['calibration_product']]) calibration_calculator = CalibrationCalculator.from_name( config['calibration_product'], config=calib_config, subarray=source.subarray) calibration_index = DL1MonitoringEventIndexContainer() dl1_container = DL1ParametersContainer() extra_im = ExtraImageInfo() extra_im.prefix = '' # get rid of the prefix # Write extra information to the DL1 file write_array_info(subarray, output_filename) write_array_info_08(subarray, output_filename) if is_simu: write_mcheader( source.simulation_config, output_filename, obs_id=source.obs_ids[0], filters=HDF5_ZSTD_FILTERS, metadata=metadata, ) with HDF5TableWriter( filename=output_filename, group_name='dl1/event', mode='a', filters=HDF5_ZSTD_FILTERS, add_prefix=True, # overwrite=True, ) as writer: if is_simu: subarray = subarray # build a mapping of tel_id back to tel_index: # (note this should be part of SubarrayDescription) idx = np.zeros(max(subarray.tel_indices) + 1) for key, val in subarray.tel_indices.items(): idx[key] = val # the final transform then needs the mapping and the number of telescopes tel_list_transform = partial( utils.expand_tel_list, max_tels=max(subarray.tel) + 1, ) writer.add_column_transform(table_name='subarray/trigger', col_name='tels_with_trigger', transform=tel_list_transform) # Forcing filters for the dl1 dataset that are currently read from the pre-existing files # This should be fixed in ctapipe and then corrected here writer._h5file.filters = HDF5_ZSTD_FILTERS logger.info(f"USING FILTERS: {writer._h5file.filters}") for i, event in enumerate(source): if i % 100 == 0: logger.info(i) event.dl0.prefix = '' event.trigger.prefix = '' if event.simulation is not None: event.simulation.prefix = 'mc' dl1_container.reset() # write sub tables if is_simu: write_subarray_tables(writer, event, metadata) if not custom_calibration: cal_mc(event) if config['mc_image_scaling_factor'] != 1: rescale_dl1_charge(event, config['mc_image_scaling_factor']) else: if i == 0: # initialize the telescope # FIXME? LST calibrator is only for one telescope # it should be inside the telescope loop (?) tel_id = calibration_calculator.tel_id #initialize the event monitoring data event.mon = source.r0_r1_calibrator.mon_data # write the first calibration event (initialized from calibration h5 file) write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=True, new_ff=True) # flat-field or pedestal: if (event.trigger.event_type == EventType.FLATFIELD or event.trigger.event_type == EventType.SKY_PEDESTAL): # process interleaved events (pedestals, ff, calibration) new_ped_event, new_ff_event = calibration_calculator.process_interleaved( event) # write monitoring containers if updated if new_ped_event or new_ff_event: write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=new_ped_event, new_ff=new_ff_event) # calibrate and gain select the event by hand for DL1 source.r0_r1_calibrator.calibrate(event) # create image for all events r1_dl1_calibrator(event) # Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten. # When the last version of the method is implemented, vol. reduction will be done at dl0 apply_volume_reduction(event, subarray, config) # FIXME? This should be eventually done after we evaluate whether the image is # a candidate muon ring. In that case the full image could be kept, or reduced # only after the ring analysis is complete. for ii, telescope_id in enumerate(event.dl1.tel.keys()): dl1_container.reset() # update the calibration index in the dl1 event container dl1_container.calibration_id = calibration_index.calibration_id dl1_container.fill_event_info(event) tel = event.dl1.tel[telescope_id] tel.prefix = '' # don't really need one # remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST' tel_name = str(subarray.tel[telescope_id])[4:] if custom_calibration: lst_calibration(event, telescope_id) write_event = True # Will determine whether this event has to be written to the # DL1 output or not. if is_simu: dl1_container.fill_mc(event, subarray.positions[telescope_id]) assert event.dl1.tel[telescope_id].image is not None try: get_dl1( event, subarray, telescope_id, dl1_container=dl1_container, custom_config=config, ) except HillasParameterizationError: logging.exception( 'HillasParameterizationError in get_dl1()') if not is_simu: dl1_container.ucts_time = 0 # convert Time to unix timestamp in (UTC) to keep compatibility # with older lstchain # FIXME: just keep it as time, table writer and reader handle it dl1_container.dragon_time = event.trigger.time.unix dl1_container.tib_time = 0 dl1_container.ucts_trigger_type = event.lst.tel[ telescope_id].evt.ucts_trigger_type dl1_container.trigger_type = event.lst.tel[ telescope_id].evt.tib_masked_trigger else: dl1_container.trigger_type = event.trigger.event_type dl1_container.az_tel = event.pointing.tel[telescope_id].azimuth dl1_container.alt_tel = event.pointing.tel[ telescope_id].altitude dl1_container.trigger_time = event.trigger.time.unix dl1_container.event_type = event.trigger.event_type # FIXME: no need to read telescope characteristics like foclen for every event! foclen = subarray.tel[ telescope_id].optics.equivalent_focal_length mirror_area = u.Quantity( subarray.tel[telescope_id].optics.mirror_area, u.m**2) dl1_container.prefix = tel.prefix # extra info for the image table extra_im.tel_id = telescope_id extra_im.selected_gain_channel = event.r1.tel[ telescope_id].selected_gain_channel for container in [extra_im, dl1_container, event.r0, tel]: add_global_metadata(container, metadata) event.r0.prefix = '' writer.write(table_name=f'telescope/image/{tel_name}', containers=[event.index, tel, extra_im]) writer.write(table_name=f'telescope/parameters/{tel_name}', containers=[event.index, dl1_container]) # Muon ring analysis, for real data only (MC is done starting from DL1 files) if not is_simu: bad_pixels = event.mon.tel[ telescope_id].calibration.unusable_pixels[0] # Set to 0 unreliable pixels: image = tel.image * (~bad_pixels) # process only promising events, in terms of # of pixels with large signals: if tag_pix_thr(image): # re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings numsamples = event.r1.tel[telescope_id].waveform.shape[ 1] # not necessarily the same as in r0! bad_pixels_hg = event.mon.tel[ telescope_id].calibration.unusable_pixels[0] bad_pixels_lg = event.mon.tel[ telescope_id].calibration.unusable_pixels[1] # Now set to 0 all samples in unreliable pixels. Important for global peak # integrator in case of crazy pixels! TBD: can this be done in a simpler # way? bad_pixels = bad_pixels_hg | bad_pixels_lg bad_waveform = np.transpose( np.array(numsamples * [bad_pixels])) # print('hg bad pixels:',np.where(bad_pixels_hg)) # print('lg bad pixels:',np.where(bad_pixels_lg)) event.r1.tel[telescope_id].waveform *= ~bad_waveform r1_dl1_calibrator_for_muon_rings(event) tel = event.dl1.tel[telescope_id] image = tel.image * (~bad_pixels) # Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum) # perhaps the event is no longer promising (e.g. if it has a large time evolution) if not tag_pix_thr(image): good_ring = False else: # read geometry from event.inst. But not needed for every event. FIXME? geom = subarray.tel[telescope_id].\ camera.geometry muonintensityparam, dist_mask, \ ring_size, size_outside_ring, muonringparam, \ good_ring, radial_distribution, \ mean_pixel_charge_around_ring,\ muonpars = \ analyze_muon_event(subarray, event.index.event_id, image, geom, foclen, mirror_area, False, '') # mirror_area, True, './') # (test) plot muon rings as png files # Now we want to obtain the waveform sample (in HG & LG) at which the ring light peaks: bright_pixels = image > min_pe_for_muon_t_calc selected_gain = event.r1.tel[ telescope_id].selected_gain_channel mask_hg = bright_pixels & (selected_gain == 0) mask_lg = bright_pixels & (selected_gain == 1) bright_pixels_waveforms_hg = event.r1.tel[ telescope_id].waveform[mask_hg, :] bright_pixels_waveforms_lg = event.r1.tel[ telescope_id].waveform[mask_lg, :] stacked_waveforms_hg = np.sum( bright_pixels_waveforms_hg, axis=0) stacked_waveforms_lg = np.sum( bright_pixels_waveforms_lg, axis=0) # stacked waveforms from all bright pixels; shape (ngains, nsamples) hg_peak_sample = np.argmax(stacked_waveforms_hg, axis=-1) lg_peak_sample = np.argmax(stacked_waveforms_lg, axis=-1) if good_ring: fill_muon_event(-1, muon_parameters, good_ring, event.index.event_id, dl1_container.dragon_time, muonintensityparam, dist_mask, muonringparam, radial_distribution, ring_size, size_outside_ring, mean_pixel_charge_around_ring, muonpars, hg_peak_sample, lg_peak_sample) # writes mc information per telescope, including photo electron image if (is_simu and config['write_pe_image'] and event.simulation.tel[telescope_id].true_image is not None and event.simulation.tel[telescope_id].true_image.any()): event.simulation.tel[telescope_id].prefix = '' writer.write(table_name=f'simulation/{tel_name}', containers=[ event.simulation.tel[telescope_id], extra_im ]) if not is_simu: # at the end of event loop ask calculation of remaining interleaved statistics new_ped, new_ff = calibration_calculator.output_interleaved_results( event) # write monitoring events write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=new_ped, new_ff=new_ff) if is_simu: # Reconstruct source position from disp for all events and write the result in the output file for tel_name in ['LST_LSTCam']: focal = OpticsDescription.from_name( tel_name.split('_')[0]).equivalent_focal_length add_disp_to_parameters_table(output_filename, dl1_params_lstcam_key, focal) # Write energy histogram from simtel file and extra metadata # ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever if source.max_events is None: write_simtel_energy_histogram(source, output_filename, obs_id=event.index.obs_id, metadata=metadata) else: dir, name = os.path.split(output_filename) name = name.replace('dl1', 'muons').replace('LST-1.1', 'LST-1') # Consider the possibilities of DL1 files with .fits.h5 & .h5 ending: name = name.replace('.fits.h5', '.fits').replace('.h5', '.fits') muon_output_filename = Path(dir, name) table = Table(muon_parameters) table.write(muon_output_filename, format='fits', overwrite=True)
def r0_to_dl1(input_filename=get_dataset_path('gamma_test_large.simtel.gz'), output_filename=None, custom_config={}, pedestal_path=None, calibration_path=None, time_calibration_path=None, pointing_file_path=None, ucts_t0_dragon=math.nan, dragon_counter0=math.nan, ucts_t0_tib=math.nan, tib_counter0=math.nan): """ Chain r0 to dl1 Save the extracted dl1 parameters in output_filename Parameters ---------- input_filename: str path to input file, default: `gamma_test_large.simtel.gz` output_filename: str or None path to output file, defaults to writing dl1 into the current directory custom_config: path to a configuration file pedestal_path: Path to the DRS4 pedestal file calibration_path: Path to the file with calibration constants and pedestals time_calibration_path: Path to the DRS4 time correction file pointing_file_path: path to the Drive log with the pointing information ucts_t0_dragon: first valid ucts_time dragon_counter0: Dragon counter corresponding to ucts_t0_dragon ucts_t0_tib: first valid ucts_time for the first valid TIB counter tib_counter0: first valid TIB counter Returns ------- """ if output_filename is None: try: run = parse_r0_filename(input_filename) output_filename = run_to_dl1_filename(run.tel_id, run.run, run.subrun) except ValueError: output_filename = r0_to_dl1_filename(Path(input_filename).name) if os.path.exists(output_filename): raise IOError(str(output_filename) + ' exists, exiting.') config = replace_config(standard_config, custom_config) custom_calibration = config["custom_calibration"] gain_selector = load_gain_selector_from_config(config) # FIXME for ctapipe 0.8, str should be removed, as Path is supported source = event_source(str(input_filename)) subarray = source.subarray is_simu = source.is_simulation source.allowed_tels = config["allowed_tels"] if config["max_events"] is not None: source.max_events = config["max_events"] metadata = global_metadata(source) write_metadata(metadata, output_filename) cal_mc = load_calibrator_from_config(config, subarray) # minimum number of pe in a pixel to include it # in calculation of muon ring time (peak sample): min_pe_for_muon_t_calc = 10. # Dictionary to store muon ring parameters muon_parameters = create_muon_table() if not is_simu: # TODO : add DRS4 calibration config in config file, read it and pass it here r0_r1_calibrator = LSTR0Corrections( pedestal_path=pedestal_path, tel_id=1, ) # all this will be cleaned up in a next PR related to the configuration files r1_dl1_calibrator = LSTCameraCalibrator( calibration_path=calibration_path, time_calibration_path=time_calibration_path, extractor_product=config['image_extractor'], gain_threshold=Config(config).gain_selector_config['threshold'], charge_scale=config['charge_scale'], config=Config(config), allowed_tels=[1], subarray=subarray) # Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but # using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous r1_dl1_calibrator_for_muon_rings = LSTCameraCalibrator( calibration_path=calibration_path, time_calibration_path=time_calibration_path, extractor_product=config['image_extractor_for_muons'], gain_threshold=Config(config).gain_selector_config['threshold'], charge_scale=config['charge_scale'], config=Config(config), allowed_tels=[1], subarray=subarray) # Component to process interleaved pedestal and flat-fields calib_config = Config(config[config['calibration_product']]) # set time calibration path for flatfield trailet () calib_config.FlasherFlatFieldCalculator.time_calibration_path = time_calibration_path calibration_calculator = CalibrationCalculator.from_name( config['calibration_product'], config=calib_config, subarray=source.subarray) calibration_index = DL1MonitoringEventIndexContainer() dl1_container = DL1ParametersContainer() if pointing_file_path: # Open drive report pointings = PointingPosition(drive_path=pointing_file_path) drive_data = pointings._read_drive_report() extra_im = ExtraImageInfo() extra_im.prefix = '' # get rid of the prefix # get the first event to write array info and mc header event_iter = iter(source) first_event = next(event_iter) # Write extra information to the DL1 file write_array_info(subarray, output_filename) write_array_info_08(subarray, output_filename) if is_simu: write_mcheader( first_event.mcheader, output_filename, obs_id=first_event.index.obs_id, filters=filters, metadata=metadata, ) with HDF5TableWriter( filename=output_filename, group_name='dl1/event', mode='a', filters=filters, add_prefix=True, # overwrite=True, ) as writer: if is_simu: subarray = subarray # build a mapping of tel_id back to tel_index: # (note this should be part of SubarrayDescription) idx = np.zeros(max(subarray.tel_indices) + 1) for key, val in subarray.tel_indices.items(): idx[key] = val # the final transform then needs the mapping and the number of telescopes tel_list_transform = partial( utils.expand_tel_list, max_tels=max(subarray.tel) + 1, ) writer.add_column_transform(table_name='subarray/trigger', col_name='tels_with_trigger', transform=tel_list_transform) # Forcing filters for the dl1 dataset that are currently read from the pre-existing files # This should be fixed in ctapipe and then corrected here writer._h5file.filters = filters logger.info(f"USING FILTERS: {writer._h5file.filters}") first_valid_ucts = None first_valid_ucts_tib = None previous_ucts_time_unix = [] previous_ucts_trigger_type = [] for i, event in enumerate(chain([first_event], event_iter)): if i % 100 == 0: logger.info(i) event.dl0.prefix = '' event.mc.prefix = 'mc' event.trigger.prefix = '' dl1_container.reset() # write sub tables if is_simu: write_subarray_tables(writer, event, metadata) if not custom_calibration: cal_mc(event) if config['mc_image_scaling_factor'] != 1: rescale_dl1_charge(event, config['mc_image_scaling_factor']) else: if i == 0: # initialize the telescope # FIXME? LST calibrator is only for one telescope # it should be inside the telescope loop (?) tel_id = calibration_calculator.tel_id # write the first calibration event (initialized from calibration h5 file) write_calibration_data( writer, calibration_index, r1_dl1_calibrator.mon_data.tel[tel_id], new_ped=True, new_ff=True) # drs4 calibrations r0_r1_calibrator.calibrate(event) # process interleaved events (pedestals, ff, calibration) new_ped_event, new_ff_event = calibration_calculator.process_interleaved( event) # write monitoring containers if updated if new_ped_event or new_ff_event: write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=new_ped_event, new_ff=new_ff_event) # calibrate and extract image from event r1_dl1_calibrator(event) # Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten. # When the last version of the method is implemented, vol. reduction will be done at dl0 apply_volume_reduction(event, subarray, config) # FIXME? This should be eventually done after we evaluate whether the image is # a candidate muon ring. In that case the full image could be kept, or reduced # only after the ring analysis is complete. for ii, telescope_id in enumerate(event.r0.tels_with_data): dl1_container.reset() # update the calibration index in the dl1 event container dl1_container.calibration_id = calibration_index.calibration_id dl1_container.fill_event_info(event) tel = event.dl1.tel[telescope_id] tel.prefix = '' # don't really need one # remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST' tel_name = str(subarray.tel[telescope_id])[4:] if custom_calibration: lst_calibration(event, telescope_id) write_event = True # Will determine whether this event has to be written to the # DL1 output or not. if is_simu: dl1_container.fill_mc(event, subarray.positions[telescope_id]) try: get_dl1(event, subarray, telescope_id, dl1_container=dl1_container, custom_config=config, use_main_island=True) except HillasParameterizationError: logging.exception( 'HillasParameterizationError in get_dl1()') if not is_simu: # GPS + WRS + UCTS is now working in its nominal configuration. # These TS are stored into ucts_time container. # TS can be alternatively calculated from the TIB and # Dragon modules counters based on the first valid UCTS TS # as the reference point. For the time being, the three TS # are stored in the DL1 files for checking purposes. module_id = 82 # Get counters from the central Dragon module if math.isnan(ucts_t0_dragon) and math.isnan(dragon_counter0) \ and math.isnan(ucts_t0_tib) and math.isnan(tib_counter0): # Dragon/TIB timestamps not based on a valid absolute reference timestamp dragon_time = (event.lst.tel[telescope_id].svc.date + event.lst.tel[telescope_id].evt. pps_counter[module_id] + event.lst.tel[telescope_id].evt. tenMHz_counter[module_id] * 10**(-7)) tib_time = ( event.lst.tel[telescope_id].svc.date + event.lst.tel[telescope_id].evt.tib_pps_counter + event.lst.tel[telescope_id].evt.tib_tenMHz_counter * 10**(-7)) if event.lst.tel[ telescope_id].evt.extdevices_presence & 2: # UCTS presence flag is OK ucts_time = event.lst.tel[ telescope_id].evt.ucts_timestamp * 1e-9 # secs if first_valid_ucts is None: first_valid_ucts = ucts_time initial_dragon_counter = ( event.lst.tel[telescope_id].evt. pps_counter[module_id] + event.lst.tel[telescope_id].evt. tenMHz_counter[module_id] * 10**(-7)) logger.warning( f"Dragon timestamps not based on a valid absolute reference timestamp. " f"Consider using the following initial values \n" f"Event ID: {event.index.event_id}, " f"First valid UCTS timestamp: {first_valid_ucts:.9f} s, " f"corresponding Dragon counter {initial_dragon_counter:.9f} s" ) if event.lst.tel[telescope_id].evt.extdevices_presence & 1 \ and first_valid_ucts_tib is None: # Both TIB and UCTS presence flags are OK first_valid_ucts_tib = ucts_time initial_tib_counter = ( event.lst.tel[telescope_id].evt. tib_pps_counter + event.lst.tel[telescope_id].evt. tib_tenMHz_counter * 10**(-7)) logger.warning( f"TIB timestamps not based on a valid absolute reference timestamp. " f"Consider using the following initial values \n" f"Event ID: {event.index.event_id}, UCTS timestamp corresponding to " f"the first valid TIB counter: {first_valid_ucts_tib:.9f} s, " f"corresponding TIB counter {initial_tib_counter:.9f} s" ) else: ucts_time = math.nan else: # Dragon/TIB timestamps based on a valid absolute reference UCTS timestamp dragon_time = ( (ucts_t0_dragon - dragon_counter0) * 1e-9 + # secs event.lst.tel[telescope_id].evt. pps_counter[module_id] + event.lst.tel[telescope_id].evt. tenMHz_counter[module_id] * 10**(-7)) tib_time = ( (ucts_t0_tib - tib_counter0) * 1e-9 + # secs event.lst.tel[telescope_id].evt.tib_pps_counter + event.lst.tel[telescope_id].evt.tib_tenMHz_counter * 10**(-7)) if event.lst.tel[ telescope_id].evt.extdevices_presence & 2: # UCTS presence flag is OK ucts_time = event.lst.tel[ telescope_id].evt.ucts_timestamp * 1e-9 # secs if first_valid_ucts is None: first_valid_ucts = ucts_time if first_valid_ucts_tib is None \ and event.lst.tel[telescope_id].evt.extdevices_presence & 1: first_valid_ucts_tib = ucts_time else: ucts_time = math.nan # FIXME: directly use unix_tai format whenever astropy v4.1 is out ucts_time_utc = unix_tai_to_time(ucts_time) dragon_time_utc = unix_tai_to_time(dragon_time) tib_time_utc = unix_tai_to_time(tib_time) dl1_container.ucts_time = ucts_time_utc.unix dl1_container.dragon_time = dragon_time_utc.unix dl1_container.tib_time = tib_time_utc.unix # Until the TIB trigger_type is fully reliable, we also add # the ucts_trigger_type to the data dl1_container.ucts_trigger_type = event.lst.tel[ telescope_id].evt.ucts_trigger_type # Due to a DAQ bug, sometimes there are 'jumps' in the # UCTS info in the raw files. After one such jump, # all the UCTS info attached to an event actually # corresponds to the next event. This one-event # shift stays like that until there is another jump # (then it becomes a 2-event shift and so on). We will # keep track of those jumps, by storing the UCTS info # of the previously read events in the list # previous_ucts_time_unix. The list has one element # for each of the jumps, so if there has been just # one jump we have the UCTS info of the previous # event only (which truly corresponds to the # current event). If there have been n jumps, we keep # the past n events. The info to be used for # the current event is always the first element of # the array, previous_ucts_time_unix[0], whereas the # current event's (wrong) ucts info is placed last in # the array. Each time the first array element is # used, it is removed and the rest move up in the # list. We have another similar array for the trigger # types, previous_ucts_trigger_type # if len(previous_ucts_time_unix) > 0: # keep the time & trigger type read for this # event (which really correspond to a later event): current_ucts_time = dl1_container.ucts_time current_ucts_trigger_type = dl1_container.ucts_trigger_type # put in dl1_container the proper time for this # event: dl1_container.ucts_time = \ previous_ucts_time_unix.pop(0) dl1_container.ucts_trigger_type = \ previous_ucts_trigger_type.pop(0) # now put the current values last in the list, # for later use: previous_ucts_time_unix.append(current_ucts_time) previous_ucts_trigger_type.\ append(current_ucts_trigger_type) # Now check consistency of UCTS and Dragon times. If # UCTS time is ahead of Dragon time by more than # 1.e-6 s, most likely the UCTS info has been # lost for this event (i.e. there has been another # 'jump' of those described above), and the one we have # actually corresponds to the next event. So we put it # back first in the list, to assign it to the next # event. We also move the other elements down in the # list, which will now be one element longer. # We leave the current event with the same time, # which will be approximately correct (depending on # event rate), and set its ucts_trigger_type to -1, # which will tell us a jump happened and hence this # event does not have proper UCTS info. if dl1_container.ucts_time - dl1_container.dragon_time > 1.e-6: previous_ucts_time_unix.\ insert( 0, dl1_container.ucts_time) previous_ucts_trigger_type.\ insert(0, dl1_container.ucts_trigger_type) dl1_container.ucts_trigger_type = -1 # Select the timestamps to be used for pointing interpolation if config['timestamps_pointing'] == "ucts": event_timestamps = dl1_container.ucts_time elif config['timestamps_pointing'] == "dragon": event_timestamps = dragon_time_utc.unix elif config['timestamps_pointing'] == "tib": event_timestamps = tib_time_utc.unix else: raise ValueError( "The timestamps_pointing option is not a valid one. \ Try ucts (default), dragon or tib.") if pointing_file_path and event_timestamps > 0: azimuth, altitude = pointings.cal_pointingposition( event_timestamps, drive_data) event.pointing.tel[telescope_id].azimuth = azimuth event.pointing.tel[telescope_id].altitude = altitude dl1_container.az_tel = azimuth dl1_container.alt_tel = altitude else: dl1_container.az_tel = u.Quantity(np.nan, u.rad) dl1_container.alt_tel = u.Quantity(np.nan, u.rad) dl1_container.trigger_time = event.r0.tel[ telescope_id].trigger_time dl1_container.trigger_type = event.r0.tel[ telescope_id].trigger_type # FIXME: no need to read telescope characteristics like foclen for every event! foclen = subarray.tel[ telescope_id].optics.equivalent_focal_length mirror_area = u.Quantity( subarray.tel[telescope_id].optics.mirror_area, u.m**2) dl1_container.prefix = tel.prefix # extra info for the image table extra_im.tel_id = telescope_id extra_im.selected_gain_channel = event.r1.tel[ telescope_id].selected_gain_channel for container in [extra_im, dl1_container, event.r0, tel]: add_global_metadata(container, metadata) event.r0.prefix = '' writer.write(table_name=f'telescope/image/{tel_name}', containers=[event.index, tel, extra_im]) writer.write(table_name=f'telescope/parameters/{tel_name}', containers=[event.index, dl1_container]) # Muon ring analysis, for real data only (MC is done starting from DL1 files) if not is_simu: bad_pixels = event.mon.tel[ telescope_id].calibration.unusable_pixels[0] # Set to 0 unreliable pixels: image = tel.image * (~bad_pixels) # process only promising events, in terms of # of pixels with large signals: if tag_pix_thr(image): # re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings numsamples = event.r1.tel[telescope_id].waveform.shape[ 2] # not necessarily the same as in r0! bad_pixels_hg = event.mon.tel[ telescope_id].calibration.unusable_pixels[0] bad_pixels_lg = event.mon.tel[ telescope_id].calibration.unusable_pixels[1] # Now set to 0 all samples in unreliable pixels. Important for global peak # integrator in case of crazy pixels! TBD: can this be done in a simpler # way? bad_waveform = np.array( ([ np.transpose( np.array(numsamples * [bad_pixels_hg])), np.transpose( np.array(numsamples * [bad_pixels_lg])) ])) # print('hg bad pixels:',np.where(bad_pixels_hg)) # print('lg bad pixels:',np.where(bad_pixels_lg)) event.r1.tel[telescope_id].waveform *= ~bad_waveform r1_dl1_calibrator_for_muon_rings(event) tel = event.dl1.tel[telescope_id] image = tel.image * (~bad_pixels) # Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum) # perhaps the event is no longer promising (e.g. if it has a large time evolution) if not tag_pix_thr(image): good_ring = False else: # read geometry from event.inst. But not needed for every event. FIXME? geom = subarray.tel[telescope_id].\ camera.geometry muonintensityparam, dist_mask, \ ring_size, size_outside_ring, muonringparam, \ good_ring, radial_distribution, \ mean_pixel_charge_around_ring,\ muonpars = \ analyze_muon_event(subarray, event.index.event_id, image, geom, foclen, mirror_area, False, '') # mirror_area, True, './') # (test) plot muon rings as png files # Now we want to obtain the waveform sample (in HG and LG) at which the ring light peaks: bright_pixels_waveforms = event.r1.tel[ telescope_id].waveform[:, image > min_pe_for_muon_t_calc, :] stacked_waveforms = np.sum(bright_pixels_waveforms, axis=-2) # stacked waveforms from all bright pixels; shape (ngains, nsamples) hg_peak_sample = np.argmax(stacked_waveforms, axis=-1)[0] lg_peak_sample = np.argmax(stacked_waveforms, axis=-1)[1] if good_ring: fill_muon_event(-1, muon_parameters, good_ring, event.index.event_id, dragon_time, muonintensityparam, dist_mask, muonringparam, radial_distribution, ring_size, size_outside_ring, mean_pixel_charge_around_ring, muonpars, hg_peak_sample, lg_peak_sample) # writes mc information per telescope, including photo electron image if is_simu \ and (event.mc.tel[telescope_id].true_image > 0).any() \ and config['write_pe_image']: event.mc.tel[telescope_id].prefix = '' writer.write( table_name=f'simulation/{tel_name}', containers=[event.mc.tel[telescope_id], extra_im]) if not is_simu: # at the end of event loop ask calculation of remaining interleaved statistics new_ped, new_ff = calibration_calculator.output_interleaved_results( event) # write monitoring events write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=new_ped, new_ff=new_ff) if first_valid_ucts is None: logger.warning("Not valid UCTS timestamp found") if first_valid_ucts_tib is None: logger.warning("Not valid TIB counter value found") if is_simu: # Reconstruct source position from disp for all events and write the result in the output file for tel_name in ['LST_LSTCam']: focal = OpticsDescription.from_name( tel_name.split('_')[0]).equivalent_focal_length add_disp_to_parameters_table(output_filename, dl1_params_lstcam_key, focal) # Write energy histogram from simtel file and extra metadata # ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever if source.max_events is None: write_simtel_energy_histogram(source, output_filename, obs_id=event.index.obs_id, metadata=metadata) else: dir, name = os.path.split(output_filename) name = name.replace('dl1', 'muons').replace('LST-1.1', 'LST-1') # Consider the possibilities of DL1 files with .fits.h5 & .h5 ending: name = name.replace('.fits.h5', '.fits').replace('.h5', '.fits') muon_output_filename = Path(dir, name) table = Table(muon_parameters) table.write(muon_output_filename, format='fits', overwrite=True) # Produce the dl1 datacheck .h5 file: check_dl1(output_filename, Path(output_filename).parent, max_cores=1, create_pdf=False)
def get_events(filename, storedata=False, test=False, concatenate=False, storeimg=False, outdir='./results/'): """ Depreciated, use r0_to_dl1. Read a Simtelarray file, extract pixels charge, calculate image parameters and timing parameters and store the result in an hdf5 file. Parameters: ----------- filename: str Name of the simtelarray file. storedata: boolean True: store extracted data in a hdf5 file concatenate: boolean True: store the extracted data at the end of an existing file storeimg: boolean True: store also pixel data outdir: srt Output directory Returns: -------- pandas DataFrame: output """ from warnings import warn warn("Deprecated: use r0_to_dl1") #Particle type: particle_type = utils.guess_type(filename) #Create data frame where DL2 data will be stored: features = [ 'obs_id', 'event_id', 'mc_energy', 'mc_alt', 'mc_az', 'mc_core_x', 'mc_core_y', 'mc_h_first_int', 'mc_type', 'gps_time', 'width', 'length', 'wl', 'phi', 'psi', 'r', 'x', 'y', 'intensity', 'skewness', 'kurtosis', 'mc_alt_tel', 'mc_az_tel', 'mc_core_distance', 'mc_x_max', 'time_gradient', 'intercept', 'src_x', 'src_y', 'disp_norm', ] output = pd.DataFrame(columns=features) #Read LST1 events: source = event_source(input_url=filename, allowed_tels={1}) #Open Simtelarray file #Cleaning levels: level1 = {'LSTCam': 6.} level2 = level1.copy() # We use as second cleaning level just half of the first cleaning level for key in level2: level2[key] *= 0.5 log10pixelHGsignal = {} survived = {} imagedata = np.array([]) for key in level1: log10pixelHGsignal[key] = [] survived[key] = [] i = 0 for event in source: if i % 100 == 0: print("EVENT_ID: ", event.r0.event_id, "TELS: ", event.r0.tels_with_data, "MC Energy:", event.mc.energy) i = i + 1 ntels = len(event.r0.tels_with_data) if test == True and i > 1000: # for quick tests break for ii, tel_id in enumerate(event.r0.tels_with_data): geom = event.inst.subarray.tel[tel_id].camera #Camera geometry tel_coords = event.inst.subarray.tel_coords[ event.inst.subarray.tel_indices[tel_id]] data = event.r0.tel[tel_id].waveform ped = event.mc.tel[tel_id].pedestal # the pedestal is the #average (for pedestal events) of the *sum* of all samples, #from sim_telarray nsamples = data.shape[2] # total number of samples # Subtract pedestal baseline. atleast_3d converts 2D to 3D matrix pedcorrectedsamples = data - np.atleast_3d(ped) / nsamples integrator = LocalPeakWindowSum() integration, pulse_time = integrator( pedcorrectedsamples ) # these are 2D matrices num_gains * num_pixels chan = 0 # high gain used for now... signals = integration[chan].astype(float) dc2pe = event.mc.tel[tel_id].dc_to_pe # numgains * numpixels signals *= dc2pe[chan] # Add all individual pixel signals to the numpy array of the # corresponding camera inside the log10pixelsignal dictionary log10pixelHGsignal[str(geom)].extend(np.log10(signals)) # Apply image cleaning cleanmask = tailcuts_clean(geom, signals, picture_thresh=level1[str(geom)], boundary_thresh=level2[str(geom)], keep_isolated_pixels=False, min_number_picture_neighbors=1) survived[str(geom)].extend(cleanmask) clean = signals.copy() clean[~cleanmask] = 0.0 # set to 0 pixels which did not # survive cleaning if np.max(clean) < 1.e-6: # skip images with no pixels continue # Calculate image parameters hillas = hillas_parameters(geom, clean) foclen = event.inst.subarray.tel[ tel_id].optics.equivalent_focal_length w = np.rad2deg(np.arctan2(hillas.width, foclen)) l = np.rad2deg(np.arctan2(hillas.length, foclen)) #Calculate Timing parameters peak_time = units.Quantity(pulse_time[chan]) * units.Unit("ns") timepars = time.timing_parameters(geom, clean, peak_time, hillas) if w >= 0: if storeimg == True: if imagedata.size == 0: imagedata = clean else: imagedata = np.vstack([imagedata, clean]) #Pixel content #Hillas parameters width = w.value length = l.value phi = hillas.phi.value psi = hillas.psi.value r = hillas.r.value x = hillas.x.value y = hillas.y.value intensity = np.log10(hillas.intensity) skewness = hillas.skewness kurtosis = hillas.kurtosis #MC data: obs_id = event.r0.obs_id event_id = event.r0.event_id mc_energy = np.log10(event.mc.energy.value * 1e3) #Log10(Energy) in GeV mc_alt = event.mc.alt.value mc_az = event.mc.az.value mc_core_x = event.mc.core_x.value mc_core_y = event.mc.core_y.value mc_h_first_int = event.mc.h_first_int.value mc_type = event.mc.shower_primary_id mc_az_tel = event.mcheader.run_array_direction[0].value mc_alt_tel = event.mcheader.run_array_direction[1].value mc_x_max = event.mc.x_max.value gps_time = event.trig.gps_time.value #Calculate mc_core_distance parameters mc_core_distance = np.sqrt( (tel_coords.x.value - event.mc.core_x.value)**2 + (tel_coords.y.value - event.mc.core_y.value)**2) #Timing parameters time_gradient = timepars['slope'].value intercept = timepars['intercept'] #Calculate disp_ and Source position in camera coordinates tel = OpticsDescription.from_name( 'LST') #Telescope description focal_length = tel.equivalent_focal_length.value sourcepos = utils.cal_cam_source_pos(mc_alt, mc_az, mc_alt_tel, mc_az_tel, focal_length) src_x = sourcepos[0] src_y = sourcepos[1] disp = utils.disp_norm(sourcepos[0], sourcepos[1], x, y) eventdf = pd.DataFrame([[ obs_id, event_id, mc_energy, mc_alt, mc_az, mc_core_x, mc_core_y, mc_h_first_int, mc_type, gps_time, width, length, width / length, phi, psi, r, x, y, intensity, skewness, kurtosis, mc_alt_tel, mc_az_tel, mc_core_distance, mc_x_max, time_gradient, intercept, src_x, src_y, disp, mc_type ]], columns=features) output = output.append(eventdf, ignore_index=True) outfile = outdir + particle_type + '_events.hdf5' if storedata == True: if (concatenate == False or (concatenate == True and np.DataSource().exists(outfile) == False)): output.to_hdf(outfile, key=particle_type + "_events", mode="w") if storeimg == True: f = h5py.File(outfile, 'r+') f.create_dataset('images', data=imagedata) f.close() else: if storeimg == True: f = h5py.File(outfile, 'r') images = f['images'] del f['images'] images = np.vstack([images, imagedata]) f.close() saved = pd.read_hdf(outfile, key=particle_type + '_events') output = saved.append(output, ignore_index=True) output.to_hdf(outfile, key=particle_type + "_events", mode="w") f = h5py.File(outfile, 'r+') f.create_dataset('images', data=images) f.close() else: saved = pd.read_hdf(outfile, key=particle_type + '_events') output = saved.append(output, ignore_index=True) output.to_hdf(outfile, key=particle_type + "_events", mode="w") del source return output
def get_events(filename, storedata=False, concatenate=False, storeimg=False, outdir='./results/'): """ Read a Simtelarray file, extract pixels charge, calculate image parameters and timing parameters and store the result in an hdf5 file. Parameters: filename: str Name of the simtelarray file. storedata: boolean True: store extracted data in a hdf5 file concatenate: boolean True: store the extracted data at the end of an existing file storeimg: boolean True: store also pixel data outdir: srt Output directory """ #Particle type: particle_type = guess_type(filename) #Create data frame where DL2 data will be stored: features = [ 'ObsID', 'EvID', 'mcEnergy', 'mcAlt', 'mcAz', 'mcCore_x', 'mcCore_y', 'mcHfirst', 'mcType', 'GPStime', 'width', 'length', 'w/l', 'phi', 'psi', 'r', 'x', 'y', 'intensity', 'skewness', 'kurtosis', 'mcAlttel', 'mcAztel', 'impact', 'mcXmax', 'time_gradient', 'intercept', 'SrcX', 'SrcY', 'disp', 'hadroness' ] output = pd.DataFrame(columns=features) #Read LST1 events: source = EventSourceFactory.produce( input_url=filename, allowed_tels={1}) #Open Simtelarray file #Cleaning levels: level1 = {'LSTCam': 6.} level2 = level1.copy() # We use as second cleaning level just half of the first cleaning level for key in level2: level2[key] *= 0.5 log10pixelHGsignal = {} survived = {} imagedata = np.array([]) for key in level1: log10pixelHGsignal[key] = [] survived[key] = [] i = 0 for event in source: if i % 100 == 0: print("EVENT_ID: ", event.r0.event_id, "TELS: ", event.r0.tels_with_data, "MC Energy:", event.mc.energy) i = i + 1 ntels = len(event.r0.tels_with_data) ''' if i > 100: # for quick tests break ''' for ii, tel_id in enumerate(event.r0.tels_with_data): geom = event.inst.subarray.tel[tel_id].camera #Camera geometry tel_coords = event.inst.subarray.tel_coords[ event.inst.subarray.tel_indices[tel_id]] data = event.r0.tel[tel_id].waveform ped = event.mc.tel[tel_id].pedestal # the pedestal is the average (for pedestal events) of the *sum* of all samples, from sim_telarray nsamples = data.shape[2] # total number of samples pedcorrectedsamples = data - np.atleast_3d( ped ) / nsamples # Subtract pedestal baseline. atleast_3d converts 2D to 3D matrix integrator = LocalPeakIntegrator(None, None) integration, peakpos, window = integrator.extract_charge( pedcorrectedsamples ) # these are 2D matrices num_gains * num_pixels chan = 0 # high gain used for now... signals = integration[chan].astype(float) dc2pe = event.mc.tel[tel_id].dc_to_pe # numgains * numpixels signals *= dc2pe[chan] # Add all individual pixel signals to the numpy array of the corresponding camera inside the log10pixelsignal dictionary log10pixelHGsignal[str(geom)].extend( np.log10(signals) ) # This seems to be faster like this, with normal python lists # Apply image cleaning cleanmask = tailcuts_clean(geom, signals, picture_thresh=level1[str(geom)], boundary_thresh=level2[str(geom)], keep_isolated_pixels=False, min_number_picture_neighbors=1) survived[str(geom)].extend( cleanmask ) # This seems to be faster like this, with normal python lists clean = signals.copy() clean[ ~cleanmask] = 0.0 # set to 0 pixels which did not survive cleaning if np.max(clean) < 1.e-6: # skip images with no pixels continue # Calculate image parameters hillas = hillas_parameters( geom, clean) # this one gives some warnings invalid value in sqrt foclen = event.inst.subarray.tel[ tel_id].optics.equivalent_focal_length w = np.rad2deg(np.arctan2(hillas.width, foclen)) l = np.rad2deg(np.arctan2(hillas.length, foclen)) #Calculate Timing parameters peak_time = units.Quantity(peakpos[chan]) * units.Unit("ns") timepars = time.timing_parameters(geom.pix_x, geom.pix_y, clean, peak_time, hillas.psi) if w >= 0: if storeimg == True: if imagedata.size == 0: imagedata = clean else: imagedata = np.vstack([imagedata, clean]) #Pixel content width = w.value length = l.value phi = hillas.phi.value psi = hillas.psi.value r = hillas.r.value x = hillas.x.value y = hillas.y.value intensity = np.log10(hillas.intensity) skewness = hillas.skewness kurtosis = hillas.kurtosis #Store parameters from event and MC: ObsID = event.r0.obs_id EvID = event.r0.event_id mcEnergy = np.log10(event.mc.energy.value * 1e3) #Log10(Energy) in GeV mcAlt = event.mc.alt.value mcAz = event.mc.az.value mcCore_x = event.mc.core_x.value mcCore_y = event.mc.core_y.value mcHfirst = event.mc.h_first_int.value mcType = event.mc.shower_primary_id mcAztel = event.mcheader.run_array_direction[0].value mcAlttel = event.mcheader.run_array_direction[1].value mcXmax = event.mc.x_max.value GPStime = event.trig.gps_time.value impact = np.sqrt( (tel_coords.x.value - event.mc.core_x.value)**2 + (tel_coords.y.value - event.mc.core_y.value)**2) time_gradient = timepars[0].value intercept = timepars[1].value #Calculate Disp and Source position in camera coordinates tel = OpticsDescription.from_name( 'LST') #Telescope description focal_length = tel.equivalent_focal_length.value sourcepos = transformations.calc_CamSourcePos( mcAlt, mcAz, mcAlttel, mcAztel, focal_length) SrcX = sourcepos[0] SrcY = sourcepos[1] disp = transformations.calc_DISP(sourcepos[0], sourcepos[1], x, y) hadroness = 0 if particle_type == 'proton': hadroness = 1 eventdf = pd.DataFrame([[ ObsID, EvID, mcEnergy, mcAlt, mcAz, mcCore_x, mcCore_y, mcHfirst, mcType, GPStime, width, length, width / length, phi, psi, r, x, y, intensity, skewness, kurtosis, mcAlttel, mcAztel, impact, mcXmax, time_gradient, intercept, SrcX, SrcY, disp, hadroness ]], columns=features) output = output.append(eventdf, ignore_index=True) outfile = outdir + particle_type + '_events.hdf5' if storedata == True: if concatenate == False or (concatenate == True and np.DataSource().exists(outfile) == False): output.to_hdf(outfile, key=particle_type + "_events", mode="w") if storeimg == True: f = h5py.File(outfile, 'r+') f.create_dataset('images', data=imagedata) f.close() else: if storeimg == True: f = h5py.File(outfile, 'r') images = f['images'] del f['images'] images = np.vstack([images, imagedata]) f.close() saved = pd.read_hdf(outfile, key=particle_type + '_events') output = saved.append(output, ignore_index=True) output.to_hdf(outfile, key=particle_type + "_events", mode="w") f = h5py.File(outfile, 'r+') f.create_dataset('images', data=images) f.close() else: saved = pd.read_hdf(outfile, key=particle_type + '_events') output = saved.append(output, ignore_index=True) output.to_hdf(outfile, key=particle_type + "_events", mode="w") del source return output
def r0_to_dl1(input_filename=get_dataset_path('gamma_test_large.simtel.gz'), output_filename=None, custom_config={}, pedestal_path=None, calibration_path=None, time_calibration_path=None, pointing_file_path=None, ucts_t0_dragon=math.nan, dragon_counter0=math.nan, ucts_t0_tib=math.nan, tib_counter0=math.nan): """ Chain r0 to dl1 Save the extracted dl1 parameters in output_filename Parameters ---------- input_filename: str path to input file, default: `gamma_test_large.simtel.gz` output_filename: str path to output file, default: `./` + basename(input_filename) custom_config: path to a configuration file pedestal_path: Path to the DRS4 pedestal file calibration_path: Path to the file with calibration constants and pedestals time_calibration_path: Path to the DRS4 time correction file pointing_file_path: path to the Drive log with the pointing information Arguments below are just temporal and will be removed whenever UCTS+EvB is proved to stably and reliably provide timestamps. ucts_t0_dragon: first valid ucts_time dragon_counter0: Dragon counter corresponding to ucts_t0_dragon ucts_t0_tib: first valid ucts_time for the first valid TIB counter tib_counter0: first valid TIB counter Returns ------- """ if output_filename is None: output_filename = ('dl1_' + os.path.basename(input_filename).rsplit('.', 1)[0] + '.h5') if os.path.exists(output_filename): raise AttributeError(output_filename + ' exists, exiting.') config = replace_config(standard_config, custom_config) custom_calibration = config["custom_calibration"] try: source = event_source(input_filename, back_seekable=True) except: # back_seekable might not be available for other sources that eventio # TODO for real data: source with calibration file and pointing file source = event_source(input_filename) is_simu = source.metadata['is_simulation'] source.allowed_tels = config["allowed_tels"] if config["max_events"] is not None: source.max_events = config["max_events"] + 1 metadata = global_metadata(source) write_metadata(metadata, output_filename) cal_mc = load_calibrator_from_config(config) # minimum number of pe in a pixel to include it in calculation of muon ring time (peak sample): min_pe_for_muon_t_calc = 10. # Dictionary to store muon ring parameters muon_parameters = create_muon_table() if not is_simu: # TODO : add DRS4 calibration config in config file, read it and pass it here r0_r1_calibrator = LSTR0Corrections(pedestal_path=pedestal_path, tel_id=1) # all this will be cleaned up in a next PR related to the configuration files r1_dl1_calibrator = LSTCameraCalibrator( calibration_path=calibration_path, time_calibration_path=time_calibration_path, extractor_product=config['image_extractor'], gain_threshold=Config(config).gain_selector_config['threshold'], config=Config(config), allowed_tels=[1], ) # Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but # using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous r1_dl1_calibrator_for_muon_rings = LSTCameraCalibrator( calibration_path=calibration_path, time_calibration_path=time_calibration_path, extractor_product=config['image_extractor_for_muons'], gain_threshold=Config(config).gain_selector_config['threshold'], config=Config(config), allowed_tels=[1], ) dl1_container = DL1ParametersContainer() if pointing_file_path: # Open drive report pointings = PointingPosition() pointings.drive_path = pointing_file_path drive_data = pointings._read_drive_report() extra_im = ExtraImageInfo() extra_im.prefix = '' # get rid of the prefix event = next(iter(source)) write_array_info(event, output_filename) ### Write extra information to the DL1 file if is_simu: write_mcheader(event.mcheader, output_filename, obs_id=event.r0.obs_id, filters=filters, metadata=metadata) subarray = event.inst.subarray with HDF5TableWriter( filename=output_filename, group_name='dl1/event', mode='a', filters=filters, add_prefix=True, # overwrite = True, ) as writer: print("USING FILTERS: ", writer._h5file.filters) if is_simu: # build a mapping of tel_id back to tel_index: # (note this should be part of SubarrayDescription) idx = np.zeros(max(subarray.tel_indices) + 1) for key, val in subarray.tel_indices.items(): idx[key] = val # the final transform then needs the mapping and the number of telescopes tel_list_transform = partial( utils.expand_tel_list, max_tels=len(event.inst.subarray.tel) + 1, ) writer.add_column_transform(table_name='subarray/trigger', col_name='tels_with_trigger', transform=tel_list_transform) ### EVENT LOOP ### for i, event in enumerate(source): if i % 100 == 0: print(i) event.dl0.prefix = '' event.mc.prefix = 'mc' event.trig.prefix = '' # write sub tables if is_simu: write_subarray_tables(writer, event, metadata) if not custom_calibration: cal_mc(event) else: r0_r1_calibrator.calibrate(event) r1_dl1_calibrator(event) # Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten. # When the last version of the method is implemented, vol. reduction will be done at dl0 check_and_apply_volume_reduction(event, config) # FIXME? This should be eventually done after we evaluate whether the image is # a candidate muon ring. In that case the full image could be kept, or reduced # only after the ring analysis is complete. for ii, telescope_id in enumerate(event.r0.tels_with_data): tel = event.dl1.tel[telescope_id] tel.prefix = '' # don't really need one # remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST' tel_name = str(event.inst.subarray.tel[telescope_id])[4:] tel_name = tel_name.replace('-003', '') if custom_calibration: lst_calibration(event, telescope_id) try: dl1_filled = get_dl1(event, telescope_id, dl1_container=dl1_container, custom_config=config, use_main_island=True) except HillasParameterizationError: logging.exception( 'HillasParameterizationError in get_dl1()') continue if dl1_filled is not None: # Some custom def dl1_container.wl = dl1_container.width / dl1_container.length # Log10(Energy) in GeV if is_simu: dl1_container.mc_energy = event.mc.energy.value dl1_container.log_mc_energy = np.log10( event.mc.energy.value * 1e3) dl1_container.fill_mc(event) dl1_container.log_intensity = np.log10( dl1_container.intensity) dl1_container.gps_time = event.trig.gps_time.value if not is_simu: # GPS + WRS + UCTS is now working in its nominal configuration. # These TS are stored into ucts_time container. # TS can be alternatively calculated from the TIB and # Dragon modules counters based on the first valid UCTS TS # as the reference point. For the time being, the three TS # are stored in the DL1 files for checking purposes. ucts_time = event.lst.tel[ telescope_id].evt.ucts_timestamp * 1e-9 # secs module_id = 82 # Get counters from the central Dragon module if math.isnan(ucts_t0_dragon) and math.isnan(dragon_counter0) \ and math.isnan(ucts_t0_tib) and math.isnan(tib_counter0): # Dragon/TIB timestamps not based on a valid absolute reference timestamp dragon_time = ( event.lst.tel[telescope_id].svc.date + event.lst.tel[telescope_id].evt. pps_counter[module_id] + event.lst.tel[telescope_id].evt. tenMHz_counter[module_id] * 10**(-7)) tib_time = ( event.lst.tel[telescope_id].svc.date + event.lst.tel[telescope_id].evt.tib_pps_counter + event.lst.tel[telescope_id].evt. tib_tenMHz_counter * 10**(-7)) else: # Dragon/TIB timestamps based on a valid absolute reference timestamp dragon_time = ( (ucts_t0_dragon - dragon_counter0) * 1e-9 + # secs event.lst.tel[telescope_id].evt. pps_counter[module_id] + event.lst.tel[telescope_id].evt. tenMHz_counter[module_id] * 10**(-7)) tib_time = ( (ucts_t0_tib - tib_counter0) * 1e-9 + # secs event.lst.tel[telescope_id].evt.tib_pps_counter + event.lst.tel[telescope_id].evt. tib_tenMHz_counter * 10**(-7)) # FIXME: directly use unix_tai format whenever astropy v4.1 is out ucts_time_utc = unix_tai_to_utc(ucts_time) dragon_time_utc = unix_tai_to_utc(dragon_time) tib_time_utc = unix_tai_to_utc(tib_time) dl1_container.ucts_time = ucts_time_utc.unix dl1_container.dragon_time = dragon_time_utc.unix dl1_container.tib_time = tib_time_utc.unix # Select the timestamps to be used for pointing interpolation if config['timestamps_pointing'] == "ucts": event_timestamps = ucts_time_utc.unix elif config['timestamps_pointing'] == "dragon": event_timestamps = dragon_time_utc.unix elif config['timestamps_pointing'] == "tib": event_timestamps = tib_time_utc.unix else: raise ValueError( "The timestamps_pointing option is not a valid one. \ Try ucts (default), dragon or tib." ) if pointing_file_path and event_timestamps > 0: azimuth, altitude = pointings.cal_pointingposition( event_timestamps, drive_data) event.pointing[telescope_id].azimuth = azimuth event.pointing[telescope_id].altitude = altitude dl1_container.az_tel = azimuth dl1_container.alt_tel = altitude else: dl1_container.az_tel = u.Quantity(np.nan, u.rad) dl1_container.alt_tel = u.Quantity(np.nan, u.rad) # Until the TIB trigger_type is fully reliable, we also add # the ucts_trigger_type to the data extra_im.ucts_trigger_type = event.lst.tel[ telescope_id].evt.ucts_trigger_type # FIXME: no need to read telescope characteristics like foclen for every event! foclen = event.inst.subarray.tel[ telescope_id].optics.equivalent_focal_length mirror_area = u.Quantity( event.inst.subarray.tel[telescope_id].optics. mirror_area, u.m**2) width = np.rad2deg(np.arctan2(dl1_container.width, foclen)) length = np.rad2deg( np.arctan2(dl1_container.length, foclen)) dl1_container.width = width.value dl1_container.length = length.value dl1_container.prefix = tel.prefix extra_im.tel_id = telescope_id extra_im.num_trig_pix = event.r0.tel[ telescope_id].num_trig_pix extra_im.trigger_time = event.r0.tel[ telescope_id].trigger_time extra_im.trigger_type = event.r0.tel[ telescope_id].trigger_type extra_im.trig_pix_id = event.r0.tel[ telescope_id].trig_pix_id for container in [extra_im, dl1_container, event.r0, tel]: add_global_metadata(container, metadata) event.r0.prefix = '' writer.write(table_name=f'telescope/image/{tel_name}', containers=[event.r0, tel, extra_im]) writer.write(table_name=f'telescope/parameters/{tel_name}', containers=[dl1_container, extra_im]) # Muon ring analysis, for real data only (MC is done starting from DL1 files) if not is_simu: bad_pixels = event.mon.tel[ telescope_id].calibration.unusable_pixels[0] # Set to 0 unreliable pixels: image = tel.image * (~bad_pixels) # process only promising events, in terms of # of pixels with large signals: if tag_pix_thr(image): # re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings numsamples = event.r1.tel[ telescope_id].waveform.shape[ 2] # not necessarily the same as in r0! bad_pixels_hg = event.mon.tel[ telescope_id].calibration.unusable_pixels[0] bad_pixels_lg = event.mon.tel[ telescope_id].calibration.unusable_pixels[1] # Now set to 0 all samples in unreliable pixels. Important for global peak # integrator in case of crazy pixels! TBD: can this be done in a simpler # way? bad_waveform = np.array(([ np.transpose( np.array(numsamples * [bad_pixels_hg])), np.transpose( np.array(numsamples * [bad_pixels_lg])) ])) # print('hg bad pixels:',np.where(bad_pixels_hg)) # print('lg bad pixels:',np.where(bad_pixels_lg)) event.r1.tel[ telescope_id].waveform *= ~bad_waveform r1_dl1_calibrator_for_muon_rings(event) tel = event.dl1.tel[telescope_id] image = tel.image * (~bad_pixels) # Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum) # perhaps the event is no longer promising (e.g. if it has a large time evolution) if not tag_pix_thr(image): good_ring = False else: # read geometry from event.inst. But not needed for every event. FIXME? geom = event.inst.subarray.tel[ telescope_id].camera muonintensityparam, size_outside_ring, muonringparam, good_ring, \ radial_distribution, mean_pixel_charge_around_ring = \ analyze_muon_event(event.r0.event_id, image, geom, foclen, mirror_area, False, '') # mirror_area, True, './') # (test) plot muon rings as png files # Now we want to obtain the waveform sample (in HG and LG) at which the ring light peaks: bright_pixels_waveforms = event.r1.tel[ telescope_id].waveform[:, image > min_pe_for_muon_t_calc, :] stacked_waveforms = np.sum( bright_pixels_waveforms, axis=-2) # stacked waveforms from all bright pixels; shape (ngains, nsamples) hg_peak_sample = np.argmax(stacked_waveforms, axis=-1)[0] lg_peak_sample = np.argmax(stacked_waveforms, axis=-1)[1] if good_ring: fill_muon_event(muon_parameters, good_ring, event.r0.event_id, dragon_time, muonintensityparam, muonringparam, radial_distribution, size_outside_ring, mean_pixel_charge_around_ring, hg_peak_sample, lg_peak_sample) # writes mc information per telescope, including photo electron image if is_simu \ and (event.mc.tel[telescope_id].photo_electron_image > 0).any() \ and config['write_pe_image']: event.mc.tel[telescope_id].prefix = '' writer.write( table_name=f'simulation/{tel_name}', containers=[event.mc.tel[telescope_id], extra_im]) if is_simu: ### Reconstruct source position from disp for all events and write the result in the output file for tel_name in ['LST_LSTCam']: focal = OpticsDescription.from_name( tel_name.split('_')[0]).equivalent_focal_length dl1_params_key = f'dl1/event/telescope/parameters/{tel_name}' add_disp_to_parameters_table(output_filename, dl1_params_key, focal) # Write energy histogram from simtel file and extra metadata # ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever if source.max_events is None: write_simtel_energy_histogram(source, output_filename, obs_id=event.dl0.obs_id, metadata=metadata) else: dir = os.path.dirname(output_filename) name = os.path.basename(output_filename) k = name.find('Run') muon_output_filename = name[0:name.find('LST-')+5] + '.' + \ name[k:k+13] + '.fits' muon_output_filename = dir + '/' + muon_output_filename.replace( "dl1", "muons") table = Table(muon_parameters) table.write(muon_output_filename, format='fits', overwrite=True)
def main(): std_config = get_standard_config() if args.config_file is not None: config = replace_config(std_config, read_configuration_file(args.config_file)) else: config = std_config print(config['tailcut']) foclen = OpticsDescription.from_name('LST').equivalent_focal_length cam_table = Table.read(args.input_file, path="instrument/telescope/camera/LSTCam") camera_geom = CameraGeometry.from_table(cam_table) dl1_container = DL1ParametersContainer() parameters_to_update = list(HillasParametersContainer().keys()) parameters_to_update.extend([ 'concentration_cog', 'concentration_core', 'concentration_pixel', 'leakage_intensity_width_1', 'leakage_intensity_width_2', 'leakage_pixels_width_1', 'leakage_pixels_width_2', 'n_islands', 'intercept', 'time_gradient', 'n_pixels', 'wl', 'r', ]) nodes_keys = get_dataset_keys(args.input_file) if args.noimage: nodes_keys.remove(dl1_images_lstcam_key) auto_merge_h5files([args.input_file], args.output_file, nodes_keys=nodes_keys) with tables.open_file(args.input_file, mode='r') as input: image_table = input.root[dl1_images_lstcam_key] with tables.open_file(args.output_file, mode='a') as output: params = output.root[dl1_params_lstcam_key].read() for ii, row in enumerate(image_table): if ii % 10000 == 0: print(ii) image = row['image'] peak_time = row['peak_time'] signal_pixels = tailcuts_clean(camera_geom, image, **config['tailcut']) n_pixels = np.count_nonzero(signal_pixels) if n_pixels > 0: num_islands, island_labels = number_of_islands( camera_geom, signal_pixels) n_pixels_on_island = np.bincount( island_labels.astype(np.int)) n_pixels_on_island[ 0] = 0 # first island is no-island and should not be considered max_island_label = np.argmax(n_pixels_on_island) signal_pixels[island_labels != max_island_label] = False hillas = hillas_parameters(camera_geom[signal_pixels], image[signal_pixels]) dl1_container.fill_hillas(hillas) dl1_container.set_timing_features( camera_geom[signal_pixels], image[signal_pixels], peak_time[signal_pixels], hillas) dl1_container.set_leakage(camera_geom, image, signal_pixels) dl1_container.set_concentration(camera_geom, image, hillas) dl1_container.n_islands = num_islands dl1_container.wl = dl1_container.width / dl1_container.length dl1_container.n_pixels = n_pixels width = np.rad2deg(np.arctan2(dl1_container.width, foclen)) length = np.rad2deg( np.arctan2(dl1_container.length, foclen)) dl1_container.width = width dl1_container.length = length dl1_container.r = np.sqrt(dl1_container.x**2 + dl1_container.y**2) else: # for consistency with r0_to_dl1.py: for key in dl1_container.keys(): dl1_container[key] = \ u.Quantity(0, dl1_container.fields[key].unit) dl1_container.width = u.Quantity(np.nan, u.m) dl1_container.length = u.Quantity(np.nan, u.m) dl1_container.wl = u.Quantity(np.nan, u.m) for p in parameters_to_update: params[ii][p] = u.Quantity(dl1_container[p]).value output.root[dl1_params_lstcam_key][:] = params
def _build_subarray_info(self, run): """ constructs a SubarrayDescription object from the info in an MCRun Parameters ---------- run: MCRun object Returns ------- SubarrayDescription : instrumental information """ subarray = SubarrayDescription("MonteCarloArray") runHeader = run.root.RunHeader tabFocalTel = runHeader.tabFocalTel.read() tabPosTelX = runHeader.tabPosTelX.read() tabPosTelY = runHeader.tabPosTelY.read() tabPosTelZ = runHeader.tabPosTelZ.read() tabPoslXYZ = np.ascontiguousarray( np.vstack((tabPosTelX, tabPosTelY, tabPosTelZ)).T) ''' # Correspance HiPeData.Telscope.Type and camera name # 0 LSTCam, 1 NectarCam, 2 FlashCam, 3 SCTCam, # 4 ASTRICam, 5 DigiCam, 6 CHEC ''' mapping_camera = { 0: 'LSTCam', 1: 'NectarCam', 2: 'FlashCam', 3: 'SCTCam', 4: 'ASTRICam', 5: 'DigiCam', 6: 'CHEC' } mapping_telName = { 0: 'LST', 1: 'MST', 2: 'MST', 3: 'MST', 4: 'SST-ASTRI', 5: 'SST-1M', 6: 'SST-2M' } for telNode in self.run.walk_nodes('/Tel', 'Group'): try: telType = uint64(telNode.telType.read()) telIndex = uint64(telNode.telIndex.read()) telId = uint64(telNode.telId.read()) cameraName = mapping_camera[telType] telName = mapping_telName[telType] camera = CameraGeometry.from_name(cameraName) camera.cam_id = cameraName foclen = tabFocalTel[telIndex] * u.m tel_pos = tabPoslXYZ[telIndex] * u.m camera.pix_x = telNode.tabPixelX.read() * u.m camera.pix_y = telNode.tabPixelY.read() * u.m #camera.rotate(-90.0*u.deg) optic = OpticsDescription.from_name(telName) optic.equivalent_focal_length = foclen telescope_description = TelescopeDescription(telName, telName, optics=optic, camera=camera) #tel.optics.mirror_area = mirror_area #tel.optics.num_mirror_tiles = num_tiles subarray.tels[telId] = telescope_description subarray.positions[telId] = tel_pos except tables.exceptions.NoSuchNodeError as e: pass return subarray
def __init__(self, **kwargs): """ Constructor Parameters ---------- kwargs: dict Parameters to be passed. NOTE: The file mask of the data to read can be passed with the 'input_url' parameter. """ try: import uproot except ImportError: raise ImportError( "The 'uproot' package is required for the DLMAGICEventSource class." ) self.file_list = glob.glob(kwargs['input_url']) self.file_list.sort() # Since EventSource can not handle file wild cards as input_url # We substitute the input_url with first file matching # the specified file mask. del kwargs['input_url'] super().__init__(input_url=self.file_list[0], **kwargs) # get run number mask = r".*_za\d+to\d+_\d_(\d+)_([A-Z]+)_.*" parsed_info = re.findall(mask, self.file_list[0]) self.run_number = parsed_info[0][0] # MAGIC telescope positions in m wrt. to the center of CTA simulations self.magic_tel_positions = { 1: [-27.24, -146.66, 50.00] * u.m, 2: [-96.44, -96.77, 51.00] * u.m } self.magic_tel_positions = self.magic_tel_positions # MAGIC telescope description optics = OpticsDescription.from_name('MAGIC') geom = CameraGeometry.from_name('MAGICCam') # Camera Readout for NectarCam used as a placeholder readout = CameraReadout( 'MAGICCam', sampling_rate=u.Quantity(1, u.GHz), reference_pulse_shape=np.array([norm.pdf(np.arange(96), 48, 6)]), reference_pulse_sample_width=u.Quantity(1, u.ns)) camera = CameraDescription('MAGICCam', geom, readout) self.magic_tel_description = TelescopeDescription(name='MAGIC', tel_type='LST', optics=optics, camera=camera) self.magic_tel_descriptions = { 1: self.magic_tel_description, 2: self.magic_tel_description } self.magic_subarray = SubarrayDescription('MAGIC', self.magic_tel_positions, self.magic_tel_descriptions) # Open ROOT files self.calib_M1, self.calib_M2, self.star_M1, self.star_M2, self.superstar = None, None, None, None, None for file in self.file_list: uproot_file = uproot.open(file) if "_Y_" in file: if "_M1_" in file: self.calib_M1 = uproot_file["Events"] self.meta = uproot_file["RunHeaders"] elif "_M2_" in file: self.calib_M2 = uproot_file["Events"] if "_I_" in file: if "_M1_" in file: self.star_M1 = uproot_file["Events"] elif "_M2_" in file: self.star_M2 = uproot_file["Events"] if "_S_" in file: self.superstar = uproot_file["Events"] self.meta = uproot_file["RunHeaders"] self._mc_header = self._parse_mc_header()
def main(): std_config = get_standard_config() log.setLevel(logging.INFO) handler = logging.StreamHandler() logging.getLogger().addHandler(handler) if args.config_file is not None: config = replace_config(std_config, read_configuration_file(args.config_file)) else: config = std_config if args.pedestal_cleaning: print("Pedestal cleaning") clean_method_name = 'tailcuts_clean_with_pedestal_threshold' sigma = config[clean_method_name]['sigma'] pedestal_thresh = get_threshold_from_dl1_file(args.input_file, sigma) cleaning_params = get_cleaning_parameters(config, clean_method_name) pic_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params log.info( f"Fraction of pixel cleaning thresholds above picture thr.:" f"{np.sum(pedestal_thresh>pic_th) / len(pedestal_thresh):.3f}") picture_th = np.clip(pedestal_thresh, pic_th, None) log.info(f"Tailcut clean with pedestal threshold config used:" f"{config['tailcuts_clean_with_pedestal_threshold']}") else: clean_method_name = 'tailcut' cleaning_params = get_cleaning_parameters(config, clean_method_name) picture_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params log.info(f"Tailcut config used: {config['tailcut']}") use_only_main_island = True if "use_only_main_island" in config[clean_method_name]: use_only_main_island = config[clean_method_name][ "use_only_main_island"] delta_time = None if "delta_time" in config[clean_method_name]: delta_time = config[clean_method_name]["delta_time"] foclen = OpticsDescription.from_name('LST').equivalent_focal_length cam_table = Table.read(args.input_file, path="instrument/telescope/camera/LSTCam") camera_geom = CameraGeometry.from_table(cam_table) dl1_container = DL1ParametersContainer() parameters_to_update = [ 'intensity', 'x', 'y', 'r', 'phi', 'length', 'width', 'psi', 'skewness', 'kurtosis', 'concentration_cog', 'concentration_core', 'concentration_pixel', 'leakage_intensity_width_1', 'leakage_intensity_width_2', 'leakage_pixels_width_1', 'leakage_pixels_width_2', 'n_islands', 'intercept', 'time_gradient', 'n_pixels', 'wl', 'log_intensity' ] nodes_keys = get_dataset_keys(args.input_file) if args.noimage: nodes_keys.remove(dl1_images_lstcam_key) auto_merge_h5files([args.input_file], args.output_file, nodes_keys=nodes_keys) with tables.open_file(args.input_file, mode='r') as input: image_table = input.root[dl1_images_lstcam_key] dl1_params_input = input.root[dl1_params_lstcam_key].colnames disp_params = { 'disp_dx', 'disp_dy', 'disp_norm', 'disp_angle', 'disp_sign' } if set(dl1_params_input).intersection(disp_params): parameters_to_update.extend(disp_params) with tables.open_file(args.output_file, mode='a') as output: params = output.root[dl1_params_lstcam_key].read() for ii, row in enumerate(image_table): dl1_container.reset() image = row['image'] peak_time = row['peak_time'] signal_pixels = tailcuts_clean(camera_geom, image, picture_th, boundary_th, isolated_pixels, min_n_neighbors) n_pixels = np.count_nonzero(signal_pixels) if n_pixels > 0: num_islands, island_labels = number_of_islands( camera_geom, signal_pixels) n_pixels_on_island = np.bincount( island_labels.astype(np.int64)) n_pixels_on_island[ 0] = 0 # first island is no-island and should not be considered max_island_label = np.argmax(n_pixels_on_island) if use_only_main_island: signal_pixels[ island_labels != max_island_label] = False # if delta_time has been set, we require at least one # neighbor within delta_time to accept a pixel in the image: if delta_time is not None: cleaned_pixel_times = peak_time # makes sure only signal pixels are used in the time # check: cleaned_pixel_times[~signal_pixels] = np.nan new_mask = apply_time_delta_cleaning( camera_geom, signal_pixels, cleaned_pixel_times, 1, delta_time) signal_pixels = new_mask # count the surviving pixels n_pixels = np.count_nonzero(signal_pixels) if n_pixels > 0: hillas = hillas_parameters(camera_geom[signal_pixels], image[signal_pixels]) dl1_container.fill_hillas(hillas) dl1_container.set_timing_features( camera_geom[signal_pixels], image[signal_pixels], peak_time[signal_pixels], hillas) dl1_container.set_leakage(camera_geom, image, signal_pixels) dl1_container.set_concentration( camera_geom, image, hillas) dl1_container.n_islands = num_islands dl1_container.wl = dl1_container.width / dl1_container.length dl1_container.n_pixels = n_pixels width = np.rad2deg( np.arctan2(dl1_container.width, foclen)) length = np.rad2deg( np.arctan2(dl1_container.length, foclen)) dl1_container.width = width dl1_container.length = length dl1_container.log_intensity = np.log10( dl1_container.intensity) if set(dl1_params_input).intersection(disp_params): disp_dx, disp_dy, disp_norm, disp_angle, disp_sign = disp( dl1_container['x'].to_value(u.m), dl1_container['y'].to_value(u.m), params['src_x'][ii], params['src_y'][ii]) dl1_container['disp_dx'] = disp_dx dl1_container['disp_dy'] = disp_dy dl1_container['disp_norm'] = disp_norm dl1_container['disp_angle'] = disp_angle dl1_container['disp_sign'] = disp_sign for p in parameters_to_update: params[ii][p] = u.Quantity(dl1_container[p]).value output.root[dl1_params_lstcam_key][:] = params
def get_expected_source_pos(data, data_type, config): """Get expected source position for source-dependent analysis . Parameters: ----------- data: Pandas DataFrame data_type: string ('mc_gamma','mc_proton','real_data') config: dictionnary containing configuration """ #For gamma MC, expected source position is actual one for each event if data_type == 'mc_gamma': expected_src_pos_x_m = data['src_x'].values expected_src_pos_y_m = data['src_y'].values #For proton MC, nominal source position is one written in config file if data_type == 'mc_proton': focal_length = OpticsDescription.from_name( 'LST').equivalent_focal_length expected_src_pos = utils.sky_to_camera( u.Quantity(data['mc_alt_tel'].values + config['mc_nominal_source_x_deg'], u.deg, copy=False), u.Quantity(data['mc_az_tel'].values + config['mc_nominal_source_y_deg'], u.deg, copy=False), focal_length, u.Quantity(data['mc_alt_tel'].values, u.deg, copy=False), u.Quantity(data['mc_az_tel'].values, u.deg, copy=False)) expected_src_pos_x_m = expected_src_pos.x.to_value(u.m) expected_src_pos_y_m = expected_src_pos.y.to_value(u.m) # For real data if data_type == 'real_data': # source is always at the ceter of camera for ON mode if config.get('observation_mode') == 'on': expected_src_pos_x_m = np.zeros(len(data)) expected_src_pos_y_m = np.zeros(len(data)) # compute source position in camera coordinate event by event for wobble mode if config.get('observation_mode') == 'wobble': if 'source_name' in config: source_coord = SkyCoord.from_name(config.get('source_name')) else: source_coord = SkyCoord(config.get('source_ra'), config.get('source_dec'), frame="icrs", unit="deg") focal_length = OpticsDescription.from_name( 'LST').equivalent_focal_length time = data['dragon_time'] obstime = Time(time, scale='utc', format='unix') pointing_alt = u.Quantity(data['alt_tel'], u.rad, copy=False) pointing_az = u.Quantity(data['az_tel'], u.rad, copy=False) source_pos = utils.radec_to_camera(source_coord, obstime, pointing_alt, pointing_az, focal_length) expected_src_pos_x_m = source_pos.x.to_value(u.m) expected_src_pos_y_m = source_pos.y.to_value(u.m) return expected_src_pos_x_m, expected_src_pos_y_m
def r0_to_dl1(input_filename=get_dataset_path('gamma_test_large.simtel.gz'), output_filename=None, custom_config={}): """ Chain r0 to dl1 Save the extracted dl1 parameters in output_filename Parameters ---------- input_filename: str path to input file, default: `gamma_test_large.simtel.gz` output_filename: str path to output file, default: `./` + basename(input_filename) config_file: path to a configuration file Returns ------- """ if output_filename is None: output_filename = ('dl1_' + os.path.basename(input_filename).split('.')[0] + '.h5') config = replace_config(standard_config, custom_config) custom_calibration = config["custom_calibration"] source = event_source(input_filename) source.allowed_tels = config["allowed_tels"] source.max_events = config["max_events"] cal = load_calibrator_from_config(config) dl1_container = DL1ParametersContainer() with HDF5TableWriter(filename=output_filename, group_name='events', overwrite=True) as writer: for i, event in enumerate(source): if i % 100 == 0: print(i) if not custom_calibration: cal(event) # for telescope_id, dl1 in event.dl1.tel.items(): for ii, telescope_id in enumerate(event.r0.tels_with_data): if custom_calibration: lst_calibration(event, telescope_id) try: dl1_filled = get_dl1(event, telescope_id, dl1_container=dl1_container, custom_config=config) except HillasParameterizationError: logging.exception( 'HillasParameterizationError in get_dl1()') continue if dl1_filled is not None: # Some custom def dl1_container.wl = dl1_container.width / dl1_container.length # Log10(Energy) in GeV dl1_container.mc_energy = np.log10(event.mc.energy.value * 1e3) dl1_container.intensity = np.log10(dl1_container.intensity) dl1_container.gps_time = event.trig.gps_time.value foclen = (event.inst.subarray.tel[telescope_id].optics. equivalent_focal_length) width = np.rad2deg(np.arctan2(dl1_container.width, foclen)) length = np.rad2deg( np.arctan2(dl1_container.length, foclen)) dl1_container.width = width.value dl1_container.length = length.value if width >= 0: # Camera geometry camera = event.inst.subarray.tel[telescope_id].camera writer.write(camera.cam_id, [dl1_container]) lst_focal = OpticsDescription.from_name('LST').equivalent_focal_length with pd.HDFStore(output_filename) as store: df = store['events/LSTCam'] source_pos_in_camera = sky_to_camera( df.mc_alt.values * u.rad, df.mc_az.values * u.rad, lst_focal, df.mc_alt_tel.values * u.rad, df.mc_az_tel.values * u.rad, ) disp_parameters = disp.disp(df.x.values * u.m, df.y.values * u.m, source_pos_in_camera.x, source_pos_in_camera.y) disp_df = pd.DataFrame(np.transpose(disp_parameters), columns=[ 'disp_dx', 'disp_dy', 'disp_norm', 'disp_angle', 'disp_sign' ]) disp_df['src_x'] = source_pos_in_camera.x.value disp_df['src_y'] = source_pos_in_camera.y.value store['events/LSTCam'] = pd.concat([store['events/LSTCam'], disp_df], axis=1) with HDF5TableWriter(filename=output_filename, group_name="simulation", mode="a") as writer: writer.write("run_config", [event.mcheader])