def test_display_dl1_event(mc_gamma_testfile): from ctapipe.io import EventSource, EventSeeker from ctapipe.calib import CameraCalibrator source = EventSource(mc_gamma_testfile, back_seekable=True) seeker = EventSeeker(source) event = seeker.get_event_index(11) # event 11 has telescopes 1 and 4 with data CameraCalibrator(subarray=source.subarray)(event) display_dl1_event(event, source.subarray.tel[1].camera.geometry, tel_id=1) display_dl1_event(event, source.subarray.tel[4].camera.geometry, tel_id=4)
def __init__(self, path, max_events=None): """ Reads simtelarray files utilising the SimTelEventSource from ctapipe Parameters ---------- path : str Path to the simtel file max_events : int Maximum number of events to read from the file """ super().__init__(path, max_events) try: from ctapipe.io import SimTelEventSource, EventSeeker except ModuleNotFoundError: msg = "Cannot find ctapipe installation" raise ModuleNotFoundError(msg) try: from target_calib import CameraConfiguration except ModuleNotFoundError: msg = ("Cannot find TARGET libraries, please follow installation " "instructions from https://forge.in2p3.fr/projects/gct/" "wiki/Installing_CHEC_Software") raise ModuleNotFoundError(msg) self.path = path reader = SimTelEventSource(input_url=path, max_events=max_events, back_seekable=True) self.seeker = EventSeeker(reader) first_event = self.seeker[0] tels = list(first_event.r0.tels_with_data) self.tel = tels[0] shape = first_event.r0.tel[self.tel].waveform.shape _, self.n_pixels, self.n_samples = shape self.n_modules = self.n_pixels // 64 self.index = 0 n_modules = 32 camera_version = "1.1.0" self._camera_config = CameraConfiguration(camera_version) tc_mapping = self._camera_config.GetMapping(n_modules == 1) self.mapping = get_clp_mapping_from_tc_mapping(tc_mapping) pix_x = first_event.inst.subarray.tel[tels[0]].camera.pix_x.value pix_y = first_event.inst.subarray.tel[tels[0]].camera.pix_y.value self.mapping['xpix'] = pix_x self.mapping['ypix'] = pix_y self.reference_pulse_path = self._camera_config.GetReferencePulsePath() self.camera_version = self._camera_config.GetVersion() self.gps_time = None self.mc_true = None self.mc = None self.pointing = None self.mcheader = None
def test_display_dl1_event(): from ctapipe.io import event_source, EventSeeker from lstchain.tests.test_lstchain import mc_gamma_testfile from ctapipe.calib import CameraCalibrator source = event_source(mc_gamma_testfile, back_seekable=True) seeker = EventSeeker(source) event = seeker[11] # event 11 has telescopes 1 and 4 with data CameraCalibrator()(event) display_dl1_event(event, tel_id=1) display_dl1_event(event, tel_id=4)
def main(): print("--> Input file: {}".format(args.input_file)) print("--> Number of events: {}".format(args.max_events)) reader = event_source(input_url=args.input_file, max_events=args.max_events) print("--> Number of files", reader.multi_file.num_inputs()) seeker = EventSeeker(reader) ev = seeker[0] tel_id = ev.r0.tels_with_data[0] n_modules = ev.lst.tel[tel_id].svc.num_modules config = Config({ "LSTR0Corrections": { "tel_id": tel_id } }) lst_r0 = LSTR0Corrections(config=config) pedestal = DragonPedestal(tel_id=tel_id, n_module=n_modules, r0_sample_start=args.start_r0_waveform) if args.deltaT: print("DeltaT correction active") for i, event in enumerate(reader): for tel_id in event.r0.tels_with_data: lst_r0.time_lapse_corr(event, tel_id) pedestal.fill_pedestal_event(event) if i%500 == 0: print("i = {}, ev id = {}".format(i, event.r0.event_id)) else: print("DeltaT correction no active") for i, event in enumerate(reader): pedestal.fill_pedestal_event(event) if i%500 == 0: print("i = {}, ev id = {}".format(i, event.r0.event_id)) # Finalize pedestal and write to fits file pedestal.finalize_pedestal() primaryhdu = fits.PrimaryHDU(ev.lst.tel[tel_id].svc.pixel_ids) secondhdu = fits.ImageHDU(np.int16(pedestal.meanped)) hdulist = fits.HDUList([primaryhdu, secondhdu]) hdulist.writeto(args.output_file)
"--input_file", help= "Path to fitz.fz file to create pedestal file. Can be one path or more", nargs='+', type=str) parser.add_argument("--output_file", help="Path where script should create pedestal file", nargs=1, type=str) args = parser.parse_args() print("Input file: ", args.input_file) print("Output file: ", args.output_file) reader = LSTEventSource(input_url=args.input_file[0]) seeker = EventSeeker(reader) ev = seeker[0] n_modules = ev.lst.tel[0].svc.num_modules ped = DragonPedestal() PedList = [] pedestal_value_array = np.zeros((n_modules, 2, 7, 4096), dtype=np.uint16) # create list of Pedestal class for each module for i in range(0, n_modules): PedList.append(DragonPedestal()) using_files_name = '' for file_path in args.input_file: using_files_name += (file_path.split("/")[-1]) using_files_name += ", "
def main(): description = ('Reduce a *_r1.tio file into a *_dl1.hdf5 file containing ' 'various parameters extracted from the waveforms') parser = argparse.ArgumentParser(description=description, formatter_class=Formatter) parser.add_argument('-f', '--files', dest='input_paths', nargs='+', help='path to the TIO r1 run files') parser.add_argument('-m', '--monitor', dest='monitor', action='store', help='path to the monitor file (OPTIONAL)') parser.add_argument('-o', '--output', dest='output_path', action='store', help='path to store the output HDF5 dl1 file ' '(OPTIONAL, will be automatically set if ' 'not specified)') parser.add_argument('-n', '--maxevents', dest='max_events', action='store', help='Number of events to process', type=int) parser.add_argument('-r', '--reducer', dest='reducer', action='store', default='AverageWF', choices=WaveformReducerFactory.subclass_names, help='WaveformReducer to use') parser.add_argument('-c', '--config', dest='configuration', help="""Configuration to pass to the waveform reducer (Usage: '{"window_shift":6, "window_size":6}') """) parser.add_argument('-p', '--plot', dest='plot', action='store_true', help="Plot stages for waveform reducers") args = parser.parse_args() if args.configuration: config = json.loads(args.configuration) config_string = args.configuration else: config = {} config_string = "" input_paths = args.input_paths n_files = len(input_paths) for i_path, input_path in enumerate(input_paths): print("PROGRESS: Reducing file {}/{}".format(i_path + 1, n_files)) kwargs = dict(input_url=input_path, max_events=args.max_events) reader = HESSIOEventSource(**kwargs) seeker = EventSeeker(reader) n_events = len(seeker) first_event = seeker[0] tels = list(first_event.r0.tels_with_data) _, n_pixels, n_samples = first_event.r0.tel[tels[0]].waveform.shape n_modules = 32 n_cells = 1 pixel_array = np.arange(n_pixels) camera_version = "1.1.0" camera_config = CameraConfiguration(camera_version) tc_mapping = camera_config.GetMapping(n_modules == 1) mapping = get_clp_mapping_from_tc_mapping(tc_mapping) if 'reference_pulse_path' not in config: reference_pulse_path = camera_config.GetReferencePulsePath() config['reference_pulse_path'] = reference_pulse_path kwargs = dict(n_pixels=n_pixels, n_samples=n_samples, plot=args.plot, mapping=mapping, **config) reducer = WaveformReducerFactory.produce(args.reducer, **kwargs) baseline_subtractor = BaselineSubtractor(seeker) input_path = reader.input_url output_path = args.output_path if not output_path: output_path = input_path.replace(".simtel.gz", "_dl1.h5") output_path = output_path.replace("run", "Run") r1 = HESSIOR1Calibrator() with DL1Writer(output_path, n_events * n_pixels, args.monitor) as writer: t_cpu = 0 start_time = 0 desc = "Processing events" for event in tqdm(seeker, total=n_events, desc=desc): iev = event.count r1.calibrate(event) waveforms = event.r1.tel[tels[0]].waveform[0] mc_true = event.mc.tel[tels[0]].photo_electron_image t_cpu = pd.to_datetime(event.trig.gps_time.value, unit='s') if not start_time: start_time = t_cpu waveforms_bs = baseline_subtractor.subtract(waveforms) bs = baseline_subtractor.baseline params = reducer.process(waveforms_bs) df_ev = pd.DataFrame( dict(iev=iev, pixel=pixel_array, first_cell_id=0, t_cpu=t_cpu, t_tack=0, baseline_subtracted=bs, **params, mc_true=mc_true)) writer.append_event(df_ev) sn_dict = {} for tm in range(n_modules): sn_dict['TM{:02d}_SN'.format(tm)] = "NaN" metadata = dict(source="CHECLabPy", date_generated=pd.datetime.now(), input_path=input_path, n_events=n_events, n_modules=n_modules, n_pixels=n_pixels, n_samples=n_samples, n_cells=n_cells, start_time=start_time, end_time=t_cpu, camera_version=camera_version, reducer=reducer.__class__.__name__, configuration=config_string, **sn_dict) writer.add_metadata(**metadata) writer.add_mapping(mapping)
def cleanmuonwriter(muon_no, proton_no, partial_no, runno): if not sys.warnoptions: warnings.simplefilter("ignore") class MultiplePeaks(Exception): pass class NoPeaksFound(Exception): pass def sig_handler(signum, frame): print("segfault") signal.signal(signal.SIGSEGV, sig_handler) # Run Options # Import raw sim_telarray output files gamma_data = "/store/muonsims/muons3/run" + str(muon_no) + ".simtel.gz"#Muons hadron_data = "/store/muonsims/proton6/run"+str(proton_no)+".simtel.gz"#Protons electron_data = "/store/muonsims/partial3/run"+str(partial_no)+".simtel.gz"#Partal Muons #Change this to partial files once done gammaflag = 0 # Should be 0 to plot muons or 1 for hadrons or 2 for partial rings. - KEEP 0 WHILST DOING FULL PROCESSING plotev = True # Whether or not to make animation plots for one single event. event_plot = 0 # Min event number to plot chan = 0 # PM Channel to use. output_filename = '/store/rogansims/cleaned' # HDF5 files output name. # Max number of events to read in for each of gammas/protons for training. maxcount = 2100 #Return to 2100 / 21 at end of testing no_files = 21 # Number of files in which to store events filerat = maxcount / no_files print('Filerat', filerat) no_tels = 1 # Number of telescopes def pos_to_index(pos, size): rnd = np.round((pos / size).to_value(unit.dimensionless_unscaled), 1) unique = np.sort(np.unique(rnd)) mask = np.append(np.diff(unique) > 0.5, True) bins = np.append(unique[mask] - 0.5, unique[-1] + 0.5) return np.digitize(rnd, bins) - 1 @jit def cam_squaremaker(data): '''Function to translate CHEC-S integrated images into square arrays for analysis purposes.''' square = np.zeros(2304) i = 0 while i < 48: if i < 8: xmin = 48 * i + 8 xmax = 48 * (i + 1) - 8 square[xmin:xmax] = data[i * 32:(i + 1) * 32] i = i + 1 elif i > 7 and i < 40: square[384:1920] = data[256:1792] i = 40 else: xmin = 48 * i + 8 xmax = 48 * (i + 1) - 8 square[xmin:xmax] = data[512 + i * 32:544 + i * 32] i = i + 1 square.resize((48, 48)) square = np.flip(square, 0) return square cmaps = [plt.cm.jet, plt.cm.winter, plt.cm.ocean, plt.cm.bone, plt.cm.gist_earth, plt.cm.hot, plt.cm.cool, plt.cm.coolwarm] hists = {} chan = 0 # which channel to look at count = 1 # Keeps track of number of events processed muon_number=1 proton_number=1 partial_number=1 key=0 caliber = CameraCalibrator() # Read in gammas/ protons from simtel for each output file. starttime=time.time() for fileno in np.arange(1, no_files + 1): # Basic principle is to load in, calibrate and parameterize the gamma ray # events, then do the same for the protons. Then mix the two together and # write them to disk. # Initialize lists for hdf5 storage. to_matlab = { 'id': [], 'event_id': [], 'label': [], 'mc_energy':[], 'tel_data': [], 'tel_integrated': []} smallfile_name = output_filename + "run" + str(runno) + "_" + str(fileno) + '.hdf5' gamma_source = SimTelEventSource(input_url=gamma_data,back_seekable=True) print('Processing Gammas') evfinder = EventSeeker(reader=gamma_source) # Determine events to load in using event seeker. startev = int(filerat * fileno - filerat) midev = int(filerat * fileno - filerat / 2.0) endev = int(filerat * fileno) print(startev, endev) for event in evfinder[startev:endev]: caliber(event) if count % 1000 == 0: print(count) if plotev == True and gammaflag >0: break to_matlab['id'].append(count) to_matlab['event_id'].append(str(event.r0.event_id) + '01') to_matlab['label'].append(0.0) energy=event.mc.energy.to(unit.GeV) print(energy.value) to_matlab['mc_energy'].append(energy.value) # Initialize arrays for given event integrated = np.zeros((no_tels, 48, 48)) for index, tel_id in enumerate(event.dl0.tels_with_data): # Loop through all triggered telescopes. cam = event.inst.subarray.tel[tel_id].camera size = np.sqrt(cam.pix_area) col = pos_to_index(cam.pix_x, size) row = pos_to_index(cam.pix_y, size) integ_charges = event.dl1.tel[tel_id]['image'] #ImageCleaningSection - From CTA documentation cleanmask=tailcuts_clean(cam, integ_charges, picture_thresh=1.7, boundary_thresh=1.5, min_number_picture_neighbors=2) cleaned = integ_charges.copy() cleaned[~cleanmask] = 0 squared = np.full((row.max() + 1, col.max() + 1), np.nan) squared[row, col] = cleaned integrated[tel_id - 1, :, :] = squared # if plotev==True and gammaflag==0: #Plot events # plt.imshow(squared) # plt.show() # plt.savefig("/home/clarkr/store/Graphs/cleanmuonplot" + str(runno) + "_" + str(muon_number) + ".png") # Send to hdf5 writer lists. # List of triggered telescopes to_matlab['tel_integrated'].append(integrated) count = count + 1 muon_number = muon_number+1 # Read in protons from simtel print('Processing Protons') proton_hessfile = SimTelEventSource(input_url=hadron_data,back_seekable=True) evfinder = EventSeeker(reader=proton_hessfile) print(startev, endev) for event in evfinder[startev:endev]: caliber = CameraCalibrator() caliber(event) if count % 1000 == 0: print(count) to_matlab['id'].append(int(count)) to_matlab['event_id'].append(str(event.r0.event_id) + '02') to_matlab['label'].append(1.0) energy=event.mc.energy.to(unit.GeV) print(energy.value) to_matlab['mc_energy'].append(energy.value) # Create arrays for event. integrated = np.zeros((no_tels, 48, 48)) for index, tel_id in enumerate(event.dl0.tels_with_data): # Loop through all triggered telescopes. cam = event.inst.subarray.tel[tel_id].camera size = np.sqrt(cam.pix_area) col = pos_to_index(cam.pix_x, size) row = pos_to_index(cam.pix_y, size) integ_charges = event.dl1.tel[tel_id]['image'] #ImageCleaningSection - From CTA documentation cleanmask=tailcuts_clean(cam, integ_charges, picture_thresh=1.7, boundary_thresh=1.5, min_number_picture_neighbors=2) cleaned = integ_charges.copy() cleaned[~cleanmask] = 0 squared = np.full((row.max() + 1, col.max() + 1), np.nan) squared[row, col] = cleaned integrated[tel_id - 1, :, :] = squared #if plotev==True and gammaflag==1: # plt.imshow(squared) # plt.show() # plt.savefig("/home/clarkr/store/Graphs/cleanprotonplot" + str(runno) + "_" + str(proton_number) + ".png") # Send to hdf5 writer lists. # List of triggered telescopes to_matlab['tel_integrated'].append(integrated) count = count + 1 proton_number = proton_number+1 print('Processing Partial') electron_hessfile = SimTelEventSource(input_url=electron_data,back_seekable=True) evfinder = EventSeeker(reader=electron_hessfile) print(startev, endev) for event in evfinder[startev:endev]: caliber = CameraCalibrator() caliber(event) if count % 1000 == 0: print(count) to_matlab['id'].append(int(count)) to_matlab['event_id'].append(str(event.r0.event_id) + '03') to_matlab['label'].append(2.0) energy=event.mc.energy.to(unit.GeV) print(energy.value) to_matlab['mc_energy'].append(energy.value) # Create arrays for event. integrated = np.zeros((no_tels, 48, 48)) for index, tel_id in enumerate(event.dl0.tels_with_data): # Loop through all triggered telescopes. cam = event.inst.subarray.tel[tel_id].camera size = np.sqrt(cam.pix_area) col = pos_to_index(cam.pix_x, size) row = pos_to_index(cam.pix_y, size) integ_charges = event.dl1.tel[tel_id]['image'] #ImageCleaningSection - From CTA documentation cleanmask=tailcuts_clean(cam, integ_charges, picture_thresh=1.7, boundary_thresh=1.5, min_number_picture_neighbors=2) cleaned = integ_charges.copy() cleaned[~cleanmask] = 0 squared = np.full((row.max() + 1, col.max() + 1), np.nan) squared[row, col] = cleaned integrated[tel_id - 1, :, :] = squared #if plotev==True and gammaflag==2: # plt.imshow(squared) #plt.show() #plt.savefig("/home/clarkr/store/Graphs/cleanedpartialplot" + str(runno) + "_" + str(partial_number) + ".png") # Send to hdf5 writer lists. # List of triggered telescopes to_matlab['tel_integrated'].append(integrated) count = count + 1 partial_number = partial_number+1 # Make everything arrays in order to randomize. to_matlab['id'] = np.asarray(to_matlab['id']) to_matlab['event_id'] = np.asarray(to_matlab['event_id']) to_matlab['label'] = np.asarray(to_matlab['label']) to_matlab['mc_energy'] = np.asarray(to_matlab['mc_energy']) to_matlab['tel_integrated'] = np.asarray(to_matlab['tel_integrated']) no_events = len(to_matlab['label']) randomize = np.arange(len(to_matlab['label']), dtype='int') # Implement uniform randomization here np.random.shuffle(randomize) to_matlab['id'] = to_matlab['id'][randomize] to_matlab['event_id'] = to_matlab['event_id'][randomize] to_matlab['label'] = to_matlab['label'][randomize] to_matlab['mc_energy'] = to_matlab['mc_energy'][randomize] to_matlab['tel_integrated'] = to_matlab['tel_integrated'][randomize] h5file = h5py.File(smallfile_name, "w") print('Writing') # HDF5 writer code lab_event = h5file.create_dataset( 'event_label', (no_events,), dtype='i', compression="gzip") id_group = h5file.create_dataset( 'id', (no_events,), dtype='i', compression="gzip") id_group2 = h5file.create_dataset( 'event_id', (no_events,), dtype='i', compression="gzip") energy_group = h5file.create_dataset( 'mc_energy', (no_events,), dtype='f', compression="gzip") squared_group = h5file.create_dataset( "squared_training", (no_events, 4, 48, 48), dtype='f', compression="gzip") for index in np.arange(0, no_events): index = int(index) lab_event[index] = np.int64(to_matlab['label'][index]) id_group[index] = np.int64(to_matlab['id'][index]) id_group2[index] = np.int64(to_matlab['event_id'][index]) energy_group[index] = np.float64(to_matlab['mc_energy'][index]) squared_group[index, :, :, :] = to_matlab['tel_integrated'][index] h5file.close() endtime=time.time() runtime=endtime-starttime print('Time for all events to be written', runtime) plt.show() #plt.savefig("/home/clarkr/store/Graphs/"+str(runno)+"_"+str(gammaflag)+".png") return
def __init__(self, path, max_events=None): """ Reads simtelarray files utilising the SimTelEventSource from ctapipe Parameters ---------- path : str Path to the simtel file max_events : int Maximum number of events to read from the file """ super().__init__(path, max_events) try: from ctapipe.io import SimTelEventSource, EventSeeker from ctapipe.coordinates import EngineeringCameraFrame except ModuleNotFoundError: msg = "Cannot find ctapipe installation" raise ModuleNotFoundError(msg) try: from target_calib import CameraConfiguration except ModuleNotFoundError: msg = ("Cannot find TARGET libraries, please follow installation " "instructions from https://forge.in2p3.fr/projects/gct/" "wiki/Installing_CHEC_Software") raise ModuleNotFoundError(msg) self.path = path reader = SimTelEventSource(input_url=path, max_events=max_events, back_seekable=True) self.seeker = EventSeeker(reader) first_event = self.seeker[0] tels = list(first_event.r0.tels_with_data) self.tel = tels[0] shape = first_event.r0.tel[self.tel].waveform.shape _, self.n_pixels, self.n_samples = shape self.n_modules = self.n_pixels // 64 n_modules = 32 camera_version = "1.1.0" self._camera_config = CameraConfiguration(camera_version) tc_mapping = self._camera_config.GetMapping(n_modules == 1) self.mapping = get_clp_mapping_from_tc_mapping(tc_mapping) n_rows = self.mapping.metadata['n_rows'] n_columns = self.mapping.metadata['n_columns'] camera_geom = first_event.inst.subarray.tel[tels[0]].camera engineering_frame = EngineeringCameraFrame(n_mirrors=2) engineering_geom = camera_geom.transform_to(engineering_frame) pix_x = engineering_geom.pix_x.value pix_y = engineering_geom.pix_y.value row, col = get_row_column(pix_x, pix_y) camera_2d = np.zeros((n_rows, n_columns), dtype=np.int) camera_2d[row, col] = np.arange(self.n_pixels, dtype=np.int) self.pixel_order = camera_2d[self.mapping['row'], self.mapping['col']] self.reference_pulse_path = self._camera_config.GetReferencePulsePath() self.camera_version = self._camera_config.GetVersion() self._iev = None self._t_cpu = None self.mc = None self.pointing = None self.mcheader = None