def write_simtel_energy_histogram(source, output_filename, obs_id=None, filters=None, metadata={}): """ Write the energy histogram from a simtel source to a HDF5 file Parameters ---------- source: `ctapipe.io.event_source` output_filename: str obs_id: float, int, str or None """ # Writing histograms with HDF5TableWriter(filename=output_filename, group_name="simulation", mode="a", filters=filters) as writer: writer.meta = metadata for hist in yield_toplevel_of_type(source.file_, Histograms): pass # find histogram id 6 (thrown energy) thrown = None for hist in source.file_.histograms: if hist['id'] == 6: thrown = hist thrown_hist = ThrownEventsHistogram() thrown_hist.fill_from_simtel(thrown) thrown_hist.obs_id = obs_id if metadata is not None: add_global_metadata(thrown_hist, metadata) writer.write('thrown_event_distribution', [thrown_hist])
def setup(self): self.log.info('Configure EventSourceFactory...') self.event_source = EventSourceFactory.produce( config=self.config, tool=self, product='HESSIOEventSource') self.event_source.allowed_tels = self.config['Analysis'][ 'allowed_tels'] self.calibrator = CameraCalibrator(config=self.config, tool=self, eventsource=self.event_source) self.writer = HDF5TableWriter(filename=self.outfile, group_name='image_infos', overwrite=True) # Define Pre-selection for images preselcuts = self.config['Preselect'] self.image_cutflow = CutFlow('Image preselection') self.image_cutflow.set_cuts( dict(no_sel=None, n_pixel=lambda s: np.count_nonzero(s) < preselcuts['n_pixel'][ 'min'], image_amplitude=lambda q: q < preselcuts['image_amplitude'][ 'min'])) # Define Pre-selection for events self.event_cutflow = CutFlow('Event preselection') self.event_cutflow.set_cuts(dict(no_sel=None))
def test_read_table(tmp_path): # write a simple hdf5 file using container = ReconstructedEnergyContainer() filename = tmp_path / "test_astropy_table.h5" with HDF5TableWriter(filename) as writer: for energy in np.logspace(1, 2, 10) * u.TeV: container.energy = energy writer.write("events", container) # try opening the result table = read_table(filename, "/events") assert "energy" in table.columns assert table["energy"].unit == u.TeV assert "CTAPIPE_VERSION" in table.meta assert table["energy"].description is not None # test using a string table = read_table(str(filename), "/events") # test write the table back out to some other format: table.write(tmp_path / "test_output.ecsv") table.write(tmp_path / "test_output.fits.gz") # test using a file handle with tables.open_file(filename) as handle: table = read_table(handle, "/events") # test a bad input with pytest.raises(ValueError): table = read_table(12345, "/events")
def setup(self): self.log.debug(f"Open file") self.eventsource = EventSource.from_config(parent=self) # if data remember how many event in the files if "LSTEventSource" in str(type(self.eventsource)): self.tot_events = len(self.eventsource.multi_file) self.log.debug(f"Input file has file {self.tot_events} events") else: self.tot_events = self.eventsource.max_events self.simulation = True self.processor = CalibrationCalculator.from_name( self.calibration_product, parent=self, subarray=self.eventsource.subarray) if self.r0calibrator_product: self.r0calibrator = CameraR0Calibrator.from_name( self.r0calibrator_product, parent=self) group_name = 'tel_' + str(self.processor.tel_id) self.log.debug(f"Open output file {self.output_file}") self.writer = HDF5TableWriter(filename=self.output_file, group_name=group_name, overwrite=True)
def setup(self): kwargs = dict(parent=self) self.eventsource = EventSource.from_config(**kwargs) self.flatfield = FlatFieldCalculator.from_name( self.flatfield_product, **kwargs ) self.pedestal = PedestalCalculator.from_name( self.pedestal_product, **kwargs ) if self.r0calibrator_product: self.r0calibrator = CameraR0Calibrator.from_name( self.r0calibrator_product, **kwargs ) msg = "tel_id not the same for all calibration components" assert self.r0calibrator.tel_id == self.pedestal.tel_id == self.flatfield.tel_id, msg self.tel_id = self.flatfield.tel_id group_name = 'tel_' + str(self.tel_id) self.writer = HDF5TableWriter( filename=self.output_file, group_name=group_name, overwrite=True )
def setup(self): kwargs = dict(parent=self) self.eventsource = EventSource.from_config(**kwargs) # remember how many event in the files self.tot_events = len(self.eventsource.multi_file) self.log.debug(f"Input file has file {self.tot_events} events") self.flatfield = FlatFieldCalculator.from_name(self.flatfield_product, **kwargs) self.pedestal = PedestalCalculator.from_name(self.pedestal_product, **kwargs) if self.r0calibrator_product: self.r0calibrator = CameraR0Calibrator.from_name( self.r0calibrator_product, **kwargs) msg = "tel_id not the same for all calibration components" assert self.r0calibrator.tel_id == self.pedestal.tel_id == self.flatfield.tel_id, msg self.tel_id = self.flatfield.tel_id group_name = 'tel_' + str(self.tel_id) self.log.debug(f"Open output file {self.output_file}") self.writer = HDF5TableWriter(filename=self.output_file, group_name=group_name, overwrite=True)
def write_mcheader(mcheader, output_filename, obs_id=None, filters=None, metadata=None): """ Write the mcheader from an event container to a HDF5 file Parameters ---------- output_filename: str event: `ctapipe.io.DataContainer` """ extramc = ExtraMCInfo() extramc.prefix = '' # get rid of the prefix if metadata is not None: add_global_metadata(mcheader, metadata) add_global_metadata(extramc, metadata) with HDF5TableWriter(filename=output_filename, group_name="simulation", mode="a", filters=filters) as writer: extramc.obs_id = obs_id writer.write("run_config", [extramc, mcheader])
def setup(self): self.log.debug(f"Open file") self.eventsource = EventSource.from_config(parent=self) tel_id = self.eventsource.lst_service.telescope_id if self.eventsource.r0_r1_calibrator.drs4_pedestal_path.tel[ tel_id] is None: raise IOError("Missing (mandatory) drs4 pedestal file in trailets") # if data remember how many event in the files if "LSTEventSource" in str(type(self.eventsource)): self.tot_events = len(self.eventsource.multi_file) self.log.debug(f"Input file has file {self.tot_events} events") else: self.tot_events = self.eventsource.max_events self.simulation = True self.processor = CalibrationCalculator.from_name( self.calibration_product, parent=self, subarray=self.eventsource.subarray) group_name = 'tel_' + str(tel_id) self.log.debug(f"Open output file {self.output_file}") self.writer = HDF5TableWriter(filename=self.output_file, group_name=group_name, overwrite=True)
def setup(self): self.log.info('Configure EventSource...') self.event_source = self.add_component( EventSource.from_config(config=self.config, parent=self)) self.calibrator = self.add_component(CameraCalibrator(parent=self)) self.writer = self.add_component( HDF5TableWriter(filename=self.outfile, group_name='image_infos', overwrite=True)) # Define Pre-selection for images preselcuts = self.config['Preselect'] self.image_cutflow = CutFlow('Image preselection') self.image_cutflow.set_cuts( dict(no_sel=None, n_pixel=lambda s: np.count_nonzero(s) < preselcuts['n_pixel'][ 'min'], image_amplitude=lambda q: q < preselcuts['image_amplitude'][ 'min'])) # Define Pre-selection for events self.event_cutflow = CutFlow('Event preselection') self.event_cutflow.set_cuts(dict(no_sel=None))
def setup(self): if self.output is None: raise ToolConfigurationError( "You need to provide an --output file") if self.output.exists() and not self.overwrite: raise ToolConfigurationError( "Outputfile {self.output} already exists, use `--overwrite` to overwrite" ) self.source = EventSource(parent=self) subarray = self.source.subarray self.calib = CameraCalibrator(subarray=subarray, parent=self) self.ring_fitter = MuonRingFitter(parent=self) self.intensity_fitter = MuonIntensityFitter(subarray=subarray, parent=self) self.cleaning = TailcutsImageCleaner(parent=self, subarray=subarray) self.writer = HDF5TableWriter(self.output, "", add_prefix=True, parent=self, mode="w") self.pixels_in_tel_frame = {} self.field_of_view = {} self.pixel_widths = {} for p in [ "min_pixels", "pedestal", "ratio_width", "completeness_threshold" ]: getattr(self, p).attach_subarray(self.source.subarray)
def setup(self): self.source: EventSource = self.add_component( EventSource.from_config(parent=self) ) if self.source.input_url == '': raise ToolConfigurationError("please specify --input <events file>") self.calib = self.add_component(CameraCalibrator(parent=self)) self.writer = self.add_component(HDF5TableWriter(self.outfile, "muons"))
def setup(self): if self.events == '': raise ToolConfigurationError( "please specify --input <events file>") self.log.debug("input: %s", self.events) self.source = event_source(self.events) self.calib = CameraCalibrator(parent=self) self.writer = HDF5TableWriter(self.outfile, "muons")
def save_event_data(events, filename, group_name): with HDF5TableWriter(filename, mode='a', group_name=group_name) as h5: for event in events: h5.write('waveforms', event.data) yield event
def setup(self): if self.events == '': raise ToolConfigurationError( "please specify --input <events file>") self.log.debug("input: %s", self.events) self.source = EventSourceFactory.produce(input_url=self.events) self.calib = CameraCalibrator(config=self.config, tool=self, eventsource=self.source) self.writer = HDF5TableWriter(self.outfile, "muons")
def setup(self): kwargs = dict(parent=self) self.eventsource = EventSource.from_config(**kwargs) self.pedestal = PedestalCalculator.from_name(self.calculator_product, **kwargs) self.group_name = 'tel_' + str(self.pedestal.tel_id) self.writer = HDF5TableWriter(filename=self.output_file, group_name=self.group_name, overwrite=True)
def setup(self): self.log.info("Configure EventSource...") self.event_source = EventSource.from_url(self.infile, parent=self) self.calibrator = CameraCalibrator(subarray=self.event_source.subarray, parent=self) self.writer = HDF5TableWriter(filename=self.outfile, group_name="image_infos", overwrite=True, parent=self)
def setup(self): kwargs = dict(parent=self) self.eventsource = EventSource.from_config(**kwargs) self.flatfield = FlatFieldCalculator.from_name(self.calculator_product, **kwargs) self.cleaner = WaveformCleaner.from_name(self.cleaner_product, **kwargs) self.writer = HDF5TableWriter(filename=self.output_file, group_name='flatfield', overwrite=True)
def setup(self): self.log.info("Configure EventSource...") EventSource.input_url.default_value = get_dataset_path( "lst_prod3_calibration_and_mcphotons.simtel.zst") self.event_source = EventSource(parent=self) self.calibrator = CameraCalibrator(subarray=self.event_source.subarray, parent=self) self.writer = HDF5TableWriter(filename=self.output, group_name="image_infos", overwrite=True, parent=self)
def setup(self): if self.output is None: raise ToolConfigurationError( 'You need to provide an --output file') if self.output.exists() and not self.overwrite: raise ToolConfigurationError( 'Outputfile {self.output} already exists, use `--overwrite` to overwrite' ) self.source = self.add_component(EventSource.from_config(parent=self)) self.extractor = self.add_component( ImageExtractor.from_name(self.extractor_name, parent=self, subarray=self.source.subarray)) self.calib = self.add_component( CameraCalibrator( subarray=self.source.subarray, parent=self, image_extractor=self.extractor, )) self.ring_fitter = self.add_component(MuonRingFitter(parent=self, )) self.intensity_fitter = self.add_component( MuonIntensityFitter( subarray=self.source.subarray, parent=self, )) self.cleaning = self.add_component( TailcutsImageCleaner( parent=self, subarray=self.source.subarray, )) self.writer = self.add_component( HDF5TableWriter( self.output, "", add_prefix=True, parent=self, mode='w', )) self.pixels_in_tel_frame = {} self.field_of_view = {} self.pixel_widths = {} for p in [ 'min_pixels', 'pedestal', 'ratio_width', 'completeness_threshold' ]: getattr(self, p).attach_subarray(self.source.subarray)
def test_condition(tmp_path): # write a simple hdf5 file using container = ReconstructedEnergyContainer() filename = tmp_path / "test_astropy_table.h5" with HDF5TableWriter(filename) as writer: for energy in [np.nan, 100, np.nan, 50, -1.0] * u.TeV: container.energy = energy writer.write("events", container) # try opening the result table = read_table(filename, "/events", condition="energy > 0") assert len(table) == 2 assert np.all(table["energy"] == [100, 50] * u.TeV)
def test_read_table_time(tmp_path): t0 = Time("2020-01-01T20:00:00.0") times = t0 + np.arange(10) * u.s # use table writer to write test file filename = tmp_path / "test_astropy_table.h5" with HDF5TableWriter(filename) as writer: for t in times: container = TelescopeTriggerContainer(time=t, n_trigger_pixels=10) writer.write("events", container) # check reading in the times works as expected table = read_table(filename, "/events") assert isinstance(table["time"], Time) assert np.allclose(times.tai.mjd, table["time"].tai.mjd)
def setup(self): self.log.debug("Opening file") self.eventsource = EventSource.from_config(parent=self) self.processor = CalibrationCalculator.from_name( self.calibration_product, parent=self, subarray=self.eventsource.subarray) tel_id = self.processor.tel_id # if real data if isinstance(self.eventsource, LSTEventSource): if tel_id != self.eventsource.lst_service.telescope_id: raise ValueError( f"Events telescope_id {self.eventsource.lst_service.telescope_id} " f"different than CalibrationCalculator telescope_id {tel_id}" ) if self.eventsource.r0_r1_calibrator.drs4_pedestal_path.tel[ tel_id] is None: raise IOError( "Missing (mandatory) drs4 pedestal file in trailets") # remember how many events in the files self.tot_events = len(self.eventsource.multi_file) self.log.debug(f"Input file has file {self.tot_events} events") else: self.tot_events = self.eventsource.max_events self.simulation = True group_name = 'tel_' + str(tel_id) self.log.debug(f"Open output file {self.output_file}") self.writer = HDF5TableWriter(filename=self.output_file, group_name=group_name, overwrite=True)
def test_read_table_slicing(tmp_path): filename = tmp_path / "test_slicing.h5" # write a simple hdf5 file using class Data(Container): index = Field(0) value = Field(0.0) rng = np.random.default_rng(0) values = rng.normal(size=100) index = np.arange(len(values)) with HDF5TableWriter(filename) as writer: for i, value in zip(index, values): container = Data(index=i, value=value) writer.write("events", container) # try opening the result table = read_table(filename, "/events", start=50) assert len(table) == 50 assert np.all(table["index"] == index[50:]) assert np.all(table["value"] == values[50:]) table = read_table(filename, "/events", stop=50) assert len(table) == 50 assert np.all(table["index"] == index[:50]) assert np.all(table["value"] == values[:50]) table = read_table(filename, "/events", start=10, stop=30) assert len(table) == 20 assert np.all(table["index"] == index[10:30]) assert np.all(table["value"] == values[10:30]) table = read_table(filename, "/events", step=5) assert len(table) == 20 assert np.all(table["index"] == index[::5]) assert np.all(table["value"] == values[::5])
def write_result_to_file(run_info_container, array_events, telescope_events, output_file, mode='a'): '''Combines run_info, array_events and telescope_events into one file using the HDF5TableWriter. ''' logging.info(f'writing to file {output_file}') with HDF5TableWriter(output_file, mode=mode, group_name='', add_prefix=True) as h5_table: run_info_container.mc['run_array_direction'] = 0 h5_table.write('runs', [run_info_container, run_info_container.mc]) for array_event in array_events: h5_table.write('array_events', [array_event, array_event.mc, array_event.reco]) for tel_event in telescope_events: h5_table.write( 'telescope_events', [ tel_event, tel_event.pointing, tel_event.hillas, tel_event.concentration, tel_event.leakage, tel_event.timing, tel_event.islands, ], )
def r0_to_dl1( input_filename=get_dataset_path('gamma_test_large.simtel.gz'), output_filename=None, custom_config={}, ): """ Chain r0 to dl1 Save the extracted dl1 parameters in output_filename Parameters ---------- input_filename: str path to input file, default: `gamma_test_large.simtel.gz` output_filename: str or None path to output file, defaults to writing dl1 into the current directory custom_config: path to a configuration file Returns ------- """ if output_filename is None: try: run = parse_r0_filename(input_filename) output_filename = run_to_dl1_filename(run.tel_id, run.run, run.subrun) except ValueError: output_filename = r0_to_dl1_filename(Path(input_filename).name) if os.path.exists(output_filename): raise IOError(str(output_filename) + ' exists, exiting.') config = replace_config(standard_config, custom_config) custom_calibration = config["custom_calibration"] source = EventSource(input_url=input_filename, config=Config(config["source_config"])) subarray = source.subarray is_simu = source.is_simulation metadata = global_metadata(source) write_metadata(metadata, output_filename) cal_mc = load_calibrator_from_config(config, subarray) # minimum number of pe in a pixel to include it # in calculation of muon ring time (peak sample): min_pe_for_muon_t_calc = 10. # Dictionary to store muon ring parameters muon_parameters = create_muon_table() # all this will be cleaned up in a next PR related to the configuration files r1_dl1_calibrator = CameraCalibrator( image_extractor_type=config['image_extractor'], config=Config(config), subarray=subarray) if not is_simu: # Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but # using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous r1_dl1_calibrator_for_muon_rings = CameraCalibrator( image_extractor_type=config['image_extractor_for_muons'], config=Config(config), subarray=subarray) # Component to process interleaved pedestal and flat-fields calib_config = Config(config[config['calibration_product']]) calibration_calculator = CalibrationCalculator.from_name( config['calibration_product'], config=calib_config, subarray=source.subarray) calibration_index = DL1MonitoringEventIndexContainer() dl1_container = DL1ParametersContainer() extra_im = ExtraImageInfo() extra_im.prefix = '' # get rid of the prefix # Write extra information to the DL1 file write_array_info(subarray, output_filename) write_array_info_08(subarray, output_filename) if is_simu: write_mcheader( source.simulation_config, output_filename, obs_id=source.obs_ids[0], filters=HDF5_ZSTD_FILTERS, metadata=metadata, ) with HDF5TableWriter( filename=output_filename, group_name='dl1/event', mode='a', filters=HDF5_ZSTD_FILTERS, add_prefix=True, # overwrite=True, ) as writer: if is_simu: subarray = subarray # build a mapping of tel_id back to tel_index: # (note this should be part of SubarrayDescription) idx = np.zeros(max(subarray.tel_indices) + 1) for key, val in subarray.tel_indices.items(): idx[key] = val # the final transform then needs the mapping and the number of telescopes tel_list_transform = partial( utils.expand_tel_list, max_tels=max(subarray.tel) + 1, ) writer.add_column_transform(table_name='subarray/trigger', col_name='tels_with_trigger', transform=tel_list_transform) # Forcing filters for the dl1 dataset that are currently read from the pre-existing files # This should be fixed in ctapipe and then corrected here writer._h5file.filters = HDF5_ZSTD_FILTERS logger.info(f"USING FILTERS: {writer._h5file.filters}") for i, event in enumerate(source): if i % 100 == 0: logger.info(i) event.dl0.prefix = '' event.trigger.prefix = '' if event.simulation is not None: event.simulation.prefix = 'mc' dl1_container.reset() # write sub tables if is_simu: write_subarray_tables(writer, event, metadata) if not custom_calibration: cal_mc(event) if config['mc_image_scaling_factor'] != 1: rescale_dl1_charge(event, config['mc_image_scaling_factor']) else: if i == 0: # initialize the telescope # FIXME? LST calibrator is only for one telescope # it should be inside the telescope loop (?) tel_id = calibration_calculator.tel_id #initialize the event monitoring data event.mon = source.r0_r1_calibrator.mon_data # write the first calibration event (initialized from calibration h5 file) write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=True, new_ff=True) # flat-field or pedestal: if (event.trigger.event_type == EventType.FLATFIELD or event.trigger.event_type == EventType.SKY_PEDESTAL): # process interleaved events (pedestals, ff, calibration) new_ped_event, new_ff_event = calibration_calculator.process_interleaved( event) # write monitoring containers if updated if new_ped_event or new_ff_event: write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=new_ped_event, new_ff=new_ff_event) # calibrate and gain select the event by hand for DL1 source.r0_r1_calibrator.calibrate(event) # create image for all events r1_dl1_calibrator(event) # Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten. # When the last version of the method is implemented, vol. reduction will be done at dl0 apply_volume_reduction(event, subarray, config) # FIXME? This should be eventually done after we evaluate whether the image is # a candidate muon ring. In that case the full image could be kept, or reduced # only after the ring analysis is complete. for ii, telescope_id in enumerate(event.dl1.tel.keys()): dl1_container.reset() # update the calibration index in the dl1 event container dl1_container.calibration_id = calibration_index.calibration_id dl1_container.fill_event_info(event) tel = event.dl1.tel[telescope_id] tel.prefix = '' # don't really need one # remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST' tel_name = str(subarray.tel[telescope_id])[4:] if custom_calibration: lst_calibration(event, telescope_id) write_event = True # Will determine whether this event has to be written to the # DL1 output or not. if is_simu: dl1_container.fill_mc(event, subarray.positions[telescope_id]) assert event.dl1.tel[telescope_id].image is not None try: get_dl1( event, subarray, telescope_id, dl1_container=dl1_container, custom_config=config, ) except HillasParameterizationError: logging.exception( 'HillasParameterizationError in get_dl1()') if not is_simu: dl1_container.ucts_time = 0 # convert Time to unix timestamp in (UTC) to keep compatibility # with older lstchain # FIXME: just keep it as time, table writer and reader handle it dl1_container.dragon_time = event.trigger.time.unix dl1_container.tib_time = 0 dl1_container.ucts_trigger_type = event.lst.tel[ telescope_id].evt.ucts_trigger_type dl1_container.trigger_type = event.lst.tel[ telescope_id].evt.tib_masked_trigger else: dl1_container.trigger_type = event.trigger.event_type dl1_container.az_tel = event.pointing.tel[telescope_id].azimuth dl1_container.alt_tel = event.pointing.tel[ telescope_id].altitude dl1_container.trigger_time = event.trigger.time.unix dl1_container.event_type = event.trigger.event_type # FIXME: no need to read telescope characteristics like foclen for every event! foclen = subarray.tel[ telescope_id].optics.equivalent_focal_length mirror_area = u.Quantity( subarray.tel[telescope_id].optics.mirror_area, u.m**2) dl1_container.prefix = tel.prefix # extra info for the image table extra_im.tel_id = telescope_id extra_im.selected_gain_channel = event.r1.tel[ telescope_id].selected_gain_channel for container in [extra_im, dl1_container, event.r0, tel]: add_global_metadata(container, metadata) event.r0.prefix = '' writer.write(table_name=f'telescope/image/{tel_name}', containers=[event.index, tel, extra_im]) writer.write(table_name=f'telescope/parameters/{tel_name}', containers=[event.index, dl1_container]) # Muon ring analysis, for real data only (MC is done starting from DL1 files) if not is_simu: bad_pixels = event.mon.tel[ telescope_id].calibration.unusable_pixels[0] # Set to 0 unreliable pixels: image = tel.image * (~bad_pixels) # process only promising events, in terms of # of pixels with large signals: if tag_pix_thr(image): # re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings numsamples = event.r1.tel[telescope_id].waveform.shape[ 1] # not necessarily the same as in r0! bad_pixels_hg = event.mon.tel[ telescope_id].calibration.unusable_pixels[0] bad_pixels_lg = event.mon.tel[ telescope_id].calibration.unusable_pixels[1] # Now set to 0 all samples in unreliable pixels. Important for global peak # integrator in case of crazy pixels! TBD: can this be done in a simpler # way? bad_pixels = bad_pixels_hg | bad_pixels_lg bad_waveform = np.transpose( np.array(numsamples * [bad_pixels])) # print('hg bad pixels:',np.where(bad_pixels_hg)) # print('lg bad pixels:',np.where(bad_pixels_lg)) event.r1.tel[telescope_id].waveform *= ~bad_waveform r1_dl1_calibrator_for_muon_rings(event) tel = event.dl1.tel[telescope_id] image = tel.image * (~bad_pixels) # Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum) # perhaps the event is no longer promising (e.g. if it has a large time evolution) if not tag_pix_thr(image): good_ring = False else: # read geometry from event.inst. But not needed for every event. FIXME? geom = subarray.tel[telescope_id].\ camera.geometry muonintensityparam, dist_mask, \ ring_size, size_outside_ring, muonringparam, \ good_ring, radial_distribution, \ mean_pixel_charge_around_ring,\ muonpars = \ analyze_muon_event(subarray, event.index.event_id, image, geom, foclen, mirror_area, False, '') # mirror_area, True, './') # (test) plot muon rings as png files # Now we want to obtain the waveform sample (in HG & LG) at which the ring light peaks: bright_pixels = image > min_pe_for_muon_t_calc selected_gain = event.r1.tel[ telescope_id].selected_gain_channel mask_hg = bright_pixels & (selected_gain == 0) mask_lg = bright_pixels & (selected_gain == 1) bright_pixels_waveforms_hg = event.r1.tel[ telescope_id].waveform[mask_hg, :] bright_pixels_waveforms_lg = event.r1.tel[ telescope_id].waveform[mask_lg, :] stacked_waveforms_hg = np.sum( bright_pixels_waveforms_hg, axis=0) stacked_waveforms_lg = np.sum( bright_pixels_waveforms_lg, axis=0) # stacked waveforms from all bright pixels; shape (ngains, nsamples) hg_peak_sample = np.argmax(stacked_waveforms_hg, axis=-1) lg_peak_sample = np.argmax(stacked_waveforms_lg, axis=-1) if good_ring: fill_muon_event(-1, muon_parameters, good_ring, event.index.event_id, dl1_container.dragon_time, muonintensityparam, dist_mask, muonringparam, radial_distribution, ring_size, size_outside_ring, mean_pixel_charge_around_ring, muonpars, hg_peak_sample, lg_peak_sample) # writes mc information per telescope, including photo electron image if (is_simu and config['write_pe_image'] and event.simulation.tel[telescope_id].true_image is not None and event.simulation.tel[telescope_id].true_image.any()): event.simulation.tel[telescope_id].prefix = '' writer.write(table_name=f'simulation/{tel_name}', containers=[ event.simulation.tel[telescope_id], extra_im ]) if not is_simu: # at the end of event loop ask calculation of remaining interleaved statistics new_ped, new_ff = calibration_calculator.output_interleaved_results( event) # write monitoring events write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=new_ped, new_ff=new_ff) if is_simu: # Reconstruct source position from disp for all events and write the result in the output file for tel_name in ['LST_LSTCam']: focal = OpticsDescription.from_name( tel_name.split('_')[0]).equivalent_focal_length add_disp_to_parameters_table(output_filename, dl1_params_lstcam_key, focal) # Write energy histogram from simtel file and extra metadata # ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever if source.max_events is None: write_simtel_energy_histogram(source, output_filename, obs_id=event.index.obs_id, metadata=metadata) else: dir, name = os.path.split(output_filename) name = name.replace('dl1', 'muons').replace('LST-1.1', 'LST-1') # Consider the possibilities of DL1 files with .fits.h5 & .h5 ending: name = name.replace('.fits.h5', '.fits').replace('.h5', '.fits') muon_output_filename = Path(dir, name) table = Table(muon_parameters) table.write(muon_output_filename, format='fits', overwrite=True)
def r0_to_dl1(input_filename=get_dataset_path('gamma_test_large.simtel.gz'), output_filename=None, custom_config={}, pedestal_path=None, calibration_path=None, time_calibration_path=None, pointing_file_path=None, ucts_t0_dragon=math.nan, dragon_counter0=math.nan, ucts_t0_tib=math.nan, tib_counter0=math.nan): """ Chain r0 to dl1 Save the extracted dl1 parameters in output_filename Parameters ---------- input_filename: str path to input file, default: `gamma_test_large.simtel.gz` output_filename: str or None path to output file, defaults to writing dl1 into the current directory custom_config: path to a configuration file pedestal_path: Path to the DRS4 pedestal file calibration_path: Path to the file with calibration constants and pedestals time_calibration_path: Path to the DRS4 time correction file pointing_file_path: path to the Drive log with the pointing information ucts_t0_dragon: first valid ucts_time dragon_counter0: Dragon counter corresponding to ucts_t0_dragon ucts_t0_tib: first valid ucts_time for the first valid TIB counter tib_counter0: first valid TIB counter Returns ------- """ if output_filename is None: try: run = parse_r0_filename(input_filename) output_filename = run_to_dl1_filename(run.tel_id, run.run, run.subrun) except ValueError: output_filename = r0_to_dl1_filename(Path(input_filename).name) if os.path.exists(output_filename): raise IOError(str(output_filename) + ' exists, exiting.') config = replace_config(standard_config, custom_config) custom_calibration = config["custom_calibration"] gain_selector = load_gain_selector_from_config(config) # FIXME for ctapipe 0.8, str should be removed, as Path is supported source = event_source(str(input_filename)) subarray = source.subarray is_simu = source.is_simulation source.allowed_tels = config["allowed_tels"] if config["max_events"] is not None: source.max_events = config["max_events"] metadata = global_metadata(source) write_metadata(metadata, output_filename) cal_mc = load_calibrator_from_config(config, subarray) # minimum number of pe in a pixel to include it # in calculation of muon ring time (peak sample): min_pe_for_muon_t_calc = 10. # Dictionary to store muon ring parameters muon_parameters = create_muon_table() if not is_simu: # TODO : add DRS4 calibration config in config file, read it and pass it here r0_r1_calibrator = LSTR0Corrections( pedestal_path=pedestal_path, tel_id=1, ) # all this will be cleaned up in a next PR related to the configuration files r1_dl1_calibrator = LSTCameraCalibrator( calibration_path=calibration_path, time_calibration_path=time_calibration_path, extractor_product=config['image_extractor'], gain_threshold=Config(config).gain_selector_config['threshold'], charge_scale=config['charge_scale'], config=Config(config), allowed_tels=[1], subarray=subarray) # Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but # using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous r1_dl1_calibrator_for_muon_rings = LSTCameraCalibrator( calibration_path=calibration_path, time_calibration_path=time_calibration_path, extractor_product=config['image_extractor_for_muons'], gain_threshold=Config(config).gain_selector_config['threshold'], charge_scale=config['charge_scale'], config=Config(config), allowed_tels=[1], subarray=subarray) # Component to process interleaved pedestal and flat-fields calib_config = Config(config[config['calibration_product']]) # set time calibration path for flatfield trailet () calib_config.FlasherFlatFieldCalculator.time_calibration_path = time_calibration_path calibration_calculator = CalibrationCalculator.from_name( config['calibration_product'], config=calib_config, subarray=source.subarray) calibration_index = DL1MonitoringEventIndexContainer() dl1_container = DL1ParametersContainer() if pointing_file_path: # Open drive report pointings = PointingPosition(drive_path=pointing_file_path) drive_data = pointings._read_drive_report() extra_im = ExtraImageInfo() extra_im.prefix = '' # get rid of the prefix # get the first event to write array info and mc header event_iter = iter(source) first_event = next(event_iter) # Write extra information to the DL1 file write_array_info(subarray, output_filename) write_array_info_08(subarray, output_filename) if is_simu: write_mcheader( first_event.mcheader, output_filename, obs_id=first_event.index.obs_id, filters=filters, metadata=metadata, ) with HDF5TableWriter( filename=output_filename, group_name='dl1/event', mode='a', filters=filters, add_prefix=True, # overwrite=True, ) as writer: if is_simu: subarray = subarray # build a mapping of tel_id back to tel_index: # (note this should be part of SubarrayDescription) idx = np.zeros(max(subarray.tel_indices) + 1) for key, val in subarray.tel_indices.items(): idx[key] = val # the final transform then needs the mapping and the number of telescopes tel_list_transform = partial( utils.expand_tel_list, max_tels=max(subarray.tel) + 1, ) writer.add_column_transform(table_name='subarray/trigger', col_name='tels_with_trigger', transform=tel_list_transform) # Forcing filters for the dl1 dataset that are currently read from the pre-existing files # This should be fixed in ctapipe and then corrected here writer._h5file.filters = filters logger.info(f"USING FILTERS: {writer._h5file.filters}") first_valid_ucts = None first_valid_ucts_tib = None previous_ucts_time_unix = [] previous_ucts_trigger_type = [] for i, event in enumerate(chain([first_event], event_iter)): if i % 100 == 0: logger.info(i) event.dl0.prefix = '' event.mc.prefix = 'mc' event.trigger.prefix = '' dl1_container.reset() # write sub tables if is_simu: write_subarray_tables(writer, event, metadata) if not custom_calibration: cal_mc(event) if config['mc_image_scaling_factor'] != 1: rescale_dl1_charge(event, config['mc_image_scaling_factor']) else: if i == 0: # initialize the telescope # FIXME? LST calibrator is only for one telescope # it should be inside the telescope loop (?) tel_id = calibration_calculator.tel_id # write the first calibration event (initialized from calibration h5 file) write_calibration_data( writer, calibration_index, r1_dl1_calibrator.mon_data.tel[tel_id], new_ped=True, new_ff=True) # drs4 calibrations r0_r1_calibrator.calibrate(event) # process interleaved events (pedestals, ff, calibration) new_ped_event, new_ff_event = calibration_calculator.process_interleaved( event) # write monitoring containers if updated if new_ped_event or new_ff_event: write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=new_ped_event, new_ff=new_ff_event) # calibrate and extract image from event r1_dl1_calibrator(event) # Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten. # When the last version of the method is implemented, vol. reduction will be done at dl0 apply_volume_reduction(event, subarray, config) # FIXME? This should be eventually done after we evaluate whether the image is # a candidate muon ring. In that case the full image could be kept, or reduced # only after the ring analysis is complete. for ii, telescope_id in enumerate(event.r0.tels_with_data): dl1_container.reset() # update the calibration index in the dl1 event container dl1_container.calibration_id = calibration_index.calibration_id dl1_container.fill_event_info(event) tel = event.dl1.tel[telescope_id] tel.prefix = '' # don't really need one # remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST' tel_name = str(subarray.tel[telescope_id])[4:] if custom_calibration: lst_calibration(event, telescope_id) write_event = True # Will determine whether this event has to be written to the # DL1 output or not. if is_simu: dl1_container.fill_mc(event, subarray.positions[telescope_id]) try: get_dl1(event, subarray, telescope_id, dl1_container=dl1_container, custom_config=config, use_main_island=True) except HillasParameterizationError: logging.exception( 'HillasParameterizationError in get_dl1()') if not is_simu: # GPS + WRS + UCTS is now working in its nominal configuration. # These TS are stored into ucts_time container. # TS can be alternatively calculated from the TIB and # Dragon modules counters based on the first valid UCTS TS # as the reference point. For the time being, the three TS # are stored in the DL1 files for checking purposes. module_id = 82 # Get counters from the central Dragon module if math.isnan(ucts_t0_dragon) and math.isnan(dragon_counter0) \ and math.isnan(ucts_t0_tib) and math.isnan(tib_counter0): # Dragon/TIB timestamps not based on a valid absolute reference timestamp dragon_time = (event.lst.tel[telescope_id].svc.date + event.lst.tel[telescope_id].evt. pps_counter[module_id] + event.lst.tel[telescope_id].evt. tenMHz_counter[module_id] * 10**(-7)) tib_time = ( event.lst.tel[telescope_id].svc.date + event.lst.tel[telescope_id].evt.tib_pps_counter + event.lst.tel[telescope_id].evt.tib_tenMHz_counter * 10**(-7)) if event.lst.tel[ telescope_id].evt.extdevices_presence & 2: # UCTS presence flag is OK ucts_time = event.lst.tel[ telescope_id].evt.ucts_timestamp * 1e-9 # secs if first_valid_ucts is None: first_valid_ucts = ucts_time initial_dragon_counter = ( event.lst.tel[telescope_id].evt. pps_counter[module_id] + event.lst.tel[telescope_id].evt. tenMHz_counter[module_id] * 10**(-7)) logger.warning( f"Dragon timestamps not based on a valid absolute reference timestamp. " f"Consider using the following initial values \n" f"Event ID: {event.index.event_id}, " f"First valid UCTS timestamp: {first_valid_ucts:.9f} s, " f"corresponding Dragon counter {initial_dragon_counter:.9f} s" ) if event.lst.tel[telescope_id].evt.extdevices_presence & 1 \ and first_valid_ucts_tib is None: # Both TIB and UCTS presence flags are OK first_valid_ucts_tib = ucts_time initial_tib_counter = ( event.lst.tel[telescope_id].evt. tib_pps_counter + event.lst.tel[telescope_id].evt. tib_tenMHz_counter * 10**(-7)) logger.warning( f"TIB timestamps not based on a valid absolute reference timestamp. " f"Consider using the following initial values \n" f"Event ID: {event.index.event_id}, UCTS timestamp corresponding to " f"the first valid TIB counter: {first_valid_ucts_tib:.9f} s, " f"corresponding TIB counter {initial_tib_counter:.9f} s" ) else: ucts_time = math.nan else: # Dragon/TIB timestamps based on a valid absolute reference UCTS timestamp dragon_time = ( (ucts_t0_dragon - dragon_counter0) * 1e-9 + # secs event.lst.tel[telescope_id].evt. pps_counter[module_id] + event.lst.tel[telescope_id].evt. tenMHz_counter[module_id] * 10**(-7)) tib_time = ( (ucts_t0_tib - tib_counter0) * 1e-9 + # secs event.lst.tel[telescope_id].evt.tib_pps_counter + event.lst.tel[telescope_id].evt.tib_tenMHz_counter * 10**(-7)) if event.lst.tel[ telescope_id].evt.extdevices_presence & 2: # UCTS presence flag is OK ucts_time = event.lst.tel[ telescope_id].evt.ucts_timestamp * 1e-9 # secs if first_valid_ucts is None: first_valid_ucts = ucts_time if first_valid_ucts_tib is None \ and event.lst.tel[telescope_id].evt.extdevices_presence & 1: first_valid_ucts_tib = ucts_time else: ucts_time = math.nan # FIXME: directly use unix_tai format whenever astropy v4.1 is out ucts_time_utc = unix_tai_to_time(ucts_time) dragon_time_utc = unix_tai_to_time(dragon_time) tib_time_utc = unix_tai_to_time(tib_time) dl1_container.ucts_time = ucts_time_utc.unix dl1_container.dragon_time = dragon_time_utc.unix dl1_container.tib_time = tib_time_utc.unix # Until the TIB trigger_type is fully reliable, we also add # the ucts_trigger_type to the data dl1_container.ucts_trigger_type = event.lst.tel[ telescope_id].evt.ucts_trigger_type # Due to a DAQ bug, sometimes there are 'jumps' in the # UCTS info in the raw files. After one such jump, # all the UCTS info attached to an event actually # corresponds to the next event. This one-event # shift stays like that until there is another jump # (then it becomes a 2-event shift and so on). We will # keep track of those jumps, by storing the UCTS info # of the previously read events in the list # previous_ucts_time_unix. The list has one element # for each of the jumps, so if there has been just # one jump we have the UCTS info of the previous # event only (which truly corresponds to the # current event). If there have been n jumps, we keep # the past n events. The info to be used for # the current event is always the first element of # the array, previous_ucts_time_unix[0], whereas the # current event's (wrong) ucts info is placed last in # the array. Each time the first array element is # used, it is removed and the rest move up in the # list. We have another similar array for the trigger # types, previous_ucts_trigger_type # if len(previous_ucts_time_unix) > 0: # keep the time & trigger type read for this # event (which really correspond to a later event): current_ucts_time = dl1_container.ucts_time current_ucts_trigger_type = dl1_container.ucts_trigger_type # put in dl1_container the proper time for this # event: dl1_container.ucts_time = \ previous_ucts_time_unix.pop(0) dl1_container.ucts_trigger_type = \ previous_ucts_trigger_type.pop(0) # now put the current values last in the list, # for later use: previous_ucts_time_unix.append(current_ucts_time) previous_ucts_trigger_type.\ append(current_ucts_trigger_type) # Now check consistency of UCTS and Dragon times. If # UCTS time is ahead of Dragon time by more than # 1.e-6 s, most likely the UCTS info has been # lost for this event (i.e. there has been another # 'jump' of those described above), and the one we have # actually corresponds to the next event. So we put it # back first in the list, to assign it to the next # event. We also move the other elements down in the # list, which will now be one element longer. # We leave the current event with the same time, # which will be approximately correct (depending on # event rate), and set its ucts_trigger_type to -1, # which will tell us a jump happened and hence this # event does not have proper UCTS info. if dl1_container.ucts_time - dl1_container.dragon_time > 1.e-6: previous_ucts_time_unix.\ insert( 0, dl1_container.ucts_time) previous_ucts_trigger_type.\ insert(0, dl1_container.ucts_trigger_type) dl1_container.ucts_trigger_type = -1 # Select the timestamps to be used for pointing interpolation if config['timestamps_pointing'] == "ucts": event_timestamps = dl1_container.ucts_time elif config['timestamps_pointing'] == "dragon": event_timestamps = dragon_time_utc.unix elif config['timestamps_pointing'] == "tib": event_timestamps = tib_time_utc.unix else: raise ValueError( "The timestamps_pointing option is not a valid one. \ Try ucts (default), dragon or tib.") if pointing_file_path and event_timestamps > 0: azimuth, altitude = pointings.cal_pointingposition( event_timestamps, drive_data) event.pointing.tel[telescope_id].azimuth = azimuth event.pointing.tel[telescope_id].altitude = altitude dl1_container.az_tel = azimuth dl1_container.alt_tel = altitude else: dl1_container.az_tel = u.Quantity(np.nan, u.rad) dl1_container.alt_tel = u.Quantity(np.nan, u.rad) dl1_container.trigger_time = event.r0.tel[ telescope_id].trigger_time dl1_container.trigger_type = event.r0.tel[ telescope_id].trigger_type # FIXME: no need to read telescope characteristics like foclen for every event! foclen = subarray.tel[ telescope_id].optics.equivalent_focal_length mirror_area = u.Quantity( subarray.tel[telescope_id].optics.mirror_area, u.m**2) dl1_container.prefix = tel.prefix # extra info for the image table extra_im.tel_id = telescope_id extra_im.selected_gain_channel = event.r1.tel[ telescope_id].selected_gain_channel for container in [extra_im, dl1_container, event.r0, tel]: add_global_metadata(container, metadata) event.r0.prefix = '' writer.write(table_name=f'telescope/image/{tel_name}', containers=[event.index, tel, extra_im]) writer.write(table_name=f'telescope/parameters/{tel_name}', containers=[event.index, dl1_container]) # Muon ring analysis, for real data only (MC is done starting from DL1 files) if not is_simu: bad_pixels = event.mon.tel[ telescope_id].calibration.unusable_pixels[0] # Set to 0 unreliable pixels: image = tel.image * (~bad_pixels) # process only promising events, in terms of # of pixels with large signals: if tag_pix_thr(image): # re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings numsamples = event.r1.tel[telescope_id].waveform.shape[ 2] # not necessarily the same as in r0! bad_pixels_hg = event.mon.tel[ telescope_id].calibration.unusable_pixels[0] bad_pixels_lg = event.mon.tel[ telescope_id].calibration.unusable_pixels[1] # Now set to 0 all samples in unreliable pixels. Important for global peak # integrator in case of crazy pixels! TBD: can this be done in a simpler # way? bad_waveform = np.array( ([ np.transpose( np.array(numsamples * [bad_pixels_hg])), np.transpose( np.array(numsamples * [bad_pixels_lg])) ])) # print('hg bad pixels:',np.where(bad_pixels_hg)) # print('lg bad pixels:',np.where(bad_pixels_lg)) event.r1.tel[telescope_id].waveform *= ~bad_waveform r1_dl1_calibrator_for_muon_rings(event) tel = event.dl1.tel[telescope_id] image = tel.image * (~bad_pixels) # Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum) # perhaps the event is no longer promising (e.g. if it has a large time evolution) if not tag_pix_thr(image): good_ring = False else: # read geometry from event.inst. But not needed for every event. FIXME? geom = subarray.tel[telescope_id].\ camera.geometry muonintensityparam, dist_mask, \ ring_size, size_outside_ring, muonringparam, \ good_ring, radial_distribution, \ mean_pixel_charge_around_ring,\ muonpars = \ analyze_muon_event(subarray, event.index.event_id, image, geom, foclen, mirror_area, False, '') # mirror_area, True, './') # (test) plot muon rings as png files # Now we want to obtain the waveform sample (in HG and LG) at which the ring light peaks: bright_pixels_waveforms = event.r1.tel[ telescope_id].waveform[:, image > min_pe_for_muon_t_calc, :] stacked_waveforms = np.sum(bright_pixels_waveforms, axis=-2) # stacked waveforms from all bright pixels; shape (ngains, nsamples) hg_peak_sample = np.argmax(stacked_waveforms, axis=-1)[0] lg_peak_sample = np.argmax(stacked_waveforms, axis=-1)[1] if good_ring: fill_muon_event(-1, muon_parameters, good_ring, event.index.event_id, dragon_time, muonintensityparam, dist_mask, muonringparam, radial_distribution, ring_size, size_outside_ring, mean_pixel_charge_around_ring, muonpars, hg_peak_sample, lg_peak_sample) # writes mc information per telescope, including photo electron image if is_simu \ and (event.mc.tel[telescope_id].true_image > 0).any() \ and config['write_pe_image']: event.mc.tel[telescope_id].prefix = '' writer.write( table_name=f'simulation/{tel_name}', containers=[event.mc.tel[telescope_id], extra_im]) if not is_simu: # at the end of event loop ask calculation of remaining interleaved statistics new_ped, new_ff = calibration_calculator.output_interleaved_results( event) # write monitoring events write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=new_ped, new_ff=new_ff) if first_valid_ucts is None: logger.warning("Not valid UCTS timestamp found") if first_valid_ucts_tib is None: logger.warning("Not valid TIB counter value found") if is_simu: # Reconstruct source position from disp for all events and write the result in the output file for tel_name in ['LST_LSTCam']: focal = OpticsDescription.from_name( tel_name.split('_')[0]).equivalent_focal_length add_disp_to_parameters_table(output_filename, dl1_params_lstcam_key, focal) # Write energy histogram from simtel file and extra metadata # ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever if source.max_events is None: write_simtel_energy_histogram(source, output_filename, obs_id=event.index.obs_id, metadata=metadata) else: dir, name = os.path.split(output_filename) name = name.replace('dl1', 'muons').replace('LST-1.1', 'LST-1') # Consider the possibilities of DL1 files with .fits.h5 & .h5 ending: name = name.replace('.fits.h5', '.fits').replace('.h5', '.fits') muon_output_filename = Path(dir, name) table = Table(muon_parameters) table.write(muon_output_filename, format='fits', overwrite=True) # Produce the dl1 datacheck .h5 file: check_dl1(output_filename, Path(output_filename).parent, max_cores=1, create_pdf=False)
def save_container(container, filename, group_name, table_name): with HDF5TableWriter(filename, mode='a', group_name=group_name) as h5: h5.write(table_name, container)
def entry(): args = docopt(__doc__) files = args['INPUT'] debug = args['--debug'] max_events = convert_max_events_args(args['--max_events']) output_path = args['OUTPUT'] if not os.path.exists(output_path): raise IOError('Path for output does not exists \n') pixel_id = convert_pixel_args(args['--pixel']) n_pixels = len(pixel_id) raw_histo_filename = 'raw_histo.pk' amplitude_histo_filename = output_path + 'amplitude_histo.pk' charge_histo_filename = output_path + 'charge_histo.pk' max_histo_filename = output_path + 'max_histo.pk' results_filename = output_path + 'fit_results.h5' dark_count_rate_filename = output_path + 'dark_count_rate.npz' crosstalk_filename = output_path + 'crosstalk.npz' electronic_noise_filename = output_path + 'electronic_noise.npz' integral_width = int(args['--integral_width']) shift = int(args['--shift']) pulse_finder_threshold = float(args['--pulse_finder_threshold']) n_samples = int(args['--n_samples']) # TODO access this in a better way ! if args['--compute']: raw_histo = raw.compute(files, max_events=max_events, pixel_id=pixel_id, output_path=output_path, filename=raw_histo_filename) baseline = raw_histo.mode() events = calibration_event_stream(files, pixel_id=pixel_id, max_events=max_events) # events = compute_baseline_with_min(events) events = fill_baseline(events, baseline) events = subtract_baseline(events) events = find_pulse_with_max(events) events = compute_charge(events, integral_width, shift) max_histo = Histogram1D(data_shape=(n_pixels, ), bin_edges=np.arange(-4095 * integral_width, 4095 * integral_width), axis_name='[LSB]') for event in events: max_histo.fill(event.data.reconstructed_charge) max_histo.save(max_histo_filename) events = calibration_event_stream(files, max_events=max_events, pixel_id=pixel_id) events = fill_baseline(events, baseline) events = subtract_baseline(events) # events = find_pulse_1(events, 0.5, 20) # events = find_pulse_2(events, widths=[5, 6], threshold_sigma=2) # events = find_pulse_fast(events, threshold=pulse_finder_threshold) # events = find_pulse_correlate(events, # threshold=pulse_finder_threshold) # events = find_pulse_gaussian_filter(events, # threshold=pulse_finder_threshold) events = find_pulse_wavelets(events, widths=[4, 5, 6], threshold_sigma=2) events = compute_charge(events, integral_width=integral_width, shift=shift) events = compute_amplitude(events) # events = fit_template(events) if debug: events = plot_event(events, 0) spe_charge = Histogram1D(data_shape=(n_pixels, ), bin_edges=np.arange(-4095 * integral_width, 4095 * integral_width)) spe_amplitude = Histogram1D(data_shape=(n_pixels, ), bin_edges=np.arange(-4095, 4095, 1)) for event in events: spe_charge.fill(event.data.reconstructed_charge) spe_amplitude.fill(event.data.reconstructed_amplitude) spe_charge.save(charge_histo_filename) spe_amplitude.save(amplitude_histo_filename) if args['--fit']: spe_charge = Histogram1D.load(charge_histo_filename) spe_amplitude = Histogram1D.load(amplitude_histo_filename) max_histo = Histogram1D.load(max_histo_filename) dark_count_rate = np.zeros(n_pixels) * np.nan electronic_noise = np.zeros(n_pixels) * np.nan for i, pixel in tqdm(enumerate(pixel_id), total=n_pixels, desc='Pixel'): x = max_histo._bin_centers() y = max_histo.data[i] n_entries = np.sum(y) mask = (y > 0) x = x[mask] y = y[mask] try: val, err = compute_gaussian_parameters_first_peak( x, y, snr=3, ) number_of_zeros = val['amplitude'] window_length = 4 * n_samples rate = compute_dark_rate(number_of_zeros, n_entries, window_length) dark_count_rate[pixel] = rate electronic_noise[pixel] = val['sigma'] except Exception as e: print('Could not compute dark count rate in pixel {}'.format( pixel)) print(e) np.savez(dark_count_rate_filename, dcr=dark_count_rate) np.savez(electronic_noise_filename, electronic_noise) spe = spe_charge name = 'charge' crosstalk = np.zeros(n_pixels) * np.nan results = SPEResultContainer() table_name = 'analysis_' + name with HDF5TableWriter(results_filename, table_name, mode='w') as h5: for i, pixel in tqdm(enumerate(pixel_id), total=n_pixels, desc='Pixel'): try: x = spe._bin_centers() y = spe.data[i] y_err = spe.errors(index=i) sigma_e = electronic_noise[i] sigma_e = sigma_e if not np.isnan(sigma_e) else None params, params_err, params_init, params_bound = \ fit_spe(x, y, y_err, sigma_e=sigma_e, snr=3, debug=debug) for key, val in params.items(): setattr(results.init, key, params_init[key]) setattr(results.param, key, params[key]) setattr(results.param_errors, key, params_err[key]) for key, val in results.items(): results[key]['pixel_id'] = pixel for key, val in results.items(): h5.write('spe_' + key, val) n_entries = params['a_1'] n_entries += params['a_2'] n_entries += params['a_3'] n_entries += params['a_4'] crosstalk[pixel] = (n_entries - params['a_1']) / n_entries except Exception as e: print('Could not fit for pixel_id : {}'.format(pixel)) print(e) np.savez(crosstalk_filename, crosstalk) if args['--save_figures']: spe_charge = Histogram1D.load(charge_histo_filename) spe_amplitude = Histogram1D.load(amplitude_histo_filename) raw_histo = Histogram1D.load( os.path.join(output_path, raw_histo_filename)) max_histo = Histogram1D.load(max_histo_filename) figure_directory = output_path + 'figures/' if not os.path.exists(figure_directory): os.makedirs(figure_directory) histograms = [spe_charge, spe_amplitude, raw_histo, max_histo] names = [ 'histogram_charge/', 'histogram_amplitude/', 'histogram_raw/', 'histo_max/' ] for i, histo in enumerate(histograms): figure = plt.figure() histogram_figure_directory = figure_directory + names[i] if not os.path.exists(histogram_figure_directory): os.makedirs(histogram_figure_directory) for j, pixel in enumerate(pixel_id): axis = figure.add_subplot(111) figure_path = histogram_figure_directory + 'pixel_{}'. \ format(pixel) try: histo.draw(index=(j, ), axis=axis, log=True, legend=False) figure.savefig(figure_path) except Exception as e: print('Could not save pixel {} to : {} \n'.format( pixel, figure_path)) print(e) axis.remove() if args['--display']: spe_charge = Histogram1D.load(charge_histo_filename) spe_amplitude = Histogram1D.load(amplitude_histo_filename) raw_histo = Histogram1D.load( os.path.join(output_path, raw_histo_filename)) max_histo = Histogram1D.load(max_histo_filename) spe_charge.draw(index=(0, ), log=True, legend=False) spe_amplitude.draw(index=(0, ), log=True, legend=False) raw_histo.draw(index=(0, ), log=True, legend=False) max_histo.draw(index=(0, ), log=True, legend=False) try: df = pd.HDFStore(results_filename, mode='r') parameters = df['analysis_charge/spe_param'] parameters_error = df['analysis_charge/spe_param_errors'] dark_count_rate = np.load(dark_count_rate_filename)['dcr'] crosstalk = np.load(crosstalk_filename)['arr_0'] except IOError as e: print(e) print('Could not find the analysis files !') plt.show() exit() label = 'mean : {:2f} \n std : {:2f}' for key, val in parameters.items(): fig = plt.figure() axes = fig.add_subplot(111) axes.hist(val, bins='auto', label=label.format(np.mean(val), np.std(val))) axes.set_xlabel(key + ' []') axes.set_ylabel('count []') axes.legend(loc='best') # fig = plt.figure() # axes = fig.add_subplot(111) # axes.hist(parameters_error[key]) # axes.set_xlabel(key + '_error' + ' []') # axes.set_ylabel('count []') crosstalk = crosstalk[np.isfinite(crosstalk)] dark_count_rate = dark_count_rate[np.isfinite(dark_count_rate)] plt.figure() plt.hist(crosstalk, bins='auto', label=label.format(np.mean(crosstalk), np.std(crosstalk))) plt.xlabel('XT []') plt.legend(loc='best') plt.figure() plt.hist(dark_count_rate[np.isfinite(dark_count_rate)], bins='auto', label=label.format(np.mean(dark_count_rate), np.std(dark_count_rate))) plt.xlabel('dark count rate [GHz]') plt.legend(loc='best') plt.show() return
def r0_to_dl1( input_filename=None, output_filename=None, custom_config={}, ): """ Chain r0 to dl1 Save the extracted dl1 parameters in output_filename Parameters ---------- input_filename: str path to input file, default: `gamma_test_large.simtel.gz` output_filename: str or None path to output file, defaults to writing dl1 into the current directory custom_config: path to a configuration file Returns ------- """ # using None as default and using `get_dataset_path` only inside the function # prevents downloading at import time. if input_filename is None: get_dataset_path('gamma_test_large.simtel.gz') if output_filename is None: try: run = parse_r0_filename(input_filename) output_filename = run_to_dl1_filename(run.tel_id, run.run, run.subrun) except ValueError: output_filename = r0_to_dl1_filename(Path(input_filename).name) if os.path.exists(output_filename): raise IOError(str(output_filename) + ' exists, exiting.') config = replace_config(standard_config, custom_config) source = EventSource(input_url=input_filename, config=Config(config["source_config"])) subarray = source.subarray is_simu = source.is_simulation metadata = global_metadata() write_metadata(metadata, output_filename) cal_mc = load_calibrator_from_config(config, subarray) # minimum number of pe in a pixel to include it # in calculation of muon ring time (peak sample): min_pe_for_muon_t_calc = 10. # Dictionary to store muon ring parameters muon_parameters = create_muon_table() # all this will be cleaned up in a next PR related to the configuration files r1_dl1_calibrator = CameraCalibrator( image_extractor_type=config['image_extractor'], config=Config(config), subarray=subarray) if not is_simu: # Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but # using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous r1_dl1_calibrator_for_muon_rings = CameraCalibrator( image_extractor_type=config['image_extractor_for_muons'], config=Config(config), subarray=subarray) # Component to process interleaved pedestal and flat-fields calib_config = Config({ config['calibration_product']: config[config['calibration_product']] }) calibration_calculator = CalibrationCalculator.from_name( config['calibration_product'], config=calib_config, subarray=source.subarray) calibration_index = DL1MonitoringEventIndexContainer() dl1_container = DL1ParametersContainer() extra_im = ExtraImageInfo() extra_im.prefix = '' # get rid of the prefix # Write extra information to the DL1 file subarray.to_hdf(output_filename) if is_simu: write_mcheader( source.simulation_config, output_filename, obs_id=source.obs_ids[0], filters=HDF5_ZSTD_FILTERS, metadata=metadata, ) nsb_tuning = False if 'waveform_nsb_tuning' in config.keys(): nsb_tuning = config['waveform_nsb_tuning']['nsb_tuning'] if nsb_tuning: if is_simu: nsb_original = extract_simulation_nsb(input_filename) pulse_template = NormalizedPulseTemplate.load_from_eventsource( subarray.tel[1].camera.readout) if 'nsb_tuning_ratio' in config['waveform_nsb_tuning'].keys(): # get value from config to possibly extract it beforehand on multiple files for averaging purposes # or gain time nsb_tuning_ratio = config['waveform_nsb_tuning'][ 'nsb_tuning_ratio'] else: # extract the pedestal variance difference between the current MC file and the target data # FIXME? fails for multiple telescopes nsb_tuning_ratio = calculate_required_additional_nsb( input_filename, config['waveform_nsb_tuning']['target_data'], config=config)[0] spe = np.loadtxt( config['waveform_nsb_tuning']['spe_location']).T spe_integral = np.cumsum(spe[1]) charge_spe_cumulative_pdf = interp1d(spe_integral, spe[0], kind='cubic', bounds_error=False, fill_value=0., assume_sorted=True) allowed_tel = np.zeros(len(nsb_original), dtype=bool) allowed_tel[np.array(config['source_config']['LSTEventSource'] ['allowed_tels'])] = True logger.info('Tuning NSB on MC waveform from ' + str(np.asarray(nsb_original)[allowed_tel]) + 'GHz to {0:d}%'.format( int(nsb_tuning_ratio * 100 + 100.5)) + ' for telescopes ids ' + str(config['source_config']['LSTEventSource'] ['allowed_tels'])) else: logger.warning( 'NSB tuning on waveform active in config but file is real data, option will be ignored' ) nsb_tuning = False with HDF5TableWriter( filename=output_filename, group_name='dl1/event', mode='a', filters=HDF5_ZSTD_FILTERS, add_prefix=True, # overwrite=True, ) as writer: setup_writer(writer, source.subarray, is_simulation=is_simu) event = None for i, event in enumerate(source): if i % 100 == 0: logger.info(i) event.dl0.prefix = '' event.trigger.prefix = '' if event.simulation is not None: event.simulation.prefix = 'mc' dl1_container.reset() # write sub tables if is_simu: write_subarray_tables(writer, event, metadata) cal_mc(event) if config['mc_image_scaling_factor'] != 1: rescale_dl1_charge(event, config['mc_image_scaling_factor']) else: if i == 0: # initialize the telescope # FIXME? LST calibrator is only for one telescope # it should be inside the telescope loop (?) tel_id = calibration_calculator.tel_id #initialize the event monitoring data event.mon = deepcopy(source.r0_r1_calibrator.mon_data) for container in [ event.mon.tel[tel_id].pedestal, event.mon.tel[tel_id].flatfield, event.mon.tel[tel_id].calibration ]: add_global_metadata(container, metadata) add_config_metadata(container, config) # write the first calibration event (initialized from calibration h5 file) write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=True, new_ff=True) # flat-field or pedestal: if (event.trigger.event_type == EventType.FLATFIELD or event.trigger.event_type == EventType.SKY_PEDESTAL): # process interleaved events (pedestals, ff, calibration) new_ped_event, new_ff_event = calibration_calculator.process_interleaved( event) # write monitoring containers if updated if new_ped_event or new_ff_event: write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=new_ped_event, new_ff=new_ff_event) # calibrate and gain select the event by hand for DL1 source.r0_r1_calibrator.calibrate(event) # Option to add nsb in waveforms if nsb_tuning: # FIXME? assumes same correction ratio for all telescopes for tel_id in config['source_config']['LSTEventSource'][ 'allowed_tels']: waveform = event.r1.tel[tel_id].waveform readout = subarray.tel[tel_id].camera.readout sampling_rate = readout.sampling_rate.to_value(u.GHz) dt = (1.0 / sampling_rate) selected_gains = event.r1.tel[tel_id].selected_gain_channel mask_high = (selected_gains == 0) tune_nsb_on_waveform(waveform, nsb_tuning_ratio, nsb_original[tel_id] * u.GHz, dt * u.ns, pulse_template, mask_high, charge_spe_cumulative_pdf) # create image for all events r1_dl1_calibrator(event) # Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten. # When the last version of the method is implemented, vol. reduction will be done at dl0 apply_volume_reduction(event, subarray, config) # FIXME? This should be eventually done after we evaluate whether the image is # a candidate muon ring. In that case the full image could be kept, or reduced # only after the ring analysis is complete. for telescope_id, dl1_tel in event.dl1.tel.items(): dl1_tel.prefix = '' # don't really need one tel_name = str(subarray.tel[telescope_id])[4:] # extra info for the image table extra_im.tel_id = telescope_id extra_im.selected_gain_channel = event.r1.tel[ telescope_id].selected_gain_channel add_global_metadata(extra_im, metadata) add_config_metadata(extra_im, config) focal_length = subarray.tel[ telescope_id].optics.equivalent_focal_length mirror_area = subarray.tel[telescope_id].optics.mirror_area dl1_container.reset() # update the calibration index in the dl1 event container dl1_container.calibration_id = calibration_index.calibration_id dl1_container.fill_event_info(event) # Will determine whether this event has to be written to the # DL1 output or not. if is_simu: dl1_container.fill_mc(event, subarray.positions[telescope_id]) assert event.dl1.tel[telescope_id].image is not None try: get_dl1( event, subarray, telescope_id, dl1_container=dl1_container, custom_config=config, ) except HillasParameterizationError: logging.exception( 'HillasParameterizationError in get_dl1()') if not is_simu: dl1_container.ucts_time = 0 # convert Time to unix timestamp in (UTC) to keep compatibility # with older lstchain # FIXME: just keep it as time, table writer and reader handle it dl1_container.dragon_time = event.trigger.time.unix dl1_container.tib_time = 0 if 'ucts_jump' in vars( event.lst.tel[telescope_id].evt.__class__): dl1_container.ucts_jump = event.lst.tel[ telescope_id].evt.ucts_jump dl1_container.ucts_trigger_type = event.lst.tel[ telescope_id].evt.ucts_trigger_type dl1_container.trigger_type = event.lst.tel[ telescope_id].evt.tib_masked_trigger else: dl1_container.trigger_type = event.trigger.event_type dl1_container.az_tel = event.pointing.tel[telescope_id].azimuth dl1_container.alt_tel = event.pointing.tel[ telescope_id].altitude dl1_container.trigger_time = event.trigger.time.unix dl1_container.event_type = event.trigger.event_type dl1_container.prefix = dl1_tel.prefix for container in [extra_im, dl1_container, event.r0, dl1_tel]: add_global_metadata(container, metadata) add_config_metadata(container, config) writer.write(table_name=f'telescope/parameters/{tel_name}', containers=[event.index, dl1_container]) writer.write(table_name=f'telescope/image/{tel_name}', containers=[event.index, dl1_tel, extra_im]) # Muon ring analysis, for real data only (MC is done starting from DL1 files) if not is_simu: bad_pixels = event.mon.tel[ telescope_id].calibration.unusable_pixels[0] # Set to 0 unreliable pixels: image = dl1_tel.image * (~bad_pixels) # process only promising events, in terms of # of pixels with large signals: if tag_pix_thr(image): # re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings numsamples = event.r1.tel[telescope_id].waveform.shape[ 1] # not necessarily the same as in r0! bad_pixels_hg = event.mon.tel[ telescope_id].calibration.unusable_pixels[0] bad_pixels_lg = event.mon.tel[ telescope_id].calibration.unusable_pixels[1] # Now set to 0 all samples in unreliable pixels. Important for global peak # integrator in case of crazy pixels! TBD: can this be done in a simpler # way? bad_pixels = bad_pixels_hg | bad_pixels_lg bad_waveform = np.transpose( np.array(numsamples * [bad_pixels])) # print('hg bad pixels:',np.where(bad_pixels_hg)) # print('lg bad pixels:',np.where(bad_pixels_lg)) event.r1.tel[telescope_id].waveform *= ~bad_waveform r1_dl1_calibrator_for_muon_rings(event) image = dl1_tel.image * (~bad_pixels) # Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum) # perhaps the event is no longer promising (e.g. if it has a large time evolution) if not tag_pix_thr(image): good_ring = False else: # read geometry from event.inst. But not needed for every event. FIXME? geom = subarray.tel[telescope_id].\ camera.geometry muonintensityparam, dist_mask, \ ring_size, size_outside_ring, muonringparam, \ good_ring, radial_distribution, \ mean_pixel_charge_around_ring,\ muonpars = \ analyze_muon_event(subarray, event.index.event_id, image, geom, focal_length, mirror_area, False, '') # mirror_area, True, './') # (test) plot muon rings as png files # Now we want to obtain the waveform sample (in HG & LG) at which the ring light peaks: bright_pixels = image > min_pe_for_muon_t_calc selected_gain = event.r1.tel[ telescope_id].selected_gain_channel mask_hg = bright_pixels & (selected_gain == 0) mask_lg = bright_pixels & (selected_gain == 1) bright_pixels_waveforms_hg = event.r1.tel[ telescope_id].waveform[mask_hg, :] bright_pixels_waveforms_lg = event.r1.tel[ telescope_id].waveform[mask_lg, :] stacked_waveforms_hg = np.sum( bright_pixels_waveforms_hg, axis=0) stacked_waveforms_lg = np.sum( bright_pixels_waveforms_lg, axis=0) # stacked waveforms from all bright pixels; shape (ngains, nsamples) hg_peak_sample = np.argmax(stacked_waveforms_hg, axis=-1) lg_peak_sample = np.argmax(stacked_waveforms_lg, axis=-1) if good_ring: fill_muon_event(-1, muon_parameters, good_ring, event.index.event_id, dl1_container.dragon_time, muonintensityparam, dist_mask, muonringparam, radial_distribution, ring_size, size_outside_ring, mean_pixel_charge_around_ring, muonpars, hg_peak_sample, lg_peak_sample) # writes mc information per telescope, including photo electron image if (is_simu and config['write_pe_image'] and event.simulation.tel[telescope_id].true_image is not None and event.simulation.tel[telescope_id].true_image.any()): event.simulation.tel[telescope_id].prefix = '' writer.write(table_name=f'simulation/{tel_name}', containers=[ event.simulation.tel[telescope_id], extra_im ]) if event is None: logger.warning('No events in file') if not is_simu and event is not None: # at the end of event loop ask calculation of remaining interleaved statistics new_ped, new_ff = calibration_calculator.output_interleaved_results( event) # write monitoring events write_calibration_data(writer, calibration_index, event.mon.tel[tel_id], new_ped=new_ped, new_ff=new_ff) if is_simu and event is not None: # Reconstruct source position from disp for all events and write the result in the output file add_disp_to_parameters_table(output_filename, dl1_params_lstcam_key, focal_length) # Write energy histogram from simtel file and extra metadata # ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever if source.max_events is None: write_simtel_energy_histogram(source, output_filename, obs_id=event.index.obs_id, metadata=metadata) if not is_simu: dir, name = os.path.split(output_filename) name = name.replace('dl1', 'muons').replace('LST-1.1', 'LST-1') # Consider the possibilities of DL1 files with .fits.h5 & .h5 ending: name = name.replace('.fits.h5', '.fits').replace('.h5', '.fits') muon_output_filename = Path(dir, name) table = Table(muon_parameters) table.write(muon_output_filename, format='fits', overwrite=True)
def r0_to_dl1(input_filename=get_dataset_path('gamma_test_large.simtel.gz'), output_filename=None): """ Chain r0 to dl1 Save the extracted dl1 parameters in output_filename Parameters ---------- input_filename: str - path to input file, default: `gamma_test_large.simtel.gz` output_filename: str - path to output file, default: `./` + basename(input_filename) Returns ------- """ import os output_filename = 'dl1_' + os.path.basename(input_filename).split('.')[0] + '.h5' if output_filename is None \ else output_filename source = event_source(input_filename) source.allowed_tels = allowed_tels source.max_events = max_events with HDF5TableWriter(filename=output_filename, group_name='events', overwrite=True) as writer: for i, event in enumerate(source): if i % 100 == 0: print(i) # cal.calibrate(event) # for telescope_id, dl1 in event.dl1.tel.items(): for ii, telescope_id in enumerate(event.r0.tels_with_data): camera = event.inst.subarray.tel[ telescope_id].camera # Camera geometry lst_calibration(event, telescope_id) dl1_container = get_dl1(event, telescope_id) if dl1_container is not None: particle_name = utils.guess_type(input_filename) # Some custom def dl1_container.mc_type = utils.particle_number( particle_name) dl1_container.hadroness = 1 if dl1_container.mc_type == 1 else 0 dl1_container.hadroness = dl1_container.mc_type dl1_container.wl = dl1_container.width / dl1_container.length dl1_container.mc_energy = np.log10( event.mc.energy.value * 1e3) # Log10(Energy) in GeV dl1_container.intensity = np.log10(dl1_container.intensity) dl1_container.gps_time = event.trig.gps_time.value foclen = event.inst.subarray.tel[ telescope_id].optics.equivalent_focal_length w = np.rad2deg(np.arctan2(dl1_container.width, foclen)) l = np.rad2deg(np.arctan2(dl1_container.length, foclen)) dl1_container.width = w.value dl1_container.length = l.value if w >= 0: writer.write(camera.cam_id, [dl1_container])