def write_simtel_energy_histogram(source,
output_filename,
obs_id=None,
filters=None,
metadata={}):
"""
Write the energy histogram from a simtel source to a HDF5 file
Parameters
----------
source: `ctapipe.io.event_source`
output_filename: str
obs_id: float, int, str or None
"""
# Writing histograms
with HDF5TableWriter(filename=output_filename,
group_name="simulation",
mode="a",
filters=filters) as writer:
writer.meta = metadata
for hist in yield_toplevel_of_type(source.file_, Histograms):
pass
# find histogram id 6 (thrown energy)
thrown = None
for hist in source.file_.histograms:
if hist['id'] == 6:
thrown = hist
thrown_hist = ThrownEventsHistogram()
thrown_hist.fill_from_simtel(thrown)
thrown_hist.obs_id = obs_id
if metadata is not None:
add_global_metadata(thrown_hist, metadata)
writer.write('thrown_event_distribution', [thrown_hist])
def setup(self):
self.log.info('Configure EventSourceFactory...')
self.event_source = EventSourceFactory.produce(
config=self.config, tool=self, product='HESSIOEventSource')
self.event_source.allowed_tels = self.config['Analysis'][
'allowed_tels']
self.calibrator = CameraCalibrator(config=self.config,
tool=self,
eventsource=self.event_source)
self.writer = HDF5TableWriter(filename=self.outfile,
group_name='image_infos',
overwrite=True)
# Define Pre-selection for images
preselcuts = self.config['Preselect']
self.image_cutflow = CutFlow('Image preselection')
self.image_cutflow.set_cuts(
dict(no_sel=None,
n_pixel=lambda s: np.count_nonzero(s) < preselcuts['n_pixel'][
'min'],
image_amplitude=lambda q: q < preselcuts['image_amplitude'][
'min']))
# Define Pre-selection for events
self.event_cutflow = CutFlow('Event preselection')
self.event_cutflow.set_cuts(dict(no_sel=None))
def test_read_table(tmp_path):
# write a simple hdf5 file using
container = ReconstructedEnergyContainer()
filename = tmp_path / "test_astropy_table.h5"
with HDF5TableWriter(filename) as writer:
for energy in np.logspace(1, 2, 10) * u.TeV:
container.energy = energy
writer.write("events", container)
# try opening the result
table = read_table(filename, "/events")
assert "energy" in table.columns
assert table["energy"].unit == u.TeV
assert "CTAPIPE_VERSION" in table.meta
assert table["energy"].description is not None
# test using a string
table = read_table(str(filename), "/events")
# test write the table back out to some other format:
table.write(tmp_path / "test_output.ecsv")
table.write(tmp_path / "test_output.fits.gz")
# test using a file handle
with tables.open_file(filename) as handle:
table = read_table(handle, "/events")
# test a bad input
with pytest.raises(ValueError):
table = read_table(12345, "/events")
def setup(self):
self.log.debug(f"Open file")
self.eventsource = EventSource.from_config(parent=self)
# if data remember how many event in the files
if "LSTEventSource" in str(type(self.eventsource)):
self.tot_events = len(self.eventsource.multi_file)
self.log.debug(f"Input file has file {self.tot_events} events")
else:
self.tot_events = self.eventsource.max_events
self.simulation = True
self.processor = CalibrationCalculator.from_name(
self.calibration_product,
parent=self,
subarray=self.eventsource.subarray)
if self.r0calibrator_product:
self.r0calibrator = CameraR0Calibrator.from_name(
self.r0calibrator_product, parent=self)
group_name = 'tel_' + str(self.processor.tel_id)
self.log.debug(f"Open output file {self.output_file}")
self.writer = HDF5TableWriter(filename=self.output_file,
group_name=group_name,
overwrite=True)
def setup(self):
kwargs = dict(parent=self)
self.eventsource = EventSource.from_config(**kwargs)
self.flatfield = FlatFieldCalculator.from_name(
self.flatfield_product,
**kwargs
)
self.pedestal = PedestalCalculator.from_name(
self.pedestal_product,
**kwargs
)
if self.r0calibrator_product:
self.r0calibrator = CameraR0Calibrator.from_name(
self.r0calibrator_product,
**kwargs
)
msg = "tel_id not the same for all calibration components"
assert self.r0calibrator.tel_id == self.pedestal.tel_id == self.flatfield.tel_id, msg
self.tel_id = self.flatfield.tel_id
group_name = 'tel_' + str(self.tel_id)
self.writer = HDF5TableWriter(
filename=self.output_file, group_name=group_name, overwrite=True
)
def setup(self):
kwargs = dict(parent=self)
self.eventsource = EventSource.from_config(**kwargs)
# remember how many event in the files
self.tot_events = len(self.eventsource.multi_file)
self.log.debug(f"Input file has file {self.tot_events} events")
self.flatfield = FlatFieldCalculator.from_name(self.flatfield_product,
**kwargs)
self.pedestal = PedestalCalculator.from_name(self.pedestal_product,
**kwargs)
if self.r0calibrator_product:
self.r0calibrator = CameraR0Calibrator.from_name(
self.r0calibrator_product, **kwargs)
msg = "tel_id not the same for all calibration components"
assert self.r0calibrator.tel_id == self.pedestal.tel_id == self.flatfield.tel_id, msg
self.tel_id = self.flatfield.tel_id
group_name = 'tel_' + str(self.tel_id)
self.log.debug(f"Open output file {self.output_file}")
self.writer = HDF5TableWriter(filename=self.output_file,
group_name=group_name,
overwrite=True)
def write_mcheader(mcheader,
output_filename,
obs_id=None,
filters=None,
metadata=None):
"""
Write the mcheader from an event container to a HDF5 file
Parameters
----------
output_filename: str
event: `ctapipe.io.DataContainer`
"""
extramc = ExtraMCInfo()
extramc.prefix = '' # get rid of the prefix
if metadata is not None:
add_global_metadata(mcheader, metadata)
add_global_metadata(extramc, metadata)
with HDF5TableWriter(filename=output_filename,
group_name="simulation",
mode="a",
filters=filters) as writer:
extramc.obs_id = obs_id
writer.write("run_config", [extramc, mcheader])
def setup(self):
self.log.debug(f"Open file")
self.eventsource = EventSource.from_config(parent=self)
tel_id = self.eventsource.lst_service.telescope_id
if self.eventsource.r0_r1_calibrator.drs4_pedestal_path.tel[
tel_id] is None:
raise IOError("Missing (mandatory) drs4 pedestal file in trailets")
# if data remember how many event in the files
if "LSTEventSource" in str(type(self.eventsource)):
self.tot_events = len(self.eventsource.multi_file)
self.log.debug(f"Input file has file {self.tot_events} events")
else:
self.tot_events = self.eventsource.max_events
self.simulation = True
self.processor = CalibrationCalculator.from_name(
self.calibration_product,
parent=self,
subarray=self.eventsource.subarray)
group_name = 'tel_' + str(tel_id)
self.log.debug(f"Open output file {self.output_file}")
self.writer = HDF5TableWriter(filename=self.output_file,
group_name=group_name,
overwrite=True)
def setup(self):
self.log.info('Configure EventSource...')
self.event_source = self.add_component(
EventSource.from_config(config=self.config, parent=self))
self.calibrator = self.add_component(CameraCalibrator(parent=self))
self.writer = self.add_component(
HDF5TableWriter(filename=self.outfile,
group_name='image_infos',
overwrite=True))
# Define Pre-selection for images
preselcuts = self.config['Preselect']
self.image_cutflow = CutFlow('Image preselection')
self.image_cutflow.set_cuts(
dict(no_sel=None,
n_pixel=lambda s: np.count_nonzero(s) < preselcuts['n_pixel'][
'min'],
image_amplitude=lambda q: q < preselcuts['image_amplitude'][
'min']))
# Define Pre-selection for events
self.event_cutflow = CutFlow('Event preselection')
self.event_cutflow.set_cuts(dict(no_sel=None))
def setup(self):
if self.output is None:
raise ToolConfigurationError(
"You need to provide an --output file")
if self.output.exists() and not self.overwrite:
raise ToolConfigurationError(
"Outputfile {self.output} already exists, use `--overwrite` to overwrite"
)
self.source = EventSource(parent=self)
subarray = self.source.subarray
self.calib = CameraCalibrator(subarray=subarray, parent=self)
self.ring_fitter = MuonRingFitter(parent=self)
self.intensity_fitter = MuonIntensityFitter(subarray=subarray,
parent=self)
self.cleaning = TailcutsImageCleaner(parent=self, subarray=subarray)
self.writer = HDF5TableWriter(self.output,
"",
add_prefix=True,
parent=self,
mode="w")
self.pixels_in_tel_frame = {}
self.field_of_view = {}
self.pixel_widths = {}
for p in [
"min_pixels", "pedestal", "ratio_width",
"completeness_threshold"
]:
getattr(self, p).attach_subarray(self.source.subarray)
def setup(self):
self.source: EventSource = self.add_component(
EventSource.from_config(parent=self)
)
if self.source.input_url == '':
raise ToolConfigurationError("please specify --input <events file>")
self.calib = self.add_component(CameraCalibrator(parent=self))
self.writer = self.add_component(HDF5TableWriter(self.outfile, "muons"))
def setup(self):
if self.events == '':
raise ToolConfigurationError(
"please specify --input <events file>")
self.log.debug("input: %s", self.events)
self.source = event_source(self.events)
self.calib = CameraCalibrator(parent=self)
self.writer = HDF5TableWriter(self.outfile, "muons")
def save_event_data(events, filename, group_name):
with HDF5TableWriter(filename, mode='a', group_name=group_name) as h5:
for event in events:
h5.write('waveforms', event.data)
yield event
def setup(self):
if self.events == '':
raise ToolConfigurationError(
"please specify --input <events file>")
self.log.debug("input: %s", self.events)
self.source = EventSourceFactory.produce(input_url=self.events)
self.calib = CameraCalibrator(config=self.config,
tool=self,
eventsource=self.source)
self.writer = HDF5TableWriter(self.outfile, "muons")
def setup(self):
kwargs = dict(parent=self)
self.eventsource = EventSource.from_config(**kwargs)
self.pedestal = PedestalCalculator.from_name(self.calculator_product,
**kwargs)
self.group_name = 'tel_' + str(self.pedestal.tel_id)
self.writer = HDF5TableWriter(filename=self.output_file,
group_name=self.group_name,
overwrite=True)
def setup(self):
self.log.info("Configure EventSource...")
self.event_source = EventSource.from_url(self.infile, parent=self)
self.calibrator = CameraCalibrator(subarray=self.event_source.subarray,
parent=self)
self.writer = HDF5TableWriter(filename=self.outfile,
group_name="image_infos",
overwrite=True,
parent=self)
def setup(self):
kwargs = dict(parent=self)
self.eventsource = EventSource.from_config(**kwargs)
self.flatfield = FlatFieldCalculator.from_name(self.calculator_product,
**kwargs)
self.cleaner = WaveformCleaner.from_name(self.cleaner_product,
**kwargs)
self.writer = HDF5TableWriter(filename=self.output_file,
group_name='flatfield',
overwrite=True)
def setup(self):
self.log.info("Configure EventSource...")
EventSource.input_url.default_value = get_dataset_path(
"lst_prod3_calibration_and_mcphotons.simtel.zst")
self.event_source = EventSource(parent=self)
self.calibrator = CameraCalibrator(subarray=self.event_source.subarray,
parent=self)
self.writer = HDF5TableWriter(filename=self.output,
group_name="image_infos",
overwrite=True,
parent=self)
def setup(self):
if self.output is None:
raise ToolConfigurationError(
'You need to provide an --output file')
if self.output.exists() and not self.overwrite:
raise ToolConfigurationError(
'Outputfile {self.output} already exists, use `--overwrite` to overwrite'
)
self.source = self.add_component(EventSource.from_config(parent=self))
self.extractor = self.add_component(
ImageExtractor.from_name(self.extractor_name,
parent=self,
subarray=self.source.subarray))
self.calib = self.add_component(
CameraCalibrator(
subarray=self.source.subarray,
parent=self,
image_extractor=self.extractor,
))
self.ring_fitter = self.add_component(MuonRingFitter(parent=self, ))
self.intensity_fitter = self.add_component(
MuonIntensityFitter(
subarray=self.source.subarray,
parent=self,
))
self.cleaning = self.add_component(
TailcutsImageCleaner(
parent=self,
subarray=self.source.subarray,
))
self.writer = self.add_component(
HDF5TableWriter(
self.output,
"",
add_prefix=True,
parent=self,
mode='w',
))
self.pixels_in_tel_frame = {}
self.field_of_view = {}
self.pixel_widths = {}
for p in [
'min_pixels', 'pedestal', 'ratio_width',
'completeness_threshold'
]:
getattr(self, p).attach_subarray(self.source.subarray)
def test_condition(tmp_path):
# write a simple hdf5 file using
container = ReconstructedEnergyContainer()
filename = tmp_path / "test_astropy_table.h5"
with HDF5TableWriter(filename) as writer:
for energy in [np.nan, 100, np.nan, 50, -1.0] * u.TeV:
container.energy = energy
writer.write("events", container)
# try opening the result
table = read_table(filename, "/events", condition="energy > 0")
assert len(table) == 2
assert np.all(table["energy"] == [100, 50] * u.TeV)
def test_read_table_time(tmp_path):
t0 = Time("2020-01-01T20:00:00.0")
times = t0 + np.arange(10) * u.s
# use table writer to write test file
filename = tmp_path / "test_astropy_table.h5"
with HDF5TableWriter(filename) as writer:
for t in times:
container = TelescopeTriggerContainer(time=t, n_trigger_pixels=10)
writer.write("events", container)
# check reading in the times works as expected
table = read_table(filename, "/events")
assert isinstance(table["time"], Time)
assert np.allclose(times.tai.mjd, table["time"].tai.mjd)
def setup(self):
self.log.debug("Opening file")
self.eventsource = EventSource.from_config(parent=self)
self.processor = CalibrationCalculator.from_name(
self.calibration_product,
parent=self,
subarray=self.eventsource.subarray)
tel_id = self.processor.tel_id
# if real data
if isinstance(self.eventsource, LSTEventSource):
if tel_id != self.eventsource.lst_service.telescope_id:
raise ValueError(
f"Events telescope_id {self.eventsource.lst_service.telescope_id} "
f"different than CalibrationCalculator telescope_id {tel_id}"
)
if self.eventsource.r0_r1_calibrator.drs4_pedestal_path.tel[
tel_id] is None:
raise IOError(
"Missing (mandatory) drs4 pedestal file in trailets")
# remember how many events in the files
self.tot_events = len(self.eventsource.multi_file)
self.log.debug(f"Input file has file {self.tot_events} events")
else:
self.tot_events = self.eventsource.max_events
self.simulation = True
group_name = 'tel_' + str(tel_id)
self.log.debug(f"Open output file {self.output_file}")
self.writer = HDF5TableWriter(filename=self.output_file,
group_name=group_name,
overwrite=True)
def test_read_table_slicing(tmp_path):
filename = tmp_path / "test_slicing.h5"
# write a simple hdf5 file using
class Data(Container):
index = Field(0)
value = Field(0.0)
rng = np.random.default_rng(0)
values = rng.normal(size=100)
index = np.arange(len(values))
with HDF5TableWriter(filename) as writer:
for i, value in zip(index, values):
container = Data(index=i, value=value)
writer.write("events", container)
# try opening the result
table = read_table(filename, "/events", start=50)
assert len(table) == 50
assert np.all(table["index"] == index[50:])
assert np.all(table["value"] == values[50:])
table = read_table(filename, "/events", stop=50)
assert len(table) == 50
assert np.all(table["index"] == index[:50])
assert np.all(table["value"] == values[:50])
table = read_table(filename, "/events", start=10, stop=30)
assert len(table) == 20
assert np.all(table["index"] == index[10:30])
assert np.all(table["value"] == values[10:30])
table = read_table(filename, "/events", step=5)
assert len(table) == 20
assert np.all(table["index"] == index[::5])
assert np.all(table["value"] == values[::5])
def write_result_to_file(run_info_container, array_events, telescope_events, output_file, mode='a'):
'''Combines run_info, array_events and telescope_events
into one file using the HDF5TableWriter.
'''
logging.info(f'writing to file {output_file}')
with HDF5TableWriter(output_file, mode=mode, group_name='', add_prefix=True) as h5_table:
run_info_container.mc['run_array_direction'] = 0
h5_table.write('runs', [run_info_container, run_info_container.mc])
for array_event in array_events:
h5_table.write('array_events', [array_event, array_event.mc, array_event.reco])
for tel_event in telescope_events:
h5_table.write(
'telescope_events',
[
tel_event,
tel_event.pointing,
tel_event.hillas,
tel_event.concentration,
tel_event.leakage,
tel_event.timing,
tel_event.islands,
],
)
def r0_to_dl1(
input_filename=get_dataset_path('gamma_test_large.simtel.gz'),
output_filename=None,
custom_config={},
):
"""
Chain r0 to dl1
Save the extracted dl1 parameters in output_filename
Parameters
----------
input_filename: str
path to input file, default: `gamma_test_large.simtel.gz`
output_filename: str or None
path to output file, defaults to writing dl1 into the current directory
custom_config: path to a configuration file
Returns
-------
"""
if output_filename is None:
try:
run = parse_r0_filename(input_filename)
output_filename = run_to_dl1_filename(run.tel_id, run.run,
run.subrun)
except ValueError:
output_filename = r0_to_dl1_filename(Path(input_filename).name)
if os.path.exists(output_filename):
raise IOError(str(output_filename) + ' exists, exiting.')
config = replace_config(standard_config, custom_config)
custom_calibration = config["custom_calibration"]
source = EventSource(input_url=input_filename,
config=Config(config["source_config"]))
subarray = source.subarray
is_simu = source.is_simulation
metadata = global_metadata(source)
write_metadata(metadata, output_filename)
cal_mc = load_calibrator_from_config(config, subarray)
# minimum number of pe in a pixel to include it
# in calculation of muon ring time (peak sample):
min_pe_for_muon_t_calc = 10.
# Dictionary to store muon ring parameters
muon_parameters = create_muon_table()
# all this will be cleaned up in a next PR related to the configuration files
r1_dl1_calibrator = CameraCalibrator(
image_extractor_type=config['image_extractor'],
config=Config(config),
subarray=subarray)
if not is_simu:
# Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but
# using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous
r1_dl1_calibrator_for_muon_rings = CameraCalibrator(
image_extractor_type=config['image_extractor_for_muons'],
config=Config(config),
subarray=subarray)
# Component to process interleaved pedestal and flat-fields
calib_config = Config(config[config['calibration_product']])
calibration_calculator = CalibrationCalculator.from_name(
config['calibration_product'],
config=calib_config,
subarray=source.subarray)
calibration_index = DL1MonitoringEventIndexContainer()
dl1_container = DL1ParametersContainer()
extra_im = ExtraImageInfo()
extra_im.prefix = '' # get rid of the prefix
# Write extra information to the DL1 file
write_array_info(subarray, output_filename)
write_array_info_08(subarray, output_filename)
if is_simu:
write_mcheader(
source.simulation_config,
output_filename,
obs_id=source.obs_ids[0],
filters=HDF5_ZSTD_FILTERS,
metadata=metadata,
)
with HDF5TableWriter(
filename=output_filename,
group_name='dl1/event',
mode='a',
filters=HDF5_ZSTD_FILTERS,
add_prefix=True,
# overwrite=True,
) as writer:
if is_simu:
subarray = subarray
# build a mapping of tel_id back to tel_index:
# (note this should be part of SubarrayDescription)
idx = np.zeros(max(subarray.tel_indices) + 1)
for key, val in subarray.tel_indices.items():
idx[key] = val
# the final transform then needs the mapping and the number of telescopes
tel_list_transform = partial(
utils.expand_tel_list,
max_tels=max(subarray.tel) + 1,
)
writer.add_column_transform(table_name='subarray/trigger',
col_name='tels_with_trigger',
transform=tel_list_transform)
# Forcing filters for the dl1 dataset that are currently read from the pre-existing files
# This should be fixed in ctapipe and then corrected here
writer._h5file.filters = HDF5_ZSTD_FILTERS
logger.info(f"USING FILTERS: {writer._h5file.filters}")
for i, event in enumerate(source):
if i % 100 == 0:
logger.info(i)
event.dl0.prefix = ''
event.trigger.prefix = ''
if event.simulation is not None:
event.simulation.prefix = 'mc'
dl1_container.reset()
# write sub tables
if is_simu:
write_subarray_tables(writer, event, metadata)
if not custom_calibration:
cal_mc(event)
if config['mc_image_scaling_factor'] != 1:
rescale_dl1_charge(event,
config['mc_image_scaling_factor'])
else:
if i == 0:
# initialize the telescope
# FIXME? LST calibrator is only for one telescope
# it should be inside the telescope loop (?)
tel_id = calibration_calculator.tel_id
#initialize the event monitoring data
event.mon = source.r0_r1_calibrator.mon_data
# write the first calibration event (initialized from calibration h5 file)
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=True,
new_ff=True)
# flat-field or pedestal:
if (event.trigger.event_type == EventType.FLATFIELD
or event.trigger.event_type == EventType.SKY_PEDESTAL):
# process interleaved events (pedestals, ff, calibration)
new_ped_event, new_ff_event = calibration_calculator.process_interleaved(
event)
# write monitoring containers if updated
if new_ped_event or new_ff_event:
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=new_ped_event,
new_ff=new_ff_event)
# calibrate and gain select the event by hand for DL1
source.r0_r1_calibrator.calibrate(event)
# create image for all events
r1_dl1_calibrator(event)
# Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten.
# When the last version of the method is implemented, vol. reduction will be done at dl0
apply_volume_reduction(event, subarray, config)
# FIXME? This should be eventually done after we evaluate whether the image is
# a candidate muon ring. In that case the full image could be kept, or reduced
# only after the ring analysis is complete.
for ii, telescope_id in enumerate(event.dl1.tel.keys()):
dl1_container.reset()
# update the calibration index in the dl1 event container
dl1_container.calibration_id = calibration_index.calibration_id
dl1_container.fill_event_info(event)
tel = event.dl1.tel[telescope_id]
tel.prefix = '' # don't really need one
# remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST'
tel_name = str(subarray.tel[telescope_id])[4:]
if custom_calibration:
lst_calibration(event, telescope_id)
write_event = True
# Will determine whether this event has to be written to the
# DL1 output or not.
if is_simu:
dl1_container.fill_mc(event,
subarray.positions[telescope_id])
assert event.dl1.tel[telescope_id].image is not None
try:
get_dl1(
event,
subarray,
telescope_id,
dl1_container=dl1_container,
custom_config=config,
)
except HillasParameterizationError:
logging.exception(
'HillasParameterizationError in get_dl1()')
if not is_simu:
dl1_container.ucts_time = 0
# convert Time to unix timestamp in (UTC) to keep compatibility
# with older lstchain
# FIXME: just keep it as time, table writer and reader handle it
dl1_container.dragon_time = event.trigger.time.unix
dl1_container.tib_time = 0
dl1_container.ucts_trigger_type = event.lst.tel[
telescope_id].evt.ucts_trigger_type
dl1_container.trigger_type = event.lst.tel[
telescope_id].evt.tib_masked_trigger
else:
dl1_container.trigger_type = event.trigger.event_type
dl1_container.az_tel = event.pointing.tel[telescope_id].azimuth
dl1_container.alt_tel = event.pointing.tel[
telescope_id].altitude
dl1_container.trigger_time = event.trigger.time.unix
dl1_container.event_type = event.trigger.event_type
# FIXME: no need to read telescope characteristics like foclen for every event!
foclen = subarray.tel[
telescope_id].optics.equivalent_focal_length
mirror_area = u.Quantity(
subarray.tel[telescope_id].optics.mirror_area, u.m**2)
dl1_container.prefix = tel.prefix
# extra info for the image table
extra_im.tel_id = telescope_id
extra_im.selected_gain_channel = event.r1.tel[
telescope_id].selected_gain_channel
for container in [extra_im, dl1_container, event.r0, tel]:
add_global_metadata(container, metadata)
event.r0.prefix = ''
writer.write(table_name=f'telescope/image/{tel_name}',
containers=[event.index, tel, extra_im])
writer.write(table_name=f'telescope/parameters/{tel_name}',
containers=[event.index, dl1_container])
# Muon ring analysis, for real data only (MC is done starting from DL1 files)
if not is_simu:
bad_pixels = event.mon.tel[
telescope_id].calibration.unusable_pixels[0]
# Set to 0 unreliable pixels:
image = tel.image * (~bad_pixels)
# process only promising events, in terms of # of pixels with large signals:
if tag_pix_thr(image):
# re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings
numsamples = event.r1.tel[telescope_id].waveform.shape[
1] # not necessarily the same as in r0!
bad_pixels_hg = event.mon.tel[
telescope_id].calibration.unusable_pixels[0]
bad_pixels_lg = event.mon.tel[
telescope_id].calibration.unusable_pixels[1]
# Now set to 0 all samples in unreliable pixels. Important for global peak
# integrator in case of crazy pixels! TBD: can this be done in a simpler
# way?
bad_pixels = bad_pixels_hg | bad_pixels_lg
bad_waveform = np.transpose(
np.array(numsamples * [bad_pixels]))
# print('hg bad pixels:',np.where(bad_pixels_hg))
# print('lg bad pixels:',np.where(bad_pixels_lg))
event.r1.tel[telescope_id].waveform *= ~bad_waveform
r1_dl1_calibrator_for_muon_rings(event)
tel = event.dl1.tel[telescope_id]
image = tel.image * (~bad_pixels)
# Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum)
# perhaps the event is no longer promising (e.g. if it has a large time evolution)
if not tag_pix_thr(image):
good_ring = False
else:
# read geometry from event.inst. But not needed for every event. FIXME?
geom = subarray.tel[telescope_id].\
camera.geometry
muonintensityparam, dist_mask, \
ring_size, size_outside_ring, muonringparam, \
good_ring, radial_distribution, \
mean_pixel_charge_around_ring,\
muonpars = \
analyze_muon_event(subarray,
event.index.event_id,
image, geom, foclen,
mirror_area, False, '')
# mirror_area, True, './')
# (test) plot muon rings as png files
# Now we want to obtain the waveform sample (in HG & LG) at which the ring light peaks:
bright_pixels = image > min_pe_for_muon_t_calc
selected_gain = event.r1.tel[
telescope_id].selected_gain_channel
mask_hg = bright_pixels & (selected_gain == 0)
mask_lg = bright_pixels & (selected_gain == 1)
bright_pixels_waveforms_hg = event.r1.tel[
telescope_id].waveform[mask_hg, :]
bright_pixels_waveforms_lg = event.r1.tel[
telescope_id].waveform[mask_lg, :]
stacked_waveforms_hg = np.sum(
bright_pixels_waveforms_hg, axis=0)
stacked_waveforms_lg = np.sum(
bright_pixels_waveforms_lg, axis=0)
# stacked waveforms from all bright pixels; shape (ngains, nsamples)
hg_peak_sample = np.argmax(stacked_waveforms_hg,
axis=-1)
lg_peak_sample = np.argmax(stacked_waveforms_lg,
axis=-1)
if good_ring:
fill_muon_event(-1, muon_parameters, good_ring,
event.index.event_id,
dl1_container.dragon_time,
muonintensityparam, dist_mask,
muonringparam, radial_distribution,
ring_size, size_outside_ring,
mean_pixel_charge_around_ring,
muonpars, hg_peak_sample,
lg_peak_sample)
# writes mc information per telescope, including photo electron image
if (is_simu and config['write_pe_image']
and event.simulation.tel[telescope_id].true_image
is not None and
event.simulation.tel[telescope_id].true_image.any()):
event.simulation.tel[telescope_id].prefix = ''
writer.write(table_name=f'simulation/{tel_name}',
containers=[
event.simulation.tel[telescope_id],
extra_im
])
if not is_simu:
# at the end of event loop ask calculation of remaining interleaved statistics
new_ped, new_ff = calibration_calculator.output_interleaved_results(
event)
# write monitoring events
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=new_ped,
new_ff=new_ff)
if is_simu:
# Reconstruct source position from disp for all events and write the result in the output file
for tel_name in ['LST_LSTCam']:
focal = OpticsDescription.from_name(
tel_name.split('_')[0]).equivalent_focal_length
add_disp_to_parameters_table(output_filename,
dl1_params_lstcam_key, focal)
# Write energy histogram from simtel file and extra metadata
# ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever
if source.max_events is None:
write_simtel_energy_histogram(source,
output_filename,
obs_id=event.index.obs_id,
metadata=metadata)
else:
dir, name = os.path.split(output_filename)
name = name.replace('dl1', 'muons').replace('LST-1.1', 'LST-1')
# Consider the possibilities of DL1 files with .fits.h5 & .h5 ending:
name = name.replace('.fits.h5', '.fits').replace('.h5', '.fits')
muon_output_filename = Path(dir, name)
table = Table(muon_parameters)
table.write(muon_output_filename, format='fits', overwrite=True)
def r0_to_dl1(input_filename=get_dataset_path('gamma_test_large.simtel.gz'),
output_filename=None,
custom_config={},
pedestal_path=None,
calibration_path=None,
time_calibration_path=None,
pointing_file_path=None,
ucts_t0_dragon=math.nan,
dragon_counter0=math.nan,
ucts_t0_tib=math.nan,
tib_counter0=math.nan):
"""
Chain r0 to dl1
Save the extracted dl1 parameters in output_filename
Parameters
----------
input_filename: str
path to input file, default: `gamma_test_large.simtel.gz`
output_filename: str or None
path to output file, defaults to writing dl1 into the current directory
custom_config: path to a configuration file
pedestal_path: Path to the DRS4 pedestal file
calibration_path: Path to the file with calibration constants and
pedestals
time_calibration_path: Path to the DRS4 time correction file
pointing_file_path: path to the Drive log with the pointing information
ucts_t0_dragon: first valid ucts_time
dragon_counter0: Dragon counter corresponding to ucts_t0_dragon
ucts_t0_tib: first valid ucts_time for the first valid TIB counter
tib_counter0: first valid TIB counter
Returns
-------
"""
if output_filename is None:
try:
run = parse_r0_filename(input_filename)
output_filename = run_to_dl1_filename(run.tel_id, run.run,
run.subrun)
except ValueError:
output_filename = r0_to_dl1_filename(Path(input_filename).name)
if os.path.exists(output_filename):
raise IOError(str(output_filename) + ' exists, exiting.')
config = replace_config(standard_config, custom_config)
custom_calibration = config["custom_calibration"]
gain_selector = load_gain_selector_from_config(config)
# FIXME for ctapipe 0.8, str should be removed, as Path is supported
source = event_source(str(input_filename))
subarray = source.subarray
is_simu = source.is_simulation
source.allowed_tels = config["allowed_tels"]
if config["max_events"] is not None:
source.max_events = config["max_events"]
metadata = global_metadata(source)
write_metadata(metadata, output_filename)
cal_mc = load_calibrator_from_config(config, subarray)
# minimum number of pe in a pixel to include it
# in calculation of muon ring time (peak sample):
min_pe_for_muon_t_calc = 10.
# Dictionary to store muon ring parameters
muon_parameters = create_muon_table()
if not is_simu:
# TODO : add DRS4 calibration config in config file, read it and pass it here
r0_r1_calibrator = LSTR0Corrections(
pedestal_path=pedestal_path,
tel_id=1,
)
# all this will be cleaned up in a next PR related to the configuration files
r1_dl1_calibrator = LSTCameraCalibrator(
calibration_path=calibration_path,
time_calibration_path=time_calibration_path,
extractor_product=config['image_extractor'],
gain_threshold=Config(config).gain_selector_config['threshold'],
charge_scale=config['charge_scale'],
config=Config(config),
allowed_tels=[1],
subarray=subarray)
# Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but
# using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous
r1_dl1_calibrator_for_muon_rings = LSTCameraCalibrator(
calibration_path=calibration_path,
time_calibration_path=time_calibration_path,
extractor_product=config['image_extractor_for_muons'],
gain_threshold=Config(config).gain_selector_config['threshold'],
charge_scale=config['charge_scale'],
config=Config(config),
allowed_tels=[1],
subarray=subarray)
# Component to process interleaved pedestal and flat-fields
calib_config = Config(config[config['calibration_product']])
# set time calibration path for flatfield trailet ()
calib_config.FlasherFlatFieldCalculator.time_calibration_path = time_calibration_path
calibration_calculator = CalibrationCalculator.from_name(
config['calibration_product'],
config=calib_config,
subarray=source.subarray)
calibration_index = DL1MonitoringEventIndexContainer()
dl1_container = DL1ParametersContainer()
if pointing_file_path:
# Open drive report
pointings = PointingPosition(drive_path=pointing_file_path)
drive_data = pointings._read_drive_report()
extra_im = ExtraImageInfo()
extra_im.prefix = '' # get rid of the prefix
# get the first event to write array info and mc header
event_iter = iter(source)
first_event = next(event_iter)
# Write extra information to the DL1 file
write_array_info(subarray, output_filename)
write_array_info_08(subarray, output_filename)
if is_simu:
write_mcheader(
first_event.mcheader,
output_filename,
obs_id=first_event.index.obs_id,
filters=filters,
metadata=metadata,
)
with HDF5TableWriter(
filename=output_filename,
group_name='dl1/event',
mode='a',
filters=filters,
add_prefix=True,
# overwrite=True,
) as writer:
if is_simu:
subarray = subarray
# build a mapping of tel_id back to tel_index:
# (note this should be part of SubarrayDescription)
idx = np.zeros(max(subarray.tel_indices) + 1)
for key, val in subarray.tel_indices.items():
idx[key] = val
# the final transform then needs the mapping and the number of telescopes
tel_list_transform = partial(
utils.expand_tel_list,
max_tels=max(subarray.tel) + 1,
)
writer.add_column_transform(table_name='subarray/trigger',
col_name='tels_with_trigger',
transform=tel_list_transform)
# Forcing filters for the dl1 dataset that are currently read from the pre-existing files
# This should be fixed in ctapipe and then corrected here
writer._h5file.filters = filters
logger.info(f"USING FILTERS: {writer._h5file.filters}")
first_valid_ucts = None
first_valid_ucts_tib = None
previous_ucts_time_unix = []
previous_ucts_trigger_type = []
for i, event in enumerate(chain([first_event], event_iter)):
if i % 100 == 0:
logger.info(i)
event.dl0.prefix = ''
event.mc.prefix = 'mc'
event.trigger.prefix = ''
dl1_container.reset()
# write sub tables
if is_simu:
write_subarray_tables(writer, event, metadata)
if not custom_calibration:
cal_mc(event)
if config['mc_image_scaling_factor'] != 1:
rescale_dl1_charge(event,
config['mc_image_scaling_factor'])
else:
if i == 0:
# initialize the telescope
# FIXME? LST calibrator is only for one telescope
# it should be inside the telescope loop (?)
tel_id = calibration_calculator.tel_id
# write the first calibration event (initialized from calibration h5 file)
write_calibration_data(
writer,
calibration_index,
r1_dl1_calibrator.mon_data.tel[tel_id],
new_ped=True,
new_ff=True)
# drs4 calibrations
r0_r1_calibrator.calibrate(event)
# process interleaved events (pedestals, ff, calibration)
new_ped_event, new_ff_event = calibration_calculator.process_interleaved(
event)
# write monitoring containers if updated
if new_ped_event or new_ff_event:
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=new_ped_event,
new_ff=new_ff_event)
# calibrate and extract image from event
r1_dl1_calibrator(event)
# Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten.
# When the last version of the method is implemented, vol. reduction will be done at dl0
apply_volume_reduction(event, subarray, config)
# FIXME? This should be eventually done after we evaluate whether the image is
# a candidate muon ring. In that case the full image could be kept, or reduced
# only after the ring analysis is complete.
for ii, telescope_id in enumerate(event.r0.tels_with_data):
dl1_container.reset()
# update the calibration index in the dl1 event container
dl1_container.calibration_id = calibration_index.calibration_id
dl1_container.fill_event_info(event)
tel = event.dl1.tel[telescope_id]
tel.prefix = '' # don't really need one
# remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST'
tel_name = str(subarray.tel[telescope_id])[4:]
if custom_calibration:
lst_calibration(event, telescope_id)
write_event = True
# Will determine whether this event has to be written to the
# DL1 output or not.
if is_simu:
dl1_container.fill_mc(event,
subarray.positions[telescope_id])
try:
get_dl1(event,
subarray,
telescope_id,
dl1_container=dl1_container,
custom_config=config,
use_main_island=True)
except HillasParameterizationError:
logging.exception(
'HillasParameterizationError in get_dl1()')
if not is_simu:
# GPS + WRS + UCTS is now working in its nominal configuration.
# These TS are stored into ucts_time container.
# TS can be alternatively calculated from the TIB and
# Dragon modules counters based on the first valid UCTS TS
# as the reference point. For the time being, the three TS
# are stored in the DL1 files for checking purposes.
module_id = 82 # Get counters from the central Dragon module
if math.isnan(ucts_t0_dragon) and math.isnan(dragon_counter0) \
and math.isnan(ucts_t0_tib) and math.isnan(tib_counter0):
# Dragon/TIB timestamps not based on a valid absolute reference timestamp
dragon_time = (event.lst.tel[telescope_id].svc.date +
event.lst.tel[telescope_id].evt.
pps_counter[module_id] +
event.lst.tel[telescope_id].evt.
tenMHz_counter[module_id] * 10**(-7))
tib_time = (
event.lst.tel[telescope_id].svc.date +
event.lst.tel[telescope_id].evt.tib_pps_counter +
event.lst.tel[telescope_id].evt.tib_tenMHz_counter
* 10**(-7))
if event.lst.tel[
telescope_id].evt.extdevices_presence & 2:
# UCTS presence flag is OK
ucts_time = event.lst.tel[
telescope_id].evt.ucts_timestamp * 1e-9 # secs
if first_valid_ucts is None:
first_valid_ucts = ucts_time
initial_dragon_counter = (
event.lst.tel[telescope_id].evt.
pps_counter[module_id] +
event.lst.tel[telescope_id].evt.
tenMHz_counter[module_id] * 10**(-7))
logger.warning(
f"Dragon timestamps not based on a valid absolute reference timestamp. "
f"Consider using the following initial values \n"
f"Event ID: {event.index.event_id}, "
f"First valid UCTS timestamp: {first_valid_ucts:.9f} s, "
f"corresponding Dragon counter {initial_dragon_counter:.9f} s"
)
if event.lst.tel[telescope_id].evt.extdevices_presence & 1 \
and first_valid_ucts_tib is None:
# Both TIB and UCTS presence flags are OK
first_valid_ucts_tib = ucts_time
initial_tib_counter = (
event.lst.tel[telescope_id].evt.
tib_pps_counter +
event.lst.tel[telescope_id].evt.
tib_tenMHz_counter * 10**(-7))
logger.warning(
f"TIB timestamps not based on a valid absolute reference timestamp. "
f"Consider using the following initial values \n"
f"Event ID: {event.index.event_id}, UCTS timestamp corresponding to "
f"the first valid TIB counter: {first_valid_ucts_tib:.9f} s, "
f"corresponding TIB counter {initial_tib_counter:.9f} s"
)
else:
ucts_time = math.nan
else:
# Dragon/TIB timestamps based on a valid absolute reference UCTS timestamp
dragon_time = (
(ucts_t0_dragon - dragon_counter0) * 1e-9 + # secs
event.lst.tel[telescope_id].evt.
pps_counter[module_id] +
event.lst.tel[telescope_id].evt.
tenMHz_counter[module_id] * 10**(-7))
tib_time = (
(ucts_t0_tib - tib_counter0) * 1e-9 + # secs
event.lst.tel[telescope_id].evt.tib_pps_counter +
event.lst.tel[telescope_id].evt.tib_tenMHz_counter
* 10**(-7))
if event.lst.tel[
telescope_id].evt.extdevices_presence & 2:
# UCTS presence flag is OK
ucts_time = event.lst.tel[
telescope_id].evt.ucts_timestamp * 1e-9 # secs
if first_valid_ucts is None:
first_valid_ucts = ucts_time
if first_valid_ucts_tib is None \
and event.lst.tel[telescope_id].evt.extdevices_presence & 1:
first_valid_ucts_tib = ucts_time
else:
ucts_time = math.nan
# FIXME: directly use unix_tai format whenever astropy v4.1 is out
ucts_time_utc = unix_tai_to_time(ucts_time)
dragon_time_utc = unix_tai_to_time(dragon_time)
tib_time_utc = unix_tai_to_time(tib_time)
dl1_container.ucts_time = ucts_time_utc.unix
dl1_container.dragon_time = dragon_time_utc.unix
dl1_container.tib_time = tib_time_utc.unix
# Until the TIB trigger_type is fully reliable, we also add
# the ucts_trigger_type to the data
dl1_container.ucts_trigger_type = event.lst.tel[
telescope_id].evt.ucts_trigger_type
# Due to a DAQ bug, sometimes there are 'jumps' in the
# UCTS info in the raw files. After one such jump,
# all the UCTS info attached to an event actually
# corresponds to the next event. This one-event
# shift stays like that until there is another jump
# (then it becomes a 2-event shift and so on). We will
# keep track of those jumps, by storing the UCTS info
# of the previously read events in the list
# previous_ucts_time_unix. The list has one element
# for each of the jumps, so if there has been just
# one jump we have the UCTS info of the previous
# event only (which truly corresponds to the
# current event). If there have been n jumps, we keep
# the past n events. The info to be used for
# the current event is always the first element of
# the array, previous_ucts_time_unix[0], whereas the
# current event's (wrong) ucts info is placed last in
# the array. Each time the first array element is
# used, it is removed and the rest move up in the
# list. We have another similar array for the trigger
# types, previous_ucts_trigger_type
#
if len(previous_ucts_time_unix) > 0:
# keep the time & trigger type read for this
# event (which really correspond to a later event):
current_ucts_time = dl1_container.ucts_time
current_ucts_trigger_type = dl1_container.ucts_trigger_type
# put in dl1_container the proper time for this
# event:
dl1_container.ucts_time = \
previous_ucts_time_unix.pop(0)
dl1_container.ucts_trigger_type = \
previous_ucts_trigger_type.pop(0)
# now put the current values last in the list,
# for later use:
previous_ucts_time_unix.append(current_ucts_time)
previous_ucts_trigger_type.\
append(current_ucts_trigger_type)
# Now check consistency of UCTS and Dragon times. If
# UCTS time is ahead of Dragon time by more than
# 1.e-6 s, most likely the UCTS info has been
# lost for this event (i.e. there has been another
# 'jump' of those described above), and the one we have
# actually corresponds to the next event. So we put it
# back first in the list, to assign it to the next
# event. We also move the other elements down in the
# list, which will now be one element longer.
# We leave the current event with the same time,
# which will be approximately correct (depending on
# event rate), and set its ucts_trigger_type to -1,
# which will tell us a jump happened and hence this
# event does not have proper UCTS info.
if dl1_container.ucts_time - dl1_container.dragon_time > 1.e-6:
previous_ucts_time_unix.\
insert( 0, dl1_container.ucts_time)
previous_ucts_trigger_type.\
insert(0, dl1_container.ucts_trigger_type)
dl1_container.ucts_trigger_type = -1
# Select the timestamps to be used for pointing interpolation
if config['timestamps_pointing'] == "ucts":
event_timestamps = dl1_container.ucts_time
elif config['timestamps_pointing'] == "dragon":
event_timestamps = dragon_time_utc.unix
elif config['timestamps_pointing'] == "tib":
event_timestamps = tib_time_utc.unix
else:
raise ValueError(
"The timestamps_pointing option is not a valid one. \
Try ucts (default), dragon or tib.")
if pointing_file_path and event_timestamps > 0:
azimuth, altitude = pointings.cal_pointingposition(
event_timestamps, drive_data)
event.pointing.tel[telescope_id].azimuth = azimuth
event.pointing.tel[telescope_id].altitude = altitude
dl1_container.az_tel = azimuth
dl1_container.alt_tel = altitude
else:
dl1_container.az_tel = u.Quantity(np.nan, u.rad)
dl1_container.alt_tel = u.Quantity(np.nan, u.rad)
dl1_container.trigger_time = event.r0.tel[
telescope_id].trigger_time
dl1_container.trigger_type = event.r0.tel[
telescope_id].trigger_type
# FIXME: no need to read telescope characteristics like foclen for every event!
foclen = subarray.tel[
telescope_id].optics.equivalent_focal_length
mirror_area = u.Quantity(
subarray.tel[telescope_id].optics.mirror_area, u.m**2)
dl1_container.prefix = tel.prefix
# extra info for the image table
extra_im.tel_id = telescope_id
extra_im.selected_gain_channel = event.r1.tel[
telescope_id].selected_gain_channel
for container in [extra_im, dl1_container, event.r0, tel]:
add_global_metadata(container, metadata)
event.r0.prefix = ''
writer.write(table_name=f'telescope/image/{tel_name}',
containers=[event.index, tel, extra_im])
writer.write(table_name=f'telescope/parameters/{tel_name}',
containers=[event.index, dl1_container])
# Muon ring analysis, for real data only (MC is done starting from DL1 files)
if not is_simu:
bad_pixels = event.mon.tel[
telescope_id].calibration.unusable_pixels[0]
# Set to 0 unreliable pixels:
image = tel.image * (~bad_pixels)
# process only promising events, in terms of # of pixels with large signals:
if tag_pix_thr(image):
# re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings
numsamples = event.r1.tel[telescope_id].waveform.shape[
2] # not necessarily the same as in r0!
bad_pixels_hg = event.mon.tel[
telescope_id].calibration.unusable_pixels[0]
bad_pixels_lg = event.mon.tel[
telescope_id].calibration.unusable_pixels[1]
# Now set to 0 all samples in unreliable pixels. Important for global peak
# integrator in case of crazy pixels! TBD: can this be done in a simpler
# way?
bad_waveform = np.array(
([
np.transpose(
np.array(numsamples * [bad_pixels_hg])),
np.transpose(
np.array(numsamples * [bad_pixels_lg]))
]))
# print('hg bad pixels:',np.where(bad_pixels_hg))
# print('lg bad pixels:',np.where(bad_pixels_lg))
event.r1.tel[telescope_id].waveform *= ~bad_waveform
r1_dl1_calibrator_for_muon_rings(event)
tel = event.dl1.tel[telescope_id]
image = tel.image * (~bad_pixels)
# Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum)
# perhaps the event is no longer promising (e.g. if it has a large time evolution)
if not tag_pix_thr(image):
good_ring = False
else:
# read geometry from event.inst. But not needed for every event. FIXME?
geom = subarray.tel[telescope_id].\
camera.geometry
muonintensityparam, dist_mask, \
ring_size, size_outside_ring, muonringparam, \
good_ring, radial_distribution, \
mean_pixel_charge_around_ring,\
muonpars = \
analyze_muon_event(subarray,
event.index.event_id,
image, geom, foclen,
mirror_area, False, '')
# mirror_area, True, './')
# (test) plot muon rings as png files
# Now we want to obtain the waveform sample (in HG and LG) at which the ring light peaks:
bright_pixels_waveforms = event.r1.tel[
telescope_id].waveform[:, image >
min_pe_for_muon_t_calc, :]
stacked_waveforms = np.sum(bright_pixels_waveforms,
axis=-2)
# stacked waveforms from all bright pixels; shape (ngains, nsamples)
hg_peak_sample = np.argmax(stacked_waveforms,
axis=-1)[0]
lg_peak_sample = np.argmax(stacked_waveforms,
axis=-1)[1]
if good_ring:
fill_muon_event(-1, muon_parameters, good_ring,
event.index.event_id, dragon_time,
muonintensityparam, dist_mask,
muonringparam, radial_distribution,
ring_size, size_outside_ring,
mean_pixel_charge_around_ring,
muonpars, hg_peak_sample,
lg_peak_sample)
# writes mc information per telescope, including photo electron image
if is_simu \
and (event.mc.tel[telescope_id].true_image > 0).any() \
and config['write_pe_image']:
event.mc.tel[telescope_id].prefix = ''
writer.write(
table_name=f'simulation/{tel_name}',
containers=[event.mc.tel[telescope_id], extra_im])
if not is_simu:
# at the end of event loop ask calculation of remaining interleaved statistics
new_ped, new_ff = calibration_calculator.output_interleaved_results(
event)
# write monitoring events
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=new_ped,
new_ff=new_ff)
if first_valid_ucts is None:
logger.warning("Not valid UCTS timestamp found")
if first_valid_ucts_tib is None:
logger.warning("Not valid TIB counter value found")
if is_simu:
# Reconstruct source position from disp for all events and write the result in the output file
for tel_name in ['LST_LSTCam']:
focal = OpticsDescription.from_name(
tel_name.split('_')[0]).equivalent_focal_length
add_disp_to_parameters_table(output_filename,
dl1_params_lstcam_key, focal)
# Write energy histogram from simtel file and extra metadata
# ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever
if source.max_events is None:
write_simtel_energy_histogram(source,
output_filename,
obs_id=event.index.obs_id,
metadata=metadata)
else:
dir, name = os.path.split(output_filename)
name = name.replace('dl1', 'muons').replace('LST-1.1', 'LST-1')
# Consider the possibilities of DL1 files with .fits.h5 & .h5 ending:
name = name.replace('.fits.h5', '.fits').replace('.h5', '.fits')
muon_output_filename = Path(dir, name)
table = Table(muon_parameters)
table.write(muon_output_filename, format='fits', overwrite=True)
# Produce the dl1 datacheck .h5 file:
check_dl1(output_filename,
Path(output_filename).parent,
max_cores=1,
create_pdf=False)
def save_container(container, filename, group_name, table_name):
with HDF5TableWriter(filename, mode='a', group_name=group_name) as h5:
h5.write(table_name, container)
def entry():
args = docopt(__doc__)
files = args['INPUT']
debug = args['--debug']
max_events = convert_max_events_args(args['--max_events'])
output_path = args['OUTPUT']
if not os.path.exists(output_path):
raise IOError('Path for output does not exists \n')
pixel_id = convert_pixel_args(args['--pixel'])
n_pixels = len(pixel_id)
raw_histo_filename = 'raw_histo.pk'
amplitude_histo_filename = output_path + 'amplitude_histo.pk'
charge_histo_filename = output_path + 'charge_histo.pk'
max_histo_filename = output_path + 'max_histo.pk'
results_filename = output_path + 'fit_results.h5'
dark_count_rate_filename = output_path + 'dark_count_rate.npz'
crosstalk_filename = output_path + 'crosstalk.npz'
electronic_noise_filename = output_path + 'electronic_noise.npz'
integral_width = int(args['--integral_width'])
shift = int(args['--shift'])
pulse_finder_threshold = float(args['--pulse_finder_threshold'])
n_samples = int(args['--n_samples']) # TODO access this in a better way !
if args['--compute']:
raw_histo = raw.compute(files,
max_events=max_events,
pixel_id=pixel_id,
output_path=output_path,
filename=raw_histo_filename)
baseline = raw_histo.mode()
events = calibration_event_stream(files,
pixel_id=pixel_id,
max_events=max_events)
# events = compute_baseline_with_min(events)
events = fill_baseline(events, baseline)
events = subtract_baseline(events)
events = find_pulse_with_max(events)
events = compute_charge(events, integral_width, shift)
max_histo = Histogram1D(data_shape=(n_pixels, ),
bin_edges=np.arange(-4095 * integral_width,
4095 * integral_width),
axis_name='[LSB]')
for event in events:
max_histo.fill(event.data.reconstructed_charge)
max_histo.save(max_histo_filename)
events = calibration_event_stream(files,
max_events=max_events,
pixel_id=pixel_id)
events = fill_baseline(events, baseline)
events = subtract_baseline(events)
# events = find_pulse_1(events, 0.5, 20)
# events = find_pulse_2(events, widths=[5, 6], threshold_sigma=2)
# events = find_pulse_fast(events, threshold=pulse_finder_threshold)
# events = find_pulse_correlate(events,
# threshold=pulse_finder_threshold)
# events = find_pulse_gaussian_filter(events,
# threshold=pulse_finder_threshold)
events = find_pulse_wavelets(events,
widths=[4, 5, 6],
threshold_sigma=2)
events = compute_charge(events,
integral_width=integral_width,
shift=shift)
events = compute_amplitude(events)
# events = fit_template(events)
if debug:
events = plot_event(events, 0)
spe_charge = Histogram1D(data_shape=(n_pixels, ),
bin_edges=np.arange(-4095 * integral_width,
4095 * integral_width))
spe_amplitude = Histogram1D(data_shape=(n_pixels, ),
bin_edges=np.arange(-4095, 4095, 1))
for event in events:
spe_charge.fill(event.data.reconstructed_charge)
spe_amplitude.fill(event.data.reconstructed_amplitude)
spe_charge.save(charge_histo_filename)
spe_amplitude.save(amplitude_histo_filename)
if args['--fit']:
spe_charge = Histogram1D.load(charge_histo_filename)
spe_amplitude = Histogram1D.load(amplitude_histo_filename)
max_histo = Histogram1D.load(max_histo_filename)
dark_count_rate = np.zeros(n_pixels) * np.nan
electronic_noise = np.zeros(n_pixels) * np.nan
for i, pixel in tqdm(enumerate(pixel_id), total=n_pixels,
desc='Pixel'):
x = max_histo._bin_centers()
y = max_histo.data[i]
n_entries = np.sum(y)
mask = (y > 0)
x = x[mask]
y = y[mask]
try:
val, err = compute_gaussian_parameters_first_peak(
x,
y,
snr=3,
)
number_of_zeros = val['amplitude']
window_length = 4 * n_samples
rate = compute_dark_rate(number_of_zeros, n_entries,
window_length)
dark_count_rate[pixel] = rate
electronic_noise[pixel] = val['sigma']
except Exception as e:
print('Could not compute dark count rate in pixel {}'.format(
pixel))
print(e)
np.savez(dark_count_rate_filename, dcr=dark_count_rate)
np.savez(electronic_noise_filename, electronic_noise)
spe = spe_charge
name = 'charge'
crosstalk = np.zeros(n_pixels) * np.nan
results = SPEResultContainer()
table_name = 'analysis_' + name
with HDF5TableWriter(results_filename, table_name, mode='w') as h5:
for i, pixel in tqdm(enumerate(pixel_id),
total=n_pixels,
desc='Pixel'):
try:
x = spe._bin_centers()
y = spe.data[i]
y_err = spe.errors(index=i)
sigma_e = electronic_noise[i]
sigma_e = sigma_e if not np.isnan(sigma_e) else None
params, params_err, params_init, params_bound = \
fit_spe(x, y, y_err, sigma_e=sigma_e,
snr=3, debug=debug)
for key, val in params.items():
setattr(results.init, key, params_init[key])
setattr(results.param, key, params[key])
setattr(results.param_errors, key, params_err[key])
for key, val in results.items():
results[key]['pixel_id'] = pixel
for key, val in results.items():
h5.write('spe_' + key, val)
n_entries = params['a_1']
n_entries += params['a_2']
n_entries += params['a_3']
n_entries += params['a_4']
crosstalk[pixel] = (n_entries - params['a_1']) / n_entries
except Exception as e:
print('Could not fit for pixel_id : {}'.format(pixel))
print(e)
np.savez(crosstalk_filename, crosstalk)
if args['--save_figures']:
spe_charge = Histogram1D.load(charge_histo_filename)
spe_amplitude = Histogram1D.load(amplitude_histo_filename)
raw_histo = Histogram1D.load(
os.path.join(output_path, raw_histo_filename))
max_histo = Histogram1D.load(max_histo_filename)
figure_directory = output_path + 'figures/'
if not os.path.exists(figure_directory):
os.makedirs(figure_directory)
histograms = [spe_charge, spe_amplitude, raw_histo, max_histo]
names = [
'histogram_charge/', 'histogram_amplitude/', 'histogram_raw/',
'histo_max/'
]
for i, histo in enumerate(histograms):
figure = plt.figure()
histogram_figure_directory = figure_directory + names[i]
if not os.path.exists(histogram_figure_directory):
os.makedirs(histogram_figure_directory)
for j, pixel in enumerate(pixel_id):
axis = figure.add_subplot(111)
figure_path = histogram_figure_directory + 'pixel_{}'. \
format(pixel)
try:
histo.draw(index=(j, ), axis=axis, log=True, legend=False)
figure.savefig(figure_path)
except Exception as e:
print('Could not save pixel {} to : {} \n'.format(
pixel, figure_path))
print(e)
axis.remove()
if args['--display']:
spe_charge = Histogram1D.load(charge_histo_filename)
spe_amplitude = Histogram1D.load(amplitude_histo_filename)
raw_histo = Histogram1D.load(
os.path.join(output_path, raw_histo_filename))
max_histo = Histogram1D.load(max_histo_filename)
spe_charge.draw(index=(0, ), log=True, legend=False)
spe_amplitude.draw(index=(0, ), log=True, legend=False)
raw_histo.draw(index=(0, ), log=True, legend=False)
max_histo.draw(index=(0, ), log=True, legend=False)
try:
df = pd.HDFStore(results_filename, mode='r')
parameters = df['analysis_charge/spe_param']
parameters_error = df['analysis_charge/spe_param_errors']
dark_count_rate = np.load(dark_count_rate_filename)['dcr']
crosstalk = np.load(crosstalk_filename)['arr_0']
except IOError as e:
print(e)
print('Could not find the analysis files !')
plt.show()
exit()
label = 'mean : {:2f} \n std : {:2f}'
for key, val in parameters.items():
fig = plt.figure()
axes = fig.add_subplot(111)
axes.hist(val,
bins='auto',
label=label.format(np.mean(val), np.std(val)))
axes.set_xlabel(key + ' []')
axes.set_ylabel('count []')
axes.legend(loc='best')
# fig = plt.figure()
# axes = fig.add_subplot(111)
# axes.hist(parameters_error[key])
# axes.set_xlabel(key + '_error' + ' []')
# axes.set_ylabel('count []')
crosstalk = crosstalk[np.isfinite(crosstalk)]
dark_count_rate = dark_count_rate[np.isfinite(dark_count_rate)]
plt.figure()
plt.hist(crosstalk,
bins='auto',
label=label.format(np.mean(crosstalk), np.std(crosstalk)))
plt.xlabel('XT []')
plt.legend(loc='best')
plt.figure()
plt.hist(dark_count_rate[np.isfinite(dark_count_rate)],
bins='auto',
label=label.format(np.mean(dark_count_rate),
np.std(dark_count_rate)))
plt.xlabel('dark count rate [GHz]')
plt.legend(loc='best')
plt.show()
return
def r0_to_dl1(
input_filename=None,
output_filename=None,
custom_config={},
):
"""
Chain r0 to dl1
Save the extracted dl1 parameters in output_filename
Parameters
----------
input_filename: str
path to input file, default: `gamma_test_large.simtel.gz`
output_filename: str or None
path to output file, defaults to writing dl1 into the current directory
custom_config: path to a configuration file
Returns
-------
"""
# using None as default and using `get_dataset_path` only inside the function
# prevents downloading at import time.
if input_filename is None:
get_dataset_path('gamma_test_large.simtel.gz')
if output_filename is None:
try:
run = parse_r0_filename(input_filename)
output_filename = run_to_dl1_filename(run.tel_id, run.run,
run.subrun)
except ValueError:
output_filename = r0_to_dl1_filename(Path(input_filename).name)
if os.path.exists(output_filename):
raise IOError(str(output_filename) + ' exists, exiting.')
config = replace_config(standard_config, custom_config)
source = EventSource(input_url=input_filename,
config=Config(config["source_config"]))
subarray = source.subarray
is_simu = source.is_simulation
metadata = global_metadata()
write_metadata(metadata, output_filename)
cal_mc = load_calibrator_from_config(config, subarray)
# minimum number of pe in a pixel to include it
# in calculation of muon ring time (peak sample):
min_pe_for_muon_t_calc = 10.
# Dictionary to store muon ring parameters
muon_parameters = create_muon_table()
# all this will be cleaned up in a next PR related to the configuration files
r1_dl1_calibrator = CameraCalibrator(
image_extractor_type=config['image_extractor'],
config=Config(config),
subarray=subarray)
if not is_simu:
# Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but
# using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous
r1_dl1_calibrator_for_muon_rings = CameraCalibrator(
image_extractor_type=config['image_extractor_for_muons'],
config=Config(config),
subarray=subarray)
# Component to process interleaved pedestal and flat-fields
calib_config = Config({
config['calibration_product']:
config[config['calibration_product']]
})
calibration_calculator = CalibrationCalculator.from_name(
config['calibration_product'],
config=calib_config,
subarray=source.subarray)
calibration_index = DL1MonitoringEventIndexContainer()
dl1_container = DL1ParametersContainer()
extra_im = ExtraImageInfo()
extra_im.prefix = '' # get rid of the prefix
# Write extra information to the DL1 file
subarray.to_hdf(output_filename)
if is_simu:
write_mcheader(
source.simulation_config,
output_filename,
obs_id=source.obs_ids[0],
filters=HDF5_ZSTD_FILTERS,
metadata=metadata,
)
nsb_tuning = False
if 'waveform_nsb_tuning' in config.keys():
nsb_tuning = config['waveform_nsb_tuning']['nsb_tuning']
if nsb_tuning:
if is_simu:
nsb_original = extract_simulation_nsb(input_filename)
pulse_template = NormalizedPulseTemplate.load_from_eventsource(
subarray.tel[1].camera.readout)
if 'nsb_tuning_ratio' in config['waveform_nsb_tuning'].keys():
# get value from config to possibly extract it beforehand on multiple files for averaging purposes
# or gain time
nsb_tuning_ratio = config['waveform_nsb_tuning'][
'nsb_tuning_ratio']
else:
# extract the pedestal variance difference between the current MC file and the target data
# FIXME? fails for multiple telescopes
nsb_tuning_ratio = calculate_required_additional_nsb(
input_filename,
config['waveform_nsb_tuning']['target_data'],
config=config)[0]
spe = np.loadtxt(
config['waveform_nsb_tuning']['spe_location']).T
spe_integral = np.cumsum(spe[1])
charge_spe_cumulative_pdf = interp1d(spe_integral,
spe[0],
kind='cubic',
bounds_error=False,
fill_value=0.,
assume_sorted=True)
allowed_tel = np.zeros(len(nsb_original), dtype=bool)
allowed_tel[np.array(config['source_config']['LSTEventSource']
['allowed_tels'])] = True
logger.info('Tuning NSB on MC waveform from ' +
str(np.asarray(nsb_original)[allowed_tel]) +
'GHz to {0:d}%'.format(
int(nsb_tuning_ratio * 100 + 100.5)) +
' for telescopes ids ' +
str(config['source_config']['LSTEventSource']
['allowed_tels']))
else:
logger.warning(
'NSB tuning on waveform active in config but file is real data, option will be ignored'
)
nsb_tuning = False
with HDF5TableWriter(
filename=output_filename,
group_name='dl1/event',
mode='a',
filters=HDF5_ZSTD_FILTERS,
add_prefix=True,
# overwrite=True,
) as writer:
setup_writer(writer, source.subarray, is_simulation=is_simu)
event = None
for i, event in enumerate(source):
if i % 100 == 0:
logger.info(i)
event.dl0.prefix = ''
event.trigger.prefix = ''
if event.simulation is not None:
event.simulation.prefix = 'mc'
dl1_container.reset()
# write sub tables
if is_simu:
write_subarray_tables(writer, event, metadata)
cal_mc(event)
if config['mc_image_scaling_factor'] != 1:
rescale_dl1_charge(event,
config['mc_image_scaling_factor'])
else:
if i == 0:
# initialize the telescope
# FIXME? LST calibrator is only for one telescope
# it should be inside the telescope loop (?)
tel_id = calibration_calculator.tel_id
#initialize the event monitoring data
event.mon = deepcopy(source.r0_r1_calibrator.mon_data)
for container in [
event.mon.tel[tel_id].pedestal,
event.mon.tel[tel_id].flatfield,
event.mon.tel[tel_id].calibration
]:
add_global_metadata(container, metadata)
add_config_metadata(container, config)
# write the first calibration event (initialized from calibration h5 file)
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=True,
new_ff=True)
# flat-field or pedestal:
if (event.trigger.event_type == EventType.FLATFIELD
or event.trigger.event_type == EventType.SKY_PEDESTAL):
# process interleaved events (pedestals, ff, calibration)
new_ped_event, new_ff_event = calibration_calculator.process_interleaved(
event)
# write monitoring containers if updated
if new_ped_event or new_ff_event:
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=new_ped_event,
new_ff=new_ff_event)
# calibrate and gain select the event by hand for DL1
source.r0_r1_calibrator.calibrate(event)
# Option to add nsb in waveforms
if nsb_tuning:
# FIXME? assumes same correction ratio for all telescopes
for tel_id in config['source_config']['LSTEventSource'][
'allowed_tels']:
waveform = event.r1.tel[tel_id].waveform
readout = subarray.tel[tel_id].camera.readout
sampling_rate = readout.sampling_rate.to_value(u.GHz)
dt = (1.0 / sampling_rate)
selected_gains = event.r1.tel[tel_id].selected_gain_channel
mask_high = (selected_gains == 0)
tune_nsb_on_waveform(waveform, nsb_tuning_ratio,
nsb_original[tel_id] * u.GHz,
dt * u.ns, pulse_template, mask_high,
charge_spe_cumulative_pdf)
# create image for all events
r1_dl1_calibrator(event)
# Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten.
# When the last version of the method is implemented, vol. reduction will be done at dl0
apply_volume_reduction(event, subarray, config)
# FIXME? This should be eventually done after we evaluate whether the image is
# a candidate muon ring. In that case the full image could be kept, or reduced
# only after the ring analysis is complete.
for telescope_id, dl1_tel in event.dl1.tel.items():
dl1_tel.prefix = '' # don't really need one
tel_name = str(subarray.tel[telescope_id])[4:]
# extra info for the image table
extra_im.tel_id = telescope_id
extra_im.selected_gain_channel = event.r1.tel[
telescope_id].selected_gain_channel
add_global_metadata(extra_im, metadata)
add_config_metadata(extra_im, config)
focal_length = subarray.tel[
telescope_id].optics.equivalent_focal_length
mirror_area = subarray.tel[telescope_id].optics.mirror_area
dl1_container.reset()
# update the calibration index in the dl1 event container
dl1_container.calibration_id = calibration_index.calibration_id
dl1_container.fill_event_info(event)
# Will determine whether this event has to be written to the
# DL1 output or not.
if is_simu:
dl1_container.fill_mc(event,
subarray.positions[telescope_id])
assert event.dl1.tel[telescope_id].image is not None
try:
get_dl1(
event,
subarray,
telescope_id,
dl1_container=dl1_container,
custom_config=config,
)
except HillasParameterizationError:
logging.exception(
'HillasParameterizationError in get_dl1()')
if not is_simu:
dl1_container.ucts_time = 0
# convert Time to unix timestamp in (UTC) to keep compatibility
# with older lstchain
# FIXME: just keep it as time, table writer and reader handle it
dl1_container.dragon_time = event.trigger.time.unix
dl1_container.tib_time = 0
if 'ucts_jump' in vars(
event.lst.tel[telescope_id].evt.__class__):
dl1_container.ucts_jump = event.lst.tel[
telescope_id].evt.ucts_jump
dl1_container.ucts_trigger_type = event.lst.tel[
telescope_id].evt.ucts_trigger_type
dl1_container.trigger_type = event.lst.tel[
telescope_id].evt.tib_masked_trigger
else:
dl1_container.trigger_type = event.trigger.event_type
dl1_container.az_tel = event.pointing.tel[telescope_id].azimuth
dl1_container.alt_tel = event.pointing.tel[
telescope_id].altitude
dl1_container.trigger_time = event.trigger.time.unix
dl1_container.event_type = event.trigger.event_type
dl1_container.prefix = dl1_tel.prefix
for container in [extra_im, dl1_container, event.r0, dl1_tel]:
add_global_metadata(container, metadata)
add_config_metadata(container, config)
writer.write(table_name=f'telescope/parameters/{tel_name}',
containers=[event.index, dl1_container])
writer.write(table_name=f'telescope/image/{tel_name}',
containers=[event.index, dl1_tel, extra_im])
# Muon ring analysis, for real data only (MC is done starting from DL1 files)
if not is_simu:
bad_pixels = event.mon.tel[
telescope_id].calibration.unusable_pixels[0]
# Set to 0 unreliable pixels:
image = dl1_tel.image * (~bad_pixels)
# process only promising events, in terms of # of pixels with large signals:
if tag_pix_thr(image):
# re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings
numsamples = event.r1.tel[telescope_id].waveform.shape[
1] # not necessarily the same as in r0!
bad_pixels_hg = event.mon.tel[
telescope_id].calibration.unusable_pixels[0]
bad_pixels_lg = event.mon.tel[
telescope_id].calibration.unusable_pixels[1]
# Now set to 0 all samples in unreliable pixels. Important for global peak
# integrator in case of crazy pixels! TBD: can this be done in a simpler
# way?
bad_pixels = bad_pixels_hg | bad_pixels_lg
bad_waveform = np.transpose(
np.array(numsamples * [bad_pixels]))
# print('hg bad pixels:',np.where(bad_pixels_hg))
# print('lg bad pixels:',np.where(bad_pixels_lg))
event.r1.tel[telescope_id].waveform *= ~bad_waveform
r1_dl1_calibrator_for_muon_rings(event)
image = dl1_tel.image * (~bad_pixels)
# Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum)
# perhaps the event is no longer promising (e.g. if it has a large time evolution)
if not tag_pix_thr(image):
good_ring = False
else:
# read geometry from event.inst. But not needed for every event. FIXME?
geom = subarray.tel[telescope_id].\
camera.geometry
muonintensityparam, dist_mask, \
ring_size, size_outside_ring, muonringparam, \
good_ring, radial_distribution, \
mean_pixel_charge_around_ring,\
muonpars = \
analyze_muon_event(subarray,
event.index.event_id,
image, geom, focal_length,
mirror_area, False, '')
# mirror_area, True, './')
# (test) plot muon rings as png files
# Now we want to obtain the waveform sample (in HG & LG) at which the ring light peaks:
bright_pixels = image > min_pe_for_muon_t_calc
selected_gain = event.r1.tel[
telescope_id].selected_gain_channel
mask_hg = bright_pixels & (selected_gain == 0)
mask_lg = bright_pixels & (selected_gain == 1)
bright_pixels_waveforms_hg = event.r1.tel[
telescope_id].waveform[mask_hg, :]
bright_pixels_waveforms_lg = event.r1.tel[
telescope_id].waveform[mask_lg, :]
stacked_waveforms_hg = np.sum(
bright_pixels_waveforms_hg, axis=0)
stacked_waveforms_lg = np.sum(
bright_pixels_waveforms_lg, axis=0)
# stacked waveforms from all bright pixels; shape (ngains, nsamples)
hg_peak_sample = np.argmax(stacked_waveforms_hg,
axis=-1)
lg_peak_sample = np.argmax(stacked_waveforms_lg,
axis=-1)
if good_ring:
fill_muon_event(-1, muon_parameters, good_ring,
event.index.event_id,
dl1_container.dragon_time,
muonintensityparam, dist_mask,
muonringparam, radial_distribution,
ring_size, size_outside_ring,
mean_pixel_charge_around_ring,
muonpars, hg_peak_sample,
lg_peak_sample)
# writes mc information per telescope, including photo electron image
if (is_simu and config['write_pe_image']
and event.simulation.tel[telescope_id].true_image
is not None and
event.simulation.tel[telescope_id].true_image.any()):
event.simulation.tel[telescope_id].prefix = ''
writer.write(table_name=f'simulation/{tel_name}',
containers=[
event.simulation.tel[telescope_id],
extra_im
])
if event is None:
logger.warning('No events in file')
if not is_simu and event is not None:
# at the end of event loop ask calculation of remaining interleaved statistics
new_ped, new_ff = calibration_calculator.output_interleaved_results(
event)
# write monitoring events
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=new_ped,
new_ff=new_ff)
if is_simu and event is not None:
# Reconstruct source position from disp for all events and write the result in the output file
add_disp_to_parameters_table(output_filename, dl1_params_lstcam_key,
focal_length)
# Write energy histogram from simtel file and extra metadata
# ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever
if source.max_events is None:
write_simtel_energy_histogram(source,
output_filename,
obs_id=event.index.obs_id,
metadata=metadata)
if not is_simu:
dir, name = os.path.split(output_filename)
name = name.replace('dl1', 'muons').replace('LST-1.1', 'LST-1')
# Consider the possibilities of DL1 files with .fits.h5 & .h5 ending:
name = name.replace('.fits.h5', '.fits').replace('.h5', '.fits')
muon_output_filename = Path(dir, name)
table = Table(muon_parameters)
table.write(muon_output_filename, format='fits', overwrite=True)
def r0_to_dl1(input_filename=get_dataset_path('gamma_test_large.simtel.gz'),
output_filename=None):
"""
Chain r0 to dl1
Save the extracted dl1 parameters in output_filename
Parameters
----------
input_filename: str - path to input file, default: `gamma_test_large.simtel.gz`
output_filename: str - path to output file, default: `./` + basename(input_filename)
Returns
-------
"""
import os
output_filename = 'dl1_' + os.path.basename(input_filename).split('.')[0] + '.h5' if output_filename is None \
else output_filename
source = event_source(input_filename)
source.allowed_tels = allowed_tels
source.max_events = max_events
with HDF5TableWriter(filename=output_filename,
group_name='events',
overwrite=True) as writer:
for i, event in enumerate(source):
if i % 100 == 0: print(i)
# cal.calibrate(event)
# for telescope_id, dl1 in event.dl1.tel.items():
for ii, telescope_id in enumerate(event.r0.tels_with_data):
camera = event.inst.subarray.tel[
telescope_id].camera # Camera geometry
lst_calibration(event, telescope_id)
dl1_container = get_dl1(event, telescope_id)
if dl1_container is not None:
particle_name = utils.guess_type(input_filename)
# Some custom def
dl1_container.mc_type = utils.particle_number(
particle_name)
dl1_container.hadroness = 1 if dl1_container.mc_type == 1 else 0
dl1_container.hadroness = dl1_container.mc_type
dl1_container.wl = dl1_container.width / dl1_container.length
dl1_container.mc_energy = np.log10(
event.mc.energy.value * 1e3) # Log10(Energy) in GeV
dl1_container.intensity = np.log10(dl1_container.intensity)
dl1_container.gps_time = event.trig.gps_time.value
foclen = event.inst.subarray.tel[
telescope_id].optics.equivalent_focal_length
w = np.rad2deg(np.arctan2(dl1_container.width, foclen))
l = np.rad2deg(np.arctan2(dl1_container.length, foclen))
dl1_container.width = w.value
dl1_container.length = l.value
if w >= 0:
writer.write(camera.cam_id, [dl1_container])
