class DisplayDL1Calib(Tool):
name = "ctapipe-display-dl1"
description = __doc__
telescope = Int(
None,
allow_none=True,
help="Telescope to view. Set to None to display all telescopes.",
).tag(config=True)
aliases = Dict(
dict(
input="EventSource.input_url",
max_events="EventSource.max_events",
T="DisplayDL1Calib.telescope",
O="ImagePlotter.output_path",
))
flags = Dict(
dict(D=(
{
"ImagePlotter": {
"display": True
}
},
"Display the photo-electron images on-screen as they are produced.",
)))
classes = List([EventSource, ImagePlotter] +
traits.classes_with_traits(ImageExtractor))
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.config.EventSource.input_url = get_dataset_path(
"gamma_test_large.simtel.gz")
self.eventsource = None
self.calibrator = None
self.plotter = None
def setup(self):
self.eventsource = EventSource.from_config(parent=self)
subarray = self.eventsource.subarray
self.calibrator = CameraCalibrator(parent=self, subarray=subarray)
self.plotter = ImagePlotter(parent=self, subarray=subarray)
def start(self):
for event in self.eventsource:
self.calibrator(event)
tel_list = event.r0.tel.keys()
if self.telescope:
if self.telescope not in tel_list:
continue
tel_list = [self.telescope]
for telid in tel_list:
self.plotter.plot(event, telid)
def finish(self):
self.plotter.finish()
def test_classes_with_traits():
from ctapipe.core import Tool
class CompA(Component):
a = Int().tag(config=True)
class CompB(Component):
classes = List([CompA])
b = Int().tag(config=True)
class CompC(Component):
c = Int().tag(config=True)
class MyTool(Tool):
classes = [CompB, CompC]
with_traits = classes_with_traits(MyTool)
assert len(with_traits) == 4
assert MyTool in with_traits
assert CompA in with_traits
assert CompB in with_traits
assert CompC in with_traits
class CalibrationHDF5Writer(Tool):
"""
Tool that generates a HDF5 file with camera calibration coefficients.
This is just an example on how the monitoring containers can be filled using
the calibration Components in calib/camera.
This example is based on an input file with pedestal and flat-field events
For getting help run:
python calc_camera_calibration.py --help
"""
name = "CalibrationHDF5Writer"
description = "Generate a HDF5 file with camera calibration coefficients"
one_event = Bool(
False, help='Stop after first calibration event').tag(config=True)
output_file = Unicode('calibration.hdf5',
help='Name of the output file').tag(config=True)
calibration_product = traits.create_class_enum_trait(
CalibrationCalculator, default_value='LSTCalibrationCalculator')
events_to_skip = Int(
1000,
help='Number of first events to skip due to bad DRS4 pedestal correction'
).tag(config=True)
mc_min_flatfield_adc = Float(
2000,
help=
'Minimum high-gain camera median charge per pixel (ADC) for flatfield MC events'
).tag(config=True)
mc_max_pedestal_adc = Float(
300,
help=
'Maximum high-gain camera median charge per pixel (ADC) for pedestal MC events'
).tag(config=True)
aliases = {
("i", "input_file"): 'EventSource.input_url',
("m", "max_events"): 'EventSource.max_events',
("o", "output_file"): 'CalibrationHDF5Writer.output_file',
("p", "pedestal_file"):
"LSTEventSource.LSTR0Corrections.drs4_pedestal_path",
("s", "systematics_file"):
"LSTCalibrationCalculator.systematic_correction_path",
("r", "run_summary_file"):
"LSTEventSource.EventTimeCalculator.run_summary_path",
("t", "time_calibration_file"):
"LSTEventSource.LSTR0Corrections.drs4_time_calibration_path",
"events_to_skip": 'CalibrationHDF5Writer.events_to_skip',
}
flags = {
**traits.flag(
"flatfield-heuristic",
"LSTEventSource.use_flatfield_heuristic",
"Use flatfield heuristic",
"Do not use flatfield heuristic",
)
}
classes = ([EventSource, CalibrationCalculator] +
traits.classes_with_traits(CalibrationCalculator) +
traits.classes_with_traits(EventSource))
def __init__(self, **kwargs):
super().__init__(**kwargs)
"""
Tool that generates a HDF5 file with camera calibration coefficients.
Input file must contain interleaved pedestal and flat-field events
For getting help run:
lstchain_create_calibration_file --help
"""
self.eventsource = None
self.processor = None
self.writer = None
self.tot_events = 0
self.simulation = False
def setup(self):
self.log.debug("Opening file")
self.eventsource = EventSource.from_config(parent=self)
self.processor = CalibrationCalculator.from_name(
self.calibration_product,
parent=self,
subarray=self.eventsource.subarray)
tel_id = self.processor.tel_id
# if real data
if isinstance(self.eventsource, LSTEventSource):
if tel_id != self.eventsource.lst_service.telescope_id:
raise ValueError(
f"Events telescope_id {self.eventsource.lst_service.telescope_id} "
f"different than CalibrationCalculator telescope_id {tel_id}"
)
if self.eventsource.r0_r1_calibrator.drs4_pedestal_path.tel[
tel_id] is None:
raise IOError(
"Missing (mandatory) drs4 pedestal file in trailets")
# remember how many events in the files
self.tot_events = len(self.eventsource.multi_file)
self.log.debug(f"Input file has file {self.tot_events} events")
else:
self.tot_events = self.eventsource.max_events
self.simulation = True
group_name = 'tel_' + str(tel_id)
self.log.debug(f"Open output file {self.output_file}")
self.writer = HDF5TableWriter(filename=self.output_file,
group_name=group_name,
overwrite=True)
def start(self):
'''Calibration coefficient calculator'''
metadata = global_metadata()
write_metadata(metadata, self.output_file)
tel_id = self.processor.tel_id
new_ped = False
new_ff = False
end_of_file = False
try:
self.log.debug("Start loop")
self.log.debug(
f"If not simulation, skip first {self.events_to_skip} events")
for count, event in enumerate(self.eventsource):
# if simulation use not calibrated and not gain selected R0 waveform
if self.simulation:
# estimate offset of each channel from the camera median pedestal value
offset = np.median(
event.mon.tel[tel_id].calibration.pedestal_per_sample,
axis=1).round()
event.r1.tel[tel_id].waveform = event.r0.tel[
tel_id].waveform.astype(
np.float32) - offset[:, np.newaxis, np.newaxis]
if count % 1000 == 0 and count > self.events_to_skip:
self.log.debug(f"Event {count}")
# if last event write results
max_events_reached = (self.eventsource.max_events is not None
and count
== self.eventsource.max_events - 1)
if count == self.tot_events - 1 or max_events_reached:
self.log.debug(f"Last event, count = {count}")
end_of_file = True
# save the config, to be retrieved as data.meta['config']
if count == 0:
if self.simulation:
initialize_pixel_status(
event.mon.tel[tel_id],
event.r1.tel[tel_id].waveform.shape)
ped_data = event.mon.tel[tel_id].pedestal
add_config_metadata(ped_data, self.config)
add_global_metadata(ped_data, metadata)
ff_data = event.mon.tel[tel_id].flatfield
add_config_metadata(ff_data, self.config)
add_global_metadata(ff_data, metadata)
status_data = event.mon.tel[tel_id].pixel_status
add_config_metadata(status_data, self.config)
add_global_metadata(status_data, metadata)
calib_data = event.mon.tel[tel_id].calibration
add_config_metadata(calib_data, self.config)
add_global_metadata(calib_data, metadata)
# skip first events which are badly drs4 corrected
if not self.simulation and count < self.events_to_skip:
continue
# if pedestal event
# use a cut on the charge for MC events if trigger not defined
if event.trigger.event_type == EventType.SKY_PEDESTAL or (
self.simulation and np.median(
np.sum(event.r1.tel[tel_id].waveform[0],
axis=1)) < self.mc_max_pedestal_adc):
new_ped = self.processor.pedestal.calculate_pedestals(
event)
# if flat-field event
# use a cut on the charge for MC events if trigger not defined
elif event.trigger.event_type == EventType.FLATFIELD or (
self.simulation and np.median(
np.sum(event.r1.tel[tel_id].waveform[0],
axis=1)) > self.mc_min_flatfield_adc):
new_ff = self.processor.flatfield.calculate_relative_gain(
event)
# write pedestal results when enough statistics or end of file
if new_ped or end_of_file:
# update the monitoring container with the last statistics
if end_of_file:
self.processor.pedestal.store_results(event)
# write the event
self.log.debug(
f"Write pedestal data at event n. {count+1}, id {event.index.event_id} "
f"stat = {ped_data.n_events} events")
# write on file
self.writer.write('pedestal', ped_data)
new_ped = False
# write flatfield results when enough statistics (also for pedestals) or end of file
if (new_ff and ped_data.n_events > 0) or end_of_file:
# update the monitoring container with the last statistics
if end_of_file:
self.processor.flatfield.store_results(event)
self.log.debug(
f"Write flatfield data at event n. {count+1}, id {event.index.event_id} "
f"stat = {ff_data.n_events} events")
# write on file
self.writer.write('flatfield', ff_data)
new_ff = False
# Then, calculate calibration coefficients
self.processor.calculate_calibration_coefficients(event)
# write calib and pixel status
self.log.debug("Write pixel_status data")
self.writer.write('pixel_status', status_data)
self.log.debug("Write calibration data")
self.writer.write('calibration', calib_data)
if self.one_event:
break
except ValueError as e:
self.log.error(e)
def finish(self):
Provenance().add_output_file(self.output_file,
role='mon.tel.calibration')
self.writer.close()
class CalibrationCalculator(Component):
"""
Parent class for the camera calibration calculators.
Fills the MonitoringCameraContainer on the base of calibration events
Parameters
----------
flatfield_calculator: lstchain.calib.camera.flatfield
The flatfield to use. If None, then FlatFieldCalculator
will be used by default.
pedestal_calculator: lstchain.calib.camera.pedestal
The pedestal to use. If None, then
PedestalCalculator will be used by default.
kwargs
"""
squared_excess_noise_factor = Float(
1.222,
help='Excess noise factor squared: 1+ Var(gain)/Mean(Gain)**2').tag(
config=True)
pedestal_product = traits.create_class_enum_trait(
PedestalCalculator, default_value='PedestalIntegrator')
flatfield_product = traits.create_class_enum_trait(
FlatFieldCalculator, default_value='FlasherFlatFieldCalculator')
classes = List([FlatFieldCalculator, PedestalCalculator] +
traits.classes_with_traits(FlatFieldCalculator) +
traits.classes_with_traits(PedestalCalculator))
def __init__(self, subarray, parent=None, config=None, **kwargs):
"""
Parent class for the camera calibration calculators.
Fills the MonitoringCameraContainer on the base of calibration events
Parameters
----------
flatfield_calculator: lstchain.calib.camera.flatfield
The flatfield to use. If None, then FlatFieldCalculator
will be used by default.
pedestal_calculator: lstchain.calib.camera.pedestal
The pedestal to use. If None, then
PedestalCalculator will be used by default.
"""
super().__init__(parent=parent, config=config, **kwargs)
self.flatfield = FlatFieldCalculator.from_name(self.flatfield_product,
parent=self,
subarray=subarray)
self.pedestal = PedestalCalculator.from_name(self.pedestal_product,
parent=self,
subarray=subarray)
msg = "tel_id not the same for all calibration components"
if self.pedestal.tel_id != self.flatfield.tel_id:
raise ValueError(msg)
self.tel_id = self.flatfield.tel_id
self.log.debug(f"{self.pedestal}")
self.log.debug(f"{self.flatfield}")
class BokehFileViewer(Tool):
name = "BokehFileViewer"
description = ("Interactively explore an event file using the bokeh "
"visualisation package")
port = Int(5006, help="Port to open bokeh server onto").tag(config=True)
disable_server = Bool(False,
help="Do not start the bokeh server "
"(useful for testing)").tag(config=True)
default_url = get_dataset_path("gamma_test_large.simtel.gz")
EventSource.input_url.default_value = default_url
extractor_product = traits.enum_trait(ImageExtractor,
default='NeighborPeakWindowSum')
aliases = Dict(
dict(
port='BokehFileViewer.port',
disable_server='BokehFileViewer.disable_server',
f='EventSource.input_url',
max_events='EventSource.max_events',
extractor='BokehFileViewer.extractor_product',
))
classes = List([
EventSource,
] + traits.classes_with_traits(ImageExtractor))
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._event = None
self._event_index = None
self._event_id = None
self._telid = None
self._channel = None
self.w_next_event = None
self.w_previous_event = None
self.w_event_index = None
self.w_event_id = None
self.w_goto_event_index = None
self.w_goto_event_id = None
self.w_telid = None
self.w_channel = None
self.w_dl1_dict = None
self.wb_extractor = None
self.layout = None
self.reader = None
self.seeker = None
self.extractor = None
self.calibrator = None
self.viewer = None
self._updating_dl1 = False
# make sure, gzip files are seekable
self.config.SimTelEventSource.back_seekable = True
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
self.reader = EventSource.from_config(parent=self)
self.seeker = EventSeeker(self.reader, parent=self)
self.extractor = ImageExtractor.from_name(
self.extractor_product,
parent=self,
subarray=self.reader.subarray,
)
self.calibrator = CameraCalibrator(
subarray=self.reader.subarray,
parent=self,
image_extractor=self.extractor,
)
self.viewer = BokehEventViewer(parent=self)
# Setup widgets
self.viewer.create()
self.viewer.enable_automatic_index_increment()
self.create_previous_event_widget()
self.create_next_event_widget()
self.create_event_index_widget()
self.create_goto_event_index_widget()
self.create_event_id_widget()
self.create_goto_event_id_widget()
self.create_telid_widget()
self.create_channel_widget()
self.create_dl1_widgets()
self.update_dl1_widget_values()
# Setup layout
self.layout = layout([[self.viewer.layout],
[
self.w_previous_event, self.w_next_event,
self.w_goto_event_index, self.w_goto_event_id
], [self.w_event_index, self.w_event_id],
[self.w_telid, self.w_channel],
[self.wb_extractor]])
def start(self):
self.event_index = 0
def finish(self):
if not self.disable_server:
def modify_doc(doc):
doc.add_root(self.layout)
doc.title = self.name
directory = os.path.abspath(os.path.dirname(__file__))
theme_path = os.path.join(directory, "theme.yaml")
template_path = os.path.join(directory, "templates")
doc.theme = Theme(filename=theme_path)
env = jinja2.Environment(
loader=jinja2.FileSystemLoader(template_path))
doc.template = env.get_template('index.html')
self.log.info('Opening Bokeh application on '
'http://localhost:{}/'.format(self.port))
server = Server({'/': modify_doc}, num_procs=1, port=self.port)
server.start()
server.io_loop.add_callback(server.show, "/")
server.io_loop.start()
@property
def event_index(self):
return self._event_index
@event_index.setter
def event_index(self, val):
try:
self.event = self.seeker[val]
except IndexError:
self.log.warning(f"Event Index {val} does not exist")
@property
def event_id(self):
return self._event_id
@event_id.setter
def event_id(self, val):
try:
self.event = self.seeker[str(val)]
except IndexError:
self.log.warning(f"Event ID {val} does not exist")
@property
def telid(self):
return self._telid
@telid.setter
def telid(self, val):
self.channel = 0
tels = list(self.event.r0.tels_with_data)
if val not in tels:
val = tels[0]
self._telid = val
self.viewer.telid = val
self.update_telid_widget()
@property
def channel(self):
return self._channel
@channel.setter
def channel(self, val):
self._channel = val
self.viewer.channel = val
self.update_channel_widget()
@property
def event(self):
return self._event
@event.setter
def event(self, val):
self.calibrator(val)
self._event = val
self.viewer.event = val
self._event_index = val.count
self._event_id = val.r0.event_id
self.update_event_index_widget()
self.update_event_id_widget()
self._telid = self.viewer.telid
self.update_telid_widget()
self._channel = self.viewer.channel
self.update_channel_widget()
def update_dl1_calibrator(self, extractor=None):
"""
Recreate the dl1 calibrator with the specified extractor and cleaner
Parameters
----------
extractor : ctapipe.image.extractor.ImageExtractor
"""
if extractor is None:
extractor = self.calibrator.image_extractor
self.extractor = extractor
self.calibrator = CameraCalibrator(
subarray=self.reader.subarray,
parent=self,
image_extractor=self.extractor,
)
self.viewer.refresh()
def create_next_event_widget(self):
self.w_next_event = Button(label=">", button_type="default", width=50)
self.w_next_event.on_click(self.on_next_event_widget_click)
def on_next_event_widget_click(self):
self.event_index += 1
def create_previous_event_widget(self):
self.w_previous_event = Button(label="<",
button_type="default",
width=50)
self.w_previous_event.on_click(self.on_previous_event_widget_click)
def on_previous_event_widget_click(self):
self.event_index -= 1
def create_event_index_widget(self):
self.w_event_index = TextInput(title="Event Index:", value='')
def update_event_index_widget(self):
if self.w_event_index:
self.w_event_index.value = str(self.event_index)
def create_event_id_widget(self):
self.w_event_id = TextInput(title="Event ID:", value='')
def update_event_id_widget(self):
if self.w_event_id:
self.w_event_id.value = str(self.event_id)
def create_goto_event_index_widget(self):
self.w_goto_event_index = Button(label="GOTO Index",
button_type="default",
width=100)
self.w_goto_event_index.on_click(self.on_goto_event_index_widget_click)
def on_goto_event_index_widget_click(self):
self.event_index = int(self.w_event_index.value)
def create_goto_event_id_widget(self):
self.w_goto_event_id = Button(label="GOTO ID",
button_type="default",
width=70)
self.w_goto_event_id.on_click(self.on_goto_event_id_widget_click)
def on_goto_event_id_widget_click(self):
self.event_id = int(self.w_event_id.value)
def create_telid_widget(self):
self.w_telid = Select(title="Telescope:", value="", options=[])
self.w_telid.on_change('value', self.on_telid_widget_change)
def update_telid_widget(self):
if self.w_telid:
tels = [str(t) for t in self.event.r0.tels_with_data]
self.w_telid.options = tels
self.w_telid.value = str(self.telid)
def on_telid_widget_change(self, _, __, ___):
if self.telid != int(self.w_telid.value):
self.telid = int(self.w_telid.value)
def create_channel_widget(self):
self.w_channel = Select(title="Channel:", value="", options=[])
self.w_channel.on_change('value', self.on_channel_widget_change)
def update_channel_widget(self):
if self.w_channel:
try:
n_chan = self.event.r0.tel[self.telid].waveform.shape[0]
except AttributeError:
n_chan = 1
channels = [str(c) for c in range(n_chan)]
self.w_channel.options = channels
self.w_channel.value = str(self.channel)
def on_channel_widget_change(self, _, __, ___):
if self.channel != int(self.w_channel.value):
self.channel = int(self.w_channel.value)
def create_dl1_widgets(self):
self.w_dl1_dict = dict(
extractor=Select(title="Extractor:",
value='',
width=5,
options=BokehFileViewer.extractor_product.values),
extractor_window_start=TextInput(title="Window Start:", value=''),
extractor_window_width=TextInput(title="Window Width:", value=''),
extractor_window_shift=TextInput(title="Window Shift:", value=''),
extractor_lwt=TextInput(title="Local Pixel Weight:", value=''))
for val in self.w_dl1_dict.values():
val.on_change('value', self.on_dl1_widget_change)
self.wb_extractor = widgetbox(
PreText(text="Charge Extractor Configuration"),
self.w_dl1_dict['extractor'],
self.w_dl1_dict['extractor_window_start'],
self.w_dl1_dict['extractor_window_width'],
self.w_dl1_dict['extractor_window_shift'],
self.w_dl1_dict['extractor_lwt'])
def update_dl1_widget_values(self):
if self.w_dl1_dict:
for key, val in self.w_dl1_dict.items():
if 'extractor' in key:
if key == 'extractor':
val.value = self.extractor.__class__.__name__
else:
key = key.replace("extractor_", "")
try:
val.value = str(getattr(self.extractor, key))
except AttributeError:
val.value = ''
def on_dl1_widget_change(self, _, __, ___):
if self.event:
if not self._updating_dl1:
self._updating_dl1 = True
cmdline = []
for key, val in self.w_dl1_dict.items():
k = key.replace("extractor_", "ImageExtractor.")
if val.value:
cmdline.append(f'--{k}={val.value}')
self.parse_command_line(cmdline)
extractor = ImageExtractor.from_name(self.extractor_product,
parent=self)
self.update_dl1_calibrator(extractor)
self.update_dl1_widget_values()
self._updating_dl1 = False
class CalibrationCalculator(Component):
"""
Parent class for the camera calibration calculators.
Fills the MonitoringCameraContainer on the base of calibration events
Parameters
----------
flatfield_calculator: lstchain.calib.camera.flatfield
The flatfield to use. If None, then FlatFieldCalculator
will be used by default.
pedestal_calculator: lstchain.calib.camera.pedestal
The pedestal to use. If None, then
PedestalCalculator will be used by default.
kwargs
"""
systematic_correction_path = Path(
default_value=None,
allow_none=True,
exists=True,
directory_ok=False,
help='Path to systematic correction file ',
).tag(config=True)
squared_excess_noise_factor = Float(
1.222,
help='Excess noise factor squared: 1+ Var(gain)/Mean(Gain)**2').tag(
config=True)
pedestal_product = traits.create_class_enum_trait(
PedestalCalculator, default_value='PedestalIntegrator')
flatfield_product = traits.create_class_enum_trait(
FlatFieldCalculator, default_value='FlasherFlatFieldCalculator')
classes = ([FlatFieldCalculator, PedestalCalculator] +
traits.classes_with_traits(FlatFieldCalculator) +
traits.classes_with_traits(PedestalCalculator))
def __init__(self, subarray, parent=None, config=None, **kwargs):
"""
Parent class for the camera calibration calculators.
Fills the MonitoringCameraContainer on the base of calibration events
Parameters
----------
flatfield_calculator: lstchain.calib.camera.flatfield
The flatfield to use. If None, then FlatFieldCalculator
will be used by default.
pedestal_calculator: lstchain.calib.camera.pedestal
The pedestal to use. If None, then
PedestalCalculator will be used by default.
"""
super().__init__(parent=parent, config=config, **kwargs)
if self.squared_excess_noise_factor <= 0:
msg = "Argument squared_excess_noise_factor must have a positive value"
raise ValueError(msg)
self.flatfield = FlatFieldCalculator.from_name(self.flatfield_product,
parent=self,
subarray=subarray)
self.pedestal = PedestalCalculator.from_name(self.pedestal_product,
parent=self,
subarray=subarray)
msg = "tel_id not the same for all calibration components"
if self.pedestal.tel_id != self.flatfield.tel_id:
raise ValueError(msg)
self.tel_id = self.flatfield.tel_id
# load systematic correction term B
self.quadratic_term = 0
if self.systematic_correction_path is not None:
try:
with h5py.File(self.systematic_correction_path, 'r') as hf:
self.quadratic_term = np.array(hf['B_term'])
except:
raise IOError(
f"Problem in reading quadratic term file {self.systematic_correction_path}"
)
self.log.debug(f"{self.pedestal}")
self.log.debug(f"{self.flatfield}")
class PedestalHDF5Writer(Tool):
'''
Example of tool that extract the pedestal value per pixel and write the pedestal
container to disk in a hdf5 file
'''
name = "PedestalHDF5Writer"
description = "Generate a HDF5 file with pedestal values"
output_file = Unicode(
'pedestal.hdf5',
help='Name of the output file'
).tag(config=True)
calculator_product = traits.enum_trait(
PedestalCalculator,
default='PedestalIntegrator'
)
r0calibrator_product = traits.enum_trait(
CameraR0Calibrator,
default='NullR0Calibrator'
)
aliases = Dict(dict(
input_file='EventSource.input_url',
max_events='EventSource.max_events',
tel_id='PedestalCalculator.tel_id',
sample_duration='PedestalCalculator.sample_duration',
sample_size='PedestalCalculator.sample_size',
n_channels='PedestalCalculator.n_channels',
charge_product = 'PedestalCalculator.charge_product'
))
classes = List([EventSource,
PedestalCalculator,
PedestalContainer,
CameraR0Calibrator,
HDF5TableWriter
] + traits.classes_with_traits(PedestalCalculator)
+ traits.classes_with_traits(CameraR0Calibrator))
def __init__(self, **kwargs):
'''
Example of tool that extract the pedestal value per pixel and write the pedestal
container to disk
'''
super().__init__(**kwargs)
self.eventsource = None
self.pedestal = None
self.container = None
self.writer = None
self.group_name = None
self.r0calibrator = None
def setup(self):
kwargs = dict(parent=self)
self.eventsource = EventSource.from_config(**kwargs)
self.pedestal = PedestalCalculator.from_name(
self.calculator_product,
**kwargs
)
self.r0calibrator = CameraR0Calibrator.from_name(
self.r0calibrator_product,
**kwargs
)
self.group_name = 'tel_' + str(self.pedestal.tel_id)
self.writer = HDF5TableWriter(
filename=self.output_file, group_name=self.group_name, overwrite=True
)
def start(self):
'''
Example of tool that extract the pedestal value per pixel and write the pedestal
container to disk
'''
write_config = True
# loop on events
for count, event in enumerate(self.eventsource):
# select only pedestal events
if event.r0.tel[self.pedestal.tel_id].trigger_type != 32:
continue
# perform R0->R1
self.r0calibrator.calibrate(event)
# fill pedestal monitoring container
if self.pedestal.calculate_pedestals(event):
ped_data = event.mon.tel[self.pedestal.tel_id].pedestal
self.log.debug(f" r0 {event.r0.tel[0].waveform.shape}")
self.log.debug(f" r1 {event.r1.tel[0].waveform.shape}")
if write_config:
ped_data.meta['config']= self.config
write_config = False
self.log.debug(f"write event in table: /{self.group_name}/pedestal")
# write data to file
self.writer.write('pedestal', ped_data)
def finish(self):
Provenance().add_output_file(
self.output_file,
role='mon.tel.pedestal'
)
self.writer.close()
class CalibrationHDF5Writer(Tool):
"""
Tool that generates a HDF5 file with camera calibration coefficients.
This is just an example on how the monitoring containers can be filled using
the calibration Components in calib/camera.
This example is based on an input file with interleaved pedestal and flat-field events
For getting help run:
python calc_camera_calibration.py --help
"""
name = "CalibrationHDF5Writer"
description = "Generate a HDF5 file with camera calibration coefficients"
output_file = Unicode('calibration.hdf5',
help='Name of the output file').tag(config=True)
minimum_charge = Float(
800,
help='Temporary cut on charge to eliminate events without led signal'
).tag(config=True)
pedestal_product = traits.enum_trait(PedestalCalculator,
default='PedestalIntegrator')
flatfield_product = traits.enum_trait(FlatFieldCalculator,
default='FlasherFlatFieldCalculator')
aliases = Dict(
dict(
input_file='EventSource.input_url',
max_events='EventSource.max_events',
flatfield_product='CalibrationHDF5Writer.flatfield_product',
pedestal_product='CalibrationHDF5Writer.pedestal_product',
))
classes = List([
EventSource, FlatFieldCalculator, FlatFieldContainer,
PedestalCalculator, PedestalContainer, WaveformCalibrationContainer
] + traits.classes_with_traits(ImageExtractor) +
traits.classes_with_traits(FlatFieldCalculator) +
traits.classes_with_traits(PedestalCalculator))
def __init__(self, **kwargs):
super().__init__(**kwargs)
"""
Tool that generates a HDF5 file with camera calibration coefficients.
Input file must contain interleaved pedestal and flat-field events
For getting help run:
python calc_camera_calibration.py --help
"""
self.eventsource = None
self.flatfield = None
self.pedestal = None
self.container = None
self.writer = None
self.tel_id = None
def setup(self):
kwargs = dict(parent=self)
self.eventsource = EventSource.from_config(**kwargs)
self.flatfield = FlatFieldCalculator.from_name(self.flatfield_product,
**kwargs)
self.pedestal = PedestalCalculator.from_name(self.pedestal_product,
**kwargs)
msg = "tel_id not the same for all calibration components"
assert self.pedestal.tel_id == self.flatfield.tel_id, msg
self.tel_id = self.flatfield.tel_id
group_name = 'tel_' + str(self.tel_id)
self.writer = HDF5TableWriter(filename=self.output_file,
group_name=group_name,
overwrite=True)
def start(self):
'''Calibration coefficient calculator'''
ped_initialized = False
ff_initialized = False
for count, event in enumerate(self.eventsource):
# get link to monitoring containers
if count == 0:
ped_data = event.mon.tel[self.tel_id].pedestal
ff_data = event.mon.tel[self.tel_id].flatfield
status_data = event.mon.tel[self.tel_id].pixel_status
calib_data = event.mon.tel[self.tel_id].calibration
ids = event.nectarcam.tel[self.tel_id].svc.pixel_ids
# if pedestal
if event.r1.tel[self.tel_id].trigger_type == 32:
if self.pedestal.calculate_pedestals(event):
self.log.debug(
f"new pedestal at event n. {count+1}, id {event.r0.event_id}"
)
# update pedestal mask
status_data.pedestal_failing_pixels = np.logical_or(
ped_data.charge_median_outliers,
ped_data.charge_std_outliers)
if not ped_initialized:
# save the config, to be retrieved as data.meta['config']
ped_data.meta['config'] = self.config
ped_initialized = True
else:
self.log.debug(f"write pedestal data")
# write only after a first event (used to initialize the mask)
self.writer.write('pedestal', ped_data)
# consider flat field events only after first pedestal event
elif (event.r1.tel[self.tel_id].trigger_type == 5
or event.r1.tel[self.tel_id].trigger_type
== 4) and (ped_initialized and np.median(
np.sum(event.r1.tel[self.tel_id].waveform[0, ids],
axis=1)) > self.minimum_charge):
if self.flatfield.calculate_relative_gain(event):
self.log.debug(
f"new flatfield at event n. {count+1}, id {event.r0.event_id}"
)
# update the flatfield mask
status_data.flatfield_failing_pixels = np.logical_or(
ff_data.charge_median_outliers,
ff_data.time_median_outliers)
# mask from pedestal and flat-fleid data
monitoring_unusable_pixels = np.logical_or(
status_data.pedestal_failing_pixels,
status_data.flatfield_failing_pixels)
# calibration unusable pixels are an OR of all maskes
calib_data.unusable_pixels = np.logical_or(
monitoring_unusable_pixels,
status_data.hardware_failing_pixels)
# Extract calibration coefficients with F-factor method
# Assume fix F2 factor, F2=1+Var(gain)/Mean(Gain)**2 must be known from elsewhere
F2 = 1.1
# calculate photon-electrons
n_pe = F2 * (ff_data.charge_median - ped_data.charge_median
)**2 / (ff_data.charge_std**2 -
ped_data.charge_std**2)
# fill WaveformCalibrationContainer (this is an example)
calib_data.time = ff_data.sample_time
calib_data.time_range = ff_data.sample_time_range
calib_data.n_pe = n_pe
calib_data.dc_to_pe = n_pe / ff_data.charge_median
calib_data.time_correction = -ff_data.relative_time_median
ped_extractor_name = self.config.get(
"PedestalCalculator").get("charge_product")
ped_width = self.config.get(ped_extractor_name).get(
"window_width")
calib_data.pedestal_per_sample = ped_data.charge_median / ped_width
# save the config, to be retrieved as data.meta['config']
if not ff_initialized:
calib_data.meta['config'] = self.config
ff_initialized = True
else:
# write only after a first event (used to initialize the mask)
self.log.debug(f"write flatfield data")
self.writer.write('flatfield', ff_data)
self.log.debug(f"write pixel_status data")
self.writer.write('pixel_status', status_data)
self.log.debug(f"write calibration data")
self.writer.write('calibration', calib_data)
def finish(self):
Provenance().add_output_file(self.output_file,
role='mon.tel.calibration')
self.writer.close()
def test_enum_classes_with_traits():
""" test that we can get a list of classes that have traits """
list_of_classes = classes_with_traits(ImageExtractor)
assert list_of_classes # should not be empty
class ChargeResolutionGenerator(Tool):
name = "ChargeResolutionGenerator"
description = ("Calculate the Charge Resolution from a sim_telarray "
"simulation and store within a HDF5 file.")
telescopes = List(
Int(),
None,
allow_none=True,
help=
"Telescopes to include from the event file. Default = All telescopes",
).tag(config=True)
output_path = Unicode(
"charge_resolution.h5",
help="Path to store the output HDF5 file").tag(config=True)
extractor_product = traits.enum_trait(ImageExtractor,
default="NeighborPeakWindowSum")
aliases = Dict(
dict(
f="SimTelEventSource.input_url",
max_events="SimTelEventSource.max_events",
T="SimTelEventSource.allowed_tels",
extractor="ChargeResolutionGenerator.extractor_product",
O="ChargeResolutionGenerator.output_path",
))
classes = List([SimTelEventSource] +
traits.classes_with_traits(ImageExtractor))
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.eventsource = None
self.calibrator = None
self.calculator = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
self.eventsource = self.add_component(SimTelEventSource(parent=self))
extractor = self.add_component(
ImageExtractor.from_name(
self.extractor_product,
parent=self,
subarray=self.eventsource.subarray,
))
self.calibrator = self.add_component(
CameraCalibrator(
parent=self,
image_extractor=extractor,
subarray=self.eventsource.subarray,
))
self.calculator = ChargeResolutionCalculator()
def start(self):
desc = "Extracting Charge Resolution"
for event in tqdm(self.eventsource, desc=desc):
self.calibrator(event)
# Check events have true charge included
if event.count == 0:
try:
pe = list(event.mc.tel.values())[0].photo_electron_image
if np.all(pe == 0):
raise KeyError
except KeyError:
self.log.exception("Source does not contain true charge!")
raise
for mc, dl1 in zip(event.mc.tel.values(), event.dl1.tel.values()):
true_charge = mc.photo_electron_image
measured_charge = dl1.image
pixels = np.arange(measured_charge.size)
self.calculator.add(pixels, true_charge, measured_charge)
def finish(self):
df_p, df_c = self.calculator.finish()
output_directory = os.path.dirname(self.output_path)
if not os.path.exists(output_directory):
self.log.info(f"Creating directory: {output_directory}")
os.makedirs(output_directory)
with pd.HDFStore(self.output_path, "w") as store:
store["charge_resolution_pixel"] = df_p
store["charge_resolution_camera"] = df_c
self.log.info("Created charge resolution file: {}".format(
self.output_path))
Provenance().add_output_file(self.output_path)
class DisplayIntegrator(Tool):
name = "ctapipe-display-integration"
description = __doc__
event_index = Int(0, help="Event index to view.").tag(config=True)
use_event_id = Bool(
False,
help=
"event_index will obtain an event using event_id instead of index.",
).tag(config=True)
telescope = Int(
None,
allow_none=True,
help="Telescope to view. Set to None to display the first"
"telescope with data.",
).tag(config=True)
channel = Enum([0, 1], 0, help="Channel to view").tag(config=True)
extractor_product = traits.enum_trait(ImageExtractor,
default="NeighborPeakWindowSum")
aliases = Dict(
dict(
f="EventSource.input_url",
max_events="EventSource.max_events",
extractor="DisplayIntegrator.extractor_product",
E="DisplayIntegrator.event_index",
T="DisplayIntegrator.telescope",
C="DisplayIntegrator.channel",
))
flags = Dict(
dict(id=(
{
"DisplayDL1Calib": {
"use_event_index": True
}
},
"event_index will obtain an event using event_id instead of index.",
)))
classes = List([EventSource] + traits.classes_with_traits(ImageExtractor))
def __init__(self, **kwargs):
super().__init__(**kwargs)
# make sure gzip files are seekable
self.config.SimTelEventSource.back_seekable = True
self.eventseeker = None
self.extractor = None
self.calibrator = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
event_source = self.add_component(EventSource.from_config(parent=self))
self.eventseeker = self.add_component(
EventSeeker(event_source, parent=self))
self.extractor = self.add_component(
ImageExtractor.from_name(self.extractor_product, parent=self))
self.calibrate = self.add_component(
CameraCalibrator(parent=self, image_extractor=self.extractor))
def start(self):
event_num = self.event_index
if self.use_event_id:
event_num = str(event_num)
event = self.eventseeker[event_num]
# Calibrate
self.calibrate(event)
# Select telescope
tels = list(event.r0.tels_with_data)
telid = self.telescope
if telid is None:
telid = tels[0]
if telid not in tels:
self.log.error("[event] please specify one of the following "
"telescopes for this event: {}".format(tels))
exit()
extractor_name = self.extractor.__class__.__name__
plot(event, telid, self.channel, extractor_name)
def finish(self):
pass
class DisplayIntegrator(Tool):
name = "ctapipe-display-integration"
description = __doc__
event_index = Int(0, help="Event index to view.").tag(config=True)
use_event_id = Bool(
False, help="event_index will obtain an event using event_id instead of index."
).tag(config=True)
telescope = Int(
None,
allow_none=True,
help="Telescope to view. Set to None to display the first"
"telescope with data.",
).tag(config=True)
channel = Enum([0, 1], 0, help="Channel to view").tag(config=True)
aliases = Dict(
dict(
f="EventSource.input_url",
max_events="EventSource.max_events",
E="DisplayIntegrator.event_index",
T="DisplayIntegrator.telescope",
C="DisplayIntegrator.channel",
)
)
flags = Dict(
dict(
id=(
{"DisplayDL1Calib": {"use_event_index": True}},
"event_index will obtain an event using event_id instead of index.",
)
)
)
classes = [EventSource] + traits.classes_with_traits(ImageExtractor)
def __init__(self, **kwargs):
super().__init__(**kwargs)
# make sure gzip files are seekable
self.config.SimTelEventSource.back_seekable = True
self.eventseeker = None
self.calibrator = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
event_source = EventSource(parent=self)
self.subarray = event_source.subarray
self.eventseeker = EventSeeker(event_source, parent=self)
self.calibrate = CameraCalibrator(parent=self, subarray=self.subarray)
def start(self):
if self.use_event_id:
event = self.eventseeker.get_event_id(self.event_index)
else:
event = self.eventseeker.get_event_index(self.event_index)
# Calibrate
self.calibrate(event)
# Select telescope
tels = list(event.r0.tel.keys())
telid = self.telescope
if telid is None:
telid = tels[0]
if telid not in tels:
self.log.error(
"[event] please specify one of the following "
"telescopes for this event: {}".format(tels)
)
exit()
extractor_name = self.calibrate.image_extractor_type.tel[telid]
plot(self.subarray, event, telid, self.channel, extractor_name)
def finish(self):
pass
class CalibrationHDF5Writer(Tool):
"""
Tool that generates a HDF5 file with camera calibration coefficients.
This is just an example on how the monitoring containers can be filled using
the calibration Components in calib/camera.
This example is based on an input file with pedestal and flat-field events
For getting help run:
python calc_camera_calibration.py --help
"""
name = "CalibrationHDF5Writer"
description = "Generate a HDF5 file with camera calibration coefficients"
one_event = Bool(
False, help='Stop after first calibration event').tag(config=True)
output_file = Unicode('calibration.hdf5',
help='Name of the output file').tag(config=True)
log_file = Unicode('None', help='Name of the log file').tag(config=True)
calibration_product = traits.create_class_enum_trait(
CalibrationCalculator, default_value='LSTCalibrationCalculator')
r0calibrator_product = traits.create_class_enum_trait(
CameraR0Calibrator, default_value='NullR0Calibrator')
aliases = Dict(
dict(
input_file='EventSource.input_url',
output_file='CalibrationHDF5Writer.output_file',
log_file='CalibrationHDF5Writer.log_file',
max_events='EventSource.max_events',
pedestal_file='LSTR0Corrections.pedestal_path',
calibration_product='CalibrationHDF5Writer.calibration_product',
r0calibrator_product='CalibrationHDF5Writer.r0calibrator_product',
))
classes = List([EventSource, CalibrationCalculator] +
traits.classes_with_traits(CameraR0Calibrator) +
traits.classes_with_traits(CalibrationCalculator))
def __init__(self, **kwargs):
super().__init__(**kwargs)
"""
Tool that generates a HDF5 file with camera calibration coefficients.
Input file must contain interleaved pedestal and flat-field events
For getting help run:
python calc_camera_calibration.py --help
"""
self.eventsource = None
self.processor = None
self.container = None
self.writer = None
self.r0calibrator = None
self.tot_events = 0
self.simulation = False
def setup(self):
self.log.debug(f"Open file")
self.eventsource = EventSource.from_config(parent=self)
# if data remember how many event in the files
if "LSTEventSource" in str(type(self.eventsource)):
self.tot_events = len(self.eventsource.multi_file)
self.log.debug(f"Input file has file {self.tot_events} events")
else:
self.tot_events = self.eventsource.max_events
self.simulation = True
self.processor = CalibrationCalculator.from_name(
self.calibration_product,
parent=self,
subarray=self.eventsource.subarray)
if self.r0calibrator_product:
self.r0calibrator = CameraR0Calibrator.from_name(
self.r0calibrator_product, parent=self)
group_name = 'tel_' + str(self.processor.tel_id)
self.log.debug(f"Open output file {self.output_file}")
self.writer = HDF5TableWriter(filename=self.output_file,
group_name=group_name,
overwrite=True)
def start(self):
'''Calibration coefficient calculator'''
tel_id = self.processor.tel_id
new_ped = False
new_ff = False
end_of_file = False
# skip the first events which are badly drs4 corrected
events_to_skip = 1000
try:
self.log.debug(f"Start loop")
for count, event in enumerate(self.eventsource):
if count % 100 == 0:
self.log.debug(f"Event {count}")
# if last event write results
max_events_reached = (self.eventsource.max_events is not None
and count
== self.eventsource.max_events - 1)
if count == self.tot_events - 1 or max_events_reached:
self.log.debug(f"Last event, count = {count}")
end_of_file = True
# save the config, to be retrieved as data.meta['config']
if count == 0:
if self.simulation:
initialize_pixel_status(
event.mon.tel[tel_id],
event.r1.tel[tel_id].waveform.shape)
ped_data = event.mon.tel[tel_id].pedestal
ped_data.meta['config'] = self.config
ff_data = event.mon.tel[tel_id].flatfield
ff_data.meta['config'] = self.config
status_data = event.mon.tel[tel_id].pixel_status
status_data.meta['config'] = self.config
calib_data = event.mon.tel[tel_id].calibration
calib_data.meta['config'] = self.config
# correct for low level calibration
self.r0calibrator.calibrate(event)
# skip first events which are badly drs4 corrected
if not self.simulation and count < events_to_skip:
continue
# reject event without trigger type
if LSTEventType.is_unknown(event.r1.tel[tel_id].trigger_type):
continue
# if pedestal event
if LSTEventType.is_pedestal(
event.r1.tel[tel_id].trigger_type
) or (self.simulation and np.median(
np.sum(event.r1.tel[tel_id].waveform[0], axis=1)) <
self.processor.minimum_hg_charge_median):
new_ped = self.processor.pedestal.calculate_pedestals(
event)
# if flat-field event: no calibration TIB for the moment,
# use a cut on the charge for ff events and on std for rejecting Magic Lidar events
elif LSTEventType.is_calibration(
event.r1.tel[tel_id].trigger_type
) or (np.median(
np.sum(event.r1.tel[tel_id].waveform[0], axis=1)) >
self.processor.minimum_hg_charge_median and
np.std(np.sum(event.r1.tel[tel_id].waveform[1], axis=1))
< self.processor.maximum_lg_charge_std):
new_ff = self.processor.flatfield.calculate_relative_gain(
event)
# write pedestal results when enough statistics or end of file
if new_ped or end_of_file:
# update the monitoring container with the last statistics
if end_of_file:
self.processor.pedestal.store_results(event)
# write the event
self.log.debug(
f"Write pedestal data at event n. {count+1}, id {event.index.event_id} "
f"stat = {ped_data.n_events} events")
# write on file
self.writer.write('pedestal', ped_data)
new_ped = False
# write flatfield results when enough statistics (also for pedestals) or end of file
if (new_ff and ped_data.n_events > 0) or end_of_file:
# update the monitoring container with the last statistics
if end_of_file:
self.processor.flatfield.store_results(event)
self.log.debug(
f"Write flatfield data at event n. {count+1}, id {event.index.event_id} "
f"stat = {ff_data.n_events} events")
# write on file
self.writer.write('flatfield', ff_data)
new_ff = False
# Then, calculate calibration coefficients
self.processor.calculate_calibration_coefficients(event)
# write calib and pixel status
self.log.debug(f"Write pixel_status data")
self.writer.write('pixel_status', status_data)
self.log.debug(f"Write calibration data")
self.writer.write('calibration', calib_data)
if self.one_event:
break
#self.writer.write('mon', event.mon.tel[tel_id])
except ValueError as e:
self.log.error(e)
def finish(self):
Provenance().add_output_file(self.output_file,
role='mon.tel.calibration')
self.writer.close()
class CalibrationHDF5Writer(Tool):
"""
Tool that generates a HDF5 file with camera calibration coefficients.
This is just an example on how the monitoring containers can be filled using
the calibration Components in calib/camera.
This example is based on an input file with interleaved pedestal and flat-field events
For getting help run:
python calc_camera_calibration.py --help
"""
name = "CalibrationHDF5Writer"
description = "Generate a HDF5 file with camera calibration coefficients"
minimum_charge = Float(
800,
help='Temporary cut on charge till the calibox TIB do not work').tag(
config=True)
one_event = Bool(
False, help='Stop after first calibration event').tag(config=True)
output_file = Unicode('calibration.hdf5',
help='Name of the output file').tag(config=True)
log_file = Unicode('None', help='Name of the log file').tag(config=True)
pedestal_product = traits.enum_trait(PedestalCalculator,
default='PedestalIntegrator')
flatfield_product = traits.enum_trait(FlatFieldCalculator,
default='FlasherFlatFieldCalculator')
r0calibrator_product = traits.enum_trait(CameraR0Calibrator,
default='NullR0Calibrator')
aliases = Dict(
dict(
input_file='EventSource.input_url',
output_file='CalibrationHDF5Writer.output_file',
log_file='CalibrationHDF5Writer.log_file',
max_events='EventSource.max_events',
pedestal_file='LSTR0Corrections.pedestal_path',
flatfield_product='CalibrationHDF5Writer.flatfield_product',
pedestal_product='CalibrationHDF5Writer.pedestal_product',
r0calibrator_product='CalibrationHDF5Writer.r0calibrator_product',
))
classes = List([EventSource, FlatFieldCalculator, PedestalCalculator] +
traits.classes_with_traits(CameraR0Calibrator) +
traits.classes_with_traits(FlatFieldCalculator) +
traits.classes_with_traits(PedestalCalculator))
def __init__(self, **kwargs):
super().__init__(**kwargs)
"""
Tool that generates a HDF5 file with camera calibration coefficients.
Input file must contain interleaved pedestal and flat-field events
For getting help run:
python calc_camera_calibration.py --help
"""
self.eventsource = None
self.flatfield = None
self.pedestal = None
self.container = None
self.writer = None
self.r0calibrator = None
self.tel_id = None
self.tot_events = 0
def setup(self):
kwargs = dict(parent=self)
self.eventsource = EventSource.from_config(**kwargs)
# remember how many event in the files
self.tot_events = len(self.eventsource.multi_file)
self.log.debug(f"Input file has file {self.tot_events} events")
self.flatfield = FlatFieldCalculator.from_name(self.flatfield_product,
**kwargs)
self.pedestal = PedestalCalculator.from_name(self.pedestal_product,
**kwargs)
if self.r0calibrator_product:
self.r0calibrator = CameraR0Calibrator.from_name(
self.r0calibrator_product, **kwargs)
msg = "tel_id not the same for all calibration components"
assert self.r0calibrator.tel_id == self.pedestal.tel_id == self.flatfield.tel_id, msg
self.tel_id = self.flatfield.tel_id
group_name = 'tel_' + str(self.tel_id)
self.log.debug(f"Open output file {self.output_file}")
self.writer = HDF5TableWriter(filename=self.output_file,
group_name=group_name,
overwrite=True)
def start(self):
'''Calibration coefficient calculator'''
new_ped = False
new_ff = False
end_of_file = False
try:
self.log.debug(f"Start loop")
for count, event in enumerate(self.eventsource):
if count % 1000 == 0:
self.log.debug(f"Event {count}")
# if last event write results
if count == self.tot_events - 1 or count == self.eventsource.max_events - 1:
self.log.debug(f"Last event, count = {count}")
end_of_file = True
# save the config, to be retrieved as data.meta['config']
if count == 0:
ped_data = event.mon.tel[self.tel_id].pedestal
ped_data.meta['config'] = self.config
ff_data = event.mon.tel[self.tel_id].flatfield
ff_data.meta['config'] = self.config
status_data = event.mon.tel[self.tel_id].pixel_status
status_data.meta['config'] = self.config
calib_data = event.mon.tel[self.tel_id].calibration
calib_data.meta['config'] = self.config
# correct for low level calibration
self.r0calibrator.calibrate(event)
# if pedestal event
if event.r1.tel[self.tel_id].trigger_type == 32:
new_ped = self.pedestal.calculate_pedestals(event)
# if flat-field event: no calibration TIB for the moment, use a cut on the charge for ff events
elif event.r1.tel[self.tel_id].trigger_type == 4 or np.median(
np.sum(event.r1.tel[self.tel_id].waveform[0],
axis=1)) > self.minimum_charge:
new_ff = self.flatfield.calculate_relative_gain(event)
# write pedestal results when enough statistics or end of file
if new_ped or end_of_file:
# update the monitoring container with the last statistics
if end_of_file:
self.pedestal.store_results(event)
# write the event
self.log.debug(
f"Write pedestal data at event n. {count+1}, id {event.r0.event_id} "
f"stat = {ped_data.n_events} events")
# write on file
self.writer.write('pedestal', ped_data)
new_ped = False
# write flatfield results when enough statistics (also for pedestals) or end of file
if (new_ff and ped_data.n_events > 0) or end_of_file:
# update the monitoring container with the last statistics
if end_of_file:
self.flatfield.store_results(event)
self.log.debug(
f"Write flatfield data at event n. {count+1}, id {event.r0.event_id} "
f"stat = {ff_data.n_events} events")
# write on file
self.writer.write('flatfield', ff_data)
new_ff = False
# Then, calculate calibration coefficients
# mask from pedestal and flat-field data
monitoring_unusable_pixels = np.logical_or(
status_data.pedestal_failing_pixels,
status_data.flatfield_failing_pixels)
# calibration unusable pixels are an OR of all masks
calib_data.unusable_pixels = np.logical_or(
monitoring_unusable_pixels,
status_data.hardware_failing_pixels)
# Extract calibration coefficients with F-factor method
# Assume fix F2 factor, F2=1+Var(gain)/Mean(Gain)**2 must be known from elsewhere
F2 = 1.2
# calculate photon-electrons
n_pe = F2 * (ff_data.charge_median - ped_data.charge_median
)**2 / (ff_data.charge_std**2 -
ped_data.charge_std**2)
# fill WaveformCalibrationContainer (this is an example)
calib_data.time = ff_data.sample_time
calib_data.time_range = ff_data.sample_time_range
calib_data.n_pe = n_pe
calib_data.dc_to_pe = n_pe / (ff_data.charge_median -
ped_data.charge_median)
calib_data.time_correction = -ff_data.relative_time_median
ped_extractor_name = self.config.get(
"PedestalCalculator").get("charge_product")
ped_width = self.config.get(ped_extractor_name).get(
"window_width")
calib_data.pedestal_per_sample = ped_data.charge_median / ped_width
# write calib and pixel status
self.log.debug(f"Write pixel_status data")
self.writer.write('pixel_status', status_data)
self.log.debug(f"Write calibration data")
self.writer.write('calibration', calib_data)
if self.one_event:
break
#self.writer.write('mon', event.mon.tel[self.tel_id])
except ValueError as e:
self.log.error(e)
def finish(self):
Provenance().add_output_file(self.output_file,
role='mon.tel.calibration')
self.writer.close()
class MuonAnalysis(Tool):
"""
Detect and extract muon ring parameters, and write the muon ring and
intensity parameters to an output table.
The resulting output can be read e.g. using for example
`pandas.read_hdf(filename, 'dl1/event/telescope/parameters/muon')`
"""
name = 'ctapipe-reconstruct-muons'
description = traits.Unicode(__doc__)
output = traits.Path(directory_ok=False,
help='HDF5 output file name').tag(config=True)
completeness_threshold = traits.FloatTelescopeParameter(
default_value=30.0,
help='Threshold for calculating the ``ring_completeness``',
).tag(config=True)
ratio_width = traits.FloatTelescopeParameter(
default_value=1.5,
help=('Ring width for intensity ratio'
' computation as multiple of pixel diameter')).tag(config=True)
overwrite = traits.Bool(
default_value=False,
help='If true, overwrite outputfile without asking').tag(config=True)
min_pixels = traits.IntTelescopeParameter(
help=('Minimum number of pixels after cleaning and ring finding'
'required to process an event'),
default_value=100,
).tag(config=True)
pedestal = traits.FloatTelescopeParameter(
help='Pedestal noise rms',
default_value=1.1,
).tag(config=True)
extractor_name = traits.create_class_enum_trait(
ImageExtractor,
default_value='GlobalPeakWindowSum',
).tag(config=True)
classes = [
CameraCalibrator,
TailcutsImageCleaner,
EventSource,
MuonRingFitter,
MuonIntensityFitter,
] + traits.classes_with_traits(ImageExtractor)
aliases = {
'i': 'EventSource.input_url',
'input': 'EventSource.input_url',
'o': 'MuonAnalysis.output',
'output': 'MuonAnalysis.output',
'max-events': 'EventSource.max_events',
'allowed-tels': 'EventSource.allowed_tels',
}
flags = {
'overwrite': ({
'MuonAnalysis': {
'overwrite': True
}
}, 'overwrite output file')
}
def setup(self):
if self.output is None:
raise ToolConfigurationError(
'You need to provide an --output file')
if self.output.exists() and not self.overwrite:
raise ToolConfigurationError(
'Outputfile {self.output} already exists, use `--overwrite` to overwrite'
)
self.source = self.add_component(EventSource.from_config(parent=self))
self.extractor = self.add_component(
ImageExtractor.from_name(self.extractor_name,
parent=self,
subarray=self.source.subarray))
self.calib = self.add_component(
CameraCalibrator(
subarray=self.source.subarray,
parent=self,
image_extractor=self.extractor,
))
self.ring_fitter = self.add_component(MuonRingFitter(parent=self, ))
self.intensity_fitter = self.add_component(
MuonIntensityFitter(
subarray=self.source.subarray,
parent=self,
))
self.cleaning = self.add_component(
TailcutsImageCleaner(
parent=self,
subarray=self.source.subarray,
))
self.writer = self.add_component(
HDF5TableWriter(
self.output,
"",
add_prefix=True,
parent=self,
mode='w',
))
self.pixels_in_tel_frame = {}
self.field_of_view = {}
self.pixel_widths = {}
for p in [
'min_pixels', 'pedestal', 'ratio_width',
'completeness_threshold'
]:
getattr(self, p).attach_subarray(self.source.subarray)
def start(self):
for event in tqdm(self.source, desc='Processing events: '):
self.process_array_event(event)
def process_array_event(self, event):
self.calib(event)
for tel_id, dl1 in event.dl1.tel.items():
self.process_telescope_event(event.index, tel_id, dl1)
self.writer.write('sim/event/subarray/shower', [event.index, event.mc])
def process_telescope_event(self, event_index, tel_id, dl1):
event_id = event_index.event_id
if self.source.subarray.tel[tel_id].optics.num_mirrors != 1:
self.log.warn(f'Skipping non-single mirror telescope {tel_id}'
' set --allowed_tels to get rid of this warning')
return
self.log.debug(f'Processing event {event_id}, telescope {tel_id}')
image = dl1.image
clean_mask = self.cleaning(tel_id, image)
if np.count_nonzero(clean_mask) <= self.min_pixels.tel[tel_id]:
self.log.debug(
f'Skipping event {event_id}-{tel_id}:'
f' has less then {self.min_pixels.tel[tel_id]} pixels after cleaning'
)
return
x, y = self.get_pixel_coords(tel_id)
# iterative ring fit.
# First use cleaning pixels, then only pixels close to the ring
# three iterations seems to be enough for most rings
mask = clean_mask
for i in range(3):
ring = self.ring_fitter(x, y, image, mask)
dist = np.sqrt((x - ring.center_x)**2 + (y - ring.center_y)**2)
mask = np.abs(dist - ring.radius) / ring.radius < 0.4
if np.count_nonzero(mask) <= self.min_pixels.tel[tel_id]:
self.log.debug(
f'Skipping event {event_id}-{tel_id}:'
f' Less then {self.min_pixels.tel[tel_id]} pixels on ring')
return
if np.isnan(
[ring.radius.value, ring.center_x.value,
ring.center_y.value]).any():
self.log.debug(
f'Skipping event {event_id}-{tel_id}: Ring fit did not succeed'
)
return
parameters = self.calculate_muon_parameters(tel_id, image, clean_mask,
ring)
# intensity_fitter does not support a mask yet, set ignored pixels to 0
image[~mask] = 0
result = self.intensity_fitter(
tel_id,
ring.center_x,
ring.center_y,
ring.radius,
image,
pedestal=self.pedestal.tel[tel_id],
)
self.log.info(
f'Muon fit: r={ring.radius:.2f}'
f', width={result.width:.4f}'
f', efficiency={result.optical_efficiency:.2%}', )
tel_event_index = TelEventIndexContainer(
**event_index,
tel_id=tel_id,
)
self.writer.write('dl1/event/telescope/parameters/muons',
[tel_event_index, ring, parameters, result])
def calculate_muon_parameters(self, tel_id, image, clean_mask, ring):
fov_radius = self.get_fov(tel_id)
x, y = self.get_pixel_coords(tel_id)
# add ring containment, not filled in fit
containment = ring_containment(
ring.radius,
ring.center_x,
ring.center_y,
fov_radius,
)
completeness = ring_completeness(
x,
y,
image,
ring.radius,
ring.center_x,
ring.center_y,
threshold=self.completeness_threshold.tel[tel_id],
)
pixel_width = self.get_pixel_width(tel_id)
intensity_ratio = intensity_ratio_inside_ring(
x[clean_mask],
y[clean_mask],
image[clean_mask],
ring.radius,
ring.center_x,
ring.center_y,
width=self.ratio_width.tel[tel_id] * pixel_width,
)
mse = mean_squared_error(x[clean_mask], y[clean_mask],
image[clean_mask], ring.radius, ring.center_x,
ring.center_y)
return MuonParametersContainer(
containment=containment,
completeness=completeness,
intensity_ratio=intensity_ratio,
mean_squared_error=mse,
)
def get_fov(self, tel_id):
'''Guesstimate fov radius for telescope with id `tel_id`'''
# memoize fov calculation
if tel_id not in self.field_of_view:
cam = self.source.subarray.tel[tel_id].camera.geometry
border = cam.get_border_pixel_mask()
x, y = self.get_pixel_coords(tel_id)
self.field_of_view[tel_id] = np.sqrt(x[border]**2 +
y[border]**2).mean()
return self.field_of_view[tel_id]
def get_pixel_width(self, tel_id):
'''Guesstimate fov radius for telescope with id `tel_id`'''
# memoize fov calculation
if tel_id not in self.pixel_widths:
x, y = self.get_pixel_coords(tel_id)
self.pixel_widths[tel_id] = CameraGeometry.guess_pixel_width(x, y)
return self.pixel_widths[tel_id]
def get_pixel_coords(self, tel_id):
'''Get pixel coords in telescope frame for telescope with id `tel_id`'''
# memoize transformation
if tel_id not in self.pixels_in_tel_frame:
telescope = self.source.subarray.tel[tel_id]
cam = telescope.camera.geometry
camera_frame = CameraFrame(
focal_length=telescope.optics.equivalent_focal_length,
rotation=cam.cam_rotation,
)
cam_coords = SkyCoord(x=cam.pix_x, y=cam.pix_y, frame=camera_frame)
tel_coord = cam_coords.transform_to(TelescopeFrame())
self.pixels_in_tel_frame[tel_id] = tel_coord
coords = self.pixels_in_tel_frame[tel_id]
return coords.fov_lon, coords.fov_lat
def finish(self):
Provenance().add_output_file(
self.output,
role='muon_efficiency_parameters',
)
self.writer.close()