class PedestalHDF5Writer(Tool):
'''
Example of tool that extract the pedestal value per pixel and write the pedestal
container to disk in a hdf5 file
'''
name = "PedestalHDF5Writer"
description = "Generate a HDF5 file with pedestal values"
output_file = Unicode(
'pedestal.hdf5',
help='Name of the output file'
).tag(config=True)
calculator_product = traits.enum_trait(
PedestalCalculator,
default='PedestalIntegrator'
)
r0calibrator_product = traits.enum_trait(
CameraR0Calibrator,
default='NullR0Calibrator'
)
aliases = Dict(dict(
input_file='EventSource.input_url',
max_events='EventSource.max_events',
tel_id='PedestalCalculator.tel_id',
sample_duration='PedestalCalculator.sample_duration',
sample_size='PedestalCalculator.sample_size',
n_channels='PedestalCalculator.n_channels',
charge_product = 'PedestalCalculator.charge_product'
))
classes = List([EventSource,
PedestalCalculator,
PedestalContainer,
CameraR0Calibrator,
HDF5TableWriter
] + traits.classes_with_traits(PedestalCalculator)
+ traits.classes_with_traits(CameraR0Calibrator))
def __init__(self, **kwargs):
'''
Example of tool that extract the pedestal value per pixel and write the pedestal
container to disk
'''
super().__init__(**kwargs)
self.eventsource = None
self.pedestal = None
self.container = None
self.writer = None
self.group_name = None
self.r0calibrator = None
def setup(self):
kwargs = dict(parent=self)
self.eventsource = EventSource.from_config(**kwargs)
self.pedestal = PedestalCalculator.from_name(
self.calculator_product,
**kwargs
)
self.r0calibrator = CameraR0Calibrator.from_name(
self.r0calibrator_product,
**kwargs
)
self.group_name = 'tel_' + str(self.pedestal.tel_id)
self.writer = HDF5TableWriter(
filename=self.output_file, group_name=self.group_name, overwrite=True
)
def start(self):
'''
Example of tool that extract the pedestal value per pixel and write the pedestal
container to disk
'''
write_config = True
# loop on events
for count, event in enumerate(self.eventsource):
# select only pedestal events
if event.r0.tel[self.pedestal.tel_id].trigger_type != 32:
continue
# perform R0->R1
self.r0calibrator.calibrate(event)
# fill pedestal monitoring container
if self.pedestal.calculate_pedestals(event):
ped_data = event.mon.tel[self.pedestal.tel_id].pedestal
self.log.debug(f" r0 {event.r0.tel[0].waveform.shape}")
self.log.debug(f" r1 {event.r1.tel[0].waveform.shape}")
if write_config:
ped_data.meta['config']= self.config
write_config = False
self.log.debug(f"write event in table: /{self.group_name}/pedestal")
# write data to file
self.writer.write('pedestal', ped_data)
def finish(self):
Provenance().add_output_file(
self.output_file,
role='mon.tel.pedestal'
)
self.writer.close()
class BokehFileViewer(Tool):
name = "BokehFileViewer"
description = ("Interactively explore an event file using the bokeh "
"visualisation package")
port = Int(5006, help="Port to open bokeh server onto").tag(config=True)
disable_server = Bool(False,
help="Do not start the bokeh server "
"(useful for testing)").tag(config=True)
default_url = get_dataset_path("gamma_test_large.simtel.gz")
EventSource.input_url.default_value = default_url
extractor_product = traits.enum_trait(ImageExtractor,
default='NeighborPeakWindowSum')
aliases = Dict(
dict(
port='BokehFileViewer.port',
disable_server='BokehFileViewer.disable_server',
f='EventSource.input_url',
max_events='EventSource.max_events',
extractor='BokehFileViewer.extractor_product',
))
classes = List([
EventSource,
] + traits.classes_with_traits(ImageExtractor))
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._event = None
self._event_index = None
self._event_id = None
self._telid = None
self._channel = None
self.w_next_event = None
self.w_previous_event = None
self.w_event_index = None
self.w_event_id = None
self.w_goto_event_index = None
self.w_goto_event_id = None
self.w_telid = None
self.w_channel = None
self.w_dl1_dict = None
self.wb_extractor = None
self.layout = None
self.reader = None
self.seeker = None
self.extractor = None
self.calibrator = None
self.viewer = None
self._updating_dl1 = False
# make sure, gzip files are seekable
self.config.SimTelEventSource.back_seekable = True
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
self.reader = EventSource.from_config(parent=self)
self.seeker = EventSeeker(self.reader, parent=self)
self.extractor = ImageExtractor.from_name(
self.extractor_product,
parent=self,
subarray=self.reader.subarray,
)
self.calibrator = CameraCalibrator(
subarray=self.reader.subarray,
parent=self,
image_extractor=self.extractor,
)
self.viewer = BokehEventViewer(parent=self)
# Setup widgets
self.viewer.create()
self.viewer.enable_automatic_index_increment()
self.create_previous_event_widget()
self.create_next_event_widget()
self.create_event_index_widget()
self.create_goto_event_index_widget()
self.create_event_id_widget()
self.create_goto_event_id_widget()
self.create_telid_widget()
self.create_channel_widget()
self.create_dl1_widgets()
self.update_dl1_widget_values()
# Setup layout
self.layout = layout([[self.viewer.layout],
[
self.w_previous_event, self.w_next_event,
self.w_goto_event_index, self.w_goto_event_id
], [self.w_event_index, self.w_event_id],
[self.w_telid, self.w_channel],
[self.wb_extractor]])
def start(self):
self.event_index = 0
def finish(self):
if not self.disable_server:
def modify_doc(doc):
doc.add_root(self.layout)
doc.title = self.name
directory = os.path.abspath(os.path.dirname(__file__))
theme_path = os.path.join(directory, "theme.yaml")
template_path = os.path.join(directory, "templates")
doc.theme = Theme(filename=theme_path)
env = jinja2.Environment(
loader=jinja2.FileSystemLoader(template_path))
doc.template = env.get_template('index.html')
self.log.info('Opening Bokeh application on '
'http://localhost:{}/'.format(self.port))
server = Server({'/': modify_doc}, num_procs=1, port=self.port)
server.start()
server.io_loop.add_callback(server.show, "/")
server.io_loop.start()
@property
def event_index(self):
return self._event_index
@event_index.setter
def event_index(self, val):
try:
self.event = self.seeker[val]
except IndexError:
self.log.warning(f"Event Index {val} does not exist")
@property
def event_id(self):
return self._event_id
@event_id.setter
def event_id(self, val):
try:
self.event = self.seeker[str(val)]
except IndexError:
self.log.warning(f"Event ID {val} does not exist")
@property
def telid(self):
return self._telid
@telid.setter
def telid(self, val):
self.channel = 0
tels = list(self.event.r0.tels_with_data)
if val not in tels:
val = tels[0]
self._telid = val
self.viewer.telid = val
self.update_telid_widget()
@property
def channel(self):
return self._channel
@channel.setter
def channel(self, val):
self._channel = val
self.viewer.channel = val
self.update_channel_widget()
@property
def event(self):
return self._event
@event.setter
def event(self, val):
self.calibrator(val)
self._event = val
self.viewer.event = val
self._event_index = val.count
self._event_id = val.r0.event_id
self.update_event_index_widget()
self.update_event_id_widget()
self._telid = self.viewer.telid
self.update_telid_widget()
self._channel = self.viewer.channel
self.update_channel_widget()
def update_dl1_calibrator(self, extractor=None):
"""
Recreate the dl1 calibrator with the specified extractor and cleaner
Parameters
----------
extractor : ctapipe.image.extractor.ImageExtractor
"""
if extractor is None:
extractor = self.calibrator.image_extractor
self.extractor = extractor
self.calibrator = CameraCalibrator(
subarray=self.reader.subarray,
parent=self,
image_extractor=self.extractor,
)
self.viewer.refresh()
def create_next_event_widget(self):
self.w_next_event = Button(label=">", button_type="default", width=50)
self.w_next_event.on_click(self.on_next_event_widget_click)
def on_next_event_widget_click(self):
self.event_index += 1
def create_previous_event_widget(self):
self.w_previous_event = Button(label="<",
button_type="default",
width=50)
self.w_previous_event.on_click(self.on_previous_event_widget_click)
def on_previous_event_widget_click(self):
self.event_index -= 1
def create_event_index_widget(self):
self.w_event_index = TextInput(title="Event Index:", value='')
def update_event_index_widget(self):
if self.w_event_index:
self.w_event_index.value = str(self.event_index)
def create_event_id_widget(self):
self.w_event_id = TextInput(title="Event ID:", value='')
def update_event_id_widget(self):
if self.w_event_id:
self.w_event_id.value = str(self.event_id)
def create_goto_event_index_widget(self):
self.w_goto_event_index = Button(label="GOTO Index",
button_type="default",
width=100)
self.w_goto_event_index.on_click(self.on_goto_event_index_widget_click)
def on_goto_event_index_widget_click(self):
self.event_index = int(self.w_event_index.value)
def create_goto_event_id_widget(self):
self.w_goto_event_id = Button(label="GOTO ID",
button_type="default",
width=70)
self.w_goto_event_id.on_click(self.on_goto_event_id_widget_click)
def on_goto_event_id_widget_click(self):
self.event_id = int(self.w_event_id.value)
def create_telid_widget(self):
self.w_telid = Select(title="Telescope:", value="", options=[])
self.w_telid.on_change('value', self.on_telid_widget_change)
def update_telid_widget(self):
if self.w_telid:
tels = [str(t) for t in self.event.r0.tels_with_data]
self.w_telid.options = tels
self.w_telid.value = str(self.telid)
def on_telid_widget_change(self, _, __, ___):
if self.telid != int(self.w_telid.value):
self.telid = int(self.w_telid.value)
def create_channel_widget(self):
self.w_channel = Select(title="Channel:", value="", options=[])
self.w_channel.on_change('value', self.on_channel_widget_change)
def update_channel_widget(self):
if self.w_channel:
try:
n_chan = self.event.r0.tel[self.telid].waveform.shape[0]
except AttributeError:
n_chan = 1
channels = [str(c) for c in range(n_chan)]
self.w_channel.options = channels
self.w_channel.value = str(self.channel)
def on_channel_widget_change(self, _, __, ___):
if self.channel != int(self.w_channel.value):
self.channel = int(self.w_channel.value)
def create_dl1_widgets(self):
self.w_dl1_dict = dict(
extractor=Select(title="Extractor:",
value='',
width=5,
options=BokehFileViewer.extractor_product.values),
extractor_window_start=TextInput(title="Window Start:", value=''),
extractor_window_width=TextInput(title="Window Width:", value=''),
extractor_window_shift=TextInput(title="Window Shift:", value=''),
extractor_lwt=TextInput(title="Local Pixel Weight:", value=''))
for val in self.w_dl1_dict.values():
val.on_change('value', self.on_dl1_widget_change)
self.wb_extractor = widgetbox(
PreText(text="Charge Extractor Configuration"),
self.w_dl1_dict['extractor'],
self.w_dl1_dict['extractor_window_start'],
self.w_dl1_dict['extractor_window_width'],
self.w_dl1_dict['extractor_window_shift'],
self.w_dl1_dict['extractor_lwt'])
def update_dl1_widget_values(self):
if self.w_dl1_dict:
for key, val in self.w_dl1_dict.items():
if 'extractor' in key:
if key == 'extractor':
val.value = self.extractor.__class__.__name__
else:
key = key.replace("extractor_", "")
try:
val.value = str(getattr(self.extractor, key))
except AttributeError:
val.value = ''
def on_dl1_widget_change(self, _, __, ___):
if self.event:
if not self._updating_dl1:
self._updating_dl1 = True
cmdline = []
for key, val in self.w_dl1_dict.items():
k = key.replace("extractor_", "ImageExtractor.")
if val.value:
cmdline.append(f'--{k}={val.value}')
self.parse_command_line(cmdline)
extractor = ImageExtractor.from_name(self.extractor_product,
parent=self)
self.update_dl1_calibrator(extractor)
self.update_dl1_widget_values()
self._updating_dl1 = False
def test_enum_trait():
""" check that enum traits are constructable from a complex class """
trait = enum_trait(ImageExtractor, default="NeighborPeakWindowSum")
assert isinstance(trait, CaselessStrEnum)
class CalibrationHDF5Writer(Tool):
"""
Tool that generates a HDF5 file with camera calibration coefficients.
This is just an example on how the monitoring containers can be filled using
the calibration Components in calib/camera.
This example is based on an input file with interleaved pedestal and flat-field events
For getting help run:
python calc_camera_calibration.py --help
"""
name = "CalibrationHDF5Writer"
description = "Generate a HDF5 file with camera calibration coefficients"
output_file = Unicode('calibration.hdf5',
help='Name of the output file').tag(config=True)
minimum_charge = Float(
800,
help='Temporary cut on charge to eliminate events without led signal'
).tag(config=True)
pedestal_product = traits.enum_trait(PedestalCalculator,
default='PedestalIntegrator')
flatfield_product = traits.enum_trait(FlatFieldCalculator,
default='FlasherFlatFieldCalculator')
aliases = Dict(
dict(
input_file='EventSource.input_url',
max_events='EventSource.max_events',
flatfield_product='CalibrationHDF5Writer.flatfield_product',
pedestal_product='CalibrationHDF5Writer.pedestal_product',
))
classes = List([
EventSource, FlatFieldCalculator, FlatFieldContainer,
PedestalCalculator, PedestalContainer, WaveformCalibrationContainer
] + traits.classes_with_traits(ImageExtractor) +
traits.classes_with_traits(FlatFieldCalculator) +
traits.classes_with_traits(PedestalCalculator))
def __init__(self, **kwargs):
super().__init__(**kwargs)
"""
Tool that generates a HDF5 file with camera calibration coefficients.
Input file must contain interleaved pedestal and flat-field events
For getting help run:
python calc_camera_calibration.py --help
"""
self.eventsource = None
self.flatfield = None
self.pedestal = None
self.container = None
self.writer = None
self.tel_id = None
def setup(self):
kwargs = dict(parent=self)
self.eventsource = EventSource.from_config(**kwargs)
self.flatfield = FlatFieldCalculator.from_name(self.flatfield_product,
**kwargs)
self.pedestal = PedestalCalculator.from_name(self.pedestal_product,
**kwargs)
msg = "tel_id not the same for all calibration components"
assert self.pedestal.tel_id == self.flatfield.tel_id, msg
self.tel_id = self.flatfield.tel_id
group_name = 'tel_' + str(self.tel_id)
self.writer = HDF5TableWriter(filename=self.output_file,
group_name=group_name,
overwrite=True)
def start(self):
'''Calibration coefficient calculator'''
ped_initialized = False
ff_initialized = False
for count, event in enumerate(self.eventsource):
# get link to monitoring containers
if count == 0:
ped_data = event.mon.tel[self.tel_id].pedestal
ff_data = event.mon.tel[self.tel_id].flatfield
status_data = event.mon.tel[self.tel_id].pixel_status
calib_data = event.mon.tel[self.tel_id].calibration
ids = event.nectarcam.tel[self.tel_id].svc.pixel_ids
# if pedestal
if event.r1.tel[self.tel_id].trigger_type == 32:
if self.pedestal.calculate_pedestals(event):
self.log.debug(
f"new pedestal at event n. {count+1}, id {event.r0.event_id}"
)
# update pedestal mask
status_data.pedestal_failing_pixels = np.logical_or(
ped_data.charge_median_outliers,
ped_data.charge_std_outliers)
if not ped_initialized:
# save the config, to be retrieved as data.meta['config']
ped_data.meta['config'] = self.config
ped_initialized = True
else:
self.log.debug(f"write pedestal data")
# write only after a first event (used to initialize the mask)
self.writer.write('pedestal', ped_data)
# consider flat field events only after first pedestal event
elif (event.r1.tel[self.tel_id].trigger_type == 5
or event.r1.tel[self.tel_id].trigger_type
== 4) and (ped_initialized and np.median(
np.sum(event.r1.tel[self.tel_id].waveform[0, ids],
axis=1)) > self.minimum_charge):
if self.flatfield.calculate_relative_gain(event):
self.log.debug(
f"new flatfield at event n. {count+1}, id {event.r0.event_id}"
)
# update the flatfield mask
status_data.flatfield_failing_pixels = np.logical_or(
ff_data.charge_median_outliers,
ff_data.time_median_outliers)
# mask from pedestal and flat-fleid data
monitoring_unusable_pixels = np.logical_or(
status_data.pedestal_failing_pixels,
status_data.flatfield_failing_pixels)
# calibration unusable pixels are an OR of all maskes
calib_data.unusable_pixels = np.logical_or(
monitoring_unusable_pixels,
status_data.hardware_failing_pixels)
# Extract calibration coefficients with F-factor method
# Assume fix F2 factor, F2=1+Var(gain)/Mean(Gain)**2 must be known from elsewhere
F2 = 1.1
# calculate photon-electrons
n_pe = F2 * (ff_data.charge_median - ped_data.charge_median
)**2 / (ff_data.charge_std**2 -
ped_data.charge_std**2)
# fill WaveformCalibrationContainer (this is an example)
calib_data.time = ff_data.sample_time
calib_data.time_range = ff_data.sample_time_range
calib_data.n_pe = n_pe
calib_data.dc_to_pe = n_pe / ff_data.charge_median
calib_data.time_correction = -ff_data.relative_time_median
ped_extractor_name = self.config.get(
"PedestalCalculator").get("charge_product")
ped_width = self.config.get(ped_extractor_name).get(
"window_width")
calib_data.pedestal_per_sample = ped_data.charge_median / ped_width
# save the config, to be retrieved as data.meta['config']
if not ff_initialized:
calib_data.meta['config'] = self.config
ff_initialized = True
else:
# write only after a first event (used to initialize the mask)
self.log.debug(f"write flatfield data")
self.writer.write('flatfield', ff_data)
self.log.debug(f"write pixel_status data")
self.writer.write('pixel_status', status_data)
self.log.debug(f"write calibration data")
self.writer.write('calibration', calib_data)
def finish(self):
Provenance().add_output_file(self.output_file,
role='mon.tel.calibration')
self.writer.close()
class DisplayDL1Calib(Tool):
name = "ctapipe-display-dl1"
description = __doc__
telescope = Int(
None,
allow_none=True,
help="Telescope to view. Set to None to display all telescopes.",
).tag(config=True)
extractor_product = traits.enum_trait(ImageExtractor,
default="NeighborPeakWindowSum")
aliases = Dict(
dict(
input="EventSource.input_url",
max_events="EventSource.max_events",
extractor="DisplayDL1Calib.extractor_product",
T="DisplayDL1Calib.telescope",
O="ImagePlotter.output_path",
))
flags = Dict(
dict(D=(
{
"ImagePlotter": {
"display": True
}
},
"Display the photo-electron images on-screen as they are produced.",
)))
classes = List([EventSource, ImagePlotter] +
traits.classes_with_traits(ImageExtractor))
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.eventsource = None
self.calibrator = None
self.plotter = None
def setup(self):
self.eventsource = self.add_component(
EventSource.from_url(
get_dataset_path("gamma_test_large.simtel.gz"), parent=self))
self.calibrator = self.add_component(
CameraCalibrator(parent=self, subarray=self.eventsource.subarray))
self.plotter = self.add_component(ImagePlotter(parent=self))
def start(self):
for event in self.eventsource:
self.calibrator(event)
tel_list = event.r0.tels_with_data
if self.telescope:
if self.telescope not in tel_list:
continue
tel_list = [self.telescope]
for telid in tel_list:
self.plotter.plot(event, telid)
def finish(self):
self.plotter.finish()
def test_enum_trait_default_is_right():
""" check default value of enum trait """
with pytest.raises(ValueError):
enum_trait(ImageExtractor, default="name_of_default_choice")
class ChargeResolutionGenerator(Tool):
name = "ChargeResolutionGenerator"
description = ("Calculate the Charge Resolution from a sim_telarray "
"simulation and store within a HDF5 file.")
telescopes = List(
Int(),
None,
allow_none=True,
help=
"Telescopes to include from the event file. Default = All telescopes",
).tag(config=True)
output_path = Unicode(
"charge_resolution.h5",
help="Path to store the output HDF5 file").tag(config=True)
extractor_product = traits.enum_trait(ImageExtractor,
default="NeighborPeakWindowSum")
aliases = Dict(
dict(
f="SimTelEventSource.input_url",
max_events="SimTelEventSource.max_events",
T="SimTelEventSource.allowed_tels",
extractor="ChargeResolutionGenerator.extractor_product",
O="ChargeResolutionGenerator.output_path",
))
classes = List([SimTelEventSource] +
traits.classes_with_traits(ImageExtractor))
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.eventsource = None
self.calibrator = None
self.calculator = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
self.eventsource = self.add_component(SimTelEventSource(parent=self))
extractor = self.add_component(
ImageExtractor.from_name(
self.extractor_product,
parent=self,
subarray=self.eventsource.subarray,
))
self.calibrator = self.add_component(
CameraCalibrator(
parent=self,
image_extractor=extractor,
subarray=self.eventsource.subarray,
))
self.calculator = ChargeResolutionCalculator()
def start(self):
desc = "Extracting Charge Resolution"
for event in tqdm(self.eventsource, desc=desc):
self.calibrator(event)
# Check events have true charge included
if event.count == 0:
try:
pe = list(event.mc.tel.values())[0].photo_electron_image
if np.all(pe == 0):
raise KeyError
except KeyError:
self.log.exception("Source does not contain true charge!")
raise
for mc, dl1 in zip(event.mc.tel.values(), event.dl1.tel.values()):
true_charge = mc.photo_electron_image
measured_charge = dl1.image
pixels = np.arange(measured_charge.size)
self.calculator.add(pixels, true_charge, measured_charge)
def finish(self):
df_p, df_c = self.calculator.finish()
output_directory = os.path.dirname(self.output_path)
if not os.path.exists(output_directory):
self.log.info(f"Creating directory: {output_directory}")
os.makedirs(output_directory)
with pd.HDFStore(self.output_path, "w") as store:
store["charge_resolution_pixel"] = df_p
store["charge_resolution_camera"] = df_c
self.log.info("Created charge resolution file: {}".format(
self.output_path))
Provenance().add_output_file(self.output_path)
class DisplayIntegrator(Tool):
name = "ctapipe-display-integration"
description = __doc__
event_index = Int(0, help="Event index to view.").tag(config=True)
use_event_id = Bool(
False,
help=
"event_index will obtain an event using event_id instead of index.",
).tag(config=True)
telescope = Int(
None,
allow_none=True,
help="Telescope to view. Set to None to display the first"
"telescope with data.",
).tag(config=True)
channel = Enum([0, 1], 0, help="Channel to view").tag(config=True)
extractor_product = traits.enum_trait(ImageExtractor,
default="NeighborPeakWindowSum")
aliases = Dict(
dict(
f="EventSource.input_url",
max_events="EventSource.max_events",
extractor="DisplayIntegrator.extractor_product",
E="DisplayIntegrator.event_index",
T="DisplayIntegrator.telescope",
C="DisplayIntegrator.channel",
))
flags = Dict(
dict(id=(
{
"DisplayDL1Calib": {
"use_event_index": True
}
},
"event_index will obtain an event using event_id instead of index.",
)))
classes = List([EventSource] + traits.classes_with_traits(ImageExtractor))
def __init__(self, **kwargs):
super().__init__(**kwargs)
# make sure gzip files are seekable
self.config.SimTelEventSource.back_seekable = True
self.eventseeker = None
self.extractor = None
self.calibrator = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
event_source = self.add_component(EventSource.from_config(parent=self))
self.eventseeker = self.add_component(
EventSeeker(event_source, parent=self))
self.extractor = self.add_component(
ImageExtractor.from_name(self.extractor_product, parent=self))
self.calibrate = self.add_component(
CameraCalibrator(parent=self, image_extractor=self.extractor))
def start(self):
event_num = self.event_index
if self.use_event_id:
event_num = str(event_num)
event = self.eventseeker[event_num]
# Calibrate
self.calibrate(event)
# Select telescope
tels = list(event.r0.tels_with_data)
telid = self.telescope
if telid is None:
telid = tels[0]
if telid not in tels:
self.log.error("[event] please specify one of the following "
"telescopes for this event: {}".format(tels))
exit()
extractor_name = self.extractor.__class__.__name__
plot(event, telid, self.channel, extractor_name)
def finish(self):
pass
class CalibrationHDF5Writer(Tool):
"""
Tool that generates a HDF5 file with camera calibration coefficients.
This is just an example on how the monitoring containers can be filled using
the calibration Components in calib/camera.
This example is based on an input file with interleaved pedestal and flat-field events
For getting help run:
python calc_camera_calibration.py --help
"""
name = "CalibrationHDF5Writer"
description = "Generate a HDF5 file with camera calibration coefficients"
minimum_charge = Float(
800,
help='Temporary cut on charge till the calibox TIB do not work').tag(
config=True)
one_event = Bool(
False, help='Stop after first calibration event').tag(config=True)
output_file = Unicode('calibration.hdf5',
help='Name of the output file').tag(config=True)
log_file = Unicode('None', help='Name of the log file').tag(config=True)
pedestal_product = traits.enum_trait(PedestalCalculator,
default='PedestalIntegrator')
flatfield_product = traits.enum_trait(FlatFieldCalculator,
default='FlasherFlatFieldCalculator')
r0calibrator_product = traits.enum_trait(CameraR0Calibrator,
default='NullR0Calibrator')
aliases = Dict(
dict(
input_file='EventSource.input_url',
output_file='CalibrationHDF5Writer.output_file',
log_file='CalibrationHDF5Writer.log_file',
max_events='EventSource.max_events',
pedestal_file='LSTR0Corrections.pedestal_path',
flatfield_product='CalibrationHDF5Writer.flatfield_product',
pedestal_product='CalibrationHDF5Writer.pedestal_product',
r0calibrator_product='CalibrationHDF5Writer.r0calibrator_product',
))
classes = List([EventSource, FlatFieldCalculator, PedestalCalculator] +
traits.classes_with_traits(CameraR0Calibrator) +
traits.classes_with_traits(FlatFieldCalculator) +
traits.classes_with_traits(PedestalCalculator))
def __init__(self, **kwargs):
super().__init__(**kwargs)
"""
Tool that generates a HDF5 file with camera calibration coefficients.
Input file must contain interleaved pedestal and flat-field events
For getting help run:
python calc_camera_calibration.py --help
"""
self.eventsource = None
self.flatfield = None
self.pedestal = None
self.container = None
self.writer = None
self.r0calibrator = None
self.tel_id = None
self.tot_events = 0
def setup(self):
kwargs = dict(parent=self)
self.eventsource = EventSource.from_config(**kwargs)
# remember how many event in the files
self.tot_events = len(self.eventsource.multi_file)
self.log.debug(f"Input file has file {self.tot_events} events")
self.flatfield = FlatFieldCalculator.from_name(self.flatfield_product,
**kwargs)
self.pedestal = PedestalCalculator.from_name(self.pedestal_product,
**kwargs)
if self.r0calibrator_product:
self.r0calibrator = CameraR0Calibrator.from_name(
self.r0calibrator_product, **kwargs)
msg = "tel_id not the same for all calibration components"
assert self.r0calibrator.tel_id == self.pedestal.tel_id == self.flatfield.tel_id, msg
self.tel_id = self.flatfield.tel_id
group_name = 'tel_' + str(self.tel_id)
self.log.debug(f"Open output file {self.output_file}")
self.writer = HDF5TableWriter(filename=self.output_file,
group_name=group_name,
overwrite=True)
def start(self):
'''Calibration coefficient calculator'''
new_ped = False
new_ff = False
end_of_file = False
try:
self.log.debug(f"Start loop")
for count, event in enumerate(self.eventsource):
if count % 1000 == 0:
self.log.debug(f"Event {count}")
# if last event write results
if count == self.tot_events - 1 or count == self.eventsource.max_events - 1:
self.log.debug(f"Last event, count = {count}")
end_of_file = True
# save the config, to be retrieved as data.meta['config']
if count == 0:
ped_data = event.mon.tel[self.tel_id].pedestal
ped_data.meta['config'] = self.config
ff_data = event.mon.tel[self.tel_id].flatfield
ff_data.meta['config'] = self.config
status_data = event.mon.tel[self.tel_id].pixel_status
status_data.meta['config'] = self.config
calib_data = event.mon.tel[self.tel_id].calibration
calib_data.meta['config'] = self.config
# correct for low level calibration
self.r0calibrator.calibrate(event)
# if pedestal event
if event.r1.tel[self.tel_id].trigger_type == 32:
new_ped = self.pedestal.calculate_pedestals(event)
# if flat-field event: no calibration TIB for the moment, use a cut on the charge for ff events
elif event.r1.tel[self.tel_id].trigger_type == 4 or np.median(
np.sum(event.r1.tel[self.tel_id].waveform[0],
axis=1)) > self.minimum_charge:
new_ff = self.flatfield.calculate_relative_gain(event)
# write pedestal results when enough statistics or end of file
if new_ped or end_of_file:
# update the monitoring container with the last statistics
if end_of_file:
self.pedestal.store_results(event)
# write the event
self.log.debug(
f"Write pedestal data at event n. {count+1}, id {event.r0.event_id} "
f"stat = {ped_data.n_events} events")
# write on file
self.writer.write('pedestal', ped_data)
new_ped = False
# write flatfield results when enough statistics (also for pedestals) or end of file
if (new_ff and ped_data.n_events > 0) or end_of_file:
# update the monitoring container with the last statistics
if end_of_file:
self.flatfield.store_results(event)
self.log.debug(
f"Write flatfield data at event n. {count+1}, id {event.r0.event_id} "
f"stat = {ff_data.n_events} events")
# write on file
self.writer.write('flatfield', ff_data)
new_ff = False
# Then, calculate calibration coefficients
# mask from pedestal and flat-field data
monitoring_unusable_pixels = np.logical_or(
status_data.pedestal_failing_pixels,
status_data.flatfield_failing_pixels)
# calibration unusable pixels are an OR of all masks
calib_data.unusable_pixels = np.logical_or(
monitoring_unusable_pixels,
status_data.hardware_failing_pixels)
# Extract calibration coefficients with F-factor method
# Assume fix F2 factor, F2=1+Var(gain)/Mean(Gain)**2 must be known from elsewhere
F2 = 1.2
# calculate photon-electrons
n_pe = F2 * (ff_data.charge_median - ped_data.charge_median
)**2 / (ff_data.charge_std**2 -
ped_data.charge_std**2)
# fill WaveformCalibrationContainer (this is an example)
calib_data.time = ff_data.sample_time
calib_data.time_range = ff_data.sample_time_range
calib_data.n_pe = n_pe
calib_data.dc_to_pe = n_pe / (ff_data.charge_median -
ped_data.charge_median)
calib_data.time_correction = -ff_data.relative_time_median
ped_extractor_name = self.config.get(
"PedestalCalculator").get("charge_product")
ped_width = self.config.get(ped_extractor_name).get(
"window_width")
calib_data.pedestal_per_sample = ped_data.charge_median / ped_width
# write calib and pixel status
self.log.debug(f"Write pixel_status data")
self.writer.write('pixel_status', status_data)
self.log.debug(f"Write calibration data")
self.writer.write('calibration', calib_data)
if self.one_event:
break
#self.writer.write('mon', event.mon.tel[self.tel_id])
except ValueError as e:
self.log.error(e)
def finish(self):
Provenance().add_output_file(self.output_file,
role='mon.tel.calibration')
self.writer.close()
class CalibrationCalculator(Component):
"""
Parent class for the camera calibration calculators.
Fills the MonitoringCameraContainer on the base of calibration events
Parameters
----------
flatfield_calculator: lstchain.calib.camera.flatfield
The flatfield to use. If None, then FlatFieldCalculator
will be used by default.
pedestal_calculator: lstchain.calib.camera.pedestal
The pedestal to use. If None, then
PedestalCalculator will be used by default.
kwargs
"""
squared_excess_noise_factor = Float(
1.2,
help='Excess noise factor squared: 1+ Var(gain)/Mean(Gain)**2').tag(
config=True)
pedestal_product = traits.enum_trait(PedestalCalculator,
default='PedestalIntegrator')
flatfield_product = traits.enum_trait(FlatFieldCalculator,
default='FlasherFlatFieldCalculator')
classes = List([FlatFieldCalculator, PedestalCalculator] +
traits.classes_with_traits(FlatFieldCalculator) +
traits.classes_with_traits(PedestalCalculator))
def __init__(self, parent=None, config=None, **kwargs):
"""
Parent class for the camera calibration calculators.
Fills the MonitoringCameraContainer on the base of calibration events
Parameters
----------
flatfield_calculator: lstchain.calib.camera.flatfield
The flatfield to use. If None, then FlatFieldCalculator
will be used by default.
pedestal_calculator: lstchain.calib.camera.pedestal
The pedestal to use. If None, then
PedestalCalculator will be used by default.
"""
super().__init__(parent=parent, config=config, **kwargs)
self.flatfield = FlatFieldCalculator.from_name(self.flatfield_product,
parent=self)
self.pedestal = PedestalCalculator.from_name(self.pedestal_product,
parent=self)
msg = "tel_id not the same for all calibration components"
if self.pedestal.tel_id != self.flatfield.tel_id:
raise ValueError(msg)
self.tel_id = self.flatfield.tel_id
self.log.debug(f"{self.pedestal}")
self.log.debug(f"{self.flatfield}")
class CalibrationHDF5Writer(Tool):
"""
Tool that generates a HDF5 file with camera calibration coefficients.
This is just an example on how the monitoring containers can be filled using
the calibration Components in calib/camera.
This example is based on an input file with pedestal and flat-field events
For getting help run:
python calc_camera_calibration.py --help
"""
name = "CalibrationHDF5Writer"
description = "Generate a HDF5 file with camera calibration coefficients"
one_event = Bool(
False, help='Stop after first calibration event').tag(config=True)
output_file = Unicode('calibration.hdf5',
help='Name of the output file').tag(config=True)
log_file = Unicode('None', help='Name of the log file').tag(config=True)
calibration_product = traits.enum_trait(CalibrationCalculator,
default='LSTCalibrationCalculator')
r0calibrator_product = traits.enum_trait(CameraR0Calibrator,
default='NullR0Calibrator')
aliases = Dict(
dict(
input_file='EventSource.input_url',
output_file='CalibrationHDF5Writer.output_file',
log_file='CalibrationHDF5Writer.log_file',
max_events='EventSource.max_events',
pedestal_file='LSTR0Corrections.pedestal_path',
calibration_product='CalibrationHDF5Writer.calibration_product',
r0calibrator_product='CalibrationHDF5Writer.r0calibrator_product',
))
classes = List([EventSource, CalibrationCalculator] +
traits.classes_with_traits(CameraR0Calibrator) +
traits.classes_with_traits(CalibrationCalculator))
def __init__(self, **kwargs):
super().__init__(**kwargs)
"""
Tool that generates a HDF5 file with camera calibration coefficients.
Input file must contain interleaved pedestal and flat-field events
For getting help run:
python calc_camera_calibration.py --help
"""
self.eventsource = None
self.processor = None
self.container = None
self.writer = None
self.r0calibrator = None
self.tot_events = 0
self.simulation = False
def setup(self):
self.log.debug(f"Open file")
self.eventsource = EventSource.from_config(parent=self)
# if data remember how many event in the files
if "LSTEventSource" in str(type(self.eventsource)):
self.tot_events = len(self.eventsource.multi_file)
self.log.debug(f"Input file has file {self.tot_events} events")
else:
self.tot_events = self.eventsource.max_events
self.simulation = True
self.processor = CalibrationCalculator.from_name(
self.calibration_product, parent=self)
if self.r0calibrator_product:
self.r0calibrator = CameraR0Calibrator.from_name(
self.r0calibrator_product, parent=self)
group_name = 'tel_' + str(self.processor.tel_id)
self.log.debug(f"Open output file {self.output_file}")
self.writer = HDF5TableWriter(filename=self.output_file,
group_name=group_name,
overwrite=True)
def start(self):
'''Calibration coefficient calculator'''
tel_id = self.processor.tel_id
new_ped = False
new_ff = False
end_of_file = False
# skip the first events which are badly drs4 corrected
events_to_skip = 1000
try:
self.log.debug(f"Start loop")
for count, event in enumerate(self.eventsource):
if count % 100 == 0:
self.log.debug(f"Event {count}")
# if last event write results
max_events_reached = (self.eventsource.max_events is not None
and count
== self.eventsource.max_events - 1)
if count == self.tot_events - 1 or max_events_reached:
self.log.debug(f"Last event, count = {count}")
end_of_file = True
# save the config, to be retrieved as data.meta['config']
if count == 0:
if self.simulation:
initialize_pixel_status(
event.mon.tel[tel_id],
event.r1.tel[tel_id].waveform.shape)
ped_data = event.mon.tel[tel_id].pedestal
ped_data.meta['config'] = self.config
ff_data = event.mon.tel[tel_id].flatfield
ff_data.meta['config'] = self.config
status_data = event.mon.tel[tel_id].pixel_status
status_data.meta['config'] = self.config
calib_data = event.mon.tel[tel_id].calibration
calib_data.meta['config'] = self.config
# correct for low level calibration
self.r0calibrator.calibrate(event)
# skip first events which are badly drs4 corrected
if not self.simulation and count < events_to_skip:
continue
# reject event without trigger type
if LSTEventType.is_unknown(event.r1.tel[tel_id].trigger_type):
continue
# if pedestal event
if LSTEventType.is_pedestal(
event.r1.tel[tel_id].trigger_type
) or (self.simulation and np.median(
np.sum(event.r1.tel[tel_id].waveform[0], axis=1)) <
self.processor.minimum_hg_charge_median):
new_ped = self.processor.pedestal.calculate_pedestals(
event)
# if flat-field event: no calibration TIB for the moment,
# use a cut on the charge for ff events and on std for rejecting Magic Lidar events
elif LSTEventType.is_calibration(
event.r1.tel[tel_id].trigger_type
) or (np.median(
np.sum(event.r1.tel[tel_id].waveform[0], axis=1)) >
self.processor.minimum_hg_charge_median and
np.std(np.sum(event.r1.tel[tel_id].waveform[1], axis=1))
< self.processor.maximum_lg_charge_std):
new_ff = self.processor.flatfield.calculate_relative_gain(
event)
# write pedestal results when enough statistics or end of file
if new_ped or end_of_file:
# update the monitoring container with the last statistics
if end_of_file:
self.processor.pedestal.store_results(event)
# write the event
self.log.debug(
f"Write pedestal data at event n. {count+1}, id {event.r0.event_id} "
f"stat = {ped_data.n_events} events")
# write on file
self.writer.write('pedestal', ped_data)
new_ped = False
# write flatfield results when enough statistics (also for pedestals) or end of file
if (new_ff and ped_data.n_events > 0) or end_of_file:
# update the monitoring container with the last statistics
if end_of_file:
self.processor.flatfield.store_results(event)
self.log.debug(
f"Write flatfield data at event n. {count+1}, id {event.r0.event_id} "
f"stat = {ff_data.n_events} events")
# write on file
self.writer.write('flatfield', ff_data)
new_ff = False
# Then, calculate calibration coefficients
self.processor.calculate_calibration_coefficients(event)
# write calib and pixel status
self.log.debug(f"Write pixel_status data")
self.writer.write('pixel_status', status_data)
self.log.debug(f"Write calibration data")
self.writer.write('calibration', calib_data)
if self.one_event:
break
#self.writer.write('mon', event.mon.tel[tel_id])
except ValueError as e:
self.log.error(e)
def finish(self):
Provenance().add_output_file(self.output_file,
role='mon.tel.calibration')
self.writer.close()
