class PedestalCalculator(Component):
"""
Parent class for the pedestal calculators.
Fills the MON.pedestal container on the base of
pedestal events (preliminary version)
"""
tel_id = Int(
0, help='id of the telescope to calculate the pedestal values').tag(
config=True)
sample_duration = Int(60,
help='sample duration in seconds').tag(config=True)
sample_size = Int(10000, help='sample size').tag(config=True)
n_channels = Int(2,
help='number of channels to be treated').tag(config=True)
charge_cut_outliers = List(
[3, 3], help='Interval (number of std) of accepted charge values').tag(
config=True)
charge_std_cut_outliers = List(
[3, 3],
help=
'Interval (number of std) of accepted charge standard deviation values'
).tag(config=True)
charge_product = Unicode(
'LocalPeakIntegrator',
help='Name of the charge extractor to be used').tag(config=True)
def __init__(self, **kwargs):
"""
Parent class for pedestal calculators.
Fills the MON.pedestal container.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
super().__init__(**kwargs)
# initialize the output
self.container = PedestalContainer()
# load the waveform charge extractor
self.extractor = ChargeExtractor.from_name(self.charge_product,
config=self.config)
self.log.info(f"extractor {self.extractor}")
@abstractmethod
def calculate_pedestals(self, event):
"""calculate relative gain from event
class FlatFieldCalculator(Component):
"""
Parent class for the flat field calculators.
Fills the MON.flatfield container.
"""
tel_id = Int(
0, help='id of the telescope to calculate the flat-field coefficients'
).tag(config=True)
sample_duration = Int(60,
help='sample duration in seconds').tag(config=True)
sample_size = Int(10000, help='sample size').tag(config=True)
n_channels = Int(2,
help='number of channels to be treated').tag(config=True)
charge_cut_outliers = List(
[-0.3, 0.3],
help=
'Interval of accepted charge values (fraction with respect to camera median value)'
).tag(config=True)
time_cut_outliers = List(
[10, 30],
help='Interval (in samples) of accepted time values').tag(config=True)
charge_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
def __init__(self, **kwargs):
"""
Parent class for the flat field calculators.
Fills the flatfield container.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
super().__init__(**kwargs)
# load the waveform charge extractor
self.extractor = ImageExtractor.from_name(self.charge_product,
config=self.config)
self.log.info(f"extractor {self.extractor}")
@abstractmethod
def calculate_relative_gain(self, event):
"""calculate relative gain from event
class EventSelector(Component):
"""
Filter values used for event filters and list of finite parameters are
taken as inputs and filter_events() is used on a table of events
called in with the Component.
For event_type, we choose the sub-array trigger, EventType.SUBARRAY.value,
32, which is for shower event candidate, as per the latest CTA R1 Event
Data Model.
"""
filters = Dict(
help="Dict of event filter parameters",
default_value={
"r": [0, 1],
"wl": [0.01, 1],
"leakage_intensity_width_2": [0, 1],
"event_type": [EventType.SUBARRAY.value, EventType.SUBARRAY.value],
},
).tag(config=True)
finite_params = List(
help="List of parameters to ensure finite values",
default_value=["intensity", "length", "width"],
).tag(config=True)
def filter_cut(self, events):
"""
Apply the event filters
"""
return filter_events(events, self.filters, self.finite_params)
class DL3FixedCuts(Component):
"""
Temporary fixed selection cuts for DL2 to DL3 conversion
"""
fixed_gh_cut = Float(
help="Fixed selection cut for gh_score (gammaness)",
default_value=0.6,
).tag(config=True)
fixed_theta_cut = Float(
help="Fixed selection cut for theta",
default_value=0.2,
).tag(config=True)
allowed_tels = List(
help="List of allowed LST telescope ids",
trait=Int(),
default_value=[1],
).tag(config=True)
def gh_cut(self, data):
return data[data["gh_score"] > self.fixed_gh_cut]
def theta_cut(self, data):
return data[data["theta"].to_value(u.deg) < self.fixed_theta_cut]
def allowed_tels_filter(self, data):
mask = np.zeros(len(data), dtype=bool)
for tel_id in self.allowed_tels:
mask |= data["tel_id"] == tel_id
return data[mask]
class ExampleQualityQuery(QualityQuery):
"""Available variables: x"""
quality_criteria = List(
default_value=[
("high_enough", "x > 3"),
("a_value_not_too_high", "x < 100"),
("smallish", "x < np.sqrt(100)"),
],
).tag(config=True)
class ShowerQualityQuery(QualityQuery):
"""Configuring shower-wise data checks."""
quality_criteria = List(
default_value=[
("> 50 phe", "lambda p: p.hillas.intensity > 50"),
("Positive width", "lambda p: p.hillas.width.value > 0"),
("> 3 pixels", "lambda p: p.morphology.num_pixels > 3"),
],
help=QualityQuery.quality_criteria.help,
).tag(config=True)
class StereoQualityQuery(QualityQuery):
"""Quality criteria for dl1 parameters checked for telescope events to enter
into stereo reconstruction"""
quality_criteria = List(
default_value=[
("> 50 phe", "lambda p: p.hillas.intensity > 50"),
("Positive width", "lambda p: p.hillas.width.value > 0"),
("> 3 pixels", "lambda p: p.morphology.num_pixels > 3"),
],
help=QualityQuery.quality_criteria.help,
).tag(config=True)
class EventSelector(Component):
"""
Filter values used for event filters and list of finite parameters are
taken as inputs and filter_events() is used on a table of events
called in with the Component.
"""
filters = Dict(
help="Dict of event filter parameters",
default_value={
"r": [0, 1],
"wl": [0.01, 1],
"leakage_intensity_width_2": [0, 1],
},
).tag(config=True)
finite_params = List(
help="List of parameters to ensure finite values",
default_value=["intensity", "length", "width"],
).tag(config=True)
def filter_cut(self, events):
return filter_events(events, self.filters, self.finite_params)
class SingleTelEventDisplay(Tool):
name = "ctapipe-display-televents"
description = Unicode(__doc__)
infile = Unicode(help="input file to read", default='').tag(config=True)
tel = Int(help='Telescope ID to display', default=0).tag(config=True)
channel = Integer(help="channel number to display", min=0,
max=1).tag(config=True)
write = Bool(help="Write out images to PNG files",
default=False).tag(config=True)
clean = Bool(help="Apply image cleaning", default=False).tag(config=True)
hillas = Bool(help="Apply and display Hillas parametrization",
default=False).tag(config=True)
samples = Bool(help="Show each sample", default=False).tag(config=True)
display = Bool(help="Display results in interactive window",
default_value=True).tag(config=True)
delay = Float(help='delay between events in s',
default_value=0.01,
min=0.001).tag(config=True)
progress = Bool(help='display progress bar',
default_value=True).tag(config=True)
aliases = Dict({
'infile': 'SingleTelEventDisplay.infile',
'tel': 'SingleTelEventDisplay.tel',
'max-events': 'EventSource.max_events',
'channel': 'SingleTelEventDisplay.channel',
'write': 'SingleTelEventDisplay.write',
'clean': 'SingleTelEventDisplay.clean',
'hillas': 'SingleTelEventDisplay.hillas',
'samples': 'SingleTelEventDisplay.samples',
'display': 'SingleTelEventDisplay.display',
'delay': 'SingleTelEventDisplay.delay',
'progress': 'SingleTelEventDisplay.progress'
})
classes = List([EventSource, CameraCalibrator])
def __init__(self, **kwargs):
super().__init__(**kwargs)
def setup(self):
print('TOLLES INFILE', self.infile)
self.event_source = EventSource.from_url(self.infile, parent=self)
self.event_source.allowed_tels = {
self.tel,
}
self.calibrator = CameraCalibrator(parent=self)
self.log.info(f'SELECTING EVENTS FROM TELESCOPE {self.tel}')
def start(self):
disp = None
for event in tqdm(self.event_source,
desc=f'Tel{self.tel}',
total=self.event_source.max_events,
disable=~self.progress):
self.log.debug(event.trig)
self.log.debug(f"Energy: {event.mc.energy}")
self.calibrator(event)
if disp is None:
geom = event.inst.subarray.tel[self.tel].camera
self.log.info(geom)
disp = CameraDisplay(geom)
# disp.enable_pixel_picker()
disp.add_colorbar()
if self.display:
plt.show(block=False)
# display the event
disp.axes.set_title('CT{:03d} ({}), event {:06d}'.format(
self.tel, geom.cam_id, event.r0.event_id))
if self.samples:
# display time-varying event
data = event.dl0.tel[self.tel].waveform[self.channel]
for ii in range(data.shape[1]):
disp.image = data[:, ii]
disp.set_limits_percent(70)
plt.suptitle(f"Sample {ii:03d}")
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f'CT{self.tel:03d}_EV{event.r0.event_id:10d}'
f'_S{ii:02d}.png')
else:
# display integrated event:
im = event.dl1.tel[self.tel].image[self.channel]
if self.clean:
mask = tailcuts_clean(geom,
im,
picture_thresh=10,
boundary_thresh=7)
im[~mask] = 0.0
disp.image = im
if self.hillas:
try:
ellipses = disp.axes.findobj(Ellipse)
if len(ellipses) > 0:
ellipses[0].remove()
params = hillas_parameters(geom, image=im)
disp.overlay_moments(params,
color='pink',
lw=3,
with_label=False)
except HillasParameterizationError:
pass
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f'CT{self.tel:03d}_EV{event.r0.event_id:010d}.png')
self.log.info("FINISHED READING DATA FILE")
if disp is None:
self.log.warning(
'No events for tel {} were found in {}. Try a '
'different EventIO file or another telescope'.format(
self.tel, self.infile), )
class ImageSumDisplayerTool(Tool):
description = Unicode(__doc__)
name = "ctapipe-display-imagesum"
infile = Unicode(
help='input simtelarray file',
default="/Users/kosack/Data/CTA/Prod3/gamma.simtel.gz").tag(
config=True)
telgroup = Integer(help='telescope group number',
default=1).tag(config=True)
max_events = Integer(help='stop after this many events if non-zero',
default_value=0,
min=0).tag(config=True)
output_suffix = Unicode(help='suffix (file extension) of output '
'filenames to write images '
'to (no writing is done if blank). '
'Images will be named [EVENTID][suffix]',
default_value="").tag(config=True)
aliases = Dict({
'infile': 'ImageSumDisplayerTool.infile',
'telgroup': 'ImageSumDisplayerTool.telgroup',
'max-events': 'ImageSumDisplayerTool.max_events',
'output-suffix': 'ImageSumDisplayerTool.output_suffix'
})
classes = List([CameraCalibrator, SimTelEventSource])
def setup(self):
# load up the telescope types table (need to first open a file, a bit of
# a hack until a proper insturment module exists) and select only the
# telescopes with the same camera type
# make sure gzip files are seekable
self.reader = SimTelEventSource(input_url=self.infile,
max_events=self.max_events,
back_seekable=True)
for event in self.reader:
camtypes = event.inst.subarray.to_table().group_by('camera_type')
event.inst.subarray.info(printer=self.log.info)
break
group = camtypes.groups[self.telgroup]
self._selected_tels = list(group['tel_id'].data)
self._base_tel = self._selected_tels[0]
self.log.info("Telescope group %d: %s", self.telgroup,
str(event.inst.subarray.tel[self._selected_tels[0]]))
self.log.info(f"SELECTED TELESCOPES:{self._selected_tels}")
self.calibrator = CameraCalibrator(parent=self)
self.reader.allowed_tels = self._selected_tels
def start(self):
geom = None
imsum = None
disp = None
for event in self.reader:
self.calibrator(event)
if geom is None:
geom = event.inst.subarray.tel[self._base_tel].camera
imsum = np.zeros(shape=geom.pix_x.shape, dtype=np.float)
disp = CameraDisplay(geom, title=geom.cam_id)
disp.add_colorbar()
disp.cmap = 'viridis'
if len(event.dl0.tels_with_data) <= 2:
continue
imsum[:] = 0
for telid in event.dl0.tels_with_data:
imsum += event.dl1.tel[telid].image
self.log.info("event={} ntels={} energy={}".format(
event.r0.event_id, len(event.dl0.tels_with_data),
event.mc.energy))
disp.image = imsum
plt.pause(0.1)
if self.output_suffix is not "":
filename = "{:020d}{}".format(event.r0.event_id,
self.output_suffix)
self.log.info(f"saving: '{filename}'")
plt.savefig(filename)
class ExampleQualityQuery(QualityQuery):
quality_criteria = List(default_value=[
("high_enough", "lambda x: x > 3"),
("a_value_not_too_high", "lambda x: x < 100"),
("smallish", "lambda x: x < np.sqrt(100)"),
], ).tag(config=True)
class LSTCameraCalibrator(CameraCalibrator):
"""
Calibrator to handle the LST camera calibration chain, in order to fill
the DL1 data level in the event container.
"""
extractor_product = Unicode(
'NeighborPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
reducer_product = Unicode(
'NullDataVolumeReducer',
help='Name of the DataVolumeReducer to use').tag(config=True)
calibration_path = Unicode(
'', allow_none=True,
help='Path to LST calibration file').tag(config=True)
time_calibration_path = Unicode(
'', allow_none=True,
help='Path to drs4 time calibration file').tag(config=True)
allowed_tels = List(
[1], help='List of telescope to be calibrated').tag(config=True)
gain_threshold = Int(
4094, allow_none=True,
help='Threshold for the gain selection in ADC').tag(config=True)
def __init__(self, **kwargs):
"""
Parameters
----------
reducer_product : ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use. If None, then
NullDataVolumeReducer will be used by default, and waveforms
will not be reduced.
extractor_product : ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, then NeighborPeakWindowSum
will be used by default.
calibration_path :
Path to LST calibration file to get the pedestal and flat-field corrections
kwargs
"""
super().__init__(**kwargs)
# load the waveform charge extractor
self.image_extractor = ImageExtractor.from_name(self.extractor_product,
config=self.config)
self.log.info(f"extractor {self.extractor_product}")
self.data_volume_reducer = DataVolumeReducer.from_name(
self.reducer_product, config=self.config)
self.log.info(f" {self.reducer_product}")
# declare gain selector if the threshold is defined
if self.gain_threshold:
self.gain_selector = gainselection.ThresholdGainSelector(
threshold=self.gain_threshold)
# declare time calibrator if correction file exist
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
self.time_corrector = None
self.log.info(
f"File {self.time_calibration_path} not found. No drs4 time corrections"
)
# calibration data container
self.mon_data = MonitoringContainer()
# initialize the MonitoringContainer() for the moment it reads it from a hdf5 file
self._initialize_correction()
def _initialize_correction(self):
"""
Read the correction from hdf5 calibration file
"""
self.mon_data.tels_with_data = self.allowed_tels
self.log.info(f"read {self.calibration_path}")
try:
with HDF5TableReader(self.calibration_path) as h5_table:
assert h5_table._h5file.isopen == True
for telid in self.allowed_tels:
# read the calibration data for the moment only one event
table = '/tel_' + str(telid) + '/calibration'
next(
h5_table.read(table,
self.mon_data.tel[telid].calibration))
# eliminate inf values (should be done probably before)
dc_to_pe = self.mon_data.tel[telid].calibration.dc_to_pe
dc_to_pe[np.isinf(dc_to_pe)] = 0
self.log.info(
f"read {self.mon_data.tel[telid].calibration.dc_to_pe}"
)
except:
self.log.error(
f"Problem in reading calibration file {self.calibration_path}")
def _calibrate_dl0(self, event, telid):
"""
create dl0 level, for the moment copy the r1
"""
waveforms = event.r1.tel[telid].waveform
if self._check_r1_empty(waveforms):
return
event.dl0.event_id = event.r1.event_id
event.mon.tel[telid].calibration = self.mon_data.tel[telid].calibration
# subtract the pedestal per sample (should we do it?) and multiply for the calibration coefficients
#
event.dl0.tel[telid].waveform = (
(event.r1.tel[telid].waveform - self.mon_data.tel[telid].
calibration.pedestal_per_sample[:, :, np.newaxis]) *
self.mon_data.tel[telid].calibration.dc_to_pe[:, :, np.newaxis])
def _calibrate_dl1(self, event, telid):
"""
create calibrated dl1 image and calibrate it
"""
waveforms = event.dl0.tel[telid].waveform
if self._check_dl0_empty(waveforms):
return
if self.image_extractor.requires_neighbors():
camera = event.inst.subarray.tel[telid].camera
self.image_extractor.neighbors = camera.neighbor_matrix_where
charge, pulse_time = self.image_extractor(waveforms)
# correct time with drs4 correction if available
if self.time_corrector:
pulse_corr_array = self.time_corrector.get_corr_pulse(
event, pulse_time)
# otherwise use the ff time correction (not drs4 corrected)
else:
pulse_corr_array = pulse_time + self.mon_data.tel[
telid].calibration.time_correction
# perform the gain selection if the threshold is defined
if self.gain_threshold:
waveforms, gain_mask = self.gain_selector(
event.r1.tel[telid].waveform)
event.dl1.tel[telid].image = charge[gain_mask,
np.arange(charge.shape[1])]
event.dl1.tel[telid].pulse_time = pulse_corr_array[
gain_mask, np.arange(pulse_corr_array.shape[1])]
# remember the mask in the lst pixel_status array (this info is missing for the moment in the
# r1 container). I follow the prescription given in the document
# "R1 & DL0 Telescope Event Interfaces and Prototype Evaluation" of K. Kosack
# bit 2 = LG
gain_mask *= 4
# bit 3 = HG
gain_mask[np.where(gain_mask == 0)] = 8
# bit 1 = pixel broken pixel (coming from the EvB)
gain_mask += event.lst.tel[telid].evt.pixel_status >> 1 & 1
# update pixel status
event.lst.tel[telid].evt.pixel_status = gain_mask
# if threshold == None
else:
event.dl1.tel[telid].image = charge
event.dl1.tel[telid].pulse_time = pulse_corr_array
class CalibrationCalculator(Component):
"""
Parent class for the camera calibration calculators.
Fills the MonitoringCameraContainer on the base of calibration events
Parameters
----------
flatfield_calculator: lstchain.calib.camera.flatfield
The flatfield to use. If None, then FlatFieldCalculator
will be used by default.
pedestal_calculator: lstchain.calib.camera.pedestal
The pedestal to use. If None, then
PedestalCalculator will be used by default.
kwargs
"""
squared_excess_noise_factor = Float(
1.222,
help='Excess noise factor squared: 1+ Var(gain)/Mean(Gain)**2').tag(
config=True)
pedestal_product = traits.create_class_enum_trait(
PedestalCalculator, default_value='PedestalIntegrator')
flatfield_product = traits.create_class_enum_trait(
FlatFieldCalculator, default_value='FlasherFlatFieldCalculator')
classes = List([FlatFieldCalculator, PedestalCalculator] +
traits.classes_with_traits(FlatFieldCalculator) +
traits.classes_with_traits(PedestalCalculator))
def __init__(self, subarray, parent=None, config=None, **kwargs):
"""
Parent class for the camera calibration calculators.
Fills the MonitoringCameraContainer on the base of calibration events
Parameters
----------
flatfield_calculator: lstchain.calib.camera.flatfield
The flatfield to use. If None, then FlatFieldCalculator
will be used by default.
pedestal_calculator: lstchain.calib.camera.pedestal
The pedestal to use. If None, then
PedestalCalculator will be used by default.
"""
super().__init__(parent=parent, config=config, **kwargs)
self.flatfield = FlatFieldCalculator.from_name(self.flatfield_product,
parent=self,
subarray=subarray)
self.pedestal = PedestalCalculator.from_name(self.pedestal_product,
parent=self,
subarray=subarray)
msg = "tel_id not the same for all calibration components"
if self.pedestal.tel_id != self.flatfield.tel_id:
raise ValueError(msg)
self.tel_id = self.flatfield.tel_id
self.log.debug(f"{self.pedestal}")
self.log.debug(f"{self.flatfield}")
class SimpleEventWriter(Tool):
name = 'ctapipe-simple-event-writer'
description = Unicode(__doc__)
infile = Unicode(help='input file to read', default='').tag(config=True)
outfile = Unicode(help='output file name',
default_value='output.h5').tag(config=True)
progress = Bool(help='display progress bar',
default_value=True).tag(config=True)
aliases = Dict({
'infile': 'EventSource.input_url',
'outfile': 'SimpleEventWriter.outfile',
'max-events': 'EventSource.max_events',
'progress': 'SimpleEventWriter.progress'
})
classes = List([EventSource, CameraCalibrator, CutFlow])
def setup(self):
self.log.info('Configure EventSource...')
self.event_source = self.add_component(
EventSource.from_config(config=self.config, parent=self))
self.calibrator = self.add_component(CameraCalibrator(parent=self))
self.writer = self.add_component(
HDF5TableWriter(filename=self.outfile,
group_name='image_infos',
overwrite=True))
# Define Pre-selection for images
preselcuts = self.config['Preselect']
self.image_cutflow = CutFlow('Image preselection')
self.image_cutflow.set_cuts(
dict(no_sel=None,
n_pixel=lambda s: np.count_nonzero(s) < preselcuts['n_pixel'][
'min'],
image_amplitude=lambda q: q < preselcuts['image_amplitude'][
'min']))
# Define Pre-selection for events
self.event_cutflow = CutFlow('Event preselection')
self.event_cutflow.set_cuts(dict(no_sel=None))
def start(self):
self.log.info('Loop on events...')
for event in tqdm(self.event_source,
desc='EventWriter',
total=self.event_source.max_events,
disable=~self.progress):
self.event_cutflow.count('no_sel')
self.calibrator(event)
for tel_id in event.dl0.tels_with_data:
self.image_cutflow.count('no_sel')
camera = event.inst.subarray.tel[tel_id].camera
dl1_tel = event.dl1.tel[tel_id]
# Image cleaning
image = dl1_tel.image # Waiting for automatic gain selection
mask = tailcuts_clean(camera,
image,
picture_thresh=10,
boundary_thresh=5)
cleaned = image.copy()
cleaned[~mask] = 0
# Preselection cuts
if self.image_cutflow.cut('n_pixel', cleaned):
continue
if self.image_cutflow.cut('image_amplitude', np.sum(cleaned)):
continue
# Image parametrisation
params = hillas_parameters(camera, cleaned)
# Save Ids, MC infos and Hillas informations
self.writer.write(camera.cam_id, [event.r0, event.mc, params])
def finish(self):
self.log.info('End of job.')
self.image_cutflow()
self.event_cutflow()
self.writer.close()
class FlasherFlatFieldCalculator(FlatFieldCalculator):
"""Calculates flat-field parameters from flasher data
based on the best algorithm described by S. Fegan in MST-CAM-TN-0060 (eq. 19)
Pixels are defined as outliers on the base of a cut on the pixel charge median
over the full sample distribution and the pixel signal time inside the
waveform time
Parameters:
----------
charge_cut_outliers : List[2]
Interval of accepted charge values (fraction with respect to camera median value)
time_cut_outliers : List[2]
Interval (in waveform samples) of accepted time values
"""
charge_median_cut_outliers = List(
[-0.3, 0.3],
help='Interval of accepted charge values (fraction with respect to camera median value)'
).tag(config=True)
charge_std_cut_outliers = List(
[-3, 3],
help='Interval (number of std) of accepted charge standard deviation around camera median value'
).tag(config=True)
time_cut_outliers = List(
[0, 60], help="Interval (in waveform samples) of accepted time values"
).tag(config=True)
time_sampling_correction_path = Path(
default_value=None,
allow_none=True,
exists=True, directory_ok=False,
help='Path to time sampling correction file'
).tag(config=True)
def __init__(self, subarray, **kwargs):
"""Calculates flat-field parameters from flasher data
based on the best algorithm described by S. Fegan in MST-CAM-TN-0060 (eq. 19)
Pixels are defined as outliers on the base of a cut on the pixel charge median
over the full sample distribution and the pixel signal time inside the
waveform time
Parameters:
----------
charge_cut_outliers : List[2]
Interval of accepted charge values (fraction with respect to camera median value)
time_cut_outliers : List[2]
Interval (in waveform samples) of accepted time values
"""
super().__init__(subarray, **kwargs)
self.log.info("Used events statistics : %d", self.sample_size)
# members to keep state in calculate_relative_gain()
self.num_events_seen = 0
self.time_start = None # trigger time of first event in sample
self.trigger_time = None # trigger time of present event
self.charge_medians = None # med. charge in camera per event in sample
self.charges = None # charge per event in sample
self.arrival_times = None # arrival time per event in sample
self.sample_masked_pixels = None # masked pixels per event in sample
# declare the charge sampling corrector
if self.time_sampling_correction_path is not None:
self.time_sampling_corrector = TimeSamplingCorrection(
time_sampling_correction_path=self.time_sampling_correction_path
)
else:
self.time_sampling_corrector = None
# fix for broken extractor setup in ctapipe baseclass
self.extractor = ImageExtractor.from_name(
self.charge_product, parent=self, subarray=subarray
)
def _extract_charge(self, event):
"""
Extract the charge and the time from a calibration event
Parameters
----------
event : general event container
"""
# copy the waveform be cause we do not want to change it for the moment
waveforms = np.copy(event.r1.tel[self.tel_id].waveform)
# In case of no gain selection the selected gain channels are [0,0,..][1,1,..]
no_gain_selection = np.zeros((waveforms.shape[0], waveforms.shape[1]), dtype=np.int64)
no_gain_selection[1] = 1
n_pixels = 1855
# correct the r1 waveform for the sampling time corrections
if self.time_sampling_corrector:
waveforms*= (self.time_sampling_corrector.get_corrections(event,self.tel_id)
[no_gain_selection, np.arange(n_pixels)])
# Extract charge and time
charge = 0
peak_pos = 0
if self.extractor:
charge, peak_pos = self.extractor(waveforms, self.tel_id, no_gain_selection)
# shift the time if time shift is already defined
# (e.g. drs4 waveform time shifts for LST)
time_shift = event.calibration.tel[self.tel_id].dl1.time_shift
if time_shift is not None:
peak_pos -= time_shift
return charge, peak_pos
def calculate_relative_gain(self, event):
"""
calculate the flatfield statistical values
and fill mon.tel[tel_id].flatfield container
Parameters
----------
event : general event container
Returns: True if the mon.tel[tel_id].flatfield is updated, False otherwise
"""
# initialize the np array at each cycle
waveform = event.r1.tel[self.tel_id].waveform
# re-initialize counter
if self.num_events_seen == self.sample_size:
self.num_events_seen = 0
pixel_mask = np.logical_or(
event.mon.tel[self.tel_id].pixel_status.hardware_failing_pixels,
event.mon.tel[self.tel_id].pixel_status.flatfield_failing_pixels)
# time
self.trigger_time = event.trigger.tel[self.tel_id].time
if self.num_events_seen == 0:
self.time_start = self.trigger_time
self.setup_sample_buffers(waveform, self.sample_size)
# extract the charge of the event and
# the peak position (assumed as time for the moment)
charge, arrival_time = self._extract_charge(event)
self.collect_sample(charge, pixel_mask, arrival_time)
sample_age = self.trigger_time - self.time_start
# check if to create a calibration event
if (self.num_events_seen > 0 and
(sample_age > self.sample_duration or
self.num_events_seen == self.sample_size)
):
# update the monitoring container
self.store_results(event)
return True
else:
return False
def store_results(self, event):
"""
Store statistical results in monitoring container
Parameters
----------
event : general event container
"""
if self.num_events_seen == 0:
raise ValueError("No flat-field events in statistics, zero results")
container = event.mon.tel[self.tel_id].flatfield
# mask the part of the array not filled
self.sample_masked_pixels[self.num_events_seen:] = 1
relative_gain_results = self.calculate_relative_gain_results(
self.charge_medians,
self.charges,
self.sample_masked_pixels
)
time_results = self.calculate_time_results(
self.arrival_times,
self.sample_masked_pixels,
self.time_start,
self.trigger_time
)
result = {
'n_events': self.num_events_seen,
**relative_gain_results,
**time_results,
}
for key, value in result.items():
setattr(container, key, value)
# update the flatfield mask
ff_charge_failing_pixels = np.logical_or(container.charge_median_outliers,
container.charge_std_outliers)
event.mon.tel[self.tel_id].pixel_status.flatfield_failing_pixels = \
np.logical_or(ff_charge_failing_pixels, container.time_median_outliers)
def setup_sample_buffers(self, waveform, sample_size):
"""Initialize sample buffers"""
n_channels = waveform.shape[0]
n_pix = waveform.shape[1]
shape = (sample_size, n_channels, n_pix)
self.charge_medians = np.zeros((sample_size, n_channels))
self.charges = np.zeros(shape)
self.arrival_times = np.zeros(shape)
self.sample_masked_pixels = np.zeros(shape)
def collect_sample(self, charge, pixel_mask, arrival_time):
"""Collect the sample data"""
# extract the charge of the event and
# the peak position (assumed as time for the moment)
good_charge = np.ma.array(charge, mask=pixel_mask)
charge_median = np.ma.median(good_charge, axis=1)
self.charges[self.num_events_seen] = charge
self.arrival_times[self.num_events_seen] = arrival_time
self.sample_masked_pixels[self.num_events_seen] = pixel_mask
self.charge_medians[self.num_events_seen] = charge_median
self.num_events_seen += 1
def calculate_time_results(
self,
trace_time,
masked_pixels_of_sample,
time_start,
trigger_time,
):
"""Calculate and return the time results """
masked_trace_time = np.ma.array(
trace_time,
mask=masked_pixels_of_sample
)
# median over the sample per pixel
pixel_median = np.ma.median(masked_trace_time, axis=0)
# mean over the sample per pixel
pixel_mean = np.ma.mean(masked_trace_time, axis=0)
# std over the sample per pixel
pixel_std = np.ma.std(masked_trace_time, axis=0)
# median of the median over the camera
median_of_pixel_median = np.ma.median(pixel_median, axis=1)
# time outliers from median
relative_median = pixel_median - median_of_pixel_median[:, np.newaxis]
time_median_outliers = np.logical_or(pixel_median < self.time_cut_outliers[0],
pixel_median > self.time_cut_outliers[1])
return {
'sample_time': (time_start +(trigger_time - time_start) / 2).unix*u.s,
'sample_time_min': time_start.unix*u.s,
'sample_time_max': trigger_time.unix*u.s,
'time_mean': np.ma.getdata(pixel_mean)*u.ns,
'time_median': np.ma.getdata(pixel_median)*u.ns,
'time_std': np.ma.getdata(pixel_std)*u.ns,
'relative_time_median': np.ma.getdata(relative_median)*u.ns,
'time_median_outliers': np.ma.getdata(time_median_outliers),
}
def calculate_relative_gain_results(
self,
event_median,
trace_integral,
masked_pixels_of_sample,
):
"""Calculate and return the sample statistics"""
masked_trace_integral = np.ma.array(
trace_integral,
mask=masked_pixels_of_sample
)
# median over the sample per pixel
pixel_median = np.ma.median(masked_trace_integral, axis=0)
# mean over the sample per pixel
pixel_mean = np.ma.mean(masked_trace_integral, axis=0)
# std over the sample per pixel
pixel_std = np.ma.std(masked_trace_integral, axis=0)
# median of the median over the camera
median_of_pixel_median = np.ma.median(pixel_median, axis=1)
# median of the std over the camera
median_of_pixel_std = np.ma.median(pixel_std, axis=1)
# std of the std over camera
std_of_pixel_std = np.ma.std(pixel_std, axis=1)
# relative gain
relative_gain_event = masked_trace_integral / event_median[:, :, np.newaxis]
# outliers from median
charge_deviation = pixel_median - median_of_pixel_median[:, np.newaxis]
charge_median_outliers = (
np.logical_or(charge_deviation < self.charge_median_cut_outliers[0] * median_of_pixel_median[:,np.newaxis],
charge_deviation > self.charge_median_cut_outliers[1] * median_of_pixel_median[:,np.newaxis]))
# outliers from standard deviation
deviation = pixel_std - median_of_pixel_std[:, np.newaxis]
charge_std_outliers = (
np.logical_or(deviation < self.charge_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis],
deviation > self.charge_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis]))
return {
'relative_gain_median': np.ma.getdata(np.ma.median(relative_gain_event, axis=0)),
'relative_gain_mean': np.ma.getdata(np.ma.mean(relative_gain_event, axis=0)),
'relative_gain_std': np.ma.getdata(np.ma.std(relative_gain_event, axis=0)),
'charge_median': np.ma.getdata(pixel_median),
'charge_mean': np.ma.getdata(pixel_mean),
'charge_std': np.ma.getdata(pixel_std),
'charge_std_outliers': np.ma.getdata(charge_std_outliers),
'charge_median_outliers': np.ma.getdata(charge_median_outliers),
}
class SimpleEventWriter(Tool):
name = "ctapipe-simple-event-writer"
description = Unicode(__doc__)
infile = Path(
default_value=get_dataset_path(
"lst_prod3_calibration_and_mcphotons.simtel.zst"),
help="input file to read",
directory_ok=False,
exists=True,
).tag(config=True)
outfile = Path(help="output file name",
directory_ok=False,
default_value="output.h5").tag(config=True)
progress = Bool(help="display progress bar",
default_value=True).tag(config=True)
aliases = Dict({
"infile": "EventSource.input_url",
"outfile": "SimpleEventWriter.outfile",
"max-events": "EventSource.max_events",
"progress": "SimpleEventWriter.progress",
})
classes = List([EventSource, CameraCalibrator])
def setup(self):
self.log.info("Configure EventSource...")
self.event_source = EventSource.from_url(self.infile, parent=self)
self.calibrator = CameraCalibrator(subarray=self.event_source.subarray,
parent=self)
self.writer = HDF5TableWriter(filename=self.outfile,
group_name="image_infos",
overwrite=True,
parent=self)
def start(self):
self.log.info("Loop on events...")
for event in tqdm(
self.event_source,
desc="EventWriter",
total=self.event_source.max_events,
disable=~self.progress,
):
self.calibrator(event)
for tel_id in event.dl0.tel.keys():
geom = self.event_source.subarray.tel[tel_id].camera.geometry
dl1_tel = event.dl1.tel[tel_id]
# Image cleaning
image = dl1_tel.image # Waiting for automatic gain selection
mask = tailcuts_clean(geom,
image,
picture_thresh=10,
boundary_thresh=5)
cleaned = image.copy()
cleaned[~mask] = 0
# Image parametrisation
params = hillas_parameters(geom, cleaned)
# Save Ids, MC infos and Hillas informations
self.writer.write(geom.camera_name,
[event.r0, event.simulation.shower, params])
def finish(self):
self.log.info("End of job.")
self.writer.close()
class FlasherFlatFieldCalculator(FlatFieldCalculator):
"""Calculates flat-field parameters from flasher data
based on the best algorithm described by S. Fegan in MST-CAM-TN-0060 (eq. 19)
Pixels are defined as outliers on the base of a cut on the pixel charge median
over the full sample distribution and the pixel signal time inside the
waveform time
Parameters:
----------
charge_cut_outliers : List[2]
Interval of accepted charge values (fraction with respect to camera median value)
time_cut_outliers : List[2]
Interval (in waveform samples) of accepted time values
"""
charge_cut_outliers = List(
[-0.3, 0.3],
help=
'Interval of accepted charge values (fraction with respect to camera median value)'
).tag(config=True)
time_cut_outliers = List(
[0, 60],
help='Interval (in waveform samples) of accepted time values').tag(
config=True)
def __init__(self, **kwargs):
"""Calculates flat-field parameters from flasher data
based on the best algorithm described by S. Fegan in MST-CAM-TN-0060 (eq. 19)
Pixels are defined as outliers on the base of a cut on the pixel charge median
over the full sample distribution and the pixel signal time inside the
waveform time
Parameters:
----------
charge_cut_outliers : List[2]
Interval of accepted charge values (fraction with respect to camera median value)
time_cut_outliers : List[2]
Interval (in waveform samples) of accepted time values
"""
super().__init__(**kwargs)
self.log.info("Used events statistics : %d", self.sample_size)
# members to keep state in calculate_relative_gain()
self.num_events_seen = 0
self.time_start = None # trigger time of first event in sample
self.charge_medians = None # med. charge in camera per event in sample
self.charges = None # charge per event in sample
self.arrival_times = None # arrival time per event in sample
self.sample_masked_pixels = None # masked pixels per event in sample
def _extract_charge(self, event):
"""
Extract the charge and the time from a calibration event
Parameters
----------
event : general event container
"""
waveforms = event.r1.tel[self.tel_id].waveform
# Extract charge and time
charge = 0
peak_pos = 0
if self.extractor:
if self.extractor.requires_neighbors():
camera = event.inst.subarray.tel[self.tel_id].camera
self.extractor.neighbours = camera.neighbor_matrix_where
charge, peak_pos = self.extractor(waveforms)
return charge, peak_pos
def calculate_relative_gain(self, event):
"""
calculate the flatfield statistical values
and fill mon.tel[tel_id].flatfield container
Parameters
----------
event : general event container
"""
# initialize the np array at each cycle
waveform = event.r1.tel[self.tel_id].waveform
container = event.mon.tel[self.tel_id].flatfield
# re-initialize counter
if self.num_events_seen == self.sample_size:
self.num_events_seen = 0
# real data
if event.meta['origin'] != 'hessio':
trigger_time = event.r1.tel[self.tel_id].trigger_time
hardware_or_pedestal_mask = np.logical_or(
event.mon.tel[
self.tel_id].pixel_status.hardware_failing_pixels,
event.mon.tel[
self.tel_id].pixel_status.pedestal_failing_pixels)
pixel_mask = np.logical_or(
hardware_or_pedestal_mask, event.mon.tel[
self.tel_id].pixel_status.flatfield_failing_pixels)
else: # patches for MC data
if event.trig.tels_with_trigger:
trigger_time = event.trig.gps_time.unix
else:
trigger_time = 0
pixel_mask = np.zeros(waveform.shape[1], dtype=bool)
if self.num_events_seen == 0:
self.time_start = trigger_time
self.setup_sample_buffers(waveform, self.sample_size)
# extract the charge of the event and
# the peak position (assumed as time for the moment)
charge, arrival_time = self._extract_charge(event)
self.collect_sample(charge, pixel_mask, arrival_time)
sample_age = trigger_time - self.time_start
# check if to create a calibration event
if (sample_age > self.sample_duration
or self.num_events_seen == self.sample_size):
relative_gain_results = self.calculate_relative_gain_results(
self.charge_medians, self.charges, self.sample_masked_pixels)
time_results = self.calculate_time_results(
self.arrival_times, self.sample_masked_pixels, self.time_start,
trigger_time)
result = {
'n_events': self.num_events_seen,
**relative_gain_results,
**time_results,
}
for key, value in result.items():
setattr(container, key, value)
return True
else:
return False
def setup_sample_buffers(self, waveform, sample_size):
"""Initialize sample buffers"""
n_channels = waveform.shape[0]
n_pix = waveform.shape[1]
shape = (sample_size, n_channels, n_pix)
self.charge_medians = np.zeros((sample_size, n_channels))
self.charges = np.zeros(shape)
self.arrival_times = np.zeros(shape)
self.sample_masked_pixels = np.zeros(shape)
def collect_sample(self, charge, pixel_mask, arrival_time):
"""Collect the sample data"""
# extract the charge of the event and
# the peak position (assumed as time for the moment)
good_charge = np.ma.array(charge, mask=pixel_mask)
charge_median = np.ma.median(good_charge, axis=1)
self.charges[self.num_events_seen] = charge
self.arrival_times[self.num_events_seen] = arrival_time
self.sample_masked_pixels[self.num_events_seen] = pixel_mask
self.charge_medians[self.num_events_seen] = charge_median
self.num_events_seen += 1
def calculate_time_results(
self,
trace_time,
masked_pixels_of_sample,
time_start,
trigger_time,
):
"""Calculate and return the time results """
masked_trace_time = np.ma.array(trace_time,
mask=masked_pixels_of_sample)
# median over the sample per pixel
pixel_median = np.ma.median(masked_trace_time, axis=0)
# mean over the sample per pixel
pixel_mean = np.ma.mean(masked_trace_time, axis=0)
# std over the sample per pixel
pixel_std = np.ma.std(masked_trace_time, axis=0)
# median of the median over the camera
median_of_pixel_median = np.ma.median(pixel_median, axis=1)
# time outliers from median
relative_median = pixel_median - median_of_pixel_median[:, np.newaxis]
time_median_outliers = np.logical_or(
pixel_median < self.time_cut_outliers[0],
pixel_median > self.time_cut_outliers[1])
return {
'sample_time': (trigger_time - time_start) / 2 * u.s,
'sample_time_range': [time_start, trigger_time] * u.s,
'time_mean': np.ma.getdata(pixel_mean),
'time_median': np.ma.getdata(pixel_median),
'time_std': np.ma.getdata(pixel_std),
'relative_time_median': np.ma.getdata(relative_median),
'time_median_outliers': np.ma.getdata(time_median_outliers),
}
def calculate_relative_gain_results(
self,
event_median,
trace_integral,
masked_pixels_of_sample,
):
"""Calculate and return the sample statistics"""
masked_trace_integral = np.ma.array(trace_integral,
mask=masked_pixels_of_sample)
# median over the sample per pixel
pixel_median = np.ma.median(masked_trace_integral, axis=0)
# mean over the sample per pixel
pixel_mean = np.ma.mean(masked_trace_integral, axis=0)
# std over the sample per pixel
pixel_std = np.ma.std(masked_trace_integral, axis=0)
# median of the median over the camera
median_of_pixel_median = np.ma.median(pixel_median, axis=1)
# relative gain
relative_gain_event = masked_trace_integral / event_median[:, :,
np.newaxis]
# outliers from median
charge_deviation = pixel_median - median_of_pixel_median[:, np.newaxis]
charge_median_outliers = (np.logical_or(
charge_deviation < self.charge_cut_outliers[0] *
median_of_pixel_median[:, np.newaxis],
charge_deviation > self.charge_cut_outliers[1] *
median_of_pixel_median[:, np.newaxis]))
return {
'relative_gain_median':
np.ma.getdata(np.ma.median(relative_gain_event, axis=0)),
'relative_gain_mean':
np.ma.getdata(np.ma.mean(relative_gain_event, axis=0)),
'relative_gain_std':
np.ma.getdata(np.ma.std(relative_gain_event, axis=0)),
'charge_median':
np.ma.getdata(pixel_median),
'charge_mean':
np.ma.getdata(pixel_mean),
'charge_std':
np.ma.getdata(pixel_std),
'charge_median_outliers':
np.ma.getdata(charge_median_outliers),
}
class PedestalIntegrator(PedestalCalculator):
"""Calculates pedestal parameters integrating the charge of pedestal events:
the pedestal value corresponds to the charge estimated with the selected
charge extractor
The pixels are set as outliers on the base of a cut on the pixel charge median
over the pedestal sample and the pixel charge standard deviation over
the pedestal sample with respect to the camera median values
Parameters:
----------
charge_median_cut_outliers : List[2]
Interval (number of std) of accepted charge values around camera median value
charge_std_cut_outliers : List[2]
Interval (number of std) of accepted charge standard deviation around camera median value
"""
charge_median_cut_outliers = List(
[-3, 3],
help=
"Interval (number of std) of accepted charge values around camera median value",
).tag(config=True)
charge_std_cut_outliers = List(
[-3, 3],
help=
"Interval (number of std) of accepted charge standard deviation around camera median value",
).tag(config=True)
def __init__(self, **kwargs):
"""Calculates pedestal parameters integrating the charge of pedestal events:
the pedestal value corresponds to the charge estimated with the selected
charge extractor
The pixels are set as outliers on the base of a cut on the pixel charge median
over the pedestal sample and the pixel charge standard deviation over
the pedestal sample with respect to the camera median values
Parameters:
----------
charge_median_cut_outliers : List[2]
Interval (number of std) of accepted charge values around camera median value
charge_std_cut_outliers : List[2]
Interval (number of std) of accepted charge standard deviation around camera median value
"""
super().__init__(**kwargs)
self.log.info("Used events statistics : %d", self.sample_size)
# members to keep state in calculate_relative_gain()
self.num_events_seen = 0
self.time_start = None # trigger time of first event in sample
self.charge_medians = None # med. charge in camera per event in sample
self.charges = None # charge per event in sample
self.sample_masked_pixels = None # pixels tp be masked per event in sample
def _extract_charge(self, event) -> DL1CameraContainer:
"""
Extract the charge and the time from a pedestal event
Parameters
----------
event: ArrayEventContainer
general event container
Returns
-------
DL1CameraContainer
"""
waveforms = event.r1.tel[self.tel_id].waveform
selected_gain_channel = event.r1.tel[self.tel_id].selected_gain_channel
# Extract charge and time
if self.extractor:
return self.extractor(waveforms, self.tel_id,
selected_gain_channel)
else:
return DL1CameraContainer(image=0, peak_pos=0, is_valid=False)
def calculate_pedestals(self, event):
"""
calculate the pedestal statistical values from
the charge extracted from pedestal events
and fill the mon.tel[tel_id].pedestal container
Parameters
----------
event : general event container
"""
# initialize the np array at each cycle
waveform = event.r1.tel[self.tel_id].waveform
container = event.mon.tel[self.tel_id].pedestal
# re-initialize counter
if self.num_events_seen == self.sample_size:
self.num_events_seen = 0
# real data
trigger_time = event.trigger.time
if event.meta["origin"] != "hessio":
pixel_mask = event.mon.tel[
self.tel_id].pixel_status.hardware_failing_pixels
else: # patches for MC data
pixel_mask = np.zeros(waveform.shape[1], dtype=bool)
if self.num_events_seen == 0:
self.time_start = trigger_time
self.setup_sample_buffers(waveform, self.sample_size)
# extract the charge of the event and
# the peak position (assumed as time for the moment)
dl1: DL1CameraContainer = self._extract_charge(event)
if not dl1.is_valid:
return False
self.collect_sample(dl1.image, pixel_mask)
sample_age = trigger_time - self.time_start
# check if to create a calibration event
if (sample_age > self.sample_duration
or self.num_events_seen == self.sample_size):
pedestal_results = calculate_pedestal_results(
self, self.charges, self.sample_masked_pixels)
time_results = calculate_time_results(self.time_start,
trigger_time)
result = {
"n_events": self.num_events_seen,
**pedestal_results,
**time_results,
}
for key, value in result.items():
setattr(container, key, value)
return True
else:
return False
def setup_sample_buffers(self, waveform, sample_size):
"""Initialize sample buffers"""
n_channels = waveform.shape[0]
n_pix = waveform.shape[1]
shape = (sample_size, n_channels, n_pix)
self.charge_medians = np.zeros((sample_size, n_channels))
self.charges = np.zeros(shape)
self.sample_masked_pixels = np.zeros(shape)
def collect_sample(self, charge, pixel_mask):
"""Collect the sample data"""
good_charge = np.ma.array(charge, mask=pixel_mask)
charge_median = np.ma.median(good_charge, axis=1)
self.charges[self.num_events_seen] = charge
self.sample_masked_pixels[self.num_events_seen] = pixel_mask
self.charge_medians[self.num_events_seen] = charge_median
self.num_events_seen += 1
class FlasherFlatFieldCalculator(FlatFieldCalculator):
"""Calculates flat-field parameters from flasher data
based on the best algorithm described by S. Fegan in MST-CAM-TN-0060 (eq. 19)
Pixels are defined as outliers on the base of a cut on the pixel charge median
over the full sample distribution and the pixel signal time inside the
waveform time
Parameters:
----------
charge_cut_outliers : List[2]
Interval of accepted charge values (fraction with respect to camera median value)
time_cut_outliers : List[2]
Interval (in waveform samples) of accepted time values
"""
charge_median_cut_outliers = List(
[-0.3, 0.3],
help=
'Interval of accepted charge values (fraction with respect to camera median value)'
).tag(config=True)
time_cut_outliers = List(
[0, 60],
help='Interval (in waveform samples) of accepted time values').tag(
config=True)
charge_std_cut_outliers = List(
[-3, 3],
help=
'Interval (number of std) of accepted charge standard deviation around camera median value'
).tag(config=True)
time_calibration_path = Unicode(
None, allow_none=True,
help='Path to drs4 time calibration file').tag(config=True)
def __init__(self, subarray, **kwargs):
"""Calculates flat-field parameters from flasher data
based on the best algorithm described by S. Fegan in MST-CAM-TN-0060 (eq. 19)
Pixels are defined as outliers on the base of a cut on the pixel charge median
over the full sample distribution and the pixel signal time inside the
waveform time
Parameters:
----------
charge_cut_outliers : List[2]
Interval of accepted charge values (fraction with respect to camera median value)
time_cut_outliers : List[2]
Interval (in waveform samples) of accepted time values
"""
super().__init__(subarray, **kwargs)
self.log.info("Used events statistics : %d", self.sample_size)
# members to keep state in calculate_relative_gain()
self.num_events_seen = 0
self.time_start = None # trigger time of first event in sample
self.trigger_time = None # trigger time of present event
self.charge_medians = None # med. charge in camera per event in sample
self.charges = None # charge per event in sample
self.arrival_times = None # arrival time per event in sample
self.sample_masked_pixels = None # masked pixels per event in sample
if self.time_calibration_path is None:
self.time_corrector = None
else:
# look for calibration path otherwise
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
msg = f"Time calibration file {self.time_calibration_path} not found!"
raise IOError(msg)
def _extract_charge(self, event):
"""
Extract the charge and the time from a calibration event
Parameters
----------
event : general event container
"""
waveforms = event.r1.tel[self.tel_id].waveform
no_gain_selection = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype=np.int)
# Extract charge and time
charge = 0
peak_pos = 0
if self.extractor:
charge, peak_pos = self.extractor(waveforms, self.tel_id,
no_gain_selection)
# correct time with drs4 correction if available
if self.time_corrector:
peak_pos = self.time_corrector.get_corr_pulse(event, peak_pos)
return charge, peak_pos
def calculate_relative_gain(self, event):
"""
calculate the flatfield statistical values
and fill mon.tel[tel_id].flatfield container
Parameters
----------
event : general event container
Returns: True if the mon.tel[tel_id].flatfield is updated, False otherwise
"""
# initialize the np array at each cycle
waveform = event.r1.tel[self.tel_id].waveform
# re-initialize counter
if self.num_events_seen == self.sample_size:
self.num_events_seen = 0
pixel_mask = np.logical_or(
event.mon.tel[self.tel_id].pixel_status.hardware_failing_pixels,
event.mon.tel[self.tel_id].pixel_status.flatfield_failing_pixels)
# real data
if event.meta['origin'] != 'hessio':
self.trigger_time = event.r1.tel[self.tel_id].trigger_time
else: # patches for MC data
if event.trig.tels_with_trigger:
self.trigger_time = event.trig.gps_time.unix
else:
self.trigger_time = 0
if self.num_events_seen == 0:
self.time_start = self.trigger_time
self.setup_sample_buffers(waveform, self.sample_size)
# extract the charge of the event and
# the peak position (assumed as time for the moment)
charge, arrival_time = self._extract_charge(event)
# correct pulse time with drs4 corrections
self.collect_sample(charge, pixel_mask, arrival_time)
sample_age = self.trigger_time - self.time_start
# check if to create a calibration event
if (self.num_events_seen > 0
and (sample_age > self.sample_duration
or self.num_events_seen == self.sample_size)):
# update the monitoring container
self.store_results(event)
return True
else:
return False
def store_results(self, event):
"""
Store stastical results in monitoring container
Parameters
----------
event : general event container
"""
if self.num_events_seen == 0:
raise ValueError(
"No flat-field events in statistics, zero results")
container = event.mon.tel[self.tel_id].flatfield
# mask the part of the array not filled
self.sample_masked_pixels[self.num_events_seen:] = 1
relative_gain_results = self.calculate_relative_gain_results(
self.charge_medians, self.charges, self.sample_masked_pixels)
time_results = self.calculate_time_results(self.arrival_times,
self.sample_masked_pixels,
self.time_start,
self.trigger_time)
result = {
'n_events': self.num_events_seen,
**relative_gain_results,
**time_results,
}
for key, value in result.items():
setattr(container, key, value)
# update the flatfield mask
ff_charge_failing_pixels = np.logical_or(
container.charge_median_outliers, container.charge_std_outliers)
event.mon.tel[self.tel_id].pixel_status.flatfield_failing_pixels = \
np.logical_or(ff_charge_failing_pixels, container.time_median_outliers)
def setup_sample_buffers(self, waveform, sample_size):
"""Initialize sample buffers"""
n_channels = waveform.shape[0]
n_pix = waveform.shape[1]
shape = (sample_size, n_channels, n_pix)
self.charge_medians = np.zeros((sample_size, n_channels))
self.charges = np.zeros(shape)
self.arrival_times = np.zeros(shape)
self.sample_masked_pixels = np.zeros(shape)
def collect_sample(self, charge, pixel_mask, arrival_time):
"""Collect the sample data"""
# extract the charge of the event and
# the peak position (assumed as time for the moment)
good_charge = np.ma.array(charge, mask=pixel_mask)
charge_median = np.ma.median(good_charge, axis=1)
self.charges[self.num_events_seen] = charge
self.arrival_times[self.num_events_seen] = arrival_time
self.sample_masked_pixels[self.num_events_seen] = pixel_mask
self.charge_medians[self.num_events_seen] = charge_median
self.num_events_seen += 1
def calculate_time_results(
self,
trace_time,
masked_pixels_of_sample,
time_start,
trigger_time,
):
"""Calculate and return the time results """
masked_trace_time = np.ma.array(trace_time,
mask=masked_pixels_of_sample)
# median over the sample per pixel
pixel_median = np.ma.median(masked_trace_time, axis=0)
# mean over the sample per pixel
pixel_mean = np.ma.mean(masked_trace_time, axis=0)
# std over the sample per pixel
pixel_std = np.ma.std(masked_trace_time, axis=0)
# median of the median over the camera
median_of_pixel_median = np.ma.median(pixel_median, axis=1)
# time outliers from median
relative_median = pixel_median - median_of_pixel_median[:, np.newaxis]
time_median_outliers = np.logical_or(
pixel_median < self.time_cut_outliers[0],
pixel_median > self.time_cut_outliers[1])
return {
'sample_time': (trigger_time - time_start) / 2 * u.s,
'sample_time_min': time_start * u.s,
'sample_time_max': trigger_time * u.s,
'time_mean': np.ma.getdata(pixel_mean) * u.ns,
'time_median': np.ma.getdata(pixel_median) * u.ns,
'time_std': np.ma.getdata(pixel_std) * u.ns,
'relative_time_median': np.ma.getdata(relative_median) * u.ns,
'time_median_outliers': np.ma.getdata(time_median_outliers),
}
def calculate_relative_gain_results(
self,
event_median,
trace_integral,
masked_pixels_of_sample,
):
"""Calculate and return the sample statistics"""
masked_trace_integral = np.ma.array(trace_integral,
mask=masked_pixels_of_sample)
# median over the sample per pixel
pixel_median = np.ma.median(masked_trace_integral, axis=0)
# mean over the sample per pixel
pixel_mean = np.ma.mean(masked_trace_integral, axis=0)
# std over the sample per pixel
pixel_std = np.ma.std(masked_trace_integral, axis=0)
# median of the median over the camera
median_of_pixel_median = np.ma.median(pixel_median, axis=1)
# median of the std over the camera
median_of_pixel_std = np.ma.median(pixel_std, axis=1)
# std of the std over camera
std_of_pixel_std = np.ma.std(pixel_std, axis=1)
# relative gain
relative_gain_event = masked_trace_integral / event_median[:, :,
np.newaxis]
# outliers from median
charge_deviation = pixel_median - median_of_pixel_median[:, np.newaxis]
charge_median_outliers = (np.logical_or(
charge_deviation < self.charge_median_cut_outliers[0] *
median_of_pixel_median[:, np.newaxis],
charge_deviation > self.charge_median_cut_outliers[1] *
median_of_pixel_median[:, np.newaxis]))
# outliers from standard deviation
deviation = pixel_std - median_of_pixel_std[:, np.newaxis]
charge_std_outliers = (np.logical_or(
deviation <
self.charge_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis],
deviation >
self.charge_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis]))
return {
'relative_gain_median':
np.ma.getdata(np.ma.median(relative_gain_event, axis=0)),
'relative_gain_mean':
np.ma.getdata(np.ma.mean(relative_gain_event, axis=0)),
'relative_gain_std':
np.ma.getdata(np.ma.std(relative_gain_event, axis=0)),
'charge_median':
np.ma.getdata(pixel_median),
'charge_mean':
np.ma.getdata(pixel_mean),
'charge_std':
np.ma.getdata(pixel_std),
'charge_std_outliers':
np.ma.getdata(charge_std_outliers),
'charge_median_outliers':
np.ma.getdata(charge_median_outliers),
}
class CompB(Component):
classes = List([CompA])
b = Int().tag(config=True)
class CalibrationCalculator(Component):
"""
Parent class for the camera calibration calculators.
Fills the MonitoringCameraContainer on the base of calibration events
Parameters
----------
flatfield_calculator: lstchain.calib.camera.flatfield
The flatfield to use. If None, then FlatFieldCalculator
will be used by default.
pedestal_calculator: lstchain.calib.camera.pedestal
The pedestal to use. If None, then
PedestalCalculator will be used by default.
kwargs
"""
systematic_correction_path = Path(
default_value=None,
allow_none=True,
exists=True,
directory_ok=False,
help='Path to systematic correction file ',
).tag(config=True)
squared_excess_noise_factor = Float(
1.222,
help='Excess noise factor squared: 1+ Var(gain)/Mean(Gain)**2').tag(
config=True)
pedestal_product = traits.create_class_enum_trait(
PedestalCalculator, default_value='PedestalIntegrator')
flatfield_product = traits.create_class_enum_trait(
FlatFieldCalculator, default_value='FlasherFlatFieldCalculator')
classes = List([FlatFieldCalculator, PedestalCalculator] +
traits.classes_with_traits(FlatFieldCalculator) +
traits.classes_with_traits(PedestalCalculator))
def __init__(self, subarray, parent=None, config=None, **kwargs):
"""
Parent class for the camera calibration calculators.
Fills the MonitoringCameraContainer on the base of calibration events
Parameters
----------
flatfield_calculator: lstchain.calib.camera.flatfield
The flatfield to use. If None, then FlatFieldCalculator
will be used by default.
pedestal_calculator: lstchain.calib.camera.pedestal
The pedestal to use. If None, then
PedestalCalculator will be used by default.
"""
super().__init__(parent=parent, config=config, **kwargs)
if self.squared_excess_noise_factor <= 0:
msg = "Argument squared_excess_noise_factor must have a positive value"
raise ValueError(msg)
self.flatfield = FlatFieldCalculator.from_name(self.flatfield_product,
parent=self,
subarray=subarray)
self.pedestal = PedestalCalculator.from_name(self.pedestal_product,
parent=self,
subarray=subarray)
msg = "tel_id not the same for all calibration components"
if self.pedestal.tel_id != self.flatfield.tel_id:
raise ValueError(msg)
self.tel_id = self.flatfield.tel_id
# load systematic correction term B
self.quadratic_term = 0
if self.systematic_correction_path is not None:
try:
with h5py.File(self.systematic_correction_path, 'r') as hf:
self.quadratic_term = np.array(hf['B_term'])
except:
raise IOError(
f"Problem in reading quadratic term file {self.systematic_correction_path}"
)
self.log.debug(f"{self.pedestal}")
self.log.debug(f"{self.flatfield}")
class LSTCameraCalibrator(CameraCalibrator):
"""
Calibrator to handle the LST camera calibration chain, in order to fill
the DL1 data level in the event container.
"""
extractor_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
reducer_product = Unicode(
'NullDataVolumeReducer',
help='Name of the DataVolumeReducer to use').tag(config=True)
calibration_path = Path(
exists=True, directory_ok=False,
help='Path to LST calibration file').tag(config=True)
time_calibration_path = Path(
exists=True,
directory_ok=False,
help='Path to drs4 time calibration file').tag(config=True)
time_sampling_correction_path = Path(
exists=True,
directory_ok=False,
help='Path to time sampling correction file',
allow_none=True,
).tag(config=True)
allowed_tels = List(
[1], help='List of telescope to be calibrated').tag(config=True)
gain_threshold = Int(
4094, allow_none=True,
help='Threshold for the gain selection in ADC').tag(config=True)
charge_scale = List(
[1, 1],
help='Multiplicative correction factor for charge estimation [HG,LG]'
).tag(config=True)
def __init__(self, subarray, **kwargs):
"""
Parameters
----------
reducer_product : ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use. If None, then
NullDataVolumeReducer will be used by default, and waveforms
will not be reduced.
extractor_product : ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, then LocalPeakWindowSum
will be used by default.
calibration_path :
Path to LST calibration file to get the pedestal and flat-field corrections
kwargs
"""
super().__init__(subarray, **kwargs)
# load the waveform charge extractor
self.image_extractor = ImageExtractor.from_name(self.extractor_product,
subarray=self.subarray,
config=self.config)
self.log.info(f"extractor {self.extractor_product}")
print("EXTRACTOR", self.image_extractor)
self.data_volume_reducer = DataVolumeReducer.from_name(
self.reducer_product, subarray=self.subarray, config=self.config)
self.log.info(f" {self.reducer_product}")
# declare gain selector if the threshold is defined
if self.gain_threshold:
self.gain_selector = gainselection.ThresholdGainSelector(
threshold=self.gain_threshold)
# declare time calibrator if correction file exist
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
raise IOError(
f"Time calibration file {self.time_calibration_path} not found!"
)
# declare the charge sampling corrector
if self.time_sampling_correction_path is not None:
# search the file in resources if not found
if not os.path.exists(self.time_sampling_correction_path):
self.time_sampling_correction_path = resource_filename(
'lstchain',
f"resources/{self.time_sampling_correction_path}")
if os.path.exists(self.time_sampling_correction_path):
self.time_sampling_corrector = TimeSamplingCorrection(
time_sampling_correction_path=self.
time_sampling_correction_path)
else:
raise IOError(
f"Sampling correction file {self.time_sampling_correction_path} not found!"
)
else:
self.time_sampling_corrector = None
# calibration data container
self.mon_data = MonitoringContainer()
# initialize the MonitoringContainer() for the moment it reads it from a hdf5 file
self._initialize_correction()
self.log.info(f"Global charge scale {self.charge_scale}")
def _initialize_correction(self):
"""
Read the correction from hdf5 calibration file
"""
self.log.info(f"read {self.calibration_path}")
try:
with HDF5TableReader(self.calibration_path) as h5_table:
for telid in self.allowed_tels:
# read the calibration data
table = '/tel_' + str(telid) + '/calibration'
next(
h5_table.read(table,
self.mon_data.tel[telid].calibration))
# read pedestal data
table = '/tel_' + str(telid) + '/pedestal'
next(
h5_table.read(table,
self.mon_data.tel[telid].pedestal))
# read flat-field data
table = '/tel_' + str(telid) + '/flatfield'
next(
h5_table.read(table,
self.mon_data.tel[telid].flatfield))
# read the pixel_status container
table = '/tel_' + str(telid) + '/pixel_status'
next(
h5_table.read(table,
self.mon_data.tel[telid].pixel_status))
except Exception:
self.log.exception(
f"Problem in reading calibration file {self.calibration_path}")
raise
def _calibrate_dl0(self, event, telid):
"""
create dl0 level, with gain-selected and calibrated waveform
"""
waveforms = event.r1.tel[telid].waveform
if self._check_r1_empty(waveforms):
return
# if not already done, initialize the event monitoring containers
if event.mon.tel[telid].calibration.dc_to_pe is None:
event.mon.tel[telid].calibration = self.mon_data.tel[
telid].calibration
event.mon.tel[telid].flatfield = self.mon_data.tel[telid].flatfield
event.mon.tel[telid].pedestal = self.mon_data.tel[telid].pedestal
event.mon.tel[telid].pixel_status = self.mon_data.tel[
telid].pixel_status
#
# subtract the pedestal per sample and multiply for the calibration coefficients
#
calibrated_waveform = (
(waveforms - self.mon_data.tel[telid].calibration.
pedestal_per_sample[:, :, np.newaxis]) *
self.mon_data.tel[telid].calibration.dc_to_pe[:, :,
np.newaxis]).astype(
np.float32)
# If requested, perform gain selection (this will be done by the EvB in future)
# find the gain selection mask
if waveforms.ndim == 3:
# if threshold defined, perform gain selection
if self.gain_threshold:
gain_mask = self.gain_selector(waveforms)
# select the samples
calibrated_waveform = calibrated_waveform[
gain_mask, np.arange(waveforms.shape[1])]
else:
# keep both HG and LG
gain_mask = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype=np.int64)
gain_mask[1] = 1
else:
# gain selection already performed in EvB: (0=HG, 1=LG)
gain_mask = event.lst.tel[telid].evt.pixel_status >> 2 & 1
# remember the calibrated and gain selected waveform
# (this should be the r1 waveform to be compliant with ctapipe (?))
event.dl0.tel[telid].waveform = calibrated_waveform
# remember which channel has been selected
event.r1.tel[telid].selected_gain_channel = gain_mask
event.dl0.tel[telid].selected_gain_channel = gain_mask
def _calibrate_dl1(self, event, telid):
"""
create calibrated dl1 image and calibrate it
"""
n_pixels = self.subarray.tels[telid].camera.geometry.n_pixels
# copy the waveform be cause I do not want to change it
waveforms = np.copy(event.dl0.tel[telid].waveform)
gain_mask = event.dl0.tel[telid].selected_gain_channel
if self._check_dl0_empty(waveforms):
return
# In case of no gain selection the selected gain channels are [0,0,..][1,1,..]
no_gain_selection = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype=np.int64)
no_gain_selection[1] = 1
# correct the dl0 waveform for the sampling time corrections
if self.time_sampling_corrector:
waveforms *= self.time_sampling_corrector.get_corrections(
event, telid)[gain_mask, np.arange(n_pixels)]
# extract the charge
charge, peak_time = self.image_extractor(waveforms, telid, gain_mask)
# correct charge for global scale
corrected_charge = charge * np.array(self.charge_scale,
dtype=np.float32)[gain_mask]
# correct time with drs4 correction if available
if self.time_corrector:
peak_time_drs4_corrected = (
peak_time - self.time_corrector.get_pulse_time_corrections(
event)[gain_mask, np.arange(n_pixels)])
# add flat-fielding time correction
peak_time_ff_corrected = (
peak_time_drs4_corrected +
self.mon_data.tel[telid].calibration.time_correction.value[
gain_mask, np.arange(n_pixels)])
# fill dl1 container
event.dl1.tel[telid].image = corrected_charge
event.dl1.tel[telid].peak_time = peak_time_ff_corrected.astype(
np.float32)
class LSTCameraCalibrator(CameraCalibrator):
"""
Calibrator to handle the LST camera calibration chain, in order to fill
the DL1 data level in the event container.
"""
extractor_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
reducer_product = Unicode(
'NullDataVolumeReducer',
help='Name of the DataVolumeReducer to use').tag(config=True)
calibration_path = Unicode(
'', help='Path to LST calibration file').tag(config=True)
time_calibration_path = Unicode(
'', help='Path to drs4 time calibration file').tag(config=True)
allowed_tels = List(
[1], help='List of telescope to be calibrated').tag(config=True)
gain_threshold = Int(
4094, allow_none=True,
help='Threshold for the gain selection in ADC').tag(config=True)
charge_scale = List(
[1, 1],
help='Multiplicative correction factor for charge estimation [HG,LG]'
).tag(config=True)
def __init__(self, subarray, **kwargs):
"""
Parameters
----------
reducer_product : ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use. If None, then
NullDataVolumeReducer will be used by default, and waveforms
will not be reduced.
extractor_product : ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, then LocalPeakWindowSum
will be used by default.
calibration_path :
Path to LST calibration file to get the pedestal and flat-field corrections
kwargs
"""
super().__init__(subarray, **kwargs)
# load the waveform charge extractor
self.image_extractor = ImageExtractor.from_name(self.extractor_product,
subarray=self.subarray,
config=self.config)
self.log.info(f"extractor {self.extractor_product}")
print("EXTRACTOR", self.image_extractor)
self.data_volume_reducer = DataVolumeReducer.from_name(
self.reducer_product, subarray=self.subarray, config=self.config)
self.log.info(f" {self.reducer_product}")
# declare gain selector if the threshold is defined
if self.gain_threshold:
self.gain_selector = gainselection.ThresholdGainSelector(
threshold=self.gain_threshold)
# declare time calibrator if correction file exist
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
raise IOError(
f"Time calibration file {self.time_calibration_path} not found!"
)
# calibration data container
self.mon_data = MonitoringContainer()
# initialize the MonitoringContainer() for the moment it reads it from a hdf5 file
self._initialize_correction()
self.log.info(f"Global charge scale {self.charge_scale}")
def _initialize_correction(self):
"""
Read the correction from hdf5 calibration file
"""
self.mon_data.tels_with_data = self.allowed_tels
self.log.info(f"read {self.calibration_path}")
try:
with HDF5TableReader(self.calibration_path) as h5_table:
for telid in self.allowed_tels:
# read the calibration data
table = '/tel_' + str(telid) + '/calibration'
next(
h5_table.read(table,
self.mon_data.tel[telid].calibration))
# read pedestal data
table = '/tel_' + str(telid) + '/pedestal'
next(
h5_table.read(table,
self.mon_data.tel[telid].pedestal))
# read flat-field data
table = '/tel_' + str(telid) + '/flatfield'
next(
h5_table.read(table,
self.mon_data.tel[telid].flatfield))
# read the pixel_status container
table = '/tel_' + str(telid) + '/pixel_status'
next(
h5_table.read(table,
self.mon_data.tel[telid].pixel_status))
except Exception:
self.log.exception(
f"Problem in reading calibration file {self.calibration_path}")
raise
def _calibrate_dl0(self, event, telid):
"""
create dl0 level, for the moment copy the r1
"""
waveforms = event.r1.tel[telid].waveform
if self._check_r1_empty(waveforms):
return
# if not already done, initialize the event monitoring containers
if event.mon.tel[telid].calibration.dc_to_pe is None:
event.mon.tel[telid].calibration = self.mon_data.tel[
telid].calibration
event.mon.tel[telid].flatfield = self.mon_data.tel[telid].flatfield
event.mon.tel[telid].pedestal = self.mon_data.tel[telid].pedestal
event.mon.tel[telid].pixel_status = self.mon_data.tel[
telid].pixel_status
#
# subtract the pedestal per sample and multiply for the calibration coefficients
#
event.dl0.tel[telid].waveform = (
(waveforms - self.mon_data.tel[telid].calibration.
pedestal_per_sample[:, :, np.newaxis]) *
self.mon_data.tel[telid].calibration.dc_to_pe[:, :,
np.newaxis]).astype(
np.float32)
def _calibrate_dl1(self, event, telid):
"""
create calibrated dl1 image and calibrate it
"""
waveforms = event.dl0.tel[telid].waveform
if self._check_dl0_empty(waveforms):
return
# for the moment we do the gain selection afterwards
# use gain mask without gain selection
# TBD: - perform calibration of the R1 waveform (not DL1)
# - gain selection before charge integration
# In case of no gain selection the selected gain channels are [0,0,..][1,1,..]
no_gain_selection = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype=np.int)
no_gain_selection[1] = 1
charge = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype='float32')
peak_time = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype='float32')
# image extraction for each channel:
for i in range(waveforms.shape[0]):
charge[i], peak_time[i] = self.image_extractor(
waveforms[i], telid, no_gain_selection[i])
# correct charge for global scale
corrected_charge = charge * np.array(self.charge_scale,
dtype=np.float32)[:, np.newaxis]
# correct time with drs4 correction if available
if self.time_corrector:
peak_time = self.time_corrector.get_corr_pulse(event, peak_time)
# add flat-fielding time correction
peak_time_ff_corrected = peak_time + self.mon_data.tel[
telid].calibration.time_correction.value
# perform the gain selection if the threshold is defined
if self.gain_threshold:
gain_mask = self.gain_selector(event.r1.tel[telid].waveform)
event.dl1.tel[telid].image = corrected_charge[
gain_mask, np.arange(charge.shape[1])]
event.dl1.tel[telid].peak_time = \
peak_time_ff_corrected[gain_mask, np.arange(peak_time_ff_corrected.shape[1])].astype(np.float32)
# remember which channel has been selected
event.r1.tel[telid].selected_gain_channel = gain_mask
# if threshold == None
else:
event.dl1.tel[telid].image = corrected_charge
event.dl1.tel[telid].peak_time = peak_time_ff_corrected
class LSTCameraCalibrator(CameraCalibrator):
"""
Calibrator to handle the LST camera calibration chain, in order to fill
the DL1 data level in the event container.
"""
extractor_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
reducer_product = Unicode(
'NullDataVolumeReducer',
help='Name of the DataVolumeReducer to use').tag(config=True)
calibration_path = Unicode(
'', help='Path to LST calibration file').tag(config=True)
time_calibration_path = Unicode(
'', help='Path to drs4 time calibration file').tag(config=True)
allowed_tels = List(
[1], help='List of telescope to be calibrated').tag(config=True)
gain_threshold = Int(
4094, allow_none=True,
help='Threshold for the gain selection in ADC').tag(config=True)
def __init__(self, **kwargs):
"""
Parameters
----------
reducer_product : ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use. If None, then
NullDataVolumeReducer will be used by default, and waveforms
will not be reduced.
extractor_product : ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, then LocalPeakWindowSum
will be used by default.
calibration_path :
Path to LST calibration file to get the pedestal and flat-field corrections
kwargs
"""
super().__init__(**kwargs)
# load the waveform charge extractor
self.image_extractor = ImageExtractor.from_name(self.extractor_product,
config=self.config)
self.log.info(f"extractor {self.extractor_product}")
print("EXTRACTOR", self.image_extractor)
self.data_volume_reducer = DataVolumeReducer.from_name(
self.reducer_product, config=self.config)
self.log.info(f" {self.reducer_product}")
# declare gain selector if the threshold is defined
if self.gain_threshold:
self.gain_selector = gainselection.ThresholdGainSelector(
threshold=self.gain_threshold)
# declare time calibrator if correction file exist
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
raise IOError(
f"Time calibration file {self.time_calibration_path} not found!"
)
# calibration data container
self.mon_data = MonitoringContainer()
# initialize the MonitoringContainer() for the moment it reads it from a hdf5 file
self._initialize_correction()
def _initialize_correction(self):
"""
Read the correction from hdf5 calibration file
"""
self.mon_data.tels_with_data = self.allowed_tels
self.log.info(f"read {self.calibration_path}")
try:
with HDF5TableReader(self.calibration_path) as h5_table:
for telid in self.allowed_tels:
# read the calibration data
table = '/tel_' + str(telid) + '/calibration'
next(
h5_table.read(table,
self.mon_data.tel[telid].calibration))
# read pedestal data
table = '/tel_' + str(telid) + '/pedestal'
next(
h5_table.read(table,
self.mon_data.tel[telid].pedestal))
# read flat-field data
table = '/tel_' + str(telid) + '/flatfield'
next(
h5_table.read(table,
self.mon_data.tel[telid].flatfield))
# read the pixel_status container
table = '/tel_' + str(telid) + '/pixel_status'
next(
h5_table.read(table,
self.mon_data.tel[telid].pixel_status))
except Exception:
self.log.exception(
f"Problem in reading calibration file {self.calibration_path}")
raise
def _calibrate_dl0(self, event, telid):
"""
create dl0 level, for the moment copy the r1
"""
waveforms = event.r1.tel[telid].waveform
if self._check_r1_empty(waveforms):
return
event.dl0.event_id = event.r1.event_id
# if not already done, initialize the event monitoring containers
if event.mon.tel[telid].calibration.dc_to_pe is None:
event.mon.tel[telid].calibration = self.mon_data.tel[
telid].calibration
event.mon.tel[telid].flatfield = self.mon_data.tel[telid].flatfield
event.mon.tel[telid].pedestal = self.mon_data.tel[telid].pedestal
event.mon.tel[telid].pixel_status = self.mon_data.tel[
telid].pixel_status
#
# subtract the pedestal per sample and multiply for the calibration coefficients
#
event.dl0.tel[telid].waveform = (
(waveforms - self.mon_data.tel[telid].calibration.
pedestal_per_sample[:, :, np.newaxis]) *
self.mon_data.tel[telid].calibration.dc_to_pe[:, :, np.newaxis])
def _calibrate_dl1(self, event, telid):
"""
create calibrated dl1 image and calibrate it
"""
waveforms = event.dl0.tel[telid].waveform
if self._check_dl0_empty(waveforms):
return
if self.image_extractor.requires_neighbors():
camera = event.inst.subarray.tel[telid].camera
self.image_extractor.neighbors = camera.neighbor_matrix_where
charge, pulse_time = self.image_extractor(waveforms)
# correct time with drs4 correction if available
if self.time_corrector:
pulse_time = self.time_corrector.get_corr_pulse(event, pulse_time)
# add flat-fielding time correction
pulse_time_ff_corrected = pulse_time + self.mon_data.tel[
telid].calibration.time_correction
# perform the gain selection if the threshold is defined
if self.gain_threshold:
waveforms, gain_mask = self.gain_selector(
event.r1.tel[telid].waveform)
event.dl1.tel[telid].image = charge[gain_mask,
np.arange(charge.shape[1])]
event.dl1.tel[telid].pulse_time = pulse_time_ff_corrected[
gain_mask,
np.arange(pulse_time_ff_corrected.shape[1])]
# remember which channel has been selected
event.r1.tel[telid].selected_gain_channel = gain_mask
# if threshold == None
else:
event.dl1.tel[telid].image = charge
event.dl1.tel[telid].pulse_time = pulse_time_ff_corrected
class PedestalIntegrator(PedestalCalculator):
"""Calculates pedestal parameters integrating the charge of pedestal events:
the pedestal value corresponds to the charge estimated with the selected
charge extractor
The pixels are set as outliers on the base of a cut on the pixel charge median
over the pedestal sample and the pixel charge standard deviation over
the pedestal sample with respect to the camera median values
Parameters:
----------
charge_median_cut_outliers : List[2]
Interval (number of std) of accepted charge values around camera median value
charge_std_cut_outliers : List[2]
Interval (number of std) of accepted charge standard deviation around camera median value
"""
charge_median_cut_outliers = List(
[-3, 3],
help=
'Interval (number of std) of accepted charge values around camera median value'
).tag(config=True)
charge_std_cut_outliers = List(
[-3, 3],
help=
'Interval (number of std) of accepted charge standard deviation around camera median value'
).tag(config=True)
def __init__(self, **kwargs):
"""Calculates pedestal parameters integrating the charge of pedestal events:
the pedestal value corresponds to the charge estimated with the selected
charge extractor
The pixels are set as outliers on the base of a cut on the pixel charge median
over the pedestal sample and the pixel charge standard deviation over
the pedestal sample with respect to the camera median values
Parameters:
----------
charge_median_cut_outliers : List[2]
Interval (number of std) of accepted charge values around camera median value
charge_std_cut_outliers : List[2]
Interval (number of std) of accepted charge standard deviation around camera median value
"""
super().__init__(**kwargs)
self.log.info("Used events statistics : %d", self.sample_size)
# members to keep state in calculate_relative_gain()
self.num_events_seen = 0
self.time_start = None # trigger time of first event in sample
self.trigger_time = None # trigger time of present event
self.charge_medians = None # med. charge in camera per event in sample
self.charges = None # charge per event in sample
self.sample_masked_pixels = None # pixels tp be masked per event in sample
def _extract_charge(self, event):
"""
Extract the charge and the time from a pedestal event
Parameters
----------
event : general event container
"""
waveforms = event.r1.tel[self.tel_id].waveform
# Extract charge and time
charge = 0
peak_pos = 0
if self.extractor:
if self.extractor.requires_neighbors():
camera = event.inst.subarray.tel[self.tel_id].camera
self.extractor.neighbours = camera.neighbor_matrix_where
charge, peak_pos = self.extractor(waveforms)
return charge, peak_pos
def calculate_pedestals(self, event):
"""
calculate the pedestal statistical values from
the charge extracted from pedestal events
and fill the mon.tel[tel_id].pedestal container
Parameters
----------
event : general event container
Returns: True if the mon.tel[tel_id].pedestal is updated, False otherwise
"""
# initialize the np array at each cycle
waveform = event.r1.tel[self.tel_id].waveform
# re-initialize counter
if self.num_events_seen == self.sample_size:
self.num_events_seen = 0
# real data
if event.meta['origin'] != 'hessio':
self.trigger_time = event.r1.tel[self.tel_id].trigger_time
pixel_mask = event.mon.tel[
self.tel_id].pixel_status.hardware_failing_pixels
else: # patches for MC data
if event.trig.tels_with_trigger:
self.trigger_time = event.trig.gps_time.unix
else:
self.trigger_time = 0
pixel_mask = np.zeros(waveform.shape[1], dtype=bool)
if self.num_events_seen == 0:
self.time_start = self.trigger_time
self.setup_sample_buffers(waveform, self.sample_size)
# extract the charge of the event and
# the peak position (assumed as time for the moment)
charge = self._extract_charge(event)[0]
self.collect_sample(charge, pixel_mask)
sample_age = self.trigger_time - self.time_start
# check if to create a calibration event
if (sample_age > self.sample_duration
or self.num_events_seen == self.sample_size):
# update the monitoring container
self.store_results(event)
return True
else:
return False
def store_results(self, event):
"""
Store statistical results in monitoring container
Parameters
----------
event : general event container
"""
if self.num_events_seen == 0:
raise ValueError("No pedestal events in statistics, zero results")
container = event.mon.tel[self.tel_id].pedestal
# mask the part of the array not filled
self.sample_masked_pixels[self.num_events_seen:] = 1
pedestal_results = calculate_pedestal_results(
self,
self.charges,
self.sample_masked_pixels,
)
time_results = calculate_time_results(
self.time_start,
self.trigger_time,
)
result = {
'n_events': self.num_events_seen,
**pedestal_results,
**time_results,
}
for key, value in result.items():
setattr(container, key, value)
# update pedestal mask
event.mon.tel[self.tel_id].pixel_status.pedestal_failing_pixels = \
np.logical_or(container.charge_median_outliers, container.charge_std_outliers)
def setup_sample_buffers(self, waveform, sample_size):
"""Initialize sample buffers"""
n_channels = waveform.shape[0]
n_pix = waveform.shape[1]
shape = (sample_size, n_channels, n_pix)
self.charge_medians = np.zeros((sample_size, n_channels))
self.charges = np.zeros(shape)
self.sample_masked_pixels = np.zeros(shape)
def collect_sample(self, charge, pixel_mask):
"""Collect the sample data"""
good_charge = np.ma.array(charge, mask=pixel_mask)
charge_median = np.ma.median(good_charge, axis=1)
self.charges[self.num_events_seen] = charge
self.sample_masked_pixels[self.num_events_seen] = pixel_mask
self.charge_medians[self.num_events_seen] = charge_median
self.num_events_seen += 1
class DL3Cuts(Component):
"""
Selection cuts for DL2 to DL3 conversion
"""
global_gh_cut = Float(
help="Global selection cut for gh_score (gammaness)",
default_value=0.6,
).tag(config=True)
gh_efficiency = Float(
help="Gamma efficiency for optimized g/h cuts in %",
default_value=0.95,
).tag(config=True)
theta_containment = Float(
help="Percentage containment region for theta cuts",
default_value=0.68,
).tag(config=True)
global_theta_cut = Float(
help="Global selection cut for theta",
default_value=0.2,
).tag(config=True)
global_alpha_cut = Float(
help="Global selection cut for alpha",
default_value=20,
).tag(config=True)
allowed_tels = List(
help="List of allowed LST telescope ids",
trait=Int(),
default_value=[1],
).tag(config=True)
def apply_global_gh_cut(self, data):
"""
Applying a global gammaness cut on a given data
"""
return data[data["gh_score"] > self.global_gh_cut]
def energy_dependent_gh_cuts(self,
data,
energy_bins,
min_value=0.1,
max_value=0.99,
smoothing=None,
min_events=10):
"""
Evaluating energy-dependent gammaness cuts, in a given
data, with provided reco energy bins, and other parameters to
pass to the pyirf.cuts.calculate_percentile_cut function
"""
gh_cuts = calculate_percentile_cut(
data["gh_score"],
data["reco_energy"],
bins=energy_bins,
min_value=min_value,
max_value=max_value,
fill_value=data["gh_score"].max(),
percentile=100 * (1 - self.gh_efficiency),
smoothing=smoothing,
min_events=min_events,
)
return gh_cuts
def apply_global_alpha_cut(self, data):
"""
Applying a global alpha cut on a given data
"""
return data[data["alpha"].to_value(u.deg) < self.global_alpha_cut]
def apply_energy_dependent_gh_cuts(self, data, gh_cuts):
"""
Applying a given energy-dependent gh cuts to a data file, along the reco
energy bins provided.
"""
data["selected_gh"] = evaluate_binned_cut(
data["gh_score"],
data["reco_energy"],
gh_cuts,
operator.ge,
)
return data[data["selected_gh"]]
def apply_global_theta_cut(self, data):
"""
Applying a global theta cut on a given data
"""
return data[data["theta"].to_value(u.deg) < self.global_theta_cut]
def energy_dependent_theta_cuts(self,
data,
energy_bins,
min_value=0.05 * u.deg,
fill_value=0.32 * u.deg,
max_value=0.32 * u.deg,
smoothing=None,
min_events=10):
"""
Evaluating an optimized energy-dependent theta cuts, in a given
data, with provided reco energy bins, and other parameters to
pass to the pyirf.cuts.calculate_percentile_cut function.
Note: Using too fine binning will result in too un-smooth cuts.
"""
theta_cuts = calculate_percentile_cut(
data["theta"],
data["reco_energy"],
bins=energy_bins,
min_value=min_value,
max_value=max_value,
fill_value=fill_value,
percentile=100 * self.theta_containment,
smoothing=smoothing,
min_events=min_events,
)
return theta_cuts
def apply_energy_dependent_theta_cuts(self, data, theta_cuts):
"""
Applying a given energy-dependent theta cuts to a data file, along the
reco energy bins provided.
"""
data["selected_theta"] = evaluate_binned_cut(
data["theta"],
data["reco_energy"],
theta_cuts,
operator.le,
)
return data[data["selected_theta"]]
def allowed_tels_filter(self, data):
"""
Applying a filter on telescopes used for observation.
"""
mask = np.zeros(len(data), dtype=bool)
for tel_id in self.allowed_tels:
mask |= data["tel_id"] == tel_id
return data[mask]
class SingleTelEventDisplay(Tool):
name = "ctapipe-display-televents"
description = Unicode(__doc__)
infile = Path(help="input file to read", exists=True,
directory_ok=False).tag(config=True)
tel = Int(help="Telescope ID to display", default_value=0).tag(config=True)
write = Bool(help="Write out images to PNG files",
default_value=False).tag(config=True)
clean = Bool(help="Apply image cleaning",
default_value=False).tag(config=True)
hillas = Bool(help="Apply and display Hillas parametrization",
default_value=False).tag(config=True)
samples = Bool(help="Show each sample",
default_value=False).tag(config=True)
display = Bool(help="Display results in interactive window",
default_value=True).tag(config=True)
delay = Float(help="delay between events in s",
default_value=0.01,
min=0.001).tag(config=True)
progress = Bool(help="display progress bar",
default_value=True).tag(config=True)
aliases = Dict({
"infile": "SingleTelEventDisplay.infile",
"tel": "SingleTelEventDisplay.tel",
"max-events": "EventSource.max_events",
"write": "SingleTelEventDisplay.write",
"clean": "SingleTelEventDisplay.clean",
"hillas": "SingleTelEventDisplay.hillas",
"samples": "SingleTelEventDisplay.samples",
"display": "SingleTelEventDisplay.display",
"delay": "SingleTelEventDisplay.delay",
"progress": "SingleTelEventDisplay.progress",
})
classes = List([EventSource, CameraCalibrator])
def __init__(self, **kwargs):
super().__init__(**kwargs)
def setup(self):
print("TOLLES INFILE", self.infile)
self.event_source = EventSource.from_url(self.infile, parent=self)
self.event_source.allowed_tels = {self.tel}
self.calibrator = CameraCalibrator(parent=self,
subarray=self.event_source.subarray)
self.log.info(f"SELECTING EVENTS FROM TELESCOPE {self.tel}")
def start(self):
disp = None
for event in tqdm(
self.event_source,
desc=f"Tel{self.tel}",
total=self.event_source.max_events,
disable=~self.progress,
):
self.log.debug(event.trigger)
self.log.debug(f"Energy: {event.simulation.shower.energy}")
self.calibrator(event)
if disp is None:
geom = self.event_source.subarray.tel[self.tel].camera.geometry
self.log.info(geom)
disp = CameraDisplay(geom)
# disp.enable_pixel_picker()
disp.add_colorbar()
if self.display:
plt.show(block=False)
# display the event
disp.axes.set_title("CT{:03d} ({}), event {:06d}".format(
self.tel, geom.camera_name, event.index.event_id))
if self.samples:
# display time-varying event
data = event.dl0.tel[self.tel].waveform
for ii in range(data.shape[1]):
disp.image = data[:, ii]
disp.set_limits_percent(70)
plt.suptitle(f"Sample {ii:03d}")
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f"CT{self.tel:03d}_EV{event.index.event_id:10d}"
f"_S{ii:02d}.png")
else:
# display integrated event:
im = event.dl1.tel[self.tel].image
if self.clean:
mask = tailcuts_clean(geom,
im,
picture_thresh=10,
boundary_thresh=7)
im[~mask] = 0.0
disp.image = im
if self.hillas:
try:
ellipses = disp.axes.findobj(Ellipse)
if len(ellipses) > 0:
ellipses[0].remove()
params = hillas_parameters(geom, image=im)
disp.overlay_moments(params,
color="pink",
lw=3,
with_label=False)
except HillasParameterizationError:
pass
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f"CT{self.tel:03d}_EV{event.index.event_id:010d}.png")
self.log.info("FINISHED READING DATA FILE")
if disp is None:
self.log.warning(
"No events for tel {} were found in {}. Try a "
"different EventIO file or another telescope".format(
self.tel, self.infile))
class LSTCameraCalibrator(CameraCalibrator):
"""
Calibrator to handle the LST camera calibration chain, in order to fill
the DL1 data level in the event container.
"""
extractor_product = Unicode(
'NeighborPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
reducer_product = Unicode(
'NullDataVolumeReducer',
help='Name of the DataVolumeReducer to use').tag(config=True)
calibration_path = Unicode(
'', allow_none=True,
help='Path to LST calibration file').tag(config=True)
allowed_tels = List(
[1], help='List of telescope to be calibrated').tag(config=True)
def __init__(self, **kwargs):
"""
Parameters
----------
reducer_product : ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use. If None, then
NullDataVolumeReducer will be used by default, and waveforms
will not be reduced.
extractor_product : ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, then NeighborPeakWindowSum
will be used by default.
calibration_path :
Path to LST calibration file to get the pedestal and flat-field corrections
kwargs
"""
super().__init__(**kwargs)
# load the waveform charge extractor
self.image_extractor = ImageExtractor.from_name(self.extractor_product,
config=self.config)
self.log.info(f"extractor {self.extractor_product}")
self.data_volume_reducer = DataVolumeReducer.from_name(
self.reducer_product, config=self.config)
self.log.info(f" {self.reducer_product}")
# calibration data container
self.mon_data = MonitoringContainer()
# initialize the MonitoringContainer() for the moment it reads it from a hdf5 file
self._initialize_correction()
def _initialize_correction(self):
"""
Read the correction from hdf5 calibration file
"""
self.mon_data.tels_with_data = self.allowed_tels
self.log.info(f"read {self.calibration_path}")
try:
with HDF5TableReader(self.calibration_path) as h5_table:
assert h5_table._h5file.isopen == True
for telid in self.allowed_tels:
# read the calibration data for the moment only one event
table = '/tel_' + str(telid) + '/calibration'
next(
h5_table.read(table,
self.mon_data.tel[telid].calibration))
# eliminate inf values (should be done probably before)
dc_to_pe = self.mon_data.tel[telid].calibration.dc_to_pe
dc_to_pe[np.isinf(dc_to_pe)] = 0
self.log.info(
f"read {self.mon_data.tel[telid].calibration.dc_to_pe}"
)
except:
self.log.error(
f"Problem in reading calibration file {self.calibration_path}")
def _calibrate_dl0(self, event, telid):
"""
create dl0 level, for the moment copy the r1
"""
waveforms = event.r1.tel[telid].waveform
if self._check_r1_empty(waveforms):
return
event.dl0.event_id = event.r1.event_id
event.mon.tel[telid].calibration = self.mon_data.tel[telid].calibration
# subtract the pedestal per sample (should we do it?) and multiply for the calibration coefficients
#
event.dl0.tel[telid].waveform = (
(event.r1.tel[telid].waveform - self.mon_data.tel[telid].
calibration.pedestal_per_sample[:, :, np.newaxis]) *
self.mon_data.tel[telid].calibration.dc_to_pe[:, :, np.newaxis])
def _calibrate_dl1(self, event, telid):
"""
create calibrated dl1 image and calibrate it
"""
waveforms = event.dl0.tel[telid].waveform
if self._check_dl0_empty(waveforms):
return
if self.image_extractor.requires_neighbors():
camera = event.inst.subarray.tel[telid].camera
self.image_extractor.neighbors = camera.neighbor_matrix_where
charge, pulse_time = self.image_extractor(waveforms)
event.dl0.event_id = event.r1.event_id
event.dl1.tel[telid].image = charge
event.dl1.tel[telid].pulse_time = pulse_time + self.mon_data.tel[
telid].calibration.time_correction
class PedestalIntegrator(PedestalCalculator):
"""Calculates pedestal parameters integrating the charge of pedestal events:
the pedestal value corresponds to the charge estimated with the selected
charge extractor
The pixels are set as outliers on the base of a cut on the pixel charge median
over the pedestal sample and the pixel charge standard deviation over
the pedestal sample with respect to the camera median values
Parameters:
----------
charge_median_cut_outliers : List[2]
Interval (number of std) of accepted charge values around camera median value
charge_std_cut_outliers : List[2]
Interval (number of std) of accepted charge standard deviation around camera median value
"""
charge_median_cut_outliers = List(
[-3, 3],
help=
'Interval (number of std) of accepted charge values around camera median value'
).tag(config=True)
charge_std_cut_outliers = List(
[-3, 3],
help=
'Interval (number of std) of accepted charge standard deviation around camera median value'
).tag(config=True)
time_sampling_correction_path = Path(
default_value=None,
allow_none=True,
directory_ok=False,
help='Path to time sampling correction file',
).tag(config=True)
def __init__(self, subarray, **kwargs):
"""Calculates pedestal parameters integrating the charge of pedestal events:
the pedestal value corresponds to the charge estimated with the selected
charge extractor
The pixels are set as outliers on the base of a cut on the pixel charge median
over the pedestal sample and the pixel charge standard deviation over
the pedestal sample with respect to the camera median values
Parameters:
----------
charge_median_cut_outliers : List[2]
Interval (number of std) of accepted charge values around camera median value
charge_std_cut_outliers : List[2]
Interval (number of std) of accepted charge standard deviation around camera median value
"""
super().__init__(subarray, **kwargs)
self.log.info("Used events statistics : %d", self.sample_size)
# members to keep state in calculate_relative_gain()
self.num_events_seen = 0
self.time_start = None # trigger time of first event in sample
self.trigger_time = None # trigger time of present event
self.charge_medians = None # med. charge in camera per event in sample
self.charges = None # charge per event in sample
self.sample_masked_pixels = None # pixels tp be masked per event in sample
# declare the charge sampling corrector
if self.time_sampling_correction_path is not None:
self.time_sampling_corrector = TimeSamplingCorrection(
time_sampling_correction_path=self.
time_sampling_correction_path)
else:
self.time_sampling_corrector = None
# fix for broken extractor setup in ctapipe baseclass
self.extractor = ImageExtractor.from_name(self.charge_product,
parent=self,
subarray=subarray)
def _extract_charge(self, event):
"""
Extract the charge and the time from a pedestal event
Parameters
----------
event : general event container
"""
# copy the waveform be cause we do not want to change it for the moment
waveforms = np.copy(event.r1.tel[self.tel_id].waveform)
# pedestal event do not have gain selection
no_gain_selection = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype=np.int64)
no_gain_selection[1] = 1
n_pixels = 1855
# correct the r1 waveform for the sampling time corrections
if self.time_sampling_corrector:
waveforms *= (self.time_sampling_corrector.get_corrections(
event, self.tel_id)[no_gain_selection,
np.arange(n_pixels)])
# Extract charge and time
charge = 0
peak_pos = 0
if self.extractor:
charge, peak_pos = self.extractor(waveforms, self.tel_id,
no_gain_selection)
return charge, peak_pos
def calculate_pedestals(self, event):
"""
calculate the pedestal statistical values from
the charge extracted from pedestal events
and fill the mon.tel[tel_id].pedestal container
Parameters
----------
event : general event container
Returns: True if the mon.tel[tel_id].pedestal is updated, False otherwise
"""
# initialize the np array at each cycle
waveform = event.r1.tel[self.tel_id].waveform
# re-initialize counter
if self.num_events_seen == self.sample_size:
self.num_events_seen = 0
pixel_mask = event.mon.tel[
self.tel_id].pixel_status.hardware_failing_pixels
self.trigger_time = event.trigger.time
if self.num_events_seen == 0:
self.time_start = self.trigger_time
self.setup_sample_buffers(waveform, self.sample_size)
# extract the charge of the event and
# the peak position (assumed as time for the moment)
charge = self._extract_charge(event)[0]
self.collect_sample(charge, pixel_mask)
sample_age = (self.trigger_time - self.time_start).to_value(u.s)
# check if to create a calibration event
if (self.num_events_seen > 0
and (sample_age > self.sample_duration
or self.num_events_seen == self.sample_size)):
# update the monitoring container
self.store_results(event)
return True
else:
return False
def store_results(self, event):
"""
Store statistical results in monitoring container
Parameters
----------
event : general event container
"""
# something wrong if you are here and no statistic is there
if self.num_events_seen == 0:
raise ValueError("No pedestal events in statistics, zero results")
container = event.mon.tel[self.tel_id].pedestal
# mask the part of the array not filled
self.sample_masked_pixels[self.num_events_seen:] = 1
pedestal_results = calculate_pedestal_results(
self,
self.charges,
self.sample_masked_pixels,
)
time_results = calculate_time_results(
self.time_start,
self.trigger_time,
)
result = {
'n_events': self.num_events_seen,
**pedestal_results,
**time_results,
}
for key, value in result.items():
setattr(container, key, value)
# update pedestal mask
event.mon.tel[self.tel_id].pixel_status.pedestal_failing_pixels = \
np.logical_or(container.charge_median_outliers, container.charge_std_outliers)
def setup_sample_buffers(self, waveform, sample_size):
"""Initialize sample buffers"""
n_channels = waveform.shape[0]
n_pix = waveform.shape[1]
shape = (sample_size, n_channels, n_pix)
self.charge_medians = np.zeros((sample_size, n_channels))
self.charges = np.zeros(shape)
self.sample_masked_pixels = np.zeros(shape)
def collect_sample(self, charge, pixel_mask):
"""Collect the sample data"""
good_charge = np.ma.array(charge, mask=pixel_mask)
charge_median = np.ma.median(good_charge, axis=1)
self.charges[self.num_events_seen] = charge
self.sample_masked_pixels[self.num_events_seen] = pixel_mask
self.charge_medians[self.num_events_seen] = charge_median
self.num_events_seen += 1
class ImageSumDisplayerTool(Tool):
description = Unicode(__doc__)
name = "ctapipe-display-imagesum"
infile = Path(
help="input simtelarray file",
default_value=get_dataset_path("gamma_test_large.simtel.gz"),
exists=True,
directory_ok=False,
).tag(config=True)
telgroup = Integer(help="telescope group number",
default_value=1).tag(config=True)
max_events = Integer(help="stop after this many events if non-zero",
default_value=0,
min=0).tag(config=True)
output_suffix = Unicode(
help="suffix (file extension) of output "
"filenames to write images "
"to (no writing is done if blank). "
"Images will be named [EVENTID][suffix]",
default_value="",
).tag(config=True)
aliases = Dict({
"infile": "ImageSumDisplayerTool.infile",
"telgroup": "ImageSumDisplayerTool.telgroup",
"max-events": "ImageSumDisplayerTool.max_events",
"output-suffix": "ImageSumDisplayerTool.output_suffix",
})
classes = List([CameraCalibrator, SimTelEventSource])
def setup(self):
# load up the telescope types table (need to first open a file, a bit of
# a hack until a proper instrument module exists) and select only the
# telescopes with the same camera type
# make sure gzip files are seekable
self.reader = SimTelEventSource(input_url=self.infile,
max_events=self.max_events,
back_seekable=True)
camtypes = self.reader.subarray.to_table().group_by("camera_type")
self.reader.subarray.info(printer=self.log.info)
group = camtypes.groups[self.telgroup]
self._selected_tels = list(group["tel_id"].data)
self._base_tel = self._selected_tels[0]
self.log.info(
"Telescope group %d: %s",
self.telgroup,
str(self.reader.subarray.tel[self._selected_tels[0]]),
)
self.log.info(f"SELECTED TELESCOPES:{self._selected_tels}")
self.calibrator = CameraCalibrator(parent=self,
subarray=self.reader.subarray)
self.reader.allowed_tels = self._selected_tels
def start(self):
geom = None
imsum = None
disp = None
for event in self.reader:
self.calibrator(event)
if geom is None:
geom = self.reader.subarray.tel[self._base_tel].camera.geometry
imsum = np.zeros(shape=geom.pix_x.shape, dtype=np.float)
disp = CameraDisplay(geom, title=geom.camera_name)
disp.add_colorbar()
disp.cmap = "viridis"
if len(event.dl0.tel.keys()) <= 2:
continue
imsum[:] = 0
for telid in event.dl0.tel.keys():
imsum += event.dl1.tel[telid].image
self.log.info("event={} ntels={} energy={}".format(
event.index.event_id,
len(event.dl0.tel.keys()),
event.simulation.shower.energy,
))
disp.image = imsum
plt.pause(0.1)
if self.output_suffix != "":
filename = "{:020d}{}".format(event.index.event_id,
self.output_suffix)
self.log.info(f"saving: '{filename}'")
plt.savefig(filename)
