class PedestalCalculator(Component):
"""
Parent class for the pedestal calculators.
Fills the MON.pedestal container on the base of
pedestal events (preliminary version)
"""
tel_id = Int(
0, help='id of the telescope to calculate the pedestal values').tag(
config=True)
sample_duration = Int(60,
help='sample duration in seconds').tag(config=True)
sample_size = Int(10000, help='sample size').tag(config=True)
n_channels = Int(2,
help='number of channels to be treated').tag(config=True)
charge_cut_outliers = List(
[3, 3], help='Interval (number of std) of accepted charge values').tag(
config=True)
charge_std_cut_outliers = List(
[3, 3],
help=
'Interval (number of std) of accepted charge standard deviation values'
).tag(config=True)
charge_product = Unicode(
'LocalPeakIntegrator',
help='Name of the charge extractor to be used').tag(config=True)
def __init__(self, **kwargs):
"""
Parent class for pedestal calculators.
Fills the MON.pedestal container.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
super().__init__(**kwargs)
# initialize the output
self.container = PedestalContainer()
# load the waveform charge extractor
self.extractor = ChargeExtractor.from_name(self.charge_product,
config=self.config)
self.log.info(f"extractor {self.extractor}")
@abstractmethod
def calculate_pedestals(self, event):
"""calculate relative gain from event
class FlatFieldCalculator(Component):
"""
Parent class for the flat field calculators.
Fills the MON.flatfield container.
"""
tel_id = Int(
0, help='id of the telescope to calculate the flat-field coefficients'
).tag(config=True)
sample_duration = Int(60,
help='sample duration in seconds').tag(config=True)
sample_size = Int(10000, help='sample size').tag(config=True)
n_channels = Int(2,
help='number of channels to be treated').tag(config=True)
charge_cut_outliers = List(
[-0.3, 0.3],
help=
'Interval of accepted charge values (fraction with respect to camera median value)'
).tag(config=True)
time_cut_outliers = List(
[10, 30],
help='Interval (in samples) of accepted time values').tag(config=True)
charge_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
def __init__(self, **kwargs):
"""
Parent class for the flat field calculators.
Fills the flatfield container.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
super().__init__(**kwargs)
# load the waveform charge extractor
self.extractor = ImageExtractor.from_name(self.charge_product,
config=self.config)
self.log.info(f"extractor {self.extractor}")
@abstractmethod
def calculate_relative_gain(self, event):
"""calculate relative gain from event
class CameraR0Calibrator(Component):
"""
The base R0-level calibrator. Change the r0 container.
The R0 calibrator performs the camera-specific R0 calibration that is
usually performed on the raw data by the camera server. This calibrator
exists in lstchain for testing and prototyping purposes.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool or None
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
tel_id = Int(1,
help='id of the telescope to calibrate'
).tag(config=True)
offset = Int(default_value=400,
help='Define the offset of the baseline').tag(config=True)
r1_sample_start = Int(default_value=3,
help='Start sample for r1 waveform',
allow_none=True).tag(config=True)
r1_sample_end = Int(default_value=39,
help='End sample for r1 waveform',
allow_none=True).tag(config=True)
def __init__(self, **kwargs):
"""
Parent class for the r0 calibrators. Change the r0 container.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool or None
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
super().__init__(**kwargs)
@abstractmethod
def calibrate(self, event):
"""
class DL3FixedCuts(Component):
"""
Temporary fixed selection cuts for DL2 to DL3 conversion
"""
fixed_gh_cut = Float(
help="Fixed selection cut for gh_score (gammaness)",
default_value=0.6,
).tag(config=True)
fixed_theta_cut = Float(
help="Fixed selection cut for theta",
default_value=0.2,
).tag(config=True)
allowed_tels = List(
help="List of allowed LST telescope ids",
trait=Int(),
default_value=[1],
).tag(config=True)
def gh_cut(self, data):
return data[data["gh_score"] > self.fixed_gh_cut]
def theta_cut(self, data):
return data[data["theta"].to_value(u.deg) < self.fixed_theta_cut]
def allowed_tels_filter(self, data):
mask = np.zeros(len(data), dtype=bool)
for tel_id in self.allowed_tels:
mask |= data["tel_id"] == tel_id
return data[mask]
class LSTR0Corrections(CameraR0Calibrator):
"""
The R0 calibrator class for LST Camera.
"""
pedestal_path = Unicode(
'', allow_none=True,
help='Path to the LST pedestal binary file').tag(config=True)
offset = Int(300,
help='Define the offset of the baseline').tag(config=True)
tel_id = Int(0, help='id of the telescope to calibrate').tag(config=True)
def __init__(self, **kwargs):
"""
The R0 calibrator for LST data.
Fill the r1 container.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
super().__init__(**kwargs)
self.n_module = 265
self.n_gain = 2
self.n_pix = 7
self.size4drs = 4 * 1024
self.roisize = 40
self.high_gain = 0
self.low_gain = 1
self.pedestal_value_array = np.zeros(
(self.n_gain, self.n_pix * self.n_module, self.size4drs + 40),
dtype=np.int16)
self.first_cap_array = np.zeros(
(self.n_module, self.n_gain, self.n_pix))
self.first_cap_time_lapse_array = np.zeros(
(self.n_module, self.n_gain, self.n_pix))
self.last_reading_time_array = np.zeros(
(self.n_module, self.n_gain, self.n_pix, self.size4drs))
self.first_cap_array_spike = np.zeros(
(self.n_module, self.n_gain, self.n_pix))
self.first_cap_old_array = np.zeros(
(self.n_module, self.n_gain, self.n_pix))
self._load_calib()
def calibrate(self, event):
for tel_id in event.r0.tels_with_data:
self.subtract_pedestal(event)
self.time_lapse_corr(event)
self.interpolate_spikes(event)
event.r1.tel[self.tel_id].trigger_type = event.r0.tel[
self.tel_id].trigger_type
event.r1.tel[self.tel_id].trigger_time = event.r1.tel[
self.tel_id].trigger_time
samples = event.r1.tel[tel_id].waveform[:, :,
self.r1_sample_start:self.
r1_sample_end]
event.r1.tel[tel_id].waveform = samples.astype('float32')
def subtract_pedestal(self, event):
"""
Subtract cell offset using pedestal file.
Fill the R1 container.
Parameters
----------
event : `ctapipe` event-container
"""
n_modules = event.lst.tel[self.tel_id].svc.num_modules
for nr_module in range(0, n_modules):
self.first_cap_array[nr_module, :, :] = self._get_first_capacitor(
event, nr_module)
expected_pixel_id = event.lst.tel[self.tel_id].svc.pixel_ids
samples = np.copy(event.r0.tel[self.tel_id].waveform)
samples.astype('int16')
samples = subtract_pedestal_jit(samples, expected_pixel_id,
self.first_cap_array,
self.pedestal_value_array, n_modules)
event.r1.tel[self.tel_id].trigger_type = event.r0.tel[
self.tel_id].trigger_type
event.r1.tel[self.tel_id].trigger_time = event.r1.tel[
self.tel_id].trigger_time
event.r1.tel[self.tel_id].waveform = samples[:, :, :]
def time_lapse_corr(self, event):
"""
Perform time lapse baseline corrections.
Fill the R1 container or
modifies R0 container
Parameters
----------
event : `ctapipe` event-container
"""
expected_pixel_id = event.lst.tel[self.tel_id].svc.pixel_ids
local_clock_list = event.lst.tel[self.tel_id].evt.local_clock_counter
n_modules = event.lst.tel[self.tel_id].svc.num_modules
for nr_module in range(0, n_modules):
self.first_cap_time_lapse_array[
nr_module, :, :] = self._get_first_capacitor(event, nr_module)
#If R1 container exist modifies it
if isinstance(event.r1.tel[self.tel_id].waveform, np.ndarray):
samples = event.r1.tel[self.tel_id].waveform
do_time_lapse_corr(samples, expected_pixel_id, local_clock_list,
self.first_cap_time_lapse_array,
self.last_reading_time_array, n_modules)
event.r1.tel[self.tel_id].trigger_type = event.r0.tel[
self.tel_id].trigger_type
event.r1.tel[self.tel_id].trigger_time = event.r1.tel[
self.tel_id].trigger_time
event.r1.tel[self.tel_id].waveform = samples[:, :, :]
else: # Modifies R0 container. This is for create pedestal file.
samples = np.copy(event.r0.tel[self.tel_id].waveform)
do_time_lapse_corr(samples, expected_pixel_id, local_clock_list,
self.first_cap_time_lapse_array,
self.last_reading_time_array, n_modules)
event.r0.tel[self.tel_id].waveform = samples[:, :, :]
def interpolate_spikes(self, event):
"""
Interpolates spike A & B.
Fill the R1 container.
Parameters
----------
event : `ctapipe` event-container
"""
self.first_cap_old_array[:, :, :] = self.first_cap_array_spike[:, :, :]
n_modules = event.lst.tel[self.tel_id].svc.num_modules
for nr_module in range(0, n_modules):
self.first_cap_array_spike[
nr_module, :, :] = self._get_first_capacitor(event, nr_module)
# Interpolate spikes should be done after pedestal subtraction and time lapse correction.
if isinstance(event.r1.tel[self.tel_id].waveform, np.ndarray):
waveform = event.r1.tel[self.tel_id].waveform[:, :, :]
expected_pixel_id = event.lst.tel[self.tel_id].svc.pixel_ids
samples = waveform.copy()
samples = samples.astype('int16')
event.r1.tel[
self.tel_id].waveform = self.interpolate_pseudo_pulses(
samples, expected_pixel_id, self.first_cap_array_spike,
self.first_cap_old_array, n_modules)
event.r1.tel[self.tel_id].trigger_type = event.r0.tel[
self.tel_id].trigger_type
event.r1.tel[self.tel_id].trigger_time = event.r1.tel[
self.tel_id].trigger_time
@staticmethod
@jit(parallel=True)
def interpolate_pseudo_pulses(waveform, expected_pixel_id, fc, fc_old,
n_modules):
"""
Interpolate Spike A & B.
Change waveform array.
Parameters
----------
waveform : ndarray
Waveform stored in a numpy array of shape
(n_gain, n_pix, n_samples).
expected_pixel_id: ndarray
Array stored expected pixel id
(n_pix*n_modules).
fc : ndarray
Value of first capacitor stored in a numpy array of shape
(n_clus, n_gain, n_pix).
fc_old : ndarray
Value of first capacitor from previous event
stored in a numpy array of shape
(n_clus, n_gain, n_pix).
n_modules : int
Number of modules
"""
roisize = 40
size4drs = 4096
n_gain = 2
n_pix = 7
for nr_module in prange(0, n_modules):
for gain in prange(0, n_gain):
for pix in prange(0, n_pix):
for k in prange(0, 4):
# looking for spike A first case
abspos = int(1024 - roisize - 2 -
fc_old[nr_module, gain, pix] + k * 1024 +
size4drs)
spike_A_position = int(
(abspos - fc[nr_module, gain, pix] + size4drs) %
size4drs)
if (spike_A_position > 2 and spike_A_position < 38):
pixel = expected_pixel_id[nr_module * 7 + pix]
interpolate_spike_A(waveform, gain,
spike_A_position, pixel)
# looking for spike A second case
abspos = int(roisize - 2 +
fc_old[nr_module, gain, pix] + k * 1024)
spike_A_position = int(
(abspos - fc[nr_module, gain, pix] + size4drs) %
size4drs)
if (spike_A_position > 2 and spike_A_position < 38):
pixel = expected_pixel_id[nr_module * 7 + pix]
interpolate_spike_A(waveform, gain,
spike_A_position, pixel)
# looking for spike B
spike_b_position = int(
(fc_old[nr_module, gain, pix] - 1 -
fc[nr_module, gain, pix] + 2 * size4drs) % size4drs)
if spike_b_position < roisize - 1:
pixel = expected_pixel_id[nr_module * 7 + pix]
interpolate_spike_B(waveform, gain, spike_b_position,
pixel)
return waveform
def _load_calib(self):
"""
Function to load pedestal file.
"""
if self.pedestal_path:
with fits.open(self.pedestal_path) as f:
pedestal_data = np.int16(f[1].data)
self.pedestal_value_array[:, :, :self.
size4drs] = pedestal_data - self.offset
self.pedestal_value_array[:, :, self.size4drs:self.size4drs + 40] \
= pedestal_data[:, :, 0:40] - self.offset
def _get_first_capacitor(self, event, nr_module):
"""
Get first capacitor values from event for nr module.
Parameters
----------
event : `ctapipe` event-container
nr_module : number of module
"""
fc = np.zeros((2, 7))
first_cap = event.lst.tel[
self.tel_id].evt.first_capacitor_id[nr_module * 8:(nr_module + 1) *
8]
# First capacitor order according Dragon v5 board data format
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [0, 0, 1, 1, 2, 2, 3]):
fc[self.high_gain, i] = first_cap[j]
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [4, 4, 5, 5, 6, 6, 7]):
fc[self.low_gain, i] = first_cap[j]
return fc
class EventSource(Component):
"""
Parent class for EventFileReaders of different sources.
A new EventFileReader should be created for each type of event file read
into ctapipe, e.g. sim_telarray files are read by the `SimTelEventSource`.
EventFileReader provides a common high-level interface for accessing event
information from different data sources (simulation or different camera
file formats). Creating an EventFileReader for a new
file format ensures that data can be accessed in a common way,
irregardless of the file format.
EventFileReader itself is an abstract class. To use an EventFileReader you
must use a subclass that is relevant for the file format you
are reading (for example you must use
`ctapipe.io.SimTelEventSource` to read a hessio format
file). Alternatively you can use `event_source()` to automatically
select the correct EventFileReader subclass for the file format you wish
to read.
To create an instance of an EventFileReader you must pass the traitlet
configuration (containing the input_url) and the
`ctapipe.core.tool.Tool`. Therefore from inside a Tool you would do:
>>> event_source = EventSource(self.config, self)
An example of how to use `ctapipe.core.tool.Tool` and `event_source()`
can be found in ctapipe/tools/display_dl1.py.
However if you are not inside a Tool, you can still create an instance and
supply an input_url via:
>>> event_source = EventSource( input_url="/path/to/file")
To loop through the events in a file:
>>> event_source = EventSource( input_url="/path/to/file")
>>> for event in event_source:
>>> print(event.count)
**NOTE**: Every time a new loop is started through the event_source, it restarts
from the first event.
Alternatively one can use EventFileReader in a `with` statement to ensure
the correct cleanups are performed when you are finished with the event_source:
>>> with EventSource( input_url="/path/to/file") as event_source:
>>> for event in event_source:
>>> print(event.count)
**NOTE**: The "event" that is returned from the generator is a pointer.
Any operation that progresses that instance of the generator further will
change the data pointed to by "event". If you wish to ensure a particular
event is kept, you should perform a `event_copy = copy.deepcopy(event)`.
Attributes
----------
input_url : str
Path to the input event file.
max_events : int
Maximum number of events to loop through in generator
metadata : dict
A dictionary containing the metadata of the file. This could include:
* is_simulation (bool indicating if the file contains simulated events)
* Telescope:Camera names (list if file contains multiple)
* Information in the file header
* Observation ID
"""
input_url = Unicode(
"", help="Path to the input file containing events.").tag(config=True)
max_events = Int(
None,
allow_none=True,
help="Maximum number of events that will be read from the file",
).tag(config=True)
allowed_tels = Set(
help=("list of allowed tel_ids, others will be ignored. "
"If left empty, all telescopes in the input stream "
"will be included")).tag(config=True)
def __init__(self, config=None, parent=None, **kwargs):
"""
Class to handle generic input files. Enables obtaining the "source"
generator, regardless of the type of file (either hessio or camera
file).
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
super().__init__(config=config, parent=parent, **kwargs)
self.metadata = dict(is_simulation=False)
input_url: Path = Path(self.input_url).expanduser()
if not input_url.exists:
raise FileNotFoundError(f"file path does not exist: '{input_url}'")
self.log.info(f"INPUT PATH = {input_url}")
if self.max_events:
self.log.info(f"Max events being read = {self.max_events}")
Provenance().add_input_file(input_url, role="DL0/Event")
@staticmethod
@abstractmethod
def is_compatible(file_path):
"""
Abstract method to be defined in child class.
Perform a set of checks to see if the input file is compatible
with this file event_source.
Parameters
----------
file_path : str
File path to the event file.
Returns
-------
compatible : bool
True if file is compatible, False if it is incompatible
"""
@property
def is_stream(self):
"""
Bool indicating if input is a stream. If it is then it is incompatible
with `ctapipe.io.eventseeker.EventSeeker`.
TODO: Define a method to detect if it is a stream
Returns
-------
bool
If True, then input is a stream.
"""
return False
@property
@abstractmethod
def subarray(self):
"""
Obtain the subarray from the EventSource
Returns
-------
ctapipe.instrument.SubarrayDecription
"""
@abstractmethod
def _generator(self):
"""
Abstract method to be defined in child class.
Generator where the filling of the `ctapipe.containers` occurs.
Returns
-------
generator
"""
def __iter__(self):
"""
Generator that iterates through `_generator`, but keeps track of
`self.max_events`.
Returns
-------
generator
"""
for event in self._generator():
yield event
if self.max_events and event.count >= self.max_events - 1:
break
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
pass
@classmethod
def from_url(cls, input_url, **kwargs):
"""
Find compatible EventSource for input_url via the `is_compatible`
method of the EventSource
Parameters
----------
input_url : str
Filename or URL pointing to an event file
kwargs
Named arguments for the EventSource
Returns
-------
instance
Instance of a compatible EventSource subclass
"""
if input_url == "" or input_url is None:
raise ToolConfigurationError(
"EventSource: No input_url was specified")
detect_and_import_io_plugins()
available_classes = non_abstract_children(cls)
for subcls in available_classes:
if subcls.is_compatible(input_url):
return subcls(input_url=input_url, **kwargs)
raise ValueError("Cannot find compatible EventSource for \n"
"\turl:{}\n"
"in available EventSources:\n"
"\t{}".format(input_url,
[c.__name__
for c in available_classes]))
@classmethod
def from_config(cls, config=None, parent=None, **kwargs):
"""
Find compatible EventSource for the EventSource.input_url traitlet
specified via the config.
This method is typically used in Tools, where the input_url is chosen via
the command line using the traitlet configuration system.
Parameters
----------
config : traitlets.config.loader.Config
Configuration created in the Tool
kwargs
Named arguments for the EventSource
Returns
-------
instance
Instance of a compatible EventSource subclass
"""
if config is None:
config = parent.config
if isinstance(config.EventSource.input_url, LazyConfigValue):
config.EventSource.input_url = cls.input_url.default_value
elif not isinstance(config.EventSource.input_url, str):
raise TraitError("Wrong type specified for input_url traitlet")
return event_source(config.EventSource.input_url,
config=config,
**kwargs)
class PointingPosition(Component):
"""
Pointion position of telescopes
"""
drive_path = Unicode(
'',
allow_none=True,
help='Path to the LST drive report file'
).tag(config=True)
tel_id = Int(
0,
help='id of the telescope to take drive report for'
).tag(config=True)
def _read_drive_report(self):
"""
Reading drive reports
Parameters:
-----------
str: drive report file
Returns:
data:`~astropy.table.Table`
A table of drive reports
"""
self.log.info("Drive report file:", self.drive_path)
if self.drive_path:
data = ascii.read(self.drive_path)
# Renaming the columns, since the drive report doesn't contain
# these information it self
data['col6'].name = 'time'
data['col8'].name = 'azimuth_avg'
data['col13'].name = 'zenith_avg'
return data
else:
raise Exception("No drive report file found")
def cal_pointingposition(self, ev_time, drive_data):
"""
Calculating pointing positions by interpolation
Parameters:
-----------
time: array
times from events
Drivereport: Container
a container filled with drive information
"""
drive_container = LSTDriveContainer()
drive_container.time = drive_data['time'].data
drive_container.azimuth_avg = drive_data['azimuth_avg'].data
drive_container.altitude_avg = 90.0 - drive_data['zenith_avg'].data
xp = drive_container.time
lower_drive_time = xp[xp < ev_time].max()
upper_drive_time = xp[xp > ev_time].min()
time_in_window = (xp >= lower_drive_time) & (xp <= upper_drive_time)
run_times = xp[time_in_window]
if len(run_times) > 1:
run_azimuth = drive_container.azimuth_avg[time_in_window]
run_altitude = drive_container.altitude_avg[time_in_window]
ev_azimuth = np.interp(ev_time, run_times, run_azimuth) * u.deg
ev_altitude = np.interp(ev_time, run_times, run_altitude) * u.deg
return ev_azimuth, ev_altitude
else:
raise Exception("No drive time in the range of event times")
class PedestalCalculator(Component):
"""
Parent class for the pedestal calculators.
Fills the MonitoringCameraContainer.PedestalContainer on the base of a given pedestal sample.
The sample is defined by a maximal interval of time (sample_duration) or a
minimal number of events (sample_duration).
The calculator is supposed to act in an event loop, extract and collect the
event charge and fill the PedestalContainer
Parameters
----------
tel_id : int
id of the telescope (default 0)
sample_duration : int
interval of time (s) used to gather the pedestal statistics
sample_size : int
number of pedestal events requested for the statistics
n_channels : int
number of waveform channel to be considered
charge_product : str
Name of the charge extractor to be used
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
kwargs
"""
tel_id = Int(
0,
help='id of the telescope to calculate the pedestal values'
).tag(config=True)
sample_duration = Int(
60,
help='sample duration in seconds'
).tag(config=True)
sample_size = Int(
10000,
help='sample size'
).tag(config=True)
n_channels = Int(
2,
help='number of channels to be treated'
).tag(config=True)
charge_product = Unicode(
'FixedWindowSum',
help='Name of the charge extractor to be used'
).tag(config=True)
def __init__(
self,
**kwargs
):
"""
Parent class for the pedestal calculators.
Fills the MonitoringCameraContainer.PedestalContainer on the base of a given pedestal sample.
The sample is defined by a maximal interval of time (sample_duration) or a
minimal number of events (sample_duration).
The calculator is supposed to act in an event loop, extract and collect the
event charge and fill the PedestalContainer
Parameters
----------
tel_id : int
id of the telescope (default 0)
sample_duration : int
interval of time (s) used to gather the pedestal statistics
sample_size : int
number of pedestal events requested for the statistics
n_channels : int
number of waveform channel to be considered
charge_product : str
Name of the charge extractor to be used
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
kwargs
"""
super().__init__(**kwargs)
# load the waveform charge extractor
self.extractor = ImageExtractor.from_name(
self.charge_product,
config=self.config
)
self.log.info(f"extractor {self.extractor}")
@abstractmethod
def calculate_pedestals(self, event):
"""
class TimeWaveformFitter(TelescopeComponent):
"""
Class used to perform event reconstruction by fitting of a model on waveforms.
"""
sigma_s = FloatTelescopeParameter(
default_value=1,
help='Width of the single photo-electron peak distribution.',
allow_none=False).tag(config=True)
crosstalk = FloatTelescopeParameter(default_value=0,
help='Average pixel crosstalk.',
allow_none=False).tag(config=True)
sigma_space = Float(
4,
help=
'Size of the region on which the fit is performed relative to the image extension.',
allow_none=False).tag(config=True)
sigma_time = Float(
3,
help=
'Time window on which the fit is performed relative to the image temporal extension.',
allow_none=False).tag(config=True)
time_before_shower = FloatTelescopeParameter(
default_value=10,
help='Additional time at the start of the fit temporal window.',
allow_none=False).tag(config=True)
time_after_shower = FloatTelescopeParameter(
default_value=20,
help='Additional time at the end of the fit temporal window.',
allow_none=False).tag(config=True)
use_weight = Bool(
False,
help=
'If True, the brightest sample is twice as important as the dimmest pixel in the '
'likelihood. If false all samples are equivalent.',
allow_none=False).tag(config=True)
no_asymmetry = Bool(
False,
help='If true, the asymmetry of the spatial model is fixed to 0.',
allow_none=False).tag(config=True)
use_interleaved = Path(
None,
help=
'Location of the dl1 file used to estimate the pedestal exploiting interleaved'
' events.',
allow_none=True).tag(config=True)
n_peaks = Int(
0,
help=
'Maximum brightness (p.e.) for which the full likelihood computation is used. '
'If the Poisson term for Np.e.>n_peak is more than 1e-6 a Gaussian approximation is used.',
allow_none=False).tag(config=True)
bound_charge_factor = FloatTelescopeParameter(
default_value=4,
help='Maximum relative change to the fitted charge parameter.',
allow_none=False).tag(config=True)
bound_t_cm_value = FloatTelescopeParameter(
default_value=10,
help='Maximum change to the t_cm parameter.',
allow_none=False).tag(config=True)
bound_centroid_control_parameter = FloatTelescopeParameter(
default_value=1,
help='Maximum change of the centroid coordinated in '
'number of seed length',
allow_none=False).tag(config=True)
bound_max_length_factor = FloatTelescopeParameter(
default_value=2,
help='Maximum relative increase to the fitted length parameter.',
allow_none=False).tag(config=True)
bound_length_asymmetry = FloatTelescopeParameter(
default_value=9,
help='Bounds for the fitted rl parameter.',
allow_none=False).tag(config=True)
bound_max_v_cm_factor = FloatTelescopeParameter(
default_value=2,
help='Maximum relative increase to the fitted v_cm parameter.',
allow_none=False).tag(config=True)
default_seed_t_cm = FloatTelescopeParameter(
default_value=0,
help='Default starting value of t_cm when the seed extraction failed.',
allow_none=False).tag(config=True)
default_seed_v_cm = FloatTelescopeParameter(
default_value=40,
help='Default starting value of v_cm when the seed extraction failed.',
allow_none=False).tag(config=True)
verbose = Int(
0,
help='4 - used for tests: create debug plots\n'
'3 - create debug plots, wait for input after each event, increase minuit verbose level\n'
'2 - create debug plots, increase minuit verbose level\n'
'1 - increase minuit verbose level\n'
'0 - silent',
allow_none=False).tag(config=True)
def __init__(self, subarray, config=None, parent=None, **kwargs):
super().__init__(subarray=subarray,
config=config,
parent=parent,
**kwargs)
self.subarray = subarray
self.template_dict = {}
self.template_time_of_max_dict = {}
for tel_id in subarray.tel:
self.template_dict[
tel_id] = NormalizedPulseTemplate.load_from_eventsource(
subarray.tel[tel_id].camera.readout)
self.template_time_of_max_dict[tel_id] = self.template_dict[
tel_id].compute_time_of_max()
poisson_peaks = np.arange(self.n_peaks + 1, dtype=int)
poisson_peaks[0] = 1
self.factorial = np.cumprod(poisson_peaks, dtype='u8')
# Find the transition charge between full likelihood computation and Gaussian approximation
# The maximum charge is selected such that each Poisson terms in the full likelihood computation
# above the n_peaks limit account for less than (1/n_peaks)%
transition_charges = {}
for config_crosstalk in self.crosstalk:
# if n_peaks is set to 0, only the Gaussian approximation is used
transition_charges[config_crosstalk[2]] = 0.0 if self.n_peaks == 0\
else self.find_transition_charge(config_crosstalk[2], 1e-2/self.n_peaks)
self.transition_charges = {}
for tel_id in subarray.tel:
self.transition_charges[tel_id] = transition_charges[
self.crosstalk.tel[tel_id]]
self.start_parameters = None
self.names_parameters = None
self.end_parameters = None
self.error_parameters = None
self.bound_parameters = None
self.fcn = None
def call_setup(self, event, telescope_id, dl1_container):
"""
Extract all event dependent quantities used for the fit.
Parameters
----------
event: ctapipe event container
Current event container.
telescope_id: int
Id of the telescope
dl1_container: DL1ParametersContainer
Contains the Hillas parameters used as seed for the fit
Returns
-------
focal_length: astropy.units.Quantity
Focal length of the telescope
fit_params: array
Array containing all the variable needed to compute the likelihood
during the fir excluding the model parameters
"""
geometry = self.subarray.tel[telescope_id].camera.geometry
unit = geometry.pix_x.unit
pix_x = geometry.pix_x.to_value(unit)
pix_y = geometry.pix_y.to_value(unit)
r_max = geometry.guess_radius().to_value(unit)
pix_radius = np.sqrt(geometry.pix_area[0].to_value(unit**2) /
np.pi) # find linear size of a pixel
readout = self.subarray.tel[telescope_id].camera.readout
sampling_rate = readout.sampling_rate.to_value(u.GHz)
dt = (1.0 / sampling_rate)
template = self.template_dict[telescope_id]
image = event.dl1.tel[telescope_id].image
hillas_signal_pixels = event.dl1.tel[telescope_id].image_mask
start_x_cm, start_y_cm = init_centroid(dl1_container,
geometry[hillas_signal_pixels],
unit,
image[hillas_signal_pixels],
self.no_asymmetry)
waveform = event.r1.tel[telescope_id].waveform
dl1_calib = event.calibration.tel[telescope_id].dl1
time_shift = dl1_calib.time_shift
# TODO check if this is correct here or if it is applied to r1 waveform earlier
if dl1_calib.pedestal_offset is not None:
waveform = waveform - dl1_calib.pedestal_offset[:, np.newaxis]
n_pixels, n_samples = waveform.shape
times = np.arange(0, n_samples) * dt
selected_gains = event.r1.tel[telescope_id].selected_gain_channel
is_high_gain = (selected_gains == 0)
# We assume that the time gradient is given in unit of 'geometry spatial unit'/ns
v = dl1_container.time_gradient
psi = dl1_container.psi.to_value(u.rad)
# We use only positive time gradients and psi is projected in [-pi,pi] from [-pi/2,pi/2]
if v < 0:
if psi >= 0:
psi = psi - np.pi
else:
psi = psi + np.pi
start_length = max(dl1_container.length.to_value(unit), pix_radius)
# With current likelihood computation, order and type of the parameters are important
start_parameters = {
'charge':
dl1_container.intensity,
't_cm':
dl1_container.intercept -
self.template_time_of_max_dict[telescope_id],
'x_cm':
start_x_cm.to_value(unit),
'y_cm':
start_y_cm.to_value(unit),
'length':
start_length,
'wl':
max(dl1_container.wl, 0.01),
'psi':
psi,
'v':
np.abs(v),
'rl':
0.0
}
# Temporal parameters extraction fails when cleaning select only 2 pixels, we use defaults values in this case
if np.isnan(start_parameters['t_cm']):
start_parameters['t_cm'] = self.default_seed_t_cm.tel[telescope_id]
if np.isnan(start_parameters['v']):
start_parameters['v'] = self.default_seed_v_cm.tel[telescope_id]
t_max = n_samples * dt
v_min, v_max = 0, max(
self.bound_max_v_cm_factor.tel[telescope_id] *
start_parameters['v'], 50)
rl_min, rl_max = -self.bound_length_asymmetry.tel[
telescope_id], self.bound_length_asymmetry.tel[telescope_id]
if self.no_asymmetry:
rl_min, rl_max = 0.0, 0.0
bound_centroid = self.bound_centroid_control_parameter.tel[
telescope_id] * start_length
bound_parameters = {
'charge': (dl1_container.intensity /
self.bound_charge_factor.tel[telescope_id],
dl1_container.intensity *
self.bound_charge_factor.tel[telescope_id]),
't_cm': (-self.bound_t_cm_value.tel[telescope_id],
t_max + self.bound_t_cm_value.tel[telescope_id]),
'x_cm': (start_x_cm.to_value(unit) - bound_centroid,
start_x_cm.to_value(unit) + bound_centroid),
'y_cm': (start_y_cm.to_value(unit) - bound_centroid,
start_y_cm.to_value(unit) + bound_centroid),
'length':
(pix_radius,
min(self.bound_max_length_factor.tel[telescope_id] * start_length,
r_max)),
'wl': (0.001, 1.0),
'psi': (-np.pi * 2.0, np.pi * 2.0),
'v': (v_min, v_max),
'rl': (rl_min, rl_max)
}
mask_pixel, mask_time = self.clean_data(pix_x, pix_y, pix_radius,
times, start_parameters,
telescope_id)
spatial_ones = np.ones(np.sum(mask_pixel))
is_high_gain = is_high_gain[mask_pixel]
sig_s = spatial_ones * self.sigma_s.tel[telescope_id]
crosstalks = spatial_ones * self.crosstalk.tel[telescope_id]
times = (np.arange(0, n_samples) * dt)[mask_time]
time_shift = time_shift[mask_pixel]
p_x = pix_x[mask_pixel]
p_y = pix_y[mask_pixel]
pix_area = geometry.pix_area[mask_pixel].to_value(unit**2)
data = waveform
error = None # TODO include option to use calibration data
filter_pixels = np.nonzero(~mask_pixel)
filter_times = np.nonzero(~mask_time)
if error is None:
std = np.std(data[~mask_pixel])
error = np.full(data.shape[0], std)
data = np.delete(data, filter_pixels, axis=0)
data = np.delete(data, filter_times, axis=1)
error = np.delete(error, filter_pixels, axis=0)
# Fill the set of non-fitted parameters needed to compute the likelihood. Order and type sensitive.
fit_params = [
data, error, is_high_gain, sig_s, crosstalks, times,
np.float32(time_shift), p_x, p_y,
np.float64(pix_area), template.dt, template.t0,
template.amplitude_LG, template.amplitude_HG, self.n_peaks,
self.transition_charges[telescope_id], self.use_weight,
self.factorial
]
self.start_parameters = start_parameters
self.names_parameters = start_parameters.keys()
self.bound_parameters = bound_parameters
return unit, fit_params
def __call__(self, event, telescope_id, dl1_container):
# setup angle to distance conversion on the camera plane for the current telescope
focal_length = self.subarray.tel[
telescope_id].optics.equivalent_focal_length
angle_dist_eq = [
(u.rad, u.m, lambda x: np.tan(x) * focal_length.to_value(u.m),
lambda x: np.arctan(x / focal_length.to_value(u.m))),
(u.rad**2, u.m**2, lambda x:
(np.tan(np.sqrt(x)) * focal_length.to_value(u.m))**2, lambda x:
(np.arctan(np.sqrt(x) / focal_length.to_value(u.m)))**2)
]
with u.set_enabled_equivalencies(angle_dist_eq):
self.start_parameters = None
self.names_parameters = None
unit_cam, fit_params = self.call_setup(event, telescope_id,
dl1_container)
self.end_parameters = None
self.error_parameters = None
self.fcn = None
return self.predict(unit_cam, fit_params)
def clean_data(self, pix_x, pix_y, pix_radius, times, start_parameters,
telescope_id):
"""
Method used to select pixels and time samples used in the fitting procedure.
The spatial selection takes pixels in an ellipsis obtained from the seed Hillas parameters extended by one pixel
size and multiplied by a factor sigma_space.
The temporal selection takes a time window centered on the seed time of center of mass and of duration equal to
the time of propagation of the signal along the length of the ellipsis times a factor sigma_time.
An additional fixed duration is also added before and after this time window through the time_before_shower and
time_after_shower arguments.
Parameters
----------
pix_x, pix_y: array-like
Pixels positions
pix_radius: float
times: array-like
Sampling times before timeshift corrections
start_parameters: dict
Seed parameters derived from the Hillas parameters
telescope_id: int
Returns
----------
mask_pixel, mask_time: array-like
Mask used to select pixels and times for the fit
"""
x_cm = start_parameters['x_cm']
y_cm = start_parameters['y_cm']
length = start_parameters['length']
width = start_parameters['wl'] * length
psi = start_parameters['psi']
dx = pix_x - x_cm
dy = pix_y - y_cm
lon = dx * np.cos(psi) + dy * np.sin(psi)
lat = dx * np.sin(psi) - dy * np.cos(psi)
mask_pixel = ((lon / (length + pix_radius))**2 +
(lat / (width + pix_radius))**2) < self.sigma_space**2
v = start_parameters['v']
t_start = (start_parameters['t_cm'] -
(np.abs(v) * length / 2 * self.sigma_time) -
self.time_before_shower.tel[telescope_id])
t_end = (start_parameters['t_cm'] +
(np.abs(v) * length / 2 * self.sigma_time) +
self.time_after_shower.tel[telescope_id])
mask_time = (times < t_end) * (times > t_start)
return mask_pixel, mask_time
def find_transition_charge(self, crosstalk, poisson_proba_min=1e-2):
"""
Find the charge below which the full likelihood computation is performed and above which a Gaussian
approximation is used. For a given pixel crosstalk it finds the maximum charge with a Generalised Poisson term
below poisson_proba_min for n_peaks photo-electrons. n_peaks here is the configured maximum number of
photo-electron considered in the full likelihood computation.
Parameters
----------
crosstalk : float
Pixels crosstalk
poisson_proba_min: float
Returns
-------
transition_charge: float32
Model charge of transition between full and approximated likelihood
"""
transition_charge = self.n_peaks / (1 + crosstalk)
step = transition_charge / 100
def poisson(mu, cross_talk):
return (mu * pow(mu + self.n_peaks * cross_talk,
(self.n_peaks - 1)) /
self.factorial[self.n_peaks] *
np.exp(-mu - self.n_peaks * cross_talk))
while poisson(transition_charge, crosstalk) > poisson_proba_min:
transition_charge -= step
logger.info(
f'Transition charge between full and approximated likelihood for camera '
f'with crosstalk = {crosstalk:.4f} is, {transition_charge:.4f}, p.e.'
)
return np.float32(transition_charge)
def fit(self, fit_params):
"""
Performs the fitting procedure.
Parameters
----------
fit_params: array
Parameters used to compute the likelihood but not fitted
"""
def f(*args):
return -2 * self.log_likelihood(*args, fit_params=fit_params)
print_level = 2 if self.verbose in [1, 2, 3] else 0
m = Minuit(f,
name=self.names_parameters,
*self.start_parameters.values())
for key, val in self.bound_parameters.items():
m.limits[key] = val
m.print_level = print_level
m.errordef = 0.5
m.simplex().migrad()
self.end_parameters = m.values.to_dict()
self.fcn = m.fval
self.error_parameters = m.errors.to_dict()
def predict(self, unit_cam, fit_params):
"""
Call the fitting procedure and fill the results.
Parameters
----------
unit_cam: astropy.units.unit
Unit used for the camera geometry and for spatial variable in the fit
fit_params: array
Parameters used to compute the likelihood but not fitted
Returns
----------
container: DL1LikelihoodParametersContainer
Filled parameter container
"""
container = DL1LikelihoodParametersContainer(lhfit_call_status=1)
try:
self.fit(fit_params)
container.lhfit_TS = self.fcn
container.lhfit_x = (self.end_parameters['x_cm'] * unit_cam).to(
u.m)
container.lhfit_x_uncertainty = (self.error_parameters['x_cm'] *
unit_cam).to(u.m)
container.lhfit_y = (self.end_parameters['y_cm'] * unit_cam).to(
u.m)
container.lhfit_y_uncertainty = (self.error_parameters['y_cm'] *
unit_cam).to(u.m)
container.lhfit_r = np.sqrt(container.lhfit_x**2 +
container.lhfit_y**2)
container.lhfit_phi = np.arctan2(container.lhfit_y,
container.lhfit_x)
if self.end_parameters['psi'] > np.pi:
self.end_parameters['psi'] -= 2 * np.pi
if self.end_parameters['psi'] < -np.pi:
self.end_parameters['psi'] += 2 * np.pi
container.lhfit_psi = self.end_parameters['psi'] * u.rad
container.lhfit_psi_uncertainty = self.error_parameters[
'psi'] * u.rad
length_asy = 1 + self.end_parameters['rl'] if self.end_parameters[
'rl'] >= 0 else 1 / (1 - self.end_parameters['rl'])
lhfit_length = ((
(1.0 + length_asy) * self.end_parameters['length'] / 2.0) *
unit_cam).to(u.deg)
container.lhfit_length = lhfit_length
lhfit_length_rel_err = self.error_parameters[
'length'] / self.end_parameters['length']
# We assume that the relative error is the same in the fitted and saved unit
container.lhfit_length_uncertainty = lhfit_length_rel_err * container.lhfit_length
container.lhfit_width = self.end_parameters[
'wl'] * container.lhfit_length
container.lhfit_time_gradient = self.end_parameters['v']
container.lhfit_time_gradient_uncertainty = self.error_parameters[
'v']
container.lhfit_ref_time = self.end_parameters['t_cm']
container.lhfit_ref_time_uncertainty = self.error_parameters[
't_cm']
container.lhfit_wl = u.Quantity(self.end_parameters['wl'])
container.lhfit_wl_uncertainty = u.Quantity(
self.error_parameters['wl'])
container.lhfit_intensity = self.end_parameters['charge']
container.lhfit_intensity_uncertainty = self.error_parameters[
'charge']
container.lhfit_log_intensity = np.log10(container.lhfit_intensity)
container.lhfit_t_68 = container.lhfit_length.value * container.lhfit_time_gradient
container.lhfit_area = container.lhfit_length * container.lhfit_width
container.lhfit_length_asymmetry = self.end_parameters['rl']
container.lhfit_length_asymmetry_uncertainty = self.error_parameters[
'rl']
except ZeroDivisionError:
# TODO Check occurrence rate and solve
container = DL1LikelihoodParametersContainer(lhfit_call_status=-1)
logger.error(
'ZeroDivisionError encounter during the fitting procedure, skipping event.'
)
return container
def __str__(self):
"""
Define the print format of TimeWaveformFitter objects.
Returns
-------
str: string
Contains the starting and bound parameters used for the fit,
and the end results with errors and associated log-likelihood
in readable format.
"""
s = 'Event processed\n'
s += 'Start parameters :\n\t{}\n'.format(self.start_parameters)
s += 'Bound parameters :\n\t{}\n'.format(self.bound_parameters)
s += 'End parameters :\n\t{}\n'.format(self.end_parameters)
s += 'Error parameters :\n\t{}\n'.format(self.error_parameters)
s += '-2Log-Likelihood :\t{}'.format(self.fcn)
return s
@staticmethod
def log_likelihood(*args, fit_params, **kwargs):
"""Compute the log-likelihood used in the fitting procedure."""
llh = log_pdf(*args, *fit_params, **kwargs)
return np.sum(llh)
class EventSource(Component):
"""
Parent class for EventSources.
EventSources read input files and generate `ArrayEvents`
when iterated over.
A new EventSource should be created for each type of event file read
into ctapipe, e.g. sim_telarray files are read by the `SimTelEventSource`.
EventSource provides a common high-level interface for accessing event
information from different data sources (simulation or different camera
file formats). Creating an EventSource for a new
file format or other event source ensures that data can be accessed in a common way,
irregardless of the file format or data origin.
EventSource itself is an abstract class, but will create an
appropriate subclass if a compatible source is found for the given
``input_url``.
>>> dataset = get_dataset_path('gamma_test_large.simtel.gz')
>>> event_source = EventSource(input_url=dataset)
<ctapipe.io.simteleventsource.SimTelEventSource at ...>
An ``EventSource`` can also be created through the configuration system,
by passing ``config`` or ``parent`` as appropriate.
E.g. if using ``EventSource`` inside of a ``Tool``, you would do:
>>> self.event_source = EventSource(parent=self)
To loop through the events in a file:
>>> event_source = EventSource(input_url="/path/to/file")
>>> for event in event_source:
>>> print(event.count)
**NOTE**: Every time a new loop is started through the event_source,
it tries to restart from the first event, which might not be supported
by the event source.
It is encouraged to use ``EventSource`` in a context manager to ensure
the correct cleanups are performed when you are finished with the event_source:
>>> with EventSource(input_url="/path/to/file") as event_source:
>>> for event in event_source:
>>> print(event.count)
**NOTE**: For effiency reasons, most sources only use a single ``ArrayEvent`` instance
and update it with new data on iteration, which might lead to surprising
behaviour if you want to access multiple events at the same time.
To keep an event and prevent its data from being overwritten with the next event's data,
perform a deepcopy: ``some_special_event = copy.deepcopy(event)``.
Attributes
----------
input_url : str
Path to the input event file.
max_events : int
Maximum number of events to loop through in generator
allowed_tels: Set[int] or None
Ids of the telescopes to be included in the data.
If given, only this subset of telescopes will be present in the
generated events. If None, all available telescopes are used.
"""
input_url = Path(
directory_ok=False,
exists=True,
help="Path to the input file containing events.",
).tag(config=True)
max_events = Int(
None,
allow_none=True,
help="Maximum number of events that will be read from the file",
).tag(config=True)
allowed_tels = Set(
default_value=None,
allow_none=True,
help=(
"list of allowed tel_ids, others will be ignored. "
"If None, all telescopes in the input stream "
"will be included"
),
).tag(config=True)
def __new__(cls, input_url=None, config=None, parent=None, **kwargs):
"""
Returns a compatible subclass for given input url, either
directly or via config / parent
"""
# needed to break recursion, as __new__ of subclass will also
# call this method
if cls is not EventSource:
return super().__new__(cls)
# check we have at least one of these to be able to determine the subclass
if input_url is None and config is None and parent is None:
raise ValueError("One of `input_url`, `config`, `parent` is required")
if input_url is None:
input_url = cls._find_input_url_in_config(config=config, parent=parent)
subcls = cls._find_compatible_source(input_url)
return super().__new__(subcls)
def __init__(self, input_url=None, config=None, parent=None, **kwargs):
"""
Class to handle generic input files. Enables obtaining the "source"
generator, regardless of the type of file (either hessio or camera
file).
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
# traitlets differentiates between not getting the kwarg
# and getting the kwarg with a None value.
# the latter overrides the value in the config with None, the former
# enables getting it from the config.
if input_url is not None:
kwargs["input_url"] = input_url
super().__init__(config=config, parent=parent, **kwargs)
self.metadata = dict(is_simulation=False)
self.log.info(f"INPUT PATH = {self.input_url}")
if self.max_events:
self.log.info(f"Max events being read = {self.max_events}")
Provenance().add_input_file(str(self.input_url), role="DL0/Event")
@staticmethod
@abstractmethod
def is_compatible(file_path):
"""
Abstract method to be defined in child class.
Perform a set of checks to see if the input file is compatible
with this file event_source.
Parameters
----------
file_path : str
File path to the event file.
Returns
-------
compatible : bool
True if file is compatible, False if it is incompatible
"""
@property
def is_stream(self):
"""
Bool indicating if input is a stream. If it is then it is incompatible
with `ctapipe.io.eventseeker.EventSeeker`.
TODO: Define a method to detect if it is a stream
Returns
-------
bool
If True, then input is a stream.
"""
return False
@property
@abstractmethod
def subarray(self):
"""
Obtain the subarray from the EventSource
Returns
-------
ctapipe.instrument.SubarrayDecription
"""
@property
@abstractmethod
def is_simulation(self):
"""
Weither the currently opened file is simulated
Returns
-------
bool
"""
@property
@abstractmethod
def datalevels(self):
"""
The datalevels provided by this event source
Returns
-------
tuple[ctapipe.io.DataLevel]
"""
def has_any_datalevel(self, datalevels):
"""
Check if any of `datalevels` is in self.datalevels
Parameters:
-----------
datalevels: Iterable
Iterable of datalevels
"""
return any(dl in self.datalevels for dl in datalevels)
@property
@abstractmethod
def obs_ids(self):
"""
The observation ids of the runs located in the file
Unmerged files should only contain a single obs id.
Returns
-------
list[int]
"""
@abstractmethod
def _generator(self):
"""
Abstract method to be defined in child class.
Generator where the filling of the `ctapipe.containers` occurs.
Returns
-------
generator
"""
def __iter__(self):
"""
Generator that iterates through `_generator`, but keeps track of
`self.max_events`.
Returns
-------
generator
"""
for event in self._generator():
yield event
if self.max_events and event.count >= self.max_events - 1:
break
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
pass
@classmethod
def _find_compatible_source(cls, input_url):
if input_url == "" or input_url is None:
raise ToolConfigurationError("EventSource: No input_url was specified")
# validate input url with the traitel validate method
# to make sure it's compatible and to raise the correct error
input_url = EventSource.input_url.validate(obj=None, value=input_url)
available_classes = non_abstract_children(cls)
for subcls in available_classes:
if subcls.is_compatible(input_url):
return subcls
raise ValueError(
"Cannot find compatible EventSource for \n"
"\turl:{}\n"
"in available EventSources:\n"
"\t{}".format(input_url, [c.__name__ for c in available_classes])
)
@classmethod
def from_url(cls, input_url, **kwargs):
"""
Find compatible EventSource for input_url via the `is_compatible`
method of the EventSource
Parameters
----------
input_url : str
Filename or URL pointing to an event file
kwargs
Named arguments for the EventSource
Returns
-------
instance
Instance of a compatible EventSource subclass
"""
subcls = cls._find_compatible_source(input_url)
return subcls(input_url=input_url, **kwargs)
@classmethod
def _find_input_url_in_config(cls, config=None, parent=None):
if config is None and parent is None:
raise ValueError("One of config or parent must be provided")
if config is not None and parent is not None:
raise ValueError("Only one of config or parent must be provided")
input_url = None
# config was passed
if config is not None:
if not isinstance(config.input_url, LazyConfigValue):
input_url = config.input_url
elif not isinstance(config.EventSource.input_url, LazyConfigValue):
input_url = config.EventSource.input_url
else:
input_url = cls.input_url.default_value
# parent was passed
else:
# first look at appropriate position in the config hierarcy
input_url = find_config_in_hierarchy(parent, "EventSource", "input_url")
# if not found, check top level
if isinstance(input_url, LazyConfigValue):
if not isinstance(parent.config.EventSource.input_url, LazyConfigValue):
input_url = parent.config.EventSource.input_url
else:
input_url = cls.input_url.default_value
return input_url
@classmethod
def from_config(cls, config=None, parent=None, **kwargs):
"""
Find compatible EventSource for the EventSource.input_url traitlet
specified via the config.
This method is typically used in Tools, where the input_url is chosen via
the command line using the traitlet configuration system.
Parameters
----------
config : traitlets.config.loader.Config
Configuration created in the Tool
kwargs
Named arguments for the EventSource
Returns
-------
instance
Instance of a compatible EventSource subclass
"""
input_url = cls._find_input_url_in_config(config=config, parent=parent)
return cls.from_url(input_url, config=config, parent=parent, **kwargs)
class TimeCorrectionCalculate(Component):
"""
The TimeCorrectionCalculate class to create h5py
file with coefficients for time correction curve
of chip DRS4.
Description of this method: "Analysis techniques
and performance of the Domino Ring Sampler version 4
based readout for the MAGIC telescopes [arxiv:1305.1007]
"""
minimum_charge = Float(
200, help='Cut on charge. Default 200 ADC').tag(config=True)
tel_id = Int(1, help='Id of the telescope to calibrate').tag(config=True)
n_combine = Int(
8, help='How many capacitors are combines in a single bin. Default 8'
).tag(config=True)
n_harmonics = Int(
16, help='Number of harmonic for Fourier series expansion. Default 16'
).tag(config=True)
n_capacitors = Int(
1024, help='Number of capacitors (1024 or 4096). Default 1024.').tag(
config=True)
charge_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
calib_file_path = Unicode(
'', allow_none=True,
help='Path to the time calibration file').tag(config=True)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.n_bins = int(self.n_capacitors / self.n_combine)
self.mean_values_per_bin = np.zeros((n_gain, n_pixels, self.n_bins))
self.entries_per_bin = np.zeros((n_gain, n_pixels, self.n_bins))
self.first_cap_array = np.zeros((n_modules, n_gain, n_channel))
# load the waveform charge extractor
self.extractor = ImageExtractor.from_name(self.charge_product,
config=self.config)
self.log.info(f"extractor {self.extractor}")
self.sum_events = 0
def calibrate_pulse_time(self, event):
"""
Fill bins using time pulse from LocalPeakWindowSum.
Parameters
----------
event : `ctapipe` event-container
"""
if event.r1.tel[self.tel_id].trigger_type == 1:
for nr_module in prange(0, n_modules):
self.first_cap_array[
nr_module, :, :] = self.get_first_capacitor(
event, nr_module)
pixel_ids = event.lst.tel[self.tel_id].svc.pixel_ids
charge, pulse_time = self.extractor(
event.r1.tel[self.tel_id].waveform)
self.calib_pulse_time_jit(charge,
pulse_time,
pixel_ids,
self.first_cap_array,
self.mean_values_per_bin,
self.entries_per_bin,
n_cap=self.n_capacitors,
n_combine=self.n_combine,
min_charge=self.minimum_charge)
self.sum_events += 1
@jit(parallel=True)
def calib_pulse_time_jit(self, charge, pulse_time, pixel_ids,
first_cap_array, mean_values_per_bin,
entries_per_bin, n_cap, n_combine, min_charge):
"""
Numba function for calibration pulse time.
Parameters
----------
pulse : ndarray
Pulse time stored in a numpy array of shape
(n_gain, n_pixels).
charge : ndarray
Charge in each pixel.
(n_gain, n_pixels).
pixel_ids: ndarray
Array stored expected pixel id
(n_pixels).
first_cap_array : ndarray
Value of first capacitor stored in a numpy array of shape
(n_clus, n_gain, n_pix).
mean_values_per_bin : ndarray
Array to fill using pulse time
stored in a numpy array of shape
(n_gain, n_pixels, n_bins).
entries_per_bin : ndarray
Array to store number of entries per bin
stored in a numpy array of shape
(n_gain, n_pixels, n_bins).
n_cap : int
Number of capacitors
n_combine : int
Number of combine capacitors in a single bin
"""
for nr_module in prange(0, n_modules):
for gain in prange(0, n_gain):
for pix in prange(0, n_channel):
pixel = pixel_ids[nr_module * 7 + pix]
if charge[gain, pixel] > min_charge: # cut change
fc = first_cap_array[nr_module, :, :]
first_cap = (fc[gain, pix]) % n_cap
bin = int(first_cap / n_combine)
mean_values_per_bin[gain, pixel,
bin] += pulse_time[gain, pixel]
entries_per_bin[gain, pixel, bin] += 1
def finalize(self):
if np.sum(self.entries_per_bin == 0) > 0:
raise RuntimeError(
"Not enough events to coverage all capacitor. "
"Please use more events to time calibration file.")
else:
self.mean_values_per_bin = self.mean_values_per_bin / self.entries_per_bin
self.save_to_hdf5_file()
def fit(self, pixel_id, gain):
"""
Fit data bins using Fourier series expansion
Parameters
----------
pixel_id : ndarray
Array stored expected pixel id of shape
(n_pixels).
gain: int
0 for high gain, 1 for low gain
"""
self.pos = np.zeros(self.n_bins)
for i in range(0, self.n_bins):
self.pos[i] = (i + 0.5) * self.n_combine
self.fan = np.zeros(self.n_harmonics) # cos coeff
self.fbn = np.zeros(self.n_harmonics) # sin coeff
for n in range(0, self.n_harmonics):
self.integrate_with_trig(self.pos,
self.mean_values_per_bin[gain, pixel_id],
n, self.fan, self.fbn)
def integrate_with_trig(self, x, y, n, an, bn):
"""
Function to expanding into Fourier series
Parameters
----------
x : ndarray
Array stored position in DRS4 ring of shape
(n_bins).
y: ndarray
Array stored mean pulse time per bin of shape
(n_bins)
n : int
n harmonic
an: ndarray
Array to fill with cos coeff of shape
(n_harmonics)
bn: ndarray
Array to fill with sin coeff of shape
(n_harmonics)
"""
suma = 0
sumb = 0
for i in range(0, self.n_bins):
suma += y[i] * self.n_combine * np.cos(
2 * np.pi * n * (x[i] / float(self.n_capacitors)))
sumb += y[i] * self.n_combine * np.sin(
2 * np.pi * n * (x[i] / float(self.n_capacitors)))
an[n] = suma * (2. / (self.n_bins * self.n_combine))
bn[n] = sumb * (2. / (self.n_bins * self.n_combine))
def get_first_capacitor(self, event, nr):
fc = np.zeros((n_gain, n_channel))
first_cap = event.lst.tel[
self.tel_id].evt.first_capacitor_id[nr * 8:(nr + 1) * 8]
# First capacitor order according Dragon v5 board data format
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [0, 0, 1, 1, 2, 2, 3]):
fc[high_gain, i] = first_cap[j]
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [4, 4, 5, 5, 6, 6, 7]):
fc[low_gain, i] = first_cap[j]
return fc
def save_to_hdf5_file(self):
"""
Function to save Fourier series expansion coeff into hdf5 file
"""
fan_array = np.zeros((n_gain, n_pixels, self.n_harmonics))
fbn_array = np.zeros((n_gain, n_pixels, self.n_harmonics))
for pix_id in range(0, n_pixels):
self.fit(pix_id, gain=high_gain)
fan_array[high_gain, pix_id, :] = self.fan
fbn_array[high_gain, pix_id, :] = self.fbn
self.fit(pix_id, gain=low_gain)
fan_array[low_gain, pix_id, :] = self.fan
fbn_array[low_gain, pix_id, :] = self.fbn
try:
hf = h5py.File(self.calib_file_path, 'w')
hf.create_dataset('fan', data=fan_array)
hf.create_dataset('fbn', data=fbn_array)
hf.attrs['n_events'] = self.sum_events
hf.attrs['n_harm'] = self.n_harmonics
except Exception as err:
print("FAILED!", err)
hf.close()
class PulseTimeCorrection(Component):
"""
The PulseTimeCorrection class to correct time pulse
using Fourier series expansion.
"""
tel_id = Int(1, help='id of the telescope to calibrate').tag(config=True)
n_capacitors = Int(
1024, help='number of capacitors (1024 or 4096)').tag(config=True)
calib_file_path = Unicode(
'', allow_none=True,
help='Path to the time calibration file').tag(config=True)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.n_harmonics = None
self.fan_array = None # array to store cos coeff for Fourier series expansion
self.fbn_array = None # array to store sin coeff for Fourier series expansion
self.first_cap_array = np.zeros((n_modules, n_gain, n_channel))
self.load_calib_file()
def load_calib_file(self):
"""
Function to load calibration file.
"""
try:
with h5py.File(self.calib_file_path, 'r') as hf:
self.n_harmonics = hf["/"].attrs['n_harm']
fan = hf.get('fan')
self.fan_array = np.array(fan)
fbn = hf.get('fbn')
self.fbn_array = np.array(fbn)
except:
self.log.error(
f"Problem in reading time from calibration file {self.calib_file_path}"
)
def get_corr_pulse(self, event, pulse):
"""
Return pulse time after time correction.
Parameters
----------
event : `ctapipe` event-container
pulse : ndarray
pulse time in each pixel.
Stored in a numpy array of shape
(2, 1855).
"""
pixel_ids = event.lst.tel[self.tel_id].svc.pixel_ids
n_modules_from_event = event.lst.tel[self.tel_id].svc.num_modules
pulse_corr = np.empty((n_gain, n_pixels))
for nr in prange(0, n_modules_from_event):
self.first_cap_array[nr, :, :] = self.get_first_capacitor(
event, nr)
self.get_corr_pulse_jit(pulse, pulse_corr, pixel_ids,
self.first_cap_array, self.fan_array,
self.fbn_array, self.n_harmonics,
self.n_capacitors)
return pulse_corr
@staticmethod
@njit(parallel=True)
def get_corr_pulse_jit(pulse, pulse_corr, pixel_ids, first_capacitor,
fan_array, fbn_array, n_harmonics, n_cap):
"""
Numba function for pulse time correction.
Parameters
----------
pulse : ndarray
Pulse time stored in a numpy array of shape
(n_gain, n_pixels).
pulse_corr : ndarray
Pulse correction time stored in a numpy array of shape
(n_gain, n_pixels).
pixel_ids: ndarray
Array stored expected pixel id
(n_pixels).
first_capacitor : ndarray
Value of first capacitor stored in a numpy array of shape
(n_clus, n_gain, n_pix).
fan_array : ndarray
Array to store coeff for Fourier series expansion
stored in a numpy array of shape
(n_gain, n_pixels, n_harmonics).
fbn_array : ndarray
Array to store coeff for Fourier series expansion
stored in a numpy array of shape
(n_gain, n_pixels, n_harmonics).
n_harmonics : int
Number of harmonics
"""
for gain in prange(0, n_gain):
for nr in prange(0, n_modules):
for pix in prange(0, n_channel):
fc = first_capacitor[nr, gain, pix]
pixel = pixel_ids[nr * 7 + pix]
pulse_corr[gain,
pixel] = pulse[gain, pixel] - get_corr_time_jit(
fc % n_cap, fan_array[gain, pixel],
fbn_array[gain, pixel], n_harmonics, n_cap)
def get_first_capacitor(self, event, nr):
"""
Get first capacitor values from event for nr module.
Parameters
----------
event : `ctapipe` event-container
nr_module : number of module
tel_id : id of the telescope
"""
fc = np.zeros((n_gain, n_channel))
first_cap = event.lst.tel[
self.tel_id].evt.first_capacitor_id[nr * 8:(nr + 1) * 8]
# First capacitor order according Dragon v5 board data format
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [0, 0, 1, 1, 2, 2, 3]):
fc[high_gain, i] = first_cap[j]
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [4, 4, 5, 5, 6, 6, 7]):
fc[low_gain, i] = first_cap[j]
return fc
class NectarCAMEventSource(EventSource):
"""
EventSource for NectarCam r0 data.
"""
n_gains = Int(2, help='Number of gains at r0/r1 level').tag(config=True)
baseline = Int(250, help='r0 waveform baseline ').tag(config=True)
def __init__(self, **kwargs):
"""
Constructor
Parameters
----------
config: traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool: ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs: dict
Additional parameters to be passed.
NOTE: The file mask of the data to read can be passed with
the 'input_url' parameter.
"""
# EventSource can not handle file wild cards as input_url
# To overcome this we substitute the input_url with first file matching
# the specified file mask (copied from MAGICEventSourceROOT).
if 'input_url' in kwargs.keys():
self.file_list = glob.glob(kwargs['input_url'])
self.file_list.sort()
kwargs['input_url'] = self.file_list[0]
super().__init__(**kwargs)
else:
super().__init__(**kwargs)
self.file_list = [self.input_url]
self.multi_file = MultiFiles(self.file_list)
self.camera_config = self.multi_file.camera_config
self.log.info("Read {} input files".format(
self.multi_file.num_inputs()))
def _generator(self):
# container for NectarCAM data
self.data = NectarCAMDataContainer()
self.data.meta['input_url'] = self.input_url
self.data.meta['origin'] = 'NectarCAM'
# fill data from the CameraConfig table
self.fill_nectarcam_service_container_from_zfile()
# Instrument information
for tel_id in self.data.nectarcam.tels_with_data:
assert (tel_id == 0) # only one telescope for the moment (id = 0)
# optics info from standard optics.fits.gz file
optics = OpticsDescription.from_name("MST")
optics.tel_subtype = '' # to correct bug in reading
# camera info from NectarCam-[geometry_version].camgeom.fits.gz file
geometry_version = 2
camera = CameraGeometry.from_name("NectarCam", geometry_version)
tel_descr = TelescopeDescription(name='MST',
tel_type='NectarCam',
optics=optics,
camera=camera)
tel_descr.optics.tel_subtype = '' # to correct bug in reading
self.n_camera_pixels = tel_descr.camera.n_pixels
tels = {tel_id: tel_descr}
# LSTs telescope position
tel_pos = {tel_id: [0., 0., 0] * u.m}
self.subarray = SubarrayDescription("MST prototype subarray")
self.subarray.tels = tels
self.subarray.positions = tel_pos
self.data.inst.subarray = self.subarray
# initialize general monitoring container
self.initialize_mon_container()
# loop on events
for count, event in enumerate(self.multi_file):
self.data.count = count
# fill specific NectarCAM event data
self.fill_nectarcam_event_container_from_zfile(event)
# fill general R0 data
self.fill_r0_container_from_zfile(event)
# copy r0 to r1
self.fill_r1_container()
# fill general monitoring data
self.fill_mon_container_from_zfile(event)
yield self.data
@staticmethod
def is_compatible(file_path):
try:
# The file contains two tables:
# 1: CameraConfig
# 2: Events
h = fits.open(file_path)[2].header
ttypes = [h[x] for x in h.keys() if 'TTYPE' in x]
except OSError:
# not even a fits file
return False
except IndexError:
# A fits file of a different format
return False
is_protobuf_zfits_file = ((h['XTENSION'] == 'BINTABLE')
and (h['EXTNAME'] == 'Events')
and (h['ZTABLE'] is True)
and (h['ORIGIN'] == 'CTA')
and (h['PBFHEAD'] == 'R1.CameraEvent'))
is_nectarcam_file = 'nectarcam_counters' in ttypes
return is_protobuf_zfits_file & is_nectarcam_file
def fill_nectarcam_service_container_from_zfile(self):
self.data.nectarcam.tels_with_data = [
self.camera_config.telescope_id,
]
svc_container = self.data.nectarcam.tel[
self.camera_config.telescope_id].svc
svc_container.telescope_id = self.camera_config.telescope_id
svc_container.cs_serial = self.camera_config.cs_serial
svc_container.configuration_id = self.camera_config.configuration_id
svc_container.acquisition_mode = self.camera_config.nectarcam.acquisition_mode
svc_container.date = self.camera_config.date
svc_container.num_pixels = self.camera_config.num_pixels
svc_container.num_samples = self.camera_config.num_samples
svc_container.pixel_ids = self.camera_config.expected_pixels_id
svc_container.data_model_version = self.camera_config.data_model_version
svc_container.num_modules = self.camera_config.nectarcam.num_modules
svc_container.module_ids = self.camera_config.nectarcam.expected_modules_id
svc_container.idaq_version = self.camera_config.nectarcam.idaq_version
svc_container.cdhs_version = self.camera_config.nectarcam.cdhs_version
svc_container.algorithms = self.camera_config.nectarcam.algorithms
# svc_container.pre_proc_algorithms = camera_config.nectarcam.pre_proc_algorithms
def fill_nectarcam_event_container_from_zfile(self, event):
event_container = self.data.nectarcam.tel[
self.camera_config.telescope_id].evt
event_container.configuration_id = event.configuration_id
event_container.event_id = event.event_id
event_container.tel_event_id = event.tel_event_id
event_container.pixel_status = event.pixel_status
event_container.ped_id = event.ped_id
event_container.module_status = event.nectarcam.module_status
event_container.extdevices_presence = event.nectarcam.extdevices_presence
#event_container.tib_data = event.nectarcam.tib_data
#event_container.cdts_data = event.nectarcam.cdts_data
event_container.swat_data = event.nectarcam.swat_data
event_container.counters = event.nectarcam.counters
# unpack TIB data
rec_fmt = '=IHIBB'
unpacked_tib = struct.unpack(rec_fmt, event.nectarcam.tib_data)
event_container.tib_event_counter = unpacked_tib[0]
event_container.tib_pps_counter = unpacked_tib[1]
event_container.tib_tenMHz_counter = unpacked_tib[2]
event_container.tib_stereo_pattern = unpacked_tib[3]
event_container.tib_masked_trigger = unpacked_tib[4]
event_container.swat_data = event.lstcam.swat_data
# unpack CDTS data
rec_fmt = '=IIIQQBBB'
unpacked_cdts = struct.unpack(rec_fmt, event.nectarcam.cdts_data)
event_container.ucts_event_counter = unpacked_cdts[0]
event_container.ucts_pps_counter = unpacked_cdts[1]
event_container.ucts_clock_counter = unpacked_cdts[2]
event_container.ucts_timestamp = unpacked_cdts[3]
event_container.ucts_camera_timestamp = unpacked_cdts[4]
event_container.ucts_trigger_type = unpacked_cdts[5]
event_container.ucts_white_rabbit_status = unpacked_cdts[6]
def fill_r0_camera_container_from_zfile(self, container, event):
container.trigger_time = event.trigger_time_s
#container.trigger_type = event.trigger_type
container.trigger_type = self.data.nectarcam.tel[
self.camera_config.telescope_id].evt.tib_masked_trigger
# verify the number of gains
if event.waveform.shape[
0] != self.camera_config.num_pixels * self.camera_config.num_samples * self.n_gains:
raise ValueError(
f"Number of gains not correct, waveform shape is {event.waveform.shape[0]}"
f" instead of "
f"{self.camera_config.num_pixels * self.camera_config.num_samples * self.n_gains}"
)
reshaped_waveform = np.array(event.waveform).reshape(
self.n_gains, self.camera_config.num_pixels,
self.camera_config.num_samples)
# initialize the waveform container to zero
container.waveform = np.zeros([
self.n_gains, self.n_camera_pixels, self.camera_config.num_samples
])
# re-order the waveform following the expected_pixels_id values (rank = pixel id)
container.waveform[:, self.camera_config.expected_pixels_id, :] \
= reshaped_waveform
def fill_r0_container_from_zfile(self, event):
"""fill the event r0 container"""
container = self.data.r0
container.obs_id = -1
container.event_id = event.event_id
container.tels_with_data = [
self.camera_config.telescope_id,
]
r0_camera_container = container.tel[self.camera_config.telescope_id]
self.fill_r0_camera_container_from_zfile(r0_camera_container, event)
def fill_r1_container(self):
"""
fill the event r1 container
In the case of nectarCAM:
r1 waveform = r0 waveform - self.baseline
"""
self.data.r1.tels_with_data = [
self.camera_config.telescope_id,
]
r1_camera_container = self.data.r1.tel[self.camera_config.telescope_id]
r1_camera_container.waveform = self.data.r0.tel[
self.camera_config.telescope_id].waveform - self.baseline
r1_camera_container.trigger_type = self.data.r0.tel[
self.camera_config.telescope_id].trigger_type
r1_camera_container.trigger_time = self.data.r0.tel[
self.camera_config.telescope_id].trigger_time
def initialize_mon_container(self):
"""
Fill with MonitoringContainer.
For the moment, initialize only the PixelStatusContainer
"""
container = self.data.mon
container.tels_with_data = [
self.camera_config.telescope_id,
]
mon_camera_container = container.tel[self.camera_config.telescope_id]
# initialize the container
status_container = PixelStatusContainer()
status_container.hardware_failing_pixels = np.zeros(
(self.n_gains, self.n_camera_pixels), dtype=bool)
status_container.pedestal_failing_pixels = np.zeros(
(self.n_gains, self.n_camera_pixels), dtype=bool)
status_container.flatfield_failing_pixels = np.zeros(
(self.n_gains, self.n_camera_pixels), dtype=bool)
mon_camera_container.pixel_status = status_container
def fill_mon_container_from_zfile(self, event):
"""
Fill with MonitoringContainer.
For the moment, initialize only the PixelStatusContainer
"""
status_container = self.data.mon.tel[
self.camera_config.telescope_id].pixel_status
# reorder the array
pixel_status = np.zeros(self.n_camera_pixels)
pixel_status[
self.camera_config.expected_pixels_id] = event.pixel_status
status_container.hardware_failing_pixels[:] = pixel_status == 0
'''
class DL3Cuts(Component):
"""
Selection cuts for DL2 to DL3 conversion
"""
global_gh_cut = Float(
help="Global selection cut for gh_score (gammaness)",
default_value=0.6,
).tag(config=True)
gh_efficiency = Float(
help="Gamma efficiency for optimized g/h cuts in %",
default_value=0.95,
).tag(config=True)
theta_containment = Float(
help="Percentage containment region for theta cuts",
default_value=0.68,
).tag(config=True)
global_theta_cut = Float(
help="Global selection cut for theta",
default_value=0.2,
).tag(config=True)
global_alpha_cut = Float(
help="Global selection cut for alpha",
default_value=20,
).tag(config=True)
allowed_tels = List(
help="List of allowed LST telescope ids",
trait=Int(),
default_value=[1],
).tag(config=True)
def apply_global_gh_cut(self, data):
"""
Applying a global gammaness cut on a given data
"""
return data[data["gh_score"] > self.global_gh_cut]
def energy_dependent_gh_cuts(self,
data,
energy_bins,
min_value=0.1,
max_value=0.99,
smoothing=None,
min_events=10):
"""
Evaluating energy-dependent gammaness cuts, in a given
data, with provided reco energy bins, and other parameters to
pass to the pyirf.cuts.calculate_percentile_cut function
"""
gh_cuts = calculate_percentile_cut(
data["gh_score"],
data["reco_energy"],
bins=energy_bins,
min_value=min_value,
max_value=max_value,
fill_value=data["gh_score"].max(),
percentile=100 * (1 - self.gh_efficiency),
smoothing=smoothing,
min_events=min_events,
)
return gh_cuts
def apply_global_alpha_cut(self, data):
"""
Applying a global alpha cut on a given data
"""
return data[data["alpha"].to_value(u.deg) < self.global_alpha_cut]
def apply_energy_dependent_gh_cuts(self, data, gh_cuts):
"""
Applying a given energy-dependent gh cuts to a data file, along the reco
energy bins provided.
"""
data["selected_gh"] = evaluate_binned_cut(
data["gh_score"],
data["reco_energy"],
gh_cuts,
operator.ge,
)
return data[data["selected_gh"]]
def apply_global_theta_cut(self, data):
"""
Applying a global theta cut on a given data
"""
return data[data["theta"].to_value(u.deg) < self.global_theta_cut]
def energy_dependent_theta_cuts(self,
data,
energy_bins,
min_value=0.05 * u.deg,
fill_value=0.32 * u.deg,
max_value=0.32 * u.deg,
smoothing=None,
min_events=10):
"""
Evaluating an optimized energy-dependent theta cuts, in a given
data, with provided reco energy bins, and other parameters to
pass to the pyirf.cuts.calculate_percentile_cut function.
Note: Using too fine binning will result in too un-smooth cuts.
"""
theta_cuts = calculate_percentile_cut(
data["theta"],
data["reco_energy"],
bins=energy_bins,
min_value=min_value,
max_value=max_value,
fill_value=fill_value,
percentile=100 * self.theta_containment,
smoothing=smoothing,
min_events=min_events,
)
return theta_cuts
def apply_energy_dependent_theta_cuts(self, data, theta_cuts):
"""
Applying a given energy-dependent theta cuts to a data file, along the
reco energy bins provided.
"""
data["selected_theta"] = evaluate_binned_cut(
data["theta"],
data["reco_energy"],
theta_cuts,
operator.le,
)
return data[data["selected_theta"]]
def allowed_tels_filter(self, data):
"""
Applying a filter on telescopes used for observation.
"""
mask = np.zeros(len(data), dtype=bool)
for tel_id in self.allowed_tels:
mask |= data["tel_id"] == tel_id
return data[mask]
class LSTCameraCalibrator(CameraCalibrator):
"""
Calibrator to handle the LST camera calibration chain, in order to fill
the DL1 data level in the event container.
"""
extractor_product = Unicode(
'NeighborPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
reducer_product = Unicode(
'NullDataVolumeReducer',
help='Name of the DataVolumeReducer to use').tag(config=True)
calibration_path = Unicode(
'', allow_none=True,
help='Path to LST calibration file').tag(config=True)
time_calibration_path = Unicode(
'', allow_none=True,
help='Path to drs4 time calibration file').tag(config=True)
allowed_tels = List(
[1], help='List of telescope to be calibrated').tag(config=True)
gain_threshold = Int(
4094, allow_none=True,
help='Threshold for the gain selection in ADC').tag(config=True)
def __init__(self, **kwargs):
"""
Parameters
----------
reducer_product : ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use. If None, then
NullDataVolumeReducer will be used by default, and waveforms
will not be reduced.
extractor_product : ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, then NeighborPeakWindowSum
will be used by default.
calibration_path :
Path to LST calibration file to get the pedestal and flat-field corrections
kwargs
"""
super().__init__(**kwargs)
# load the waveform charge extractor
self.image_extractor = ImageExtractor.from_name(self.extractor_product,
config=self.config)
self.log.info(f"extractor {self.extractor_product}")
self.data_volume_reducer = DataVolumeReducer.from_name(
self.reducer_product, config=self.config)
self.log.info(f" {self.reducer_product}")
# declare gain selector if the threshold is defined
if self.gain_threshold:
self.gain_selector = gainselection.ThresholdGainSelector(
threshold=self.gain_threshold)
# declare time calibrator if correction file exist
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
self.time_corrector = None
self.log.info(
f"File {self.time_calibration_path} not found. No drs4 time corrections"
)
# calibration data container
self.mon_data = MonitoringContainer()
# initialize the MonitoringContainer() for the moment it reads it from a hdf5 file
self._initialize_correction()
def _initialize_correction(self):
"""
Read the correction from hdf5 calibration file
"""
self.mon_data.tels_with_data = self.allowed_tels
self.log.info(f"read {self.calibration_path}")
try:
with HDF5TableReader(self.calibration_path) as h5_table:
assert h5_table._h5file.isopen == True
for telid in self.allowed_tels:
# read the calibration data for the moment only one event
table = '/tel_' + str(telid) + '/calibration'
next(
h5_table.read(table,
self.mon_data.tel[telid].calibration))
# eliminate inf values (should be done probably before)
dc_to_pe = self.mon_data.tel[telid].calibration.dc_to_pe
dc_to_pe[np.isinf(dc_to_pe)] = 0
self.log.info(
f"read {self.mon_data.tel[telid].calibration.dc_to_pe}"
)
except:
self.log.error(
f"Problem in reading calibration file {self.calibration_path}")
def _calibrate_dl0(self, event, telid):
"""
create dl0 level, for the moment copy the r1
"""
waveforms = event.r1.tel[telid].waveform
if self._check_r1_empty(waveforms):
return
event.dl0.event_id = event.r1.event_id
event.mon.tel[telid].calibration = self.mon_data.tel[telid].calibration
# subtract the pedestal per sample (should we do it?) and multiply for the calibration coefficients
#
event.dl0.tel[telid].waveform = (
(event.r1.tel[telid].waveform - self.mon_data.tel[telid].
calibration.pedestal_per_sample[:, :, np.newaxis]) *
self.mon_data.tel[telid].calibration.dc_to_pe[:, :, np.newaxis])
def _calibrate_dl1(self, event, telid):
"""
create calibrated dl1 image and calibrate it
"""
waveforms = event.dl0.tel[telid].waveform
if self._check_dl0_empty(waveforms):
return
if self.image_extractor.requires_neighbors():
camera = event.inst.subarray.tel[telid].camera
self.image_extractor.neighbors = camera.neighbor_matrix_where
charge, pulse_time = self.image_extractor(waveforms)
# correct time with drs4 correction if available
if self.time_corrector:
pulse_corr_array = self.time_corrector.get_corr_pulse(
event, pulse_time)
# otherwise use the ff time correction (not drs4 corrected)
else:
pulse_corr_array = pulse_time + self.mon_data.tel[
telid].calibration.time_correction
# perform the gain selection if the threshold is defined
if self.gain_threshold:
waveforms, gain_mask = self.gain_selector(
event.r1.tel[telid].waveform)
event.dl1.tel[telid].image = charge[gain_mask,
np.arange(charge.shape[1])]
event.dl1.tel[telid].pulse_time = pulse_corr_array[
gain_mask, np.arange(pulse_corr_array.shape[1])]
# remember the mask in the lst pixel_status array (this info is missing for the moment in the
# r1 container). I follow the prescription given in the document
# "R1 & DL0 Telescope Event Interfaces and Prototype Evaluation" of K. Kosack
# bit 2 = LG
gain_mask *= 4
# bit 3 = HG
gain_mask[np.where(gain_mask == 0)] = 8
# bit 1 = pixel broken pixel (coming from the EvB)
gain_mask += event.lst.tel[telid].evt.pixel_status >> 1 & 1
# update pixel status
event.lst.tel[telid].evt.pixel_status = gain_mask
# if threshold == None
else:
event.dl1.tel[telid].image = charge
event.dl1.tel[telid].pulse_time = pulse_corr_array
class LSTCameraCalibrator(CameraCalibrator):
"""
Calibrator to handle the LST camera calibration chain, in order to fill
the DL1 data level in the event container.
"""
extractor_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
reducer_product = Unicode(
'NullDataVolumeReducer',
help='Name of the DataVolumeReducer to use').tag(config=True)
calibration_path = Unicode(
'', help='Path to LST calibration file').tag(config=True)
time_calibration_path = Unicode(
'', help='Path to drs4 time calibration file').tag(config=True)
allowed_tels = List(
[1], help='List of telescope to be calibrated').tag(config=True)
gain_threshold = Int(
4094, allow_none=True,
help='Threshold for the gain selection in ADC').tag(config=True)
def __init__(self, **kwargs):
"""
Parameters
----------
reducer_product : ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use. If None, then
NullDataVolumeReducer will be used by default, and waveforms
will not be reduced.
extractor_product : ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, then LocalPeakWindowSum
will be used by default.
calibration_path :
Path to LST calibration file to get the pedestal and flat-field corrections
kwargs
"""
super().__init__(**kwargs)
# load the waveform charge extractor
self.image_extractor = ImageExtractor.from_name(self.extractor_product,
config=self.config)
self.log.info(f"extractor {self.extractor_product}")
print("EXTRACTOR", self.image_extractor)
self.data_volume_reducer = DataVolumeReducer.from_name(
self.reducer_product, config=self.config)
self.log.info(f" {self.reducer_product}")
# declare gain selector if the threshold is defined
if self.gain_threshold:
self.gain_selector = gainselection.ThresholdGainSelector(
threshold=self.gain_threshold)
# declare time calibrator if correction file exist
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
raise IOError(
f"Time calibration file {self.time_calibration_path} not found!"
)
# calibration data container
self.mon_data = MonitoringContainer()
# initialize the MonitoringContainer() for the moment it reads it from a hdf5 file
self._initialize_correction()
def _initialize_correction(self):
"""
Read the correction from hdf5 calibration file
"""
self.mon_data.tels_with_data = self.allowed_tels
self.log.info(f"read {self.calibration_path}")
try:
with HDF5TableReader(self.calibration_path) as h5_table:
for telid in self.allowed_tels:
# read the calibration data
table = '/tel_' + str(telid) + '/calibration'
next(
h5_table.read(table,
self.mon_data.tel[telid].calibration))
# read pedestal data
table = '/tel_' + str(telid) + '/pedestal'
next(
h5_table.read(table,
self.mon_data.tel[telid].pedestal))
# read flat-field data
table = '/tel_' + str(telid) + '/flatfield'
next(
h5_table.read(table,
self.mon_data.tel[telid].flatfield))
# read the pixel_status container
table = '/tel_' + str(telid) + '/pixel_status'
next(
h5_table.read(table,
self.mon_data.tel[telid].pixel_status))
except Exception:
self.log.exception(
f"Problem in reading calibration file {self.calibration_path}")
raise
def _calibrate_dl0(self, event, telid):
"""
create dl0 level, for the moment copy the r1
"""
waveforms = event.r1.tel[telid].waveform
if self._check_r1_empty(waveforms):
return
event.dl0.event_id = event.r1.event_id
# if not already done, initialize the event monitoring containers
if event.mon.tel[telid].calibration.dc_to_pe is None:
event.mon.tel[telid].calibration = self.mon_data.tel[
telid].calibration
event.mon.tel[telid].flatfield = self.mon_data.tel[telid].flatfield
event.mon.tel[telid].pedestal = self.mon_data.tel[telid].pedestal
event.mon.tel[telid].pixel_status = self.mon_data.tel[
telid].pixel_status
#
# subtract the pedestal per sample and multiply for the calibration coefficients
#
event.dl0.tel[telid].waveform = (
(waveforms - self.mon_data.tel[telid].calibration.
pedestal_per_sample[:, :, np.newaxis]) *
self.mon_data.tel[telid].calibration.dc_to_pe[:, :, np.newaxis])
def _calibrate_dl1(self, event, telid):
"""
create calibrated dl1 image and calibrate it
"""
waveforms = event.dl0.tel[telid].waveform
if self._check_dl0_empty(waveforms):
return
if self.image_extractor.requires_neighbors():
camera = event.inst.subarray.tel[telid].camera
self.image_extractor.neighbors = camera.neighbor_matrix_where
charge, pulse_time = self.image_extractor(waveforms)
# correct time with drs4 correction if available
if self.time_corrector:
pulse_time = self.time_corrector.get_corr_pulse(event, pulse_time)
# add flat-fielding time correction
pulse_time_ff_corrected = pulse_time + self.mon_data.tel[
telid].calibration.time_correction
# perform the gain selection if the threshold is defined
if self.gain_threshold:
waveforms, gain_mask = self.gain_selector(
event.r1.tel[telid].waveform)
event.dl1.tel[telid].image = charge[gain_mask,
np.arange(charge.shape[1])]
event.dl1.tel[telid].pulse_time = pulse_time_ff_corrected[
gain_mask,
np.arange(pulse_time_ff_corrected.shape[1])]
# remember which channel has been selected
event.r1.tel[telid].selected_gain_channel = gain_mask
# if threshold == None
else:
event.dl1.tel[telid].image = charge
event.dl1.tel[telid].pulse_time = pulse_time_ff_corrected
class NectarCAMEventSource(EventSource):
"""
EventSource for NectarCam r0 data.
"""
n_gains = Int(2, help='Number of gains at r0/r1 level').tag(config=True)
baseline = Int(250, help='r0 waveform baseline ').tag(config=True)
geometry_version = Int(
3, help='Version of the camera geometry to be used ').tag(config=True)
def __init__(self, **kwargs):
"""
Constructor
Parameters
----------
config: traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool: ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs: dict
Additional parameters to be passed.
NOTE: The file mask of the data to read can be passed with
the 'input_url' parameter.
"""
# EventSource can not handle file wild cards as input_url
# To overcome this we substitute the input_url with first file matching
# the specified file mask (copied from MAGICEventSourceROOT).
if 'input_url' in kwargs.keys():
self.file_list = glob.glob(str(kwargs['input_url']))
self.file_list.sort()
kwargs['input_url'] = self.file_list[0]
super().__init__(**kwargs)
else:
super().__init__(**kwargs)
self.file_list = [self.input_url]
self.multi_file = MultiFiles(self.file_list)
self.camera_config = self.multi_file.camera_config
self.data = None
self.log.info("Read {} input files".format(
self.multi_file.num_inputs()))
self._tel_id = self.camera_config.telescope_id
self._subarray_info = self.prepare_subarray_info(self._tel_id)
@property
def subarray(self):
return self._subarray_info
def prepare_subarray_info(self, tel_id=0):
"""
Constructs a SubarrayDescription object.
Parameters
----------
tel_id: int
Telescope identifier.
Returns
-------
SubarrayDescription :
instrumental information
"""
tel_descriptions = {} # tel_id : TelescopeDescription
tel_positions = {} # tel_id : TelescopeDescription
# optics info from standard optics.fits.gz file
optics = OpticsDescription.from_name("MST")
optics.tel_subtype = '' # to correct bug in reading
# camera info from NectarCam-[geometry_version].camgeom.fits.gz file
camera = CameraGeometry.from_name("NectarCam", self.geometry_version)
tel_descr = TelescopeDescription(name='MST',
tel_type='NectarCam',
optics=optics,
camera=camera)
tel_descr.optics.tel_subtype = '' # to correct bug in reading
self.n_camera_pixels = tel_descr.camera.n_pixels
# MST telescope position
tel_positions[tel_id] = [0., 0., 0] * u.m
tel_descriptions[tel_id] = tel_descr
return SubarrayDescription(
"Adlershof",
tel_positions=tel_positions,
tel_descriptions=tel_descriptions,
)
@property
def is_simulation(self):
return False
@property
def datalevels(self):
return (DataLevel.R0, DataLevel.R1)
@property
def obs_id(self):
return self.camera_config.nectarcam.run_id
def _generator(self):
# container for NectarCAM data
self.data = NectarCAMDataContainer()
self.data.meta['input_url'] = self.input_url
self.data.meta['origin'] = 'NectarCAM'
# fill data from the CameraConfig table
self.fill_nectarcam_service_container_from_zfile()
# initialize general monitoring container
self.initialize_mon_container()
# loop on events
for count, event in enumerate(self.multi_file):
self.data.count = count
# fill specific NectarCAM event data
self.fill_nectarcam_event_container_from_zfile(event)
# fill general R0 data
self.fill_r0_container_from_zfile(event)
# copy r0 to r1
self.fill_r1_container()
# fill general monitoring data
self.fill_mon_container_from_zfile(event)
yield self.data
@staticmethod
def is_compatible(file_path):
try:
# The file contains two tables:
# 1: CameraConfig
# 2: Events
h = fits.open(file_path)[2].header
ttypes = [h[x] for x in h.keys() if 'TTYPE' in x]
except OSError:
# not even a fits file
return False
except IndexError:
# A fits file of a different format
return False
is_protobuf_zfits_file = ((h['XTENSION'] == 'BINTABLE')
and (h['EXTNAME'] == 'Events')
and (h['ZTABLE'] is True)
and (h['ORIGIN'] == 'CTA')
and (h['PBFHEAD'] == 'R1.CameraEvent'))
is_nectarcam_file = 'nectarcam_counters' in ttypes
return is_protobuf_zfits_file & is_nectarcam_file
def fill_nectarcam_service_container_from_zfile(self):
self.data.nectarcam.tels_with_data = [
self.camera_config.telescope_id,
]
svc_container = self.data.nectarcam.tel[
self.camera_config.telescope_id].svc
svc_container.telescope_id = self.camera_config.telescope_id
svc_container.cs_serial = self.camera_config.cs_serial
svc_container.configuration_id = self.camera_config.configuration_id
svc_container.acquisition_mode = self.camera_config.nectarcam.acquisition_mode
svc_container.date = self.camera_config.date
svc_container.num_pixels = self.camera_config.num_pixels
svc_container.num_samples = self.camera_config.num_samples
svc_container.pixel_ids = self.camera_config.expected_pixels_id
svc_container.data_model_version = self.camera_config.data_model_version
svc_container.num_modules = self.camera_config.nectarcam.num_modules
svc_container.module_ids = self.camera_config.nectarcam.expected_modules_id
svc_container.idaq_version = self.camera_config.nectarcam.idaq_version
svc_container.cdhs_version = self.camera_config.nectarcam.cdhs_version
svc_container.algorithms = self.camera_config.nectarcam.algorithms
# svc_container.pre_proc_algorithms = camera_config.nectarcam.pre_proc_algorithms
def fill_nectarcam_event_container_from_zfile(self, event):
event_container = self.data.nectarcam.tel[
self.camera_config.telescope_id].evt
event_container.configuration_id = event.configuration_id
event_container.event_id = event.event_id
event_container.tel_event_id = event.tel_event_id
event_container.pixel_status = event.pixel_status
event_container.ped_id = event.ped_id
event_container.module_status = event.nectarcam.module_status
event_container.extdevices_presence = event.nectarcam.extdevices_presence
#event_container.tib_data = event.nectarcam.tib_data
#event_container.cdts_data = event.nectarcam.cdts_data
event_container.swat_data = event.nectarcam.swat_data
event_container.counters = event.nectarcam.counters
# unpack TIB data
rec_fmt = '=IHIBB'
unpacked_tib = struct.unpack(rec_fmt, event.nectarcam.tib_data)
event_container.tib_event_counter = unpacked_tib[0]
event_container.tib_pps_counter = unpacked_tib[1]
event_container.tib_tenMHz_counter = unpacked_tib[2]
event_container.tib_stereo_pattern = unpacked_tib[3]
event_container.tib_masked_trigger = unpacked_tib[4]
event_container.swat_data = event.lstcam.swat_data
# unpack CDTS data
is_old_cdts = len(event.nectarcam.cdts_data) < 36
rec_fmt = '=IIIQQBBB' if is_old_cdts else '=QIIIIIBBBBI'
unpacked_cdts = struct.unpack(rec_fmt, event.nectarcam.cdts_data)
if is_old_cdts:
event_container.ucts_event_counter = unpacked_cdts[0]
event_container.ucts_pps_counter = unpacked_cdts[1]
event_container.ucts_clock_counter = unpacked_cdts[2]
event_container.ucts_timestamp = unpacked_cdts[3]
event_container.ucts_camera_timestamp = unpacked_cdts[4]
event_container.ucts_trigger_type = unpacked_cdts[5]
event_container.ucts_white_rabbit_status = unpacked_cdts[6]
else:
event_container.ucts_timestamp = unpacked_cdts[0]
event_container.ucts_address = unpacked_cdts[1]
event_container.ucts_event_counter = unpacked_cdts[2]
event_container.ucts_busy_counter = unpacked_cdts[3]
event_container.ucts_pps_counter = unpacked_cdts[4]
event_container.ucts_clock_counter = unpacked_cdts[5]
event_container.ucts_trigger_type = unpacked_cdts[6]
event_container.ucts_white_rabbit_status = unpacked_cdts[7]
event_container.ucts_stereo_pattern = unpacked_cdts[8]
event_container.ucts_num_in_bunch = unpacked_cdts[9]
event_container.cdts_version = unpacked_cdts[10]
# Unpack FEB counters and trigger pattern
self.unpack_feb_data(event)
def unpack_feb_data(self, event):
'''Unpack FEB counters and trigger pattern'''
event_container = self.data.nectarcam.tel[
self.camera_config.telescope_id].evt
# Deduce data format version
bytes_per_module = len(event.nectarcam.counters
) // self.camera_config.nectarcam.num_modules
# Remain compatible with data before addition of trigger pattern
module_fmt = 'IHHIBBBBBBBB' if bytes_per_module > 16 else 'IHHIBBBB'
n_fields = len(module_fmt)
rec_fmt = '=' + module_fmt * self.camera_config.nectarcam.num_modules
# Unpack
unpacked_feb = struct.unpack(rec_fmt, event.nectarcam.counters)
# Initialize field containers
n_camera_modules = self.n_camera_pixels // 7
event_container.feb_abs_event_id = np.zeros(shape=(n_camera_modules, ),
dtype=np.uint32)
event_container.feb_event_id = np.zeros(shape=(n_camera_modules, ),
dtype=np.uint16)
event_container.feb_pps_cnt = np.zeros(shape=(n_camera_modules, ),
dtype=np.uint16)
event_container.feb_ts1 = np.zeros(shape=(n_camera_modules, ),
dtype=np.uint32)
event_container.feb_ts2_trig = np.zeros(shape=(n_camera_modules, ),
dtype=np.int16)
event_container.feb_ts2_pps = np.zeros(shape=(n_camera_modules, ),
dtype=np.int16)
if bytes_per_module > 16:
n_patterns = 4
event_container.trigger_pattern = np.zeros(
shape=(n_patterns, self.n_camera_pixels), dtype=bool)
# Unpack absolute event ID
event_container.feb_abs_event_id[
self.camera_config.nectarcam.
expected_modules_id] = unpacked_feb[0::n_fields]
# Unpack PPS counter
event_container.feb_pps_cnt[
self.camera_config.nectarcam.
expected_modules_id] = unpacked_feb[1::n_fields]
# Unpack relative event ID
event_container.feb_event_id[
self.camera_config.nectarcam.
expected_modules_id] = unpacked_feb[2::n_fields]
# Unpack TS1 counter
event_container.feb_ts1[
self.camera_config.nectarcam.
expected_modules_id] = unpacked_feb[3::n_fields]
# Unpack TS2 counters
ts2_decimal = lambda bits: bits - (1 << 8
) if bits & 0x80 != 0 else bits
ts2_decimal_vec = np.vectorize(ts2_decimal)
event_container.feb_ts2_trig[self.camera_config.nectarcam.
expected_modules_id] = ts2_decimal_vec(
unpacked_feb[4::n_fields])
event_container.feb_ts2_pps[self.camera_config.nectarcam.
expected_modules_id] = ts2_decimal_vec(
unpacked_feb[5::n_fields])
# Loop over modules
for module_idx, module_id in enumerate(
self.camera_config.nectarcam.expected_modules_id):
offset = module_id * 7
if bytes_per_module > 16:
field_id = 8
# Decode trigger pattern
for pattern_id in range(n_patterns):
value = unpacked_feb[n_fields * module_idx + field_id +
pattern_id]
module_pattern = [
int(digit)
for digit in reversed(bin(value)[2:].zfill(7))
]
event_container.trigger_pattern[pattern_id, offset:offset +
7] = module_pattern
# Unpack native charge
if len(event.nectarcam.charges_gain1) > 0:
event_container.native_charge = np.zeros(
shape=(self.n_gains, self.n_camera_pixels), dtype=np.uint16)
rec_fmt = '=' + 'H' * self.camera_config.num_pixels
for gain_id in range(self.n_gains):
unpacked_charge = struct.unpack(
rec_fmt,
getattr(event.nectarcam, f'charges_gain{gain_id + 1}'))
event_container.native_charge[
gain_id,
self.camera_config.expected_pixels_id] = unpacked_charge
def fill_r0_camera_container_from_zfile(self, container, event):
container.trigger_time = event.trigger_time_s
#container.trigger_type = event.trigger_type
container.trigger_type = self.data.nectarcam.tel[
self.camera_config.telescope_id].evt.tib_masked_trigger
# verify the number of gains
if event.waveform.shape[
0] != self.camera_config.num_pixels * self.camera_config.num_samples * self.n_gains:
raise ValueError(
f"Number of gains not correct, waveform shape is {event.waveform.shape[0]}"
f" instead of "
f"{self.camera_config.num_pixels * self.camera_config.num_samples * self.n_gains}"
)
reshaped_waveform = np.array(event.waveform).reshape(
self.n_gains, self.camera_config.num_pixels,
self.camera_config.num_samples)
# initialize the waveform container to zero
container.waveform = np.zeros([
self.n_gains, self.n_camera_pixels, self.camera_config.num_samples
])
# re-order the waveform following the expected_pixels_id values (rank = pixel id)
container.waveform[:, self.camera_config.expected_pixels_id, :] \
= reshaped_waveform
def fill_r0_container_from_zfile(self, event):
"""fill the event r0 container"""
self.data.index.obs_id = self.obs_id
self.data.index.event_id = event.event_id
container = self.data.r0
container.tels_with_data = [
self.camera_config.telescope_id,
]
r0_camera_container = container.tel[self.camera_config.telescope_id]
self.fill_r0_camera_container_from_zfile(r0_camera_container, event)
def fill_r1_container(self):
"""
fill the event r1 container
In the case of nectarCAM:
r1 waveform = r0 waveform - self.baseline
"""
self.data.r1.tels_with_data = [
self.camera_config.telescope_id,
]
r1_camera_container = self.data.r1.tel[self.camera_config.telescope_id]
r1_camera_container.waveform = self.data.r0.tel[
self.camera_config.telescope_id].waveform - self.baseline
r1_camera_container.trigger_type = self.data.r0.tel[
self.camera_config.telescope_id].trigger_type
r1_camera_container.trigger_time = self.data.r0.tel[
self.camera_config.telescope_id].trigger_time
def initialize_mon_container(self):
"""
Fill with MonitoringContainer.
For the moment, initialize only the PixelStatusContainer
"""
container = self.data.mon
container.tels_with_data = [
self.camera_config.telescope_id,
]
mon_camera_container = container.tel[self.camera_config.telescope_id]
# initialize the container
status_container = PixelStatusContainer()
status_container.hardware_failing_pixels = np.zeros(
(self.n_gains, self.n_camera_pixels), dtype=bool)
status_container.pedestal_failing_pixels = np.zeros(
(self.n_gains, self.n_camera_pixels), dtype=bool)
status_container.flatfield_failing_pixels = np.zeros(
(self.n_gains, self.n_camera_pixels), dtype=bool)
mon_camera_container.pixel_status = status_container
def fill_mon_container_from_zfile(self, event):
"""
Fill with MonitoringContainer.
For the moment, initialize only the PixelStatusContainer
"""
status_container = self.data.mon.tel[
self.camera_config.telescope_id].pixel_status
# reorder the array
pixel_status = np.zeros(self.n_camera_pixels)
pixel_status[
self.camera_config.expected_pixels_id] = event.pixel_status
status_container.hardware_failing_pixels[:] = pixel_status == 0
'''
class TimeCorrectionCalculate(Component):
"""
The TimeCorrectionCalculate class to create h5py
file with coefficients for time correction curve
of chip DRS4.
Description of this method: "Analysis techniques
and performance of the Domino Ring Sampler version 4
based readout for the MAGIC telescopes [arxiv:1305.1007]
"""
minimum_charge = Float(
200, help='Cut on charge. Default 200 ADC').tag(config=True)
tel_id = Int(1, help='Id of the telescope to calibrate').tag(config=True)
n_combine = Int(
8, help='How many capacitors are combines in a single bin. Default 8'
).tag(config=True)
n_harmonics = Int(
16, help='Number of harmonic for Fourier series expansion. Default 16'
).tag(config=True)
n_capacitors = Int(
1024, help='Number of capacitors (1024 or 4096). Default 1024.').tag(
config=True)
charge_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
calib_file_path = Unicode(
'', allow_none=True,
help='Path to the time calibration file').tag(config=True)
def __init__(self, subarray, **kwargs):
"""
The TimeCorrectionCalculate class to create h5py
file with coefficients for time correction curve of chip DRS4.
Description of this method: "Analysis techniques and performance
of the Domino Ring Sampler version 4 based readout
for the MAGIC telescopes [arxiv:1305.1007]
Parameters
----------
subarray: ctapipe.instrument.SubarrayDescription
Description of the subarray. Provides information about the
camera which are useful in charge extraction, such as reference
pulse shape, sampling rate, neighboring pixels. Also required for
configuring the TelescopeParameter traitlets.
kwargs
"""
super().__init__(**kwargs)
self.n_bins = int(self.n_capacitors / self.n_combine)
self.mean_values_per_bin = np.zeros((n_gain, n_pixels, self.n_bins))
self.entries_per_bin = np.zeros((n_gain, n_pixels, self.n_bins))
self.first_cap_array = np.zeros((n_modules, n_gain, n_channel))
# load the waveform charge extractor
self.extractor = ImageExtractor.from_name(self.charge_product,
config=self.config,
subarray=subarray)
self.log.info(f"extractor {self.extractor}")
self.sum_events = 0
def calibrate_peak_time(self, event):
"""
Fill bins using time pulse from LocalPeakWindowSum.
Parameters
----------
event : `ctapipe` event-container
"""
if event.trigger.event_type == EventType.FLATFIELD:
for nr_module in prange(0, n_modules):
self.first_cap_array[
nr_module, :, :] = self.get_first_capacitor(
event, nr_module)
pixel_ids = event.lst.tel[self.tel_id].svc.pixel_ids
waveforms = event.r1.tel[self.tel_id].waveform
no_gain_selection = np.zeros(
(waveforms.shape[0], waveforms.shape[1]), dtype=np.int64)
# select both gain
charge, peak_time = self.extractor(
event.r1.tel[self.tel_id].waveform[:, :, :], self.tel_id,
no_gain_selection)
self.calib_peak_time_jit(charge,
peak_time,
pixel_ids,
self.first_cap_array,
self.mean_values_per_bin,
self.entries_per_bin,
n_cap=self.n_capacitors,
n_combine=self.n_combine,
min_charge=self.minimum_charge)
self.sum_events += 1
@staticmethod
@njit(parallel=True)
def calib_peak_time_jit(charge, peak_time, pixel_ids, first_cap_array,
mean_values_per_bin, entries_per_bin, n_cap,
n_combine, min_charge):
"""
Numba function for calibration pulse time.
Parameters
----------
pulse : ndarray
Pulse time stored in a numpy array of shape
(n_gain, n_pixels).
charge : ndarray
Charge in each pixel.
(n_gain, n_pixels).
pixel_ids: ndarray
Array stored expected pixel id
(n_pixels).
first_cap_array : ndarray
Value of first capacitor stored in a numpy array of shape
(n_clus, n_gain, n_pix).
mean_values_per_bin : ndarray
Array to fill using pulse time
stored in a numpy array of shape
(n_gain, n_pixels, n_bins).
entries_per_bin : ndarray
Array to store number of entries per bin
stored in a numpy array of shape
(n_gain, n_pixels, n_bins).
n_cap : int
Number of capacitors
n_combine : int
Number of combine capacitors in a single bin
"""
for nr_module in prange(n_modules):
for gain in prange(n_gain):
for pix in prange(n_channel):
pixel = pixel_ids[nr_module * 7 + pix]
if charge[gain, pixel] > min_charge: # cut change
fc = first_cap_array[nr_module, :, :]
first_cap = (fc[gain, pix]) % n_cap
bin = int(first_cap / n_combine)
mean_values_per_bin[gain, pixel,
bin] += peak_time[gain, pixel]
entries_per_bin[gain, pixel, bin] += 1
def finalize(self):
n_total = self.entries_per_bin.size
n_available = np.count_nonzero(self.entries_per_bin)
if n_available < n_total:
raise RuntimeError(
"No data available for some capacitors. "
"It might help to use more events to create the calibration file. "
f"Available: {n_available / n_total:.3%}, Missing: {n_total - n_available}"
)
else:
self.mean_values_per_bin = self.mean_values_per_bin / self.entries_per_bin
self.save_to_hdf5_file()
def fit(self, pixel_id, gain):
"""
Fit data bins using Fourier series expansion
Parameters
----------
pixel_id : ndarray
Array stored expected pixel id of shape
(n_pixels).
gain: int
0 for high gain, 1 for low gain
"""
self.pos = np.zeros(self.n_bins)
for i in range(0, self.n_bins):
self.pos[i] = (i + 0.5) * self.n_combine
self.fan = np.zeros(self.n_harmonics) # cos coeff
self.fbn = np.zeros(self.n_harmonics) # sin coeff
for n in range(0, self.n_harmonics):
self.integrate_with_trig(self.pos,
self.mean_values_per_bin[gain, pixel_id],
n, self.fan, self.fbn)
def integrate_with_trig(self, x, y, n, an, bn):
"""
Function to expanding into Fourier series
Parameters
----------
x : ndarray
Array stored position in DRS4 ring of shape
(n_bins).
y: ndarray
Array stored mean pulse time per bin of shape
(n_bins)
n : int
n harmonic
an: ndarray
Array to fill with cos coeff of shape
(n_harmonics)
bn: ndarray
Array to fill with sin coeff of shape
(n_harmonics)
"""
suma = 0
sumb = 0
for i in range(0, self.n_bins):
suma += y[i] * self.n_combine * np.cos(
2 * np.pi * n * (x[i] / float(self.n_capacitors)))
sumb += y[i] * self.n_combine * np.sin(
2 * np.pi * n * (x[i] / float(self.n_capacitors)))
an[n] = suma * (2. / (self.n_bins * self.n_combine))
bn[n] = sumb * (2. / (self.n_bins * self.n_combine))
def get_first_capacitor(self, event, nr):
fc = np.zeros((n_gain, n_channel))
first_cap = event.lst.tel[
self.tel_id].evt.first_capacitor_id[nr * 8:(nr + 1) * 8]
# First capacitor order according Dragon v5 board data format
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [0, 0, 1, 1, 2, 2, 3]):
fc[high_gain, i] = first_cap[j]
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [4, 4, 5, 5, 6, 6, 7]):
fc[low_gain, i] = first_cap[j]
return fc
def save_to_hdf5_file(self):
"""
Function to save Fourier series expansion coeff into hdf5 file
"""
fan_array = np.zeros((n_gain, n_pixels, self.n_harmonics))
fbn_array = np.zeros((n_gain, n_pixels, self.n_harmonics))
for pix_id in range(n_pixels):
self.fit(pix_id, gain=high_gain)
fan_array[high_gain, pix_id, :] = self.fan
fbn_array[high_gain, pix_id, :] = self.fbn
self.fit(pix_id, gain=low_gain)
fan_array[low_gain, pix_id, :] = self.fan
fbn_array[low_gain, pix_id, :] = self.fbn
try:
with h5py.File(self.calib_file_path, 'w') as hf:
hf.create_dataset('fan', data=fan_array)
hf.create_dataset('fbn', data=fbn_array)
hf.attrs['n_events'] = self.sum_events
hf.attrs['n_harm'] = self.n_harmonics
# need pytables and time calib container
# to use lstchain.io.add_config_metadata
hf.attrs['config'] = str(self.config)
metadata = global_metadata()
write_metadata(metadata, self.calib_file_path)
except Exception:
raise IOError(f"Failed to create the file {self.calib_file_path}")
class LSTCameraCalibrator(CameraCalibrator):
"""
Calibrator to handle the LST camera calibration chain, in order to fill
the DL1 data level in the event container.
"""
extractor_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
reducer_product = Unicode(
'NullDataVolumeReducer',
help='Name of the DataVolumeReducer to use').tag(config=True)
calibration_path = Unicode(
'', help='Path to LST calibration file').tag(config=True)
time_calibration_path = Unicode(
'', help='Path to drs4 time calibration file').tag(config=True)
allowed_tels = List(
[1], help='List of telescope to be calibrated').tag(config=True)
gain_threshold = Int(
4094, allow_none=True,
help='Threshold for the gain selection in ADC').tag(config=True)
charge_scale = List(
[1, 1],
help='Multiplicative correction factor for charge estimation [HG,LG]'
).tag(config=True)
def __init__(self, subarray, **kwargs):
"""
Parameters
----------
reducer_product : ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use. If None, then
NullDataVolumeReducer will be used by default, and waveforms
will not be reduced.
extractor_product : ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, then LocalPeakWindowSum
will be used by default.
calibration_path :
Path to LST calibration file to get the pedestal and flat-field corrections
kwargs
"""
super().__init__(subarray, **kwargs)
# load the waveform charge extractor
self.image_extractor = ImageExtractor.from_name(self.extractor_product,
subarray=self.subarray,
config=self.config)
self.log.info(f"extractor {self.extractor_product}")
print("EXTRACTOR", self.image_extractor)
self.data_volume_reducer = DataVolumeReducer.from_name(
self.reducer_product, subarray=self.subarray, config=self.config)
self.log.info(f" {self.reducer_product}")
# declare gain selector if the threshold is defined
if self.gain_threshold:
self.gain_selector = gainselection.ThresholdGainSelector(
threshold=self.gain_threshold)
# declare time calibrator if correction file exist
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
raise IOError(
f"Time calibration file {self.time_calibration_path} not found!"
)
# calibration data container
self.mon_data = MonitoringContainer()
# initialize the MonitoringContainer() for the moment it reads it from a hdf5 file
self._initialize_correction()
self.log.info(f"Global charge scale {self.charge_scale}")
def _initialize_correction(self):
"""
Read the correction from hdf5 calibration file
"""
self.mon_data.tels_with_data = self.allowed_tels
self.log.info(f"read {self.calibration_path}")
try:
with HDF5TableReader(self.calibration_path) as h5_table:
for telid in self.allowed_tels:
# read the calibration data
table = '/tel_' + str(telid) + '/calibration'
next(
h5_table.read(table,
self.mon_data.tel[telid].calibration))
# read pedestal data
table = '/tel_' + str(telid) + '/pedestal'
next(
h5_table.read(table,
self.mon_data.tel[telid].pedestal))
# read flat-field data
table = '/tel_' + str(telid) + '/flatfield'
next(
h5_table.read(table,
self.mon_data.tel[telid].flatfield))
# read the pixel_status container
table = '/tel_' + str(telid) + '/pixel_status'
next(
h5_table.read(table,
self.mon_data.tel[telid].pixel_status))
except Exception:
self.log.exception(
f"Problem in reading calibration file {self.calibration_path}")
raise
def _calibrate_dl0(self, event, telid):
"""
create dl0 level, for the moment copy the r1
"""
waveforms = event.r1.tel[telid].waveform
if self._check_r1_empty(waveforms):
return
# if not already done, initialize the event monitoring containers
if event.mon.tel[telid].calibration.dc_to_pe is None:
event.mon.tel[telid].calibration = self.mon_data.tel[
telid].calibration
event.mon.tel[telid].flatfield = self.mon_data.tel[telid].flatfield
event.mon.tel[telid].pedestal = self.mon_data.tel[telid].pedestal
event.mon.tel[telid].pixel_status = self.mon_data.tel[
telid].pixel_status
#
# subtract the pedestal per sample and multiply for the calibration coefficients
#
event.dl0.tel[telid].waveform = (
(waveforms - self.mon_data.tel[telid].calibration.
pedestal_per_sample[:, :, np.newaxis]) *
self.mon_data.tel[telid].calibration.dc_to_pe[:, :,
np.newaxis]).astype(
np.float32)
def _calibrate_dl1(self, event, telid):
"""
create calibrated dl1 image and calibrate it
"""
waveforms = event.dl0.tel[telid].waveform
if self._check_dl0_empty(waveforms):
return
# for the moment we do the gain selection afterwards
# use gain mask without gain selection
# TBD: - perform calibration of the R1 waveform (not DL1)
# - gain selection before charge integration
# In case of no gain selection the selected gain channels are [0,0,..][1,1,..]
no_gain_selection = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype=np.int)
no_gain_selection[1] = 1
charge = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype='float32')
peak_time = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype='float32')
# image extraction for each channel:
for i in range(waveforms.shape[0]):
charge[i], peak_time[i] = self.image_extractor(
waveforms[i], telid, no_gain_selection[i])
# correct charge for global scale
corrected_charge = charge * np.array(self.charge_scale,
dtype=np.float32)[:, np.newaxis]
# correct time with drs4 correction if available
if self.time_corrector:
peak_time = self.time_corrector.get_corr_pulse(event, peak_time)
# add flat-fielding time correction
peak_time_ff_corrected = peak_time + self.mon_data.tel[
telid].calibration.time_correction.value
# perform the gain selection if the threshold is defined
if self.gain_threshold:
gain_mask = self.gain_selector(event.r1.tel[telid].waveform)
event.dl1.tel[telid].image = corrected_charge[
gain_mask, np.arange(charge.shape[1])]
event.dl1.tel[telid].peak_time = \
peak_time_ff_corrected[gain_mask, np.arange(peak_time_ff_corrected.shape[1])].astype(np.float32)
# remember which channel has been selected
event.r1.tel[telid].selected_gain_channel = gain_mask
# if threshold == None
else:
event.dl1.tel[telid].image = corrected_charge
event.dl1.tel[telid].peak_time = peak_time_ff_corrected
class DragonPedestal(Component):
"""
The DragonPedestal class to create pedestal
for LST readout system using chip DRS4.
"""
r0_sample_start = Int(default_value=11,
help='Start sample for waveform'
).tag(config=True)
def __init__(self, tel_id, n_module, **kwargs):
super().__init__(**kwargs)
self.tel_id = tel_id
self.n_module = n_module # This is number of module read from data
# Readout system of LST has 265 modules.
# Each module has 7 channels (pixels)
self.n_pixels = n_module_in_camera*n_channel
self.meanped = np.zeros((n_gain, self.n_pixels, size4drs))
self.numped = np.zeros((n_gain, self.n_pixels, size4drs))
self.first_cap_array = np.zeros((self.n_module, n_gain, n_channel))
self.failing_pixels_array = np.full((self.n_pixels), False)
def fill_pedestal_event(self, event):
expected_pixel_id = event.lst.tel[self.tel_id].svc.pixel_ids
waveform = event.r0.tel[self.tel_id].waveform
for nr_module in prange(0, self.n_module):
self.first_cap_array[nr_module, :, :] = self.get_first_capacitor(event, nr_module)
self._fill_pedestal_event_jit(waveform,
expected_pixel_id,
self.first_cap_array,
self.meanped,
self.numped,
self.n_module,
self.r0_sample_start)
@staticmethod
@jit(parallel=True)
def _fill_pedestal_event_jit(waveform, expected_pixel_id, first_cap_array,
meanped, numped, n_module, start_sample_r0):
for nr_module in prange(0, n_module):
first_cap = first_cap_array[nr_module, :, :]
for gain in prange(0, n_gain):
for pix in prange(0, n_channel):
fc = first_cap[gain, pix]
pixel = expected_pixel_id[nr_module * 7 + pix]
posads0 = int((start_sample_r0+fc)%size4drs)
if posads0 + roi_size < size4drs:
# the first 9 samples have occasionally increased signal due to Tsutomu pattern,
# hence we skip them. Start sample might be set as script argument. Default = 11.
meanped[gain, pixel, posads0:((posads0-start_sample_r0) + roi_size-2)] += waveform[gain, pixel, start_sample_r0:roi_size-2]
numped[gain, pixel, posads0:((posads0-start_sample_r0) + roi_size-2)] += 1
else:
for k in prange(start_sample_r0, roi_size - 2):
# the first 9 samples have occasionally increased signal due to Tsutomu pattern,
# hence we skip them. Start sample might be set as script argument. Default = 11.
posads = int((k + fc) % size4drs)
val = waveform[gain, pixel, k]
meanped[gain, pixel, posads] += val
numped[gain, pixel, posads] += 1
def finalize_pedestal(self):
self.meanped = self.meanped / self.numped
pixels_with_nan_value = np.where(np.isnan(self.meanped).any(axis=0))
if len(pixels_with_nan_value[0]) > 0:
# Find failing pixels id
index_failing_pixels = np.unique(pixels_with_nan_value[0])
self.failing_pixels_array[index_failing_pixels] = True
print("Failing pixels:")
print(index_failing_pixels)
def get_first_capacitor(self, event, nr):
fc = np.zeros((2, 7))
first_cap = event.lst.tel[self.tel_id].evt.first_capacitor_id[nr * 8:(nr + 1) * 8]
# First capacitor order according Dragon v5 board data format
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [0, 0, 1, 1, 2, 2, 3]):
fc[high_gain, i] = first_cap[j]
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [4, 4, 5, 5, 6, 6, 7]):
fc[low_gain, i] = first_cap[j]
return fc
class CleanigPedestalImage(Component):
"""
Class to chceck pedestal image
"""
tel_id = Int(1, help='Id of the telescope to calibrate').tag(config=True)
charge_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
calib_file = Unicode('',
allow_none=True,
help='Path to the calibration file').tag(config=True)
calib_time_file = Unicode(
'', allow_none=True,
help="Path to the time calibration file").tag(config=True)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.cleaning_parameters = self.config["tailcut"]
print(self.config)
self.r1_dl1_calibrator = LSTCameraCalibrator(
calibration_path=self.calib_file,
time_calibration_path=self.calib_time_file,
extractor_product=self.charge_product,
config=self.config,
gain_threshold=Config(
self.config).gain_selector_config['threshold'],
allowed_tels=[1])
def run(self, list_of_file, max_events):
signal_place_after_clean = np.zeros(1855)
sum_ped_ev = 0
alive_ped_ev = 0
for input_file in list_of_file:
print(input_file)
r0_r1_calibrator = LSTR0Corrections(pedestal_path=None,
r1_sample_start=3,
r1_sample_end=39)
reader = LSTEventSource(input_url=input_file,
max_events=max_events)
for i, ev in enumerate(reader):
r0_r1_calibrator.calibrate(ev)
if i % 10000 == 0:
print(ev.r0.event_id)
if ev.lst.tel[1].evt.tib_masked_trigger == 32:
sum_ped_ev += 1
self.r1_dl1_calibrator(ev)
img = ev.dl1.tel[1].image
geom = ev.inst.subarray.tel[1].camera
clean = tailcuts_clean(geom, img,
**self.cleaning_parameters)
cleaned = img.copy()
cleaned[~clean] = 0.0
signal_place_after_clean[np.where(clean == True)] += 1
if np.sum(cleaned > 0) > 0:
alive_ped_ev += 1
fig, ax = plt.subplots(figsize=(10, 8))
geom = ev.inst.subarray.tel[1].camera
disp0 = CameraDisplay(geom, ax=ax)
disp0.image = signal_place_after_clean / sum_ped_ev
disp0.add_colorbar(ax=ax,
label="N times signal remain after cleaning [%]")
disp0.cmap = 'gnuplot2'
ax.set_title("{} \n {}/{}".format(
input_file.split("/")[-1][8:21], alive_ped_ev, sum_ped_ev),
fontsize=25)
print("{}/{}".format(alive_ped_ev, sum_ped_ev))
ax.set_xlabel(" ")
ax.set_ylabel(" ")
plt.tight_layout()
plt.show()
def remove_star_and_run(self, list_of_file, max_events,
noise_pixels_id_list):
signal_place_after_clean = np.zeros(1855)
sum_ped_ev = 0
alive_ped_ev = 0
for input_file in list_of_file:
print(input_file)
r0_r1_calibrator = LSTR0Corrections(pedestal_path=None,
r1_sample_start=3,
r1_sample_end=39)
reader = LSTEventSource(input_url=input_file,
max_events=max_events)
for i, ev in enumerate(reader):
r0_r1_calibrator.calibrate(ev)
if i % 10000 == 0:
print(ev.r0.event_id)
if ev.lst.tel[1].evt.tib_masked_trigger == 32:
sum_ped_ev += 1
self.r1_dl1_calibrator(ev)
img = ev.dl1.tel[1].image
img[noise_pixels_id_list] = 0
geom = ev.inst.subarray.tel[1].camera
clean = tailcuts_clean(geom, img,
**self.cleaning_parameters)
cleaned = img.copy()
cleaned[~clean] = 0.0
signal_place_after_clean[np.where(clean == True)] += 1
if np.sum(cleaned > 0) > 0:
alive_ped_ev += 1
fig, ax = plt.subplots(figsize=(10, 8))
geom = ev.inst.subarray.tel[1].camera
disp0 = CameraDisplay(geom, ax=ax)
disp0.image = signal_place_after_clean / sum_ped_ev
disp0.highlight_pixels(noise_pixels_id_list, linewidth=3)
disp0.add_colorbar(ax=ax,
label="N times signal remain after cleaning [%]")
disp0.cmap = 'gnuplot2'
ax.set_title("{} \n {}/{}".format(
input_file.split("/")[-1][8:21], alive_ped_ev, sum_ped_ev),
fontsize=25)
print("{}/{}".format(alive_ped_ev, sum_ped_ev))
ax.set_xlabel(" ")
ax.set_ylabel(" ")
plt.tight_layout()
plt.show()
def plot_camera_display(self, image, input_file, noise_pixels_id_list,
alive_ped_ev, sum_ped_ev):
fig, ax = plt.subplots(figsize=(10, 8))
geom = CameraGeometry.from_name('LSTCam-003')
disp0 = CameraDisplay(geom, ax=ax)
disp0.image = image
disp0.highlight_pixels(noise_pixels_id_list, linewidth=3)
disp0.add_colorbar(ax=ax,
label="N times signal remain after cleaning [%]")
disp0.cmap = 'gnuplot2'
ax.set_title("{} \n {}/{}".format(
input_file.split("/")[-1][8:21], alive_ped_ev, sum_ped_ev),
fontsize=25)
print("{}/{}".format(alive_ped_ev, sum_ped_ev))
ax.set_xlabel(" ")
ax.set_ylabel(" ")
plt.tight_layout()
plt.show()
def check_interleave_pedestal_cleaning(self, list_of_file, max_events,
sigma, dl1_file):
high_gain = 0
ped_mean_pe, ped_rms_pe = get_bias_and_rms(dl1_file)
bad_pixel_ids = np.where(ped_rms_pe[1, high_gain, :] == 0)[0]
print(bad_pixel_ids)
th = get_threshold(ped_mean_pe[1, high_gain, :],
ped_rms_pe[1, high_gain, :], sigma)
make_camera_binary_image(th, sigma,
self.cleaning_parameters['picture_thresh'],
bad_pixel_ids)
signal_place_after_clean = np.zeros(1855)
sum_ped_ev = 0
alive_ped_ev = 0
for input_file in list_of_file:
print(input_file)
r0_r1_calibrator = LSTR0Corrections(pedestal_path=None,
r1_sample_start=3,
r1_sample_end=39)
reader = LSTEventSource(input_url=input_file,
max_events=max_events)
for i, ev in enumerate(reader):
r0_r1_calibrator.calibrate(ev)
if i % 10000 == 0:
print(ev.r0.event_id)
if ev.lst.tel[1].evt.tib_masked_trigger == 32:
sum_ped_ev += 1
self.r1_dl1_calibrator(ev)
img = ev.dl1.tel[1].image
img[bad_pixel_ids] = 0
geom = ev.inst.subarray.tel[1].camera
clean = tailcuts_pedestal_clean(geom, img, th,
**self.cleaning_parameters)
cleaned = img.copy()
cleaned[~clean] = 0.0
signal_place_after_clean[np.where(clean == True)] += 1
if np.sum(cleaned > 0) > 0:
alive_ped_ev += 1
noise_remain = signal_place_after_clean / sum_ped_ev
self.plot_camera_display(noise_remain, input_file, bad_pixel_ids,
alive_ped_ev, sum_ped_ev)
class SingleTelEventDisplay(Tool):
name = "ctapipe-display-televents"
description = Unicode(__doc__)
infile = Unicode(help="input file to read", default='').tag(config=True)
tel = Int(help='Telescope ID to display', default=0).tag(config=True)
channel = Integer(help="channel number to display", min=0,
max=1).tag(config=True)
write = Bool(help="Write out images to PNG files",
default=False).tag(config=True)
clean = Bool(help="Apply image cleaning", default=False).tag(config=True)
hillas = Bool(help="Apply and display Hillas parametrization",
default=False).tag(config=True)
samples = Bool(help="Show each sample", default=False).tag(config=True)
display = Bool(help="Display results in interactive window",
default_value=True).tag(config=True)
delay = Float(help='delay between events in s',
default_value=0.01,
min=0.001).tag(config=True)
progress = Bool(help='display progress bar',
default_value=True).tag(config=True)
aliases = Dict({
'infile': 'SingleTelEventDisplay.infile',
'tel': 'SingleTelEventDisplay.tel',
'max-events': 'EventSource.max_events',
'channel': 'SingleTelEventDisplay.channel',
'write': 'SingleTelEventDisplay.write',
'clean': 'SingleTelEventDisplay.clean',
'hillas': 'SingleTelEventDisplay.hillas',
'samples': 'SingleTelEventDisplay.samples',
'display': 'SingleTelEventDisplay.display',
'delay': 'SingleTelEventDisplay.delay',
'progress': 'SingleTelEventDisplay.progress'
})
classes = List([EventSource, CameraCalibrator])
def __init__(self, **kwargs):
super().__init__(**kwargs)
def setup(self):
print('TOLLES INFILE', self.infile)
self.event_source = EventSource.from_url(self.infile, parent=self)
self.event_source.allowed_tels = {
self.tel,
}
self.calibrator = CameraCalibrator(parent=self)
self.log.info(f'SELECTING EVENTS FROM TELESCOPE {self.tel}')
def start(self):
disp = None
for event in tqdm(self.event_source,
desc=f'Tel{self.tel}',
total=self.event_source.max_events,
disable=~self.progress):
self.log.debug(event.trig)
self.log.debug(f"Energy: {event.mc.energy}")
self.calibrator(event)
if disp is None:
geom = event.inst.subarray.tel[self.tel].camera
self.log.info(geom)
disp = CameraDisplay(geom)
# disp.enable_pixel_picker()
disp.add_colorbar()
if self.display:
plt.show(block=False)
# display the event
disp.axes.set_title('CT{:03d} ({}), event {:06d}'.format(
self.tel, geom.cam_id, event.r0.event_id))
if self.samples:
# display time-varying event
data = event.dl0.tel[self.tel].waveform[self.channel]
for ii in range(data.shape[1]):
disp.image = data[:, ii]
disp.set_limits_percent(70)
plt.suptitle(f"Sample {ii:03d}")
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f'CT{self.tel:03d}_EV{event.r0.event_id:10d}'
f'_S{ii:02d}.png')
else:
# display integrated event:
im = event.dl1.tel[self.tel].image[self.channel]
if self.clean:
mask = tailcuts_clean(geom,
im,
picture_thresh=10,
boundary_thresh=7)
im[~mask] = 0.0
disp.image = im
if self.hillas:
try:
ellipses = disp.axes.findobj(Ellipse)
if len(ellipses) > 0:
ellipses[0].remove()
params = hillas_parameters(geom, image=im)
disp.overlay_moments(params,
color='pink',
lw=3,
with_label=False)
except HillasParameterizationError:
pass
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f'CT{self.tel:03d}_EV{event.r0.event_id:010d}.png')
self.log.info("FINISHED READING DATA FILE")
if disp is None:
self.log.warning(
'No events for tel {} were found in {}. Try a '
'different EventIO file or another telescope'.format(
self.tel, self.infile), )
class LSTEventSource(EventSource):
"""EventSource for LST r0 data."""
n_gains = Int(
2,
help='Number of gains at r0/r1 level'
).tag(config=True)
baseline = Int(
400,
help='r0 waveform baseline (default from EvB v3)'
).tag(config=True)
multi_streams = Bool(
True,
help='Read in parallel all streams '
).tag(config=True)
def __init__(self, **kwargs):
"""
Constructor
Parameters
----------
n_gains = number of gains expected in input file
baseline = baseline to be subtracted at r1 level (not used for the moment)
multi_streams = enable the reading of input files from all streams
config: traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.\
kwargs: dict
Additional parameters to be passed.
NOTE: The file mask of the data to read can be passed with
the 'input_url' parameter.
"""
# EventSource can not handle file wild cards as input_url
# To overcome this we substitute the input_url with first file matching
# the specified file mask (copied from MAGICEventSourceROOT).
super().__init__(**kwargs)
if self.multi_streams:
# test how many streams are there:
# file name must be [stream name]Run[all the rest]
# All the files with the same [all the rest] are opened
if '/' in self.input_url:
dir, name = self.input_url.rsplit('/', 1)
else:
dir = getcwd()
name = self.input_url
if 'Run' in name:
stream, run = name.split('Run', 1)
else:
run = name
ls = listdir(dir)
self.file_list = []
for file_name in ls:
if run in file_name:
full_name = dir + '/' + file_name
self.file_list.append(full_name)
Provenance().add_input_file(full_name, role='dl0.sub.evt')
else:
self.file_list = [self.input_url]
self.multi_file = MultiFiles(self.file_list)
self.camera_config = self.multi_file.camera_config
self.log.info(
"Read {} input files".format(
self.multi_file.num_inputs()
)
)
def rewind(self):
self.multi_file.rewind()
def _generator(self):
# container for LST data
self.data = LSTDataContainer()
self.data.meta['input_url'] = self.input_url
self.data.meta['max_events'] = self.max_events
self.data.meta['origin'] = 'LSTCAM'
# fill LST data from the CameraConfig table
self.fill_lst_service_container_from_zfile()
# Instrument information
for tel_id in self.data.lst.tels_with_data:
assert (tel_id == 0 or tel_id == 1) # only LST1 (for the moment id = 0)
# optics info from standard optics.fits.gz file
optics = OpticsDescription.from_name("LST")
# camera info from LSTCam-[geometry_version].camgeom.fits.gz file
geometry_version = 2
camera = CameraGeometry.from_name("LSTCam", geometry_version)
tel_descr = TelescopeDescription(
name='LST', tel_type='LST', optics=optics, camera=camera
)
self.n_camera_pixels = tel_descr.camera.n_pixels
tels = {tel_id: tel_descr}
# LSTs telescope position taken from MC from the moment
tel_pos = {tel_id: [50., 50., 16] * u.m}
subarray = SubarrayDescription("LST1 subarray")
subarray.tels = tels
subarray.positions = tel_pos
self.data.inst.subarray = subarray
# initialize general monitoring container
self.initialize_mon_container()
# loop on events
for count, event in enumerate(self.multi_file):
self.data.count = count
# fill specific LST event data
self.fill_lst_event_container_from_zfile(event)
# fill general monitoring data
self.fill_mon_container_from_zfile(event)
# fill general R0 data
self.fill_r0_container_from_zfile(event)
yield self.data
@staticmethod
def is_compatible(file_path):
from astropy.io import fits
try:
# The file contains two tables:
# 1: CameraConfig
# 2: Events
h = fits.open(file_path)[2].header
ttypes = [
h[x] for x in h.keys() if 'TTYPE' in x
]
except OSError:
# not even a fits file
return False
except IndexError:
# A fits file of a different format
return False
is_protobuf_zfits_file = (
(h['XTENSION'] == 'BINTABLE') and
(h['EXTNAME'] == 'Events') and
(h['ZTABLE'] is True) and
(h['ORIGIN'] == 'CTA') and
(h['PBFHEAD'] == 'R1.CameraEvent')
)
is_lst_file = 'lstcam_counters' in ttypes
return is_protobuf_zfits_file & is_lst_file
def fill_lst_service_container_from_zfile(self):
"""
Fill LSTServiceContainer with specific LST service data data
(from the CameraConfig table of zfit file)
"""
self.data.lst.tels_with_data = [self.camera_config.telescope_id, ]
svc_container = self.data.lst.tel[self.camera_config.telescope_id].svc
svc_container.telescope_id = self.camera_config.telescope_id
svc_container.cs_serial = self.camera_config.cs_serial
svc_container.configuration_id = self.camera_config.configuration_id
svc_container.date = self.camera_config.date
svc_container.num_pixels = self.camera_config.num_pixels
svc_container.num_samples = self.camera_config.num_samples
svc_container.pixel_ids = self.camera_config.expected_pixels_id
svc_container.data_model_version = self.camera_config.data_model_version
svc_container.num_modules = self.camera_config.lstcam.num_modules
svc_container.module_ids = self.camera_config.lstcam.expected_modules_id
svc_container.idaq_version = self.camera_config.lstcam.idaq_version
svc_container.cdhs_version = self.camera_config.lstcam.cdhs_version
svc_container.algorithms = self.camera_config.lstcam.algorithms
svc_container.pre_proc_algorithms = self.camera_config.lstcam.pre_proc_algorithms
def fill_lst_event_container_from_zfile(self, event):
"""
Fill LSTEventContainer with specific LST service data
(from the Event table of zfit file)
"""
event_container = self.data.lst.tel[self.camera_config.telescope_id].evt
event_container.configuration_id = event.configuration_id
event_container.event_id = event.event_id
event_container.tel_event_id = event.tel_event_id
event_container.pixel_status = event.pixel_status
event_container.ped_id = event.ped_id
event_container.module_status = event.lstcam.module_status
event_container.extdevices_presence = event.lstcam.extdevices_presence
# unpack TIB data
rec_fmt = '=IHIBB'
unpacked_tib = struct.unpack(rec_fmt, event.lstcam.tib_data)
event_container.tib_event_counter = unpacked_tib[0]
event_container.tib_pps_counter = unpacked_tib[1]
event_container.tib_tenMHz_counter = unpacked_tib[2]
event_container.tib_stereo_pattern = unpacked_tib[3]
event_container.tib_masked_trigger = unpacked_tib[4]
event_container.swat_data = event.lstcam.swat_data
# unpack CDTS data
rec_fmt = '=IIIQQBBB'
unpacked_cdts = struct.unpack(rec_fmt, event.lstcam.cdts_data)
event_container.ucts_event_counter = unpacked_cdts[0]
event_container.ucts_pps_counter = unpacked_cdts[1]
event_container.ucts_clock_counter = unpacked_cdts[2]
event_container.ucts_timestamp = unpacked_cdts[3]
event_container.ucts_camera_timestamp = unpacked_cdts[4]
event_container.ucts_trigger_type = unpacked_cdts[5]
event_container.ucts_white_rabbit_status = unpacked_cdts[6]
# unpack Dragon counters
rec_fmt = '=HIIIQ'
rec_len = struct.calcsize(rec_fmt)
rec_unpack = struct.Struct(rec_fmt).unpack_from
event_container.pps_counter = np.zeros(self.camera_config.lstcam.num_modules)
event_container.tenMHz_counter = np.zeros(self.camera_config.lstcam.num_modules)
event_container.event_counter = np.zeros(self.camera_config.lstcam.num_modules)
event_container.trigger_counter = np.zeros(self.camera_config.lstcam.num_modules)
event_container.local_clock_counter = np.zeros(self.camera_config.lstcam.num_modules)
for mod in range(self.camera_config.lstcam.num_modules):
words=event.lstcam.counters[mod*rec_len:(mod+1)*rec_len]
unpacked_counter = rec_unpack(words)
event_container.pps_counter[mod] = unpacked_counter[0]
event_container.tenMHz_counter[mod] = unpacked_counter[1]
event_container.event_counter[mod] = unpacked_counter[2]
event_container.trigger_counter[mod] = unpacked_counter[3]
event_container.local_clock_counter[mod] = unpacked_counter[4]
event_container.chips_flags = event.lstcam.chips_flags
event_container.first_capacitor_id = event.lstcam.first_capacitor_id
event_container.drs_tag_status = event.lstcam.drs_tag_status
event_container.drs_tag = event.lstcam.drs_tag
def fill_r0_camera_container_from_zfile(self, r0_container, event):
"""
Fill with R0CameraContainer
"""
# temporary patch to have an event time set
r0_container.trigger_time = (
self.data.lst.tel[self.camera_config.telescope_id].evt.tib_pps_counter +
self.data.lst.tel[self.camera_config.telescope_id].evt.tib_tenMHz_counter * 10**(-7))
if r0_container.trigger_time is None:
r0_container.trigger_time = 0
#r0_container.trigger_type = event.trigger_type
r0_container.trigger_type = self.data.lst.tel[self.camera_config.telescope_id].evt.tib_masked_trigger
# verify the number of gains
if event.waveform.shape[0] != self.camera_config.num_pixels * self.camera_config.num_samples * self.n_gains:
raise ValueError(f"Number of gains not correct, waveform shape is {event.waveform.shape[0]}"
f" instead of "
f"{self.camera_config.num_pixels * self.camera_config.num_samples * self.n_gains}")
reshaped_waveform = np.array(
event.waveform
).reshape(
self.n_gains,
self.camera_config.num_pixels,
self.camera_config.num_samples
)
# initialize the waveform container to zero
r0_container.waveform = np.zeros([self.n_gains, self.n_camera_pixels,
self.camera_config.num_samples])
# re-order the waveform following the expected_pixels_id values
# (rank = pixel id)
r0_container.waveform[:, self.camera_config.expected_pixels_id, :] =\
reshaped_waveform
def fill_r0_container_from_zfile(self, event):
"""
Fill with R0Container
"""
container = self.data.r0
container.obs_id = -1
container.event_id = event.event_id
container.tels_with_data = [self.camera_config.telescope_id, ]
r0_camera_container = container.tel[self.camera_config.telescope_id]
self.fill_r0_camera_container_from_zfile(
r0_camera_container,
event
)
def initialize_mon_container(self):
"""
Fill with MonitoringContainer.
For the moment, initialize only the PixelStatusContainer
"""
container = self.data.mon
container.tels_with_data = [self.camera_config.telescope_id, ]
mon_camera_container = container.tel[self.camera_config.telescope_id]
# initialize the container
status_container = PixelStatusContainer()
status_container.hardware_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)
status_container.pedestal_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)
status_container.flatfield_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)
mon_camera_container.pixel_status = status_container
def fill_mon_container_from_zfile(self, event):
"""
Fill with MonitoringContainer.
For the moment, initialize only the PixelStatusContainer
"""
status_container = self.data.mon.tel[self.camera_config.telescope_id].pixel_status
# reorder the array
pixel_status = np.zeros(self.n_camera_pixels)
pixel_status[self.camera_config.expected_pixels_id] = event.pixel_status
status_container.hardware_failing_pixels[:] = pixel_status == 0
class LSTCameraCalibrator(CameraCalibrator):
"""
Calibrator to handle the LST camera calibration chain, in order to fill
the DL1 data level in the event container.
"""
extractor_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
reducer_product = Unicode(
'NullDataVolumeReducer',
help='Name of the DataVolumeReducer to use').tag(config=True)
calibration_path = Path(
exists=True, directory_ok=False,
help='Path to LST calibration file').tag(config=True)
time_calibration_path = Path(
exists=True,
directory_ok=False,
help='Path to drs4 time calibration file').tag(config=True)
time_sampling_correction_path = Path(
exists=True,
directory_ok=False,
help='Path to time sampling correction file',
allow_none=True,
).tag(config=True)
allowed_tels = List(
[1], help='List of telescope to be calibrated').tag(config=True)
gain_threshold = Int(
4094, allow_none=True,
help='Threshold for the gain selection in ADC').tag(config=True)
charge_scale = List(
[1, 1],
help='Multiplicative correction factor for charge estimation [HG,LG]'
).tag(config=True)
def __init__(self, subarray, **kwargs):
"""
Parameters
----------
reducer_product : ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use. If None, then
NullDataVolumeReducer will be used by default, and waveforms
will not be reduced.
extractor_product : ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, then LocalPeakWindowSum
will be used by default.
calibration_path :
Path to LST calibration file to get the pedestal and flat-field corrections
kwargs
"""
super().__init__(subarray, **kwargs)
# load the waveform charge extractor
self.image_extractor = ImageExtractor.from_name(self.extractor_product,
subarray=self.subarray,
config=self.config)
self.log.info(f"extractor {self.extractor_product}")
print("EXTRACTOR", self.image_extractor)
self.data_volume_reducer = DataVolumeReducer.from_name(
self.reducer_product, subarray=self.subarray, config=self.config)
self.log.info(f" {self.reducer_product}")
# declare gain selector if the threshold is defined
if self.gain_threshold:
self.gain_selector = gainselection.ThresholdGainSelector(
threshold=self.gain_threshold)
# declare time calibrator if correction file exist
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
raise IOError(
f"Time calibration file {self.time_calibration_path} not found!"
)
# declare the charge sampling corrector
if self.time_sampling_correction_path is not None:
# search the file in resources if not found
if not os.path.exists(self.time_sampling_correction_path):
self.time_sampling_correction_path = resource_filename(
'lstchain',
f"resources/{self.time_sampling_correction_path}")
if os.path.exists(self.time_sampling_correction_path):
self.time_sampling_corrector = TimeSamplingCorrection(
time_sampling_correction_path=self.
time_sampling_correction_path)
else:
raise IOError(
f"Sampling correction file {self.time_sampling_correction_path} not found!"
)
else:
self.time_sampling_corrector = None
# calibration data container
self.mon_data = MonitoringContainer()
# initialize the MonitoringContainer() for the moment it reads it from a hdf5 file
self._initialize_correction()
self.log.info(f"Global charge scale {self.charge_scale}")
def _initialize_correction(self):
"""
Read the correction from hdf5 calibration file
"""
self.log.info(f"read {self.calibration_path}")
try:
with HDF5TableReader(self.calibration_path) as h5_table:
for telid in self.allowed_tels:
# read the calibration data
table = '/tel_' + str(telid) + '/calibration'
next(
h5_table.read(table,
self.mon_data.tel[telid].calibration))
# read pedestal data
table = '/tel_' + str(telid) + '/pedestal'
next(
h5_table.read(table,
self.mon_data.tel[telid].pedestal))
# read flat-field data
table = '/tel_' + str(telid) + '/flatfield'
next(
h5_table.read(table,
self.mon_data.tel[telid].flatfield))
# read the pixel_status container
table = '/tel_' + str(telid) + '/pixel_status'
next(
h5_table.read(table,
self.mon_data.tel[telid].pixel_status))
except Exception:
self.log.exception(
f"Problem in reading calibration file {self.calibration_path}")
raise
def _calibrate_dl0(self, event, telid):
"""
create dl0 level, with gain-selected and calibrated waveform
"""
waveforms = event.r1.tel[telid].waveform
if self._check_r1_empty(waveforms):
return
# if not already done, initialize the event monitoring containers
if event.mon.tel[telid].calibration.dc_to_pe is None:
event.mon.tel[telid].calibration = self.mon_data.tel[
telid].calibration
event.mon.tel[telid].flatfield = self.mon_data.tel[telid].flatfield
event.mon.tel[telid].pedestal = self.mon_data.tel[telid].pedestal
event.mon.tel[telid].pixel_status = self.mon_data.tel[
telid].pixel_status
#
# subtract the pedestal per sample and multiply for the calibration coefficients
#
calibrated_waveform = (
(waveforms - self.mon_data.tel[telid].calibration.
pedestal_per_sample[:, :, np.newaxis]) *
self.mon_data.tel[telid].calibration.dc_to_pe[:, :,
np.newaxis]).astype(
np.float32)
# If requested, perform gain selection (this will be done by the EvB in future)
# find the gain selection mask
if waveforms.ndim == 3:
# if threshold defined, perform gain selection
if self.gain_threshold:
gain_mask = self.gain_selector(waveforms)
# select the samples
calibrated_waveform = calibrated_waveform[
gain_mask, np.arange(waveforms.shape[1])]
else:
# keep both HG and LG
gain_mask = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype=np.int64)
gain_mask[1] = 1
else:
# gain selection already performed in EvB: (0=HG, 1=LG)
gain_mask = event.lst.tel[telid].evt.pixel_status >> 2 & 1
# remember the calibrated and gain selected waveform
# (this should be the r1 waveform to be compliant with ctapipe (?))
event.dl0.tel[telid].waveform = calibrated_waveform
# remember which channel has been selected
event.r1.tel[telid].selected_gain_channel = gain_mask
event.dl0.tel[telid].selected_gain_channel = gain_mask
def _calibrate_dl1(self, event, telid):
"""
create calibrated dl1 image and calibrate it
"""
n_pixels = self.subarray.tels[telid].camera.geometry.n_pixels
# copy the waveform be cause I do not want to change it
waveforms = np.copy(event.dl0.tel[telid].waveform)
gain_mask = event.dl0.tel[telid].selected_gain_channel
if self._check_dl0_empty(waveforms):
return
# In case of no gain selection the selected gain channels are [0,0,..][1,1,..]
no_gain_selection = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype=np.int64)
no_gain_selection[1] = 1
# correct the dl0 waveform for the sampling time corrections
if self.time_sampling_corrector:
waveforms *= self.time_sampling_corrector.get_corrections(
event, telid)[gain_mask, np.arange(n_pixels)]
# extract the charge
charge, peak_time = self.image_extractor(waveforms, telid, gain_mask)
# correct charge for global scale
corrected_charge = charge * np.array(self.charge_scale,
dtype=np.float32)[gain_mask]
# correct time with drs4 correction if available
if self.time_corrector:
peak_time_drs4_corrected = (
peak_time - self.time_corrector.get_pulse_time_corrections(
event)[gain_mask, np.arange(n_pixels)])
# add flat-fielding time correction
peak_time_ff_corrected = (
peak_time_drs4_corrected +
self.mon_data.tel[telid].calibration.time_correction.value[
gain_mask, np.arange(n_pixels)])
# fill dl1 container
event.dl1.tel[telid].image = corrected_charge
event.dl1.tel[telid].peak_time = peak_time_ff_corrected.astype(
np.float32)
class ChargeResolutionCalculator(Component):
"""
Class to handle the calculation of Charge Resolution.
Attributes
----------
max_pe : int
Maximum pe to calculate the charge resolution up to.
sum_dict : dict
Dictionary to store the running sum for each true charge.
n_dict : dict
Dictionary to store the running number for each true charge.
variation_hist_nbins : float
Number of bins for the variation histogram.
variation_hist_range : list
X and Y range for the variation histogram.
variation_hist : `np.histogram2d`
variation_xedges : ndarray
Edges of the X bins for the variation histogram.
variation_yedges : ndarray
Edges of the Y bins for the variation histogram.
"""
max_pe = Int(2000,
help='Maximum pe to calculate the charge resolution '
'up to').tag(config=True)
binning = Int(60,
allow_none=True,
help='Number of bins for the Charge Resolution. If None, '
'no binning is performed.').tag(config=True)
log_bins = Bool(True,
help='Bin the x axis linearly instead of '
'logarithmic.').tag(config=True)
def __init__(self, config=None, tool=None, **kwargs):
"""
Calculator of charge resolution.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
super().__init__(config=config, parent=tool, **kwargs)
self.sum_array = np.zeros(self.max_pe)
self.n_array = np.zeros(self.max_pe)
self.sum_dict = {}
self.n_dict = {}
self.variation_hist_nbins = log10(self.max_pe) * 50
self.variation_hist_range = [[log10(1), log10(self.max_pe)],
[log10(1), log10(self.max_pe)]]
h, xedges, yedges = np.histogram2d([np.nan], [np.nan],
bins=self.variation_hist_nbins,
range=self.variation_hist_range)
self.variation_hist = h
self.variation_xedges = xedges
self.variation_yedges = yedges
self.storage_arrays = [
'max_pe', 'sum_array', 'n_array', 'variation_hist_nbins',
'variation_hist_range', 'variation_hist', 'variation_xedges',
'variation_yedges'
]
def add_charges(self, true_charge, measured_charge):
"""
Fill the class parameters with a numpy array of true charge and
measured (calibrated) charge from an event. The two arrays must be the
same size.
Parameters
----------
true_charge : ndarray
Array of true (MC) charge.
Obtained from event.mc.tel[telid].image[channel]
measured_charge : ndarray
Array of measured (dl1 calibrated) charge.
Obtained from event.mc.tel[tel_id].photo_electron_image
"""
above_0 = (measured_charge > 0) & (true_charge > 0)
x = true_charge[above_0]
y = measured_charge[above_0]
h, _, _ = np.histogram2d(np.log10(y),
np.log10(x),
bins=self.variation_hist_nbins,
range=self.variation_hist_range)
self.variation_hist += h
in_range = (true_charge > 0) & (true_charge <= self.max_pe)
true_q = true_charge[in_range]
measured_q = measured_charge[in_range]
np.add.at(self.sum_array, true_q - 1, np.power(measured_q - true_q, 2))
np.add.at(self.n_array, true_q - 1, 1)
def get_charge_resolution(self):
"""
Calculate and obtain the charge resolution graph arrays.
Returns
-------
true_charge : ndarray
The X axis true charges.
chargeres : ndarray
The Y axis charge resolution values.
chargeres_error : ndarray
The error on the charge resolution.
scaled_chargeres : ndarray
The Y axis charge resolution divided by the Goal.
scaled_chargeres_error : ndarray
The error on the charge resolution divided by the Goal.
"""
self.log.debug('[chargeres] Calculating charge resolution')
n_1 = self.n_array > 0
n = self.n_array[n_1]
true = (np.arange(self.max_pe) + 1)[n_1]
sum_ = self.sum_array[n_1]
res = np.sqrt((sum_ / n) + true) / true
res_error = res * (1 / np.sqrt(2 * n))
scale = self.goal(true)
scaled_res = res / scale
scaled_res_error = res_error / scale
if self.binning is not None:
x = true
if self.log_bins:
x = np.log10(true)
def binning(array):
return bs(x, array, 'mean', bins=self.binning)
def sum_errors(array):
return np.sqrt(np.sum(np.power(array, 2))) / array.size
def bin_errors(array):
return bs(x, array, sum_errors, bins=self.binning)
true, _, _ = binning(true)
res, _, _ = binning(res)
res_error, _, _ = bin_errors(res_error)
scaled_res, _, _ = binning(scaled_res)
scaled_res_error, _, _ = bin_errors(scaled_res_error)
return true, res, res_error, scaled_res, scaled_res_error
@staticmethod
def limit_curves(npe, n_nsb, n_add, enf, sigma2):
"""
Equation for calculating the Goal and Requirement curves, as defined
in SCI-MC/121113.
https://portal.cta-observatory.org/recordscentre/Records/SCI/
SCI-MC/measurment_errors_system_performance_1YQCBC.pdf
Parameters
----------
npe : ndarray
Number of photoeletrons (variable).
n_nsb : float
Number of NSB photons.
n_add : float
Number of photoelectrons from additional noise sources.
enf : float
Excess noise factor.
sigma2 : float
Percentage ofmultiplicative errors.
"""
return (np.sqrt((n_nsb + n_add) + np.power(enf, 2) * npe +
np.power(sigma2 * npe, 2)) / npe).astype(float)
@staticmethod
def requirement(npe):
"""
CTA requirement curve.
Parameters
----------
npe : ndarray
Number of photoeletrons (variable).
"""
n_nsb = sqrt(4.0 + 3.0)
n_add = 0
enf = 1.2
sigma2 = 0.1
defined_npe = 1000
# If npe is not an array, temporarily convert it to one
npe = np.array([npe])
lc = ChargeResolutionCalculator.limit_curves
requirement = lc(npe, n_nsb, n_add, enf, sigma2)
requirement[npe > defined_npe] = np.nan
return requirement[0]
@staticmethod
def goal(npe):
"""
CTA goal curve.
Parameters
----------
npe : ndarray
Number of photoeletrons (variable).
"""
n_nsb = 2
n_add = 0
enf = 1.1152
sigma2 = 0.05
defined_npe = 2000
# If npe is not an array, temporarily convert it to one
npe = np.array([npe])
lc = ChargeResolutionCalculator.limit_curves
goal = lc(npe, n_nsb, n_add, enf, sigma2)
goal[npe > defined_npe] = np.nan
return goal[0]
@staticmethod
def poisson(npe):
"""
Poisson limit curve.
Parameters
----------
npe : ndarray
Number of photoeletrons (variable).
"""
# If npe is not an array, temporarily convert it to one
npe = np.array([npe])
poisson = np.sqrt(npe) / npe
return poisson[0]
def save(self, path):
output_dir = dirname(path)
if not exists(output_dir):
self.log.info("[output] Creating directory: {}".format(output_dir))
makedirs(output_dir)
self.log.info("Saving Charge Resolution file: {}".format(path))
with open_file(path, mode="w", title="ChargeResolutionFile") as f:
group = f.create_group("/", 'ChargeResolution', '')
for arr in self.storage_arrays:
f.create_array(group, arr, getattr(self, arr), arr)
def load(self, path):
self.log.info("Loading Charge Resolution file: {}".format(path))
with open_file(path, mode="r") as f:
for arr in self.storage_arrays:
setattr(self, arr, f.get_node("/ChargeResolution", arr).read())
class DataBinning(Component):
"""
Collects information on generating energy and angular bins for
generating IRFs as per pyIRF requirements.
"""
true_energy_min = Float(
help="Minimum value for True Energy bins in TeV units",
default_value=0.01,
).tag(config=True)
true_energy_max = Float(
help="Maximum value for True Energy bins in TeV units",
default_value=100,
).tag(config=True)
true_energy_n_bins_per_decade = Float(
help="Number of edges per decade for True Energy bins",
default_value=5.5,
).tag(config=True)
reco_energy_min = Float(
help="Minimum value for Reco Energy bins in TeV units",
default_value=0.01,
).tag(config=True)
reco_energy_max = Float(
help="Maximum value for Reco Energy bins in TeV units",
default_value=100,
).tag(config=True)
reco_energy_n_bins_per_decade = Float(
help="Number of edges per decade for Reco Energy bins",
default_value=5.5,
).tag(config=True)
energy_migration_min = Float(
help="Minimum value of Energy Migration matrix",
default_value=0.2,
).tag(config=True)
energy_migration_max = Float(
help="Maximum value of Energy Migration matrix",
default_value=5,
).tag(config=True)
energy_migration_n_bins = Int(
help="Number of bins in log scale for Energy Migration matrix",
default_value=31,
).tag(config=True)
fov_offset_min = Float(
help="Minimum value for FoV Offset bins",
default_value=0.1,
).tag(config=True)
fov_offset_max = Float(
help="Maximum value for FoV offset bins",
default_value=1.1,
).tag(config=True)
fov_offset_n_edges = Int(
help="Number of edges for FoV offset bins",
default_value=9,
).tag(config=True)
bkg_fov_offset_min = Float(
help="Minimum value for FoV offset bins for Background IRF",
default_value=0,
).tag(config=True)
bkg_fov_offset_max = Float(
help="Maximum value for FoV offset bins for Background IRF",
default_value=10,
).tag(config=True)
bkg_fov_offset_n_edges = Int(
help="Number of edges for FoV offset bins for Background IRF",
default_value=21,
).tag(config=True)
source_offset_min = Float(
help="Minimum value for Source offset for PSF IRF",
default_value=0.0001,
).tag(config=True)
source_offset_max = Float(
help="Maximum value for Source offset for PSF IRF",
default_value=1.0001,
).tag(config=True)
source_offset_n_edges = Int(
help="Number of edges for Source offset for PSF IRF",
default_value=1000,
).tag(config=True)
def true_energy_bins(self):
"""
Creates bins per decade for true MC energy using pyirf function.
The overflow binning added is not needed at the current stage
It can be used as - add_overflow_bins(***)[1:-1]
"""
true_energy = create_bins_per_decade(
self.true_energy_min * u.TeV,
self.true_energy_max * u.TeV,
self.true_energy_n_bins_per_decade,
)
return true_energy
def reco_energy_bins(self):
"""
Creates bins per decade for reconstructed MC energy using pyirf function.
The overflow binning added is not needed at the current stage
It can be used as - add_overflow_bins(***)[1:-1]
"""
reco_energy = create_bins_per_decade(
self.reco_energy_min * u.TeV,
self.reco_energy_max * u.TeV,
self.reco_energy_n_bins_per_decade,
)
return reco_energy
def energy_migration_bins(self):
"""
Creates bins for energy migration.
"""
energy_migration = np.geomspace(
self.energy_migration_min,
self.energy_migration_max,
self.energy_migration_n_bins,
)
return energy_migration
def fov_offset_bins(self):
"""
Creates bins for single/multiple FoV offset
"""
fov_offset = (
np.linspace(
self.fov_offset_min,
self.fov_offset_max,
self.fov_offset_n_edges,
)
* u.deg
)
return fov_offset
def bkg_fov_offset_bins(self):
"""
Creates bins for FoV offset for Background IRF,
Using the same binning as in pyirf example.
"""
background_offset = (
np.linspace(
self.bkg_fov_offset_min,
self.bkg_fov_offset_max,
self.bkg_fov_offset_n_edges,
)
* u.deg
)
return background_offset
def source_offset_bins(self):
"""
Creates bins for source offset for generating PSF IRF.
Using the same binning as in pyirf example.
"""
source_offset = (
np.linspace(
self.source_offset_min,
self.source_offset_max,
self.source_offset_n_edges,
)
* u.deg
)
return source_offset
class FlatFieldCalculator(Component):
"""
Parent class for the flat-field calculators.
Fills the MonitoringCameraContainer.FlatfieldContainer on the base of a given
flat-field event sample.
The sample is defined by a maximal interval of time (sample_duration) or a
minimal number of events (sample_duration).
The calculator is supposed to be called in an event loop, extract and collect the
event charge and fill the PedestalContainer
Parameters
----------
tel_id : int
id of the telescope (default 0)
sample_duration : int
interval of time (s) used to gather the pedestal statistics
sample_size : int
number of pedestal events requested for the statistics
n_channels : int
number of waveform channel to be considered
charge_product : str
Name of the charge extractor to be used
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
kwargs
"""
tel_id = Int(
0, help="id of the telescope to calculate the flat-field coefficients"
).tag(config=True)
sample_duration = Int(60, help="sample duration in seconds").tag(config=True)
sample_size = Int(10000, help="sample size").tag(config=True)
n_channels = Int(2, help="number of channels to be treated").tag(config=True)
charge_product = Unicode(
"LocalPeakWindowSum", help="Name of the charge extractor to be used"
).tag(config=True)
def __init__(self, subarray, **kwargs):
"""
Parent class for the flat-field calculators.
Fills the MonitoringCameraContainer.FlatfieldContainer on the base of a given
flat-field event sample.
The sample is defined by a maximal interval of time (sample_duration) or a
minimal number of events (sample_duration).
The calculator is supposed to be called in an event loop, extract and collect the
event charge and fill the PedestalContainer
Parameters
----------
subarray: ctapipe.instrument.SubarrayDescription
Description of the subarray
tel_id : int
id of the telescope (default 0)
sample_duration : int
interval of time (s) used to gather the pedestal statistics
sample_size : int
number of pedestal events requested for the statistics
n_channels : int
number of waveform channel to be considered
charge_product : str
Name of the charge extractor to be used
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
kwargs
"""
super().__init__(**kwargs)
# load the waveform charge extractor
self.extractor = ImageExtractor.from_name(
self.charge_product, config=self.config, subarray=subarray
)
self.log.info(f"extractor {self.extractor}")
@abstractmethod
def calculate_relative_gain(self, event):
"""
class SingleTelEventDisplay(Tool):
name = "ctapipe-display-televents"
description = Unicode(__doc__)
infile = Path(help="input file to read", exists=True,
directory_ok=False).tag(config=True)
tel = Int(help="Telescope ID to display", default_value=0).tag(config=True)
write = Bool(help="Write out images to PNG files",
default_value=False).tag(config=True)
clean = Bool(help="Apply image cleaning",
default_value=False).tag(config=True)
hillas = Bool(help="Apply and display Hillas parametrization",
default_value=False).tag(config=True)
samples = Bool(help="Show each sample",
default_value=False).tag(config=True)
display = Bool(help="Display results in interactive window",
default_value=True).tag(config=True)
delay = Float(help="delay between events in s",
default_value=0.01,
min=0.001).tag(config=True)
progress = Bool(help="display progress bar",
default_value=True).tag(config=True)
aliases = Dict({
"infile": "SingleTelEventDisplay.infile",
"tel": "SingleTelEventDisplay.tel",
"max-events": "EventSource.max_events",
"write": "SingleTelEventDisplay.write",
"clean": "SingleTelEventDisplay.clean",
"hillas": "SingleTelEventDisplay.hillas",
"samples": "SingleTelEventDisplay.samples",
"display": "SingleTelEventDisplay.display",
"delay": "SingleTelEventDisplay.delay",
"progress": "SingleTelEventDisplay.progress",
})
classes = List([EventSource, CameraCalibrator])
def __init__(self, **kwargs):
super().__init__(**kwargs)
def setup(self):
print("TOLLES INFILE", self.infile)
self.event_source = EventSource.from_url(self.infile, parent=self)
self.event_source.allowed_tels = {self.tel}
self.calibrator = CameraCalibrator(parent=self,
subarray=self.event_source.subarray)
self.log.info(f"SELECTING EVENTS FROM TELESCOPE {self.tel}")
def start(self):
disp = None
for event in tqdm(
self.event_source,
desc=f"Tel{self.tel}",
total=self.event_source.max_events,
disable=~self.progress,
):
self.log.debug(event.trigger)
self.log.debug(f"Energy: {event.simulation.shower.energy}")
self.calibrator(event)
if disp is None:
geom = self.event_source.subarray.tel[self.tel].camera.geometry
self.log.info(geom)
disp = CameraDisplay(geom)
# disp.enable_pixel_picker()
disp.add_colorbar()
if self.display:
plt.show(block=False)
# display the event
disp.axes.set_title("CT{:03d} ({}), event {:06d}".format(
self.tel, geom.camera_name, event.index.event_id))
if self.samples:
# display time-varying event
data = event.dl0.tel[self.tel].waveform
for ii in range(data.shape[1]):
disp.image = data[:, ii]
disp.set_limits_percent(70)
plt.suptitle(f"Sample {ii:03d}")
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f"CT{self.tel:03d}_EV{event.index.event_id:10d}"
f"_S{ii:02d}.png")
else:
# display integrated event:
im = event.dl1.tel[self.tel].image
if self.clean:
mask = tailcuts_clean(geom,
im,
picture_thresh=10,
boundary_thresh=7)
im[~mask] = 0.0
disp.image = im
if self.hillas:
try:
ellipses = disp.axes.findobj(Ellipse)
if len(ellipses) > 0:
ellipses[0].remove()
params = hillas_parameters(geom, image=im)
disp.overlay_moments(params,
color="pink",
lw=3,
with_label=False)
except HillasParameterizationError:
pass
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f"CT{self.tel:03d}_EV{event.index.event_id:010d}.png")
self.log.info("FINISHED READING DATA FILE")
if disp is None:
self.log.warning(
"No events for tel {} were found in {}. Try a "
"different EventIO file or another telescope".format(
self.tel, self.infile))
class LSTEventSource(EventSource):
"""EventSource for LST r0 data."""
n_gains = Int(
2,
help='Number of gains at r0/r1 level'
).tag(config=True)
baseline = Int(
400,
help='r0 waveform baseline (default from EvB v3)'
).tag(config=True)
multi_streams = Bool(
True,
help='Read in parallel all streams '
).tag(config=True)
def __init__(self, **kwargs):
"""
Constructor
Parameters
----------
n_gains = number of gains expected in input file
baseline = baseline to be subtracted at r1 level (not used for the moment)
multi_streams = enable the reading of input files from all streams
config: traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.\
kwargs: dict
Additional parameters to be passed.
NOTE: The file mask of the data to read can be passed with
the 'input_url' parameter.
"""
super().__init__(**kwargs)
if self.multi_streams:
# test how many streams are there:
# file name must be [stream name]Run[all the rest]
# All the files with the same [all the rest] are opened
path, name = os.path.split(os.path.abspath(self.input_url))
if 'Run' in name:
stream, run = name.split('Run', 1)
else:
run = name
ls = listdir(path)
self.file_list = []
for file_name in ls:
if run in file_name:
full_name = os.path.join(path, file_name)
self.file_list.append(full_name)
else:
self.file_list = [self.input_url]
self.multi_file = MultiFiles(self.file_list)
self.geometry_version = 4
self.camera_config = self.multi_file.camera_config
self.log.info(
"Read {} input files".format(
self.multi_file.num_inputs()
)
)
self.tel_id = self.camera_config.telescope_id
self._subarray = self.create_subarray(self.tel_id)
self.n_camera_pixels = self.subarray.tel[self.tel_id].camera.n_pixels
@property
def subarray(self):
return self._subarray
@property
def is_simulation(self):
return False
@property
def obs_id(self):
# currently no obs id is available from the input files
return self.camera_config.configuration_id
@property
def datalevels(self):
return (DataLevel.R0, )
def rewind(self):
self.multi_file.rewind()
def create_subarray(self, tel_id=1):
"""
Obtain the subarray from the EventSource
Returns
-------
ctapipe.instrument.SubarrayDecription
"""
# camera info from LSTCam-[geometry_version].camgeom.fits.gz file
camera = load_camera_geometry(version=self.geometry_version)
tel_descr = TelescopeDescription(
name='LST', tel_type='LST', optics=OPTICS, camera=camera
)
tels = {tel_id: tel_descr}
# LSTs telescope position taken from MC from the moment
tel_pos = {tel_id: [50., 50., 16] * u.m}
subarray = SubarrayDescription("LST1 subarray")
subarray.tels = tels
subarray.positions = tel_pos
return subarray
def _generator(self):
# container for LST data
self.data = LSTDataContainer()
self.data.meta['input_url'] = self.input_url
self.data.meta['max_events'] = self.max_events
self.data.meta['origin'] = 'LSTCAM'
# fill LST data from the CameraConfig table
self.fill_lst_service_container_from_zfile()
# initialize general monitoring container
self.initialize_mon_container()
# loop on events
for count, event in enumerate(self.multi_file):
self.data.count = count
self.data.index.event_id = event.event_id
self.data.index.obs_id = self.obs_id
# fill specific LST event data
self.fill_lst_event_container_from_zfile(event)
# fill general monitoring data
self.fill_mon_container_from_zfile(event)
# fill general R0 data
self.fill_r0_container_from_zfile(event)
yield self.data
@staticmethod
def is_compatible(file_path):
from astropy.io import fits
try:
# The file contains two tables:
# 1: CameraConfig
# 2: Events
h = fits.open(file_path)[2].header
ttypes = [
h[x] for x in h.keys() if 'TTYPE' in x
]
except OSError:
# not even a fits file
return False
except IndexError:
# A fits file of a different format
return False
is_protobuf_zfits_file = (
(h['XTENSION'] == 'BINTABLE') and
(h['EXTNAME'] == 'Events') and
(h['ZTABLE'] is True) and
(h['ORIGIN'] == 'CTA') and
(h['PBFHEAD'] == 'R1.CameraEvent')
)
is_lst_file = 'lstcam_counters' in ttypes
return is_protobuf_zfits_file & is_lst_file
def fill_lst_service_container_from_zfile(self):
"""
Fill LSTServiceContainer with specific LST service data data
(from the CameraConfig table of zfit file)
"""
self.data.lst.tels_with_data = [self.tel_id, ]
svc_container = self.data.lst.tel[self.tel_id].svc
svc_container.telescope_id = self.tel_id
svc_container.cs_serial = self.camera_config.cs_serial
svc_container.configuration_id = self.camera_config.configuration_id
svc_container.date = self.camera_config.date
svc_container.num_pixels = self.camera_config.num_pixels
svc_container.num_samples = self.camera_config.num_samples
svc_container.pixel_ids = self.camera_config.expected_pixels_id
svc_container.data_model_version = self.camera_config.data_model_version
svc_container.num_modules = self.camera_config.lstcam.num_modules
svc_container.module_ids = self.camera_config.lstcam.expected_modules_id
svc_container.idaq_version = self.camera_config.lstcam.idaq_version
svc_container.cdhs_version = self.camera_config.lstcam.cdhs_version
svc_container.algorithms = self.camera_config.lstcam.algorithms
svc_container.pre_proc_algorithms = self.camera_config.lstcam.pre_proc_algorithms
def fill_lst_event_container_from_zfile(self, event):
"""
Fill LSTEventContainer with specific LST service data
(from the Event table of zfit file)
"""
event_container = self.data.lst.tel[self.tel_id].evt
event_container.configuration_id = event.configuration_id
event_container.event_id = event.event_id
event_container.tel_event_id = event.tel_event_id
event_container.pixel_status = event.pixel_status
event_container.ped_id = event.ped_id
event_container.module_status = event.lstcam.module_status
event_container.extdevices_presence = event.lstcam.extdevices_presence
# if TIB data are there
if event_container.extdevices_presence & 1:
# unpack TIB data
rec_fmt = '=IHIBB'
unpacked_tib = struct.unpack(rec_fmt, event.lstcam.tib_data)
event_container.tib_event_counter = unpacked_tib[0]
event_container.tib_pps_counter = unpacked_tib[1]
event_container.tib_tenMHz_counter = unpacked_tib[2]
event_container.tib_stereo_pattern = unpacked_tib[3]
event_container.tib_masked_trigger = unpacked_tib[4]
# if UCTS data are there
if event_container.extdevices_presence & 2:
if int(self.data.lst.tel[self.tel_id].svc.idaq_version) > 37201:
# unpack UCTS-CDTS data (new version)
rec_fmt = '=QIIIIIBBBBI'
unpacked_cdts = struct.unpack(rec_fmt, event.lstcam.cdts_data)
event_container.ucts_timestamp = unpacked_cdts[0]
event_container.ucts_address = unpacked_cdts[1] # new
event_container.ucts_event_counter = unpacked_cdts[2]
event_container.ucts_busy_counter = unpacked_cdts[3] # new
event_container.ucts_pps_counter = unpacked_cdts[4]
event_container.ucts_clock_counter = unpacked_cdts[5]
event_container.ucts_trigger_type = unpacked_cdts[6]
event_container.ucts_white_rabbit_status = unpacked_cdts[7]
event_container.ucts_stereo_pattern = unpacked_cdts[8] # new
event_container.ucts_num_in_bunch = unpacked_cdts[9] # new
event_container.ucts_cdts_version = unpacked_cdts[10] # new
else:
# unpack UCTS-CDTS data (old version)
rec_fmt = '=IIIQQBBB'
unpacked_cdts = struct.unpack(rec_fmt, event.lstcam.cdts_data)
event_container.ucts_event_counter = unpacked_cdts[0]
event_container.ucts_pps_counter = unpacked_cdts[1]
event_container.ucts_clock_counter = unpacked_cdts[2]
event_container.ucts_timestamp = unpacked_cdts[3]
event_container.ucts_camera_timestamp = unpacked_cdts[4]
event_container.ucts_trigger_type = unpacked_cdts[5]
event_container.ucts_white_rabbit_status = unpacked_cdts[6]
# if SWAT data are there
if event_container.extdevices_presence & 4:
# unpack SWAT data
rec_fmt = '=QIIBBIBI'
unpacked_swat = struct.unpack(rec_fmt, event.lstcam.swat_data)
event_container.swat_timestamp = unpacked_swat[0]
event_container.swat_counter1 = unpacked_swat[1]
event_container.swat_counter2 = unpacked_swat[2]
event_container.swat_event_type = unpacked_swat[3]
event_container.swat_camera_flag = unpacked_swat[4]
event_container.swat_camera_event_num = unpacked_swat[5]
event_container.swat_array_flag = unpacked_swat[6]
event_container.swat_array_event_num = unpacked_swat[7]
# unpack Dragon counters
rec_fmt = '=HIIIQ'
rec_len = struct.calcsize(rec_fmt)
rec_unpack = struct.Struct(rec_fmt).unpack_from
event_container.pps_counter = np.zeros(self.camera_config.lstcam.num_modules)
event_container.tenMHz_counter = np.zeros(self.camera_config.lstcam.num_modules)
event_container.event_counter = np.zeros(self.camera_config.lstcam.num_modules)
event_container.trigger_counter = np.zeros(self.camera_config.lstcam.num_modules)
event_container.local_clock_counter = np.zeros(self.camera_config.lstcam.num_modules)
for mod in range(self.camera_config.lstcam.num_modules):
words=event.lstcam.counters[mod*rec_len:(mod+1)*rec_len]
unpacked_counter = rec_unpack(words)
event_container.pps_counter[mod] = unpacked_counter[0]
event_container.tenMHz_counter[mod] = unpacked_counter[1]
event_container.event_counter[mod] = unpacked_counter[2]
event_container.trigger_counter[mod] = unpacked_counter[3]
event_container.local_clock_counter[mod] = unpacked_counter[4]
event_container.chips_flags = event.lstcam.chips_flags
event_container.first_capacitor_id = event.lstcam.first_capacitor_id
event_container.drs_tag_status = event.lstcam.drs_tag_status
event_container.drs_tag = event.lstcam.drs_tag
def fill_r0_camera_container_from_zfile(self, r0_container, event):
"""
Fill with R0CameraContainer
"""
# look for correct trigger_time (TAI time in s), first in UCTS and then in TIB
#if self.data.lst.tel[self.tel_id].evt.ucts_timestamp > 0:
# r0_container.trigger_time = self.data.lst.tel[self.tel_id].evt.ucts_timestamp/1e9
# consider for the moment only TIB time since UCTS seems not correct
#if self.data.lst.tel[self.tel_id].evt.tib_pps_counter > 0:
# r0_container.trigger_time = (
# self.data.lst.tel[self.tel_id].svc.date +
# self.data.lst.tel[self.tel_id].evt.tib_pps_counter +
# self.data.lst.tel[self.tel_id].evt.tib_tenMHz_counter * 10**(-7))
#else:
# r0_container.trigger_time = 0
#consider for the moment trigger time from central dragon module
module_rank = np.where(self.data.lst.tel[self.tel_id].svc.module_ids == 132)
r0_container.trigger_time = (
self.data.lst.tel[self.tel_id].svc.date +
self.data.lst.tel[self.tel_id].evt.pps_counter[module_rank] +
self.data.lst.tel[self.tel_id].evt.tenMHz_counter[module_rank] * 10**(-7))
# look for correct trigger type first in UCTS and then in TIB
#if self.data.lst.tel[self.tel_id].evt.ucts_trigger_type > 0:
# r0_container.trigger_type = self.data.lst.tel[self.tel_id].evt.ucts_trigger_type
# consider for the moment only TIB trigger since UCTS seems not correct
if self.data.lst.tel[self.tel_id].evt.tib_masked_trigger > 0:
r0_container.trigger_type = self.data.lst.tel[self.tel_id].evt.tib_masked_trigger
else:
r0_container.trigger_type = -1
# verify the number of gains
if event.waveform.shape[0] != self.camera_config.num_pixels * self.camera_config.num_samples * self.n_gains:
raise ValueError(f"Number of gains not correct, waveform shape is {event.waveform.shape[0]}"
f" instead of "
f"{self.camera_config.num_pixels * self.camera_config.num_samples * self.n_gains}")
reshaped_waveform = np.array(
event.waveform
).reshape(
self.n_gains,
self.camera_config.num_pixels,
self.camera_config.num_samples
)
# initialize the waveform container to zero
r0_container.waveform = np.zeros([self.n_gains, self.n_camera_pixels,
self.camera_config.num_samples])
# re-order the waveform following the expected_pixels_id values
# (rank = pixel id)
r0_container.waveform[:, self.camera_config.expected_pixels_id, :] =\
reshaped_waveform
def fill_r0_container_from_zfile(self, event):
"""
Fill with R0Container
"""
container = self.data.r0
container.tels_with_data = [self.tel_id, ]
r0_camera_container = container.tel[self.tel_id]
self.fill_r0_camera_container_from_zfile(
r0_camera_container,
event
)
def initialize_mon_container(self):
"""
Fill with MonitoringContainer.
For the moment, initialize only the PixelStatusContainer
"""
container = self.data.mon
container.tels_with_data = [self.tel_id, ]
mon_camera_container = container.tel[self.tel_id]
# initialize the container
status_container = PixelStatusContainer()
status_container.hardware_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)
status_container.pedestal_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)
status_container.flatfield_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)
mon_camera_container.pixel_status = status_container
def fill_mon_container_from_zfile(self, event):
"""
Fill with MonitoringContainer.
For the moment, initialize only the PixelStatusContainer
"""
status_container = self.data.mon.tel[self.tel_id].pixel_status
# reorder the array
pixel_status = np.zeros(self.n_camera_pixels)
pixel_status[self.camera_config.expected_pixels_id] = event.pixel_status
status_container.hardware_failing_pixels[:] = pixel_status == 0