class ImageSumDisplayerTool(Tool):
description = Unicode(__doc__)
name = "ctapipe-image-sum-display"
infile = Unicode(
help='input simtelarray file',
default="/Users/kosack/Data/CTA/Prod3/gamma.simtel.gz").tag(
config=True)
telgroup = Integer(help='telescope group number',
default=1).tag(config=True)
aliases = Dict({
'infile': 'ImageSumDisplayerTool.infile',
'telgroup': 'ImageSumDisplayerTool.telgroup'
})
def setup(self):
# load up the telescope types table (need to first open a file, a bit of
# a hack until a proper insturment module exists) and select only the
# telescopes with the same camera type
data = next(hessio_event_source(self.infile, max_events=1))
camtypes = get_camera_types(data.inst)
group = camtypes.groups[self.telgroup]
self._selected_tels = group['tel_id'].data
self._base_tel = self._selected_tels[0]
self.log.info("Telescope group %d: %s with %s camera", self.telgroup,
group[0]['tel_type'], group[0]['cam_type'])
self.log.info("SELECTED TELESCOPES:{}".format(self._selected_tels))
def start(self):
geom = None
imsum = None
disp = None
for data in hessio_event_source(self.infile,
allowed_tels=self._selected_tels,
max_events=None):
if geom is None:
x, y = data.inst.pixel_pos[self._base_tel]
flen = data.inst.optical_foclen[self._base_tel]
geom = CameraGeometry.guess(x, y, flen)
imsum = np.zeros(shape=x.shape, dtype=np.float)
disp = CameraDisplay(geom, title=geom.cam_id)
disp.add_colorbar()
disp.cmap = 'viridis'
if len(data.r0.tels_with_data) <= 2:
continue
imsum[:] = 0
for telid in data.r0.tels_with_data:
imsum += data.r0.tel[telid].adc_sums[0]
self.log.info("event={} ntels={} energy={}" \
.format(data.r0.event_id,
len(data.r0.tels_with_data),
data.mc.energy))
disp.image = imsum
plt.pause(0.1)
class DumpInstrumentTool(Tool):
description = Unicode(__doc__)
name = 'ctapipe-dump-instrument'
infile = Unicode(help='input simtelarray file').tag(config=True)
format = Enum(['fits', 'ecsv', 'hdf5'],
default_value='fits',
help='Format '
'of '
'output '
'file',
config=True)
aliases = Dict(
dict(infile='DumpInstrumentTool.infile',
format='DumpInstrumentTool.format'))
def setup(self):
source = hessio_event_source(self.infile)
data = next(source) # get one event, so the instrument table is
# filled in
self.inst = data.inst # keep a pointer to the instrument stuff
pass
def start(self):
self.write_camera_geometries()
def finish(self):
pass
def _get_file_format_info(self, format_name, table_type, cam_name):
""" returns file extension + dict of required parameters for
Table.write"""
if format_name == 'fits':
return 'fits.gz', dict()
elif format_name == 'ecsv':
return 'ecsv.txt', dict(format='ascii.ecsv')
elif format_name == 'hdf5':
return 'h5', dict(path="/" + table_type + "/" + cam_name)
else:
raise NameError("format not supported")
def write_camera_geometries(self):
cam_types = get_camera_types(self.inst)
print_camera_types(self.inst, printer=self.log.info)
for cam_name in cam_types:
ext, args = self._get_file_format_info(self.format, 'CAMGEOM',
cam_name)
self.log.debug("writing {}".format(cam_name))
tel_id = cam_types[cam_name].pop()
pix = self.inst.pixel_pos[tel_id]
flen = self.inst.optical_foclen[tel_id]
geom = CameraGeometry.guess(*pix, flen)
table = geom.to_table()
table.meta['SOURCE'] = self.infile
table.write("{}.camgeom.{}".format(cam_name, ext), **args)
class PedestalCalculator(Component):
"""
Parent class for the pedestal calculators.
Fills the MON.pedestal container on the base of
pedestal events (preliminary version)
"""
tel_id = Int(
0, help='id of the telescope to calculate the pedestal values').tag(
config=True)
sample_duration = Int(60,
help='sample duration in seconds').tag(config=True)
sample_size = Int(10000, help='sample size').tag(config=True)
n_channels = Int(2,
help='number of channels to be treated').tag(config=True)
charge_cut_outliers = List(
[3, 3], help='Interval (number of std) of accepted charge values').tag(
config=True)
charge_std_cut_outliers = List(
[3, 3],
help=
'Interval (number of std) of accepted charge standard deviation values'
).tag(config=True)
charge_product = Unicode(
'LocalPeakIntegrator',
help='Name of the charge extractor to be used').tag(config=True)
def __init__(self, **kwargs):
"""
Parent class for pedestal calculators.
Fills the MON.pedestal container.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
super().__init__(**kwargs)
# initialize the output
self.container = PedestalContainer()
# load the waveform charge extractor
self.extractor = ChargeExtractor.from_name(self.charge_product,
config=self.config)
self.log.info(f"extractor {self.extractor}")
@abstractmethod
def calculate_pedestals(self, event):
"""calculate relative gain from event
class FlatFieldCalculator(Component):
"""
Parent class for the flat field calculators.
Fills the MON.flatfield container.
"""
tel_id = Int(
0, help='id of the telescope to calculate the flat-field coefficients'
).tag(config=True)
sample_duration = Int(60,
help='sample duration in seconds').tag(config=True)
sample_size = Int(10000, help='sample size').tag(config=True)
n_channels = Int(2,
help='number of channels to be treated').tag(config=True)
charge_cut_outliers = List(
[-0.3, 0.3],
help=
'Interval of accepted charge values (fraction with respect to camera median value)'
).tag(config=True)
time_cut_outliers = List(
[10, 30],
help='Interval (in samples) of accepted time values').tag(config=True)
charge_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
def __init__(self, **kwargs):
"""
Parent class for the flat field calculators.
Fills the flatfield container.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
super().__init__(**kwargs)
# load the waveform charge extractor
self.extractor = ImageExtractor.from_name(self.charge_product,
config=self.config)
self.log.info(f"extractor {self.extractor}")
@abstractmethod
def calculate_relative_gain(self, event):
"""calculate relative gain from event
class MakeHistFirstCap(Tool):
name = "make-hist-first-cap"
infile = Unicode(help='input LST file', default="").tag(config=True)
max_events = Integer(help='stop after this many events if non-zero',
default_value=0,
min=0).tag(config=True)
output_suffix = Unicode(help='suffix (file extension) of output '
'filenames to write images '
'to (no writing is done if blank). '
'Images will be named [EVENTID][suffix]',
default_value="").tag(config=True)
aliases = Dict({
'infile': 'MakeHistFirstCap.infile',
'max-events': 'MakeHistFirstCap.max_events',
'output-suffix': 'MakeHistFirstCap.output_suffix'
})
def setup(self):
# load LST data
self.log.info("Read file:{}".format(self.infile))
self.reader = LSTEventSource(input_url=self.infile,
max_events=self.max_events)
def start(self):
fc = []
for event in self.reader:
fc.extend(get_first_capacitor_array(event))
plt.figure()
plt.hist(fc, bins=4096)
plt.ylabel("Number")
plt.xlabel("Stop Cell")
plt.show()
class EventSource(Component):
"""
Parent class for EventFileReaders of different sources.
A new EventFileReader should be created for each type of event file read
into ctapipe, e.g. sim_telarray files are read by the `SimTelEventSource`.
EventFileReader provides a common high-level interface for accessing event
information from different data sources (simulation or different camera
file formats). Creating an EventFileReader for a new
file format ensures that data can be accessed in a common way,
irregardless of the file format.
EventFileReader itself is an abstract class. To use an EventFileReader you
must use a subclass that is relevant for the file format you
are reading (for example you must use
`ctapipe.io.SimTelEventSource` to read a hessio format
file). Alternatively you can use `event_source()` to automatically
select the correct EventFileReader subclass for the file format you wish
to read.
To create an instance of an EventFileReader you must pass the traitlet
configuration (containing the input_url) and the
`ctapipe.core.tool.Tool`. Therefore from inside a Tool you would do:
>>> event_source = EventSource(self.config, self)
An example of how to use `ctapipe.core.tool.Tool` and `event_source()`
can be found in ctapipe/tools/display_dl1.py.
However if you are not inside a Tool, you can still create an instance and
supply an input_url via:
>>> event_source = EventSource( input_url="/path/to/file")
To loop through the events in a file:
>>> event_source = EventSource( input_url="/path/to/file")
>>> for event in event_source:
>>> print(event.count)
**NOTE**: Every time a new loop is started through the event_source, it restarts
from the first event.
Alternatively one can use EventFileReader in a `with` statement to ensure
the correct cleanups are performed when you are finished with the event_source:
>>> with EventSource( input_url="/path/to/file") as event_source:
>>> for event in event_source:
>>> print(event.count)
**NOTE**: The "event" that is returned from the generator is a pointer.
Any operation that progresses that instance of the generator further will
change the data pointed to by "event". If you wish to ensure a particular
event is kept, you should perform a `event_copy = copy.deepcopy(event)`.
Attributes
----------
input_url : str
Path to the input event file.
max_events : int
Maximum number of events to loop through in generator
metadata : dict
A dictionary containing the metadata of the file. This could include:
* is_simulation (bool indicating if the file contains simulated events)
* Telescope:Camera names (list if file contains multiple)
* Information in the file header
* Observation ID
"""
input_url = Unicode(
"", help="Path to the input file containing events.").tag(config=True)
max_events = Int(
None,
allow_none=True,
help="Maximum number of events that will be read from the file",
).tag(config=True)
allowed_tels = Set(
help=("list of allowed tel_ids, others will be ignored. "
"If left empty, all telescopes in the input stream "
"will be included")).tag(config=True)
def __init__(self, config=None, parent=None, **kwargs):
"""
Class to handle generic input files. Enables obtaining the "source"
generator, regardless of the type of file (either hessio or camera
file).
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
super().__init__(config=config, parent=parent, **kwargs)
self.metadata = dict(is_simulation=False)
input_url: Path = Path(self.input_url).expanduser()
if not input_url.exists:
raise FileNotFoundError(f"file path does not exist: '{input_url}'")
self.log.info(f"INPUT PATH = {input_url}")
if self.max_events:
self.log.info(f"Max events being read = {self.max_events}")
Provenance().add_input_file(input_url, role="DL0/Event")
@staticmethod
@abstractmethod
def is_compatible(file_path):
"""
Abstract method to be defined in child class.
Perform a set of checks to see if the input file is compatible
with this file event_source.
Parameters
----------
file_path : str
File path to the event file.
Returns
-------
compatible : bool
True if file is compatible, False if it is incompatible
"""
@property
def is_stream(self):
"""
Bool indicating if input is a stream. If it is then it is incompatible
with `ctapipe.io.eventseeker.EventSeeker`.
TODO: Define a method to detect if it is a stream
Returns
-------
bool
If True, then input is a stream.
"""
return False
@property
@abstractmethod
def subarray(self):
"""
Obtain the subarray from the EventSource
Returns
-------
ctapipe.instrument.SubarrayDecription
"""
@abstractmethod
def _generator(self):
"""
Abstract method to be defined in child class.
Generator where the filling of the `ctapipe.containers` occurs.
Returns
-------
generator
"""
def __iter__(self):
"""
Generator that iterates through `_generator`, but keeps track of
`self.max_events`.
Returns
-------
generator
"""
for event in self._generator():
yield event
if self.max_events and event.count >= self.max_events - 1:
break
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
pass
@classmethod
def from_url(cls, input_url, **kwargs):
"""
Find compatible EventSource for input_url via the `is_compatible`
method of the EventSource
Parameters
----------
input_url : str
Filename or URL pointing to an event file
kwargs
Named arguments for the EventSource
Returns
-------
instance
Instance of a compatible EventSource subclass
"""
if input_url == "" or input_url is None:
raise ToolConfigurationError(
"EventSource: No input_url was specified")
detect_and_import_io_plugins()
available_classes = non_abstract_children(cls)
for subcls in available_classes:
if subcls.is_compatible(input_url):
return subcls(input_url=input_url, **kwargs)
raise ValueError("Cannot find compatible EventSource for \n"
"\turl:{}\n"
"in available EventSources:\n"
"\t{}".format(input_url,
[c.__name__
for c in available_classes]))
@classmethod
def from_config(cls, config=None, parent=None, **kwargs):
"""
Find compatible EventSource for the EventSource.input_url traitlet
specified via the config.
This method is typically used in Tools, where the input_url is chosen via
the command line using the traitlet configuration system.
Parameters
----------
config : traitlets.config.loader.Config
Configuration created in the Tool
kwargs
Named arguments for the EventSource
Returns
-------
instance
Instance of a compatible EventSource subclass
"""
if config is None:
config = parent.config
if isinstance(config.EventSource.input_url, LazyConfigValue):
config.EventSource.input_url = cls.input_url.default_value
elif not isinstance(config.EventSource.input_url, str):
raise TraitError("Wrong type specified for input_url traitlet")
return event_source(config.EventSource.input_url,
config=config,
**kwargs)
class SingleTelEventDisplay(Tool):
name = "ctapipe-display-televents"
description = Unicode(__doc__)
infile = Path(help="input file to read", exists=True,
directory_ok=False).tag(config=True)
tel = Int(help="Telescope ID to display", default_value=0).tag(config=True)
write = Bool(help="Write out images to PNG files",
default_value=False).tag(config=True)
clean = Bool(help="Apply image cleaning",
default_value=False).tag(config=True)
hillas = Bool(help="Apply and display Hillas parametrization",
default_value=False).tag(config=True)
samples = Bool(help="Show each sample",
default_value=False).tag(config=True)
display = Bool(help="Display results in interactive window",
default_value=True).tag(config=True)
delay = Float(help="delay between events in s",
default_value=0.01,
min=0.001).tag(config=True)
progress = Bool(help="display progress bar",
default_value=True).tag(config=True)
aliases = Dict({
"infile": "SingleTelEventDisplay.infile",
"tel": "SingleTelEventDisplay.tel",
"max-events": "EventSource.max_events",
"write": "SingleTelEventDisplay.write",
"clean": "SingleTelEventDisplay.clean",
"hillas": "SingleTelEventDisplay.hillas",
"samples": "SingleTelEventDisplay.samples",
"display": "SingleTelEventDisplay.display",
"delay": "SingleTelEventDisplay.delay",
"progress": "SingleTelEventDisplay.progress",
})
classes = List([EventSource, CameraCalibrator])
def __init__(self, **kwargs):
super().__init__(**kwargs)
def setup(self):
print("TOLLES INFILE", self.infile)
self.event_source = EventSource.from_url(self.infile, parent=self)
self.event_source.allowed_tels = {self.tel}
self.calibrator = CameraCalibrator(parent=self,
subarray=self.event_source.subarray)
self.log.info(f"SELECTING EVENTS FROM TELESCOPE {self.tel}")
def start(self):
disp = None
for event in tqdm(
self.event_source,
desc=f"Tel{self.tel}",
total=self.event_source.max_events,
disable=~self.progress,
):
self.log.debug(event.trigger)
self.log.debug(f"Energy: {event.simulation.shower.energy}")
self.calibrator(event)
if disp is None:
geom = self.event_source.subarray.tel[self.tel].camera.geometry
self.log.info(geom)
disp = CameraDisplay(geom)
# disp.enable_pixel_picker()
disp.add_colorbar()
if self.display:
plt.show(block=False)
# display the event
disp.axes.set_title("CT{:03d} ({}), event {:06d}".format(
self.tel, geom.camera_name, event.index.event_id))
if self.samples:
# display time-varying event
data = event.dl0.tel[self.tel].waveform
for ii in range(data.shape[1]):
disp.image = data[:, ii]
disp.set_limits_percent(70)
plt.suptitle(f"Sample {ii:03d}")
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f"CT{self.tel:03d}_EV{event.index.event_id:10d}"
f"_S{ii:02d}.png")
else:
# display integrated event:
im = event.dl1.tel[self.tel].image
if self.clean:
mask = tailcuts_clean(geom,
im,
picture_thresh=10,
boundary_thresh=7)
im[~mask] = 0.0
disp.image = im
if self.hillas:
try:
ellipses = disp.axes.findobj(Ellipse)
if len(ellipses) > 0:
ellipses[0].remove()
params = hillas_parameters(geom, image=im)
disp.overlay_moments(params,
color="pink",
lw=3,
with_label=False)
except HillasParameterizationError:
pass
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f"CT{self.tel:03d}_EV{event.index.event_id:010d}.png")
self.log.info("FINISHED READING DATA FILE")
if disp is None:
self.log.warning(
"No events for tel {} were found in {}. Try a "
"different EventIO file or another telescope".format(
self.tel, self.infile))
class PulseTimeCorrection(Component):
"""
The PulseTimeCorrection class to correct time pulse
using Fourier series expansion.
"""
tel_id = Int(1, help='id of the telescope to calibrate').tag(config=True)
n_capacitors = Int(
1024, help='number of capacitors (1024 or 4096)').tag(config=True)
calib_file_path = Unicode(
'', allow_none=True,
help='Path to the time calibration file').tag(config=True)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.n_harmonics = None
self.fan_array = None # array to store cos coeff for Fourier series expansion
self.fbn_array = None # array to store sin coeff for Fourier series expansion
self.first_cap_array = np.zeros((n_modules, n_gain, n_channel))
self.load_calib_file()
def load_calib_file(self):
"""
Function to load calibration file.
"""
try:
with h5py.File(self.calib_file_path, 'r') as hf:
self.n_harmonics = hf["/"].attrs['n_harm']
fan = hf.get('fan')
self.fan_array = np.array(fan)
fbn = hf.get('fbn')
self.fbn_array = np.array(fbn)
except:
self.log.error(
f"Problem in reading time from calibration file {self.calib_file_path}"
)
def get_corr_pulse(self, event, pulse):
"""
Return pulse time after time correction.
Parameters
----------
event : `ctapipe` event-container
pulse : ndarray
pulse time in each pixel.
Stored in a numpy array of shape
(2, 1855).
"""
pixel_ids = event.lst.tel[self.tel_id].svc.pixel_ids
n_modules_from_event = event.lst.tel[self.tel_id].svc.num_modules
pulse_corr = np.empty((n_gain, n_pixels))
for nr in prange(0, n_modules_from_event):
self.first_cap_array[nr, :, :] = self.get_first_capacitor(
event, nr)
self.get_corr_pulse_jit(pulse, pulse_corr, pixel_ids,
self.first_cap_array, self.fan_array,
self.fbn_array, self.n_harmonics,
self.n_capacitors)
return pulse_corr
@staticmethod
@njit(parallel=True)
def get_corr_pulse_jit(pulse, pulse_corr, pixel_ids, first_capacitor,
fan_array, fbn_array, n_harmonics, n_cap):
"""
Numba function for pulse time correction.
Parameters
----------
pulse : ndarray
Pulse time stored in a numpy array of shape
(n_gain, n_pixels).
pulse_corr : ndarray
Pulse correction time stored in a numpy array of shape
(n_gain, n_pixels).
pixel_ids: ndarray
Array stored expected pixel id
(n_pixels).
first_capacitor : ndarray
Value of first capacitor stored in a numpy array of shape
(n_clus, n_gain, n_pix).
fan_array : ndarray
Array to store coeff for Fourier series expansion
stored in a numpy array of shape
(n_gain, n_pixels, n_harmonics).
fbn_array : ndarray
Array to store coeff for Fourier series expansion
stored in a numpy array of shape
(n_gain, n_pixels, n_harmonics).
n_harmonics : int
Number of harmonics
"""
for gain in prange(0, n_gain):
for nr in prange(0, n_modules):
for pix in prange(0, n_channel):
fc = first_capacitor[nr, gain, pix]
pixel = pixel_ids[nr * 7 + pix]
pulse_corr[gain,
pixel] = pulse[gain, pixel] - get_corr_time_jit(
fc % n_cap, fan_array[gain, pixel],
fbn_array[gain, pixel], n_harmonics, n_cap)
def get_first_capacitor(self, event, nr):
"""
Get first capacitor values from event for nr module.
Parameters
----------
event : `ctapipe` event-container
nr_module : number of module
tel_id : id of the telescope
"""
fc = np.zeros((n_gain, n_channel))
first_cap = event.lst.tel[
self.tel_id].evt.first_capacitor_id[nr * 8:(nr + 1) * 8]
# First capacitor order according Dragon v5 board data format
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [0, 0, 1, 1, 2, 2, 3]):
fc[high_gain, i] = first_cap[j]
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [4, 4, 5, 5, 6, 6, 7]):
fc[low_gain, i] = first_cap[j]
return fc
class DumpInstrumentTool(Tool):
description = Unicode(__doc__)
name = "ctapipe-dump-instrument"
infile = Path(exists=True, help="input simtelarray file").tag(config=True)
format = Enum(
["fits", "ecsv", "hdf5"],
default_value="fits",
help="Format of output file",
config=True,
)
aliases = Dict(
dict(infile="DumpInstrumentTool.infile",
format="DumpInstrumentTool.format"))
def setup(self):
with event_source(self.infile) as source:
self.subarray = source.subarray
def start(self):
if self.format == "hdf5":
self.subarray.to_hdf("subarray.h5")
else:
self.write_camera_geometries()
self.write_optics_descriptions()
self.write_subarray_description()
def finish(self):
pass
@staticmethod
def _get_file_format_info(format_name, table_type, table_name):
""" returns file extension + dict of required parameters for
Table.write"""
if format_name == "fits":
return "fits.gz", dict()
elif format_name == "ecsv":
return "ecsv.txt", dict(format="ascii.ecsv")
else:
raise NameError(f"format {format_name} not supported")
def write_camera_geometries(self):
cam_types = get_camera_types(self.subarray)
self.subarray.info(printer=self.log.info)
for cam_name in cam_types:
ext, args = self._get_file_format_info(self.format, "CAMGEOM",
cam_name)
self.log.debug(f"writing {cam_name}")
tel_id = cam_types[cam_name].pop()
geom = self.subarray.tel[tel_id].camera.geometry
table = geom.to_table()
table.meta["SOURCE"] = str(self.infile)
filename = f"{cam_name}.camgeom.{ext}"
try:
table.write(filename, **args)
Provenance().add_output_file(filename, "dl0.tel.svc.camera")
except IOError as err:
self.log.warning(
"couldn't write camera definition '%s' because: %s",
filename, err)
def write_optics_descriptions(self):
sub = self.subarray
ext, args = self._get_file_format_info(self.format, sub.name, "optics")
tab = sub.to_table(kind="optics")
tab.meta["SOURCE"] = str(self.infile)
filename = f"{sub.name}.optics.{ext}"
try:
tab.write(filename, **args)
Provenance().add_output_file(filename, "dl0.sub.svc.optics")
except IOError as err:
self.log.warning(
"couldn't write optics description '%s' because: %s", filename,
err)
def write_subarray_description(self):
sub = self.subarray
ext, args = self._get_file_format_info(self.format, sub.name,
"subarray")
tab = sub.to_table(kind="subarray")
tab.meta["SOURCE"] = str(self.infile)
filename = f"{sub.name}.subarray.{ext}"
try:
tab.write(filename, **args)
Provenance().add_output_file(filename, "dl0.sub.svc.subarray")
except IOError as err:
self.log.warning(
"couldn't write subarray description '%s' because: %s",
filename, err)
class TimeCorrectionCalculate(Component):
"""
The TimeCorrectionCalculate class to create h5py
file with coefficients for time correction curve
of chip DRS4.
Description of this method: "Analysis techniques
and performance of the Domino Ring Sampler version 4
based readout for the MAGIC telescopes [arxiv:1305.1007]
"""
minimum_charge = Float(
200, help='Cut on charge. Default 200 ADC').tag(config=True)
tel_id = Int(1, help='Id of the telescope to calibrate').tag(config=True)
n_combine = Int(
8, help='How many capacitors are combines in a single bin. Default 8'
).tag(config=True)
n_harmonics = Int(
16, help='Number of harmonic for Fourier series expansion. Default 16'
).tag(config=True)
n_capacitors = Int(
1024, help='Number of capacitors (1024 or 4096). Default 1024.').tag(
config=True)
charge_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
calib_file_path = Unicode(
'', allow_none=True,
help='Path to the time calibration file').tag(config=True)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.n_bins = int(self.n_capacitors / self.n_combine)
self.mean_values_per_bin = np.zeros((n_gain, n_pixels, self.n_bins))
self.entries_per_bin = np.zeros((n_gain, n_pixels, self.n_bins))
self.first_cap_array = np.zeros((n_modules, n_gain, n_channel))
# load the waveform charge extractor
self.extractor = ImageExtractor.from_name(self.charge_product,
config=self.config)
self.log.info(f"extractor {self.extractor}")
self.sum_events = 0
def calibrate_pulse_time(self, event):
"""
Fill bins using time pulse from LocalPeakWindowSum.
Parameters
----------
event : `ctapipe` event-container
"""
if event.r1.tel[self.tel_id].trigger_type == 1:
for nr_module in prange(0, n_modules):
self.first_cap_array[
nr_module, :, :] = self.get_first_capacitor(
event, nr_module)
pixel_ids = event.lst.tel[self.tel_id].svc.pixel_ids
charge, pulse_time = self.extractor(
event.r1.tel[self.tel_id].waveform)
self.calib_pulse_time_jit(charge,
pulse_time,
pixel_ids,
self.first_cap_array,
self.mean_values_per_bin,
self.entries_per_bin,
n_cap=self.n_capacitors,
n_combine=self.n_combine,
min_charge=self.minimum_charge)
self.sum_events += 1
@jit(parallel=True)
def calib_pulse_time_jit(self, charge, pulse_time, pixel_ids,
first_cap_array, mean_values_per_bin,
entries_per_bin, n_cap, n_combine, min_charge):
"""
Numba function for calibration pulse time.
Parameters
----------
pulse : ndarray
Pulse time stored in a numpy array of shape
(n_gain, n_pixels).
charge : ndarray
Charge in each pixel.
(n_gain, n_pixels).
pixel_ids: ndarray
Array stored expected pixel id
(n_pixels).
first_cap_array : ndarray
Value of first capacitor stored in a numpy array of shape
(n_clus, n_gain, n_pix).
mean_values_per_bin : ndarray
Array to fill using pulse time
stored in a numpy array of shape
(n_gain, n_pixels, n_bins).
entries_per_bin : ndarray
Array to store number of entries per bin
stored in a numpy array of shape
(n_gain, n_pixels, n_bins).
n_cap : int
Number of capacitors
n_combine : int
Number of combine capacitors in a single bin
"""
for nr_module in prange(0, n_modules):
for gain in prange(0, n_gain):
for pix in prange(0, n_channel):
pixel = pixel_ids[nr_module * 7 + pix]
if charge[gain, pixel] > min_charge: # cut change
fc = first_cap_array[nr_module, :, :]
first_cap = (fc[gain, pix]) % n_cap
bin = int(first_cap / n_combine)
mean_values_per_bin[gain, pixel,
bin] += pulse_time[gain, pixel]
entries_per_bin[gain, pixel, bin] += 1
def finalize(self):
if np.sum(self.entries_per_bin == 0) > 0:
raise RuntimeError(
"Not enough events to coverage all capacitor. "
"Please use more events to time calibration file.")
else:
self.mean_values_per_bin = self.mean_values_per_bin / self.entries_per_bin
self.save_to_hdf5_file()
def fit(self, pixel_id, gain):
"""
Fit data bins using Fourier series expansion
Parameters
----------
pixel_id : ndarray
Array stored expected pixel id of shape
(n_pixels).
gain: int
0 for high gain, 1 for low gain
"""
self.pos = np.zeros(self.n_bins)
for i in range(0, self.n_bins):
self.pos[i] = (i + 0.5) * self.n_combine
self.fan = np.zeros(self.n_harmonics) # cos coeff
self.fbn = np.zeros(self.n_harmonics) # sin coeff
for n in range(0, self.n_harmonics):
self.integrate_with_trig(self.pos,
self.mean_values_per_bin[gain, pixel_id],
n, self.fan, self.fbn)
def integrate_with_trig(self, x, y, n, an, bn):
"""
Function to expanding into Fourier series
Parameters
----------
x : ndarray
Array stored position in DRS4 ring of shape
(n_bins).
y: ndarray
Array stored mean pulse time per bin of shape
(n_bins)
n : int
n harmonic
an: ndarray
Array to fill with cos coeff of shape
(n_harmonics)
bn: ndarray
Array to fill with sin coeff of shape
(n_harmonics)
"""
suma = 0
sumb = 0
for i in range(0, self.n_bins):
suma += y[i] * self.n_combine * np.cos(
2 * np.pi * n * (x[i] / float(self.n_capacitors)))
sumb += y[i] * self.n_combine * np.sin(
2 * np.pi * n * (x[i] / float(self.n_capacitors)))
an[n] = suma * (2. / (self.n_bins * self.n_combine))
bn[n] = sumb * (2. / (self.n_bins * self.n_combine))
def get_first_capacitor(self, event, nr):
fc = np.zeros((n_gain, n_channel))
first_cap = event.lst.tel[
self.tel_id].evt.first_capacitor_id[nr * 8:(nr + 1) * 8]
# First capacitor order according Dragon v5 board data format
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [0, 0, 1, 1, 2, 2, 3]):
fc[high_gain, i] = first_cap[j]
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [4, 4, 5, 5, 6, 6, 7]):
fc[low_gain, i] = first_cap[j]
return fc
def save_to_hdf5_file(self):
"""
Function to save Fourier series expansion coeff into hdf5 file
"""
fan_array = np.zeros((n_gain, n_pixels, self.n_harmonics))
fbn_array = np.zeros((n_gain, n_pixels, self.n_harmonics))
for pix_id in range(0, n_pixels):
self.fit(pix_id, gain=high_gain)
fan_array[high_gain, pix_id, :] = self.fan
fbn_array[high_gain, pix_id, :] = self.fbn
self.fit(pix_id, gain=low_gain)
fan_array[low_gain, pix_id, :] = self.fan
fbn_array[low_gain, pix_id, :] = self.fbn
try:
hf = h5py.File(self.calib_file_path, 'w')
hf.create_dataset('fan', data=fan_array)
hf.create_dataset('fbn', data=fbn_array)
hf.attrs['n_events'] = self.sum_events
hf.attrs['n_harm'] = self.n_harmonics
except Exception as err:
print("FAILED!", err)
hf.close()
class LSTCameraCalibrator(CameraCalibrator):
"""
Calibrator to handle the LST camera calibration chain, in order to fill
the DL1 data level in the event container.
"""
extractor_product = Unicode(
'NeighborPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
reducer_product = Unicode(
'NullDataVolumeReducer',
help='Name of the DataVolumeReducer to use').tag(config=True)
calibration_path = Unicode(
'', allow_none=True,
help='Path to LST calibration file').tag(config=True)
time_calibration_path = Unicode(
'', allow_none=True,
help='Path to drs4 time calibration file').tag(config=True)
allowed_tels = List(
[1], help='List of telescope to be calibrated').tag(config=True)
gain_threshold = Int(
4094, allow_none=True,
help='Threshold for the gain selection in ADC').tag(config=True)
def __init__(self, **kwargs):
"""
Parameters
----------
reducer_product : ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use. If None, then
NullDataVolumeReducer will be used by default, and waveforms
will not be reduced.
extractor_product : ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, then NeighborPeakWindowSum
will be used by default.
calibration_path :
Path to LST calibration file to get the pedestal and flat-field corrections
kwargs
"""
super().__init__(**kwargs)
# load the waveform charge extractor
self.image_extractor = ImageExtractor.from_name(self.extractor_product,
config=self.config)
self.log.info(f"extractor {self.extractor_product}")
self.data_volume_reducer = DataVolumeReducer.from_name(
self.reducer_product, config=self.config)
self.log.info(f" {self.reducer_product}")
# declare gain selector if the threshold is defined
if self.gain_threshold:
self.gain_selector = gainselection.ThresholdGainSelector(
threshold=self.gain_threshold)
# declare time calibrator if correction file exist
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
self.time_corrector = None
self.log.info(
f"File {self.time_calibration_path} not found. No drs4 time corrections"
)
# calibration data container
self.mon_data = MonitoringContainer()
# initialize the MonitoringContainer() for the moment it reads it from a hdf5 file
self._initialize_correction()
def _initialize_correction(self):
"""
Read the correction from hdf5 calibration file
"""
self.mon_data.tels_with_data = self.allowed_tels
self.log.info(f"read {self.calibration_path}")
try:
with HDF5TableReader(self.calibration_path) as h5_table:
assert h5_table._h5file.isopen == True
for telid in self.allowed_tels:
# read the calibration data for the moment only one event
table = '/tel_' + str(telid) + '/calibration'
next(
h5_table.read(table,
self.mon_data.tel[telid].calibration))
# eliminate inf values (should be done probably before)
dc_to_pe = self.mon_data.tel[telid].calibration.dc_to_pe
dc_to_pe[np.isinf(dc_to_pe)] = 0
self.log.info(
f"read {self.mon_data.tel[telid].calibration.dc_to_pe}"
)
except:
self.log.error(
f"Problem in reading calibration file {self.calibration_path}")
def _calibrate_dl0(self, event, telid):
"""
create dl0 level, for the moment copy the r1
"""
waveforms = event.r1.tel[telid].waveform
if self._check_r1_empty(waveforms):
return
event.dl0.event_id = event.r1.event_id
event.mon.tel[telid].calibration = self.mon_data.tel[telid].calibration
# subtract the pedestal per sample (should we do it?) and multiply for the calibration coefficients
#
event.dl0.tel[telid].waveform = (
(event.r1.tel[telid].waveform - self.mon_data.tel[telid].
calibration.pedestal_per_sample[:, :, np.newaxis]) *
self.mon_data.tel[telid].calibration.dc_to_pe[:, :, np.newaxis])
def _calibrate_dl1(self, event, telid):
"""
create calibrated dl1 image and calibrate it
"""
waveforms = event.dl0.tel[telid].waveform
if self._check_dl0_empty(waveforms):
return
if self.image_extractor.requires_neighbors():
camera = event.inst.subarray.tel[telid].camera
self.image_extractor.neighbors = camera.neighbor_matrix_where
charge, pulse_time = self.image_extractor(waveforms)
# correct time with drs4 correction if available
if self.time_corrector:
pulse_corr_array = self.time_corrector.get_corr_pulse(
event, pulse_time)
# otherwise use the ff time correction (not drs4 corrected)
else:
pulse_corr_array = pulse_time + self.mon_data.tel[
telid].calibration.time_correction
# perform the gain selection if the threshold is defined
if self.gain_threshold:
waveforms, gain_mask = self.gain_selector(
event.r1.tel[telid].waveform)
event.dl1.tel[telid].image = charge[gain_mask,
np.arange(charge.shape[1])]
event.dl1.tel[telid].pulse_time = pulse_corr_array[
gain_mask, np.arange(pulse_corr_array.shape[1])]
# remember the mask in the lst pixel_status array (this info is missing for the moment in the
# r1 container). I follow the prescription given in the document
# "R1 & DL0 Telescope Event Interfaces and Prototype Evaluation" of K. Kosack
# bit 2 = LG
gain_mask *= 4
# bit 3 = HG
gain_mask[np.where(gain_mask == 0)] = 8
# bit 1 = pixel broken pixel (coming from the EvB)
gain_mask += event.lst.tel[telid].evt.pixel_status >> 1 & 1
# update pixel status
event.lst.tel[telid].evt.pixel_status = gain_mask
# if threshold == None
else:
event.dl1.tel[telid].image = charge
event.dl1.tel[telid].pulse_time = pulse_corr_array
class LSTR0Corrections(CameraR0Calibrator):
"""
The R0 calibrator class for LST Camera.
"""
pedestal_path = Unicode(
'', allow_none=True,
help='Path to the LST pedestal binary file').tag(config=True)
def __init__(self, config=None, tool=None, offset=300, **kwargs):
"""
The R0 calibrator for LST data.
Change the r0 container.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
super().__init__(config=config, tool=tool, **kwargs)
self.telid = 0
self.n_module = 265
self.n_gain = 2
self.n_pix = 7
self.size4drs = 4 * 1024
self.roisize = 40
self.offset = offset
self.high_gain = 0
self.low_gain = 1
self.pedestal_value_array = np.zeros(
(self.n_gain, self.n_pix * self.n_module, self.size4drs + 40),
dtype=np.int16)
self.first_cap_array = np.zeros(
(self.n_module, self.n_gain, self.n_pix))
self.first_cap_time_lapse_array = np.zeros(
(self.n_module, self.n_gain, self.n_pix))
self.last_reading_time_array = np.zeros(
(self.n_module, self.n_gain, self.n_pix, self.size4drs))
self.first_cap_array_spike = np.zeros(
(self.n_module, self.n_gain, self.n_pix))
self.first_cap_old_array = np.zeros(
(self.n_module, self.n_gain, self.n_pix))
self._load_calib()
def subtract_pedestal(self, event):
"""
Subtract cell offset using pedestal file.
Change the R0 container.
Parameters
----------
event : `ctapipe` event-container
"""
n_modules = event.lst.tel[0].svc.num_modules
for nr_module in range(0, n_modules):
self.first_cap_array[nr_module, :, :] = self._get_first_capacitor(
event, nr_module)
expected_pixel_id = event.lst.tel[0].svc.pixel_ids
event.r0.tel[self.telid].waveform[:, :, :] = subtract_pedestal_jit(
event.r0.tel[self.telid].waveform, expected_pixel_id,
self.first_cap_array, self.pedestal_value_array, n_modules)
event.r0.tel[self.telid].waveform.astype(np.uint16)
def time_lapse_corr(self, event):
"""
Perform time lapse baseline corrections.
Change the R0 container.
Parameters
----------
event : `ctapipe` event-container
"""
expected_pixel_id = event.lst.tel[0].svc.pixel_ids
local_clock_list = event.lst.tel[0].evt.local_clock_counter
n_modules = event.lst.tel[0].svc.num_modules
for nr_module in range(0, n_modules):
self.first_cap_time_lapse_array[
nr_module, :, :] = self._get_first_capacitor(event, nr_module)
do_time_lapse_corr(event.r0.tel[0].waveform, expected_pixel_id,
local_clock_list, self.first_cap_time_lapse_array,
self.last_reading_time_array, n_modules)
event.r0.tel[self.telid].waveform.astype(np.uint16)
def interpolate_spikes(self, event):
"""
Interpolates spike A & B.
Change the R0 container.
Parameters
----------
event : `ctapipe` event-container
"""
self.first_cap_old_array[:, :, :] = self.first_cap_array_spike[:, :, :]
n_modules = event.lst.tel[0].svc.num_modules
for nr_module in range(0, n_modules):
self.first_cap_array_spike[
nr_module, :, :] = self._get_first_capacitor(event, nr_module)
waveform = event.r0.tel[0].waveform[:, :, :]
expected_pixel_id = event.lst.tel[0].svc.pixel_ids
wf = waveform.copy()
wf = wf.astype('int16')
event.r0.tel[0].waveform = self.interpolate_pseudo_pulses(
wf, expected_pixel_id, self.first_cap_array_spike,
self.first_cap_old_array, n_modules)
event.r0.tel[self.telid].waveform.astype(np.uint16)
@staticmethod
@jit(parallel=True)
def interpolate_pseudo_pulses(waveform, expected_pixel_id, fc, fc_old,
n_modules):
"""
Interpolate Spike A & B.
Change waveform array.
Parameters
----------
waveform : ndarray
Waveform stored in a numpy array of shape
(n_gain, n_pix, n_samples).
expected_pixel_id: ndarray
Array stored expected pixel id
(n_pix*n_modules).
fc : ndarray
Value of first capacitor stored in a numpy array of shape
(n_clus, n_gain, n_pix).
fc_old : ndarray
Value of first capacitor from previous event
stored in a numpy array of shape
(n_clus, n_gain, n_pix).
n_modules : int
Number of modules
"""
roisize = 40
size4drs = 4096
n_gain = 2
n_pix = 7
for nr_module in prange(0, n_modules):
for gain in prange(0, n_gain):
for pix in prange(0, n_pix):
for k in prange(0, 4):
# looking for spike A first case
abspos = int(1024 - roisize - 2 -
fc_old[nr_module, gain, pix] + k * 1024 +
size4drs)
spike_A_position = int(
(abspos - fc[nr_module, gain, pix] + size4drs) %
size4drs)
if (spike_A_position > 2 and spike_A_position < 38):
pixel = expected_pixel_id[nr_module * 7 + pix]
interpolate_spike_A(waveform, gain,
spike_A_position, pixel)
# looking for spike A second case
abspos = int(roisize - 2 +
fc_old[nr_module, gain, pix] + k * 1024 +
size4drs)
spike_A_position = int(
(abspos - fc[nr_module, gain, pix] + size4drs) %
size4drs)
if (spike_A_position > 2 and spike_A_position < 38):
pixel = expected_pixel_id[nr_module * 7 + pix]
interpolate_spike_A(waveform, gain,
spike_A_position, pixel)
# looking for spike B
spike_b_position = int(
(fc_old[nr_module, gain, pix] - 1 -
fc[nr_module, gain, pix] + 2 * size4drs) % size4drs)
if spike_b_position < roisize - 1:
pixel = expected_pixel_id[nr_module * 7 + pix]
interpolate_spike_B(waveform, gain, spike_b_position,
pixel)
return waveform
def _load_calib(self):
"""
Function to load pedestal file.
"""
if self.pedestal_path:
with fits.open(self.pedestal_path) as f:
pedestal_data = np.int16(f[1].data)
self.pedestal_value_array[:, :, :self.
size4drs] = pedestal_data - self.offset
self.pedestal_value_array[:, :, self.size4drs:self.size4drs + 40] \
= pedestal_data[:, :, 0:40] - self.offset
def _get_first_capacitor(self, event, nr_module):
"""
Get first capacitor values from event for nr module.
Parameters
----------
event : `ctapipe` event-container
nr_module : number of module
"""
fc = np.zeros((2, 7))
first_cap = event.lst.tel[0].evt.first_capacitor_id[nr_module *
8:(nr_module + 1) *
8]
# First capacitor order according Dragon v5 board data format
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [0, 0, 1, 1, 2, 2, 3]):
fc[self.high_gain, i] = first_cap[j]
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [4, 4, 5, 5, 6, 6, 7]):
fc[self.low_gain, i] = first_cap[j]
return fc
class TimeSamplingCorrection(Component):
"""
The PulseTimeCorrection class to correct time pulse
using Fourier series expansion.
"""
time_sampling_correction_path = Unicode(
'',
help='Path to the waveform sampling correction file',
allow_none = True,
).tag(config=True)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.time_sampling_coefficients = None
self.load_sampling_coefficient_file()
def load_sampling_coefficient_file(self):
"""
Function to load sampling coefficient file.
"""
try:
with h5py.File(self.time_sampling_correction_path, 'r') as hf:
self.time_sampling_coefficients = np.array(hf['sampling_interval_coefficient'])
except:
self.log.error(f"Problem in reading sampling coefficient file {self.time_sampling_correction_path}")
def get_corrections(self, event, telid):
"""
Get the time/charge sampling corrections for one event and one telescope
Parameters
----------
event: general event container
telid: id of the telescope
Return
------
sampling_corrections: np.array (n_gains, n_pixels, n_samples) with the correction factors
"""
n_gains = 2
n_modules = 265 # number of modules in LST's camera.
n_pixel_per_module = 7 # number of pixels per one module.
size_drs4 = 1024
n_pixels = 1855
roi = 36
# shift the first capacitor to the value used in the r1
r1_roi_start = 3
fc_all = event.lst.tel[telid].evt.first_capacitor_id
# get the first capacitor per pixel and gain
fc = np.zeros((n_gains, n_pixels), dtype=np.int16)
for k in range(n_modules):
for n in range(n_pixel_per_module):
fc[0][k * 7 + n] = fc_all[8 * k + n // 2]
fc[1][k * 7 + n] = fc_all[8 * k + n // 2 + 4]
# reorder first capacitor as in waveform
fc_ordered = np.zeros((n_gains, n_pixels), dtype=np.int16)
fc_ordered[:, event.lst.tel[telid].svc.pixel_ids] = fc
# shift the first capacitor to the value used in the r1
fc_ordered = fc_ordered + r1_roi_start
# first capacitor in the drs4
fc_drs4 = (fc_ordered[:, :]) % size_drs4
# how many slices to the end of buffer
fc_to_last = size_drs4 - fc_drs4[:, :]
# initialize the charge correction vector to one
sampling_corrections = np.ones((n_gains, n_pixels, roi))
# loop over the gains and pixels
for gain in range(n_gains):
for pix in range(n_pixels):
# if I am at the end of the 1024
if 0 < fc_to_last[gain, pix] < roi:
# I complete the buffer
sampling_corrections[gain, pix, :fc_to_last[gain, pix]] = (
sampling_corrections[gain, pix,:fc_to_last[gain, pix]] *
self.time_sampling_coefficients[gain, pix, fc_drs4[gain, pix]:])
# I start again from the beginning of the buffer
sampling_corrections[gain, pix, fc_to_last[gain, pix]:] = (
sampling_corrections[gain, pix,fc_to_last[gain, pix]:] *
self.time_sampling_coefficients[gain, pix,:roi - fc_to_last[gain, pix]])
else:
sampling_corrections[gain, pix, :] = (
sampling_corrections[gain, pix] *
self.time_sampling_coefficients[gain, pix, fc_drs4[gain, pix]:fc_drs4[gain, pix] + roi])
return sampling_corrections
class FlatFieldCalculator(Component):
"""
Parent class for the flat-field calculators.
Fills the MonitoringCameraContainer.FlatfieldContainer on the base of a given
flat-field event sample.
The sample is defined by a maximal interval of time (sample_duration) or a
minimal number of events (sample_duration).
The calculator is supposed to be called in an event loop, extract and collect the
event charge and fill the PedestalContainer
Parameters
----------
tel_id : int
id of the telescope (default 0)
sample_duration : int
interval of time (s) used to gather the pedestal statistics
sample_size : int
number of pedestal events requested for the statistics
n_channels : int
number of waveform channel to be considered
charge_product : str
Name of the charge extractor to be used
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
kwargs
"""
tel_id = Int(
0, help="id of the telescope to calculate the flat-field coefficients"
).tag(config=True)
sample_duration = Int(60, help="sample duration in seconds").tag(config=True)
sample_size = Int(10000, help="sample size").tag(config=True)
n_channels = Int(2, help="number of channels to be treated").tag(config=True)
charge_product = Unicode(
"LocalPeakWindowSum", help="Name of the charge extractor to be used"
).tag(config=True)
def __init__(self, subarray, **kwargs):
"""
Parent class for the flat-field calculators.
Fills the MonitoringCameraContainer.FlatfieldContainer on the base of a given
flat-field event sample.
The sample is defined by a maximal interval of time (sample_duration) or a
minimal number of events (sample_duration).
The calculator is supposed to be called in an event loop, extract and collect the
event charge and fill the PedestalContainer
Parameters
----------
subarray: ctapipe.instrument.SubarrayDescription
Description of the subarray
tel_id : int
id of the telescope (default 0)
sample_duration : int
interval of time (s) used to gather the pedestal statistics
sample_size : int
number of pedestal events requested for the statistics
n_channels : int
number of waveform channel to be considered
charge_product : str
Name of the charge extractor to be used
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
kwargs
"""
super().__init__(**kwargs)
# load the waveform charge extractor
self.extractor = ImageExtractor.from_name(
self.charge_product, config=self.config, subarray=subarray
)
self.log.info(f"extractor {self.extractor}")
@abstractmethod
def calculate_relative_gain(self, event):
"""
class FlasherFlatFieldCalculator(FlatFieldCalculator):
"""Calculates flat-field parameters from flasher data
based on the best algorithm described by S. Fegan in MST-CAM-TN-0060 (eq. 19)
Pixels are defined as outliers on the base of a cut on the pixel charge median
over the full sample distribution and the pixel signal time inside the
waveform time
Parameters:
----------
charge_cut_outliers : List[2]
Interval of accepted charge values (fraction with respect to camera median value)
time_cut_outliers : List[2]
Interval (in waveform samples) of accepted time values
"""
charge_median_cut_outliers = List(
[-0.3, 0.3],
help=
'Interval of accepted charge values (fraction with respect to camera median value)'
).tag(config=True)
time_cut_outliers = List(
[0, 60],
help='Interval (in waveform samples) of accepted time values').tag(
config=True)
charge_std_cut_outliers = List(
[-3, 3],
help=
'Interval (number of std) of accepted charge standard deviation around camera median value'
).tag(config=True)
time_calibration_path = Unicode(
None, allow_none=True,
help='Path to drs4 time calibration file').tag(config=True)
def __init__(self, subarray, **kwargs):
"""Calculates flat-field parameters from flasher data
based on the best algorithm described by S. Fegan in MST-CAM-TN-0060 (eq. 19)
Pixels are defined as outliers on the base of a cut on the pixel charge median
over the full sample distribution and the pixel signal time inside the
waveform time
Parameters:
----------
charge_cut_outliers : List[2]
Interval of accepted charge values (fraction with respect to camera median value)
time_cut_outliers : List[2]
Interval (in waveform samples) of accepted time values
"""
super().__init__(subarray, **kwargs)
self.log.info("Used events statistics : %d", self.sample_size)
# members to keep state in calculate_relative_gain()
self.num_events_seen = 0
self.time_start = None # trigger time of first event in sample
self.trigger_time = None # trigger time of present event
self.charge_medians = None # med. charge in camera per event in sample
self.charges = None # charge per event in sample
self.arrival_times = None # arrival time per event in sample
self.sample_masked_pixels = None # masked pixels per event in sample
if self.time_calibration_path is None:
self.time_corrector = None
else:
# look for calibration path otherwise
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
msg = f"Time calibration file {self.time_calibration_path} not found!"
raise IOError(msg)
def _extract_charge(self, event):
"""
Extract the charge and the time from a calibration event
Parameters
----------
event : general event container
"""
waveforms = event.r1.tel[self.tel_id].waveform
no_gain_selection = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype=np.int)
# Extract charge and time
charge = 0
peak_pos = 0
if self.extractor:
charge, peak_pos = self.extractor(waveforms, self.tel_id,
no_gain_selection)
# correct time with drs4 correction if available
if self.time_corrector:
peak_pos = self.time_corrector.get_corr_pulse(event, peak_pos)
return charge, peak_pos
def calculate_relative_gain(self, event):
"""
calculate the flatfield statistical values
and fill mon.tel[tel_id].flatfield container
Parameters
----------
event : general event container
Returns: True if the mon.tel[tel_id].flatfield is updated, False otherwise
"""
# initialize the np array at each cycle
waveform = event.r1.tel[self.tel_id].waveform
# re-initialize counter
if self.num_events_seen == self.sample_size:
self.num_events_seen = 0
pixel_mask = np.logical_or(
event.mon.tel[self.tel_id].pixel_status.hardware_failing_pixels,
event.mon.tel[self.tel_id].pixel_status.flatfield_failing_pixels)
# real data
if event.meta['origin'] != 'hessio':
self.trigger_time = event.r1.tel[self.tel_id].trigger_time
else: # patches for MC data
if event.trig.tels_with_trigger:
self.trigger_time = event.trig.gps_time.unix
else:
self.trigger_time = 0
if self.num_events_seen == 0:
self.time_start = self.trigger_time
self.setup_sample_buffers(waveform, self.sample_size)
# extract the charge of the event and
# the peak position (assumed as time for the moment)
charge, arrival_time = self._extract_charge(event)
# correct pulse time with drs4 corrections
self.collect_sample(charge, pixel_mask, arrival_time)
sample_age = self.trigger_time - self.time_start
# check if to create a calibration event
if (self.num_events_seen > 0
and (sample_age > self.sample_duration
or self.num_events_seen == self.sample_size)):
# update the monitoring container
self.store_results(event)
return True
else:
return False
def store_results(self, event):
"""
Store stastical results in monitoring container
Parameters
----------
event : general event container
"""
if self.num_events_seen == 0:
raise ValueError(
"No flat-field events in statistics, zero results")
container = event.mon.tel[self.tel_id].flatfield
# mask the part of the array not filled
self.sample_masked_pixels[self.num_events_seen:] = 1
relative_gain_results = self.calculate_relative_gain_results(
self.charge_medians, self.charges, self.sample_masked_pixels)
time_results = self.calculate_time_results(self.arrival_times,
self.sample_masked_pixels,
self.time_start,
self.trigger_time)
result = {
'n_events': self.num_events_seen,
**relative_gain_results,
**time_results,
}
for key, value in result.items():
setattr(container, key, value)
# update the flatfield mask
ff_charge_failing_pixels = np.logical_or(
container.charge_median_outliers, container.charge_std_outliers)
event.mon.tel[self.tel_id].pixel_status.flatfield_failing_pixels = \
np.logical_or(ff_charge_failing_pixels, container.time_median_outliers)
def setup_sample_buffers(self, waveform, sample_size):
"""Initialize sample buffers"""
n_channels = waveform.shape[0]
n_pix = waveform.shape[1]
shape = (sample_size, n_channels, n_pix)
self.charge_medians = np.zeros((sample_size, n_channels))
self.charges = np.zeros(shape)
self.arrival_times = np.zeros(shape)
self.sample_masked_pixels = np.zeros(shape)
def collect_sample(self, charge, pixel_mask, arrival_time):
"""Collect the sample data"""
# extract the charge of the event and
# the peak position (assumed as time for the moment)
good_charge = np.ma.array(charge, mask=pixel_mask)
charge_median = np.ma.median(good_charge, axis=1)
self.charges[self.num_events_seen] = charge
self.arrival_times[self.num_events_seen] = arrival_time
self.sample_masked_pixels[self.num_events_seen] = pixel_mask
self.charge_medians[self.num_events_seen] = charge_median
self.num_events_seen += 1
def calculate_time_results(
self,
trace_time,
masked_pixels_of_sample,
time_start,
trigger_time,
):
"""Calculate and return the time results """
masked_trace_time = np.ma.array(trace_time,
mask=masked_pixels_of_sample)
# median over the sample per pixel
pixel_median = np.ma.median(masked_trace_time, axis=0)
# mean over the sample per pixel
pixel_mean = np.ma.mean(masked_trace_time, axis=0)
# std over the sample per pixel
pixel_std = np.ma.std(masked_trace_time, axis=0)
# median of the median over the camera
median_of_pixel_median = np.ma.median(pixel_median, axis=1)
# time outliers from median
relative_median = pixel_median - median_of_pixel_median[:, np.newaxis]
time_median_outliers = np.logical_or(
pixel_median < self.time_cut_outliers[0],
pixel_median > self.time_cut_outliers[1])
return {
'sample_time': (trigger_time - time_start) / 2 * u.s,
'sample_time_min': time_start * u.s,
'sample_time_max': trigger_time * u.s,
'time_mean': np.ma.getdata(pixel_mean) * u.ns,
'time_median': np.ma.getdata(pixel_median) * u.ns,
'time_std': np.ma.getdata(pixel_std) * u.ns,
'relative_time_median': np.ma.getdata(relative_median) * u.ns,
'time_median_outliers': np.ma.getdata(time_median_outliers),
}
def calculate_relative_gain_results(
self,
event_median,
trace_integral,
masked_pixels_of_sample,
):
"""Calculate and return the sample statistics"""
masked_trace_integral = np.ma.array(trace_integral,
mask=masked_pixels_of_sample)
# median over the sample per pixel
pixel_median = np.ma.median(masked_trace_integral, axis=0)
# mean over the sample per pixel
pixel_mean = np.ma.mean(masked_trace_integral, axis=0)
# std over the sample per pixel
pixel_std = np.ma.std(masked_trace_integral, axis=0)
# median of the median over the camera
median_of_pixel_median = np.ma.median(pixel_median, axis=1)
# median of the std over the camera
median_of_pixel_std = np.ma.median(pixel_std, axis=1)
# std of the std over camera
std_of_pixel_std = np.ma.std(pixel_std, axis=1)
# relative gain
relative_gain_event = masked_trace_integral / event_median[:, :,
np.newaxis]
# outliers from median
charge_deviation = pixel_median - median_of_pixel_median[:, np.newaxis]
charge_median_outliers = (np.logical_or(
charge_deviation < self.charge_median_cut_outliers[0] *
median_of_pixel_median[:, np.newaxis],
charge_deviation > self.charge_median_cut_outliers[1] *
median_of_pixel_median[:, np.newaxis]))
# outliers from standard deviation
deviation = pixel_std - median_of_pixel_std[:, np.newaxis]
charge_std_outliers = (np.logical_or(
deviation <
self.charge_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis],
deviation >
self.charge_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis]))
return {
'relative_gain_median':
np.ma.getdata(np.ma.median(relative_gain_event, axis=0)),
'relative_gain_mean':
np.ma.getdata(np.ma.mean(relative_gain_event, axis=0)),
'relative_gain_std':
np.ma.getdata(np.ma.std(relative_gain_event, axis=0)),
'charge_median':
np.ma.getdata(pixel_median),
'charge_mean':
np.ma.getdata(pixel_mean),
'charge_std':
np.ma.getdata(pixel_std),
'charge_std_outliers':
np.ma.getdata(charge_std_outliers),
'charge_median_outliers':
np.ma.getdata(charge_median_outliers),
}
class CleanigPedestalImage(Component):
"""
Class to chceck pedestal image
"""
tel_id = Int(1, help='Id of the telescope to calibrate').tag(config=True)
charge_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
calib_file = Unicode('',
allow_none=True,
help='Path to the calibration file').tag(config=True)
calib_time_file = Unicode(
'', allow_none=True,
help="Path to the time calibration file").tag(config=True)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.cleaning_parameters = self.config["tailcut"]
print(self.config)
self.r1_dl1_calibrator = LSTCameraCalibrator(
calibration_path=self.calib_file,
time_calibration_path=self.calib_time_file,
extractor_product=self.charge_product,
config=self.config,
gain_threshold=Config(
self.config).gain_selector_config['threshold'],
allowed_tels=[1])
def run(self, list_of_file, max_events):
signal_place_after_clean = np.zeros(1855)
sum_ped_ev = 0
alive_ped_ev = 0
for input_file in list_of_file:
print(input_file)
r0_r1_calibrator = LSTR0Corrections(pedestal_path=None,
r1_sample_start=3,
r1_sample_end=39)
reader = LSTEventSource(input_url=input_file,
max_events=max_events)
for i, ev in enumerate(reader):
r0_r1_calibrator.calibrate(ev)
if i % 10000 == 0:
print(ev.r0.event_id)
if ev.lst.tel[1].evt.tib_masked_trigger == 32:
sum_ped_ev += 1
self.r1_dl1_calibrator(ev)
img = ev.dl1.tel[1].image
geom = ev.inst.subarray.tel[1].camera
clean = tailcuts_clean(geom, img,
**self.cleaning_parameters)
cleaned = img.copy()
cleaned[~clean] = 0.0
signal_place_after_clean[np.where(clean == True)] += 1
if np.sum(cleaned > 0) > 0:
alive_ped_ev += 1
fig, ax = plt.subplots(figsize=(10, 8))
geom = ev.inst.subarray.tel[1].camera
disp0 = CameraDisplay(geom, ax=ax)
disp0.image = signal_place_after_clean / sum_ped_ev
disp0.add_colorbar(ax=ax,
label="N times signal remain after cleaning [%]")
disp0.cmap = 'gnuplot2'
ax.set_title("{} \n {}/{}".format(
input_file.split("/")[-1][8:21], alive_ped_ev, sum_ped_ev),
fontsize=25)
print("{}/{}".format(alive_ped_ev, sum_ped_ev))
ax.set_xlabel(" ")
ax.set_ylabel(" ")
plt.tight_layout()
plt.show()
def remove_star_and_run(self, list_of_file, max_events,
noise_pixels_id_list):
signal_place_after_clean = np.zeros(1855)
sum_ped_ev = 0
alive_ped_ev = 0
for input_file in list_of_file:
print(input_file)
r0_r1_calibrator = LSTR0Corrections(pedestal_path=None,
r1_sample_start=3,
r1_sample_end=39)
reader = LSTEventSource(input_url=input_file,
max_events=max_events)
for i, ev in enumerate(reader):
r0_r1_calibrator.calibrate(ev)
if i % 10000 == 0:
print(ev.r0.event_id)
if ev.lst.tel[1].evt.tib_masked_trigger == 32:
sum_ped_ev += 1
self.r1_dl1_calibrator(ev)
img = ev.dl1.tel[1].image
img[noise_pixels_id_list] = 0
geom = ev.inst.subarray.tel[1].camera
clean = tailcuts_clean(geom, img,
**self.cleaning_parameters)
cleaned = img.copy()
cleaned[~clean] = 0.0
signal_place_after_clean[np.where(clean == True)] += 1
if np.sum(cleaned > 0) > 0:
alive_ped_ev += 1
fig, ax = plt.subplots(figsize=(10, 8))
geom = ev.inst.subarray.tel[1].camera
disp0 = CameraDisplay(geom, ax=ax)
disp0.image = signal_place_after_clean / sum_ped_ev
disp0.highlight_pixels(noise_pixels_id_list, linewidth=3)
disp0.add_colorbar(ax=ax,
label="N times signal remain after cleaning [%]")
disp0.cmap = 'gnuplot2'
ax.set_title("{} \n {}/{}".format(
input_file.split("/")[-1][8:21], alive_ped_ev, sum_ped_ev),
fontsize=25)
print("{}/{}".format(alive_ped_ev, sum_ped_ev))
ax.set_xlabel(" ")
ax.set_ylabel(" ")
plt.tight_layout()
plt.show()
def plot_camera_display(self, image, input_file, noise_pixels_id_list,
alive_ped_ev, sum_ped_ev):
fig, ax = plt.subplots(figsize=(10, 8))
geom = CameraGeometry.from_name('LSTCam-003')
disp0 = CameraDisplay(geom, ax=ax)
disp0.image = image
disp0.highlight_pixels(noise_pixels_id_list, linewidth=3)
disp0.add_colorbar(ax=ax,
label="N times signal remain after cleaning [%]")
disp0.cmap = 'gnuplot2'
ax.set_title("{} \n {}/{}".format(
input_file.split("/")[-1][8:21], alive_ped_ev, sum_ped_ev),
fontsize=25)
print("{}/{}".format(alive_ped_ev, sum_ped_ev))
ax.set_xlabel(" ")
ax.set_ylabel(" ")
plt.tight_layout()
plt.show()
def check_interleave_pedestal_cleaning(self, list_of_file, max_events,
sigma, dl1_file):
high_gain = 0
ped_mean_pe, ped_rms_pe = get_bias_and_rms(dl1_file)
bad_pixel_ids = np.where(ped_rms_pe[1, high_gain, :] == 0)[0]
print(bad_pixel_ids)
th = get_threshold(ped_mean_pe[1, high_gain, :],
ped_rms_pe[1, high_gain, :], sigma)
make_camera_binary_image(th, sigma,
self.cleaning_parameters['picture_thresh'],
bad_pixel_ids)
signal_place_after_clean = np.zeros(1855)
sum_ped_ev = 0
alive_ped_ev = 0
for input_file in list_of_file:
print(input_file)
r0_r1_calibrator = LSTR0Corrections(pedestal_path=None,
r1_sample_start=3,
r1_sample_end=39)
reader = LSTEventSource(input_url=input_file,
max_events=max_events)
for i, ev in enumerate(reader):
r0_r1_calibrator.calibrate(ev)
if i % 10000 == 0:
print(ev.r0.event_id)
if ev.lst.tel[1].evt.tib_masked_trigger == 32:
sum_ped_ev += 1
self.r1_dl1_calibrator(ev)
img = ev.dl1.tel[1].image
img[bad_pixel_ids] = 0
geom = ev.inst.subarray.tel[1].camera
clean = tailcuts_pedestal_clean(geom, img, th,
**self.cleaning_parameters)
cleaned = img.copy()
cleaned[~clean] = 0.0
signal_place_after_clean[np.where(clean == True)] += 1
if np.sum(cleaned > 0) > 0:
alive_ped_ev += 1
noise_remain = signal_place_after_clean / sum_ped_ev
self.plot_camera_display(noise_remain, input_file, bad_pixel_ids,
alive_ped_ev, sum_ped_ev)
class LSTR0Corrections(CameraR0Calibrator):
"""
The R0 calibrator class for LST Camera.
"""
pedestal_path = Unicode(
'', allow_none=True,
help='Path to the LST pedestal binary file').tag(config=True)
def __init__(self, **kwargs):
"""
The R0 calibrator for LST data.
Fill the r1 container.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
super().__init__(**kwargs)
self.n_module = 265
self.n_gain = 2
self.n_pix = 7
self.size4drs = 4 * 1024
self.roisize = 40
self.high_gain = 0
self.low_gain = 1
self.last_run_with_old_firmware = 1574
self.pedestal_value_array = np.zeros(
(self.n_gain, self.n_pix * self.n_module, self.size4drs + 40),
dtype=np.int16)
self.first_cap_array = np.zeros(
(self.n_module, self.n_gain, self.n_pix))
self.first_cap_time_lapse_array = np.zeros(
(self.n_module, self.n_gain, self.n_pix))
self.last_reading_time_array = np.zeros(
(self.n_module, self.n_gain, self.n_pix, self.size4drs))
self.first_cap_array_spike = np.zeros(
(self.n_module, self.n_gain, self.n_pix))
self.first_cap_old_array = np.zeros(
(self.n_module, self.n_gain, self.n_pix))
self._load_calib()
def calibrate(self, event):
for tel_id in event.r0.tels_with_data:
self.subtract_pedestal(event, tel_id)
self.time_lapse_corr(event, tel_id)
self.interpolate_spikes(event, tel_id)
event.r1.tel[tel_id].trigger_type = event.r0.tel[
tel_id].trigger_type
event.r1.tel[tel_id].trigger_time = event.r0.tel[
tel_id].trigger_time
samples = event.r1.tel[tel_id].waveform[:, :,
self.r1_sample_start:self.
r1_sample_end]
event.r1.tel[tel_id].waveform = (samples - self.offset).astype(
np.float32)
def subtract_pedestal(self, event, tel_id):
"""
Subtract cell offset using pedestal file.
Fill the R1 container.
Parameters
----------
event : `ctapipe` event-container
tel_id : id of the telescope
"""
n_modules = event.lst.tel[tel_id].svc.num_modules
for nr_module in range(0, n_modules):
self.first_cap_array[nr_module, :, :] = self._get_first_capacitor(
event, nr_module, tel_id)
expected_pixel_id = event.lst.tel[tel_id].svc.pixel_ids
samples = event.r0.tel[tel_id].waveform.astype(np.float32)
samples = subtract_pedestal_jit(samples, expected_pixel_id,
self.first_cap_array,
self.pedestal_value_array, n_modules)
event.r1.tel[self.tel_id].trigger_type = event.r0.tel[
self.tel_id].trigger_type
event.r1.tel[self.tel_id].trigger_time = event.r1.tel[
self.tel_id].trigger_time
event.r1.tel[self.tel_id].waveform = samples[:, :, :]
def time_lapse_corr(self, event, tel_id):
"""
Perform time lapse baseline corrections.
Fill the R1 container or modifies R0 container.
Parameters
----------
event : `ctapipe` event-container
tel_id : id of the telescope
"""
run_id = event.lst.tel[tel_id].svc.configuration_id
expected_pixel_id = event.lst.tel[tel_id].svc.pixel_ids
local_clock_list = event.lst.tel[tel_id].evt.local_clock_counter
n_modules = event.lst.tel[tel_id].svc.num_modules
for nr_module in range(0, n_modules):
self.first_cap_time_lapse_array[
nr_module, :, :] = self._get_first_capacitor(
event, nr_module, tel_id)
#If R1 container exist modifies it
if isinstance(event.r1.tel[self.tel_id].waveform, np.ndarray):
samples = event.r1.tel[self.tel_id].waveform
# We have 2 functions: one for data from 2018/10/10 to 2019/11/04 and
# one for data from 2019/11/05 (from Run 1574) after update firmware.
# The old readout (before 2019/11/05) is shifted by 1 cell.
if run_id > self.last_run_with_old_firmware:
do_time_lapse_corr(samples, expected_pixel_id,
local_clock_list,
self.first_cap_time_lapse_array,
self.last_reading_time_array, n_modules)
else:
do_time_lapse_corr_data_from_20181010_to_20191104(
samples, expected_pixel_id, local_clock_list,
self.first_cap_time_lapse_array,
self.last_reading_time_array, n_modules)
event.r1.tel[self.tel_id].trigger_type = event.r0.tel[
self.tel_id].trigger_type
event.r1.tel[self.tel_id].trigger_time = event.r0.tel[
self.tel_id].trigger_time
event.r1.tel[self.tel_id].waveform = samples[:, :, :]
else: # Modifies R0 container. This is for create pedestal file.
samples = np.copy(event.r0.tel[self.tel_id].waveform)
if run_id > self.last_run_with_old_firmware:
do_time_lapse_corr(samples, expected_pixel_id,
local_clock_list,
self.first_cap_time_lapse_array,
self.last_reading_time_array, n_modules)
else:
do_time_lapse_corr_data_from_20181010_to_20191104(
samples, expected_pixel_id, local_clock_list,
self.first_cap_time_lapse_array,
self.last_reading_time_array, n_modules)
event.r0.tel[self.tel_id].waveform = samples[:, :, :]
def interpolate_spikes(self, event, tel_id):
"""
Interpolates spike A & B.
Fill the R1 container.
Parameters
----------
event : `ctapipe` event-container
tel_id : id of the telescope
"""
run_id = event.lst.tel[tel_id].svc.configuration_id
self.first_cap_old_array[:, :, :] = self.first_cap_array_spike[:, :, :]
n_modules = event.lst.tel[tel_id].svc.num_modules
for nr_module in range(0, n_modules):
self.first_cap_array_spike[
nr_module, :, :] = self._get_first_capacitor(
event, nr_module, tel_id)
# Interpolate spikes should be done after pedestal subtraction and time lapse correction.
if isinstance(event.r1.tel[tel_id].waveform, np.ndarray):
waveform = event.r1.tel[tel_id].waveform[:, :, :]
expected_pixel_id = event.lst.tel[tel_id].svc.pixel_ids
samples = waveform.copy()
# We have 2 functions: one for data from 2018/10/10 to 2019/11/04 and
# one for data from 2019/11/05 (from Run 1574) after update firmware.
# The old readout (before 2019/11/05) is shifted by 1 cell.
if run_id > self.last_run_with_old_firmware:
event.r1.tel[
self.tel_id].waveform = self.interpolate_pseudo_pulses(
samples, expected_pixel_id, self.first_cap_array_spike,
self.first_cap_old_array, n_modules)
else:
event.r1.tel[self.tel_id].waveform = \
self.interpolate_pseudo_pulses_data_from_20181010_to_20191104(samples,
expected_pixel_id,
self.first_cap_array_spike,
self.first_cap_old_array,
n_modules)
event.r1.tel[self.tel_id].trigger_type = event.r0.tel[
self.tel_id].trigger_type
event.r1.tel[self.tel_id].trigger_time = event.r0.tel[
self.tel_id].trigger_time
@staticmethod
@jit(parallel=True)
def interpolate_pseudo_pulses(waveform, expected_pixel_id, fc, fc_old,
n_modules):
"""
Interpolate Spike A & B.
Change waveform array.
Parameters
----------
waveform : ndarray
Waveform stored in a numpy array of shape
(n_gain, n_pix, n_samples).
expected_pixel_id: ndarray
Array stored expected pixel id
(n_pix*n_modules).
fc : ndarray
Value of first capacitor stored in a numpy array of shape
(n_clus, n_gain, n_pix).
fc_old : ndarray
Value of first capacitor from previous event
stored in a numpy array of shape
(n_clus, n_gain, n_pix).
n_modules : int
Number of modules
"""
roi_size = 40
size1drs = 1024
size4drs = 4096
n_gain = 2
n_pix = 7
for nr_module in prange(0, n_modules):
for gain in prange(0, n_gain):
for pix in prange(0, n_pix):
for k in prange(0, 4):
# looking for spike A first case
abspos = int(size1drs + 1 - roi_size - 2 -
fc_old[nr_module, gain, pix] +
k * size1drs + size4drs)
spike_A_position = int(
(abspos - fc[nr_module, gain, pix] + size4drs) %
size4drs)
if (spike_A_position > 2
and spike_A_position < roi_size - 2):
# The correction is only needed for even
# last capacitor (lc) in the first half of the
# DRS4 ring
if ((fc_old[nr_module, gain, pix] +
(roi_size - 1)) % 2 == 0
and (fc_old[nr_module, gain, pix] +
(roi_size - 1)) % size1drs <=
size1drs // 2 - 1):
pixel = expected_pixel_id[nr_module * 7 + pix]
interpolate_spike_A(waveform, gain,
spike_A_position, pixel)
# looking for spike A second case
abspos = int(roi_size - 1 +
fc_old[nr_module, gain, pix] +
k * size1drs)
spike_A_position = int(
(abspos - fc[nr_module, gain, pix] + size4drs) %
size4drs)
if (spike_A_position > 2 and spike_A_position <
(roi_size - 2)):
# The correction is only needed for even last capacitor (lc) in the
# first half of the DRS4 ring
if ((fc_old[nr_module, gain, pix] +
(roi_size - 1)) % 2 == 0
and (fc_old[nr_module, gain, pix] +
(roi_size - 1)) % size1drs <=
size1drs // 2 - 1):
pixel = expected_pixel_id[nr_module * 7 + pix]
interpolate_spike_A(waveform, gain,
spike_A_position, pixel)
return waveform
@staticmethod
@jit(parallel=True)
def interpolate_pseudo_pulses_data_from_20181010_to_20191104(
waveform, expected_pixel_id, fc, fc_old, n_modules):
"""
Interpolate Spike A & B.
This is function for data from 2018/10/10 to 2019/11/04 with old firmware.
Change waveform array.
Parameters
----------
waveform : ndarray
Waveform stored in a numpy array of shape
(n_gain, n_pix, n_samples).
expected_pixel_id: ndarray
Array stored expected pixel id
(n_pix*n_modules).
fc : ndarray
Value of first capacitor stored in a numpy array of shape
(n_clus, n_gain, n_pix).
fc_old : ndarray
Value of first capacitor from previous event
stored in a numpy array of shape
(n_clus, n_gain, n_pix).
n_modules : int
Number of modules
"""
roi_size = 40
size1drs = 1024
size4drs = 4096
n_gain = 2
n_pix = 7
for nr_module in prange(0, n_modules):
for gain in prange(0, n_gain):
for pix in prange(0, n_pix):
for k in prange(0, 4):
# looking for spike A first case
abspos = int(size1drs - roi_size - 2 -
fc_old[nr_module, gain, pix] +
k * size1drs + size4drs)
spike_A_position = int(
(abspos - fc[nr_module, gain, pix] + size4drs) %
size4drs)
if (spike_A_position > 2
and spike_A_position < roi_size - 2):
# The correction is only needed for even
# last capacitor (lc) in the first half of the
# DRS4 ring
if ((fc_old[nr_module, gain, pix] +
(roi_size - 1)) % 2 == 0
and (fc_old[nr_module, gain, pix] +
(roi_size - 1)) % size1drs <=
size1drs // 2 - 2):
pixel = expected_pixel_id[nr_module * 7 + pix]
interpolate_spike_A(waveform, gain,
spike_A_position, pixel)
# looking for spike A second case
abspos = int(roi_size - 2 +
fc_old[nr_module, gain, pix] +
k * size1drs)
spike_A_position = int(
(abspos - fc[nr_module, gain, pix] + size4drs) %
size4drs)
if (spike_A_position > 2 and spike_A_position <
(roi_size - 2)):
# The correction is only needed for even last capacitor (lc) in the
# first half of the DRS4 ring
if ((fc_old[nr_module, gain, pix] +
(roi_size - 1)) % 2 == 0
and (fc_old[nr_module, gain, pix] +
(roi_size - 1)) % size1drs <=
size1drs // 2 - 2):
pixel = expected_pixel_id[nr_module * 7 + pix]
interpolate_spike_A(waveform, gain,
spike_A_position, pixel)
return waveform
def _load_calib(self):
"""
Function to load pedestal file.
"""
if self.pedestal_path:
with fits.open(self.pedestal_path) as f:
pedestal_data = np.int16(f[1].data)
self.pedestal_value_array[:, :, :self.size4drs] = \
pedestal_data - self.offset
self.pedestal_value_array[:, :, self.size4drs:self.size4drs + 40] \
= pedestal_data[:, :, 0:40] - self.offset
def _get_first_capacitor(self, event, nr_module, tel_id):
"""
Get first capacitor values from event for nr module.
Parameters
----------
event : `ctapipe` event-container
nr_module : number of module
tel_id : id of the telescope
"""
fc = np.zeros((2, 7))
first_cap = event.lst.tel[tel_id].evt.first_capacitor_id[nr_module *
8:(nr_module +
1) * 8]
# First capacitor order according Dragon v5 board data format
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [0, 0, 1, 1, 2, 2, 3]):
fc[self.high_gain, i] = first_cap[j]
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [4, 4, 5, 5, 6, 6, 7]):
fc[self.low_gain, i] = first_cap[j]
return fc
class SingleTelEventDisplay(Tool):
name = "ctapipe-display-televents"
description = Unicode(__doc__)
infile = Unicode(help="input file to read", default='').tag(config=True)
tel = Int(help='Telescope ID to display', default=0).tag(config=True)
channel = Integer(help="channel number to display", min=0,
max=1).tag(config=True)
write = Bool(help="Write out images to PNG files",
default=False).tag(config=True)
clean = Bool(help="Apply image cleaning", default=False).tag(config=True)
hillas = Bool(help="Apply and display Hillas parametrization",
default=False).tag(config=True)
samples = Bool(help="Show each sample", default=False).tag(config=True)
display = Bool(help="Display results in interactive window",
default_value=True).tag(config=True)
delay = Float(help='delay between events in s',
default_value=0.01,
min=0.001).tag(config=True)
progress = Bool(help='display progress bar',
default_value=True).tag(config=True)
aliases = Dict({
'infile': 'SingleTelEventDisplay.infile',
'tel': 'SingleTelEventDisplay.tel',
'max-events': 'EventSource.max_events',
'channel': 'SingleTelEventDisplay.channel',
'write': 'SingleTelEventDisplay.write',
'clean': 'SingleTelEventDisplay.clean',
'hillas': 'SingleTelEventDisplay.hillas',
'samples': 'SingleTelEventDisplay.samples',
'display': 'SingleTelEventDisplay.display',
'delay': 'SingleTelEventDisplay.delay',
'progress': 'SingleTelEventDisplay.progress'
})
classes = List([EventSource, CameraCalibrator])
def __init__(self, **kwargs):
super().__init__(**kwargs)
def setup(self):
print('TOLLES INFILE', self.infile)
self.event_source = EventSource.from_url(self.infile, parent=self)
self.event_source.allowed_tels = {
self.tel,
}
self.calibrator = CameraCalibrator(parent=self)
self.log.info(f'SELECTING EVENTS FROM TELESCOPE {self.tel}')
def start(self):
disp = None
for event in tqdm(self.event_source,
desc=f'Tel{self.tel}',
total=self.event_source.max_events,
disable=~self.progress):
self.log.debug(event.trig)
self.log.debug(f"Energy: {event.mc.energy}")
self.calibrator(event)
if disp is None:
geom = event.inst.subarray.tel[self.tel].camera
self.log.info(geom)
disp = CameraDisplay(geom)
# disp.enable_pixel_picker()
disp.add_colorbar()
if self.display:
plt.show(block=False)
# display the event
disp.axes.set_title('CT{:03d} ({}), event {:06d}'.format(
self.tel, geom.cam_id, event.r0.event_id))
if self.samples:
# display time-varying event
data = event.dl0.tel[self.tel].waveform[self.channel]
for ii in range(data.shape[1]):
disp.image = data[:, ii]
disp.set_limits_percent(70)
plt.suptitle(f"Sample {ii:03d}")
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f'CT{self.tel:03d}_EV{event.r0.event_id:10d}'
f'_S{ii:02d}.png')
else:
# display integrated event:
im = event.dl1.tel[self.tel].image[self.channel]
if self.clean:
mask = tailcuts_clean(geom,
im,
picture_thresh=10,
boundary_thresh=7)
im[~mask] = 0.0
disp.image = im
if self.hillas:
try:
ellipses = disp.axes.findobj(Ellipse)
if len(ellipses) > 0:
ellipses[0].remove()
params = hillas_parameters(geom, image=im)
disp.overlay_moments(params,
color='pink',
lw=3,
with_label=False)
except HillasParameterizationError:
pass
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f'CT{self.tel:03d}_EV{event.r0.event_id:010d}.png')
self.log.info("FINISHED READING DATA FILE")
if disp is None:
self.log.warning(
'No events for tel {} were found in {}. Try a '
'different EventIO file or another telescope'.format(
self.tel, self.infile), )
class LSTCameraCalibrator(CameraCalibrator):
"""
Calibrator to handle the LST camera calibration chain, in order to fill
the DL1 data level in the event container.
"""
extractor_product = Unicode(
'NeighborPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
reducer_product = Unicode(
'NullDataVolumeReducer',
help='Name of the DataVolumeReducer to use').tag(config=True)
calibration_path = Unicode(
'', allow_none=True,
help='Path to LST calibration file').tag(config=True)
allowed_tels = List(
[1], help='List of telescope to be calibrated').tag(config=True)
def __init__(self, **kwargs):
"""
Parameters
----------
reducer_product : ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use. If None, then
NullDataVolumeReducer will be used by default, and waveforms
will not be reduced.
extractor_product : ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, then NeighborPeakWindowSum
will be used by default.
calibration_path :
Path to LST calibration file to get the pedestal and flat-field corrections
kwargs
"""
super().__init__(**kwargs)
# load the waveform charge extractor
self.image_extractor = ImageExtractor.from_name(self.extractor_product,
config=self.config)
self.log.info(f"extractor {self.extractor_product}")
self.data_volume_reducer = DataVolumeReducer.from_name(
self.reducer_product, config=self.config)
self.log.info(f" {self.reducer_product}")
# calibration data container
self.mon_data = MonitoringContainer()
# initialize the MonitoringContainer() for the moment it reads it from a hdf5 file
self._initialize_correction()
def _initialize_correction(self):
"""
Read the correction from hdf5 calibration file
"""
self.mon_data.tels_with_data = self.allowed_tels
self.log.info(f"read {self.calibration_path}")
try:
with HDF5TableReader(self.calibration_path) as h5_table:
assert h5_table._h5file.isopen == True
for telid in self.allowed_tels:
# read the calibration data for the moment only one event
table = '/tel_' + str(telid) + '/calibration'
next(
h5_table.read(table,
self.mon_data.tel[telid].calibration))
# eliminate inf values (should be done probably before)
dc_to_pe = self.mon_data.tel[telid].calibration.dc_to_pe
dc_to_pe[np.isinf(dc_to_pe)] = 0
self.log.info(
f"read {self.mon_data.tel[telid].calibration.dc_to_pe}"
)
except:
self.log.error(
f"Problem in reading calibration file {self.calibration_path}")
def _calibrate_dl0(self, event, telid):
"""
create dl0 level, for the moment copy the r1
"""
waveforms = event.r1.tel[telid].waveform
if self._check_r1_empty(waveforms):
return
event.dl0.event_id = event.r1.event_id
event.mon.tel[telid].calibration = self.mon_data.tel[telid].calibration
# subtract the pedestal per sample (should we do it?) and multiply for the calibration coefficients
#
event.dl0.tel[telid].waveform = (
(event.r1.tel[telid].waveform - self.mon_data.tel[telid].
calibration.pedestal_per_sample[:, :, np.newaxis]) *
self.mon_data.tel[telid].calibration.dc_to_pe[:, :, np.newaxis])
def _calibrate_dl1(self, event, telid):
"""
create calibrated dl1 image and calibrate it
"""
waveforms = event.dl0.tel[telid].waveform
if self._check_dl0_empty(waveforms):
return
if self.image_extractor.requires_neighbors():
camera = event.inst.subarray.tel[telid].camera
self.image_extractor.neighbors = camera.neighbor_matrix_where
charge, pulse_time = self.image_extractor(waveforms)
event.dl0.event_id = event.r1.event_id
event.dl1.tel[telid].image = charge
event.dl1.tel[telid].pulse_time = pulse_time + self.mon_data.tel[
telid].calibration.time_correction
class TailCutsDataVolumeReducer(DataVolumeReducer):
"""
Reduce the time integrated shower image in 3 Steps:
1) Select pixels with tailcuts_clean.
2) Add iteratively all pixels with Signal S >= boundary_thresh
with ctapipe module dilate until no new pixels were added.
3) Adding new pixels with dilate to get more conservative.
Attributes
----------
image_extractor_type: String
Name of the image_extractor to be used.
n_end_dilates: IntTelescopeParameter
Number of how many times to dilate at the end.
do_boundary_dilation: BoolTelescopeParameter
If set to 'False', the iteration steps in 2) are skipped and
normal TailcutCleaning is used.
"""
image_extractor_type = Unicode(
default_value="NeighborPeakWindowSum",
help="Name of the image_extractor"
"to be used.",
).tag(config=True)
n_end_dilates = IntTelescopeParameter(
default_value=1,
help="Number of how many times to dilate at the end.").tag(config=True)
do_boundary_dilation = BoolTelescopeParameter(
default_value=True,
help="If set to 'False', the iteration steps in 2) are skipped and"
"normal TailcutCleaning is used.",
).tag(config=True)
def __init__(self, subarray, config=None, parent=None, **kwargs):
"""
Parameters
----------
subarray: ctapipe.instrument.SubarrayDescription
Description of the subarray
config: traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
kwargs
"""
super().__init__(config=config,
parent=parent,
subarray=subarray,
**kwargs)
self.cleaner = TailcutsImageCleaner(parent=self,
subarray=self.subarray)
self.image_extractor = ImageExtractor.from_name(
self.image_extractor_type, subarray=self.subarray, parent=self)
def select_pixels(self, waveforms, telid=None, selected_gain_channel=None):
camera_geom = self.subarray.tel[telid].camera.geometry
# Pulse-integrate waveforms
charge, _ = self.image_extractor(
waveforms,
telid=telid,
selected_gain_channel=selected_gain_channel)
# 1) Step: TailcutCleaning at first
mask = self.cleaner(telid, charge)
pixels_above_boundary_thresh = (
charge >= self.cleaner.boundary_threshold_pe.tel[telid])
mask_in_loop = np.array([])
# 2) Step: Add iteratively all pixels with Signal
# S > boundary_thresh with ctapipe module
# 'dilate' until no new pixels were added.
while (not np.array_equal(mask, mask_in_loop)
and self.do_boundary_dilation.tel[telid]):
mask_in_loop = mask
mask = dilate(camera_geom, mask) & pixels_above_boundary_thresh
# 3) Step: Adding Pixels with 'dilate' to get more conservative.
for _ in range(self.n_end_dilates.tel[telid]):
mask = dilate(camera_geom, mask)
return mask
class ImageSumDisplayerTool(Tool):
description = Unicode(__doc__)
name = "ctapipe-display-imagesum"
infile = Unicode(
help='input simtelarray file',
default="/Users/kosack/Data/CTA/Prod3/gamma.simtel.gz").tag(
config=True)
telgroup = Integer(help='telescope group number',
default=1).tag(config=True)
max_events = Integer(help='stop after this many events if non-zero',
default_value=0,
min=0).tag(config=True)
output_suffix = Unicode(help='suffix (file extension) of output '
'filenames to write images '
'to (no writing is done if blank). '
'Images will be named [EVENTID][suffix]',
default_value="").tag(config=True)
aliases = Dict({
'infile': 'ImageSumDisplayerTool.infile',
'telgroup': 'ImageSumDisplayerTool.telgroup',
'max-events': 'ImageSumDisplayerTool.max_events',
'output-suffix': 'ImageSumDisplayerTool.output_suffix'
})
classes = List([CameraCalibrator, SimTelEventSource])
def setup(self):
# load up the telescope types table (need to first open a file, a bit of
# a hack until a proper insturment module exists) and select only the
# telescopes with the same camera type
# make sure gzip files are seekable
self.reader = SimTelEventSource(input_url=self.infile,
max_events=self.max_events,
back_seekable=True)
for event in self.reader:
camtypes = event.inst.subarray.to_table().group_by('camera_type')
event.inst.subarray.info(printer=self.log.info)
break
group = camtypes.groups[self.telgroup]
self._selected_tels = list(group['tel_id'].data)
self._base_tel = self._selected_tels[0]
self.log.info("Telescope group %d: %s", self.telgroup,
str(event.inst.subarray.tel[self._selected_tels[0]]))
self.log.info(f"SELECTED TELESCOPES:{self._selected_tels}")
self.calibrator = CameraCalibrator(parent=self)
self.reader.allowed_tels = self._selected_tels
def start(self):
geom = None
imsum = None
disp = None
for event in self.reader:
self.calibrator(event)
if geom is None:
geom = event.inst.subarray.tel[self._base_tel].camera
imsum = np.zeros(shape=geom.pix_x.shape, dtype=np.float)
disp = CameraDisplay(geom, title=geom.cam_id)
disp.add_colorbar()
disp.cmap = 'viridis'
if len(event.dl0.tels_with_data) <= 2:
continue
imsum[:] = 0
for telid in event.dl0.tels_with_data:
imsum += event.dl1.tel[telid].image
self.log.info("event={} ntels={} energy={}".format(
event.r0.event_id, len(event.dl0.tels_with_data),
event.mc.energy))
disp.image = imsum
plt.pause(0.1)
if self.output_suffix is not "":
filename = "{:020d}{}".format(event.r0.event_id,
self.output_suffix)
self.log.info(f"saving: '{filename}'")
plt.savefig(filename)
class DumpTriggersTool(Tool):
description = Unicode(__doc__)
# =============================================
# configuration parameters:
# =============================================
infile = Unicode(help='input simtelarray file').tag(config=True,
allow_none=False)
outfile = Unicode('triggers.fits',
help='output filename (*.fits, *.h5)').tag(config=True)
overwrite = Bool(False,
help="overwrite existing output file").tag(config=True)
# =============================================
# map low-level options to high-level command-line options
# =============================================
aliases = Dict({
'infile': 'DumpTriggersTool.infile',
'outfile': 'DumpTriggersTool.outfile'
})
flags = Dict({
'overwrite': ({
'DumpTriggersTool': {
'overwrite': True
}
}, 'Enable overwriting of output file')
})
examples = ('ctapipe-dump-triggers --infile gamma.simtel.gz '
'--outfile trig.fits --overwrite'
'\n\n'
'If you want to see more output, use --log_level=DEBUG')
# =============================================
# The methods of the Tool (initialize, start, finish):
# =============================================
def add_event_to_table(self, event_id):
"""
add the current pyhessio event to a row in the `self.events` table
"""
ts, tns = pyhessio.get_central_event_gps_time()
gpstime = Time(ts * u.s, tns * u.ns, format='gps', scale='utc')
if self._prev_gpstime is None:
self._prev_gpstime = gpstime
if self._current_starttime is None:
self._current_starttime = gpstime
relative_time = gpstime - self._current_starttime
delta_t = gpstime - self._prev_gpstime
self._prev_gpstime = gpstime
# build the trigger pattern as a fixed-length array
# (better for storage in FITS format)
trigtels = pyhessio.get_telescope_with_data_list()
self._current_trigpattern[:] = 0 # zero the trigger pattern
self._current_trigpattern[trigtels] = 1 # set the triggered tels to 1
# insert the row into the table
self.events.add_row((event_id, relative_time.sec, delta_t.sec,
len(trigtels), self._current_trigpattern))
def setup(self):
""" setup function, called before `start()` """
if self.infile == '':
raise ValueError("No 'infile' parameter was specified. "
"Use --help for info")
self.events = Table(
names=['EVENT_ID', 'T_REL', 'DELTA_T', 'N_TRIG', 'TRIGGERED_TELS'],
dtype=[np.int64, np.float64, np.float64, np.int32, np.uint8])
self.events['TRIGGERED_TELS'].shape = (0, MAX_TELS)
self.events['T_REL'].unit = u.s
self.events['T_REL'].description = 'Time relative to first event'
self.events['DELTA_T'].unit = u.s
self.events.meta['INPUT'] = self.infile
self._current_trigpattern = np.zeros(MAX_TELS)
self._current_starttime = None
self._prev_gpstime = None
pyhessio.file_open(self.infile)
def start(self):
""" main event loop """
for run_id, event_id in pyhessio.move_to_next_event():
self.add_event_to_table(event_id)
def finish(self):
"""
finish up and write out results (called automatically after
`start()`)
"""
pyhessio.close_file()
# write out the final table
if self.outfile.endswith('fits') or self.outfile.endswith('fits.gz'):
self.events.write(self.outfile, overwrite=self.overwrite)
elif self.outfile.endswith('h5'):
self.events.write(self.outfile,
path='/events',
overwrite=self.overwrite)
else:
self.events.write(self.outfile)
self.log.info("Table written to '{}'".format(self.outfile))
self.log.info('\n %s', self.events)
class DumpInstrumentTool(Tool):
description = Unicode(__doc__)
name = 'ctapipe-dump-instrument'
infile = Unicode(help='input simtelarray file').tag(config=True)
format = Enum(['fits', 'ecsv', 'hdf5'],
default_value='fits',
help='Format of output file',
config=True)
aliases = Dict(dict(infile='DumpInstrumentTool.infile',
format='DumpInstrumentTool.format'))
def setup(self):
with event_source(self.infile) as source:
data = next(iter(source)) # get one event, so the instrument table is there
self.inst = data.inst # keep a reference to the instrument stuff
def start(self):
self.write_camera_geometries()
self.write_optics_descriptions()
self.write_subarray_description()
def finish(self):
pass
@staticmethod
def _get_file_format_info(format_name, table_type, table_name):
""" returns file extension + dict of required parameters for
Table.write"""
if format_name == 'fits':
return 'fits.gz', dict()
elif format_name == 'ecsv':
return 'ecsv.txt', dict(format='ascii.ecsv')
elif format_name == 'hdf5':
return 'h5', dict(path="/" + table_type + "/" + table_name)
else:
raise NameError("format not supported")
def write_camera_geometries(self):
cam_types = get_camera_types(self.inst.subarray)
self.inst.subarray.info(printer=self.log.info)
for cam_name in cam_types:
ext, args = self._get_file_format_info(self.format,
'CAMGEOM',
cam_name)
self.log.debug("writing {}".format(cam_name))
tel_id = cam_types[cam_name].pop()
geom = self.inst.subarray.tel[tel_id].camera
table = geom.to_table()
table.meta['SOURCE'] = self.infile
filename = "{}.camgeom.{}".format(cam_name, ext)
try:
table.write(filename, **args)
Provenance().add_output_file(filename, 'dl0.tel.svc.camera')
except IOError as err:
self.log.warn("couldn't write camera definition '%s' because: "
"%s", filename, err)
def write_optics_descriptions(self):
sub = self.inst.subarray
ext, args = self._get_file_format_info(self.format, sub.name, 'optics')
tab = sub.to_table(kind='optics')
tab.meta['SOURCE'] = self.infile
filename = '{}.optics.{}'.format(sub.name, ext)
try:
tab.write(filename, **args)
Provenance().add_output_file(filename, 'dl0.sub.svc.optics')
except IOError as err:
self.log.warn("couldn't write optics description '%s' because: "
"%s", filename, err)
def write_subarray_description(self):
sub = self.inst.subarray
ext, args = self._get_file_format_info(self.format, sub.name,
'subarray')
tab = sub.to_table(kind='subarray')
tab.meta['SOURCE'] = self.infile
filename = '{}.subarray.{}'.format(sub.name, ext)
try:
tab.write(filename, **args)
Provenance().add_output_file(filename, 'dl0.sub.svc.subarray')
except IOError as err:
self.log.warn("couldn't write subarray description '%s' because: "
"%s", filename, err)
class PedestalCalculator(Component):
"""
Parent class for the pedestal calculators.
Fills the MonitoringCameraContainer.PedestalContainer on the base of a given pedestal sample.
The sample is defined by a maximal interval of time (sample_duration) or a
minimal number of events (sample_duration).
The calculator is supposed to act in an event loop, extract and collect the
event charge and fill the PedestalContainer
Parameters
----------
tel_id : int
id of the telescope (default 0)
sample_duration : int
interval of time (s) used to gather the pedestal statistics
sample_size : int
number of pedestal events requested for the statistics
n_channels : int
number of waveform channel to be considered
charge_product : str
Name of the charge extractor to be used
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
kwargs
"""
tel_id = Int(
0,
help='id of the telescope to calculate the pedestal values'
).tag(config=True)
sample_duration = Int(
60,
help='sample duration in seconds'
).tag(config=True)
sample_size = Int(
10000,
help='sample size'
).tag(config=True)
n_channels = Int(
2,
help='number of channels to be treated'
).tag(config=True)
charge_product = Unicode(
'FixedWindowSum',
help='Name of the charge extractor to be used'
).tag(config=True)
def __init__(
self,
**kwargs
):
"""
Parent class for the pedestal calculators.
Fills the MonitoringCameraContainer.PedestalContainer on the base of a given pedestal sample.
The sample is defined by a maximal interval of time (sample_duration) or a
minimal number of events (sample_duration).
The calculator is supposed to act in an event loop, extract and collect the
event charge and fill the PedestalContainer
Parameters
----------
tel_id : int
id of the telescope (default 0)
sample_duration : int
interval of time (s) used to gather the pedestal statistics
sample_size : int
number of pedestal events requested for the statistics
n_channels : int
number of waveform channel to be considered
charge_product : str
Name of the charge extractor to be used
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
kwargs
"""
super().__init__(**kwargs)
# load the waveform charge extractor
self.extractor = ImageExtractor.from_name(
self.charge_product,
config=self.config
)
self.log.info(f"extractor {self.extractor}")
@abstractmethod
def calculate_pedestals(self, event):
"""
class LSTCameraCalibrator(CameraCalibrator):
"""
Calibrator to handle the LST camera calibration chain, in order to fill
the DL1 data level in the event container.
"""
extractor_product = Unicode(
'LocalPeakWindowSum',
help='Name of the charge extractor to be used').tag(config=True)
reducer_product = Unicode(
'NullDataVolumeReducer',
help='Name of the DataVolumeReducer to use').tag(config=True)
calibration_path = Path(
exists=True, directory_ok=False,
help='Path to LST calibration file').tag(config=True)
time_calibration_path = Path(
exists=True,
directory_ok=False,
help='Path to drs4 time calibration file').tag(config=True)
time_sampling_correction_path = Path(
exists=True,
directory_ok=False,
help='Path to time sampling correction file',
allow_none=True,
).tag(config=True)
allowed_tels = List(
[1], help='List of telescope to be calibrated').tag(config=True)
gain_threshold = Int(
4094, allow_none=True,
help='Threshold for the gain selection in ADC').tag(config=True)
charge_scale = List(
[1, 1],
help='Multiplicative correction factor for charge estimation [HG,LG]'
).tag(config=True)
def __init__(self, subarray, **kwargs):
"""
Parameters
----------
reducer_product : ctapipe.image.reducer.DataVolumeReducer
The DataVolumeReducer to use. If None, then
NullDataVolumeReducer will be used by default, and waveforms
will not be reduced.
extractor_product : ctapipe.image.extractor.ImageExtractor
The ImageExtractor to use. If None, then LocalPeakWindowSum
will be used by default.
calibration_path :
Path to LST calibration file to get the pedestal and flat-field corrections
kwargs
"""
super().__init__(subarray, **kwargs)
# load the waveform charge extractor
self.image_extractor = ImageExtractor.from_name(self.extractor_product,
subarray=self.subarray,
config=self.config)
self.log.info(f"extractor {self.extractor_product}")
print("EXTRACTOR", self.image_extractor)
self.data_volume_reducer = DataVolumeReducer.from_name(
self.reducer_product, subarray=self.subarray, config=self.config)
self.log.info(f" {self.reducer_product}")
# declare gain selector if the threshold is defined
if self.gain_threshold:
self.gain_selector = gainselection.ThresholdGainSelector(
threshold=self.gain_threshold)
# declare time calibrator if correction file exist
if os.path.exists(self.time_calibration_path):
self.time_corrector = PulseTimeCorrection(
calib_file_path=self.time_calibration_path)
else:
raise IOError(
f"Time calibration file {self.time_calibration_path} not found!"
)
# declare the charge sampling corrector
if self.time_sampling_correction_path is not None:
# search the file in resources if not found
if not os.path.exists(self.time_sampling_correction_path):
self.time_sampling_correction_path = resource_filename(
'lstchain',
f"resources/{self.time_sampling_correction_path}")
if os.path.exists(self.time_sampling_correction_path):
self.time_sampling_corrector = TimeSamplingCorrection(
time_sampling_correction_path=self.
time_sampling_correction_path)
else:
raise IOError(
f"Sampling correction file {self.time_sampling_correction_path} not found!"
)
else:
self.time_sampling_corrector = None
# calibration data container
self.mon_data = MonitoringContainer()
# initialize the MonitoringContainer() for the moment it reads it from a hdf5 file
self._initialize_correction()
self.log.info(f"Global charge scale {self.charge_scale}")
def _initialize_correction(self):
"""
Read the correction from hdf5 calibration file
"""
self.log.info(f"read {self.calibration_path}")
try:
with HDF5TableReader(self.calibration_path) as h5_table:
for telid in self.allowed_tels:
# read the calibration data
table = '/tel_' + str(telid) + '/calibration'
next(
h5_table.read(table,
self.mon_data.tel[telid].calibration))
# read pedestal data
table = '/tel_' + str(telid) + '/pedestal'
next(
h5_table.read(table,
self.mon_data.tel[telid].pedestal))
# read flat-field data
table = '/tel_' + str(telid) + '/flatfield'
next(
h5_table.read(table,
self.mon_data.tel[telid].flatfield))
# read the pixel_status container
table = '/tel_' + str(telid) + '/pixel_status'
next(
h5_table.read(table,
self.mon_data.tel[telid].pixel_status))
except Exception:
self.log.exception(
f"Problem in reading calibration file {self.calibration_path}")
raise
def _calibrate_dl0(self, event, telid):
"""
create dl0 level, with gain-selected and calibrated waveform
"""
waveforms = event.r1.tel[telid].waveform
if self._check_r1_empty(waveforms):
return
# if not already done, initialize the event monitoring containers
if event.mon.tel[telid].calibration.dc_to_pe is None:
event.mon.tel[telid].calibration = self.mon_data.tel[
telid].calibration
event.mon.tel[telid].flatfield = self.mon_data.tel[telid].flatfield
event.mon.tel[telid].pedestal = self.mon_data.tel[telid].pedestal
event.mon.tel[telid].pixel_status = self.mon_data.tel[
telid].pixel_status
#
# subtract the pedestal per sample and multiply for the calibration coefficients
#
calibrated_waveform = (
(waveforms - self.mon_data.tel[telid].calibration.
pedestal_per_sample[:, :, np.newaxis]) *
self.mon_data.tel[telid].calibration.dc_to_pe[:, :,
np.newaxis]).astype(
np.float32)
# If requested, perform gain selection (this will be done by the EvB in future)
# find the gain selection mask
if waveforms.ndim == 3:
# if threshold defined, perform gain selection
if self.gain_threshold:
gain_mask = self.gain_selector(waveforms)
# select the samples
calibrated_waveform = calibrated_waveform[
gain_mask, np.arange(waveforms.shape[1])]
else:
# keep both HG and LG
gain_mask = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype=np.int64)
gain_mask[1] = 1
else:
# gain selection already performed in EvB: (0=HG, 1=LG)
gain_mask = event.lst.tel[telid].evt.pixel_status >> 2 & 1
# remember the calibrated and gain selected waveform
# (this should be the r1 waveform to be compliant with ctapipe (?))
event.dl0.tel[telid].waveform = calibrated_waveform
# remember which channel has been selected
event.r1.tel[telid].selected_gain_channel = gain_mask
event.dl0.tel[telid].selected_gain_channel = gain_mask
def _calibrate_dl1(self, event, telid):
"""
create calibrated dl1 image and calibrate it
"""
n_pixels = self.subarray.tels[telid].camera.geometry.n_pixels
# copy the waveform be cause I do not want to change it
waveforms = np.copy(event.dl0.tel[telid].waveform)
gain_mask = event.dl0.tel[telid].selected_gain_channel
if self._check_dl0_empty(waveforms):
return
# In case of no gain selection the selected gain channels are [0,0,..][1,1,..]
no_gain_selection = np.zeros((waveforms.shape[0], waveforms.shape[1]),
dtype=np.int64)
no_gain_selection[1] = 1
# correct the dl0 waveform for the sampling time corrections
if self.time_sampling_corrector:
waveforms *= self.time_sampling_corrector.get_corrections(
event, telid)[gain_mask, np.arange(n_pixels)]
# extract the charge
charge, peak_time = self.image_extractor(waveforms, telid, gain_mask)
# correct charge for global scale
corrected_charge = charge * np.array(self.charge_scale,
dtype=np.float32)[gain_mask]
# correct time with drs4 correction if available
if self.time_corrector:
peak_time_drs4_corrected = (
peak_time - self.time_corrector.get_pulse_time_corrections(
event)[gain_mask, np.arange(n_pixels)])
# add flat-fielding time correction
peak_time_ff_corrected = (
peak_time_drs4_corrected +
self.mon_data.tel[telid].calibration.time_correction.value[
gain_mask, np.arange(n_pixels)])
# fill dl1 container
event.dl1.tel[telid].image = corrected_charge
event.dl1.tel[telid].peak_time = peak_time_ff_corrected.astype(
np.float32)
class SimpleEventWriter(Tool):
name = "ctapipe-simple-event-writer"
description = Unicode(__doc__)
infile = Path(
default_value=get_dataset_path(
"lst_prod3_calibration_and_mcphotons.simtel.zst"),
help="input file to read",
directory_ok=False,
exists=True,
).tag(config=True)
outfile = Path(help="output file name",
directory_ok=False,
default_value="output.h5").tag(config=True)
progress = Bool(help="display progress bar",
default_value=True).tag(config=True)
aliases = Dict({
"infile": "EventSource.input_url",
"outfile": "SimpleEventWriter.outfile",
"max-events": "EventSource.max_events",
"progress": "SimpleEventWriter.progress",
})
classes = List([EventSource, CameraCalibrator])
def setup(self):
self.log.info("Configure EventSource...")
self.event_source = EventSource.from_url(self.infile, parent=self)
self.calibrator = CameraCalibrator(subarray=self.event_source.subarray,
parent=self)
self.writer = HDF5TableWriter(filename=self.outfile,
group_name="image_infos",
overwrite=True,
parent=self)
def start(self):
self.log.info("Loop on events...")
for event in tqdm(
self.event_source,
desc="EventWriter",
total=self.event_source.max_events,
disable=~self.progress,
):
self.calibrator(event)
for tel_id in event.dl0.tel.keys():
geom = self.event_source.subarray.tel[tel_id].camera.geometry
dl1_tel = event.dl1.tel[tel_id]
# Image cleaning
image = dl1_tel.image # Waiting for automatic gain selection
mask = tailcuts_clean(geom,
image,
picture_thresh=10,
boundary_thresh=5)
cleaned = image.copy()
cleaned[~mask] = 0
# Image parametrisation
params = hillas_parameters(geom, cleaned)
# Save Ids, MC infos and Hillas informations
self.writer.write(geom.camera_name,
[event.r0, event.simulation.shower, params])
def finish(self):
self.log.info("End of job.")
self.writer.close()
class SimpleEventWriter(Tool):
name = 'ctapipe-simple-event-writer'
description = Unicode(__doc__)
infile = Unicode(help='input file to read', default='').tag(config=True)
outfile = Unicode(help='output file name',
default_value='output.h5').tag(config=True)
progress = Bool(help='display progress bar',
default_value=True).tag(config=True)
aliases = Dict({
'infile': 'EventSource.input_url',
'outfile': 'SimpleEventWriter.outfile',
'max-events': 'EventSource.max_events',
'progress': 'SimpleEventWriter.progress'
})
classes = List([EventSource, CameraCalibrator, CutFlow])
def setup(self):
self.log.info('Configure EventSource...')
self.event_source = self.add_component(
EventSource.from_config(config=self.config, parent=self))
self.calibrator = self.add_component(CameraCalibrator(parent=self))
self.writer = self.add_component(
HDF5TableWriter(filename=self.outfile,
group_name='image_infos',
overwrite=True))
# Define Pre-selection for images
preselcuts = self.config['Preselect']
self.image_cutflow = CutFlow('Image preselection')
self.image_cutflow.set_cuts(
dict(no_sel=None,
n_pixel=lambda s: np.count_nonzero(s) < preselcuts['n_pixel'][
'min'],
image_amplitude=lambda q: q < preselcuts['image_amplitude'][
'min']))
# Define Pre-selection for events
self.event_cutflow = CutFlow('Event preselection')
self.event_cutflow.set_cuts(dict(no_sel=None))
def start(self):
self.log.info('Loop on events...')
for event in tqdm(self.event_source,
desc='EventWriter',
total=self.event_source.max_events,
disable=~self.progress):
self.event_cutflow.count('no_sel')
self.calibrator(event)
for tel_id in event.dl0.tels_with_data:
self.image_cutflow.count('no_sel')
camera = event.inst.subarray.tel[tel_id].camera
dl1_tel = event.dl1.tel[tel_id]
# Image cleaning
image = dl1_tel.image # Waiting for automatic gain selection
mask = tailcuts_clean(camera,
image,
picture_thresh=10,
boundary_thresh=5)
cleaned = image.copy()
cleaned[~mask] = 0
# Preselection cuts
if self.image_cutflow.cut('n_pixel', cleaned):
continue
if self.image_cutflow.cut('image_amplitude', np.sum(cleaned)):
continue
# Image parametrisation
params = hillas_parameters(camera, cleaned)
# Save Ids, MC infos and Hillas informations
self.writer.write(camera.cam_id, [event.r0, event.mc, params])
def finish(self):
self.log.info('End of job.')
self.image_cutflow()
self.event_cutflow()
self.writer.close()
class DumpTriggersTool(Tool):
description = Unicode(__doc__)
name = 'ctapipe-dump-triggers'
# =============================================
# configuration parameters:
# =============================================
infile = Path(exists=True,
directory_ok=False,
help='input simtelarray file').tag(config=True)
outfile = Path(
default_value='triggers.fits',
directory_ok=False,
help='output filename (*.fits, *.h5)',
).tag(config=True)
overwrite = Bool(False,
help="overwrite existing output file").tag(config=True)
# =============================================
# map low-level options to high-level command-line options
# =============================================
aliases = Dict({
'infile': 'DumpTriggersTool.infile',
'outfile': 'DumpTriggersTool.outfile'
})
flags = Dict({
'overwrite': ({
'DumpTriggersTool': {
'overwrite': True
}
}, 'Enable overwriting of output file')
})
examples = ('ctapipe-dump-triggers --infile gamma.simtel.gz '
'--outfile trig.fits --overwrite'
'\n\n'
'If you want to see more output, use --log_level=DEBUG')
# =============================================
# The methods of the Tool (initialize, start, finish):
# =============================================
def add_event_to_table(self, event):
"""
add the current hessio event to a row in the `self.events` table
"""
time = event.trigger.time
if self._prev_time is None:
self._prev_time = time
if self._current_starttime is None:
self._current_starttime = time
relative_time = time - self._current_starttime
delta_t = time - self._prev_time
self._prev_time = time
# build the trigger pattern as a fixed-length array
# (better for storage in FITS format)
# trigtels = event.get_telescope_with_data_list()
trigtels = event.dl0.tels_with_data
self._current_trigpattern[:] = 0 # zero the trigger pattern
self._current_trigpattern[list(trigtels)] = 1 # set the triggered tels
# to 1
# insert the row into the table
self.events.add_row(
(event.index.event_id, relative_time.sec, delta_t.sec,
len(trigtels), self._current_trigpattern))
def setup(self):
""" setup function, called before `start()` """
if self.infile == '':
raise ToolConfigurationError(
"No 'infile' parameter was specified. ")
self.events = Table(
names=['EVENT_ID', 'T_REL', 'DELTA_T', 'N_TRIG', 'TRIGGERED_TELS'],
dtype=[np.int64, np.float64, np.float64, np.int32, np.uint8])
self.events['TRIGGERED_TELS'].shape = (0, MAX_TELS)
self.events['T_REL'].unit = u.s
self.events['T_REL'].description = 'Time relative to first event'
self.events['DELTA_T'].unit = u.s
self.events.meta['INPUT'] = self.infile
self._current_trigpattern = np.zeros(MAX_TELS)
self._current_starttime = None
self._prev_time = None
def start(self):
""" main event loop """
with event_source(self.infile) as source:
for event in source:
self.add_event_to_table(event)
def finish(self):
"""
finish up and write out results (called automatically after
`start()`)
"""
# write out the final table
try:
if '.fits' in self.outfile.suffixes:
self.events.write(self.outfile, overwrite=self.overwrite)
elif self.outfile.suffix in ('.hdf5', '.h5', '.hdf'):
self.events.write(self.outfile,
path='/events',
overwrite=self.overwrite)
else:
self.events.write(self.outfile)
Provenance().add_output_file(self.outfile)
except IOError as err:
self.log.warning("Couldn't write output (%s)", err)
self.log.info('\n %s', self.events)
class ImageSumDisplayerTool(Tool):
description = Unicode(__doc__)
name = "ctapipe-display-imagesum"
infile = Path(
help="input simtelarray file",
default_value=get_dataset_path("gamma_test_large.simtel.gz"),
exists=True,
directory_ok=False,
).tag(config=True)
telgroup = Integer(help="telescope group number", default_value=1).tag(config=True)
max_events = Integer(
help="stop after this many events if non-zero", default_value=0, min=0
).tag(config=True)
output_suffix = Unicode(
help="suffix (file extension) of output "
"filenames to write images "
"to (no writing is done if blank). "
"Images will be named [EVENTID][suffix]",
default_value="",
).tag(config=True)
aliases = Dict(
{
"infile": "ImageSumDisplayerTool.infile",
"telgroup": "ImageSumDisplayerTool.telgroup",
"max-events": "ImageSumDisplayerTool.max_events",
"output-suffix": "ImageSumDisplayerTool.output_suffix",
}
)
classes = [CameraCalibrator, SimTelEventSource]
def setup(self):
# load up the telescope types table (need to first open a file, a bit of
# a hack until a proper instrument module exists) and select only the
# telescopes with the same camera type
# make sure gzip files are seekable
self.reader = SimTelEventSource(
input_url=self.infile, max_events=self.max_events, back_seekable=True
)
camtypes = self.reader.subarray.to_table().group_by("camera_type")
self.reader.subarray.info(printer=self.log.info)
group = camtypes.groups[self.telgroup]
self._selected_tels = list(group["tel_id"].data)
self._base_tel = self._selected_tels[0]
self.log.info(
"Telescope group %d: %s",
self.telgroup,
str(self.reader.subarray.tel[self._selected_tels[0]]),
)
self.log.info(f"SELECTED TELESCOPES:{self._selected_tels}")
self.calibrator = CameraCalibrator(parent=self, subarray=self.reader.subarray)
self.reader = SimTelEventSource(
input_url=self.infile,
max_events=self.max_events,
back_seekable=True,
allowed_tels=set(self._selected_tels),
)
def start(self):
geom = None
imsum = None
disp = None
for event in self.reader:
self.calibrator(event)
if geom is None:
geom = self.reader.subarray.tel[self._base_tel].camera.geometry
imsum = np.zeros(shape=geom.pix_x.shape, dtype=np.float64)
disp = CameraDisplay(geom, title=geom.camera_name)
disp.add_colorbar()
disp.cmap = "viridis"
if len(event.dl0.tel.keys()) <= 2:
continue
imsum[:] = 0
for telid in event.dl0.tel.keys():
imsum += event.dl1.tel[telid].image
self.log.info(
"event={} ntels={} energy={}".format(
event.index.event_id,
len(event.dl0.tel.keys()),
event.simulation.shower.energy,
)
)
disp.image = imsum
plt.pause(0.1)
if self.output_suffix != "":
filename = "{:020d}{}".format(event.index.event_id, self.output_suffix)
self.log.info(f"saving: '{filename}'")
plt.savefig(filename)
class PointingPosition(Component):
"""
Pointion position of telescopes
"""
drive_path = Unicode(
'',
allow_none=True,
help='Path to the LST drive report file'
).tag(config=True)
tel_id = Int(
0,
help='id of the telescope to take drive report for'
).tag(config=True)
def _read_drive_report(self):
"""
Reading drive reports
Parameters:
-----------
str: drive report file
Returns:
data:`~astropy.table.Table`
A table of drive reports
"""
self.log.info("Drive report file:", self.drive_path)
if self.drive_path:
data = ascii.read(self.drive_path)
# Renaming the columns, since the drive report doesn't contain
# these information it self
data['col6'].name = 'time'
data['col8'].name = 'azimuth_avg'
data['col13'].name = 'zenith_avg'
return data
else:
raise Exception("No drive report file found")
def cal_pointingposition(self, ev_time, drive_data):
"""
Calculating pointing positions by interpolation
Parameters:
-----------
time: array
times from events
Drivereport: Container
a container filled with drive information
"""
drive_container = LSTDriveContainer()
drive_container.time = drive_data['time'].data
drive_container.azimuth_avg = drive_data['azimuth_avg'].data
drive_container.altitude_avg = 90.0 - drive_data['zenith_avg'].data
xp = drive_container.time
lower_drive_time = xp[xp < ev_time].max()
upper_drive_time = xp[xp > ev_time].min()
time_in_window = (xp >= lower_drive_time) & (xp <= upper_drive_time)
run_times = xp[time_in_window]
if len(run_times) > 1:
run_azimuth = drive_container.azimuth_avg[time_in_window]
run_altitude = drive_container.altitude_avg[time_in_window]
ev_azimuth = np.interp(ev_time, run_times, run_azimuth) * u.deg
ev_altitude = np.interp(ev_time, run_times, run_altitude) * u.deg
return ev_azimuth, ev_altitude
else:
raise Exception("No drive time in the range of event times")
