class SingleTelEventDisplay(Tool):
name = "ctapipe-display-televents"
description = Unicode(__doc__)
infile = Path(help="input file to read", exists=True,
directory_ok=False).tag(config=True)
tel = Int(help="Telescope ID to display", default_value=0).tag(config=True)
write = Bool(help="Write out images to PNG files",
default_value=False).tag(config=True)
clean = Bool(help="Apply image cleaning",
default_value=False).tag(config=True)
hillas = Bool(help="Apply and display Hillas parametrization",
default_value=False).tag(config=True)
samples = Bool(help="Show each sample",
default_value=False).tag(config=True)
display = Bool(help="Display results in interactive window",
default_value=True).tag(config=True)
delay = Float(help="delay between events in s",
default_value=0.01,
min=0.001).tag(config=True)
progress = Bool(help="display progress bar",
default_value=True).tag(config=True)
aliases = Dict({
"infile": "SingleTelEventDisplay.infile",
"tel": "SingleTelEventDisplay.tel",
"max-events": "EventSource.max_events",
"write": "SingleTelEventDisplay.write",
"clean": "SingleTelEventDisplay.clean",
"hillas": "SingleTelEventDisplay.hillas",
"samples": "SingleTelEventDisplay.samples",
"display": "SingleTelEventDisplay.display",
"delay": "SingleTelEventDisplay.delay",
"progress": "SingleTelEventDisplay.progress",
})
classes = List([EventSource, CameraCalibrator])
def __init__(self, **kwargs):
super().__init__(**kwargs)
def setup(self):
print("TOLLES INFILE", self.infile)
self.event_source = EventSource.from_url(self.infile, parent=self)
self.event_source.allowed_tels = {self.tel}
self.calibrator = CameraCalibrator(parent=self,
subarray=self.event_source.subarray)
self.log.info(f"SELECTING EVENTS FROM TELESCOPE {self.tel}")
def start(self):
disp = None
for event in tqdm(
self.event_source,
desc=f"Tel{self.tel}",
total=self.event_source.max_events,
disable=~self.progress,
):
self.log.debug(event.trigger)
self.log.debug(f"Energy: {event.simulation.shower.energy}")
self.calibrator(event)
if disp is None:
geom = self.event_source.subarray.tel[self.tel].camera.geometry
self.log.info(geom)
disp = CameraDisplay(geom)
# disp.enable_pixel_picker()
disp.add_colorbar()
if self.display:
plt.show(block=False)
# display the event
disp.axes.set_title("CT{:03d} ({}), event {:06d}".format(
self.tel, geom.camera_name, event.index.event_id))
if self.samples:
# display time-varying event
data = event.dl0.tel[self.tel].waveform
for ii in range(data.shape[1]):
disp.image = data[:, ii]
disp.set_limits_percent(70)
plt.suptitle(f"Sample {ii:03d}")
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f"CT{self.tel:03d}_EV{event.index.event_id:10d}"
f"_S{ii:02d}.png")
else:
# display integrated event:
im = event.dl1.tel[self.tel].image
if self.clean:
mask = tailcuts_clean(geom,
im,
picture_thresh=10,
boundary_thresh=7)
im[~mask] = 0.0
disp.image = im
if self.hillas:
try:
ellipses = disp.axes.findobj(Ellipse)
if len(ellipses) > 0:
ellipses[0].remove()
params = hillas_parameters(geom, image=im)
disp.overlay_moments(params,
color="pink",
lw=3,
with_label=False)
except HillasParameterizationError:
pass
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f"CT{self.tel:03d}_EV{event.index.event_id:010d}.png")
self.log.info("FINISHED READING DATA FILE")
if disp is None:
self.log.warning(
"No events for tel {} were found in {}. Try a "
"different EventIO file or another telescope".format(
self.tel, self.infile))
class EventViewer(Component):
"""
Event viewer class built on top of the other plotters to allow a single view of both the camera images
and the projected Hillas parameters for a single event. Can be further modified to show the reconstructed
shower direction and core position if needed. Plus further info
"""
name = 'EventViewer'
test = Bool(True, help='').tag(config=True)
def __init__(self, draw_hillas_planes=False):
"""
Parameters
----------
draw_hillas_planes: bool
Determines whether a projection of the Hillas parameters in the nominal and tilted systems should be drawn
"""
self.array_view = None
self.nominal_view = None
self.geom = dict()
self.cam_display = dict()
self.draw_hillas_planes = draw_hillas_planes
def draw_source(self, source):
"""
Loop over events and draw each
Parameters
----------
source: ctapipe source object
Returns
-------
None
"""
for event in source:
self.draw_event(event)
return
def draw_event(self, event, hillas_parameters=None):
"""
Draw display for a given event
Parameters
----------
event: ctapipe event object
hillas_parameters: dict
Dictionary of Hillas parameters (in nominal system)
Returns
-------
None
"""
tel_list = event.r0.tels_with_data
images = event.dl1
# First close any plots that already exist
plt.close()
ntels = len(tel_list)
fig = plt.figure(figsize=(20, 20 * 0.66))
# If we want to draw the Hillas parameters in different planes we need to split our figure
if self.draw_hillas_planes:
y_axis_split = 2
else:
y_axis_split = 1
outer_grid = gridspec.GridSpec(1,
y_axis_split,
width_ratios=[y_axis_split, 1])
# Create a square grid for camera images
nn = int(ceil(sqrt(ntels)))
nx = nn
ny = nn
while nx * ny >= ntels:
ny -= 1
ny += 1
while nx * ny >= ntels:
nx -= 1
nx += 1
camera_grid = gridspec.GridSpecFromSubplotSpec(
ny, nx, subplot_spec=outer_grid[0])
# Loop over camera images of all telescopes and create plots
for ii, tel_id in zip(range(ntels), tel_list):
# Cache of camera geometries, this may go away soon
if tel_id not in self.geom:
self.geom[tel_id] = CameraGeometry.guess(
event.inst.pixel_pos[tel_id][0],
event.inst.pixel_pos[tel_id][1],
event.inst.optical_foclen[tel_id])
ax = plt.subplot(camera_grid[ii])
self.get_camera_view(tel_id, images.tel[tel_id].image[0], ax)
# If we want to draw the Hillas parameters in different planes we need to make a couple more viewers
if self.draw_hillas_planes:
# Split the second sub figure into two further figures
reco_grid = gridspec.GridSpecFromSubplotSpec(
2, 1, subplot_spec=outer_grid[1])
# Create plot of telescope positions at ground level
array = ArrayPlotter(
telescopes=tel_list,
instrument=event.inst, # system=tilted_system,
ax=plt.subplot(reco_grid[0]))
# Draw MC position (this should change later)
array.draw_position(event.mc.core_x,
event.mc.core_y,
use_centre=True)
array.draw_array(((-300, 300), (-300, 300)))
# If we have valid Hillas parameters we should draw them in the Nominal system
if hillas_parameters is not None:
array.overlay_hillas(hillas_parameters, draw_axes=True)
nominal = NominalPlotter(hillas_parameters=hillas_parameters,
draw_axes=True,
ax=plt.subplot(reco_grid[1]))
nominal.draw_array()
plt.show()
return
def get_camera_view(self, tel_id, image, axis):
"""
Create camera viewer for a given camera image
Parameters
----------
tel_id: int
Telescope ID number
image: ndarray
Array of calibrated pixel intensities
axis: matplotlib axis
Axis on which to draw plot
Returns
-------
Camera display
"""
#if tel_id not in self.cam_display:
# Argh this is annoying, for some reason we cannot cahe the displays
self.cam_display[tel_id] = visualization.CameraDisplay(
self.geom[tel_id], title="CT{0}".format(tel_id))
self.cam_display[tel_id].add_colorbar()
self.cam_display[tel_id].pixels.set_antialiaseds(False)
self.cam_display[tel_id].autoupdate = True
self.cam_display[tel_id].cmap = "viridis"
self.cam_display[tel_id].ax = axis
self.cam_display[tel_id].image = image
self.cam_display[tel_id].set_limits_percent(95)
return self.cam_display[tel_id]
class SimpleEventWriter(Tool):
name = 'ctapipe-simple-event-writer'
description = Unicode(__doc__)
infile = Unicode(help='input file to read', default='').tag(config=True)
outfile = Unicode(help='output file name',
default_value='output.h5').tag(config=True)
progress = Bool(help='display progress bar',
default_value=True).tag(config=True)
aliases = Dict({
'infile': 'EventSource.input_url',
'outfile': 'SimpleEventWriter.outfile',
'max-events': 'EventSource.max_events',
'progress': 'SimpleEventWriter.progress'
})
classes = List([EventSource, CameraCalibrator, CutFlow])
def setup(self):
self.log.info('Configure EventSource...')
self.event_source = self.add_component(
EventSource.from_config(config=self.config, parent=self))
self.calibrator = self.add_component(CameraCalibrator(parent=self))
self.writer = self.add_component(
HDF5TableWriter(filename=self.outfile,
group_name='image_infos',
overwrite=True))
# Define Pre-selection for images
preselcuts = self.config['Preselect']
self.image_cutflow = CutFlow('Image preselection')
self.image_cutflow.set_cuts(
dict(no_sel=None,
n_pixel=lambda s: np.count_nonzero(s) < preselcuts['n_pixel'][
'min'],
image_amplitude=lambda q: q < preselcuts['image_amplitude'][
'min']))
# Define Pre-selection for events
self.event_cutflow = CutFlow('Event preselection')
self.event_cutflow.set_cuts(dict(no_sel=None))
def start(self):
self.log.info('Loop on events...')
for event in tqdm(self.event_source,
desc='EventWriter',
total=self.event_source.max_events,
disable=~self.progress):
self.event_cutflow.count('no_sel')
self.calibrator(event)
for tel_id in event.dl0.tels_with_data:
self.image_cutflow.count('no_sel')
camera = event.inst.subarray.tel[tel_id].camera
dl1_tel = event.dl1.tel[tel_id]
# Image cleaning
image = dl1_tel.image # Waiting for automatic gain selection
mask = tailcuts_clean(camera,
image,
picture_thresh=10,
boundary_thresh=5)
cleaned = image.copy()
cleaned[~mask] = 0
# Preselection cuts
if self.image_cutflow.cut('n_pixel', cleaned):
continue
if self.image_cutflow.cut('image_amplitude', np.sum(cleaned)):
continue
# Image parametrisation
params = hillas_parameters(camera, cleaned)
# Save Ids, MC infos and Hillas informations
self.writer.write(camera.cam_id, [event.r0, event.mc, params])
def finish(self):
self.log.info('End of job.')
self.image_cutflow()
self.event_cutflow()
self.writer.close()
class DumpTriggersTool(Tool):
description = Unicode(__doc__)
name = 'ctapipe-dump-triggers'
# =============================================
# configuration parameters:
# =============================================
infile = Path(exists=True,
directory_ok=False,
help='input simtelarray file').tag(config=True)
outfile = Path(
default_value='triggers.fits',
directory_ok=False,
help='output filename (*.fits, *.h5)',
).tag(config=True)
overwrite = Bool(False,
help="overwrite existing output file").tag(config=True)
# =============================================
# map low-level options to high-level command-line options
# =============================================
aliases = Dict({
'infile': 'DumpTriggersTool.infile',
'outfile': 'DumpTriggersTool.outfile'
})
flags = Dict({
'overwrite': ({
'DumpTriggersTool': {
'overwrite': True
}
}, 'Enable overwriting of output file')
})
examples = ('ctapipe-dump-triggers --infile gamma.simtel.gz '
'--outfile trig.fits --overwrite'
'\n\n'
'If you want to see more output, use --log_level=DEBUG')
# =============================================
# The methods of the Tool (initialize, start, finish):
# =============================================
def add_event_to_table(self, event):
"""
add the current hessio event to a row in the `self.events` table
"""
time = event.trigger.time
if self._prev_time is None:
self._prev_time = time
if self._current_starttime is None:
self._current_starttime = time
relative_time = time - self._current_starttime
delta_t = time - self._prev_time
self._prev_time = time
# build the trigger pattern as a fixed-length array
# (better for storage in FITS format)
# trigtels = event.get_telescope_with_data_list()
trigtels = event.dl0.tels_with_data
self._current_trigpattern[:] = 0 # zero the trigger pattern
self._current_trigpattern[list(trigtels)] = 1 # set the triggered tels
# to 1
# insert the row into the table
self.events.add_row(
(event.index.event_id, relative_time.sec, delta_t.sec,
len(trigtels), self._current_trigpattern))
def setup(self):
""" setup function, called before `start()` """
if self.infile == '':
raise ToolConfigurationError(
"No 'infile' parameter was specified. ")
self.events = Table(
names=['EVENT_ID', 'T_REL', 'DELTA_T', 'N_TRIG', 'TRIGGERED_TELS'],
dtype=[np.int64, np.float64, np.float64, np.int32, np.uint8])
self.events['TRIGGERED_TELS'].shape = (0, MAX_TELS)
self.events['T_REL'].unit = u.s
self.events['T_REL'].description = 'Time relative to first event'
self.events['DELTA_T'].unit = u.s
self.events.meta['INPUT'] = self.infile
self._current_trigpattern = np.zeros(MAX_TELS)
self._current_starttime = None
self._prev_time = None
def start(self):
""" main event loop """
with event_source(self.infile) as source:
for event in source:
self.add_event_to_table(event)
def finish(self):
"""
finish up and write out results (called automatically after
`start()`)
"""
# write out the final table
try:
if '.fits' in self.outfile.suffixes:
self.events.write(self.outfile, overwrite=self.overwrite)
elif self.outfile.suffix in ('.hdf5', '.h5', '.hdf'):
self.events.write(self.outfile,
path='/events',
overwrite=self.overwrite)
else:
self.events.write(self.outfile)
Provenance().add_output_file(self.outfile)
except IOError as err:
self.log.warning("Couldn't write output (%s)", err)
self.log.info('\n %s', self.events)
class SingleTelEventDisplay(Tool):
name = "ctapipe-display-televents"
description = Unicode(__doc__)
infile = Unicode(help="input file to read", default='').tag(config=True)
tel = Int(help='Telescope ID to display', default=0).tag(config=True)
channel = Integer(help="channel number to display", min=0,
max=1).tag(config=True)
write = Bool(help="Write out images to PNG files",
default=False).tag(config=True)
clean = Bool(help="Apply image cleaning", default=False).tag(config=True)
hillas = Bool(help="Apply and display Hillas parametrization",
default=False).tag(config=True)
samples = Bool(help="Show each sample", default=False).tag(config=True)
display = Bool(help="Display results in interactive window",
default_value=True).tag(config=True)
delay = Float(help='delay between events in s',
default_value=0.01,
min=0.001).tag(config=True)
progress = Bool(help='display progress bar',
default_value=True).tag(config=True)
aliases = Dict({
'infile': 'SingleTelEventDisplay.infile',
'tel': 'SingleTelEventDisplay.tel',
'max-events': 'EventSource.max_events',
'channel': 'SingleTelEventDisplay.channel',
'write': 'SingleTelEventDisplay.write',
'clean': 'SingleTelEventDisplay.clean',
'hillas': 'SingleTelEventDisplay.hillas',
'samples': 'SingleTelEventDisplay.samples',
'display': 'SingleTelEventDisplay.display',
'delay': 'SingleTelEventDisplay.delay',
'progress': 'SingleTelEventDisplay.progress'
})
classes = List([EventSource, CameraCalibrator])
def __init__(self, **kwargs):
super().__init__(**kwargs)
def setup(self):
print('TOLLES INFILE', self.infile)
self.event_source = EventSource.from_url(self.infile, parent=self)
self.event_source.allowed_tels = {
self.tel,
}
self.calibrator = CameraCalibrator(parent=self)
self.log.info(f'SELECTING EVENTS FROM TELESCOPE {self.tel}')
def start(self):
disp = None
for event in tqdm(self.event_source,
desc=f'Tel{self.tel}',
total=self.event_source.max_events,
disable=~self.progress):
self.log.debug(event.trig)
self.log.debug(f"Energy: {event.mc.energy}")
self.calibrator(event)
if disp is None:
geom = event.inst.subarray.tel[self.tel].camera
self.log.info(geom)
disp = CameraDisplay(geom)
# disp.enable_pixel_picker()
disp.add_colorbar()
if self.display:
plt.show(block=False)
# display the event
disp.axes.set_title('CT{:03d} ({}), event {:06d}'.format(
self.tel, geom.cam_id, event.r0.event_id))
if self.samples:
# display time-varying event
data = event.dl0.tel[self.tel].waveform[self.channel]
for ii in range(data.shape[1]):
disp.image = data[:, ii]
disp.set_limits_percent(70)
plt.suptitle(f"Sample {ii:03d}")
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f'CT{self.tel:03d}_EV{event.r0.event_id:10d}'
f'_S{ii:02d}.png')
else:
# display integrated event:
im = event.dl1.tel[self.tel].image[self.channel]
if self.clean:
mask = tailcuts_clean(geom,
im,
picture_thresh=10,
boundary_thresh=7)
im[~mask] = 0.0
disp.image = im
if self.hillas:
try:
ellipses = disp.axes.findobj(Ellipse)
if len(ellipses) > 0:
ellipses[0].remove()
params = hillas_parameters(geom, image=im)
disp.overlay_moments(params,
color='pink',
lw=3,
with_label=False)
except HillasParameterizationError:
pass
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f'CT{self.tel:03d}_EV{event.r0.event_id:010d}.png')
self.log.info("FINISHED READING DATA FILE")
if disp is None:
self.log.warning(
'No events for tel {} were found in {}. Try a '
'different EventIO file or another telescope'.format(
self.tel, self.infile), )
class LSTEventSource(EventSource):
"""EventSource for LST r0 data."""
n_gains = Int(
2,
help='Number of gains at r0/r1 level'
).tag(config=True)
baseline = Int(
400,
help='r0 waveform baseline (default from EvB v3)'
).tag(config=True)
multi_streams = Bool(
True,
help='Read in parallel all streams '
).tag(config=True)
def __init__(self, **kwargs):
"""
Constructor
Parameters
----------
n_gains = number of gains expected in input file
baseline = baseline to be subtracted at r1 level (not used for the moment)
multi_streams = enable the reading of input files from all streams
config: traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.\
kwargs: dict
Additional parameters to be passed.
NOTE: The file mask of the data to read can be passed with
the 'input_url' parameter.
"""
super().__init__(**kwargs)
if self.multi_streams:
# test how many streams are there:
# file name must be [stream name]Run[all the rest]
# All the files with the same [all the rest] are opened
path, name = os.path.split(os.path.abspath(self.input_url))
if 'Run' in name:
stream, run = name.split('Run', 1)
else:
run = name
ls = listdir(path)
self.file_list = []
for file_name in ls:
if run in file_name:
full_name = os.path.join(path, file_name)
self.file_list.append(full_name)
else:
self.file_list = [self.input_url]
self.multi_file = MultiFiles(self.file_list)
self.geometry_version = 4
self.camera_config = self.multi_file.camera_config
self.log.info(
"Read {} input files".format(
self.multi_file.num_inputs()
)
)
self.tel_id = self.camera_config.telescope_id
self._subarray = self.create_subarray(self.tel_id)
self.n_camera_pixels = self.subarray.tel[self.tel_id].camera.n_pixels
@property
def subarray(self):
return self._subarray
@property
def is_simulation(self):
return False
@property
def obs_id(self):
# currently no obs id is available from the input files
return self.camera_config.configuration_id
@property
def datalevels(self):
return (DataLevel.R0, )
def rewind(self):
self.multi_file.rewind()
def create_subarray(self, tel_id=1):
"""
Obtain the subarray from the EventSource
Returns
-------
ctapipe.instrument.SubarrayDecription
"""
# camera info from LSTCam-[geometry_version].camgeom.fits.gz file
camera = load_camera_geometry(version=self.geometry_version)
tel_descr = TelescopeDescription(
name='LST', tel_type='LST', optics=OPTICS, camera=camera
)
tels = {tel_id: tel_descr}
# LSTs telescope position taken from MC from the moment
tel_pos = {tel_id: [50., 50., 16] * u.m}
subarray = SubarrayDescription("LST1 subarray")
subarray.tels = tels
subarray.positions = tel_pos
return subarray
def _generator(self):
# container for LST data
self.data = LSTDataContainer()
self.data.meta['input_url'] = self.input_url
self.data.meta['max_events'] = self.max_events
self.data.meta['origin'] = 'LSTCAM'
# fill LST data from the CameraConfig table
self.fill_lst_service_container_from_zfile()
# initialize general monitoring container
self.initialize_mon_container()
# loop on events
for count, event in enumerate(self.multi_file):
self.data.count = count
self.data.index.event_id = event.event_id
self.data.index.obs_id = self.obs_id
# fill specific LST event data
self.fill_lst_event_container_from_zfile(event)
# fill general monitoring data
self.fill_mon_container_from_zfile(event)
# fill general R0 data
self.fill_r0_container_from_zfile(event)
yield self.data
@staticmethod
def is_compatible(file_path):
from astropy.io import fits
try:
# The file contains two tables:
# 1: CameraConfig
# 2: Events
h = fits.open(file_path)[2].header
ttypes = [
h[x] for x in h.keys() if 'TTYPE' in x
]
except OSError:
# not even a fits file
return False
except IndexError:
# A fits file of a different format
return False
is_protobuf_zfits_file = (
(h['XTENSION'] == 'BINTABLE') and
(h['EXTNAME'] == 'Events') and
(h['ZTABLE'] is True) and
(h['ORIGIN'] == 'CTA') and
(h['PBFHEAD'] == 'R1.CameraEvent')
)
is_lst_file = 'lstcam_counters' in ttypes
return is_protobuf_zfits_file & is_lst_file
def fill_lst_service_container_from_zfile(self):
"""
Fill LSTServiceContainer with specific LST service data data
(from the CameraConfig table of zfit file)
"""
self.data.lst.tels_with_data = [self.tel_id, ]
svc_container = self.data.lst.tel[self.tel_id].svc
svc_container.telescope_id = self.tel_id
svc_container.cs_serial = self.camera_config.cs_serial
svc_container.configuration_id = self.camera_config.configuration_id
svc_container.date = self.camera_config.date
svc_container.num_pixels = self.camera_config.num_pixels
svc_container.num_samples = self.camera_config.num_samples
svc_container.pixel_ids = self.camera_config.expected_pixels_id
svc_container.data_model_version = self.camera_config.data_model_version
svc_container.num_modules = self.camera_config.lstcam.num_modules
svc_container.module_ids = self.camera_config.lstcam.expected_modules_id
svc_container.idaq_version = self.camera_config.lstcam.idaq_version
svc_container.cdhs_version = self.camera_config.lstcam.cdhs_version
svc_container.algorithms = self.camera_config.lstcam.algorithms
svc_container.pre_proc_algorithms = self.camera_config.lstcam.pre_proc_algorithms
def fill_lst_event_container_from_zfile(self, event):
"""
Fill LSTEventContainer with specific LST service data
(from the Event table of zfit file)
"""
event_container = self.data.lst.tel[self.tel_id].evt
event_container.configuration_id = event.configuration_id
event_container.event_id = event.event_id
event_container.tel_event_id = event.tel_event_id
event_container.pixel_status = event.pixel_status
event_container.ped_id = event.ped_id
event_container.module_status = event.lstcam.module_status
event_container.extdevices_presence = event.lstcam.extdevices_presence
# if TIB data are there
if event_container.extdevices_presence & 1:
# unpack TIB data
rec_fmt = '=IHIBB'
unpacked_tib = struct.unpack(rec_fmt, event.lstcam.tib_data)
event_container.tib_event_counter = unpacked_tib[0]
event_container.tib_pps_counter = unpacked_tib[1]
event_container.tib_tenMHz_counter = unpacked_tib[2]
event_container.tib_stereo_pattern = unpacked_tib[3]
event_container.tib_masked_trigger = unpacked_tib[4]
# if UCTS data are there
if event_container.extdevices_presence & 2:
if int(self.data.lst.tel[self.tel_id].svc.idaq_version) > 37201:
# unpack UCTS-CDTS data (new version)
rec_fmt = '=QIIIIIBBBBI'
unpacked_cdts = struct.unpack(rec_fmt, event.lstcam.cdts_data)
event_container.ucts_timestamp = unpacked_cdts[0]
event_container.ucts_address = unpacked_cdts[1] # new
event_container.ucts_event_counter = unpacked_cdts[2]
event_container.ucts_busy_counter = unpacked_cdts[3] # new
event_container.ucts_pps_counter = unpacked_cdts[4]
event_container.ucts_clock_counter = unpacked_cdts[5]
event_container.ucts_trigger_type = unpacked_cdts[6]
event_container.ucts_white_rabbit_status = unpacked_cdts[7]
event_container.ucts_stereo_pattern = unpacked_cdts[8] # new
event_container.ucts_num_in_bunch = unpacked_cdts[9] # new
event_container.ucts_cdts_version = unpacked_cdts[10] # new
else:
# unpack UCTS-CDTS data (old version)
rec_fmt = '=IIIQQBBB'
unpacked_cdts = struct.unpack(rec_fmt, event.lstcam.cdts_data)
event_container.ucts_event_counter = unpacked_cdts[0]
event_container.ucts_pps_counter = unpacked_cdts[1]
event_container.ucts_clock_counter = unpacked_cdts[2]
event_container.ucts_timestamp = unpacked_cdts[3]
event_container.ucts_camera_timestamp = unpacked_cdts[4]
event_container.ucts_trigger_type = unpacked_cdts[5]
event_container.ucts_white_rabbit_status = unpacked_cdts[6]
# if SWAT data are there
if event_container.extdevices_presence & 4:
# unpack SWAT data
rec_fmt = '=QIIBBIBI'
unpacked_swat = struct.unpack(rec_fmt, event.lstcam.swat_data)
event_container.swat_timestamp = unpacked_swat[0]
event_container.swat_counter1 = unpacked_swat[1]
event_container.swat_counter2 = unpacked_swat[2]
event_container.swat_event_type = unpacked_swat[3]
event_container.swat_camera_flag = unpacked_swat[4]
event_container.swat_camera_event_num = unpacked_swat[5]
event_container.swat_array_flag = unpacked_swat[6]
event_container.swat_array_event_num = unpacked_swat[7]
# unpack Dragon counters
rec_fmt = '=HIIIQ'
rec_len = struct.calcsize(rec_fmt)
rec_unpack = struct.Struct(rec_fmt).unpack_from
event_container.pps_counter = np.zeros(self.camera_config.lstcam.num_modules)
event_container.tenMHz_counter = np.zeros(self.camera_config.lstcam.num_modules)
event_container.event_counter = np.zeros(self.camera_config.lstcam.num_modules)
event_container.trigger_counter = np.zeros(self.camera_config.lstcam.num_modules)
event_container.local_clock_counter = np.zeros(self.camera_config.lstcam.num_modules)
for mod in range(self.camera_config.lstcam.num_modules):
words=event.lstcam.counters[mod*rec_len:(mod+1)*rec_len]
unpacked_counter = rec_unpack(words)
event_container.pps_counter[mod] = unpacked_counter[0]
event_container.tenMHz_counter[mod] = unpacked_counter[1]
event_container.event_counter[mod] = unpacked_counter[2]
event_container.trigger_counter[mod] = unpacked_counter[3]
event_container.local_clock_counter[mod] = unpacked_counter[4]
event_container.chips_flags = event.lstcam.chips_flags
event_container.first_capacitor_id = event.lstcam.first_capacitor_id
event_container.drs_tag_status = event.lstcam.drs_tag_status
event_container.drs_tag = event.lstcam.drs_tag
def fill_r0_camera_container_from_zfile(self, r0_container, event):
"""
Fill with R0CameraContainer
"""
# look for correct trigger_time (TAI time in s), first in UCTS and then in TIB
#if self.data.lst.tel[self.tel_id].evt.ucts_timestamp > 0:
# r0_container.trigger_time = self.data.lst.tel[self.tel_id].evt.ucts_timestamp/1e9
# consider for the moment only TIB time since UCTS seems not correct
#if self.data.lst.tel[self.tel_id].evt.tib_pps_counter > 0:
# r0_container.trigger_time = (
# self.data.lst.tel[self.tel_id].svc.date +
# self.data.lst.tel[self.tel_id].evt.tib_pps_counter +
# self.data.lst.tel[self.tel_id].evt.tib_tenMHz_counter * 10**(-7))
#else:
# r0_container.trigger_time = 0
#consider for the moment trigger time from central dragon module
module_rank = np.where(self.data.lst.tel[self.tel_id].svc.module_ids == 132)
r0_container.trigger_time = (
self.data.lst.tel[self.tel_id].svc.date +
self.data.lst.tel[self.tel_id].evt.pps_counter[module_rank] +
self.data.lst.tel[self.tel_id].evt.tenMHz_counter[module_rank] * 10**(-7))
# look for correct trigger type first in UCTS and then in TIB
#if self.data.lst.tel[self.tel_id].evt.ucts_trigger_type > 0:
# r0_container.trigger_type = self.data.lst.tel[self.tel_id].evt.ucts_trigger_type
# consider for the moment only TIB trigger since UCTS seems not correct
if self.data.lst.tel[self.tel_id].evt.tib_masked_trigger > 0:
r0_container.trigger_type = self.data.lst.tel[self.tel_id].evt.tib_masked_trigger
else:
r0_container.trigger_type = -1
# verify the number of gains
if event.waveform.shape[0] != self.camera_config.num_pixels * self.camera_config.num_samples * self.n_gains:
raise ValueError(f"Number of gains not correct, waveform shape is {event.waveform.shape[0]}"
f" instead of "
f"{self.camera_config.num_pixels * self.camera_config.num_samples * self.n_gains}")
reshaped_waveform = np.array(
event.waveform
).reshape(
self.n_gains,
self.camera_config.num_pixels,
self.camera_config.num_samples
)
# initialize the waveform container to zero
r0_container.waveform = np.zeros([self.n_gains, self.n_camera_pixels,
self.camera_config.num_samples])
# re-order the waveform following the expected_pixels_id values
# (rank = pixel id)
r0_container.waveform[:, self.camera_config.expected_pixels_id, :] =\
reshaped_waveform
def fill_r0_container_from_zfile(self, event):
"""
Fill with R0Container
"""
container = self.data.r0
container.tels_with_data = [self.tel_id, ]
r0_camera_container = container.tel[self.tel_id]
self.fill_r0_camera_container_from_zfile(
r0_camera_container,
event
)
def initialize_mon_container(self):
"""
Fill with MonitoringContainer.
For the moment, initialize only the PixelStatusContainer
"""
container = self.data.mon
container.tels_with_data = [self.tel_id, ]
mon_camera_container = container.tel[self.tel_id]
# initialize the container
status_container = PixelStatusContainer()
status_container.hardware_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)
status_container.pedestal_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)
status_container.flatfield_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)
mon_camera_container.pixel_status = status_container
def fill_mon_container_from_zfile(self, event):
"""
Fill with MonitoringContainer.
For the moment, initialize only the PixelStatusContainer
"""
status_container = self.data.mon.tel[self.tel_id].pixel_status
# reorder the array
pixel_status = np.zeros(self.n_camera_pixels)
pixel_status[self.camera_config.expected_pixels_id] = event.pixel_status
status_container.hardware_failing_pixels[:] = pixel_status == 0
class SimpleEventWriter(Tool):
name = "ctapipe-simple-event-writer"
description = Unicode(__doc__)
infile = Path(
default_value=get_dataset_path(
"lst_prod3_calibration_and_mcphotons.simtel.zst"),
help="input file to read",
directory_ok=False,
exists=True,
).tag(config=True)
outfile = Path(help="output file name",
directory_ok=False,
default_value="output.h5").tag(config=True)
progress = Bool(help="display progress bar",
default_value=True).tag(config=True)
aliases = Dict({
"infile": "EventSource.input_url",
"outfile": "SimpleEventWriter.outfile",
"max-events": "EventSource.max_events",
"progress": "SimpleEventWriter.progress",
})
classes = List([EventSource, CameraCalibrator])
def setup(self):
self.log.info("Configure EventSource...")
self.event_source = EventSource.from_url(self.infile, parent=self)
self.calibrator = CameraCalibrator(subarray=self.event_source.subarray,
parent=self)
self.writer = HDF5TableWriter(filename=self.outfile,
group_name="image_infos",
overwrite=True,
parent=self)
def start(self):
self.log.info("Loop on events...")
for event in tqdm(
self.event_source,
desc="EventWriter",
total=self.event_source.max_events,
disable=~self.progress,
):
self.calibrator(event)
for tel_id in event.dl0.tel.keys():
geom = self.event_source.subarray.tel[tel_id].camera.geometry
dl1_tel = event.dl1.tel[tel_id]
# Image cleaning
image = dl1_tel.image # Waiting for automatic gain selection
mask = tailcuts_clean(geom,
image,
picture_thresh=10,
boundary_thresh=5)
cleaned = image.copy()
cleaned[~mask] = 0
# Image parametrisation
params = hillas_parameters(geom, cleaned)
# Save Ids, MC infos and Hillas informations
self.writer.write(geom.camera_name,
[event.r0, event.simulation.shower, params])
def finish(self):
self.log.info("End of job.")
self.writer.close()
class LSTR0Corrections(TelescopeComponent):
"""
The base R0-level calibrator. Changes the r0 container.
The R0 calibrator performs the camera-specific R0 calibration that is
usually performed on the raw data by the camera server.
This calibrator exists in lstchain for testing and prototyping purposes.
"""
offset = IntTelescopeParameter(
default_value=0,
help=(
'Define offset to be subtracted from the waveform *additionally*'
' to the drs4 pedestal offset. This only needs to be given when'
' the drs4 pedestal calibration is not applied or the offset of the'
' drs4 run is different from the data run'
)
).tag(config=True)
r1_sample_start = IntTelescopeParameter(
default_value=3,
help='Start sample for r1 waveform',
allow_none=True,
).tag(config=True)
r1_sample_end = IntTelescopeParameter(
default_value=39,
help='End sample for r1 waveform',
allow_none=True,
).tag(config=True)
drs4_pedestal_path = TelescopeParameter(
trait=Path(exists=True, directory_ok=False),
allow_none=True,
default_value=None,
help=(
'Path to the LST pedestal file'
', required when `apply_drs4_pedestal_correction=True`'
),
).tag(config=True)
calibration_path = Path(
exists=True, directory_ok=False,
help='Path to LST calibration file',
).tag(config=True)
drs4_time_calibration_path = TelescopeParameter(
trait=Path(exists=True, directory_ok=False),
help='Path to the time calibration file',
default_value=None,
allow_none=True,
).tag(config=True)
calib_scale_high_gain = FloatTelescopeParameter(
default_value=1.0,
help='High gain waveform is multiplied by this number'
).tag(config=True)
calib_scale_low_gain = FloatTelescopeParameter(
default_value=1.0,
help='Low gain waveform is multiplied by this number'
).tag(config=True)
select_gain = Bool(
default_value=True,
help='Set to False to keep both gains.'
).tag(config=True)
apply_drs4_pedestal_correction = Bool(
default_value=True,
help=(
'Set to False to disable drs4 pedestal correction.'
' Providing the drs4_pedestal_path is required to perform this calibration'
),
).tag(config=True)
apply_timelapse_correction = Bool(
default_value=True,
help='Set to False to disable drs4 timelapse correction'
).tag(config=True)
apply_spike_correction = Bool(
default_value=True,
help='Set to False to disable drs4 spike correction'
).tag(config=True)
add_calibration_timeshift = Bool(
default_value=True,
help=(
'If true, time correction from the calibration'
' file is added to calibration.dl1.time'
),
).tag(config=True)
gain_selection_threshold = Float(
default_value=3500,
help='Threshold for the ThresholdGainSelector.'
).tag(config=True)
def __init__(self, subarray, config=None, parent=None, **kwargs):
"""
The R0 calibrator for LST data.
Fill the r1 container.
Parameters
----------
"""
super().__init__(
subarray=subarray, config=config, parent=parent, **kwargs
)
self.mon_data = None
self.last_readout_time = {}
self.first_cap = {}
self.first_cap_old = {}
self.fbn = {}
self.fan = {}
for tel_id in self.subarray.tel:
shape = (N_GAINS, N_PIXELS, N_CAPACITORS_PIXEL)
self.last_readout_time[tel_id] = np.zeros(shape, dtype='uint64')
shape = (N_GAINS, N_PIXELS)
self.first_cap[tel_id] = np.zeros(shape, dtype=int)
self.first_cap_old[tel_id] = np.zeros(shape, dtype=int)
if self.select_gain:
self.gain_selector = ThresholdGainSelector(
threshold=self.gain_selection_threshold,
parent=self
)
else:
self.gain_selector = None
if self.calibration_path is not None:
self.mon_data = self._read_calibration_file(self.calibration_path)
def apply_drs4_corrections(self, event: LSTArrayEventContainer):
self.update_first_capacitors(event)
for tel_id, r0 in event.r0.tel.items():
r1 = event.r1.tel[tel_id]
# If r1 was not yet filled, copy of r0 converted
if r1.waveform is None:
r1.waveform = r0.waveform
# float32 can represent all values of uint16 exactly,
# so this does not loose precision.
r1.waveform = r1.waveform.astype(np.float32, copy=False)
# apply drs4 corrections
if self.apply_drs4_pedestal_correction:
self.subtract_pedestal(event, tel_id)
if self.apply_timelapse_correction:
self.time_lapse_corr(event, tel_id)
if self.apply_spike_correction:
self.interpolate_spikes(event, tel_id)
# remove samples at beginning / end of waveform
start = self.r1_sample_start.tel[tel_id]
end = self.r1_sample_end.tel[tel_id]
r1.waveform = r1.waveform[..., start:end]
if self.offset.tel[tel_id] != 0:
r1.waveform -= self.offset.tel[tel_id]
mon = event.mon.tel[tel_id]
if r1.selected_gain_channel is None:
r1.waveform[mon.pixel_status.hardware_failing_pixels] = 0.0
else:
broken = mon.pixel_status.hardware_failing_pixels[r1.selected_gain_channel, PIXEL_INDEX]
r1.waveform[broken] = 0.0
def update_first_capacitors(self, event: LSTArrayEventContainer):
for tel_id, lst in event.lst.tel.items():
self.first_cap_old[tel_id] = self.first_cap[tel_id]
self.first_cap[tel_id] = get_first_capacitors_for_pixels(
lst.evt.first_capacitor_id,
lst.svc.pixel_ids,
)
def calibrate(self, event: LSTArrayEventContainer):
for tel_id in event.r0.tel:
r1 = event.r1.tel[tel_id]
# if `apply_drs4_corrections` is False, we did not fill in the
# waveform yet.
if r1.waveform is None:
r1.waveform = event.r0.tel[tel_id].waveform
r1.waveform = r1.waveform.astype(np.float32, copy=False)
# do gain selection before converting to pe
# like eventbuilder will do
if self.select_gain and r1.selected_gain_channel is None:
r1.selected_gain_channel = self.gain_selector(r1.waveform)
r1.waveform = r1.waveform[r1.selected_gain_channel, PIXEL_INDEX]
# apply monitoring data corrections,
# subtract pedestal and convert to pe
if self.mon_data is not None:
calibration = self.mon_data.tel[tel_id].calibration
convert_to_pe(
waveform=r1.waveform,
calibration=calibration,
selected_gain_channel=r1.selected_gain_channel
)
broken_pixels = event.mon.tel[tel_id].pixel_status.hardware_failing_pixels
if r1.selected_gain_channel is None:
r1.waveform[broken_pixels] = 0.0
else:
r1.waveform[broken_pixels[r1.selected_gain_channel, PIXEL_INDEX]] = 0.0
# store calibration data needed for dl1 calibration in ctapipe
# first drs4 time shift (zeros if no calib file was given)
time_shift = self.get_drs4_time_correction(
tel_id, self.first_cap[tel_id],
selected_gain_channel=r1.selected_gain_channel,
)
# time shift from flat fielding
if self.mon_data is not None and self.add_calibration_timeshift:
time_corr = self.mon_data.tel[tel_id].calibration.time_correction
# time_shift is subtracted in ctapipe,
# but time_correction should be added
if r1.selected_gain_channel is not None:
time_shift -= time_corr[r1.selected_gain_channel, PIXEL_INDEX].to_value(u.ns)
else:
time_shift -= time_corr.to_value(u.ns)
event.calibration.tel[tel_id].dl1.time_shift = time_shift
# needed for charge scaling in ctpaipe dl1 calib
if r1.selected_gain_channel is not None:
relative_factor = np.empty(N_PIXELS)
relative_factor[r1.selected_gain_channel == HIGH_GAIN] = self.calib_scale_high_gain.tel[tel_id]
relative_factor[r1.selected_gain_channel == LOW_GAIN] = self.calib_scale_low_gain.tel[tel_id]
else:
relative_factor = np.empty((N_GAINS, N_PIXELS))
relative_factor[HIGH_GAIN] = self.calib_scale_high_gain.tel[tel_id]
relative_factor[LOW_GAIN] = self.calib_scale_low_gain.tel[tel_id]
event.calibration.tel[tel_id].dl1.relative_factor = relative_factor
@staticmethod
def _read_calibration_file(path):
"""
Read the correction from hdf5 calibration file
"""
mon = MonitoringContainer()
with tables.open_file(path) as f:
tel_ids = [
int(key[4:]) for key in f.root._v_children.keys()
if key.startswith('tel_')
]
for tel_id in tel_ids:
with HDF5TableReader(path) as h5_table:
base = f'/tel_{tel_id}'
# read the calibration data
table = base + '/calibration'
next(h5_table.read(table, mon.tel[tel_id].calibration))
# read pedestal data
table = base + '/pedestal'
next(h5_table.read(table, mon.tel[tel_id].pedestal))
# read flat-field data
table = base + '/flatfield'
next(h5_table.read(table, mon.tel[tel_id].flatfield))
# read the pixel_status container
table = base + '/pixel_status'
next(h5_table.read(table, mon.tel[tel_id].pixel_status))
return mon
@staticmethod
def load_drs4_time_calibration_file(path):
"""
Function to load calibration file.
"""
with tables.open_file(path, 'r') as f:
fan = f.root.fan[:]
fbn = f.root.fbn[:]
return fan, fbn
def load_drs4_time_calibration_file_for_tel(self, tel_id):
self.fan[tel_id], self.fbn[tel_id] = self.load_drs4_time_calibration_file(
self.drs4_time_calibration_path.tel[tel_id]
)
def get_drs4_time_correction(self, tel_id, first_capacitors, selected_gain_channel=None):
"""
Return pulse time after time correction.
"""
if self.drs4_time_calibration_path.tel[tel_id] is None:
if selected_gain_channel is None:
return np.zeros((N_GAINS, N_PIXELS))
else:
return np.zeros(N_PIXELS)
# load calib file if not already done
if tel_id not in self.fan:
self.load_drs4_time_calibration_file_for_tel(tel_id)
if selected_gain_channel is not None:
return calc_drs4_time_correction_gain_selected(
first_capacitors,
selected_gain_channel,
self.fan[tel_id],
self.fbn[tel_id],
)
else:
return calc_drs4_time_correction_both_gains(
first_capacitors,
self.fan[tel_id],
self.fbn[tel_id],
)
@staticmethod
@lru_cache(maxsize=4)
def _get_drs4_pedestal_data(path):
"""
Function to load pedestal file.
To make boundary conditions unnecessary,
the first N_SAMPLES values are repeated at the end of the array
The result is cached so we can repeatedly call this method
using the configured path without reading it each time.
"""
if path is None:
raise ValueError(
"DRS4 pedestal correction requested"
" but no file provided for telescope"
)
pedestal_data = np.empty(
(N_GAINS, N_PIXELS_MODULE * N_MODULES, N_CAPACITORS_PIXEL + N_SAMPLES),
dtype=np.int16
)
with fits.open(path) as f:
pedestal_data[:, :, :N_CAPACITORS_PIXEL] = f[1].data
pedestal_data[:, :, N_CAPACITORS_PIXEL:] = pedestal_data[:, :, :N_SAMPLES]
return pedestal_data
def subtract_pedestal(self, event, tel_id):
"""
Subtract cell offset using pedestal file.
Fill the R1 container.
Parameters
----------
event : `ctapipe` event-container
tel_id : id of the telescope
"""
pedestal = self._get_drs4_pedestal_data(
self.drs4_pedestal_path.tel[tel_id]
)
if event.r1.tel[tel_id].selected_gain_channel is None:
subtract_pedestal(
event.r1.tel[tel_id].waveform,
self.first_cap[tel_id],
pedestal,
)
else:
subtract_pedestal_gain_selected(
event.r1.tel[tel_id].waveform,
self.first_cap[tel_id],
pedestal,
event.r1.tel[tel_id].selected_gain_channel,
)
def time_lapse_corr(self, event, tel_id):
"""
Perform time lapse baseline corrections.
Fill the R1 container or modifies R0 container.
Parameters
----------
event : `ctapipe` event-container
tel_id : id of the telescope
"""
lst = event.lst.tel[tel_id]
# If R1 container exists, update it inplace
if isinstance(event.r1.tel[tel_id].waveform, np.ndarray):
container = event.r1.tel[tel_id]
else:
# Modify R0 container. This is to create pedestal files.
container = event.r0.tel[tel_id]
waveform = container.waveform.copy()
# We have 2 functions: one for data from 2018/10/10 to 2019/11/04 and
# one for data from 2019/11/05 (from Run 1574) after update firmware.
# The old readout (before 2019/11/05) is shifted by 1 cell.
run_id = event.lst.tel[tel_id].svc.configuration_id
# not yet gain selected
if event.r1.tel[tel_id].selected_gain_channel is None:
apply_timelapse_correction(
waveform=waveform,
local_clock_counter=lst.evt.local_clock_counter,
first_capacitors=self.first_cap[tel_id],
last_readout_time=self.last_readout_time[tel_id],
expected_pixels_id=lst.svc.pixel_ids,
run_id=run_id,
)
else:
apply_timelapse_correction_gain_selected(
waveform=waveform,
local_clock_counter=lst.evt.local_clock_counter,
first_capacitors=self.first_cap[tel_id],
last_readout_time=self.last_readout_time[tel_id],
expected_pixels_id=lst.svc.pixel_ids,
selected_gain_channel=event.r1.tel[tel_id].selected_gain_channel,
run_id=run_id,
)
container.waveform = waveform
def interpolate_spikes(self, event, tel_id):
"""
Interpolates spike A & B.
Fill the R1 container.
Parameters
----------
event : `ctapipe` event-container
tel_id : id of the telescope
"""
run_id = event.lst.tel[tel_id].svc.configuration_id
r1 = event.r1.tel[tel_id]
if r1.selected_gain_channel is None:
interpolate_spikes(
waveform=r1.waveform,
first_capacitors=self.first_cap[tel_id],
previous_first_capacitors=self.first_cap_old[tel_id],
run_id=run_id,
)
else:
interpolate_spikes_gain_selected(
waveform=r1.waveform,
first_capacitors=self.first_cap[tel_id],
previous_first_capacitors=self.first_cap_old[tel_id],
selected_gain_channel=r1.selected_gain_channel,
run_id=run_id,
)
class DumpTriggersTool(Tool):
description = Unicode(__doc__)
# =============================================
# configuration parameters:
# =============================================
infile = Unicode(help='input simtelarray file').tag(config=True,
allow_none=False)
outfile = Unicode('triggers.fits',
help='output filename (*.fits, *.h5)').tag(config=True)
overwrite = Bool(False,
help="overwrite existing output file").tag(config=True)
# =============================================
# map low-level options to high-level command-line options
# =============================================
aliases = Dict({
'infile': 'DumpTriggersTool.infile',
'outfile': 'DumpTriggersTool.outfile'
})
flags = Dict({
'overwrite': ({
'DumpTriggersTool': {
'overwrite': True
}
}, 'Enable overwriting of output file')
})
examples = ('ctapipe-dump-triggers --infile gamma.simtel.gz '
'--outfile trig.fits --overwrite'
'\n\n'
'If you want to see more output, use --log_level=DEBUG')
# =============================================
# The methods of the Tool (initialize, start, finish):
# =============================================
def add_event_to_table(self, event_id):
"""
add the current pyhessio event to a row in the `self.events` table
"""
ts, tns = pyhessio.get_central_event_gps_time()
gpstime = Time(ts * u.s, tns * u.ns, format='gps', scale='utc')
if self._prev_gpstime is None:
self._prev_gpstime = gpstime
if self._current_starttime is None:
self._current_starttime = gpstime
relative_time = gpstime - self._current_starttime
delta_t = gpstime - self._prev_gpstime
self._prev_gpstime = gpstime
# build the trigger pattern as a fixed-length array
# (better for storage in FITS format)
trigtels = pyhessio.get_telescope_with_data_list()
self._current_trigpattern[:] = 0 # zero the trigger pattern
self._current_trigpattern[trigtels] = 1 # set the triggered tels to 1
# insert the row into the table
self.events.add_row((event_id, relative_time.sec, delta_t.sec,
len(trigtels), self._current_trigpattern))
def setup(self):
""" setup function, called before `start()` """
if self.infile == '':
raise ValueError("No 'infile' parameter was specified. "
"Use --help for info")
self.events = Table(
names=['EVENT_ID', 'T_REL', 'DELTA_T', 'N_TRIG', 'TRIGGERED_TELS'],
dtype=[np.int64, np.float64, np.float64, np.int32, np.uint8])
self.events['TRIGGERED_TELS'].shape = (0, MAX_TELS)
self.events['T_REL'].unit = u.s
self.events['T_REL'].description = 'Time relative to first event'
self.events['DELTA_T'].unit = u.s
self.events.meta['INPUT'] = self.infile
self._current_trigpattern = np.zeros(MAX_TELS)
self._current_starttime = None
self._prev_gpstime = None
pyhessio.file_open(self.infile)
def start(self):
""" main event loop """
for run_id, event_id in pyhessio.move_to_next_event():
self.add_event_to_table(event_id)
def finish(self):
"""
finish up and write out results (called automatically after
`start()`)
"""
pyhessio.close_file()
# write out the final table
if self.outfile.endswith('fits') or self.outfile.endswith('fits.gz'):
self.events.write(self.outfile, overwrite=self.overwrite)
elif self.outfile.endswith('h5'):
self.events.write(self.outfile,
path='/events',
overwrite=self.overwrite)
else:
self.events.write(self.outfile)
self.log.info("Table written to '{}'".format(self.outfile))
self.log.info('\n %s', self.events)
class LSTEventSource(EventSource):
"""EventSource for LST r0 data."""
n_gains = Int(
2,
help='Number of gains at r0/r1 level'
).tag(config=True)
baseline = Int(
400,
help='r0 waveform baseline (default from EvB v3)'
).tag(config=True)
multi_streams = Bool(
True,
help='Read in parallel all streams '
).tag(config=True)
def __init__(self, **kwargs):
"""
Constructor
Parameters
----------
n_gains = number of gains expected in input file
baseline = baseline to be subtracted at r1 level (not used for the moment)
multi_streams = enable the reading of input files from all streams
config: traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.\
kwargs: dict
Additional parameters to be passed.
NOTE: The file mask of the data to read can be passed with
the 'input_url' parameter.
"""
# EventSource can not handle file wild cards as input_url
# To overcome this we substitute the input_url with first file matching
# the specified file mask (copied from MAGICEventSourceROOT).
super().__init__(**kwargs)
if self.multi_streams:
# test how many streams are there:
# file name must be [stream name]Run[all the rest]
# All the files with the same [all the rest] are opened
if '/' in self.input_url:
dir, name = self.input_url.rsplit('/', 1)
else:
dir = getcwd()
name = self.input_url
if 'Run' in name:
stream, run = name.split('Run', 1)
else:
run = name
ls = listdir(dir)
self.file_list = []
for file_name in ls:
if run in file_name:
full_name = dir + '/' + file_name
self.file_list.append(full_name)
Provenance().add_input_file(full_name, role='dl0.sub.evt')
else:
self.file_list = [self.input_url]
self.multi_file = MultiFiles(self.file_list)
self.camera_config = self.multi_file.camera_config
self.log.info(
"Read {} input files".format(
self.multi_file.num_inputs()
)
)
def rewind(self):
self.multi_file.rewind()
def _generator(self):
# container for LST data
self.data = LSTDataContainer()
self.data.meta['input_url'] = self.input_url
self.data.meta['max_events'] = self.max_events
self.data.meta['origin'] = 'LSTCAM'
# fill LST data from the CameraConfig table
self.fill_lst_service_container_from_zfile()
# Instrument information
for tel_id in self.data.lst.tels_with_data:
assert (tel_id == 0 or tel_id == 1) # only LST1 (for the moment id = 0)
# optics info from standard optics.fits.gz file
optics = OpticsDescription.from_name("LST")
# camera info from LSTCam-[geometry_version].camgeom.fits.gz file
geometry_version = 2
camera = CameraGeometry.from_name("LSTCam", geometry_version)
tel_descr = TelescopeDescription(
name='LST', tel_type='LST', optics=optics, camera=camera
)
self.n_camera_pixels = tel_descr.camera.n_pixels
tels = {tel_id: tel_descr}
# LSTs telescope position taken from MC from the moment
tel_pos = {tel_id: [50., 50., 16] * u.m}
subarray = SubarrayDescription("LST1 subarray")
subarray.tels = tels
subarray.positions = tel_pos
self.data.inst.subarray = subarray
# initialize general monitoring container
self.initialize_mon_container()
# loop on events
for count, event in enumerate(self.multi_file):
self.data.count = count
# fill specific LST event data
self.fill_lst_event_container_from_zfile(event)
# fill general monitoring data
self.fill_mon_container_from_zfile(event)
# fill general R0 data
self.fill_r0_container_from_zfile(event)
yield self.data
@staticmethod
def is_compatible(file_path):
from astropy.io import fits
try:
# The file contains two tables:
# 1: CameraConfig
# 2: Events
h = fits.open(file_path)[2].header
ttypes = [
h[x] for x in h.keys() if 'TTYPE' in x
]
except OSError:
# not even a fits file
return False
except IndexError:
# A fits file of a different format
return False
is_protobuf_zfits_file = (
(h['XTENSION'] == 'BINTABLE') and
(h['EXTNAME'] == 'Events') and
(h['ZTABLE'] is True) and
(h['ORIGIN'] == 'CTA') and
(h['PBFHEAD'] == 'R1.CameraEvent')
)
is_lst_file = 'lstcam_counters' in ttypes
return is_protobuf_zfits_file & is_lst_file
def fill_lst_service_container_from_zfile(self):
"""
Fill LSTServiceContainer with specific LST service data data
(from the CameraConfig table of zfit file)
"""
self.data.lst.tels_with_data = [self.camera_config.telescope_id, ]
svc_container = self.data.lst.tel[self.camera_config.telescope_id].svc
svc_container.telescope_id = self.camera_config.telescope_id
svc_container.cs_serial = self.camera_config.cs_serial
svc_container.configuration_id = self.camera_config.configuration_id
svc_container.date = self.camera_config.date
svc_container.num_pixels = self.camera_config.num_pixels
svc_container.num_samples = self.camera_config.num_samples
svc_container.pixel_ids = self.camera_config.expected_pixels_id
svc_container.data_model_version = self.camera_config.data_model_version
svc_container.num_modules = self.camera_config.lstcam.num_modules
svc_container.module_ids = self.camera_config.lstcam.expected_modules_id
svc_container.idaq_version = self.camera_config.lstcam.idaq_version
svc_container.cdhs_version = self.camera_config.lstcam.cdhs_version
svc_container.algorithms = self.camera_config.lstcam.algorithms
svc_container.pre_proc_algorithms = self.camera_config.lstcam.pre_proc_algorithms
def fill_lst_event_container_from_zfile(self, event):
"""
Fill LSTEventContainer with specific LST service data
(from the Event table of zfit file)
"""
event_container = self.data.lst.tel[self.camera_config.telescope_id].evt
event_container.configuration_id = event.configuration_id
event_container.event_id = event.event_id
event_container.tel_event_id = event.tel_event_id
event_container.pixel_status = event.pixel_status
event_container.ped_id = event.ped_id
event_container.module_status = event.lstcam.module_status
event_container.extdevices_presence = event.lstcam.extdevices_presence
# unpack TIB data
rec_fmt = '=IHIBB'
unpacked_tib = struct.unpack(rec_fmt, event.lstcam.tib_data)
event_container.tib_event_counter = unpacked_tib[0]
event_container.tib_pps_counter = unpacked_tib[1]
event_container.tib_tenMHz_counter = unpacked_tib[2]
event_container.tib_stereo_pattern = unpacked_tib[3]
event_container.tib_masked_trigger = unpacked_tib[4]
event_container.swat_data = event.lstcam.swat_data
# unpack CDTS data
rec_fmt = '=IIIQQBBB'
unpacked_cdts = struct.unpack(rec_fmt, event.lstcam.cdts_data)
event_container.ucts_event_counter = unpacked_cdts[0]
event_container.ucts_pps_counter = unpacked_cdts[1]
event_container.ucts_clock_counter = unpacked_cdts[2]
event_container.ucts_timestamp = unpacked_cdts[3]
event_container.ucts_camera_timestamp = unpacked_cdts[4]
event_container.ucts_trigger_type = unpacked_cdts[5]
event_container.ucts_white_rabbit_status = unpacked_cdts[6]
# unpack Dragon counters
rec_fmt = '=HIIIQ'
rec_len = struct.calcsize(rec_fmt)
rec_unpack = struct.Struct(rec_fmt).unpack_from
event_container.pps_counter = np.zeros(self.camera_config.lstcam.num_modules)
event_container.tenMHz_counter = np.zeros(self.camera_config.lstcam.num_modules)
event_container.event_counter = np.zeros(self.camera_config.lstcam.num_modules)
event_container.trigger_counter = np.zeros(self.camera_config.lstcam.num_modules)
event_container.local_clock_counter = np.zeros(self.camera_config.lstcam.num_modules)
for mod in range(self.camera_config.lstcam.num_modules):
words=event.lstcam.counters[mod*rec_len:(mod+1)*rec_len]
unpacked_counter = rec_unpack(words)
event_container.pps_counter[mod] = unpacked_counter[0]
event_container.tenMHz_counter[mod] = unpacked_counter[1]
event_container.event_counter[mod] = unpacked_counter[2]
event_container.trigger_counter[mod] = unpacked_counter[3]
event_container.local_clock_counter[mod] = unpacked_counter[4]
event_container.chips_flags = event.lstcam.chips_flags
event_container.first_capacitor_id = event.lstcam.first_capacitor_id
event_container.drs_tag_status = event.lstcam.drs_tag_status
event_container.drs_tag = event.lstcam.drs_tag
def fill_r0_camera_container_from_zfile(self, r0_container, event):
"""
Fill with R0CameraContainer
"""
# temporary patch to have an event time set
r0_container.trigger_time = (
self.data.lst.tel[self.camera_config.telescope_id].evt.tib_pps_counter +
self.data.lst.tel[self.camera_config.telescope_id].evt.tib_tenMHz_counter * 10**(-7))
if r0_container.trigger_time is None:
r0_container.trigger_time = 0
#r0_container.trigger_type = event.trigger_type
r0_container.trigger_type = self.data.lst.tel[self.camera_config.telescope_id].evt.tib_masked_trigger
# verify the number of gains
if event.waveform.shape[0] != self.camera_config.num_pixels * self.camera_config.num_samples * self.n_gains:
raise ValueError(f"Number of gains not correct, waveform shape is {event.waveform.shape[0]}"
f" instead of "
f"{self.camera_config.num_pixels * self.camera_config.num_samples * self.n_gains}")
reshaped_waveform = np.array(
event.waveform
).reshape(
self.n_gains,
self.camera_config.num_pixels,
self.camera_config.num_samples
)
# initialize the waveform container to zero
r0_container.waveform = np.zeros([self.n_gains, self.n_camera_pixels,
self.camera_config.num_samples])
# re-order the waveform following the expected_pixels_id values
# (rank = pixel id)
r0_container.waveform[:, self.camera_config.expected_pixels_id, :] =\
reshaped_waveform
def fill_r0_container_from_zfile(self, event):
"""
Fill with R0Container
"""
container = self.data.r0
container.obs_id = -1
container.event_id = event.event_id
container.tels_with_data = [self.camera_config.telescope_id, ]
r0_camera_container = container.tel[self.camera_config.telescope_id]
self.fill_r0_camera_container_from_zfile(
r0_camera_container,
event
)
def initialize_mon_container(self):
"""
Fill with MonitoringContainer.
For the moment, initialize only the PixelStatusContainer
"""
container = self.data.mon
container.tels_with_data = [self.camera_config.telescope_id, ]
mon_camera_container = container.tel[self.camera_config.telescope_id]
# initialize the container
status_container = PixelStatusContainer()
status_container.hardware_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)
status_container.pedestal_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)
status_container.flatfield_failing_pixels = np.zeros((self.n_gains, self.n_camera_pixels), dtype=bool)
mon_camera_container.pixel_status = status_container
def fill_mon_container_from_zfile(self, event):
"""
Fill with MonitoringContainer.
For the moment, initialize only the PixelStatusContainer
"""
status_container = self.data.mon.tel[self.camera_config.telescope_id].pixel_status
# reorder the array
pixel_status = np.zeros(self.n_camera_pixels)
pixel_status[self.camera_config.expected_pixels_id] = event.pixel_status
status_container.hardware_failing_pixels[:] = pixel_status == 0
class ChargeResolutionCalculator(Component):
"""
Class to handle the calculation of Charge Resolution.
Attributes
----------
max_pe : int
Maximum pe to calculate the charge resolution up to.
sum_dict : dict
Dictionary to store the running sum for each true charge.
n_dict : dict
Dictionary to store the running number for each true charge.
variation_hist_nbins : float
Number of bins for the variation histogram.
variation_hist_range : list
X and Y range for the variation histogram.
variation_hist : `np.histogram2d`
variation_xedges : ndarray
Edges of the X bins for the variation histogram.
variation_yedges : ndarray
Edges of the Y bins for the variation histogram.
"""
max_pe = Int(2000,
help='Maximum pe to calculate the charge resolution '
'up to').tag(config=True)
binning = Int(60,
allow_none=True,
help='Number of bins for the Charge Resolution. If None, '
'no binning is performed.').tag(config=True)
log_bins = Bool(True,
help='Bin the x axis linearly instead of '
'logarithmic.').tag(config=True)
def __init__(self, config=None, tool=None, **kwargs):
"""
Calculator of charge resolution.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
super().__init__(config=config, parent=tool, **kwargs)
self.sum_array = np.zeros(self.max_pe)
self.n_array = np.zeros(self.max_pe)
self.sum_dict = {}
self.n_dict = {}
self.variation_hist_nbins = log10(self.max_pe) * 50
self.variation_hist_range = [[log10(1), log10(self.max_pe)],
[log10(1), log10(self.max_pe)]]
h, xedges, yedges = np.histogram2d([np.nan], [np.nan],
bins=self.variation_hist_nbins,
range=self.variation_hist_range)
self.variation_hist = h
self.variation_xedges = xedges
self.variation_yedges = yedges
self.storage_arrays = [
'max_pe', 'sum_array', 'n_array', 'variation_hist_nbins',
'variation_hist_range', 'variation_hist', 'variation_xedges',
'variation_yedges'
]
def add_charges(self, true_charge, measured_charge):
"""
Fill the class parameters with a numpy array of true charge and
measured (calibrated) charge from an event. The two arrays must be the
same size.
Parameters
----------
true_charge : ndarray
Array of true (MC) charge.
Obtained from event.mc.tel[telid].image[channel]
measured_charge : ndarray
Array of measured (dl1 calibrated) charge.
Obtained from event.mc.tel[tel_id].photo_electron_image
"""
above_0 = (measured_charge > 0) & (true_charge > 0)
x = true_charge[above_0]
y = measured_charge[above_0]
h, _, _ = np.histogram2d(np.log10(y),
np.log10(x),
bins=self.variation_hist_nbins,
range=self.variation_hist_range)
self.variation_hist += h
in_range = (true_charge > 0) & (true_charge <= self.max_pe)
true_q = true_charge[in_range]
measured_q = measured_charge[in_range]
np.add.at(self.sum_array, true_q - 1, np.power(measured_q - true_q, 2))
np.add.at(self.n_array, true_q - 1, 1)
def get_charge_resolution(self):
"""
Calculate and obtain the charge resolution graph arrays.
Returns
-------
true_charge : ndarray
The X axis true charges.
chargeres : ndarray
The Y axis charge resolution values.
chargeres_error : ndarray
The error on the charge resolution.
scaled_chargeres : ndarray
The Y axis charge resolution divided by the Goal.
scaled_chargeres_error : ndarray
The error on the charge resolution divided by the Goal.
"""
self.log.debug('[chargeres] Calculating charge resolution')
n_1 = self.n_array > 0
n = self.n_array[n_1]
true = (np.arange(self.max_pe) + 1)[n_1]
sum_ = self.sum_array[n_1]
res = np.sqrt((sum_ / n) + true) / true
res_error = res * (1 / np.sqrt(2 * n))
scale = self.goal(true)
scaled_res = res / scale
scaled_res_error = res_error / scale
if self.binning is not None:
x = true
if self.log_bins:
x = np.log10(true)
def binning(array):
return bs(x, array, 'mean', bins=self.binning)
def sum_errors(array):
return np.sqrt(np.sum(np.power(array, 2))) / array.size
def bin_errors(array):
return bs(x, array, sum_errors, bins=self.binning)
true, _, _ = binning(true)
res, _, _ = binning(res)
res_error, _, _ = bin_errors(res_error)
scaled_res, _, _ = binning(scaled_res)
scaled_res_error, _, _ = bin_errors(scaled_res_error)
return true, res, res_error, scaled_res, scaled_res_error
@staticmethod
def limit_curves(npe, n_nsb, n_add, enf, sigma2):
"""
Equation for calculating the Goal and Requirement curves, as defined
in SCI-MC/121113.
https://portal.cta-observatory.org/recordscentre/Records/SCI/
SCI-MC/measurment_errors_system_performance_1YQCBC.pdf
Parameters
----------
npe : ndarray
Number of photoeletrons (variable).
n_nsb : float
Number of NSB photons.
n_add : float
Number of photoelectrons from additional noise sources.
enf : float
Excess noise factor.
sigma2 : float
Percentage ofmultiplicative errors.
"""
return (np.sqrt((n_nsb + n_add) + np.power(enf, 2) * npe +
np.power(sigma2 * npe, 2)) / npe).astype(float)
@staticmethod
def requirement(npe):
"""
CTA requirement curve.
Parameters
----------
npe : ndarray
Number of photoeletrons (variable).
"""
n_nsb = sqrt(4.0 + 3.0)
n_add = 0
enf = 1.2
sigma2 = 0.1
defined_npe = 1000
# If npe is not an array, temporarily convert it to one
npe = np.array([npe])
lc = ChargeResolutionCalculator.limit_curves
requirement = lc(npe, n_nsb, n_add, enf, sigma2)
requirement[npe > defined_npe] = np.nan
return requirement[0]
@staticmethod
def goal(npe):
"""
CTA goal curve.
Parameters
----------
npe : ndarray
Number of photoeletrons (variable).
"""
n_nsb = 2
n_add = 0
enf = 1.1152
sigma2 = 0.05
defined_npe = 2000
# If npe is not an array, temporarily convert it to one
npe = np.array([npe])
lc = ChargeResolutionCalculator.limit_curves
goal = lc(npe, n_nsb, n_add, enf, sigma2)
goal[npe > defined_npe] = np.nan
return goal[0]
@staticmethod
def poisson(npe):
"""
Poisson limit curve.
Parameters
----------
npe : ndarray
Number of photoeletrons (variable).
"""
# If npe is not an array, temporarily convert it to one
npe = np.array([npe])
poisson = np.sqrt(npe) / npe
return poisson[0]
def save(self, path):
output_dir = dirname(path)
if not exists(output_dir):
self.log.info("[output] Creating directory: {}".format(output_dir))
makedirs(output_dir)
self.log.info("Saving Charge Resolution file: {}".format(path))
with open_file(path, mode="w", title="ChargeResolutionFile") as f:
group = f.create_group("/", 'ChargeResolution', '')
for arr in self.storage_arrays:
f.create_array(group, arr, getattr(self, arr), arr)
def load(self, path):
self.log.info("Loading Charge Resolution file: {}".format(path))
with open_file(path, mode="r") as f:
for arr in self.storage_arrays:
setattr(self, arr, f.get_node("/ChargeResolution", arr).read())
class SomeComponent(TelescopeComponent):
path = TelescopeParameter(Path(), default_value=None).tag(config=True)
val = TelescopeParameter(Float(), default_value=1.0).tag(config=True)
flag = TelescopeParameter(Bool(), default_value=True).tag(config=True)
class DRS4PedestalAndSpikeHeight(Tool):
name = 'lstchain_create_drs4_pedestal_file'
output_path = Path(
directory_ok=False,
help=
'Path for the output hdf5 file of pedestal baseline and spike heights',
).tag(config=True)
skip_samples_front = Integer(
default_value=10,
help='Do not include first N samples in pedestal calculation').tag(
config=True)
skip_samples_end = Integer(
default_value=1,
help='Do not include last N samples in pedestal calculation').tag(
config=True)
progress_bar = Bool(help="Show progress bar during processing",
default_value=True).tag(config=True)
full_statistics = Bool(help=(
"If True, write spike{1,2,3} mean, count, std for each capacitor."
" Otherwise, only mean spike height for each gain, pixel is written"),
default_value=False).tag(config=True)
overwrite = Bool(help=("If true, overwrite output without asking,"
" else fail if output file already exists"),
default_value=False).tag(config=True)
aliases = {
('i', 'input'): 'LSTEventSource.input_url',
('o', 'output'): 'DRS4PedestalAndSpikeHeight.output_path',
('m', 'max-events'): 'LSTEventSource.max_events',
}
flags = {
**flag(
"overwrite",
"DRS4PedestalAndSpikeHeight.overwrite",
"Overwrite output file if it exists",
"Fail if output file already exists",
),
**flag(
"progress",
"DRS4PedestalAndSpikeHeight.progress_bar",
"Show a progress bar during event processing",
"Do not show a progress bar during event processing",
),
**flag(
"full-statistics",
"DRS4PedestalAndSpikeHeight.full_statistics",
"Whether to write the full statistics about spikes or not",
),
}
classes = [LSTEventSource]
def setup(self):
self.output_path = self.output_path.expanduser().resolve()
if self.output_path.exists():
if self.overwrite:
self.log.warning("Overwriting %s", self.output_path)
self.output_path.unlink()
else:
raise ToolConfigurationError(
f"Output file {self.output_path} exists"
", use the `overwrite` option or choose another `output_path` "
)
self.log.debug("output path: %s", self.output_path)
Provenance().add_output_file(str(self.output_path), role="DL1/Event")
self.source = LSTEventSource(
parent=self,
pointing_information=False,
trigger_information=False,
)
# set some config options, these are necessary for this tool,
# so we set them here and not via the config system
self.source.r0_r1_calibrator.r1_sample_start = 0
self.source.r0_r1_calibrator.r1_sample_end = N_SAMPLES
self.source.r0_r1_calibrator.offset = 0
self.source.r0_r1_calibrator.apply_spike_correction = False
self.source.r0_r1_calibrator.apply_timelapse_correction = True
self.source.r0_r1_calibrator.apply_drs4_pedestal_correction = False
n_stats = N_GAINS * N_PIXELS * N_CAPACITORS_PIXEL
self.baseline_stats = OnlineStats(n_stats)
self.spike0_stats = OnlineStats(n_stats)
self.spike1_stats = OnlineStats(n_stats)
self.spike2_stats = OnlineStats(n_stats)
def start(self):
tel_id = self.source.tel_id
for event in tqdm(self.source, disable=not self.progress_bar):
fill_stats(
event.r1.tel[tel_id].waveform,
self.source.r0_r1_calibrator.first_cap[tel_id],
self.source.r0_r1_calibrator.first_cap_old[tel_id],
self.source.r0_r1_calibrator.last_readout_time[tel_id],
self.baseline_stats,
self.spike0_stats,
self.spike1_stats,
self.spike2_stats,
skip_samples_front=self.skip_samples_front,
skip_samples_end=self.skip_samples_end,
)
def mean_spike_height(self):
'''Calculate mean spike height for each gain, pixel'''
shape = (N_GAINS, N_PIXELS, N_CAPACITORS_PIXEL)
mean_baseline = self.baseline_stats.mean.reshape(shape)
spike_heights = np.full((N_GAINS, N_PIXELS, 3),
np.nan,
dtype=np.float32)
for i in range(3):
stats = getattr(self, f'spike{i}_stats')
counts = stats.counts.reshape(shape)
spike_height = stats.mean.reshape(shape) - mean_baseline
spike_height[counts == 0] = 0
# np.ma does not raise an error if the weights sum to 0
mean_height = np.ma.average(spike_height, weights=counts, axis=2)
# convert masked array to dense, replacing invalid values with nan
spike_heights[:, :, i] = mean_height.filled(np.nan)
unknown_spike_heights = np.isnan(spike_heights).any(axis=2)
n_unknown_spike_heights = np.count_nonzero(unknown_spike_heights)
if n_unknown_spike_heights > 0:
self.log.warning(
f'Could not determine spike height for {n_unknown_spike_heights} channels'
)
self.log.warning(
f'Gain, pixel: {np.nonzero(unknown_spike_heights)}')
# replace any unknown pixels with the mean over the camera
camera_mean_spike_height = np.nanmean(spike_heights, axis=(0, 1))
self.log.warning(
f'Using camera mean of {camera_mean_spike_height} for these pixels'
)
spike_heights[unknown_spike_heights] = camera_mean_spike_height
return spike_heights
def finish(self):
tel_id = self.source.tel_id
self.log.info('Writing output to %s', self.output_path)
key = f'r1/monitoring/drs4_baseline/tel_{tel_id:03d}'
shape = (N_GAINS, N_PIXELS, N_CAPACITORS_PIXEL)
baseline_mean = self.baseline_stats.mean.reshape(shape)
baseline_std = self.baseline_stats.std.reshape(shape)
baseline_counts = self.baseline_stats.counts.reshape(shape).astype(
np.uint16)
n_negative = np.count_nonzero(baseline_mean < 0)
if n_negative > 0:
gain, pixel, capacitor = np.nonzero(baseline_mean < 0)
self.log.critical(
f'{n_negative} baseline values are smaller than 0')
self.log.info("Gain | Pixel | Capacitor | Baseline ")
for g, p, c in zip(gain, pixel, capacitor):
self.log.info(
f"{g:4d} | {p:4d} | {c:9d} | {baseline_mean[g][p][c]:6.1f}"
)
n_small = np.count_nonzero(baseline_mean < 25)
if n_small > 0:
gain, pixel, capacitor = np.nonzero(baseline_mean < 25)
self.log.warning(f'{n_small} baseline values are smaller than 25')
self.log.info("Gain | Pixel | Capacitor | Baseline ")
for g, p, c in zip(gain, pixel, capacitor):
self.log.info(
f"{g:4d} | {p:4d} | {c:9d} | {baseline_mean[g][p][c]:6.1f}"
)
# Convert baseline mean and spike heights to uint16, handle missing
# values and values smaller 0, larger maxint
baseline_mean = convert_to_uint16(baseline_mean)
baseline_std = convert_to_uint16(baseline_std)
spike_height = convert_to_uint16(self.mean_spike_height())
with HDF5TableWriter(self.output_path) as writer:
Provenance().add_output_file(str(self.output_path))
drs4_calibration = DRS4CalibrationContainer(
baseline_mean=baseline_mean,
baseline_std=baseline_std,
baseline_counts=baseline_counts,
spike_height=spike_height,
)
writer.write(key, drs4_calibration)
class LSTEventSource(EventSource):
"""
EventSource for LST R0 data.
"""
multi_streams = Bool(True,
help='Read in parallel all streams ').tag(config=True)
min_flatfield_adc = Float(
default_value=3000.0,
help=
('Events with that have more than ``min_flatfield_pixel_fraction``'
' of the pixels inside [``min_flatfield_adc``, ``max_flatfield_adc``]'
' get tagged as EventType.FLATFIELD'),
).tag(config=True)
max_flatfield_adc = Float(
default_value=12000.0,
help=
('Events with that have more than ``min_flatfield_pixel_fraction``'
' of the pixels inside [``min_flatfield_adc``, ``max_flatfield_adc``]'
' get tagged as EventType.FLATFIELD'),
).tag(config=True)
min_flatfield_pixel_fraction = Float(
default_value=0.8,
help=(
'Events with that have more than ``min_flatfield_pixel_fraction``'
' of the pixels inside [``min_flatfield_pe``, ``max_flatfield_pe``]'
' get tagged as EventType.FLATFIELD'),
).tag(config=True)
default_trigger_type = Enum(
['ucts', 'tib'],
default_value='ucts',
help=
('Default source for trigger type information.'
' For older data, tib might be the better choice but for data newer'
' than 2020-06-25, ucts is the preferred option. The source will still'
' fallback to the other device if the chosen default device is not '
' available')).tag(config=True)
use_flatfield_heuristic = Bool(
default_value=True,
help=('If true, try to identify flat field events independent of the'
' trigger type in the event. See option ``min_flatfield_adc``'),
).tag(config=True)
calibrate_flatfields_and_pedestals = Bool(
default_value=True,
help='If True, flat field and pedestal events are also calibrated.'
).tag(config=True)
apply_drs4_corrections = Bool(
default_value=True,
help=(
'Apply DRS4 corrections.'
' If True, this will fill R1 waveforms with the corrections applied'
' Use the options for the LSTR0Corrections to configure which'
' corrections are applied'),
).tag(config=True)
trigger_information = Bool(
default_value=True, help='Fill trigger information.').tag(config=True)
pointing_information = Bool(
default_value=True,
help=('Fill pointing information.'
' Requires specifying `PointingSource.drive_report_path`'),
).tag(config=True)
classes = [PointingSource, EventTimeCalculator, LSTR0Corrections]
def __init__(self, input_url=None, **kwargs):
'''
Create a new LSTEventSource.
Parameters
----------
input_url: Path
Path to or url understood by ``ctapipe.core.traits.Path``.
If ``multi_streams`` is ``True``, the source will try to read all
streams matching the given ``input_url``
**kwargs:
Any of the traitlets. See ``LSTEventSource.class_print_help``
'''
super().__init__(input_url=input_url, **kwargs)
if self.multi_streams:
# test how many streams are there:
# file name must be [stream name]Run[all the rest]
# All the files with the same [all the rest] are opened
path, name = os.path.split(os.path.abspath(self.input_url))
if 'Run' in name:
_, run = name.split('Run', 1)
else:
run = name
ls = listdir(path)
self.file_list = []
for file_name in ls:
if run in file_name:
full_name = os.path.join(path, file_name)
self.file_list.append(full_name)
else:
self.file_list = [self.input_url]
self.multi_file = MultiFiles(self.file_list)
self.geometry_version = 4
self.camera_config = self.multi_file.camera_config
self.log.info("Read {} input files".format(
self.multi_file.num_inputs()))
self.tel_id = self.camera_config.telescope_id
self._subarray = self.create_subarray(self.geometry_version,
self.tel_id)
self.r0_r1_calibrator = LSTR0Corrections(subarray=self._subarray,
parent=self)
self.time_calculator = EventTimeCalculator(
subarray=self.subarray,
run_id=self.camera_config.configuration_id,
expected_modules_id=self.camera_config.lstcam.expected_modules_id,
parent=self,
)
self.pointing_source = PointingSource(subarray=self.subarray,
parent=self)
self.lst_service = self.fill_lst_service_container(
self.tel_id, self.camera_config)
@property
def subarray(self):
return self._subarray
@property
def is_simulation(self):
return False
@property
def obs_ids(self):
# currently no obs id is available from the input files
return [
self.camera_config.configuration_id,
]
@property
def datalevels(self):
if self.r0_r1_calibrator.calibration_path is not None:
return (DataLevel.R0, DataLevel.R1)
return (DataLevel.R0, )
def rewind(self):
self.multi_file.rewind()
@staticmethod
def create_subarray(geometry_version, tel_id=1):
"""
Obtain the subarray from the EventSource
Returns
-------
ctapipe.instrument.SubarrayDescription
"""
# camera info from LSTCam-[geometry_version].camgeom.fits.gz file
camera_geom = load_camera_geometry(version=geometry_version)
# get info on the camera readout:
daq_time_per_sample, pulse_shape_time_step, pulse_shapes = read_pulse_shapes(
)
camera_readout = CameraReadout(
'LSTCam',
1 / daq_time_per_sample,
pulse_shapes,
pulse_shape_time_step,
)
camera = CameraDescription('LSTCam', camera_geom, camera_readout)
lst_tel_descr = TelescopeDescription(name='LST',
tel_type='LST',
optics=OPTICS,
camera=camera)
tel_descriptions = {tel_id: lst_tel_descr}
# LSTs telescope position taken from MC from the moment
tel_positions = {tel_id: [50., 50., 16] * u.m}
subarray = SubarrayDescription(
name=f"LST-{tel_id} subarray",
tel_descriptions=tel_descriptions,
tel_positions=tel_positions,
)
return subarray
def _generator(self):
# container for LST data
array_event = LSTArrayEventContainer()
array_event.meta['input_url'] = self.input_url
array_event.meta['max_events'] = self.max_events
array_event.meta['origin'] = 'LSTCAM'
# also add service container to the event section
array_event.lst.tel[self.tel_id].svc = self.lst_service
# initialize general monitoring container
self.initialize_mon_container(array_event)
# loop on events
for count, zfits_event in enumerate(self.multi_file):
array_event.count = count
array_event.index.event_id = zfits_event.event_id
array_event.index.obs_id = self.obs_ids[0]
# Skip "empty" events that occur at the end of some runs
if zfits_event.event_id == 0:
self.log.warning('Event with event_id=0 found, skipping')
continue
self.fill_r0r1_container(array_event, zfits_event)
self.fill_lst_event_container(array_event, zfits_event)
if self.trigger_information:
self.fill_trigger_info(array_event)
self.fill_mon_container(array_event, zfits_event)
if self.pointing_information:
self.fill_pointing_info(array_event)
# apply low level corrections
if self.apply_drs4_corrections:
self.r0_r1_calibrator.apply_drs4_corrections(array_event)
# flat field tagging is performed on r1 data, so can only
# be done after the drs4 corrections are applied
if self.use_flatfield_heuristic:
self.tag_flatfield_events(array_event)
# gain select and calibrate to pe
if self.r0_r1_calibrator.calibration_path is not None:
# skip flatfield and pedestal events if asked
if (self.calibrate_flatfields_and_pedestals
or array_event.trigger.event_type
not in {EventType.FLATFIELD, EventType.SKY_PEDESTAL}):
self.r0_r1_calibrator.calibrate(array_event)
yield array_event
@staticmethod
def is_compatible(file_path):
from astropy.io import fits
try:
# The file contains two tables:
# 1: CameraConfig
# 2: Events
h = fits.open(file_path)[2].header
ttypes = [h[x] for x in h.keys() if 'TTYPE' in x]
except OSError:
# not even a fits file
return False
except IndexError:
# A fits file of a different format
return False
is_protobuf_zfits_file = ((h['XTENSION'] == 'BINTABLE')
and (h['EXTNAME'] == 'Events')
and (h['ZTABLE'] is True)
and (h['ORIGIN'] == 'CTA')
and (h['PBFHEAD'] == 'R1.CameraEvent'))
is_lst_file = 'lstcam_counters' in ttypes
return is_protobuf_zfits_file & is_lst_file
@staticmethod
def fill_lst_service_container(tel_id, camera_config):
"""
Fill LSTServiceContainer with specific LST service data data
(from the CameraConfig table of zfit file)
"""
return LSTServiceContainer(
telescope_id=tel_id,
cs_serial=camera_config.cs_serial,
configuration_id=camera_config.configuration_id,
date=camera_config.date,
num_pixels=camera_config.num_pixels,
num_samples=camera_config.num_samples,
pixel_ids=camera_config.expected_pixels_id,
data_model_version=camera_config.data_model_version,
num_modules=camera_config.lstcam.num_modules,
module_ids=camera_config.lstcam.expected_modules_id,
idaq_version=camera_config.lstcam.idaq_version,
cdhs_version=camera_config.lstcam.cdhs_version,
algorithms=camera_config.lstcam.algorithms,
pre_proc_algorithms=camera_config.lstcam.pre_proc_algorithms,
)
def fill_lst_event_container(self, array_event, zfits_event):
"""
Fill LSTEventContainer with specific LST service data
(from the Event table of zfit file)
"""
tel_id = self.tel_id
lst_evt = array_event.lst.tel[tel_id].evt
lst_evt.configuration_id = zfits_event.configuration_id
lst_evt.event_id = zfits_event.event_id
lst_evt.tel_event_id = zfits_event.tel_event_id
lst_evt.pixel_status = zfits_event.pixel_status
lst_evt.ped_id = zfits_event.ped_id
lst_evt.module_status = zfits_event.lstcam.module_status
lst_evt.extdevices_presence = zfits_event.lstcam.extdevices_presence
# if TIB data are there
if lst_evt.extdevices_presence & 1:
tib = zfits_event.lstcam.tib_data.view(TIB_DTYPE)[0]
lst_evt.tib_event_counter = tib['event_counter']
lst_evt.tib_pps_counter = tib['pps_counter']
lst_evt.tib_tenMHz_counter = tib['tenMHz_counter']
lst_evt.tib_stereo_pattern = tib['stereo_pattern']
lst_evt.tib_masked_trigger = tib['masked_trigger']
# if UCTS data are there
if lst_evt.extdevices_presence & 2:
if int(array_event.lst.tel[tel_id].svc.idaq_version) > 37201:
cdts = zfits_event.lstcam.cdts_data.view(
CDTS_AFTER_37201_DTYPE)[0]
lst_evt.ucts_timestamp = cdts[0]
lst_evt.ucts_address = cdts[1] # new
lst_evt.ucts_event_counter = cdts[2]
lst_evt.ucts_busy_counter = cdts[3] # new
lst_evt.ucts_pps_counter = cdts[4]
lst_evt.ucts_clock_counter = cdts[5]
lst_evt.ucts_trigger_type = cdts[6]
lst_evt.ucts_white_rabbit_status = cdts[7]
lst_evt.ucts_stereo_pattern = cdts[8] # new
lst_evt.ucts_num_in_bunch = cdts[9] # new
lst_evt.ucts_cdts_version = cdts[10] # new
else:
# unpack UCTS-CDTS data (old version)
cdts = zfits_event.lstcam.cdts_data.view(
CDTS_BEFORE_37201_DTYPE)[0]
lst_evt.ucts_event_counter = cdts[0]
lst_evt.ucts_pps_counter = cdts[1]
lst_evt.ucts_clock_counter = cdts[2]
lst_evt.ucts_timestamp = cdts[3]
lst_evt.ucts_camera_timestamp = cdts[4]
lst_evt.ucts_trigger_type = cdts[5]
lst_evt.ucts_white_rabbit_status = cdts[6]
# if SWAT data are there
if lst_evt.extdevices_presence & 4:
# unpack SWAT data
unpacked_swat = zfits_event.lstcam.swat_data.view(SWAT_DTYPE)[0]
lst_evt.swat_timestamp = unpacked_swat[0]
lst_evt.swat_counter1 = unpacked_swat[1]
lst_evt.swat_counter2 = unpacked_swat[2]
lst_evt.swat_event_type = unpacked_swat[3]
lst_evt.swat_camera_flag = unpacked_swat[4]
lst_evt.swat_camera_event_num = unpacked_swat[5]
lst_evt.swat_array_flag = unpacked_swat[6]
lst_evt.swat_array_event_num = unpacked_swat[7]
# unpack Dragon counters
counters = zfits_event.lstcam.counters.view(DRAGON_COUNTERS_DTYPE)
lst_evt.pps_counter = counters['pps_counter']
lst_evt.tenMHz_counter = counters['tenMHz_counter']
lst_evt.event_counter = counters['event_counter']
lst_evt.trigger_counter = counters['trigger_counter']
lst_evt.local_clock_counter = counters['local_clock_counter']
lst_evt.chips_flags = zfits_event.lstcam.chips_flags
lst_evt.first_capacitor_id = zfits_event.lstcam.first_capacitor_id
lst_evt.drs_tag_status = zfits_event.lstcam.drs_tag_status
lst_evt.drs_tag = zfits_event.lstcam.drs_tag
lst_evt.ucts_jump = False
def fill_trigger_info(self, array_event):
tel_id = self.tel_id
trigger = array_event.trigger
trigger.time = self.time_calculator(tel_id, array_event)
trigger.tels_with_trigger = [tel_id]
trigger.tel[tel_id].time = trigger.time
lst = array_event.lst.tel[tel_id]
tib_available = lst.evt.extdevices_presence & 1
ucts_available = lst.evt.extdevices_presence & 2
# decide which source to use, if both are available,
# the option decides, if not, fallback to the avilable source
# if no source available, warn and do not fill trigger info
if tib_available and ucts_available:
if self.default_trigger_type == 'ucts':
trigger_bits = lst.evt.ucts_trigger_type
else:
trigger_bits = lst.evt.tib_masked_trigger
elif tib_available:
trigger_bits = lst.evt.tib_masked_trigger
elif ucts_available:
trigger_bits = lst.evt.ucts_trigger_type
else:
self.log.warning('No trigger info available.')
trigger.event_type = EventType.UNKNOWN
return
if (ucts_available and lst.evt.ucts_trigger_type == 42
and self.default_trigger_type == "ucts"):
self.log.warning(
'Event with UCTS trigger_type 42 found.'
' Probably means unreliable or shifted UCTS data.'
' Consider switching to TIB using `default_trigger_type="tib"`'
)
# first bit mono trigger, second stereo.
# If *only* those two are set, we assume it's a physics event
# for all other we only check if the flag is present
if (trigger_bits & TriggerBits.PHYSICS) and not (trigger_bits
& TriggerBits.OTHER):
trigger.event_type = EventType.SUBARRAY
elif trigger_bits & TriggerBits.CALIBRATION:
trigger.event_type = EventType.FLATFIELD
elif trigger_bits & TriggerBits.PEDESTAL:
trigger.event_type = EventType.SKY_PEDESTAL
elif trigger_bits & TriggerBits.SINGLE_PE:
trigger.event_type = EventType.SINGLE_PE
else:
self.log.warning(
f'Event {array_event.index.event_id} has unknown event type, trigger: {trigger_bits:08b}'
)
trigger.event_type = EventType.UNKNOWN
def tag_flatfield_events(self, array_event):
'''
Use a heuristic based on R1 waveforms to recognize flat field events
Currently, tagging of flat field events does not work,
they are reported as physics events, here a heuristic identifies
those events. Since trigger types might be wrong due to ucts errors,
we try to identify flat field events in all trigger types.
DRS4 corrections but not the p.e. calibration must be applied
'''
tel_id = self.tel_id
waveform = array_event.r1.tel[tel_id].waveform
# needs to work for gain already selected or not
if waveform.ndim == 3:
image = waveform[HIGH_GAIN].sum(axis=1)
else:
image = waveform.sum(axis=1)
in_range = (image >= self.min_flatfield_adc) & (image <=
self.max_flatfield_adc)
n_in_range = np.count_nonzero(in_range)
looks_like_ff = n_in_range >= self.min_flatfield_pixel_fraction * image.size
if looks_like_ff:
array_event.trigger.event_type = EventType.FLATFIELD
self.log.debug('Setting event type of event'
f' {array_event.index.event_id} to FLATFIELD')
elif array_event.trigger.event_type == EventType.FLATFIELD:
self.log.warning(
'Found FF event that does not fulfill FF criteria: %d',
array_event.index.event_id,
)
array_event.trigger.event_type = EventType.UNKNOWN
def fill_pointing_info(self, array_event):
tel_id = self.tel_id
pointing = self.pointing_source.get_pointing_position_altaz(
tel_id,
array_event.trigger.time,
)
array_event.pointing.tel[tel_id] = pointing
array_event.pointing.array_altitude = pointing.altitude
array_event.pointing.array_azimuth = pointing.azimuth
ra, dec = self.pointing_source.get_pointing_position_icrs(
tel_id, array_event.trigger.time)
array_event.pointing.array_ra = ra
array_event.pointing.array_dec = dec
def fill_r0r1_camera_container(self, zfits_event):
"""
Fill the r0 or r1 container, depending on whether gain
selection has already happened (r1) or not (r0)
This will create waveforms of shape (N_GAINS, N_PIXELS, N_SAMPLES),
or (N_PIXELS, N_SAMPLES) respectively regardless of the n_pixels, n_samples
in the file.
Missing or broken pixels are filled using maxval of the waveform dtype.
"""
n_pixels = self.camera_config.num_pixels
n_samples = self.camera_config.num_samples
expected_pixels = self.camera_config.expected_pixels_id
has_low_gain = (zfits_event.pixel_status
& PixelStatus.LOW_GAIN_STORED).astype(bool)
has_high_gain = (zfits_event.pixel_status
& PixelStatus.HIGH_GAIN_STORED).astype(bool)
not_broken = (has_low_gain | has_high_gain).astype(bool)
# broken pixels have both false, so gain selected means checking
# if there are any pixels where exactly one of high or low gain is stored
gain_selected = np.any(has_low_gain != has_high_gain)
# fill value for broken pixels
dtype = zfits_event.waveform.dtype
fill = np.iinfo(dtype).max
# we assume that either all pixels are gain selected or none
# only broken pixels are allowed to be missing completely
if gain_selected:
selected_gain = np.where(has_high_gain, 0, 1)
waveform = np.full((n_pixels, n_samples), fill, dtype=dtype)
waveform[not_broken] = zfits_event.waveform.reshape(
(-1, n_samples))
reordered_waveform = np.full((N_PIXELS, N_SAMPLES),
fill,
dtype=dtype)
reordered_waveform[expected_pixels] = waveform
reordered_selected_gain = np.full(N_PIXELS, -1, dtype=np.int8)
reordered_selected_gain[expected_pixels] = selected_gain
r0 = R0CameraContainer()
r1 = R1CameraContainer(
waveform=reordered_waveform,
selected_gain_channel=reordered_selected_gain,
)
else:
reshaped_waveform = zfits_event.waveform.reshape(
N_GAINS, n_pixels, n_samples)
# re-order the waveform following the expected_pixels_id values
# could also just do waveform = reshaped_waveform[np.argsort(expected_ids)]
reordered_waveform = np.full((N_GAINS, N_PIXELS, N_SAMPLES),
fill,
dtype=dtype)
reordered_waveform[:, expected_pixels, :] = reshaped_waveform
r0 = R0CameraContainer(waveform=reordered_waveform)
r1 = R1CameraContainer()
return r0, r1
def fill_r0r1_container(self, array_event, zfits_event):
"""
Fill with R0Container
"""
r0, r1 = self.fill_r0r1_camera_container(zfits_event)
array_event.r0.tel[self.tel_id] = r0
array_event.r1.tel[self.tel_id] = r1
def initialize_mon_container(self, array_event):
"""
Fill with MonitoringContainer.
For the moment, initialize only the PixelStatusContainer
"""
container = array_event.mon
mon_camera_container = container.tel[self.tel_id]
shape = (N_GAINS, N_PIXELS)
# all pixels broken by default
status_container = PixelStatusContainer(
hardware_failing_pixels=np.ones(shape, dtype=bool),
pedestal_failing_pixels=np.zeros(shape, dtype=bool),
flatfield_failing_pixels=np.zeros(shape, dtype=bool),
)
mon_camera_container.pixel_status = status_container
def fill_mon_container(self, array_event, zfits_event):
"""
Fill with MonitoringContainer.
For the moment, initialize only the PixelStatusContainer
"""
status_container = array_event.mon.tel[self.tel_id].pixel_status
# reorder the array
expected_pixels_id = self.camera_config.expected_pixels_id
reordered_pixel_status = np.zeros(N_PIXELS,
dtype=zfits_event.pixel_status.dtype)
reordered_pixel_status[expected_pixels_id] = zfits_event.pixel_status
channel_info = get_channel_info(reordered_pixel_status)
status_container.hardware_failing_pixels[:] = channel_info == 0
def __exit__(self, exc_type, exc_value, traceback):
self.close()
def __len__(self):
if self.max_events is not None:
return min(self.max_events, len(self.multi_file))
return len(self.multi_file)
def close(self):
self.multi_file.close()
class SimTelEventSource(EventSource):
skip_calibration_events = Bool(
True, help="Skip calibration events").tag(config=True)
back_seekable = Bool(
False,
help=("Require the event source to be backwards seekable."
" This will reduce in slower read speed for gzipped files"
" and is not possible for zstd compressed files"),
).tag(config=True)
focal_length_choice = CaselessStrEnum(
["nominal", "effective"],
default_value="nominal",
help=
("if both nominal and effective focal lengths are available in the "
"SimTelArray file, which one to use when constructing the "
"SubarrayDescription (which will be used in CameraFrame to TelescopeFrame "
"coordinate transforms. The 'nominal' focal length is the one used during "
"the simulation, the 'effective' focal length is computed using specialized "
"ray-tracing from a point light source"),
).tag(config=True)
def __init__(self, config=None, parent=None, gain_selector=None, **kwargs):
"""
EventSource for simtelarray files using the pyeventio library.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
gain_selector : ctapipe.calib.camera.gainselection.GainSelector
The GainSelector to use. If None, then ThresholdGainSelector will be used.
kwargs
"""
super().__init__(config=config, parent=parent, **kwargs)
self.metadata["is_simulation"] = True
self._camera_cache = {}
# traitlets creates an empty set as default,
# which ctapipe treats as no restriction on the telescopes
# but eventio treats an emty set as "no telescopes allowed"
# so we explicitly pass None in that case
self.file_ = SimTelFile(
Path(self.input_url).expanduser(),
allowed_telescopes=set(self.allowed_tels)
if self.allowed_tels else None,
skip_calibration=self.skip_calibration_events,
zcat=not self.back_seekable,
)
if self.back_seekable and self.is_stream:
raise IOError(
"back seekable was required but not possible for inputfile")
self._subarray_info = self.prepare_subarray_info(
self.file_.telescope_descriptions, self.file_.header)
self.start_pos = self.file_.tell()
# Waveforms from simtelarray have both gain channels
# Gain selection is performed by this EventSource to produce R1 waveforms
if gain_selector is None:
gain_selector = ThresholdGainSelector(parent=self)
self.gain_selector = gain_selector
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def close(self):
self.file_.close()
@property
def is_stream(self):
return not isinstance(self.file_._filehandle,
(BufferedReader, GzipFile))
def prepare_subarray_info(self, telescope_descriptions, header):
"""
Constructs a SubarrayDescription object from the
``telescope_descriptions`` given by ``SimTelFile``
Parameters
----------
telescope_descriptions: dict
telescope descriptions as given by ``SimTelFile.telescope_descriptions``
header: dict
header as returned by ``SimTelFile.header``
Returns
-------
SubarrayDescription :
instrumental information
"""
tel_descriptions = {} # tel_id : TelescopeDescription
tel_positions = {} # tel_id : TelescopeDescription
for tel_id, telescope_description in telescope_descriptions.items():
cam_settings = telescope_description["camera_settings"]
pixel_settings = telescope_description["pixel_settings"]
n_pixels = cam_settings["n_pixels"]
focal_length = u.Quantity(cam_settings["focal_length"], u.m)
if self.focal_length_choice == "effective":
try:
focal_length = u.Quantity(
cam_settings["effective_focal_length"], u.m)
except KeyError as err:
raise RuntimeError(
f"the SimTelEventSource option 'focal_length_choice' was set to "
f"{self.focal_length_choice}, but the effective focal length "
f"was not present in the file. ({err})")
try:
telescope = guess_telescope(n_pixels, focal_length)
except ValueError:
telescope = UNKNOWN_TELESCOPE
camera = self._camera_cache.get(telescope.camera_name)
if camera is None:
camera = build_camera(cam_settings, pixel_settings, telescope)
self._camera_cache[telescope.camera_name] = camera
optics = OpticsDescription(
name=telescope.name,
num_mirrors=telescope.n_mirrors,
equivalent_focal_length=focal_length,
mirror_area=u.Quantity(cam_settings["mirror_area"], u.m**2),
num_mirror_tiles=cam_settings["n_mirrors"],
)
tel_descriptions[tel_id] = TelescopeDescription(
name=telescope.name,
tel_type=telescope.type,
optics=optics,
camera=camera,
)
tel_idx = np.where(header["tel_id"] == tel_id)[0][0]
tel_positions[tel_id] = header["tel_pos"][tel_idx] * u.m
return SubarrayDescription(
"MonteCarloArray",
tel_positions=tel_positions,
tel_descriptions=tel_descriptions,
)
@staticmethod
def is_compatible(file_path):
return is_eventio(Path(file_path).expanduser())
@property
def subarray(self):
return self._subarray_info
def _generator(self):
if self.file_.tell() > self.start_pos:
self.file_._next_header_pos = 0
warnings.warn("Backseeking to start of file.")
try:
yield from self.__generator()
except EOFError:
msg = 'EOFError reading from "{input_url}". Might be truncated'.format(
input_url=self.input_url)
self.log.warning(msg)
warnings.warn(msg)
def __generator(self):
data = EventAndMonDataContainer()
data.meta["origin"] = "hessio"
data.meta["input_url"] = self.input_url
data.meta["max_events"] = self.max_events
for counter, array_event in enumerate(self.file_):
# next lines are just for debugging
self.array_event = array_event
data.event_type = array_event["type"]
# calibration events do not have an event id
if data.event_type == "calibration":
event_id = -1
else:
event_id = array_event["event_id"]
data.inst.subarray = self._subarray_info
obs_id = self.file_.header["run"]
tels_with_data = set(array_event["telescope_events"].keys())
data.count = counter
data.index.obs_id = obs_id
data.index.event_id = event_id
data.r0.obs_id = obs_id # deprecated
data.r0.event_id = event_id # deprecated
data.r0.tels_with_data = tels_with_data
data.r1.obs_id = obs_id # deprecated
data.r1.event_id = event_id # deprecated
data.r1.tels_with_data = tels_with_data
data.dl0.obs_id = obs_id # deprecated
data.dl0.event_id = event_id # deprecated
data.dl0.tels_with_data = tels_with_data
# handle telescope filtering by taking the intersection of
# tels_with_data and allowed_tels
if len(self.allowed_tels) > 0:
selected = tels_with_data & self.allowed_tels
if len(selected) == 0:
continue # skip event
data.r0.tels_with_data = selected
data.r1.tels_with_data = selected
data.dl0.tels_with_data = selected
trigger_information = array_event["trigger_information"]
data.trig.tels_with_trigger = trigger_information[
"triggered_telescopes"]
time_s, time_ns = trigger_information["gps_time"]
data.trig.gps_time = Time(time_s * u.s,
time_ns * u.ns,
format="unix",
scale="utc")
if data.event_type == "data":
self.fill_mc_information(data, array_event)
# this should be done in a nicer way to not re-allocate the
# data each time (right now it's just deleted and garbage
# collected)
data.r0.tel.clear()
data.r1.tel.clear()
data.dl0.tel.clear()
data.dl1.tel.clear()
data.mc.tel.clear() # clear the previous telescopes
telescope_events = array_event["telescope_events"]
tracking_positions = array_event["tracking_positions"]
for tel_id, telescope_event in telescope_events.items():
tel_index = self.file_.header["tel_id"].tolist().index(tel_id)
adc_samples = telescope_event.get("adc_samples")
if adc_samples is None:
adc_samples = telescope_event["adc_sums"][:, :, np.newaxis]
_, n_pixels, n_samples = adc_samples.shape
mc = data.mc.tel[tel_id]
mc.dc_to_pe = array_event["laser_calibrations"][tel_id][
"calib"]
mc.pedestal = array_event["camera_monitorings"][tel_id][
"pedestal"]
mc.photo_electron_image = (array_event.get(
"photoelectrons",
{}).get(tel_index,
{}).get("photoelectrons",
np.zeros(n_pixels, dtype="float32")))
tracking_position = tracking_positions[tel_id]
mc.azimuth_raw = tracking_position["azimuth_raw"]
mc.altitude_raw = tracking_position["altitude_raw"]
mc.azimuth_cor = tracking_position.get("azimuth_cor", np.nan)
mc.altitude_cor = tracking_position.get("altitude_cor", np.nan)
if np.isnan(mc.azimuth_cor):
data.pointing[tel_id].azimuth = u.Quantity(
mc.azimuth_raw, u.rad)
else:
data.pointing[tel_id].azimuth = u.Quantity(
mc.azimuth_cor, u.rad)
if np.isnan(mc.altitude_cor):
data.pointing[tel_id].altitude = u.Quantity(
mc.altitude_raw, u.rad)
else:
data.pointing[tel_id].altitude = u.Quantity(
mc.altitude_cor, u.rad)
r0 = data.r0.tel[tel_id]
r1 = data.r1.tel[tel_id]
r0.waveform = adc_samples
r1.waveform, r1.selected_gain_channel = apply_simtel_r1_calibration(
adc_samples, mc.pedestal, mc.dc_to_pe, self.gain_selector)
pixel_lists = telescope_event["pixel_lists"]
r0.num_trig_pix = pixel_lists.get(0, {"pixels": 0})["pixels"]
if r0.num_trig_pix > 0:
r0.trig_pix_id = pixel_lists[0]["pixel_list"]
yield data
def fill_mc_information(self, data, array_event):
mc_event = array_event["mc_event"]
mc_shower = array_event["mc_shower"]
data.mc.energy = mc_shower["energy"] * u.TeV
data.mc.alt = Angle(mc_shower["altitude"], u.rad)
data.mc.az = Angle(mc_shower["azimuth"], u.rad)
data.mc.core_x = mc_event["xcore"] * u.m
data.mc.core_y = mc_event["ycore"] * u.m
first_int = mc_shower["h_first_int"] * u.m
data.mc.h_first_int = first_int
data.mc.x_max = mc_shower["xmax"] * u.g / (u.cm**2)
data.mc.shower_primary_id = mc_shower["primary_id"]
# mc run header data
data.mcheader.run_array_direction = Angle(
self.file_.header["direction"] * u.rad)
mc_run_head = self.file_.mc_run_headers[-1]
data.mcheader.corsika_version = mc_run_head["shower_prog_vers"]
data.mcheader.simtel_version = mc_run_head["detector_prog_vers"]
data.mcheader.energy_range_min = mc_run_head["E_range"][0] * u.TeV
data.mcheader.energy_range_max = mc_run_head["E_range"][1] * u.TeV
data.mcheader.prod_site_B_total = mc_run_head["B_total"] * u.uT
data.mcheader.prod_site_B_declination = Angle(
mc_run_head["B_declination"] * u.rad)
data.mcheader.prod_site_B_inclination = Angle(
mc_run_head["B_inclination"] * u.rad)
data.mcheader.prod_site_alt = mc_run_head["obsheight"] * u.m
data.mcheader.spectral_index = mc_run_head["spectral_index"]
data.mcheader.shower_prog_start = mc_run_head["shower_prog_start"]
data.mcheader.shower_prog_id = mc_run_head["shower_prog_id"]
data.mcheader.detector_prog_start = mc_run_head["detector_prog_start"]
data.mcheader.detector_prog_id = mc_run_head["detector_prog_id"]
data.mcheader.num_showers = mc_run_head["n_showers"]
data.mcheader.shower_reuse = mc_run_head["n_use"]
data.mcheader.max_alt = mc_run_head["alt_range"][1] * u.rad
data.mcheader.min_alt = mc_run_head["alt_range"][0] * u.rad
data.mcheader.max_az = mc_run_head["az_range"][1] * u.rad
data.mcheader.min_az = mc_run_head["az_range"][0] * u.rad
data.mcheader.diffuse = mc_run_head["diffuse"]
data.mcheader.max_viewcone_radius = mc_run_head["viewcone"][1] * u.deg
data.mcheader.min_viewcone_radius = mc_run_head["viewcone"][0] * u.deg
data.mcheader.max_scatter_range = mc_run_head["core_range"][1] * u.m
data.mcheader.min_scatter_range = mc_run_head["core_range"][0] * u.m
data.mcheader.core_pos_mode = mc_run_head["core_pos_mode"]
data.mcheader.injection_height = mc_run_head["injection_height"] * u.m
data.mcheader.atmosphere = mc_run_head["atmosphere"]
data.mcheader.corsika_iact_options = mc_run_head[
"corsika_iact_options"]
data.mcheader.corsika_low_E_model = mc_run_head["corsika_low_E_model"]
data.mcheader.corsika_high_E_model = mc_run_head[
"corsika_high_E_model"]
data.mcheader.corsika_bunchsize = mc_run_head["corsika_bunchsize"]
data.mcheader.corsika_wlen_min = mc_run_head["corsika_wlen_min"] * u.nm
data.mcheader.corsika_wlen_max = mc_run_head["corsika_wlen_max"] * u.nm
data.mcheader.corsika_low_E_detail = mc_run_head[
"corsika_low_E_detail"]
data.mcheader.corsika_high_E_detail = mc_run_head[
"corsika_high_E_detail"]
class TimeWaveformFitter(TelescopeComponent):
"""
Class used to perform event reconstruction by fitting of a model on waveforms.
"""
sigma_s = FloatTelescopeParameter(
default_value=1,
help='Width of the single photo-electron peak distribution.',
allow_none=False).tag(config=True)
crosstalk = FloatTelescopeParameter(default_value=0,
help='Average pixel crosstalk.',
allow_none=False).tag(config=True)
sigma_space = Float(
4,
help=
'Size of the region on which the fit is performed relative to the image extension.',
allow_none=False).tag(config=True)
sigma_time = Float(
3,
help=
'Time window on which the fit is performed relative to the image temporal extension.',
allow_none=False).tag(config=True)
time_before_shower = FloatTelescopeParameter(
default_value=10,
help='Additional time at the start of the fit temporal window.',
allow_none=False).tag(config=True)
time_after_shower = FloatTelescopeParameter(
default_value=20,
help='Additional time at the end of the fit temporal window.',
allow_none=False).tag(config=True)
use_weight = Bool(
False,
help=
'If True, the brightest sample is twice as important as the dimmest pixel in the '
'likelihood. If false all samples are equivalent.',
allow_none=False).tag(config=True)
no_asymmetry = Bool(
False,
help='If true, the asymmetry of the spatial model is fixed to 0.',
allow_none=False).tag(config=True)
use_interleaved = Path(
None,
help=
'Location of the dl1 file used to estimate the pedestal exploiting interleaved'
' events.',
allow_none=True).tag(config=True)
n_peaks = Int(
0,
help=
'Maximum brightness (p.e.) for which the full likelihood computation is used. '
'If the Poisson term for Np.e.>n_peak is more than 1e-6 a Gaussian approximation is used.',
allow_none=False).tag(config=True)
bound_charge_factor = FloatTelescopeParameter(
default_value=4,
help='Maximum relative change to the fitted charge parameter.',
allow_none=False).tag(config=True)
bound_t_cm_value = FloatTelescopeParameter(
default_value=10,
help='Maximum change to the t_cm parameter.',
allow_none=False).tag(config=True)
bound_centroid_control_parameter = FloatTelescopeParameter(
default_value=1,
help='Maximum change of the centroid coordinated in '
'number of seed length',
allow_none=False).tag(config=True)
bound_max_length_factor = FloatTelescopeParameter(
default_value=2,
help='Maximum relative increase to the fitted length parameter.',
allow_none=False).tag(config=True)
bound_length_asymmetry = FloatTelescopeParameter(
default_value=9,
help='Bounds for the fitted rl parameter.',
allow_none=False).tag(config=True)
bound_max_v_cm_factor = FloatTelescopeParameter(
default_value=2,
help='Maximum relative increase to the fitted v_cm parameter.',
allow_none=False).tag(config=True)
default_seed_t_cm = FloatTelescopeParameter(
default_value=0,
help='Default starting value of t_cm when the seed extraction failed.',
allow_none=False).tag(config=True)
default_seed_v_cm = FloatTelescopeParameter(
default_value=40,
help='Default starting value of v_cm when the seed extraction failed.',
allow_none=False).tag(config=True)
verbose = Int(
0,
help='4 - used for tests: create debug plots\n'
'3 - create debug plots, wait for input after each event, increase minuit verbose level\n'
'2 - create debug plots, increase minuit verbose level\n'
'1 - increase minuit verbose level\n'
'0 - silent',
allow_none=False).tag(config=True)
def __init__(self, subarray, config=None, parent=None, **kwargs):
super().__init__(subarray=subarray,
config=config,
parent=parent,
**kwargs)
self.subarray = subarray
self.template_dict = {}
self.template_time_of_max_dict = {}
for tel_id in subarray.tel:
self.template_dict[
tel_id] = NormalizedPulseTemplate.load_from_eventsource(
subarray.tel[tel_id].camera.readout)
self.template_time_of_max_dict[tel_id] = self.template_dict[
tel_id].compute_time_of_max()
poisson_peaks = np.arange(self.n_peaks + 1, dtype=int)
poisson_peaks[0] = 1
self.factorial = np.cumprod(poisson_peaks, dtype='u8')
# Find the transition charge between full likelihood computation and Gaussian approximation
# The maximum charge is selected such that each Poisson terms in the full likelihood computation
# above the n_peaks limit account for less than (1/n_peaks)%
transition_charges = {}
for config_crosstalk in self.crosstalk:
# if n_peaks is set to 0, only the Gaussian approximation is used
transition_charges[config_crosstalk[2]] = 0.0 if self.n_peaks == 0\
else self.find_transition_charge(config_crosstalk[2], 1e-2/self.n_peaks)
self.transition_charges = {}
for tel_id in subarray.tel:
self.transition_charges[tel_id] = transition_charges[
self.crosstalk.tel[tel_id]]
self.start_parameters = None
self.names_parameters = None
self.end_parameters = None
self.error_parameters = None
self.bound_parameters = None
self.fcn = None
def call_setup(self, event, telescope_id, dl1_container):
"""
Extract all event dependent quantities used for the fit.
Parameters
----------
event: ctapipe event container
Current event container.
telescope_id: int
Id of the telescope
dl1_container: DL1ParametersContainer
Contains the Hillas parameters used as seed for the fit
Returns
-------
focal_length: astropy.units.Quantity
Focal length of the telescope
fit_params: array
Array containing all the variable needed to compute the likelihood
during the fir excluding the model parameters
"""
geometry = self.subarray.tel[telescope_id].camera.geometry
unit = geometry.pix_x.unit
pix_x = geometry.pix_x.to_value(unit)
pix_y = geometry.pix_y.to_value(unit)
r_max = geometry.guess_radius().to_value(unit)
pix_radius = np.sqrt(geometry.pix_area[0].to_value(unit**2) /
np.pi) # find linear size of a pixel
readout = self.subarray.tel[telescope_id].camera.readout
sampling_rate = readout.sampling_rate.to_value(u.GHz)
dt = (1.0 / sampling_rate)
template = self.template_dict[telescope_id]
image = event.dl1.tel[telescope_id].image
hillas_signal_pixels = event.dl1.tel[telescope_id].image_mask
start_x_cm, start_y_cm = init_centroid(dl1_container,
geometry[hillas_signal_pixels],
unit,
image[hillas_signal_pixels],
self.no_asymmetry)
waveform = event.r1.tel[telescope_id].waveform
dl1_calib = event.calibration.tel[telescope_id].dl1
time_shift = dl1_calib.time_shift
# TODO check if this is correct here or if it is applied to r1 waveform earlier
if dl1_calib.pedestal_offset is not None:
waveform = waveform - dl1_calib.pedestal_offset[:, np.newaxis]
n_pixels, n_samples = waveform.shape
times = np.arange(0, n_samples) * dt
selected_gains = event.r1.tel[telescope_id].selected_gain_channel
is_high_gain = (selected_gains == 0)
# We assume that the time gradient is given in unit of 'geometry spatial unit'/ns
v = dl1_container.time_gradient
psi = dl1_container.psi.to_value(u.rad)
# We use only positive time gradients and psi is projected in [-pi,pi] from [-pi/2,pi/2]
if v < 0:
if psi >= 0:
psi = psi - np.pi
else:
psi = psi + np.pi
start_length = max(dl1_container.length.to_value(unit), pix_radius)
# With current likelihood computation, order and type of the parameters are important
start_parameters = {
'charge':
dl1_container.intensity,
't_cm':
dl1_container.intercept -
self.template_time_of_max_dict[telescope_id],
'x_cm':
start_x_cm.to_value(unit),
'y_cm':
start_y_cm.to_value(unit),
'length':
start_length,
'wl':
max(dl1_container.wl, 0.01),
'psi':
psi,
'v':
np.abs(v),
'rl':
0.0
}
# Temporal parameters extraction fails when cleaning select only 2 pixels, we use defaults values in this case
if np.isnan(start_parameters['t_cm']):
start_parameters['t_cm'] = self.default_seed_t_cm.tel[telescope_id]
if np.isnan(start_parameters['v']):
start_parameters['v'] = self.default_seed_v_cm.tel[telescope_id]
t_max = n_samples * dt
v_min, v_max = 0, max(
self.bound_max_v_cm_factor.tel[telescope_id] *
start_parameters['v'], 50)
rl_min, rl_max = -self.bound_length_asymmetry.tel[
telescope_id], self.bound_length_asymmetry.tel[telescope_id]
if self.no_asymmetry:
rl_min, rl_max = 0.0, 0.0
bound_centroid = self.bound_centroid_control_parameter.tel[
telescope_id] * start_length
bound_parameters = {
'charge': (dl1_container.intensity /
self.bound_charge_factor.tel[telescope_id],
dl1_container.intensity *
self.bound_charge_factor.tel[telescope_id]),
't_cm': (-self.bound_t_cm_value.tel[telescope_id],
t_max + self.bound_t_cm_value.tel[telescope_id]),
'x_cm': (start_x_cm.to_value(unit) - bound_centroid,
start_x_cm.to_value(unit) + bound_centroid),
'y_cm': (start_y_cm.to_value(unit) - bound_centroid,
start_y_cm.to_value(unit) + bound_centroid),
'length':
(pix_radius,
min(self.bound_max_length_factor.tel[telescope_id] * start_length,
r_max)),
'wl': (0.001, 1.0),
'psi': (-np.pi * 2.0, np.pi * 2.0),
'v': (v_min, v_max),
'rl': (rl_min, rl_max)
}
mask_pixel, mask_time = self.clean_data(pix_x, pix_y, pix_radius,
times, start_parameters,
telescope_id)
spatial_ones = np.ones(np.sum(mask_pixel))
is_high_gain = is_high_gain[mask_pixel]
sig_s = spatial_ones * self.sigma_s.tel[telescope_id]
crosstalks = spatial_ones * self.crosstalk.tel[telescope_id]
times = (np.arange(0, n_samples) * dt)[mask_time]
time_shift = time_shift[mask_pixel]
p_x = pix_x[mask_pixel]
p_y = pix_y[mask_pixel]
pix_area = geometry.pix_area[mask_pixel].to_value(unit**2)
data = waveform
error = None # TODO include option to use calibration data
filter_pixels = np.nonzero(~mask_pixel)
filter_times = np.nonzero(~mask_time)
if error is None:
std = np.std(data[~mask_pixel])
error = np.full(data.shape[0], std)
data = np.delete(data, filter_pixels, axis=0)
data = np.delete(data, filter_times, axis=1)
error = np.delete(error, filter_pixels, axis=0)
# Fill the set of non-fitted parameters needed to compute the likelihood. Order and type sensitive.
fit_params = [
data, error, is_high_gain, sig_s, crosstalks, times,
np.float32(time_shift), p_x, p_y,
np.float64(pix_area), template.dt, template.t0,
template.amplitude_LG, template.amplitude_HG, self.n_peaks,
self.transition_charges[telescope_id], self.use_weight,
self.factorial
]
self.start_parameters = start_parameters
self.names_parameters = start_parameters.keys()
self.bound_parameters = bound_parameters
return unit, fit_params
def __call__(self, event, telescope_id, dl1_container):
# setup angle to distance conversion on the camera plane for the current telescope
focal_length = self.subarray.tel[
telescope_id].optics.equivalent_focal_length
angle_dist_eq = [
(u.rad, u.m, lambda x: np.tan(x) * focal_length.to_value(u.m),
lambda x: np.arctan(x / focal_length.to_value(u.m))),
(u.rad**2, u.m**2, lambda x:
(np.tan(np.sqrt(x)) * focal_length.to_value(u.m))**2, lambda x:
(np.arctan(np.sqrt(x) / focal_length.to_value(u.m)))**2)
]
with u.set_enabled_equivalencies(angle_dist_eq):
self.start_parameters = None
self.names_parameters = None
unit_cam, fit_params = self.call_setup(event, telescope_id,
dl1_container)
self.end_parameters = None
self.error_parameters = None
self.fcn = None
return self.predict(unit_cam, fit_params)
def clean_data(self, pix_x, pix_y, pix_radius, times, start_parameters,
telescope_id):
"""
Method used to select pixels and time samples used in the fitting procedure.
The spatial selection takes pixels in an ellipsis obtained from the seed Hillas parameters extended by one pixel
size and multiplied by a factor sigma_space.
The temporal selection takes a time window centered on the seed time of center of mass and of duration equal to
the time of propagation of the signal along the length of the ellipsis times a factor sigma_time.
An additional fixed duration is also added before and after this time window through the time_before_shower and
time_after_shower arguments.
Parameters
----------
pix_x, pix_y: array-like
Pixels positions
pix_radius: float
times: array-like
Sampling times before timeshift corrections
start_parameters: dict
Seed parameters derived from the Hillas parameters
telescope_id: int
Returns
----------
mask_pixel, mask_time: array-like
Mask used to select pixels and times for the fit
"""
x_cm = start_parameters['x_cm']
y_cm = start_parameters['y_cm']
length = start_parameters['length']
width = start_parameters['wl'] * length
psi = start_parameters['psi']
dx = pix_x - x_cm
dy = pix_y - y_cm
lon = dx * np.cos(psi) + dy * np.sin(psi)
lat = dx * np.sin(psi) - dy * np.cos(psi)
mask_pixel = ((lon / (length + pix_radius))**2 +
(lat / (width + pix_radius))**2) < self.sigma_space**2
v = start_parameters['v']
t_start = (start_parameters['t_cm'] -
(np.abs(v) * length / 2 * self.sigma_time) -
self.time_before_shower.tel[telescope_id])
t_end = (start_parameters['t_cm'] +
(np.abs(v) * length / 2 * self.sigma_time) +
self.time_after_shower.tel[telescope_id])
mask_time = (times < t_end) * (times > t_start)
return mask_pixel, mask_time
def find_transition_charge(self, crosstalk, poisson_proba_min=1e-2):
"""
Find the charge below which the full likelihood computation is performed and above which a Gaussian
approximation is used. For a given pixel crosstalk it finds the maximum charge with a Generalised Poisson term
below poisson_proba_min for n_peaks photo-electrons. n_peaks here is the configured maximum number of
photo-electron considered in the full likelihood computation.
Parameters
----------
crosstalk : float
Pixels crosstalk
poisson_proba_min: float
Returns
-------
transition_charge: float32
Model charge of transition between full and approximated likelihood
"""
transition_charge = self.n_peaks / (1 + crosstalk)
step = transition_charge / 100
def poisson(mu, cross_talk):
return (mu * pow(mu + self.n_peaks * cross_talk,
(self.n_peaks - 1)) /
self.factorial[self.n_peaks] *
np.exp(-mu - self.n_peaks * cross_talk))
while poisson(transition_charge, crosstalk) > poisson_proba_min:
transition_charge -= step
logger.info(
f'Transition charge between full and approximated likelihood for camera '
f'with crosstalk = {crosstalk:.4f} is, {transition_charge:.4f}, p.e.'
)
return np.float32(transition_charge)
def fit(self, fit_params):
"""
Performs the fitting procedure.
Parameters
----------
fit_params: array
Parameters used to compute the likelihood but not fitted
"""
def f(*args):
return -2 * self.log_likelihood(*args, fit_params=fit_params)
print_level = 2 if self.verbose in [1, 2, 3] else 0
m = Minuit(f,
name=self.names_parameters,
*self.start_parameters.values())
for key, val in self.bound_parameters.items():
m.limits[key] = val
m.print_level = print_level
m.errordef = 0.5
m.simplex().migrad()
self.end_parameters = m.values.to_dict()
self.fcn = m.fval
self.error_parameters = m.errors.to_dict()
def predict(self, unit_cam, fit_params):
"""
Call the fitting procedure and fill the results.
Parameters
----------
unit_cam: astropy.units.unit
Unit used for the camera geometry and for spatial variable in the fit
fit_params: array
Parameters used to compute the likelihood but not fitted
Returns
----------
container: DL1LikelihoodParametersContainer
Filled parameter container
"""
container = DL1LikelihoodParametersContainer(lhfit_call_status=1)
try:
self.fit(fit_params)
container.lhfit_TS = self.fcn
container.lhfit_x = (self.end_parameters['x_cm'] * unit_cam).to(
u.m)
container.lhfit_x_uncertainty = (self.error_parameters['x_cm'] *
unit_cam).to(u.m)
container.lhfit_y = (self.end_parameters['y_cm'] * unit_cam).to(
u.m)
container.lhfit_y_uncertainty = (self.error_parameters['y_cm'] *
unit_cam).to(u.m)
container.lhfit_r = np.sqrt(container.lhfit_x**2 +
container.lhfit_y**2)
container.lhfit_phi = np.arctan2(container.lhfit_y,
container.lhfit_x)
if self.end_parameters['psi'] > np.pi:
self.end_parameters['psi'] -= 2 * np.pi
if self.end_parameters['psi'] < -np.pi:
self.end_parameters['psi'] += 2 * np.pi
container.lhfit_psi = self.end_parameters['psi'] * u.rad
container.lhfit_psi_uncertainty = self.error_parameters[
'psi'] * u.rad
length_asy = 1 + self.end_parameters['rl'] if self.end_parameters[
'rl'] >= 0 else 1 / (1 - self.end_parameters['rl'])
lhfit_length = ((
(1.0 + length_asy) * self.end_parameters['length'] / 2.0) *
unit_cam).to(u.deg)
container.lhfit_length = lhfit_length
lhfit_length_rel_err = self.error_parameters[
'length'] / self.end_parameters['length']
# We assume that the relative error is the same in the fitted and saved unit
container.lhfit_length_uncertainty = lhfit_length_rel_err * container.lhfit_length
container.lhfit_width = self.end_parameters[
'wl'] * container.lhfit_length
container.lhfit_time_gradient = self.end_parameters['v']
container.lhfit_time_gradient_uncertainty = self.error_parameters[
'v']
container.lhfit_ref_time = self.end_parameters['t_cm']
container.lhfit_ref_time_uncertainty = self.error_parameters[
't_cm']
container.lhfit_wl = u.Quantity(self.end_parameters['wl'])
container.lhfit_wl_uncertainty = u.Quantity(
self.error_parameters['wl'])
container.lhfit_intensity = self.end_parameters['charge']
container.lhfit_intensity_uncertainty = self.error_parameters[
'charge']
container.lhfit_log_intensity = np.log10(container.lhfit_intensity)
container.lhfit_t_68 = container.lhfit_length.value * container.lhfit_time_gradient
container.lhfit_area = container.lhfit_length * container.lhfit_width
container.lhfit_length_asymmetry = self.end_parameters['rl']
container.lhfit_length_asymmetry_uncertainty = self.error_parameters[
'rl']
except ZeroDivisionError:
# TODO Check occurrence rate and solve
container = DL1LikelihoodParametersContainer(lhfit_call_status=-1)
logger.error(
'ZeroDivisionError encounter during the fitting procedure, skipping event.'
)
return container
def __str__(self):
"""
Define the print format of TimeWaveformFitter objects.
Returns
-------
str: string
Contains the starting and bound parameters used for the fit,
and the end results with errors and associated log-likelihood
in readable format.
"""
s = 'Event processed\n'
s += 'Start parameters :\n\t{}\n'.format(self.start_parameters)
s += 'Bound parameters :\n\t{}\n'.format(self.bound_parameters)
s += 'End parameters :\n\t{}\n'.format(self.end_parameters)
s += 'Error parameters :\n\t{}\n'.format(self.error_parameters)
s += '-2Log-Likelihood :\t{}'.format(self.fcn)
return s
@staticmethod
def log_likelihood(*args, fit_params, **kwargs):
"""Compute the log-likelihood used in the fitting procedure."""
llh = log_pdf(*args, *fit_params, **kwargs)
return np.sum(llh)