def __init__(self,
input_url,
config=None,
parent=None,
gain_selector=None,
**kwargs):
"""
EventSource for simtelarray files using the pyeventio library.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
gain_selector : ctapipe.calib.camera.gainselection.GainSelector
The GainSelector to use. If None, then ThresholdGainSelector will be used.
kwargs
"""
super().__init__(input_url=input_url,
config=config,
parent=parent,
**kwargs)
self._camera_cache = {}
self.file_ = SimTelFile(
self.input_url.expanduser(),
allowed_telescopes=self.allowed_tels,
skip_calibration=self.skip_calibration_events,
zcat=not self.back_seekable,
)
if self.back_seekable and self.is_stream:
raise IOError(
"back seekable was required but not possible for inputfile")
self._subarray_info = self.prepare_subarray_info(
self.file_.telescope_descriptions, self.file_.header)
self._mc_header = self._parse_mc_header()
self.start_pos = self.file_.tell()
# Waveforms from simtelarray have both gain channels
# Gain selection is performed by this EventSource to produce R1 waveforms
if gain_selector is None:
gain_selector = ThresholdGainSelector(parent=self)
self.gain_selector = gain_selector
def __init__(self, config=None, parent=None, **kwargs):
super().__init__(config=config, parent=parent, **kwargs)
self.metadata['is_simulation'] = True
# traitlets creates an empty set as default,
# which ctapipe treats as no restriction on the telescopes
# but eventio treats an emty set as "no telescopes allowed"
# so we explicitly pass None in that case
self.file_ = SimTelFile(self.input_url,
allowed_telescopes=self.allowed_tels
if self.allowed_tels else None,
skip_calibration=self.skip_calibration_events)
self._subarray_info = self.prepare_subarray_info(
self.file_.telescope_descriptions, self.file_.header)
self.start_pos = self.file_.tell()
def __init__(self, config=None, tool=None, **kwargs):
super().__init__(config=config, tool=tool, **kwargs)
self.metadata['is_simulation'] = True
self.file_ = SimTelFile(self.input_url)
self._subarray_info = self.prepare_subarray_info(
self.file_.telescope_descriptions, self.file_.header)
def process_mc(simtelfile, dl2_file, mc_type):
"""
Process the MC simulated and reconstructed to extract the relevant
parameters to compute the sensitivity
Paramenters
---------
simtel: simtelarray file
dl2_file: `pandas.DataFrame` dl2 parameters
mc_type: 'string' type of particle
Returns
---------
gammaness: `numpy.ndarray`
angdist2: `numpy.ndarray` angular distance squared
e_reco: `numpy.ndarray` reconstructed energies
n_reco: `int` number of reconstructed events
mc_par: `dict` with simulated parameters
"""
source = SimTelFile(simtelfile)
sim_par = read_sim_par(source)
events = pd.read_hdf(dl2_file)
e_reco = 10**events.mc_energy.to_numpy() * u.GeV
gammaness = events.gammaness
#Get source position in radians
focal_length = source.telescope_descriptions[1]['camera_settings']['focal_length'] * u.m
# If the particle is a gamma ray, it returns the squared angular distance
# from the reconstructed gamma-ray position and the simulated incoming position
if mc_type=='gamma':
events = events[events.mc_type==0]
alt2 = events.mc_alt
az2 = np.arctan(np.tan(events.mc_az))
# If the particle is not a gamma-ray (diffuse protons/electrons), it returns
# the squared angular distance of the reconstructed position w.r.t. the
# center of the camera
else:
events = events[events.mc_type!=0]
alt2 = events.mc_alt_tel
az2 = np.arctan(np.tan(events.mc_az_tel))
src_pos_reco = reco_source_position_sky(events.x.values * u.m,
events.y.values * u.m,
events.disp_dx_rec.values * u.m,
events.disp_dy_rec.values * u.m,
focal_length,
events.mc_alt_tel.values * u.rad,
events.mc_az_tel.values * u.rad)
alt1 = src_pos_reco.alt.rad
az1 = np.arctan(np.tan(src_pos_reco.az.rad))
angdist2 = (angular_separation(az1, alt1, az2, alt2).to_numpy() * u.rad)**2
return gammaness, angdist2.to(u.deg**2), e_reco, sim_par
def process_mc(simtelfile, dl2_file):
"""
Process the MC simulated and reconstructed to extract the relevant
parameters to compute the sensitivity
Paramenters
---------
simtel: simtelarray file
dl2_file: `pandas.DataFrame` dl2 parameters
Returns
---------
gammaness: `numpy.ndarray`
theta2: `numpy.ndarray`
e_reco: `numpy.ndarray` reconstructed energies
n_reco: `int` number of reconstructed events
mc_par: `dict` with simulated parameters
"""
source = SimTelFile(simtelfile)
sim_par = read_sim_par(source)
events = pd.read_hdf(dl2_file)
e_reco = 10**events.mc_energy * u.GeV
# n_reco = e_reco.shape[0]
gammaness = events.gammaness
theta2 = (events.src_x - events.src_x_rec)**2 + \
(events.src_y - events.src_y_rec)**2 * u.deg**2
return gammaness, theta2, e_reco, sim_par
def __init__(self, config=None, parent=None, gain_selector=None, **kwargs):
"""
EventSource for simtelarray files using the pyeventio library.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
gain_selector : ctapipe.calib.camera.gainselection.GainSelector
The GainSelector to use. If None, then ThresholdGainSelector will be used.
kwargs
"""
super().__init__(config=config, parent=parent, **kwargs)
self.metadata['is_simulation'] = True
self._camera_cache = {}
# traitlets creates an empty set as default,
# which ctapipe treats as no restriction on the telescopes
# but eventio treats an emty set as "no telescopes allowed"
# so we explicitly pass None in that case
self.file_ = SimTelFile(
Path(self.input_url).expanduser(),
allowed_telescopes=set(self.allowed_tels)
if self.allowed_tels else None,
skip_calibration=self.skip_calibration_events,
zcat=not self.back_seekable,
)
if self.back_seekable and self.is_stream:
raise IOError(
'back seekable was required but not possible for inputfile')
self._subarray_info = self.prepare_subarray_info(
self.file_.telescope_descriptions, self.file_.header)
self.start_pos = self.file_.tell()
# Waveforms from simtelarray have both gain channels
# Gain selection is performed by this EventSource to produce R1 waveforms
if gain_selector is None:
gain_selector = ThresholdGainSelector(parent=self)
self.gain_selector = gain_selector
def __init__(self, config=None, parent=None, **kwargs):
super().__init__(config=config, parent=parent, **kwargs)
self.metadata['is_simulation'] = True
self._camera_cache = {}
# traitlets creates an empty set as default,
# which ctapipe treats as no restriction on the telescopes
# but eventio treats an emty set as "no telescopes allowed"
# so we explicitly pass None in that case
self.file_ = SimTelFile(
self.input_url,
allowed_telescopes=set(self.allowed_tels) if self.allowed_tels else None,
skip_calibration=self.skip_calibration_events,
zcat=not self.back_seekable,
)
if self.back_seekable and self.is_stream:
raise IOError('back seekable was required but not possible for inputfile')
self._subarray_info = self.prepare_subarray_info(
self.file_.telescope_descriptions,
self.file_.header
)
self.start_pos = self.file_.tell()
def get_spectral_w_pars(filename):
"""
Return parameters required to calculate spectral weighting of an event
Parameters
----------
filename: string, simtelarray file
Returns
-------
array of parameters
"""
particle = utils.guess_type(filename)
source = SimTelFile(filename)
emin, emax = source.mc_run_headers[0]['E_range'] * 1e3 #GeV
spectral_index = source.mc_run_headers[0]['spectral_index']
num_showers = source.mc_run_headers[0]['num_showers']
num_use = source.mc_run_headers[0]['num_use']
Simulated_Events = num_showers * num_use
Max_impact = source.mc_run_headers[0]['core_range'][1] * 1e2 #cm
Area_sim = np.pi * math.pow(Max_impact, 2)
cone = source.mc_run_headers[0]['viewcone'][1]
cone = cone * np.pi / 180
if (cone == 0):
Omega = 1
else:
Omega = 2 * np.pi * (1 - np.cos(cone))
if particle == 'proton':
K_w = 9.6e-11 # GeV^-1 cm^-2 s^-1
index_w = -2.7
E0 = 1000. # GeV
if particle == 'gamma':
K_w = 2.83e-11 # GeV^-1 cm^-2 s^-1
index_w = -2.62
E0 = 1000. # GeV
K = Simulated_Events * (1 + spectral_index) / (emax**(1 + spectral_index) -
emin**(1 + spectral_index))
Int_e1_e2 = K * E0**spectral_index
N_ = Int_e1_e2 * (emax**(index_w + 1) -
emin**(index_w + 1)) / (E0**index_w) / (index_w + 1)
R = K_w * Area_sim * Omega * (emax**(index_w + 1) - emin**(index_w + 1)
) / (E0**index_w) / (index_w + 1)
return E0, spectral_index, index_w, R, N_
def __init__(self, config=None, parent=None, **kwargs):
super().__init__(config=config, parent=parent, **kwargs)
self.metadata['is_simulation'] = True
# traitlets creates an empty set as default,
# which ctapipe treats as no restriction on the telescopes
# but eventio treats an emty set as "no telescopes allowed"
# so we explicitly pass None in that case
self.file_ = SimTelFile(
self.input_url,
allowed_telescopes=self.allowed_tels if self.allowed_tels else None,
skip_calibration=self.skip_calibration_events
)
self._subarray_info = self.prepare_subarray_info(
self.file_.telescope_descriptions,
self.file_.header
)
self.start_pos = self.file_.tell()
import time
from pprint import pprint
from eventio.simtel.simtelfile import SimTelFile
from argparse import ArgumentParser
parser = ArgumentParser()
parser.add_argument('inputfile')
parser.add_argument('-p', '--print', action='store_true')
args = parser.parse_args()
start_time = time.time()
with SimTelFile(args.inputfile) as f:
for i, event in enumerate(f):
print('Event count: {: 04d}, E = {:8.3f} Tev, #Telescopes={: 3d}'.format(
i, event['mc_shower']['energy'], len(event['telescope_events'])
))
if args.print:
pprint(event)
print(' Duration:', time.time() - start_time)
class SimTelEventSource(EventSource):
skip_calibration_events = Bool(True, help='Skip calibration events').tag(config=True)
def __init__(self, config=None, parent=None, **kwargs):
super().__init__(config=config, parent=parent, **kwargs)
self.metadata['is_simulation'] = True
# traitlets creates an empty set as default,
# which ctapipe treats as no restriction on the telescopes
# but eventio treats an emty set as "no telescopes allowed"
# so we explicitly pass None in that case
self.file_ = SimTelFile(
self.input_url,
allowed_telescopes=self.allowed_tels if self.allowed_tels else None,
skip_calibration=self.skip_calibration_events
)
self._subarray_info = self.prepare_subarray_info(
self.file_.telescope_descriptions,
self.file_.header
)
self.start_pos = self.file_.tell()
@staticmethod
def prepare_subarray_info(telescope_descriptions, header):
"""
Constructs a SubarrayDescription object from the
``telescope_descriptions`` given by ``SimTelFile``
Parameters
----------
telescope_descriptions: dict
telescope descriptions as given by ``SimTelFile.telescope_descriptions``
header: dict
header as returned by ``SimTelFile.header``
Returns
-------
SubarrayDescription :
instrumental information
"""
tel_descriptions = {} # tel_id : TelescopeDescription
tel_positions = {} # tel_id : TelescopeDescription
for tel_id, telescope_description in telescope_descriptions.items():
cam_settings = telescope_description['camera_settings']
n_pixels = cam_settings['n_pixels']
focal_length = u.Quantity(cam_settings['focal_length'], u.m)
try:
telescope = guess_telescope(n_pixels, focal_length)
except ValueError:
telescope = UNKNOWN_TELESCOPE
pixel_shape = cam_settings['pixel_shape'][0]
try:
pix_type, pix_rotation = CameraGeometry.simtel_shape_to_type(pixel_shape)
except ValueError:
warnings.warn(
f'Unkown pixel_shape {pixel_shape} for tel_id {tel_id}',
UnknownPixelShapeWarning,
)
pix_type = 'hexagon'
pix_rotation = '0d'
camera = CameraGeometry(
telescope.camera_name,
pix_id=np.arange(n_pixels),
pix_x=u.Quantity(cam_settings['pixel_x'], u.m),
pix_y=u.Quantity(cam_settings['pixel_y'], u.m),
pix_area=u.Quantity(cam_settings['pixel_area'], u.m**2),
pix_type=pix_type,
pix_rotation=pix_rotation,
cam_rotation=-Angle(cam_settings['cam_rot'], u.rad),
apply_derotation=True,
)
optics = OpticsDescription(
name=telescope.name,
num_mirrors=cam_settings['n_mirrors'],
equivalent_focal_length=focal_length,
mirror_area=u.Quantity(cam_settings['mirror_area'], u.m**2),
num_mirror_tiles=cam_settings['n_mirrors'],
)
tel_descriptions[tel_id] = TelescopeDescription(
name=telescope.name,
type=telescope.type,
camera=camera,
optics=optics,
)
tel_idx = np.where(header['tel_id'] == tel_id)[0][0]
tel_positions[tel_id] = header['tel_pos'][tel_idx] * u.m
return SubarrayDescription(
"MonteCarloArray",
tel_positions=tel_positions,
tel_descriptions=tel_descriptions,
)
@staticmethod
def is_compatible(file_path):
return is_eventio(file_path)
def _generator(self):
if self.file_.tell() > self.start_pos:
self.file_._next_header_pos = 0
warnings.warn('Backseeking to start of file.')
try:
yield from self.__generator()
except EOFError:
msg = 'EOFError reading from "{input_url}". Might be truncated'.format(
input_url=self.input_url
)
self.log.warning(msg)
warnings.warn(msg)
def __generator(self):
data = DataContainer()
data.meta['origin'] = 'hessio'
data.meta['input_url'] = self.input_url
data.meta['max_events'] = self.max_events
for counter, array_event in enumerate(self.file_):
# next lines are just for debugging
self.array_event = array_event
data.event_type = array_event['type']
# calibration events do not have an event id
if data.event_type == 'calibration':
event_id = -1
else:
event_id = array_event['event_id']
data.inst.subarray = self._subarray_info
obs_id = self.file_.header['run']
tels_with_data = set(array_event['telescope_events'].keys())
data.count = counter
data.r0.obs_id = obs_id
data.r0.event_id = event_id
data.r0.tels_with_data = tels_with_data
data.r1.obs_id = obs_id
data.r1.event_id = event_id
data.r1.tels_with_data = tels_with_data
data.dl0.obs_id = obs_id
data.dl0.event_id = event_id
data.dl0.tels_with_data = tels_with_data
# handle telescope filtering by taking the intersection of
# tels_with_data and allowed_tels
if len(self.allowed_tels) > 0:
selected = tels_with_data & self.allowed_tels
if len(selected) == 0:
continue # skip event
data.r0.tels_with_data = selected
data.r1.tels_with_data = selected
data.dl0.tels_with_data = selected
trigger_information = array_event['trigger_information']
data.trig.tels_with_trigger = trigger_information['triggered_telescopes']
time_s, time_ns = trigger_information['gps_time']
data.trig.gps_time = Time(time_s * u.s, time_ns * u.ns,
format='unix', scale='utc')
if data.event_type == 'data':
self.fill_mc_information(data, array_event)
# this should be done in a nicer way to not re-allocate the
# data each time (right now it's just deleted and garbage
# collected)
data.r0.tel.clear()
data.r1.tel.clear()
data.dl0.tel.clear()
data.dl1.tel.clear()
data.mc.tel.clear() # clear the previous telescopes
telescope_events = array_event['telescope_events']
tracking_positions = array_event['tracking_positions']
for tel_id, telescope_event in telescope_events.items():
tel_index = self.file_.header['tel_id'].tolist().index(tel_id)
telescope_description = self.file_.telescope_descriptions[tel_id]
adc_samples = telescope_event.get('adc_samples')
if adc_samples is None:
adc_samples = telescope_event['adc_sums'][:, :, np.newaxis]
r0 = data.r0.tel[tel_id]
r0.waveform = adc_samples
r0.num_samples = adc_samples.shape[-1]
# We should not calculate stuff in an event source
# if this is not needed, we calculate it for nothing
r0.image = adc_samples.sum(axis=-1)
pixel_lists = telescope_event['pixel_lists']
r0.num_trig_pix = pixel_lists.get(0, {'pixels': 0})['pixels']
if r0.num_trig_pix > 0:
r0.trig_pix_id = pixel_lists[0]['pixel_list']
pixel_settings = telescope_description['pixel_settings']
n_pixel = r0.waveform.shape[-2]
mc = data.mc.tel[tel_id]
mc.dc_to_pe = array_event['laser_calibrations'][tel_id]['calib']
mc.pedestal = array_event['camera_monitorings'][tel_id]['pedestal']
mc.reference_pulse_shape = pixel_settings['ref_shape'].astype('float64')
mc.meta['refstep'] = float(pixel_settings['ref_step'])
mc.time_slice = float(pixel_settings['time_slice'])
mc.photo_electron_image = (
array_event
.get('photoelectrons', {})
.get(tel_index, {})
.get('photoelectrons', np.zeros(n_pixel, dtype='float32'))
)
tracking_position = tracking_positions[tel_id]
mc.azimuth_raw = tracking_position['azimuth_raw']
mc.altitude_raw = tracking_position['altitude_raw']
mc.azimuth_cor = tracking_position.get('azimuth_cor', 0)
mc.altitude_cor = tracking_position.get('altitude_cor', 0)
yield data
def fill_mc_information(self, data, array_event):
mc_event = array_event['mc_event']
mc_shower = array_event['mc_shower']
data.mc.energy = mc_shower['energy'] * u.TeV
data.mc.alt = Angle(mc_shower['altitude'], u.rad)
data.mc.az = Angle(mc_shower['azimuth'], u.rad)
data.mc.core_x = mc_event['xcore'] * u.m
data.mc.core_y = mc_event['ycore'] * u.m
first_int = mc_shower['h_first_int'] * u.m
data.mc.h_first_int = first_int
data.mc.x_max = mc_shower['xmax'] * u.g / (u.cm**2)
data.mc.shower_primary_id = mc_shower['primary_id']
# mc run header data
data.mcheader.run_array_direction = Angle(
self.file_.header['direction'] * u.rad
)
mc_run_head = self.file_.mc_run_headers[-1]
data.mcheader.corsika_version = mc_run_head['shower_prog_vers']
data.mcheader.simtel_version = mc_run_head['detector_prog_vers']
data.mcheader.energy_range_min = mc_run_head['E_range'][0] * u.TeV
data.mcheader.energy_range_max = mc_run_head['E_range'][1] * u.TeV
data.mcheader.prod_site_B_total = mc_run_head['B_total'] * u.uT
data.mcheader.prod_site_B_declination = Angle(
mc_run_head['B_declination'] * u.rad)
data.mcheader.prod_site_B_inclination = Angle(
mc_run_head['B_inclination'] * u.rad)
data.mcheader.prod_site_alt = mc_run_head['obsheight'] * u.m
data.mcheader.spectral_index = mc_run_head['spectral_index']
data.mcheader.shower_prog_start = mc_run_head['shower_prog_start']
data.mcheader.shower_prog_id = mc_run_head['shower_prog_id']
data.mcheader.detector_prog_start = mc_run_head['detector_prog_start']
data.mcheader.detector_prog_id = mc_run_head['detector_prog_id']
data.mcheader.num_showers = mc_run_head['n_showers']
data.mcheader.shower_reuse = mc_run_head['n_use']
data.mcheader.max_alt = mc_run_head['alt_range'][1] * u.rad
data.mcheader.min_alt = mc_run_head['alt_range'][0] * u.rad
data.mcheader.max_az = mc_run_head['az_range'][1] * u.rad
data.mcheader.min_az = mc_run_head['az_range'][0] * u.rad
data.mcheader.diffuse = mc_run_head['diffuse']
data.mcheader.max_viewcone_radius = mc_run_head['viewcone'][1] * u.deg
data.mcheader.min_viewcone_radius = mc_run_head['viewcone'][0] * u.deg
data.mcheader.max_scatter_range = mc_run_head['core_range'][1] * u.m
data.mcheader.min_scatter_range = mc_run_head['core_range'][0] * u.m
data.mcheader.core_pos_mode = mc_run_head['core_pos_mode']
data.mcheader.injection_height = mc_run_head['injection_height'] * u.m
data.mcheader.atmosphere = mc_run_head['atmosphere']
data.mcheader.corsika_iact_options = mc_run_head['corsika_iact_options']
data.mcheader.corsika_low_E_model = mc_run_head['corsika_low_E_model']
data.mcheader.corsika_high_E_model = mc_run_head['corsika_high_E_model']
data.mcheader.corsika_bunchsize = mc_run_head['corsika_bunchsize']
data.mcheader.corsika_wlen_min = mc_run_head['corsika_wlen_min'] * u.nm
data.mcheader.corsika_wlen_max = mc_run_head['corsika_wlen_max'] * u.nm
data.mcheader.corsika_low_E_detail = mc_run_head['corsika_low_E_detail']
data.mcheader.corsika_high_E_detail = mc_run_head['corsika_high_E_detail']
class SimTelEventSource(EventSource):
skip_calibration_events = Bool(True, help="Skip calibration events").tag(
config=True
)
back_seekable = Bool(
False,
help=(
"Require the event source to be backwards seekable."
" This will reduce in slower read speed for gzipped files"
" and is not possible for zstd compressed files"
),
).tag(config=True)
focal_length_choice = CaselessStrEnum(
["nominal", "effective"],
default_value="nominal",
help=(
"if both nominal and effective focal lengths are available in the "
"SimTelArray file, which one to use when constructing the "
"SubarrayDescription (which will be used in CameraFrame to TelescopeFrame "
"coordinate transforms. The 'nominal' focal length is the one used during "
"the simulation, the 'effective' focal length is computed using specialized "
"ray-tracing from a point light source"
),
).tag(config=True)
def __init__(
self, input_url, config=None, parent=None, gain_selector=None, **kwargs
):
"""
EventSource for simtelarray files using the pyeventio library.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
gain_selector : ctapipe.calib.camera.gainselection.GainSelector
The GainSelector to use. If None, then ThresholdGainSelector will be used.
kwargs
"""
super().__init__(input_url=input_url, config=config, parent=parent, **kwargs)
self._camera_cache = {}
self.file_ = SimTelFile(
self.input_url.expanduser(),
allowed_telescopes=self.allowed_tels,
skip_calibration=self.skip_calibration_events,
zcat=not self.back_seekable,
)
if self.back_seekable and self.is_stream:
raise IOError("back seekable was required but not possible for inputfile")
self._subarray_info = self.prepare_subarray_info(
self.file_.telescope_descriptions, self.file_.header
)
self._mc_header = self._parse_mc_header()
self.start_pos = self.file_.tell()
# Waveforms from simtelarray have both gain channels
# Gain selection is performed by this EventSource to produce R1 waveforms
if gain_selector is None:
gain_selector = ThresholdGainSelector(parent=self)
self.gain_selector = gain_selector
@observe("allowed_tels")
def _observe_allowed_tels(self, change):
# this can run in __init__ before file_ is created
if hasattr(self, "file_"):
allowed_tels = set(self.allowed_tels) if self.allowed_tels else None
self.file_.allowed_telescopes = allowed_tels
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def close(self):
self.file_.close()
@property
def is_simulation(self):
return True
@property
def datalevels(self):
return (DataLevel.R0, DataLevel.R1, DataLevel.DL0)
@property
def obs_id(self):
return self.file_.header["run"]
@property
def mc_header(self):
return self._mc_header
@property
def is_stream(self):
return not isinstance(self.file_._filehandle, (BufferedReader, GzipFile))
def prepare_subarray_info(self, telescope_descriptions, header):
"""
Constructs a SubarrayDescription object from the
``telescope_descriptions`` given by ``SimTelFile``
Parameters
----------
telescope_descriptions: dict
telescope descriptions as given by ``SimTelFile.telescope_descriptions``
header: dict
header as returned by ``SimTelFile.header``
Returns
-------
SubarrayDescription :
instrumental information
"""
tel_descriptions = {} # tel_id : TelescopeDescription
tel_positions = {} # tel_id : TelescopeDescription
for tel_id, telescope_description in telescope_descriptions.items():
cam_settings = telescope_description["camera_settings"]
pixel_settings = telescope_description["pixel_settings"]
n_pixels = cam_settings["n_pixels"]
focal_length = u.Quantity(cam_settings["focal_length"], u.m)
if self.focal_length_choice == "effective":
try:
focal_length = u.Quantity(
cam_settings["effective_focal_length"], u.m
)
except KeyError as err:
raise RuntimeError(
f"the SimTelEventSource option 'focal_length_choice' was set to "
f"{self.focal_length_choice}, but the effective focal length "
f"was not present in the file. ({err})"
)
try:
telescope = guess_telescope(n_pixels, focal_length)
except ValueError:
telescope = UNKNOWN_TELESCOPE
camera = self._camera_cache.get(telescope.camera_name)
if camera is None:
camera = build_camera(cam_settings, pixel_settings, telescope)
self._camera_cache[telescope.camera_name] = camera
optics = OpticsDescription(
name=telescope.name,
num_mirrors=telescope.n_mirrors,
equivalent_focal_length=focal_length,
mirror_area=u.Quantity(cam_settings["mirror_area"], u.m ** 2),
num_mirror_tiles=cam_settings["n_mirrors"],
)
tel_descriptions[tel_id] = TelescopeDescription(
name=telescope.name,
tel_type=telescope.type,
optics=optics,
camera=camera,
)
tel_idx = np.where(header["tel_id"] == tel_id)[0][0]
tel_positions[tel_id] = header["tel_pos"][tel_idx] * u.m
return SubarrayDescription(
"MonteCarloArray",
tel_positions=tel_positions,
tel_descriptions=tel_descriptions,
)
@staticmethod
def is_compatible(file_path):
return is_eventio(Path(file_path).expanduser())
@property
def subarray(self):
return self._subarray_info
def _generator(self):
if self.file_.tell() > self.start_pos:
self.file_._next_header_pos = 0
warnings.warn("Backseeking to start of file.")
try:
yield from self._generate_events()
except EOFError:
msg = 'EOFError reading from "{input_url}". Might be truncated'.format(
input_url=self.input_url
)
self.log.warning(msg)
warnings.warn(msg)
def _generate_events(self):
data = EventAndMonDataContainer()
data.meta["origin"] = "hessio"
data.meta["input_url"] = self.input_url
data.meta["max_events"] = self.max_events
data.mcheader = self._mc_header
self._fill_array_pointing(data)
for counter, array_event in enumerate(self.file_):
event_id = array_event.get("event_id", -1)
obs_id = self.file_.header["run"]
tels_with_data = set(array_event["telescope_events"].keys())
data.count = counter
data.index.obs_id = obs_id
data.index.event_id = event_id
data.r0.tels_with_data = tels_with_data
data.r1.tels_with_data = tels_with_data
data.dl0.tels_with_data = tels_with_data
self._fill_trigger_info(data, array_event)
if data.trigger.event_type == EventType.SUBARRAY:
self._fill_mc_event_information(data, array_event)
# this should be done in a nicer way to not re-allocate the
# data each time (right now it's just deleted and garbage
# collected)
data.r0.tel.clear()
data.r1.tel.clear()
data.dl0.tel.clear()
data.dl1.tel.clear()
data.mc.tel.clear()
data.pointing.tel.clear()
telescope_events = array_event["telescope_events"]
tracking_positions = array_event["tracking_positions"]
for tel_id, telescope_event in telescope_events.items():
adc_samples = telescope_event.get("adc_samples")
if adc_samples is None:
adc_samples = telescope_event["adc_sums"][:, :, np.newaxis]
n_gains, n_pixels, n_samples = adc_samples.shape
mc = data.mc.tel[tel_id]
mc.dc_to_pe = array_event["laser_calibrations"][tel_id]["calib"]
mc.pedestal = array_event["camera_monitorings"][tel_id]["pedestal"]
mc.true_image = (
array_event.get("photoelectrons", {})
.get(tel_id - 1, {})
.get("photoelectrons", np.zeros(n_pixels, dtype="float32"))
)
self._fill_event_pointing(
data.pointing.tel[tel_id], mc, tracking_positions[tel_id],
)
r0 = data.r0.tel[tel_id]
r1 = data.r1.tel[tel_id]
r0.waveform = adc_samples
r1.waveform, r1.selected_gain_channel = apply_simtel_r1_calibration(
adc_samples, mc.pedestal, mc.dc_to_pe, self.gain_selector
)
pixel_lists = telescope_event["pixel_lists"]
r0.num_trig_pix = pixel_lists.get(0, {"pixels": 0})["pixels"]
if r0.num_trig_pix > 0:
r0.trig_pix_id = pixel_lists[0]["pixel_list"]
yield data
@staticmethod
def _fill_event_pointing(pointing, mc, tracking_position):
mc.azimuth_raw = tracking_position["azimuth_raw"]
mc.altitude_raw = tracking_position["altitude_raw"]
mc.azimuth_cor = tracking_position.get("azimuth_cor", np.nan)
mc.altitude_cor = tracking_position.get("altitude_cor", np.nan)
# take pointing corrected position if available
if np.isnan(mc.azimuth_cor):
pointing.azimuth = u.Quantity(mc.azimuth_raw, u.rad)
else:
pointing.azimuth = u.Quantity(mc.azimuth_cor, u.rad)
# take pointing corrected position if available
if np.isnan(mc.altitude_cor):
pointing.altitude = u.Quantity(mc.altitude_raw, u.rad)
else:
pointing.altitude = u.Quantity(mc.altitude_cor, u.rad)
@staticmethod
def _fill_trigger_info(data, array_event):
trigger = array_event["trigger_information"]
if array_event["type"] == "data":
data.trigger.event_type = EventType.SUBARRAY
elif array_event["type"] == "calibration":
# if using eventio >= 1.1.1, we can use the calibration_type
data.trigger.event_type = SIMTEL_TO_CTA_EVENT_TYPE.get(
array_event.get("calibration_type", -1), EventType.OTHER_CALIBRATION
)
else:
data.trigger.event_type = EventType.UNKNOWN
data.trigger.tels_with_trigger = trigger["triggered_telescopes"]
data.trigger.time = parse_simtel_time(trigger["gps_time"])
for tel_id, time in zip(
trigger["triggered_telescopes"], trigger["trigger_times"]
):
# time is relative to central trigger in nano seconds
data.trigger.tel[tel_id].time = data.trigger.time + u.Quantity(time, u.ns)
def _fill_array_pointing(self, data):
if self.file_.header["tracking_mode"] == 0:
az, alt = self.file_.header["direction"]
data.pointing.array_altitude = u.Quantity(alt, u.rad)
data.pointing.array_azimuth = u.Quantity(az, u.rad)
else:
ra, dec = self.file_.header["direction"]
data.pointing.array_ra = u.Quantity(ra, u.rad)
data.pointing.array_dec = u.Quantity(dec, u.rad)
def _parse_mc_header(self):
mc_run_head = self.file_.mc_run_headers[-1]
return MCHeaderContainer(
corsika_version=mc_run_head["shower_prog_vers"],
simtel_version=mc_run_head["detector_prog_vers"],
energy_range_min=mc_run_head["E_range"][0] * u.TeV,
energy_range_max=mc_run_head["E_range"][1] * u.TeV,
prod_site_B_total=mc_run_head["B_total"] * u.uT,
prod_site_B_declination=Angle(mc_run_head["B_declination"], u.rad,),
prod_site_B_inclination=Angle(mc_run_head["B_inclination"], u.rad,),
prod_site_alt=mc_run_head["obsheight"] * u.m,
spectral_index=mc_run_head["spectral_index"],
shower_prog_start=mc_run_head["shower_prog_start"],
shower_prog_id=mc_run_head["shower_prog_id"],
detector_prog_start=mc_run_head["detector_prog_start"],
detector_prog_id=mc_run_head["detector_prog_id"],
num_showers=mc_run_head["n_showers"],
shower_reuse=mc_run_head["n_use"],
max_alt=mc_run_head["alt_range"][1] * u.rad,
min_alt=mc_run_head["alt_range"][0] * u.rad,
max_az=mc_run_head["az_range"][1] * u.rad,
min_az=mc_run_head["az_range"][0] * u.rad,
diffuse=mc_run_head["diffuse"],
max_viewcone_radius=mc_run_head["viewcone"][1] * u.deg,
min_viewcone_radius=mc_run_head["viewcone"][0] * u.deg,
max_scatter_range=mc_run_head["core_range"][1] * u.m,
min_scatter_range=mc_run_head["core_range"][0] * u.m,
core_pos_mode=mc_run_head["core_pos_mode"],
injection_height=mc_run_head["injection_height"] * u.m,
atmosphere=mc_run_head["atmosphere"],
corsika_iact_options=mc_run_head["corsika_iact_options"],
corsika_low_E_model=mc_run_head["corsika_low_E_model"],
corsika_high_E_model=mc_run_head["corsika_high_E_model"],
corsika_bunchsize=mc_run_head["corsika_bunchsize"],
corsika_wlen_min=mc_run_head["corsika_wlen_min"] * u.nm,
corsika_wlen_max=mc_run_head["corsika_wlen_max"] * u.nm,
corsika_low_E_detail=mc_run_head["corsika_low_E_detail"],
corsika_high_E_detail=mc_run_head["corsika_high_E_detail"],
run_array_direction=Angle(self.file_.header["direction"] * u.rad),
)
@staticmethod
def _fill_mc_event_information(data, array_event):
mc_event = array_event["mc_event"]
mc_shower = array_event["mc_shower"]
data.mc.energy = u.Quantity(mc_shower["energy"], u.TeV)
data.mc.alt = Angle(mc_shower["altitude"], u.rad)
data.mc.az = Angle(mc_shower["azimuth"], u.rad)
data.mc.core_x = u.Quantity(mc_event["xcore"], u.m)
data.mc.core_y = u.Quantity(mc_event["ycore"], u.m)
first_int = u.Quantity(mc_shower["h_first_int"], u.m)
data.mc.h_first_int = first_int
data.mc.x_max = u.Quantity(mc_shower["xmax"], X_MAX_UNIT)
data.mc.shower_primary_id = mc_shower["primary_id"]
class SimTelEventSource(EventSource):
skip_calibration_events = Bool(
True, help='Skip calibration events').tag(config=True)
back_seekable = Bool(
False,
help=('Require the event source to be backwards seekable.'
' This will reduce in slower read speed for gzipped files'
' and is not possible for zstd compressed files')).tag(
config=True)
def __init__(self, config=None, parent=None, gain_selector=None, **kwargs):
"""
EventSource for simtelarray files using the pyeventio library.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
gain_selector : ctapipe.calib.camera.gainselection.GainSelector
The GainSelector to use. If None, then ThresholdGainSelector will be used.
kwargs
"""
super().__init__(config=config, parent=parent, **kwargs)
self.metadata['is_simulation'] = True
self._camera_cache = {}
# traitlets creates an empty set as default,
# which ctapipe treats as no restriction on the telescopes
# but eventio treats an emty set as "no telescopes allowed"
# so we explicitly pass None in that case
self.file_ = SimTelFile(
Path(self.input_url).expanduser(),
allowed_telescopes=set(self.allowed_tels)
if self.allowed_tels else None,
skip_calibration=self.skip_calibration_events,
zcat=not self.back_seekable,
)
if self.back_seekable and self.is_stream:
raise IOError(
'back seekable was required but not possible for inputfile')
self._subarray_info = self.prepare_subarray_info(
self.file_.telescope_descriptions, self.file_.header)
self.start_pos = self.file_.tell()
# Waveforms from simtelarray have both gain channels
# Gain selection is performed by this EventSource to produce R1 waveforms
if gain_selector is None:
gain_selector = ThresholdGainSelector(parent=self)
self.gain_selector = gain_selector
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def close(self):
self.file_.close()
@property
def is_stream(self):
return not isinstance(self.file_._filehandle,
(BufferedReader, GzipFile))
def prepare_subarray_info(self, telescope_descriptions, header):
"""
Constructs a SubarrayDescription object from the
``telescope_descriptions`` given by ``SimTelFile``
Parameters
----------
telescope_descriptions: dict
telescope descriptions as given by ``SimTelFile.telescope_descriptions``
header: dict
header as returned by ``SimTelFile.header``
Returns
-------
SubarrayDescription :
instrumental information
"""
tel_descriptions = {} # tel_id : TelescopeDescription
tel_positions = {} # tel_id : TelescopeDescription
for tel_id, telescope_description in telescope_descriptions.items():
cam_settings = telescope_description['camera_settings']
n_pixels = cam_settings['n_pixels']
focal_length = u.Quantity(cam_settings['focal_length'], u.m)
try:
telescope = guess_telescope(n_pixels, focal_length)
except ValueError:
telescope = UNKNOWN_TELESCOPE
camera = self._camera_cache.get(telescope.camera_name)
if camera is None:
camera = build_camera_geometry(cam_settings, telescope)
self._camera_cache[telescope.camera_name] = camera
optics = OpticsDescription(
name=telescope.name,
num_mirrors=telescope.n_mirrors,
equivalent_focal_length=focal_length,
mirror_area=u.Quantity(cam_settings['mirror_area'], u.m**2),
num_mirror_tiles=cam_settings['n_mirrors'],
)
tel_descriptions[tel_id] = TelescopeDescription(
name=telescope.name,
tel_type=telescope.type,
optics=optics,
camera=camera)
tel_idx = np.where(header['tel_id'] == tel_id)[0][0]
tel_positions[tel_id] = header['tel_pos'][tel_idx] * u.m
return SubarrayDescription(
"MonteCarloArray",
tel_positions=tel_positions,
tel_descriptions=tel_descriptions,
)
@staticmethod
def is_compatible(file_path):
return is_eventio(Path(file_path).expanduser())
def _generator(self):
if self.file_.tell() > self.start_pos:
self.file_._next_header_pos = 0
warnings.warn('Backseeking to start of file.')
try:
yield from self.__generator()
except EOFError:
msg = 'EOFError reading from "{input_url}". Might be truncated'.format(
input_url=self.input_url)
self.log.warning(msg)
warnings.warn(msg)
def __generator(self):
data = DataContainer()
data.meta['origin'] = 'hessio'
data.meta['input_url'] = self.input_url
data.meta['max_events'] = self.max_events
for counter, array_event in enumerate(self.file_):
# next lines are just for debugging
self.array_event = array_event
data.event_type = array_event['type']
# calibration events do not have an event id
if data.event_type == 'calibration':
event_id = -1
else:
event_id = array_event['event_id']
data.inst.subarray = self._subarray_info
obs_id = self.file_.header['run']
tels_with_data = set(array_event['telescope_events'].keys())
data.count = counter
data.index.obs_id = obs_id
data.index.event_id = event_id
data.r0.obs_id = obs_id # deprecated
data.r0.event_id = event_id # deprecated
data.r0.tels_with_data = tels_with_data
data.r1.obs_id = obs_id # deprecated
data.r1.event_id = event_id # deprecated
data.r1.tels_with_data = tels_with_data
data.dl0.obs_id = obs_id # deprecated
data.dl0.event_id = event_id # deprecated
data.dl0.tels_with_data = tels_with_data
# handle telescope filtering by taking the intersection of
# tels_with_data and allowed_tels
if len(self.allowed_tels) > 0:
selected = tels_with_data & self.allowed_tels
if len(selected) == 0:
continue # skip event
data.r0.tels_with_data = selected
data.r1.tels_with_data = selected
data.dl0.tels_with_data = selected
trigger_information = array_event['trigger_information']
data.trig.tels_with_trigger = trigger_information[
'triggered_telescopes']
time_s, time_ns = trigger_information['gps_time']
data.trig.gps_time = Time(time_s * u.s,
time_ns * u.ns,
format='unix',
scale='utc')
if data.event_type == 'data':
self.fill_mc_information(data, array_event)
# this should be done in a nicer way to not re-allocate the
# data each time (right now it's just deleted and garbage
# collected)
data.r0.tel.clear()
data.r1.tel.clear()
data.dl0.tel.clear()
data.dl1.tel.clear()
data.mc.tel.clear() # clear the previous telescopes
telescope_events = array_event['telescope_events']
tracking_positions = array_event['tracking_positions']
for tel_id, telescope_event in telescope_events.items():
tel_index = self.file_.header['tel_id'].tolist().index(tel_id)
telescope_description = self.file_.telescope_descriptions[
tel_id]
adc_samples = telescope_event.get('adc_samples')
if adc_samples is None:
adc_samples = telescope_event['adc_sums'][:, :, np.newaxis]
_, n_pixels, n_samples = adc_samples.shape
pixel_settings = telescope_description['pixel_settings']
mc = data.mc.tel[tel_id]
mc.dc_to_pe = array_event['laser_calibrations'][tel_id][
'calib']
mc.pedestal = array_event['camera_monitorings'][tel_id][
'pedestal']
mc.reference_pulse_shape = pixel_settings['ref_shape'].astype(
'float64')
mc.meta['refstep'] = float(pixel_settings['ref_step'])
mc.time_slice = float(pixel_settings['time_slice'])
mc.photo_electron_image = (array_event.get(
'photoelectrons',
{}).get(tel_index,
{}).get('photoelectrons',
np.zeros(n_pixels, dtype='float32')))
tracking_position = tracking_positions[tel_id]
mc.azimuth_raw = tracking_position['azimuth_raw']
mc.altitude_raw = tracking_position['altitude_raw']
mc.azimuth_cor = tracking_position.get('azimuth_cor', np.nan)
mc.altitude_cor = tracking_position.get('altitude_cor', np.nan)
if np.isnan(mc.azimuth_cor):
data.pointing[tel_id].azimuth = u.Quantity(
mc.azimuth_raw, u.rad)
else:
data.pointing[tel_id].azimuth = u.Quantity(
mc.azimuth_cor, u.rad)
if np.isnan(mc.altitude_cor):
data.pointing[tel_id].altitude = u.Quantity(
mc.altitude_raw, u.rad)
else:
data.pointing[tel_id].altitude = u.Quantity(
mc.altitude_cor, u.rad)
r0 = data.r0.tel[tel_id]
r1 = data.r1.tel[tel_id]
r0.waveform = adc_samples
r1.waveform, r1.selected_gain_channel = apply_simtel_r1_calibration(
adc_samples, mc.pedestal, mc.dc_to_pe, self.gain_selector)
pixel_lists = telescope_event['pixel_lists']
r0.num_trig_pix = pixel_lists.get(0, {'pixels': 0})['pixels']
if r0.num_trig_pix > 0:
r0.trig_pix_id = pixel_lists[0]['pixel_list']
yield data
def fill_mc_information(self, data, array_event):
mc_event = array_event['mc_event']
mc_shower = array_event['mc_shower']
data.mc.energy = mc_shower['energy'] * u.TeV
data.mc.alt = Angle(mc_shower['altitude'], u.rad)
data.mc.az = Angle(mc_shower['azimuth'], u.rad)
data.mc.core_x = mc_event['xcore'] * u.m
data.mc.core_y = mc_event['ycore'] * u.m
first_int = mc_shower['h_first_int'] * u.m
data.mc.h_first_int = first_int
data.mc.x_max = mc_shower['xmax'] * u.g / (u.cm**2)
data.mc.shower_primary_id = mc_shower['primary_id']
# mc run header data
data.mcheader.run_array_direction = Angle(
self.file_.header['direction'] * u.rad)
mc_run_head = self.file_.mc_run_headers[-1]
data.mcheader.corsika_version = mc_run_head['shower_prog_vers']
data.mcheader.simtel_version = mc_run_head['detector_prog_vers']
data.mcheader.energy_range_min = mc_run_head['E_range'][0] * u.TeV
data.mcheader.energy_range_max = mc_run_head['E_range'][1] * u.TeV
data.mcheader.prod_site_B_total = mc_run_head['B_total'] * u.uT
data.mcheader.prod_site_B_declination = Angle(
mc_run_head['B_declination'] * u.rad)
data.mcheader.prod_site_B_inclination = Angle(
mc_run_head['B_inclination'] * u.rad)
data.mcheader.prod_site_alt = mc_run_head['obsheight'] * u.m
data.mcheader.spectral_index = mc_run_head['spectral_index']
data.mcheader.shower_prog_start = mc_run_head['shower_prog_start']
data.mcheader.shower_prog_id = mc_run_head['shower_prog_id']
data.mcheader.detector_prog_start = mc_run_head['detector_prog_start']
data.mcheader.detector_prog_id = mc_run_head['detector_prog_id']
data.mcheader.num_showers = mc_run_head['n_showers']
data.mcheader.shower_reuse = mc_run_head['n_use']
data.mcheader.max_alt = mc_run_head['alt_range'][1] * u.rad
data.mcheader.min_alt = mc_run_head['alt_range'][0] * u.rad
data.mcheader.max_az = mc_run_head['az_range'][1] * u.rad
data.mcheader.min_az = mc_run_head['az_range'][0] * u.rad
data.mcheader.diffuse = mc_run_head['diffuse']
data.mcheader.max_viewcone_radius = mc_run_head['viewcone'][1] * u.deg
data.mcheader.min_viewcone_radius = mc_run_head['viewcone'][0] * u.deg
data.mcheader.max_scatter_range = mc_run_head['core_range'][1] * u.m
data.mcheader.min_scatter_range = mc_run_head['core_range'][0] * u.m
data.mcheader.core_pos_mode = mc_run_head['core_pos_mode']
data.mcheader.injection_height = mc_run_head['injection_height'] * u.m
data.mcheader.atmosphere = mc_run_head['atmosphere']
data.mcheader.corsika_iact_options = mc_run_head[
'corsika_iact_options']
data.mcheader.corsika_low_E_model = mc_run_head['corsika_low_E_model']
data.mcheader.corsika_high_E_model = mc_run_head[
'corsika_high_E_model']
data.mcheader.corsika_bunchsize = mc_run_head['corsika_bunchsize']
data.mcheader.corsika_wlen_min = mc_run_head['corsika_wlen_min'] * u.nm
data.mcheader.corsika_wlen_max = mc_run_head['corsika_wlen_max'] * u.nm
data.mcheader.corsika_low_E_detail = mc_run_head[
'corsika_low_E_detail']
data.mcheader.corsika_high_E_detail = mc_run_head[
'corsika_high_E_detail']
class SimTelEventSource(EventSource):
skip_calibration_events = Bool(
True, help='Skip calibration events').tag(config=True)
def __init__(self, config=None, parent=None, **kwargs):
super().__init__(config=config, parent=parent, **kwargs)
self.metadata['is_simulation'] = True
# traitlets creates an empty set as default,
# which ctapipe treats as no restriction on the telescopes
# but eventio treats an emty set as "no telescopes allowed"
# so we explicitly pass None in that case
self.file_ = SimTelFile(self.input_url,
allowed_telescopes=self.allowed_tels
if self.allowed_tels else None,
skip_calibration=self.skip_calibration_events)
self._subarray_info = self.prepare_subarray_info(
self.file_.telescope_descriptions, self.file_.header)
self.start_pos = self.file_.tell()
@staticmethod
def prepare_subarray_info(telescope_descriptions, header):
"""
Constructs a SubarrayDescription object from the
``telescope_descriptions`` given by ``SimTelFile``
Parameters
----------
telescope_descriptions: dict
telescope descriptions as given by ``SimTelFile.telescope_descriptions``
header: dict
header as returned by ``SimTelFile.header``
Returns
-------
SubarrayDescription :
instrumental information
"""
tel_descriptions = {} # tel_id : TelescopeDescription
tel_positions = {} # tel_id : TelescopeDescription
for tel_id, telescope_description in telescope_descriptions.items():
cam_settings = telescope_description['camera_settings']
n_pixels = cam_settings['n_pixels']
focal_length = u.Quantity(cam_settings['focal_length'], u.m)
try:
telescope = guess_telescope(n_pixels, focal_length)
except ValueError:
telescope = UNKNOWN_TELESCOPE
pixel_shape = cam_settings['pixel_shape'][0]
try:
pix_type, pix_rotation = CameraGeometry.simtel_shape_to_type(
pixel_shape)
except ValueError:
warnings.warn(
f'Unkown pixel_shape {pixel_shape} for tel_id {tel_id}',
UnknownPixelShapeWarning,
)
pix_type = 'hexagon'
pix_rotation = '0d'
camera = CameraGeometry(
telescope.camera_name,
pix_id=np.arange(n_pixels),
pix_x=u.Quantity(cam_settings['pixel_x'], u.m),
pix_y=u.Quantity(cam_settings['pixel_y'], u.m),
pix_area=u.Quantity(cam_settings['pixel_area'], u.m**2),
pix_type=pix_type,
pix_rotation=pix_rotation,
cam_rotation=-Angle(cam_settings['cam_rot'], u.rad),
apply_derotation=True,
)
optics = OpticsDescription(
name=telescope.name,
num_mirrors=cam_settings['n_mirrors'],
equivalent_focal_length=focal_length,
mirror_area=u.Quantity(cam_settings['mirror_area'], u.m**2),
num_mirror_tiles=cam_settings['n_mirrors'],
)
tel_descriptions[tel_id] = TelescopeDescription(
name=telescope.name,
type=telescope.type,
camera=camera,
optics=optics,
)
tel_idx = np.where(header['tel_id'] == tel_id)[0][0]
tel_positions[tel_id] = header['tel_pos'][tel_idx] * u.m
return SubarrayDescription(
"MonteCarloArray",
tel_positions=tel_positions,
tel_descriptions=tel_descriptions,
)
@staticmethod
def is_compatible(file_path):
return is_eventio(file_path)
def _generator(self):
if self.file_.tell() > self.start_pos:
self.file_._next_header_pos = 0
warnings.warn('Backseeking to start of file.')
try:
yield from self.__generator()
except EOFError:
msg = 'EOFError reading from "{input_url}". Might be truncated'.format(
input_url=self.input_url)
self.log.warning(msg)
warnings.warn(msg)
def __generator(self):
data = DataContainer()
data.meta['origin'] = 'hessio'
data.meta['input_url'] = self.input_url
data.meta['max_events'] = self.max_events
for counter, array_event in enumerate(self.file_):
# next lines are just for debugging
self.array_event = array_event
data.event_type = array_event['type']
# calibration events do not have an event id
if data.event_type == 'calibration':
event_id = -1
else:
event_id = array_event['event_id']
data.inst.subarray = self._subarray_info
obs_id = self.file_.header['run']
tels_with_data = set(array_event['telescope_events'].keys())
data.count = counter
data.r0.obs_id = obs_id
data.r0.event_id = event_id
data.r0.tels_with_data = tels_with_data
data.r1.obs_id = obs_id
data.r1.event_id = event_id
data.r1.tels_with_data = tels_with_data
data.dl0.obs_id = obs_id
data.dl0.event_id = event_id
data.dl0.tels_with_data = tels_with_data
# handle telescope filtering by taking the intersection of
# tels_with_data and allowed_tels
if len(self.allowed_tels) > 0:
selected = tels_with_data & self.allowed_tels
if len(selected) == 0:
continue # skip event
data.r0.tels_with_data = selected
data.r1.tels_with_data = selected
data.dl0.tels_with_data = selected
trigger_information = array_event['trigger_information']
data.trig.tels_with_trigger = trigger_information[
'triggered_telescopes']
time_s, time_ns = trigger_information['gps_time']
data.trig.gps_time = Time(time_s * u.s,
time_ns * u.ns,
format='unix',
scale='utc')
if data.event_type == 'data':
self.fill_mc_information(data, array_event)
# this should be done in a nicer way to not re-allocate the
# data each time (right now it's just deleted and garbage
# collected)
data.r0.tel.clear()
data.r1.tel.clear()
data.dl0.tel.clear()
data.dl1.tel.clear()
data.mc.tel.clear() # clear the previous telescopes
telescope_events = array_event['telescope_events']
tracking_positions = array_event['tracking_positions']
for tel_id, telescope_event in telescope_events.items():
tel_index = self.file_.header['tel_id'].tolist().index(tel_id)
telescope_description = self.file_.telescope_descriptions[
tel_id]
adc_samples = telescope_event.get('adc_samples')
if adc_samples is None:
adc_samples = telescope_event['adc_sums'][:, :, np.newaxis]
r0 = data.r0.tel[tel_id]
r0.waveform = adc_samples
r0.num_samples = adc_samples.shape[-1]
# We should not calculate stuff in an event source
# if this is not needed, we calculate it for nothing
r0.image = adc_samples.sum(axis=-1)
pixel_lists = telescope_event['pixel_lists']
r0.num_trig_pix = pixel_lists.get(0, {'pixels': 0})['pixels']
if r0.num_trig_pix > 0:
r0.trig_pix_id = pixel_lists[0]['pixel_list']
pixel_settings = telescope_description['pixel_settings']
n_pixel = r0.waveform.shape[-2]
mc = data.mc.tel[tel_id]
mc.dc_to_pe = array_event['laser_calibrations'][tel_id][
'calib']
mc.pedestal = array_event['camera_monitorings'][tel_id][
'pedestal']
mc.reference_pulse_shape = pixel_settings['ref_shape'].astype(
'float64')
mc.meta['refstep'] = float(pixel_settings['ref_step'])
mc.time_slice = float(pixel_settings['time_slice'])
mc.photo_electron_image = (array_event.get(
'photoelectrons',
{}).get(tel_index,
{}).get('photoelectrons',
np.zeros(n_pixel, dtype='float32')))
tracking_position = tracking_positions[tel_id]
mc.azimuth_raw = tracking_position['azimuth_raw']
mc.altitude_raw = tracking_position['altitude_raw']
mc.azimuth_cor = tracking_position.get('azimuth_cor', 0)
mc.altitude_cor = tracking_position.get('altitude_cor', 0)
yield data
def fill_mc_information(self, data, array_event):
mc_event = array_event['mc_event']
mc_shower = array_event['mc_shower']
data.mc.energy = mc_shower['energy'] * u.TeV
data.mc.alt = Angle(mc_shower['altitude'], u.rad)
data.mc.az = Angle(mc_shower['azimuth'], u.rad)
data.mc.core_x = mc_event['xcore'] * u.m
data.mc.core_y = mc_event['ycore'] * u.m
first_int = mc_shower['h_first_int'] * u.m
data.mc.h_first_int = first_int
data.mc.x_max = mc_shower['xmax'] * u.g / (u.cm**2)
data.mc.shower_primary_id = mc_shower['primary_id']
# mc run header data
data.mcheader.run_array_direction = Angle(
self.file_.header['direction'] * u.rad)
mc_run_head = self.file_.mc_run_headers[-1]
data.mcheader.corsika_version = mc_run_head['shower_prog_vers']
data.mcheader.simtel_version = mc_run_head['detector_prog_vers']
data.mcheader.energy_range_min = mc_run_head['E_range'][0] * u.TeV
data.mcheader.energy_range_max = mc_run_head['E_range'][1] * u.TeV
data.mcheader.prod_site_B_total = mc_run_head['B_total'] * u.uT
data.mcheader.prod_site_B_declination = Angle(
mc_run_head['B_declination'] * u.rad)
data.mcheader.prod_site_B_inclination = Angle(
mc_run_head['B_inclination'] * u.rad)
data.mcheader.prod_site_alt = mc_run_head['obsheight'] * u.m
data.mcheader.spectral_index = mc_run_head['spectral_index']
data.mcheader.shower_prog_start = mc_run_head['shower_prog_start']
data.mcheader.shower_prog_id = mc_run_head['shower_prog_id']
data.mcheader.detector_prog_start = mc_run_head['detector_prog_start']
data.mcheader.detector_prog_id = mc_run_head['detector_prog_id']
data.mcheader.num_showers = mc_run_head['n_showers']
data.mcheader.shower_reuse = mc_run_head['n_use']
data.mcheader.max_alt = mc_run_head['alt_range'][1] * u.rad
data.mcheader.min_alt = mc_run_head['alt_range'][0] * u.rad
data.mcheader.max_az = mc_run_head['az_range'][1] * u.rad
data.mcheader.min_az = mc_run_head['az_range'][0] * u.rad
data.mcheader.diffuse = mc_run_head['diffuse']
data.mcheader.max_viewcone_radius = mc_run_head['viewcone'][1] * u.deg
data.mcheader.min_viewcone_radius = mc_run_head['viewcone'][0] * u.deg
data.mcheader.max_scatter_range = mc_run_head['core_range'][1] * u.m
data.mcheader.min_scatter_range = mc_run_head['core_range'][0] * u.m
data.mcheader.core_pos_mode = mc_run_head['core_pos_mode']
data.mcheader.injection_height = mc_run_head['injection_height'] * u.m
data.mcheader.atmosphere = mc_run_head['atmosphere']
data.mcheader.corsika_iact_options = mc_run_head[
'corsika_iact_options']
data.mcheader.corsika_low_E_model = mc_run_head['corsika_low_E_model']
data.mcheader.corsika_high_E_model = mc_run_head[
'corsika_high_E_model']
data.mcheader.corsika_bunchsize = mc_run_head['corsika_bunchsize']
data.mcheader.corsika_wlen_min = mc_run_head['corsika_wlen_min'] * u.nm
data.mcheader.corsika_wlen_max = mc_run_head['corsika_wlen_max'] * u.nm
data.mcheader.corsika_low_E_detail = mc_run_head[
'corsika_low_E_detail']
data.mcheader.corsika_high_E_detail = mc_run_head[
'corsika_high_E_detail']
def simtel_event_source(url,
camera=None,
max_events=None,
allowed_tels=None,
requested_event=None,
use_event_id=False,
event_id=None,
disable_bar=False):
"""A generator that streams data from an EventIO/HESSIO MC data file
(e.g. a standard CTA data file.)
Parameters
----------
url : str
path to file to open
max_events : int, optional
maximum number of events to read
allowed_tels : list[int]
select only a subset of telescope, if None, all are read. This can
be used for example emulate the final CTA data format, where there
would be 1 telescope per file (whereas in current monte-carlo,
they are all interleaved into one file)
requested_event : int
Seek to a paricular event index
use_event_id : bool
If True ,'requested_event' now seeks for a particular event id instead
of index
disable_bar : Unused, for compatibility with other readers
"""
if event_id is not None:
raise ValueError('Event id feature not implemented yet! \n'
'Use event_id=None')
if not SimTelEventSource.is_compatible(url):
raise ValueError(url, 'is not a valid simtel file')
data = DataContainer()
data.meta['origin'] = "hessio"
# some hessio_event_source specific parameters
data.meta['input'] = url
# data.meta['max_events'] = max_events
with SimTelFile(url) as file:
telescope_descriptions = file.telescope_descriptions
subarray_info = SimTelEventSource.prepare_subarray_info(
telescope_descriptions, file.header)
counter = 0
for array_event in tqdm(file, disable=disable_bar):
# Seek to requested event
if requested_event is not None:
current = counter
if use_event_id:
current = event_id
if not current == requested_event:
counter += 1
continue
run_id = file.header['run']
event_id = array_event['event_id']
tels_with_data = set(array_event['telescope_events'].keys())
data.inst.subarray = subarray_info
data.r0.run_id = run_id
data.r0.event_id = event_id
data.r0.tels_with_data = tels_with_data
data.r1.run_id = run_id
data.r1.event_id = event_id
data.r1.tels_with_data = tels_with_data
data.dl0.run_id = run_id
data.dl0.event_id = event_id
data.dl0.tels_with_data = tels_with_data
# handle telescope filtering by taking the intersection of
# tels_with_data and allowed_tels
if allowed_tels is not None:
selected = data.r0.tels_with_data & allowed_tels
if len(selected) == 0:
continue # skip event
data.r0.tels_with_data = selected
data.r1.tels_with_data = selected
data.dl0.tels_with_data = selected
trigger_information = array_event['trigger_information']
mc_event = array_event['mc_event']
mc_shower = array_event['mc_shower']
data.trig.tels_with_trigger = \
trigger_information['triggered_telescopes']
time_s, time_ns = trigger_information['gps_time']
data.trig.gps_time = Time(time_s * u.s,
time_ns * u.ns,
format='unix',
scale='utc')
data.mc.energy = mc_shower['energy'] * u.TeV
data.mc.alt = Angle(mc_shower['altitude'], u.rad)
data.mc.az = Angle(mc_shower['azimuth'], u.rad)
data.mc.core_x = mc_event['xcore'] * u.m
data.mc.core_y = mc_event['ycore'] * u.m
first_int = mc_shower['h_first_int'] * u.m
data.mc.h_first_int = first_int
data.mc.x_max = mc_shower['xmax'] * u.g / (u.cm**2)
data.mc.shower_primary_id = mc_shower['primary_id']
# mc run header data
data.mcheader.run_array_direction = Angle(
file.header['direction'] * u.rad)
mc_run_head = file.mc_run_headers[-1]
data.mcheader.corsika_version = mc_run_head['shower_prog_vers']
data.mcheader.simtel_version = mc_run_head['detector_prog_vers']
data.mcheader.energy_range_min = mc_run_head['E_range'][0] * u.TeV
data.mcheader.energy_range_max = mc_run_head['E_range'][1] * u.TeV
data.mcheader.prod_site_B_total = mc_run_head['B_total'] * u.uT
data.mcheader.prod_site_B_declination = Angle(
mc_run_head['B_declination'] * u.rad)
data.mcheader.prod_site_B_inclination = Angle(
mc_run_head['B_inclination'] * u.rad)
data.mcheader.prod_site_alt = mc_run_head['obsheight'] * u.m
data.mcheader.spectral_index = mc_run_head['spectral_index']
# this should be done in a nicer way to not re-allocate the
# data each time (right now it's just deleted and garbage
# collected)
data.r0.tel.clear()
data.r1.tel.clear()
data.dl0.tel.clear()
data.dl1.tel.clear()
data.mc.tel.clear() # clear the previous telescopes
telescope_events = array_event['telescope_events']
tracking_positions = array_event['tracking_positions']
for tel_id, telescope_event in telescope_events.items():
telescope_description = telescope_descriptions[tel_id]
camera_monitorings = array_event['camera_monitorings'][tel_id]
pedestal = camera_monitorings['pedestal']
laser_calib = array_event['laser_calibrations'][tel_id]
data.mc.tel[tel_id].dc_to_pe = laser_calib['calib']
data.mc.tel[tel_id].pedestal = pedestal
adc_samples = telescope_event.get('adc_samples')
n_pixel = adc_samples.shape[-2]
if adc_samples is None:
adc_samples = telescope_event['adc_sums'][:, :, np.newaxis]
data.r0.tel[tel_id].adc_samples = adc_samples
data.r0.tel[tel_id].num_samples = adc_samples.shape[-1]
# We should not calculate stuff in an event source
# if this is not needed, we calculate it for nothing
data.r0.tel[tel_id].adc_sums = adc_samples.sum(axis=-1)
baseline = pedestal / adc_samples.shape[1]
data.r0.tel[tel_id].digicam_baseline = np.squeeze(baseline)
data.r0.tel[tel_id].camera_event_number = event_id
pixel_settings = telescope_description['pixel_settings']
data.mc.tel[tel_id].reference_pulse_shape = pixel_settings[
'refshape'].astype('float64')
data.mc.tel[tel_id].meta['refstep'] = float(
pixel_settings['ref_step'])
data.mc.tel[tel_id].time_slice = float(
pixel_settings['time_slice'])
data.mc.tel[tel_id].photo_electron_image = array_event.get(
'photoelectrons', {}).get(tel_id)
if data.mc.tel[tel_id].photo_electron_image is None:
data.mc.tel[tel_id].photo_electron_image = np.zeros(
(n_pixel, ), dtype='i2')
tracking_position = tracking_positions[tel_id]
data.mc.tel[tel_id].azimuth_raw = tracking_position[
'azimuth_raw']
data.mc.tel[tel_id].altitude_raw = tracking_position[
'altitude_raw']
data.mc.tel[tel_id].azimuth_cor = tracking_position.get(
'azimuth_cor', 0)
data.mc.tel[tel_id].altitude_cor = tracking_position.get(
'altitude_cor', 0)
yield data
counter += 1
if max_events and counter >= max_events:
return
class SimTelEventSource(EventSource):
""" Read events from a SimTelArray data file (in EventIO format)."""
skip_calibration_events = Bool(True, help="Skip calibration events").tag(
config=True
)
back_seekable = Bool(
False,
help=(
"Require the event source to be backwards seekable."
" This will reduce in slower read speed for gzipped files"
" and is not possible for zstd compressed files"
),
).tag(config=True)
focal_length_choice = CaselessStrEnum(
["nominal", "effective"],
default_value="nominal",
help=(
"if both nominal and effective focal lengths are available in the "
"SimTelArray file, which one to use when constructing the "
"SubarrayDescription (which will be used in CameraFrame to TelescopeFrame "
"coordinate transforms. The 'nominal' focal length is the one used during "
"the simulation, the 'effective' focal length is computed using specialized "
"ray-tracing from a point light source"
),
).tag(config=True)
gain_selector_type = create_class_enum_trait(
base_class=GainSelector, default_value="ThresholdGainSelector"
).tag(config=True)
def __init__(self, input_url=None, config=None, parent=None, **kwargs):
"""
EventSource for simtelarray files using the pyeventio library.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
gain_selector : ctapipe.calib.camera.gainselection.GainSelector
The GainSelector to use. If None, then ThresholdGainSelector will be used.
kwargs
"""
super().__init__(input_url=input_url, config=config, parent=parent, **kwargs)
self._camera_cache = {}
self.file_ = SimTelFile(
self.input_url.expanduser(),
allowed_telescopes=self.allowed_tels,
skip_calibration=self.skip_calibration_events,
zcat=not self.back_seekable,
)
if self.back_seekable and self.is_stream:
raise IOError("back seekable was required but not possible for inputfile")
self._subarray_info = self.prepare_subarray_info(
self.file_.telescope_descriptions, self.file_.header
)
self._simulation_config = self._parse_simulation_header()
self.start_pos = self.file_.tell()
self.gain_selector = GainSelector.from_name(
self.gain_selector_type, parent=self
)
self.log.debug(f"Using gain selector {self.gain_selector}")
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def close(self):
self.file_.close()
@property
def is_simulation(self):
return True
@property
def datalevels(self):
return (DataLevel.R0, DataLevel.R1)
@property
def obs_ids(self):
# ToDo: This does not support merged simtel files!
return [self.file_.header["run"]]
@property
def simulation_config(self) -> SimulationConfigContainer:
return self._simulation_config
@property
def is_stream(self):
return not isinstance(self.file_._filehandle, (BufferedReader, GzipFile))
def prepare_subarray_info(self, telescope_descriptions, header):
"""
Constructs a SubarrayDescription object from the
``telescope_descriptions`` given by ``SimTelFile``
Parameters
----------
telescope_descriptions: dict
telescope descriptions as given by ``SimTelFile.telescope_descriptions``
header: dict
header as returned by ``SimTelFile.header``
Returns
-------
SubarrayDescription :
instrumental information
"""
tel_descriptions = {} # tel_id : TelescopeDescription
tel_positions = {} # tel_id : TelescopeDescription
for tel_id, telescope_description in telescope_descriptions.items():
cam_settings = telescope_description["camera_settings"]
pixel_settings = telescope_description["pixel_settings"]
n_pixels = cam_settings["n_pixels"]
focal_length = u.Quantity(cam_settings["focal_length"], u.m)
if self.focal_length_choice == "effective":
try:
focal_length = u.Quantity(
cam_settings["effective_focal_length"], u.m
)
except KeyError as err:
raise RuntimeError(
f"the SimTelEventSource option 'focal_length_choice' was set to "
f"{self.focal_length_choice}, but the effective focal length "
f"was not present in the file. ({err})"
)
try:
telescope = guess_telescope(n_pixels, focal_length)
except ValueError:
telescope = UNKNOWN_TELESCOPE
optics = OpticsDescription(
name=telescope.name,
num_mirrors=telescope.n_mirrors,
equivalent_focal_length=focal_length,
mirror_area=u.Quantity(cam_settings["mirror_area"], u.m ** 2),
num_mirror_tiles=cam_settings["n_mirrors"],
)
camera = self._camera_cache.get(telescope.camera_name)
if camera is None:
camera = build_camera(
cam_settings,
pixel_settings,
telescope,
frame=CameraFrame(focal_length=optics.equivalent_focal_length),
)
self._camera_cache[telescope.camera_name] = camera
tel_descriptions[tel_id] = TelescopeDescription(
name=telescope.name,
tel_type=telescope.type,
optics=optics,
camera=camera,
)
tel_idx = np.where(header["tel_id"] == tel_id)[0][0]
tel_positions[tel_id] = header["tel_pos"][tel_idx] * u.m
subarray = SubarrayDescription(
name="MonteCarloArray",
tel_positions=tel_positions,
tel_descriptions=tel_descriptions,
)
if self.allowed_tels:
subarray = subarray.select_subarray(self.allowed_tels)
return subarray
@staticmethod
def is_compatible(file_path):
return is_eventio(Path(file_path).expanduser())
@property
def subarray(self):
return self._subarray_info
def _generator(self):
if self.file_.tell() > self.start_pos:
self.file_._next_header_pos = 0
warnings.warn("Backseeking to start of file.")
try:
yield from self._generate_events()
except EOFError:
msg = 'EOFError reading from "{input_url}". Might be truncated'.format(
input_url=self.input_url
)
self.log.warning(msg)
warnings.warn(msg)
def _generate_events(self):
data = ArrayEventContainer()
data.simulation = SimulatedEventContainer()
data.meta["origin"] = "hessio"
data.meta["input_url"] = self.input_url
data.meta["max_events"] = self.max_events
self._fill_array_pointing(data)
for counter, array_event in enumerate(self.file_):
event_id = array_event.get("event_id", -1)
obs_id = self.file_.header["run"]
data.count = counter
data.index.obs_id = obs_id
data.index.event_id = event_id
self._fill_trigger_info(data, array_event)
if data.trigger.event_type == EventType.SUBARRAY:
self._fill_simulated_event_information(data, array_event)
# this should be done in a nicer way to not re-allocate the
# data each time (right now it's just deleted and garbage
# collected)
data.r0.tel.clear()
data.r1.tel.clear()
data.dl0.tel.clear()
data.dl1.tel.clear()
data.pointing.tel.clear()
data.simulation.tel.clear()
telescope_events = array_event["telescope_events"]
tracking_positions = array_event["tracking_positions"]
for tel_id, telescope_event in telescope_events.items():
adc_samples = telescope_event.get("adc_samples")
if adc_samples is None:
adc_samples = telescope_event["adc_sums"][:, :, np.newaxis]
n_gains, n_pixels, n_samples = adc_samples.shape
true_image = (
array_event.get("photoelectrons", {})
.get(tel_id - 1, {})
.get("photoelectrons", None)
)
data.simulation.tel[tel_id] = SimulatedCameraContainer(
true_image=true_image
)
data.pointing.tel[tel_id] = self._fill_event_pointing(
tracking_positions[tel_id]
)
r0 = data.r0.tel[tel_id]
r1 = data.r1.tel[tel_id]
r0.waveform = adc_samples
cam_mon = array_event["camera_monitorings"][tel_id]
pedestal = cam_mon["pedestal"] / cam_mon["n_ped_slices"]
dc_to_pe = array_event["laser_calibrations"][tel_id]["calib"]
# fill dc_to_pe and pedestal_per_sample info into monitoring
# container
mon = data.mon.tel[tel_id]
mon.calibration.dc_to_pe = dc_to_pe
mon.calibration.pedestal_per_sample = pedestal
r1.waveform, r1.selected_gain_channel = apply_simtel_r1_calibration(
adc_samples, pedestal, dc_to_pe, self.gain_selector
)
# get time_shift from laser calibration
time_calib = array_event["laser_calibrations"][tel_id]["tm_calib"]
pix_index = np.arange(n_pixels)
dl1_calib = data.calibration.tel[tel_id].dl1
dl1_calib.time_shift = time_calib[r1.selected_gain_channel, pix_index]
yield data
@staticmethod
def _fill_event_pointing(tracking_position):
azimuth_raw = tracking_position["azimuth_raw"]
altitude_raw = tracking_position["altitude_raw"]
azimuth_cor = tracking_position.get("azimuth_cor", np.nan)
altitude_cor = tracking_position.get("altitude_cor", np.nan)
# take pointing corrected position if available
if np.isnan(azimuth_cor):
azimuth = u.Quantity(azimuth_raw, u.rad, copy=False)
else:
azimuth = u.Quantity(azimuth_cor, u.rad, copy=False)
# take pointing corrected position if available
if np.isnan(altitude_cor):
altitude = u.Quantity(altitude_raw, u.rad, copy=False)
else:
altitude = u.Quantity(altitude_cor, u.rad, copy=False)
return TelescopePointingContainer(azimuth=azimuth, altitude=altitude)
def _fill_trigger_info(self, data, array_event):
trigger = array_event["trigger_information"]
if array_event["type"] == "data":
data.trigger.event_type = EventType.SUBARRAY
elif array_event["type"] == "calibration":
# if using eventio >= 1.1.1, we can use the calibration_type
data.trigger.event_type = SIMTEL_TO_CTA_EVENT_TYPE.get(
array_event.get("calibration_type", -1), EventType.OTHER_CALIBRATION
)
else:
data.trigger.event_type = EventType.UNKNOWN
data.trigger.tels_with_trigger = trigger["triggered_telescopes"]
if self.allowed_tels:
data.trigger.tels_with_trigger = np.intersect1d(
data.trigger.tels_with_trigger,
self.subarray.tel_ids,
assume_unique=True,
)
central_time = parse_simtel_time(trigger["gps_time"])
data.trigger.time = central_time
for tel_id, time in zip(
trigger["triggered_telescopes"], trigger["trigger_times"]
):
if self.allowed_tels and tel_id not in self.allowed_tels:
continue
# telescope time is relative to central trigger in ns
time = Time(
central_time.jd1,
central_time.jd2 + time / NANOSECONDS_PER_DAY,
scale=central_time.scale,
format="jd",
)
# triggered pixel info
n_trigger_pixels = -1
trigger_pixels = None
tel_event = array_event["telescope_events"].get(tel_id)
if tel_event:
# code 0 = trigger pixels
pixel_list = tel_event["pixel_lists"].get(0)
if pixel_list:
n_trigger_pixels = pixel_list["pixels"]
trigger_pixels = pixel_list["pixel_list"]
trigger = data.trigger.tel[tel_id] = TelescopeTriggerContainer(
time=time,
n_trigger_pixels=n_trigger_pixels,
trigger_pixels=trigger_pixels,
)
def _fill_array_pointing(self, data):
if self.file_.header["tracking_mode"] == 0:
az, alt = self.file_.header["direction"]
data.pointing.array_altitude = u.Quantity(alt, u.rad)
data.pointing.array_azimuth = u.Quantity(az, u.rad)
else:
ra, dec = self.file_.header["direction"]
data.pointing.array_ra = u.Quantity(ra, u.rad)
data.pointing.array_dec = u.Quantity(dec, u.rad)
def _parse_simulation_header(self):
mc_run_head = self.file_.mc_run_headers[-1]
return SimulationConfigContainer(
corsika_version=mc_run_head["shower_prog_vers"],
simtel_version=mc_run_head["detector_prog_vers"],
energy_range_min=mc_run_head["E_range"][0] * u.TeV,
energy_range_max=mc_run_head["E_range"][1] * u.TeV,
prod_site_B_total=mc_run_head["B_total"] * u.uT,
prod_site_B_declination=Angle(mc_run_head["B_declination"], u.rad),
prod_site_B_inclination=Angle(mc_run_head["B_inclination"], u.rad),
prod_site_alt=mc_run_head["obsheight"] * u.m,
spectral_index=mc_run_head["spectral_index"],
shower_prog_start=mc_run_head["shower_prog_start"],
shower_prog_id=mc_run_head["shower_prog_id"],
detector_prog_start=mc_run_head["detector_prog_start"],
detector_prog_id=mc_run_head["detector_prog_id"],
num_showers=mc_run_head["n_showers"],
shower_reuse=mc_run_head["n_use"],
max_alt=mc_run_head["alt_range"][1] * u.rad,
min_alt=mc_run_head["alt_range"][0] * u.rad,
max_az=mc_run_head["az_range"][1] * u.rad,
min_az=mc_run_head["az_range"][0] * u.rad,
diffuse=mc_run_head["diffuse"],
max_viewcone_radius=mc_run_head["viewcone"][1] * u.deg,
min_viewcone_radius=mc_run_head["viewcone"][0] * u.deg,
max_scatter_range=mc_run_head["core_range"][1] * u.m,
min_scatter_range=mc_run_head["core_range"][0] * u.m,
core_pos_mode=mc_run_head["core_pos_mode"],
injection_height=mc_run_head["injection_height"] * u.m,
atmosphere=mc_run_head["atmosphere"],
corsika_iact_options=mc_run_head["corsika_iact_options"],
corsika_low_E_model=mc_run_head["corsika_low_E_model"],
corsika_high_E_model=mc_run_head["corsika_high_E_model"],
corsika_bunchsize=mc_run_head["corsika_bunchsize"],
corsika_wlen_min=mc_run_head["corsika_wlen_min"] * u.nm,
corsika_wlen_max=mc_run_head["corsika_wlen_max"] * u.nm,
corsika_low_E_detail=mc_run_head["corsika_low_E_detail"],
corsika_high_E_detail=mc_run_head["corsika_high_E_detail"],
)
@staticmethod
def _fill_simulated_event_information(data, array_event):
mc_event = array_event["mc_event"]
mc_shower = array_event["mc_shower"]
data.simulation.shower = SimulatedShowerContainer(
energy=u.Quantity(mc_shower["energy"], u.TeV),
alt=Angle(mc_shower["altitude"], u.rad),
az=Angle(mc_shower["azimuth"], u.rad),
core_x=u.Quantity(mc_event["xcore"], u.m),
core_y=u.Quantity(mc_event["ycore"], u.m),
h_first_int=u.Quantity(mc_shower["h_first_int"], u.m),
x_max=u.Quantity(mc_shower["xmax"], X_MAX_UNIT),
shower_primary_id=mc_shower["primary_id"],
)
class SimTelEventSource(EventSource):
skip_calibration_events = Bool(
True, help="Skip calibration events").tag(config=True)
back_seekable = Bool(
False,
help=("Require the event source to be backwards seekable."
" This will reduce in slower read speed for gzipped files"
" and is not possible for zstd compressed files"),
).tag(config=True)
focal_length_choice = CaselessStrEnum(
["nominal", "effective"],
default_value="nominal",
help=
("if both nominal and effective focal lengths are available in the "
"SimTelArray file, which one to use when constructing the "
"SubarrayDescription (which will be used in CameraFrame to TelescopeFrame "
"coordinate transforms. The 'nominal' focal length is the one used during "
"the simulation, the 'effective' focal length is computed using specialized "
"ray-tracing from a point light source"),
).tag(config=True)
def __init__(self, config=None, parent=None, gain_selector=None, **kwargs):
"""
EventSource for simtelarray files using the pyeventio library.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
gain_selector : ctapipe.calib.camera.gainselection.GainSelector
The GainSelector to use. If None, then ThresholdGainSelector will be used.
kwargs
"""
super().__init__(config=config, parent=parent, **kwargs)
self.metadata["is_simulation"] = True
self._camera_cache = {}
# traitlets creates an empty set as default,
# which ctapipe treats as no restriction on the telescopes
# but eventio treats an emty set as "no telescopes allowed"
# so we explicitly pass None in that case
self.file_ = SimTelFile(
Path(self.input_url).expanduser(),
allowed_telescopes=set(self.allowed_tels)
if self.allowed_tels else None,
skip_calibration=self.skip_calibration_events,
zcat=not self.back_seekable,
)
if self.back_seekable and self.is_stream:
raise IOError(
"back seekable was required but not possible for inputfile")
self._subarray_info = self.prepare_subarray_info(
self.file_.telescope_descriptions, self.file_.header)
self.start_pos = self.file_.tell()
# Waveforms from simtelarray have both gain channels
# Gain selection is performed by this EventSource to produce R1 waveforms
if gain_selector is None:
gain_selector = ThresholdGainSelector(parent=self)
self.gain_selector = gain_selector
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def close(self):
self.file_.close()
@property
def is_stream(self):
return not isinstance(self.file_._filehandle,
(BufferedReader, GzipFile))
def prepare_subarray_info(self, telescope_descriptions, header):
"""
Constructs a SubarrayDescription object from the
``telescope_descriptions`` given by ``SimTelFile``
Parameters
----------
telescope_descriptions: dict
telescope descriptions as given by ``SimTelFile.telescope_descriptions``
header: dict
header as returned by ``SimTelFile.header``
Returns
-------
SubarrayDescription :
instrumental information
"""
tel_descriptions = {} # tel_id : TelescopeDescription
tel_positions = {} # tel_id : TelescopeDescription
for tel_id, telescope_description in telescope_descriptions.items():
cam_settings = telescope_description["camera_settings"]
pixel_settings = telescope_description["pixel_settings"]
n_pixels = cam_settings["n_pixels"]
focal_length = u.Quantity(cam_settings["focal_length"], u.m)
if self.focal_length_choice == "effective":
try:
focal_length = u.Quantity(
cam_settings["effective_focal_length"], u.m)
except KeyError as err:
raise RuntimeError(
f"the SimTelEventSource option 'focal_length_choice' was set to "
f"{self.focal_length_choice}, but the effective focal length "
f"was not present in the file. ({err})")
try:
telescope = guess_telescope(n_pixels, focal_length)
except ValueError:
telescope = UNKNOWN_TELESCOPE
camera = self._camera_cache.get(telescope.camera_name)
if camera is None:
camera = build_camera(cam_settings, pixel_settings, telescope)
self._camera_cache[telescope.camera_name] = camera
optics = OpticsDescription(
name=telescope.name,
num_mirrors=telescope.n_mirrors,
equivalent_focal_length=focal_length,
mirror_area=u.Quantity(cam_settings["mirror_area"], u.m**2),
num_mirror_tiles=cam_settings["n_mirrors"],
)
tel_descriptions[tel_id] = TelescopeDescription(
name=telescope.name,
tel_type=telescope.type,
optics=optics,
camera=camera,
)
tel_idx = np.where(header["tel_id"] == tel_id)[0][0]
tel_positions[tel_id] = header["tel_pos"][tel_idx] * u.m
return SubarrayDescription(
"MonteCarloArray",
tel_positions=tel_positions,
tel_descriptions=tel_descriptions,
)
@staticmethod
def is_compatible(file_path):
return is_eventio(Path(file_path).expanduser())
@property
def subarray(self):
return self._subarray_info
def _generator(self):
if self.file_.tell() > self.start_pos:
self.file_._next_header_pos = 0
warnings.warn("Backseeking to start of file.")
try:
yield from self.__generator()
except EOFError:
msg = 'EOFError reading from "{input_url}". Might be truncated'.format(
input_url=self.input_url)
self.log.warning(msg)
warnings.warn(msg)
def __generator(self):
data = EventAndMonDataContainer()
data.meta["origin"] = "hessio"
data.meta["input_url"] = self.input_url
data.meta["max_events"] = self.max_events
for counter, array_event in enumerate(self.file_):
# next lines are just for debugging
self.array_event = array_event
data.event_type = array_event["type"]
# calibration events do not have an event id
if data.event_type == "calibration":
event_id = -1
else:
event_id = array_event["event_id"]
data.inst.subarray = self._subarray_info
obs_id = self.file_.header["run"]
tels_with_data = set(array_event["telescope_events"].keys())
data.count = counter
data.index.obs_id = obs_id
data.index.event_id = event_id
data.r0.obs_id = obs_id # deprecated
data.r0.event_id = event_id # deprecated
data.r0.tels_with_data = tels_with_data
data.r1.obs_id = obs_id # deprecated
data.r1.event_id = event_id # deprecated
data.r1.tels_with_data = tels_with_data
data.dl0.obs_id = obs_id # deprecated
data.dl0.event_id = event_id # deprecated
data.dl0.tels_with_data = tels_with_data
# handle telescope filtering by taking the intersection of
# tels_with_data and allowed_tels
if len(self.allowed_tels) > 0:
selected = tels_with_data & self.allowed_tels
if len(selected) == 0:
continue # skip event
data.r0.tels_with_data = selected
data.r1.tels_with_data = selected
data.dl0.tels_with_data = selected
trigger_information = array_event["trigger_information"]
data.trig.tels_with_trigger = trigger_information[
"triggered_telescopes"]
time_s, time_ns = trigger_information["gps_time"]
data.trig.gps_time = Time(time_s * u.s,
time_ns * u.ns,
format="unix",
scale="utc")
if data.event_type == "data":
self.fill_mc_information(data, array_event)
# this should be done in a nicer way to not re-allocate the
# data each time (right now it's just deleted and garbage
# collected)
data.r0.tel.clear()
data.r1.tel.clear()
data.dl0.tel.clear()
data.dl1.tel.clear()
data.mc.tel.clear() # clear the previous telescopes
telescope_events = array_event["telescope_events"]
tracking_positions = array_event["tracking_positions"]
for tel_id, telescope_event in telescope_events.items():
tel_index = self.file_.header["tel_id"].tolist().index(tel_id)
adc_samples = telescope_event.get("adc_samples")
if adc_samples is None:
adc_samples = telescope_event["adc_sums"][:, :, np.newaxis]
_, n_pixels, n_samples = adc_samples.shape
mc = data.mc.tel[tel_id]
mc.dc_to_pe = array_event["laser_calibrations"][tel_id][
"calib"]
mc.pedestal = array_event["camera_monitorings"][tel_id][
"pedestal"]
mc.photo_electron_image = (array_event.get(
"photoelectrons",
{}).get(tel_index,
{}).get("photoelectrons",
np.zeros(n_pixels, dtype="float32")))
tracking_position = tracking_positions[tel_id]
mc.azimuth_raw = tracking_position["azimuth_raw"]
mc.altitude_raw = tracking_position["altitude_raw"]
mc.azimuth_cor = tracking_position.get("azimuth_cor", np.nan)
mc.altitude_cor = tracking_position.get("altitude_cor", np.nan)
if np.isnan(mc.azimuth_cor):
data.pointing[tel_id].azimuth = u.Quantity(
mc.azimuth_raw, u.rad)
else:
data.pointing[tel_id].azimuth = u.Quantity(
mc.azimuth_cor, u.rad)
if np.isnan(mc.altitude_cor):
data.pointing[tel_id].altitude = u.Quantity(
mc.altitude_raw, u.rad)
else:
data.pointing[tel_id].altitude = u.Quantity(
mc.altitude_cor, u.rad)
r0 = data.r0.tel[tel_id]
r1 = data.r1.tel[tel_id]
r0.waveform = adc_samples
r1.waveform, r1.selected_gain_channel = apply_simtel_r1_calibration(
adc_samples, mc.pedestal, mc.dc_to_pe, self.gain_selector)
pixel_lists = telescope_event["pixel_lists"]
r0.num_trig_pix = pixel_lists.get(0, {"pixels": 0})["pixels"]
if r0.num_trig_pix > 0:
r0.trig_pix_id = pixel_lists[0]["pixel_list"]
yield data
def fill_mc_information(self, data, array_event):
mc_event = array_event["mc_event"]
mc_shower = array_event["mc_shower"]
data.mc.energy = mc_shower["energy"] * u.TeV
data.mc.alt = Angle(mc_shower["altitude"], u.rad)
data.mc.az = Angle(mc_shower["azimuth"], u.rad)
data.mc.core_x = mc_event["xcore"] * u.m
data.mc.core_y = mc_event["ycore"] * u.m
first_int = mc_shower["h_first_int"] * u.m
data.mc.h_first_int = first_int
data.mc.x_max = mc_shower["xmax"] * u.g / (u.cm**2)
data.mc.shower_primary_id = mc_shower["primary_id"]
# mc run header data
data.mcheader.run_array_direction = Angle(
self.file_.header["direction"] * u.rad)
mc_run_head = self.file_.mc_run_headers[-1]
data.mcheader.corsika_version = mc_run_head["shower_prog_vers"]
data.mcheader.simtel_version = mc_run_head["detector_prog_vers"]
data.mcheader.energy_range_min = mc_run_head["E_range"][0] * u.TeV
data.mcheader.energy_range_max = mc_run_head["E_range"][1] * u.TeV
data.mcheader.prod_site_B_total = mc_run_head["B_total"] * u.uT
data.mcheader.prod_site_B_declination = Angle(
mc_run_head["B_declination"] * u.rad)
data.mcheader.prod_site_B_inclination = Angle(
mc_run_head["B_inclination"] * u.rad)
data.mcheader.prod_site_alt = mc_run_head["obsheight"] * u.m
data.mcheader.spectral_index = mc_run_head["spectral_index"]
data.mcheader.shower_prog_start = mc_run_head["shower_prog_start"]
data.mcheader.shower_prog_id = mc_run_head["shower_prog_id"]
data.mcheader.detector_prog_start = mc_run_head["detector_prog_start"]
data.mcheader.detector_prog_id = mc_run_head["detector_prog_id"]
data.mcheader.num_showers = mc_run_head["n_showers"]
data.mcheader.shower_reuse = mc_run_head["n_use"]
data.mcheader.max_alt = mc_run_head["alt_range"][1] * u.rad
data.mcheader.min_alt = mc_run_head["alt_range"][0] * u.rad
data.mcheader.max_az = mc_run_head["az_range"][1] * u.rad
data.mcheader.min_az = mc_run_head["az_range"][0] * u.rad
data.mcheader.diffuse = mc_run_head["diffuse"]
data.mcheader.max_viewcone_radius = mc_run_head["viewcone"][1] * u.deg
data.mcheader.min_viewcone_radius = mc_run_head["viewcone"][0] * u.deg
data.mcheader.max_scatter_range = mc_run_head["core_range"][1] * u.m
data.mcheader.min_scatter_range = mc_run_head["core_range"][0] * u.m
data.mcheader.core_pos_mode = mc_run_head["core_pos_mode"]
data.mcheader.injection_height = mc_run_head["injection_height"] * u.m
data.mcheader.atmosphere = mc_run_head["atmosphere"]
data.mcheader.corsika_iact_options = mc_run_head[
"corsika_iact_options"]
data.mcheader.corsika_low_E_model = mc_run_head["corsika_low_E_model"]
data.mcheader.corsika_high_E_model = mc_run_head[
"corsika_high_E_model"]
data.mcheader.corsika_bunchsize = mc_run_head["corsika_bunchsize"]
data.mcheader.corsika_wlen_min = mc_run_head["corsika_wlen_min"] * u.nm
data.mcheader.corsika_wlen_max = mc_run_head["corsika_wlen_max"] * u.nm
data.mcheader.corsika_low_E_detail = mc_run_head[
"corsika_low_E_detail"]
data.mcheader.corsika_high_E_detail = mc_run_head[
"corsika_high_E_detail"]