def ctapipe_subarray(self):
from ctapipe.instrument import TelescopeDescription, SubarrayDescription, \
CameraGeometry, CameraReadout, CameraDescription, OpticsDescription
import astropy.units as u
geom = CameraGeometry("sstcam", self.mapping.pixel.i,
u.Quantity(self.mapping.pixel.x, 'm'),
u.Quantity(self.mapping.pixel.y, 'm'),
u.Quantity(self.mapping.pixel.size, 'm')**2,
'square')
readout = CameraReadout(
"sstcam", u.Quantity(1 / self.waveform_sample_width, "GHz"),
self.photoelectron_pulse.amplitude[None, :],
u.Quantity(self.photoelectron_pulse.sample_width, "ns"))
camera = CameraDescription("sstcam", geom, readout)
optics = OpticsDescription.from_name('SST-ASTRI')
telescope = TelescopeDescription("SST", "SST", optics, camera)
subarray = SubarrayDescription(
'toy',
tel_positions={1: [0, 0, 0] * u.m},
tel_descriptions={1: telescope},
)
return subarray
def main():
std_config = get_standard_config()
if args.config_file is not None:
config = replace_config(std_config, read_configuration_file(args.config_file))
else:
config = std_config
print(config['tailcut'])
geom = CameraGeometry.from_name('LSTCam-002')
foclen = OpticsDescription.from_name('LST').equivalent_focal_length
dl1_container = DL1ParametersContainer()
parameters_to_update = list(HillasParametersContainer().keys())
parameters_to_update.extend(['wl', 'r', 'leakage', 'n_islands', 'intercept', 'time_gradient'])
nodes_keys = get_dataset_keys(args.input_file)
if args.noimage:
nodes_keys.remove(dl1_images_lstcam_key)
auto_merge_h5files([args.input_file], args.output_file, nodes_keys=nodes_keys)
with tables.open_file(args.input_file, mode='r') as input:
image_table = input.root[dl1_images_lstcam_key]
with tables.open_file(args.output_file, mode='a') as output:
params = output.root[dl1_params_lstcam_key].read()
for ii, row in enumerate(image_table):
if ii%10000 == 0:
print(ii)
image = row['image']
pulse_time = row['pulse_time']
signal_pixels = tailcuts_clean(geom, image, **config['tailcut'])
if image[signal_pixels].shape[0] > 0:
num_islands, island_labels = number_of_islands(geom, signal_pixels)
hillas = hillas_parameters(geom[signal_pixels], image[signal_pixels])
dl1_container.fill_hillas(hillas)
dl1_container.set_timing_features(geom[signal_pixels],
image[signal_pixels],
pulse_time[signal_pixels],
hillas)
dl1_container.set_leakage(geom, image, signal_pixels)
dl1_container.n_islands = num_islands
dl1_container.wl = dl1_container.width / dl1_container.length
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(np.arctan2(dl1_container.length, foclen))
dl1_container.width = width.value
dl1_container.length = length.value
dl1_container.r = np.sqrt(dl1_container.x**2 + dl1_container.y**2)
for p in parameters_to_update:
params[ii][p] = Quantity(dl1_container[p]).value
else:
for p in parameters_to_update:
params[ii][p] = 0
output.root[dl1_params_lstcam_key][:] = params
def _generator(self):
# container for LST data
self.data = LSTDataContainer()
self.data.meta['input_url'] = self.input_url
self.data.meta['max_events'] = self.max_events
self.data.meta['origin'] = 'LSTCAM'
# fill LST data from the CameraConfig table
self.fill_lst_service_container_from_zfile()
# Instrument information
for tel_id in self.data.lst.tels_with_data:
assert (tel_id == 0
or tel_id == 1) # only LST1 (for the moment id = 0)
# optics info from standard optics.fits.gz file
optics = OpticsDescription.from_name("LST")
# camera info from LSTCam-[geometry_version].camgeom.fits.gz file
geometry_version = 2
camera = CameraGeometry.from_name("LSTCam", geometry_version)
tel_descr = TelescopeDescription(name='LST',
tel_type='LST',
optics=optics,
camera=camera)
self.n_camera_pixels = tel_descr.camera.n_pixels
tels = {tel_id: tel_descr}
# LSTs telescope position taken from MC from the moment
tel_pos = {tel_id: [50., 50., 16] * u.m}
subarray = SubarrayDescription("LST1 subarray")
subarray.tels = tels
subarray.positions = tel_pos
self.data.inst.subarray = subarray
# initialize general monitoring container
self.initialize_mon_container()
# loop on events
for count, event in enumerate(self.multi_file):
self.data.count = count
# fill specific LST event data
self.fill_lst_event_container_from_zfile(event)
# fill general monitoring data
self.fill_mon_container_from_zfile(event)
# fill general R0 data
self.fill_r0_container_from_zfile(event)
yield self.data
def __init__(self, config=None, tool=None, **kwargs):
"""
Constructor
Parameters
----------
config: traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool: ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs: dict
Additional parameters to be passed.
NOTE: The file mask of the data to read can be passed with
the 'input_url' parameter.
"""
file_list = glob.glob(kwargs['input_url'])
file_list.sort()
# EventSource can not handle file wild cards as input_url
# To overcome this we substitute the input_url with first file matching
# the specified file mask.
del kwargs['input_url']
super().__init__(config=config, tool=tool, input_url=file_list[0], **kwargs)
try:
import uproot
except ImportError:
msg = "The `uproot` python module is required to access the MAGIC data"
self.log.error(msg)
raise
# Retrieving the list of run numbers corresponding to the data files
run_numbers = list(map(self._get_run_number, file_list))
self.run_numbers = np.unique(run_numbers)
# # Setting up the current run with the first run present in the data
# self.current_run = self._set_active_run(run_number=0)
self.current_run = None
# MAGIC telescope positions in m wrt. to the center of CTA simulations
self.magic_tel_positions = {
1: [-27.24, -146.66, 50.00] * u.m,
2: [-96.44, -96.77, 51.00] * u.m
}
# MAGIC telescope description
optics = OpticsDescription.from_name('MAGIC')
geom = CameraGeometry.from_name('MAGICCam')
self.magic_tel_description = TelescopeDescription(optics=optics, camera=geom)
self.magic_tel_descriptions = {1: self.magic_tel_description, 2: self.magic_tel_description}
self.magic_subarray = SubarrayDescription('MAGIC', self.magic_tel_positions, self.magic_tel_descriptions)
def _generator(self):
# container for NectarCAM data
self.data = NectarCAMDataContainer()
self.data.meta['input_url'] = self.input_url
# fill data from the CameraConfig table
self.fill_nectarcam_service_container_from_zfile()
# Instrument information
for tel_id in self.data.nectarcam.tels_with_data:
assert (tel_id == 0) # only one telescope for the moment (id = 0)
# optics info from standard optics.fits.gz file
optics = OpticsDescription.from_name("MST")
optics.tel_subtype = '' # to correct bug in reading
# camera info from NectarCam-[geometry_version].camgeom.fits.gz file
geometry_version = 2
camera = CameraGeometry.from_name("NectarCam", geometry_version)
tel_descr = TelescopeDescription(optics, camera)
tel_descr.optics.tel_subtype = '' # to correct bug in reading
self.n_camera_pixels = tel_descr.camera.n_pixels
tels = {tel_id: tel_descr}
# LSTs telescope position
tel_pos = {tel_id: [0., 0., 0] * u.m}
self.subarray = SubarrayDescription("MST prototype subarray")
self.subarray.tels = tels
self.subarray.positions = tel_pos
self.data.inst.subarray = self.subarray
# loop on events
for count, event in enumerate(self.multi_file):
self.data.count = count
# fill specific NectarCAM event data
self.fill_nectarcam_event_container_from_zfile(event)
# fill general R0 data
self.fill_r0_container_from_zfile(event)
yield self.data
def _generator(self):
# container for LST data
self.data = LSTDataContainer()
self.data.meta['input_url'] = self.input_url
self.data.meta['max_events'] = self.max_events
# fill LST data from the CameraConfig table
self.fill_lst_service_container_from_zfile()
# Instrument information
for tel_id in self.data.lst.tels_with_data:
assert (tel_id == 0) # only LST1 for the moment (id = 0)
# optics info from standard optics.fits.gz file
optics = OpticsDescription.from_name("LST")
optics.tel_subtype = '' # to correct bug in reading
# camera info from LSTCam-[geometry_version].camgeom.fits.gz file
geometry_version = 2
camera = CameraGeometry.from_name("LSTCam", geometry_version)
tel_descr = TelescopeDescription(optics, camera)
self.n_camera_pixels = tel_descr.camera.n_pixels
tels = {tel_id: tel_descr}
# LSTs telescope position taken from MC from the moment
tel_pos = {tel_id: [50., 50., 16] * u.m}
subarray = SubarrayDescription("LST1 subarray")
subarray.tels = tels
subarray.positions = tel_pos
self.data.inst.subarray = subarray
# loop on events
for count, event in enumerate(self.multi_file):
self.data.count = count
# fill specific LST event data
self.fill_lst_event_container_from_zfile(event)
# fill general R0 data
self.fill_r0_container_from_zfile(event)
yield self.data
def _generator(self):
# container for NectarCAM data
self.data = NectarCAMDataContainer()
self.data.meta['input_url'] = self.input_url
# fill data from the CameraConfig table
self.fill_nectarcam_service_container_from_zfile()
# Instrument information
for tel_id in self.data.nectarcam.tels_with_data:
assert (tel_id == 0) # only one telescope for the moment (id = 0)
# optics info from standard optics.fits.gz file
optics = OpticsDescription.from_name("MST")
optics.tel_subtype = '' # to correct bug in reading
# camera info from NectarCam-[geometry_version].camgeom.fits.gz file
geometry_version = 2
camera = CameraGeometry.from_name("NectarCam", geometry_version)
tel_descr = TelescopeDescription(optics, camera)
tel_descr.optics.tel_subtype = '' # to correct bug in reading
self.n_camera_pixels = tel_descr.camera.n_pixels
tels = {tel_id: tel_descr}
# LSTs telescope position
tel_pos = {tel_id: [0., 0., 0] * u.m}
self.subarray = SubarrayDescription("MST prototype subarray")
self.subarray.tels = tels
self.subarray.positions = tel_pos
self.data.inst.subarray = self.subarray
# loop on events
for count, event in enumerate(self.multi_file):
self.data.count = count
# fill specific NectarCAM event data
self.fill_nectarcam_event_container_from_zfile(event)
# fill general R0 data
self.fill_r0_container_from_zfile(event)
yield self.data
def get_expected_source_pos(data, data_type, config):
"""Get expected source position for source-dependent analysis .
Parameters:
-----------
data: Pandas DataFrame
data_type: 'mc_gamma','mc_proton','real_data'
config: dictionnary containing configuration
"""
#For gamma MC, expected source position is actual one for each event
if data_type == 'mc_gamma':
expected_src_pos_x_m = data['src_x'].values
expected_src_pos_y_m = data['src_y'].values
#For proton MC, nominal source position is one written in config file
if data_type == 'mc_proton':
focal_length = OpticsDescription.from_name(
'LST').equivalent_focal_length
expected_src_pos = utils.sky_to_camera(
u.Quantity(data['mc_alt_tel'].values +
config['mc_nominal_source_x_deg'],
u.deg,
copy=False),
u.Quantity(data['mc_az_tel'].values +
config['mc_nominal_source_y_deg'],
u.deg,
copy=False), focal_length,
u.Quantity(data['mc_alt_tel'].values, u.deg, copy=False),
u.Quantity(data['mc_az_tel'].values, u.deg, copy=False))
expected_src_pos_x_m = expected_src_pos.x.to_value()
expected_src_pos_y_m = expected_src_pos.y.to_value()
# For real data
# TODO: expected source position for real data should be obtained by using tel_alt,az and source RA,Dec
# For the moment, expected source position is defined as camera center (ON mode)
if data_type == 'real_data':
expected_src_pos_x_m = np.zeros(len(data))
expected_src_pos_y_m = np.zeros(len(data))
return expected_src_pos_x_m, expected_src_pos_y_m
def prepare_subarray_info(self, tel_id=0):
"""
Constructs a SubarrayDescription object.
Parameters
----------
tel_id: int
Telescope identifier.
Returns
-------
SubarrayDescription :
instrumental information
"""
tel_descriptions = {} # tel_id : TelescopeDescription
tel_positions = {} # tel_id : TelescopeDescription
# optics info from standard optics.fits.gz file
optics = OpticsDescription.from_name("MST")
optics.tel_subtype = '' # to correct bug in reading
# camera info from NectarCam-[geometry_version].camgeom.fits.gz file
camera = CameraGeometry.from_name("NectarCam", self.geometry_version)
tel_descr = TelescopeDescription(name='MST',
tel_type='NectarCam',
optics=optics,
camera=camera)
tel_descr.optics.tel_subtype = '' # to correct bug in reading
self.n_camera_pixels = tel_descr.camera.n_pixels
# MST telescope position
tel_positions[tel_id] = [0., 0., 0] * u.m
tel_descriptions[tel_id] = tel_descr
return SubarrayDescription(
"Adlershof",
tel_positions=tel_positions,
tel_descriptions=tel_descriptions,
)
import pandas as pd
from astropy.table import Table
import RFGHsep_fromfile as rf
#Read data into pandas DataFrame
filetype = 'hdf5'
gammafile = "/scratch/bernardos/LST1/Events/gamma_events_diff.hdf5" #File with events
protonfile = "/scratch/bernardos/LST1/Events/proton_events.hdf5" #File with events
dat_gamma = Table.read(gammafile, format=filetype, path='gamma')
dat_proton = Table.read(protonfile, format=filetype, path='proton')
df_gamma = dat_gamma.to_pandas()
df_proton = dat_proton.to_pandas()
#Get some telescope parameters
tel = OpticsDescription.from_name('LST') #Telescope description
focal_length = tel.equivalent_focal_length.value #Telescope focal length
#Calculate source position and Disp distance:
sourcepos_gamma = Disp.calc_CamSourcePos(df_gamma['mcAlt'].get_values(),
df_gamma['mcAz'].get_values(),
df_gamma['mcAlttel'].get_values(),
df_gamma['mcAztel'].get_values(),
focal_length)
disp_gamma = Disp.calc_DISP(sourcepos_gamma[0], sourcepos_gamma[1],
df_gamma['x'].get_values(),
df_gamma['y'].get_values())
sourcepos_proton = Disp.calc_CamSourcePos(df_proton['mcAlt'].get_values(),
df_proton['mcAz'].get_values(),
df_proton['mcAlttel'].get_values(),
def r0_to_dl1(input_filename=get_dataset_path('gamma_test_large.simtel.gz'),
output_filename=None,
custom_config={},
pedestal_path=None,
calibration_path=None,
time_calibration_path=None,
pointing_file_path=None):
"""
Chain r0 to dl1
Save the extracted dl1 parameters in output_filename
Parameters
----------
input_filename: str
path to input file, default: `gamma_test_large.simtel.gz`
output_filename: str
path to output file, default: `./` + basename(input_filename)
config_file: path to a configuration file
pointing_file_path: path to the Drive log with the pointing information
Returns
-------
"""
if output_filename is None:
output_filename = ('dl1_' +
os.path.basename(input_filename).split('.')[0] +
'.h5')
config = replace_config(standard_config, custom_config)
custom_calibration = config["custom_calibration"]
try:
source = event_source(input_filename, back_seekable=True)
except:
# back_seekable might not be available for other sources that eventio
# TODO for real data: source with calibration file and pointing file
source = event_source(input_filename)
is_simu = source.metadata['is_simulation']
source.allowed_tels = config["allowed_tels"]
source.max_events = config["max_events"]
metadata = global_metadata(source)
write_metadata(metadata, output_filename)
cal = load_calibrator_from_config(config)
if not is_simu:
# TODO : add calibration config in config file, read it and pass it here
r0_r1_calibrator = LSTR0Corrections(pedestal_path=pedestal_path,
tel_id=1)
# all this will be cleaned up in a next PR related to the configuration files
r1_dl1_calibrator = LSTCameraCalibrator(
calibration_path=calibration_path,
time_calibration_path=time_calibration_path,
image_extractor=config['image_extractor'],
gain_threshold=Config(config).gain_selector_config['threshold'],
config=Config(config),
allowed_tels=[1],
)
dl1_container = DL1ParametersContainer()
if pointing_file_path:
# Open drive report
pointings = PointingPosition()
pointings.drive_path = pointing_file_path
drive_data = pointings._read_drive_report()
extra_im = ExtraImageInfo()
extra_im.prefix = '' # get rid of the prefix
event = next(iter(source))
write_array_info(event, output_filename)
### Write extra information to the DL1 file
if is_simu:
write_mcheader(event.mcheader,
output_filename,
obs_id=event.r0.obs_id,
filters=filters,
metadata=metadata)
subarray = event.inst.subarray
with HDF5TableWriter(
filename=output_filename,
group_name='dl1/event',
mode='a',
filters=filters,
add_prefix=True,
# overwrite=True,
) as writer:
print("USING FILTERS: ", writer._h5file.filters)
if is_simu:
# build a mapping of tel_id back to tel_index:
# (note this should be part of SubarrayDescription)
idx = np.zeros(max(subarray.tel_indices) + 1)
for key, val in subarray.tel_indices.items():
idx[key] = val
# the final transform then needs the mapping and the number of telescopes
tel_list_transform = partial(
utils.expand_tel_list,
max_tels=len(event.inst.subarray.tel) + 1,
)
writer.add_column_transform(table_name='subarray/trigger',
col_name='tels_with_trigger',
transform=tel_list_transform)
### EVENT LOOP ###
for i, event in enumerate(source):
if i % 100 == 0:
print(i)
event.dl0.prefix = ''
event.mc.prefix = 'mc'
event.trig.prefix = ''
# write sub tables
if is_simu:
write_subarray_tables(writer, event, metadata)
if not custom_calibration and is_simu:
cal(event)
if not is_simu:
r0_r1_calibrator.calibrate(event)
r1_dl1_calibrator(event)
for ii, telescope_id in enumerate(event.r0.tels_with_data):
tel = event.dl1.tel[telescope_id]
tel.prefix = '' # don't really need one
# remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST'
tel_name = str(event.inst.subarray.tel[telescope_id])[4:]
tel_name = tel_name.replace('-003', '')
if custom_calibration:
lst_calibration(event, telescope_id)
try:
dl1_filled = get_dl1(event,
telescope_id,
dl1_container=dl1_container,
custom_config=config,
use_main_island=True)
except HillasParameterizationError:
logging.exception(
'HillasParameterizationError in get_dl1()')
continue
if dl1_filled is not None:
# Some custom def
dl1_container.wl = dl1_container.width / dl1_container.length
# Log10(Energy) in GeV
if is_simu:
dl1_container.mc_energy = event.mc.energy.value
dl1_container.log_mc_energy = np.log10(
event.mc.energy.value * 1e3)
dl1_container.fill_mc(event)
dl1_container.log_intensity = np.log10(
dl1_container.intensity)
dl1_container.gps_time = event.trig.gps_time.value
if not is_simu:
# GPS time is not available for the time being. Meanwhile,
# timestamps can be extracted from the UCTS and alternatively
# calculated from the TIB/Dragon modules counters + NTP time
# corresponding to the start of the run. For the time being,
# we will store the three of them in the dl1 files.
# This will be deprecated and modified back to directly use
# gps_time whenever the GPS starts working reliably.
# gps_time = event.r0.tel[telescope_id].trigger_time
ucts_time = event.lst.tel[
telescope_id].evt.ucts_timestamp * 1e-9 # nsecs
# Get counters from the central Dragon module
module_id = 82
dragon_time = (event.lst.tel[telescope_id].svc.date +
event.lst.tel[telescope_id].evt.
pps_counter[module_id] +
event.lst.tel[telescope_id].evt.
tenMHz_counter[module_id] * 10**(-7))
tib_time = (
event.lst.tel[telescope_id].svc.date +
event.lst.tel[telescope_id].evt.tib_pps_counter +
event.lst.tel[telescope_id].evt.tib_tenMHz_counter
* 10**(-7))
#dl1_container.gps_time = gps_time
dl1_container.tib_time = tib_time
dl1_container.ucts_time = ucts_time
dl1_container.dragon_time = dragon_time
# Select the timestamps to be used for pointing interpolation
if config['timestamps_pointing'] == "ucts":
event_timestamps = ucts_time
elif config['timestamps_pointing'] == "dragon":
event_timestamps = dragon_time
elif config['timestamps_pointing'] == "tib":
event_timestamps = tib_time
else:
raise ValueError(
"The timestamps_pointing option is not a valid one. \
Try ucts (default), dragon or tib.")
if pointing_file_path and event_timestamps > 0:
azimuth, altitude = pointings.cal_pointingposition(
event_timestamps, drive_data)
event.pointing[telescope_id].azimuth = azimuth
event.pointing[telescope_id].altitude = altitude
dl1_container.az_tel = azimuth
dl1_container.alt_tel = altitude
foclen = event.inst.subarray.tel[
telescope_id].optics.equivalent_focal_length
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(
np.arctan2(dl1_container.length, foclen))
dl1_container.width = width.value
dl1_container.length = length.value
dl1_container.prefix = tel.prefix
extra_im.tel_id = telescope_id
extra_im.num_trig_pix = event.r0.tel[
telescope_id].num_trig_pix
extra_im.trigger_time = event.r0.tel[
telescope_id].trigger_time
extra_im.trigger_type = event.r0.tel[
telescope_id].trigger_type
extra_im.trig_pix_id = event.r0.tel[
telescope_id].trig_pix_id
for container in [extra_im, dl1_container, event.r0, tel]:
add_global_metadata(container, metadata)
event.r0.prefix = ''
writer.write(table_name=f'telescope/image/{tel_name}',
containers=[event.r0, tel, extra_im])
writer.write(table_name=f'telescope/parameters/{tel_name}',
containers=[dl1_container, extra_im])
# writes mc information per telescope, including photo electron image
if is_simu \
and (event.mc.tel[telescope_id].photo_electron_image > 0).any() \
and config['write_pe_image']:
event.mc.tel[telescope_id].prefix = ''
writer.write(
table_name=f'simulation/{tel_name}',
containers=[event.mc.tel[telescope_id], extra_im])
if is_simu:
### Reconstruct source position from disp for all events and write the result in the output file
for tel_name in ['LST_LSTCam']:
focal = OpticsDescription.from_name(
tel_name.split('_')[0]).equivalent_focal_length
dl1_params_key = f'dl1/event/telescope/parameters/{tel_name}'
add_disp_to_parameters_table(output_filename, dl1_params_key,
focal)
# Write energy histogram from simtel file and extra metadata
if is_simu:
write_simtel_energy_histogram(source,
output_filename,
obs_id=event.dl0.obs_id,
metadata=metadata)
def apply_models(dl1, classifier, reg_energy, reg_disp_vector, custom_config={}):
"""Apply previously trained Random Forests to a set of data
depending on a set of features.
Parameters:
-----------
data: Pandas DataFrame
features: list
classifier: Random Forest Classifier
RF for Gamma/Hadron separation
reg_energy: Random Forest Regressor
RF for Energy reconstruction
reg_disp: Random Forest Regressor
RF for disp_norm reconstruction
"""
config = replace_config(standard_config, custom_config)
regression_features = config["regression_features"]
classification_features = config["classification_features"]
dl2 = dl1.copy()
#Reconstruction of Energy and disp_norm distance
dl2['log_reco_energy'] = reg_energy.predict(dl2[regression_features])
dl2['reco_energy'] = 10**(dl2['log_reco_energy']-3)
disp_vector = reg_disp_vector.predict(dl2[regression_features])
dl2['reco_disp_dx'] = disp_vector[:, 0]
dl2['reco_disp_dy'] = disp_vector[:, 1]
#Construction of Source position in camera coordinates from disp_norm distance.
dl2['reco_src_x'], dl2['reco_src_y'] = disp.disp_to_pos(dl2.reco_disp_dx,
dl2.reco_disp_dy,
dl2.x,
dl2.y,
)
focal_length = OpticsDescription.from_name('LST').equivalent_focal_length
if 'mc_alt_tel' in dl2.columns:
alt_tel = dl2['mc_alt_tel'].values
az_tel = dl2['mc_az_tel'].values
elif 'alt_tel' in dl2.columns:
alt_tel = dl2['alt_tel'].values
az_tel = dl2['az_tel'].values
else:
alt_tel = - np.pi/2. * np.ones(len(dl2))
az_tel = - np.pi/2. * np.ones(len(dl2))
src_pos_reco = utils.reco_source_position_sky(dl2.x.values * u.m,
dl2.y.values * u.m,
dl2.reco_disp_dx.values * u.m,
dl2.reco_disp_dy.values * u.m,
focal_length,
alt_tel * u.rad,
az_tel * u.rad)
dl2['reco_alt'] = src_pos_reco.alt.rad
dl2['reco_az'] = src_pos_reco.az.rad
dl2['reco_type'] = classifier.predict(dl2[classification_features]).astype(int)
probs = classifier.predict_proba(dl2[classification_features])[0:, 0]
dl2['gammaness'] = probs
return dl2
def r0_to_dl1(
input_filename=get_dataset_path('gamma_test_large.simtel.gz'),
output_filename=None,
custom_config={},
pedestal_path=None,
calibration_path=None,
pointing_file_path=None,
):
"""
Chain r0 to dl1
Save the extracted dl1 parameters in output_filename
Parameters
----------
input_filename: str
path to input file, default: `gamma_test_large.simtel.gz`
output_filename: str
path to output file, default: `./` + basename(input_filename)
config_file: path to a configuration file
pointing_file_path: path to the Drive log with the pointing information
Returns
-------
"""
if output_filename is None:
output_filename = ('dl1_' +
os.path.basename(input_filename).split('.')[0] +
'.h5')
config = replace_config(standard_config, custom_config)
custom_calibration = config["custom_calibration"]
try:
source = event_source(input_filename, back_seekable=True)
except:
# back_seekable might not be available for other sources that eventio
# TODO for real data: source with calibration file and pointing file
source = event_source(input_filename)
is_simu = source.metadata['is_simulation']
source.allowed_tels = config["allowed_tels"]
source.max_events = config["max_events"]
metadata = global_metadata(source)
write_metadata(metadata, output_filename)
cal = load_calibrator_from_config(config)
if not is_simu:
# TODO : add calibration config in config file, read it and pass it here
charge_config = Config(
{"LocalPeakWindowSum": {
"window_shift": 5,
"window_width": 12
}})
r0_r1_calibrator = LSTR0Corrections(
pedestal_path=pedestal_path,
r1_sample_start=2, # numbers in config ?
r1_sample_end=38,
)
r1_dl1_calibrator = LSTCameraCalibrator(
calibration_path=calibration_path,
image_extractor=config['image_extractor'],
config=charge_config,
allowed_tels=[1],
)
dl1_container = DL1ParametersContainer()
if pointing_file_path:
# Open drive report
pointings = PointingPosition()
pointings.drive_path = pointing_file_path
drive_data = pointings._read_drive_report()
extra_im = ExtraImageInfo()
extra_im.prefix = '' # get rid of the prefix
event = next(iter(source))
write_array_info(event, output_filename)
### Write extra information to the DL1 file
if is_simu:
write_mcheader(event.mcheader,
output_filename,
obs_id=event.r0.obs_id,
filters=filters,
metadata=metadata)
subarray = event.inst.subarray
with HDF5TableWriter(
filename=output_filename,
group_name='dl1/event',
mode='a',
filters=filters,
add_prefix=True,
# overwrite=True,
) as writer:
print("USING FILTERS: ", writer._h5file.filters)
if is_simu:
# build a mapping of tel_id back to tel_index:
# (note this should be part of SubarrayDescription)
idx = np.zeros(max(subarray.tel_indices) + 1)
for key, val in subarray.tel_indices.items():
idx[key] = val
# the final transform then needs the mapping and the number of telescopes
tel_list_transform = partial(
utils.expand_tel_list,
max_tels=len(event.inst.subarray.tel) + 1,
)
writer.add_column_transform(table_name='subarray/trigger',
col_name='tels_with_trigger',
transform=tel_list_transform)
### EVENT LOOP ###
for i, event in enumerate(source):
if i % 100 == 0:
print(i)
event.dl0.prefix = ''
event.mc.prefix = 'mc'
event.trig.prefix = ''
# write sub tables
if is_simu:
write_subarray_tables(writer, event, metadata)
if not custom_calibration and is_simu:
cal(event)
if not is_simu:
r0_r1_calibrator.calibrate(event)
r1_dl1_calibrator(event)
for ii, telescope_id in enumerate(event.r0.tels_with_data):
if not is_simu:
combine_channels(event, telescope_id, 4095)
tel = event.dl1.tel[telescope_id]
tel.prefix = '' # don't really need one
# remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST'
tel_name = str(event.inst.subarray.tel[telescope_id])[4:]
tel_name = tel_name.replace('-002', '')
if custom_calibration:
lst_calibration(event, telescope_id)
try:
dl1_filled = get_dl1(event,
telescope_id,
dl1_container=dl1_container,
custom_config=config,
use_main_island=True)
except HillasParameterizationError:
logging.exception(
'HillasParameterizationError in get_dl1()')
continue
if dl1_filled is not None:
# Some custom def
dl1_container.wl = dl1_container.width / dl1_container.length
# Log10(Energy) in GeV
if is_simu:
dl1_container.mc_energy = event.mc.energy.value
dl1_container.log_mc_energy = np.log10(
event.mc.energy.value * 1e3)
dl1_container.fill_mc(event)
dl1_container.log_intensity = np.log10(
dl1_container.intensity)
dl1_container.gps_time = event.trig.gps_time.value
if not is_simu:
# For real data, GPS time is not available for the time being.
# In the mean time, it is taken from TIB pps and 10 MHz counters
# since UCTS timestamps do not seem to be trustable. This will be
# deprecated and modified back to directly use gps_time whenever
# the GPS starts working.
# TAI time in s taken from TIB
tai_time = event.r0.tel[telescope_id].trigger_time
utc_time = Time(datetime.utcfromtimestamp(tai_time))
gps_time = utc_time.gps
dl1_container.gps_time = gps_time
if pointing_file_path:
azimuth, altitude = pointings.cal_pointingposition(
utc_time.unix, drive_data)
event.pointing[telescope_id].azimuth = azimuth
event.pointing[telescope_id].altitude = altitude
dl1_container.az_tel = azimuth
dl1_container.alt_tel = altitude
foclen = event.inst.subarray.tel[
telescope_id].optics.equivalent_focal_length
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(
np.arctan2(dl1_container.length, foclen))
dl1_container.width = width.value
dl1_container.length = length.value
dl1_container.prefix = tel.prefix
extra_im.tel_id = telescope_id
for container in [extra_im, dl1_container, event.r0, tel]:
add_global_metadata(container, metadata)
event.r0.prefix = ''
writer.write(table_name=f'telescope/image/{tel_name}',
containers=[event.r0, tel, extra_im])
writer.write(table_name=f'telescope/parameters/{tel_name}',
containers=[dl1_container])
# writes mc information per telescope, including photo electron image
if is_simu \
and (event.mc.tel[telescope_id].photo_electron_image > 0).any() \
and config['write_pe_image']:
event.mc.tel[telescope_id].prefix = ''
writer.write(
table_name=f'simulation/{tel_name}',
containers=[event.mc.tel[telescope_id], extra_im])
if is_simu:
### Reconstruct source position from disp for all events and write the result in the output file
for tel_name in ['LST_LSTCam']:
focal = OpticsDescription.from_name(
tel_name.split('_')[0]).equivalent_focal_length
dl1_params_key = f'dl1/event/telescope/parameters/{tel_name}'
add_disp_to_parameters_table(output_filename, dl1_params_key,
focal)
# Write energy histogram from simtel file and extra metadata
if is_simu:
write_simtel_energy_histogram(source,
output_filename,
obs_id=event.dl0.obs_id,
metadata=metadata)
def __init__(self, **kwargs):
"""
Constructor
Parameters
----------
kwargs: dict
Parameters to be passed.
NOTE: The file mask of the data to read can be passed with
the 'input_url' parameter.
"""
try:
import uproot
except ImportError:
raise ImportError(
"The 'uproot' package is required for the DLMAGICEventSource class."
)
self.file_list = glob.glob(kwargs["input_url"])
self.file_list.sort()
# Since EventSource can not handle file wild cards as input_url
# We substitute the input_url with first file matching
# the specified file mask.
del kwargs["input_url"]
super().__init__(input_url=self.file_list[0], **kwargs)
# Translate MAGIC shower primary id to CTA convention
self.magic_to_cta_shower_primary_id = {
1: 0, # gamma
14: 101, # MAGIC proton
3: 1, # MAGIC electron
}
# MAGIC telescope positions in m wrt. to the center of CTA simulations
self.magic_tel_positions = {
1: [-27.24, -146.66, 50.00] * u.m,
2: [-96.44, -96.77, 51.00] * u.m,
}
self.magic_tel_positions = self.magic_tel_positions
# MAGIC telescope description
optics = OpticsDescription.from_name("MAGIC")
geom = CameraGeometry.from_name("MAGICCam")
# Camera Readout for NectarCam used as a placeholder
readout = CameraReadout(
"MAGICCam",
sampling_rate=u.Quantity(1, u.GHz),
reference_pulse_shape=np.array([norm.pdf(np.arange(96), 48, 6)]),
reference_pulse_sample_width=u.Quantity(1, u.ns),
)
camera = CameraDescription("MAGICCam", geom, readout)
self.magic_tel_description = TelescopeDescription(name="MAGIC",
tel_type="LST",
optics=optics,
camera=camera)
self.magic_tel_descriptions = {
1: self.magic_tel_description,
2: self.magic_tel_description,
}
self.magic_subarray = SubarrayDescription("MAGIC",
self.magic_tel_positions,
self.magic_tel_descriptions)
# Open ROOT files
self.calib_M1, self.calib_M2, self.star_M1, self.star_M2, self.superstar = (
None,
None,
None,
None,
None,
)
for file in self.file_list:
uproot_file = uproot.open(file)
if "_Y_" in file:
if "_M1_" in file:
self.calib_M1 = uproot_file["Events"]
self.meta = uproot_file["RunHeaders"]
elif "_M2_" in file:
self.calib_M2 = uproot_file["Events"]
if "_I_" in file:
if "_M1_" in file:
self.star_M1 = uproot_file["Events"]
elif "_M2_" in file:
self.star_M2 = uproot_file["Events"]
if "_S_" in file:
self.superstar = uproot_file["Events"]
self.meta = uproot_file["RunHeaders"]
# figure out if MC or Data run
self.mc = "MMcCorsikaRunHeader." in self.meta.keys()
# get the run number directly from the root file
if self.mc:
self.run_number = int(
uproot_file["RunHeaders"]["MMcCorsikaRunHeader."]
["MMcCorsikaRunHeader.fRunNumber"].array()[0])
else:
self.run_number = int(uproot_file["RunHeaders"]["MRawRunHeader_1."]
["MRawRunHeader_1.fRunNumber"].array()[0])
self._header = self._parse_header()
def apply_models(dl1,
classifier,
reg_energy,
reg_disp_vector,
custom_config={}):
"""Apply previously trained Random Forests to a set of data
depending on a set of features.
Parameters:
-----------
data: Pandas DataFrame
features: list
classifier: Random Forest Classifier
RF for Gamma/Hadron separation
reg_energy: Random Forest Regressor
RF for Energy reconstruction
reg_disp: Random Forest Regressor
RF for disp_norm reconstruction
"""
config = replace_config(standard_config, custom_config)
dl2 = dl1.copy()
regression_features = config["regression_features"]
classification_features = config["classification_features"]
#Reconstruction of Energy and disp_norm distance
dl2['log_reco_energy'] = reg_energy.predict(dl2[regression_features])
dl2['reco_energy'] = 10**(dl2['log_reco_energy'])
disp_vector = reg_disp_vector.predict(dl2[regression_features])
dl2['reco_disp_dx'] = disp_vector[:, 0]
dl2['reco_disp_dy'] = disp_vector[:, 1]
#Construction of Source position in camera coordinates from disp_norm distance.
dl2['reco_src_x'], dl2['reco_src_y'] = disp.disp_to_pos(
dl2.reco_disp_dx,
dl2.reco_disp_dy,
dl2.x,
dl2.y,
)
focal_length = OpticsDescription.from_name('LST').equivalent_focal_length
if 'mc_alt_tel' in dl2.columns:
alt_tel = dl2['mc_alt_tel'].values
az_tel = dl2['mc_az_tel'].values
elif 'alt_tel' in dl2.columns:
alt_tel = dl2['alt_tel'].values
az_tel = dl2['az_tel'].values
else:
alt_tel = -np.pi / 2. * np.ones(len(dl2))
az_tel = -np.pi / 2. * np.ones(len(dl2))
src_pos_reco = utils.reco_source_position_sky(
dl2.x.values * u.m, dl2.y.values * u.m, dl2.reco_disp_dx.values * u.m,
dl2.reco_disp_dy.values * u.m, focal_length, alt_tel * u.rad,
az_tel * u.rad)
dl2['reco_alt'] = src_pos_reco.alt.rad
dl2['reco_az'] = src_pos_reco.az.rad
dl2['reco_type'] = classifier.predict(
dl2[classification_features]).astype(int)
probs = classifier.predict_proba(dl2[classification_features])
# This check is valid as long as we train on only two classes (gammas and protons)
if probs.shape[1] > 2:
raise ValueError(
"The classifier is predicting more than two classes, "
"the predicted probabilty to assign as gammaness is unclear."
"Please check training data")
# gammaness is the prediction probability for the first class (0)
dl2['gammaness'] = probs[:, 0]
return dl2
def main():
std_config = get_standard_config()
log.setLevel(logging.INFO)
handler = logging.StreamHandler()
logging.getLogger().addHandler(handler)
if args.config_file is not None:
config = replace_config(std_config, read_configuration_file(args.config_file))
else:
config = std_config
log.info(f"Tailcut config used: {config['tailcut']}")
foclen = OpticsDescription.from_name('LST').equivalent_focal_length
cam_table = Table.read(args.input_file, path="instrument/telescope/camera/LSTCam")
camera_geom = CameraGeometry.from_table(cam_table)
dl1_container = DL1ParametersContainer()
parameters_to_update = list(HillasParametersContainer().keys())
parameters_to_update.extend([
'concentration_cog',
'concentration_core',
'concentration_pixel',
'leakage_intensity_width_1',
'leakage_intensity_width_2',
'leakage_pixels_width_1',
'leakage_pixels_width_2',
'n_islands',
'intercept',
'time_gradient',
'n_pixels',
'wl',
'log_intensity'
])
nodes_keys = get_dataset_keys(args.input_file)
if args.noimage:
nodes_keys.remove(dl1_images_lstcam_key)
auto_merge_h5files([args.input_file], args.output_file, nodes_keys=nodes_keys)
with tables.open_file(args.input_file, mode='r') as input:
image_table = input.root[dl1_images_lstcam_key]
dl1_params_input = input.root[dl1_params_lstcam_key].colnames
disp_params = {'disp_dx', 'disp_dy', 'disp_norm', 'disp_angle', 'disp_sign'}
if set(dl1_params_input).intersection(disp_params):
parameters_to_update.extend(disp_params)
with tables.open_file(args.output_file, mode='a') as output:
params = output.root[dl1_params_lstcam_key].read()
for ii, row in enumerate(image_table):
dl1_container.reset()
image = row['image']
peak_time = row['peak_time']
signal_pixels = tailcuts_clean(camera_geom, image, **config['tailcut'])
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
num_islands, island_labels = number_of_islands(camera_geom, signal_pixels)
n_pixels_on_island = np.bincount(island_labels.astype(np.int))
n_pixels_on_island[0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
signal_pixels[island_labels != max_island_label] = False
hillas = hillas_parameters(camera_geom[signal_pixels], image[signal_pixels])
dl1_container.fill_hillas(hillas)
dl1_container.set_timing_features(camera_geom[signal_pixels],
image[signal_pixels],
peak_time[signal_pixels],
hillas)
dl1_container.set_leakage(camera_geom, image, signal_pixels)
dl1_container.set_concentration(camera_geom, image, hillas)
dl1_container.n_islands = num_islands
dl1_container.wl = dl1_container.width / dl1_container.length
dl1_container.n_pixels = n_pixels
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(np.arctan2(dl1_container.length, foclen))
dl1_container.width = width
dl1_container.length = length
dl1_container.log_intensity = np.log10(dl1_container.intensity)
if set(dl1_params_input).intersection(disp_params):
disp_dx, disp_dy, disp_norm, disp_angle, disp_sign = disp(
dl1_container['x'].to_value(u.m),
dl1_container['y'].to_value(u.m),
params['src_x'][ii],
params['src_y'][ii]
)
dl1_container['disp_dx'] = disp_dx
dl1_container['disp_dy'] = disp_dy
dl1_container['disp_norm'] = disp_norm
dl1_container['disp_angle'] = disp_angle
dl1_container['disp_sign'] = disp_sign
for p in parameters_to_update:
params[ii][p] = u.Quantity(dl1_container[p]).value
output.root[dl1_params_lstcam_key][:] = params
def r0_to_dl1(
input_filename=get_dataset_path('gamma_test_large.simtel.gz'),
output_filename=None,
custom_config={},
):
"""
Chain r0 to dl1
Save the extracted dl1 parameters in output_filename
Parameters
----------
input_filename: str
path to input file, default: `gamma_test_large.simtel.gz`
output_filename: str or None
path to output file, defaults to writing dl1 into the current directory
custom_config: path to a configuration file
Returns
-------
"""
if output_filename is None:
try:
run = parse_r0_filename(input_filename)
output_filename = run_to_dl1_filename(run.tel_id, run.run,
run.subrun)
except ValueError:
output_filename = r0_to_dl1_filename(Path(input_filename).name)
if os.path.exists(output_filename):
raise IOError(str(output_filename) + ' exists, exiting.')
config = replace_config(standard_config, custom_config)
custom_calibration = config["custom_calibration"]
source = EventSource(input_url=input_filename,
config=Config(config["source_config"]))
subarray = source.subarray
is_simu = source.is_simulation
metadata = global_metadata(source)
write_metadata(metadata, output_filename)
cal_mc = load_calibrator_from_config(config, subarray)
# minimum number of pe in a pixel to include it
# in calculation of muon ring time (peak sample):
min_pe_for_muon_t_calc = 10.
# Dictionary to store muon ring parameters
muon_parameters = create_muon_table()
# all this will be cleaned up in a next PR related to the configuration files
r1_dl1_calibrator = CameraCalibrator(
image_extractor_type=config['image_extractor'],
config=Config(config),
subarray=subarray)
if not is_simu:
# Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but
# using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous
r1_dl1_calibrator_for_muon_rings = CameraCalibrator(
image_extractor_type=config['image_extractor_for_muons'],
config=Config(config),
subarray=subarray)
# Component to process interleaved pedestal and flat-fields
calib_config = Config(config[config['calibration_product']])
calibration_calculator = CalibrationCalculator.from_name(
config['calibration_product'],
config=calib_config,
subarray=source.subarray)
calibration_index = DL1MonitoringEventIndexContainer()
dl1_container = DL1ParametersContainer()
extra_im = ExtraImageInfo()
extra_im.prefix = '' # get rid of the prefix
# Write extra information to the DL1 file
write_array_info(subarray, output_filename)
write_array_info_08(subarray, output_filename)
if is_simu:
write_mcheader(
source.simulation_config,
output_filename,
obs_id=source.obs_ids[0],
filters=HDF5_ZSTD_FILTERS,
metadata=metadata,
)
with HDF5TableWriter(
filename=output_filename,
group_name='dl1/event',
mode='a',
filters=HDF5_ZSTD_FILTERS,
add_prefix=True,
# overwrite=True,
) as writer:
if is_simu:
subarray = subarray
# build a mapping of tel_id back to tel_index:
# (note this should be part of SubarrayDescription)
idx = np.zeros(max(subarray.tel_indices) + 1)
for key, val in subarray.tel_indices.items():
idx[key] = val
# the final transform then needs the mapping and the number of telescopes
tel_list_transform = partial(
utils.expand_tel_list,
max_tels=max(subarray.tel) + 1,
)
writer.add_column_transform(table_name='subarray/trigger',
col_name='tels_with_trigger',
transform=tel_list_transform)
# Forcing filters for the dl1 dataset that are currently read from the pre-existing files
# This should be fixed in ctapipe and then corrected here
writer._h5file.filters = HDF5_ZSTD_FILTERS
logger.info(f"USING FILTERS: {writer._h5file.filters}")
for i, event in enumerate(source):
if i % 100 == 0:
logger.info(i)
event.dl0.prefix = ''
event.trigger.prefix = ''
if event.simulation is not None:
event.simulation.prefix = 'mc'
dl1_container.reset()
# write sub tables
if is_simu:
write_subarray_tables(writer, event, metadata)
if not custom_calibration:
cal_mc(event)
if config['mc_image_scaling_factor'] != 1:
rescale_dl1_charge(event,
config['mc_image_scaling_factor'])
else:
if i == 0:
# initialize the telescope
# FIXME? LST calibrator is only for one telescope
# it should be inside the telescope loop (?)
tel_id = calibration_calculator.tel_id
#initialize the event monitoring data
event.mon = source.r0_r1_calibrator.mon_data
# write the first calibration event (initialized from calibration h5 file)
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=True,
new_ff=True)
# flat-field or pedestal:
if (event.trigger.event_type == EventType.FLATFIELD
or event.trigger.event_type == EventType.SKY_PEDESTAL):
# process interleaved events (pedestals, ff, calibration)
new_ped_event, new_ff_event = calibration_calculator.process_interleaved(
event)
# write monitoring containers if updated
if new_ped_event or new_ff_event:
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=new_ped_event,
new_ff=new_ff_event)
# calibrate and gain select the event by hand for DL1
source.r0_r1_calibrator.calibrate(event)
# create image for all events
r1_dl1_calibrator(event)
# Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten.
# When the last version of the method is implemented, vol. reduction will be done at dl0
apply_volume_reduction(event, subarray, config)
# FIXME? This should be eventually done after we evaluate whether the image is
# a candidate muon ring. In that case the full image could be kept, or reduced
# only after the ring analysis is complete.
for ii, telescope_id in enumerate(event.dl1.tel.keys()):
dl1_container.reset()
# update the calibration index in the dl1 event container
dl1_container.calibration_id = calibration_index.calibration_id
dl1_container.fill_event_info(event)
tel = event.dl1.tel[telescope_id]
tel.prefix = '' # don't really need one
# remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST'
tel_name = str(subarray.tel[telescope_id])[4:]
if custom_calibration:
lst_calibration(event, telescope_id)
write_event = True
# Will determine whether this event has to be written to the
# DL1 output or not.
if is_simu:
dl1_container.fill_mc(event,
subarray.positions[telescope_id])
assert event.dl1.tel[telescope_id].image is not None
try:
get_dl1(
event,
subarray,
telescope_id,
dl1_container=dl1_container,
custom_config=config,
)
except HillasParameterizationError:
logging.exception(
'HillasParameterizationError in get_dl1()')
if not is_simu:
dl1_container.ucts_time = 0
# convert Time to unix timestamp in (UTC) to keep compatibility
# with older lstchain
# FIXME: just keep it as time, table writer and reader handle it
dl1_container.dragon_time = event.trigger.time.unix
dl1_container.tib_time = 0
dl1_container.ucts_trigger_type = event.lst.tel[
telescope_id].evt.ucts_trigger_type
dl1_container.trigger_type = event.lst.tel[
telescope_id].evt.tib_masked_trigger
else:
dl1_container.trigger_type = event.trigger.event_type
dl1_container.az_tel = event.pointing.tel[telescope_id].azimuth
dl1_container.alt_tel = event.pointing.tel[
telescope_id].altitude
dl1_container.trigger_time = event.trigger.time.unix
dl1_container.event_type = event.trigger.event_type
# FIXME: no need to read telescope characteristics like foclen for every event!
foclen = subarray.tel[
telescope_id].optics.equivalent_focal_length
mirror_area = u.Quantity(
subarray.tel[telescope_id].optics.mirror_area, u.m**2)
dl1_container.prefix = tel.prefix
# extra info for the image table
extra_im.tel_id = telescope_id
extra_im.selected_gain_channel = event.r1.tel[
telescope_id].selected_gain_channel
for container in [extra_im, dl1_container, event.r0, tel]:
add_global_metadata(container, metadata)
event.r0.prefix = ''
writer.write(table_name=f'telescope/image/{tel_name}',
containers=[event.index, tel, extra_im])
writer.write(table_name=f'telescope/parameters/{tel_name}',
containers=[event.index, dl1_container])
# Muon ring analysis, for real data only (MC is done starting from DL1 files)
if not is_simu:
bad_pixels = event.mon.tel[
telescope_id].calibration.unusable_pixels[0]
# Set to 0 unreliable pixels:
image = tel.image * (~bad_pixels)
# process only promising events, in terms of # of pixels with large signals:
if tag_pix_thr(image):
# re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings
numsamples = event.r1.tel[telescope_id].waveform.shape[
1] # not necessarily the same as in r0!
bad_pixels_hg = event.mon.tel[
telescope_id].calibration.unusable_pixels[0]
bad_pixels_lg = event.mon.tel[
telescope_id].calibration.unusable_pixels[1]
# Now set to 0 all samples in unreliable pixels. Important for global peak
# integrator in case of crazy pixels! TBD: can this be done in a simpler
# way?
bad_pixels = bad_pixels_hg | bad_pixels_lg
bad_waveform = np.transpose(
np.array(numsamples * [bad_pixels]))
# print('hg bad pixels:',np.where(bad_pixels_hg))
# print('lg bad pixels:',np.where(bad_pixels_lg))
event.r1.tel[telescope_id].waveform *= ~bad_waveform
r1_dl1_calibrator_for_muon_rings(event)
tel = event.dl1.tel[telescope_id]
image = tel.image * (~bad_pixels)
# Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum)
# perhaps the event is no longer promising (e.g. if it has a large time evolution)
if not tag_pix_thr(image):
good_ring = False
else:
# read geometry from event.inst. But not needed for every event. FIXME?
geom = subarray.tel[telescope_id].\
camera.geometry
muonintensityparam, dist_mask, \
ring_size, size_outside_ring, muonringparam, \
good_ring, radial_distribution, \
mean_pixel_charge_around_ring,\
muonpars = \
analyze_muon_event(subarray,
event.index.event_id,
image, geom, foclen,
mirror_area, False, '')
# mirror_area, True, './')
# (test) plot muon rings as png files
# Now we want to obtain the waveform sample (in HG & LG) at which the ring light peaks:
bright_pixels = image > min_pe_for_muon_t_calc
selected_gain = event.r1.tel[
telescope_id].selected_gain_channel
mask_hg = bright_pixels & (selected_gain == 0)
mask_lg = bright_pixels & (selected_gain == 1)
bright_pixels_waveforms_hg = event.r1.tel[
telescope_id].waveform[mask_hg, :]
bright_pixels_waveforms_lg = event.r1.tel[
telescope_id].waveform[mask_lg, :]
stacked_waveforms_hg = np.sum(
bright_pixels_waveforms_hg, axis=0)
stacked_waveforms_lg = np.sum(
bright_pixels_waveforms_lg, axis=0)
# stacked waveforms from all bright pixels; shape (ngains, nsamples)
hg_peak_sample = np.argmax(stacked_waveforms_hg,
axis=-1)
lg_peak_sample = np.argmax(stacked_waveforms_lg,
axis=-1)
if good_ring:
fill_muon_event(-1, muon_parameters, good_ring,
event.index.event_id,
dl1_container.dragon_time,
muonintensityparam, dist_mask,
muonringparam, radial_distribution,
ring_size, size_outside_ring,
mean_pixel_charge_around_ring,
muonpars, hg_peak_sample,
lg_peak_sample)
# writes mc information per telescope, including photo electron image
if (is_simu and config['write_pe_image']
and event.simulation.tel[telescope_id].true_image
is not None and
event.simulation.tel[telescope_id].true_image.any()):
event.simulation.tel[telescope_id].prefix = ''
writer.write(table_name=f'simulation/{tel_name}',
containers=[
event.simulation.tel[telescope_id],
extra_im
])
if not is_simu:
# at the end of event loop ask calculation of remaining interleaved statistics
new_ped, new_ff = calibration_calculator.output_interleaved_results(
event)
# write monitoring events
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=new_ped,
new_ff=new_ff)
if is_simu:
# Reconstruct source position from disp for all events and write the result in the output file
for tel_name in ['LST_LSTCam']:
focal = OpticsDescription.from_name(
tel_name.split('_')[0]).equivalent_focal_length
add_disp_to_parameters_table(output_filename,
dl1_params_lstcam_key, focal)
# Write energy histogram from simtel file and extra metadata
# ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever
if source.max_events is None:
write_simtel_energy_histogram(source,
output_filename,
obs_id=event.index.obs_id,
metadata=metadata)
else:
dir, name = os.path.split(output_filename)
name = name.replace('dl1', 'muons').replace('LST-1.1', 'LST-1')
# Consider the possibilities of DL1 files with .fits.h5 & .h5 ending:
name = name.replace('.fits.h5', '.fits').replace('.h5', '.fits')
muon_output_filename = Path(dir, name)
table = Table(muon_parameters)
table.write(muon_output_filename, format='fits', overwrite=True)
def r0_to_dl1(input_filename=get_dataset_path('gamma_test_large.simtel.gz'),
output_filename=None,
custom_config={},
pedestal_path=None,
calibration_path=None,
time_calibration_path=None,
pointing_file_path=None,
ucts_t0_dragon=math.nan,
dragon_counter0=math.nan,
ucts_t0_tib=math.nan,
tib_counter0=math.nan):
"""
Chain r0 to dl1
Save the extracted dl1 parameters in output_filename
Parameters
----------
input_filename: str
path to input file, default: `gamma_test_large.simtel.gz`
output_filename: str or None
path to output file, defaults to writing dl1 into the current directory
custom_config: path to a configuration file
pedestal_path: Path to the DRS4 pedestal file
calibration_path: Path to the file with calibration constants and
pedestals
time_calibration_path: Path to the DRS4 time correction file
pointing_file_path: path to the Drive log with the pointing information
ucts_t0_dragon: first valid ucts_time
dragon_counter0: Dragon counter corresponding to ucts_t0_dragon
ucts_t0_tib: first valid ucts_time for the first valid TIB counter
tib_counter0: first valid TIB counter
Returns
-------
"""
if output_filename is None:
try:
run = parse_r0_filename(input_filename)
output_filename = run_to_dl1_filename(run.tel_id, run.run,
run.subrun)
except ValueError:
output_filename = r0_to_dl1_filename(Path(input_filename).name)
if os.path.exists(output_filename):
raise IOError(str(output_filename) + ' exists, exiting.')
config = replace_config(standard_config, custom_config)
custom_calibration = config["custom_calibration"]
gain_selector = load_gain_selector_from_config(config)
# FIXME for ctapipe 0.8, str should be removed, as Path is supported
source = event_source(str(input_filename))
subarray = source.subarray
is_simu = source.is_simulation
source.allowed_tels = config["allowed_tels"]
if config["max_events"] is not None:
source.max_events = config["max_events"]
metadata = global_metadata(source)
write_metadata(metadata, output_filename)
cal_mc = load_calibrator_from_config(config, subarray)
# minimum number of pe in a pixel to include it
# in calculation of muon ring time (peak sample):
min_pe_for_muon_t_calc = 10.
# Dictionary to store muon ring parameters
muon_parameters = create_muon_table()
if not is_simu:
# TODO : add DRS4 calibration config in config file, read it and pass it here
r0_r1_calibrator = LSTR0Corrections(
pedestal_path=pedestal_path,
tel_id=1,
)
# all this will be cleaned up in a next PR related to the configuration files
r1_dl1_calibrator = LSTCameraCalibrator(
calibration_path=calibration_path,
time_calibration_path=time_calibration_path,
extractor_product=config['image_extractor'],
gain_threshold=Config(config).gain_selector_config['threshold'],
charge_scale=config['charge_scale'],
config=Config(config),
allowed_tels=[1],
subarray=subarray)
# Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but
# using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous
r1_dl1_calibrator_for_muon_rings = LSTCameraCalibrator(
calibration_path=calibration_path,
time_calibration_path=time_calibration_path,
extractor_product=config['image_extractor_for_muons'],
gain_threshold=Config(config).gain_selector_config['threshold'],
charge_scale=config['charge_scale'],
config=Config(config),
allowed_tels=[1],
subarray=subarray)
# Component to process interleaved pedestal and flat-fields
calib_config = Config(config[config['calibration_product']])
# set time calibration path for flatfield trailet ()
calib_config.FlasherFlatFieldCalculator.time_calibration_path = time_calibration_path
calibration_calculator = CalibrationCalculator.from_name(
config['calibration_product'],
config=calib_config,
subarray=source.subarray)
calibration_index = DL1MonitoringEventIndexContainer()
dl1_container = DL1ParametersContainer()
if pointing_file_path:
# Open drive report
pointings = PointingPosition(drive_path=pointing_file_path)
drive_data = pointings._read_drive_report()
extra_im = ExtraImageInfo()
extra_im.prefix = '' # get rid of the prefix
# get the first event to write array info and mc header
event_iter = iter(source)
first_event = next(event_iter)
# Write extra information to the DL1 file
write_array_info(subarray, output_filename)
write_array_info_08(subarray, output_filename)
if is_simu:
write_mcheader(
first_event.mcheader,
output_filename,
obs_id=first_event.index.obs_id,
filters=filters,
metadata=metadata,
)
with HDF5TableWriter(
filename=output_filename,
group_name='dl1/event',
mode='a',
filters=filters,
add_prefix=True,
# overwrite=True,
) as writer:
if is_simu:
subarray = subarray
# build a mapping of tel_id back to tel_index:
# (note this should be part of SubarrayDescription)
idx = np.zeros(max(subarray.tel_indices) + 1)
for key, val in subarray.tel_indices.items():
idx[key] = val
# the final transform then needs the mapping and the number of telescopes
tel_list_transform = partial(
utils.expand_tel_list,
max_tels=max(subarray.tel) + 1,
)
writer.add_column_transform(table_name='subarray/trigger',
col_name='tels_with_trigger',
transform=tel_list_transform)
# Forcing filters for the dl1 dataset that are currently read from the pre-existing files
# This should be fixed in ctapipe and then corrected here
writer._h5file.filters = filters
logger.info(f"USING FILTERS: {writer._h5file.filters}")
first_valid_ucts = None
first_valid_ucts_tib = None
previous_ucts_time_unix = []
previous_ucts_trigger_type = []
for i, event in enumerate(chain([first_event], event_iter)):
if i % 100 == 0:
logger.info(i)
event.dl0.prefix = ''
event.mc.prefix = 'mc'
event.trigger.prefix = ''
dl1_container.reset()
# write sub tables
if is_simu:
write_subarray_tables(writer, event, metadata)
if not custom_calibration:
cal_mc(event)
if config['mc_image_scaling_factor'] != 1:
rescale_dl1_charge(event,
config['mc_image_scaling_factor'])
else:
if i == 0:
# initialize the telescope
# FIXME? LST calibrator is only for one telescope
# it should be inside the telescope loop (?)
tel_id = calibration_calculator.tel_id
# write the first calibration event (initialized from calibration h5 file)
write_calibration_data(
writer,
calibration_index,
r1_dl1_calibrator.mon_data.tel[tel_id],
new_ped=True,
new_ff=True)
# drs4 calibrations
r0_r1_calibrator.calibrate(event)
# process interleaved events (pedestals, ff, calibration)
new_ped_event, new_ff_event = calibration_calculator.process_interleaved(
event)
# write monitoring containers if updated
if new_ped_event or new_ff_event:
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=new_ped_event,
new_ff=new_ff_event)
# calibrate and extract image from event
r1_dl1_calibrator(event)
# Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten.
# When the last version of the method is implemented, vol. reduction will be done at dl0
apply_volume_reduction(event, subarray, config)
# FIXME? This should be eventually done after we evaluate whether the image is
# a candidate muon ring. In that case the full image could be kept, or reduced
# only after the ring analysis is complete.
for ii, telescope_id in enumerate(event.r0.tels_with_data):
dl1_container.reset()
# update the calibration index in the dl1 event container
dl1_container.calibration_id = calibration_index.calibration_id
dl1_container.fill_event_info(event)
tel = event.dl1.tel[telescope_id]
tel.prefix = '' # don't really need one
# remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST'
tel_name = str(subarray.tel[telescope_id])[4:]
if custom_calibration:
lst_calibration(event, telescope_id)
write_event = True
# Will determine whether this event has to be written to the
# DL1 output or not.
if is_simu:
dl1_container.fill_mc(event,
subarray.positions[telescope_id])
try:
get_dl1(event,
subarray,
telescope_id,
dl1_container=dl1_container,
custom_config=config,
use_main_island=True)
except HillasParameterizationError:
logging.exception(
'HillasParameterizationError in get_dl1()')
if not is_simu:
# GPS + WRS + UCTS is now working in its nominal configuration.
# These TS are stored into ucts_time container.
# TS can be alternatively calculated from the TIB and
# Dragon modules counters based on the first valid UCTS TS
# as the reference point. For the time being, the three TS
# are stored in the DL1 files for checking purposes.
module_id = 82 # Get counters from the central Dragon module
if math.isnan(ucts_t0_dragon) and math.isnan(dragon_counter0) \
and math.isnan(ucts_t0_tib) and math.isnan(tib_counter0):
# Dragon/TIB timestamps not based on a valid absolute reference timestamp
dragon_time = (event.lst.tel[telescope_id].svc.date +
event.lst.tel[telescope_id].evt.
pps_counter[module_id] +
event.lst.tel[telescope_id].evt.
tenMHz_counter[module_id] * 10**(-7))
tib_time = (
event.lst.tel[telescope_id].svc.date +
event.lst.tel[telescope_id].evt.tib_pps_counter +
event.lst.tel[telescope_id].evt.tib_tenMHz_counter
* 10**(-7))
if event.lst.tel[
telescope_id].evt.extdevices_presence & 2:
# UCTS presence flag is OK
ucts_time = event.lst.tel[
telescope_id].evt.ucts_timestamp * 1e-9 # secs
if first_valid_ucts is None:
first_valid_ucts = ucts_time
initial_dragon_counter = (
event.lst.tel[telescope_id].evt.
pps_counter[module_id] +
event.lst.tel[telescope_id].evt.
tenMHz_counter[module_id] * 10**(-7))
logger.warning(
f"Dragon timestamps not based on a valid absolute reference timestamp. "
f"Consider using the following initial values \n"
f"Event ID: {event.index.event_id}, "
f"First valid UCTS timestamp: {first_valid_ucts:.9f} s, "
f"corresponding Dragon counter {initial_dragon_counter:.9f} s"
)
if event.lst.tel[telescope_id].evt.extdevices_presence & 1 \
and first_valid_ucts_tib is None:
# Both TIB and UCTS presence flags are OK
first_valid_ucts_tib = ucts_time
initial_tib_counter = (
event.lst.tel[telescope_id].evt.
tib_pps_counter +
event.lst.tel[telescope_id].evt.
tib_tenMHz_counter * 10**(-7))
logger.warning(
f"TIB timestamps not based on a valid absolute reference timestamp. "
f"Consider using the following initial values \n"
f"Event ID: {event.index.event_id}, UCTS timestamp corresponding to "
f"the first valid TIB counter: {first_valid_ucts_tib:.9f} s, "
f"corresponding TIB counter {initial_tib_counter:.9f} s"
)
else:
ucts_time = math.nan
else:
# Dragon/TIB timestamps based on a valid absolute reference UCTS timestamp
dragon_time = (
(ucts_t0_dragon - dragon_counter0) * 1e-9 + # secs
event.lst.tel[telescope_id].evt.
pps_counter[module_id] +
event.lst.tel[telescope_id].evt.
tenMHz_counter[module_id] * 10**(-7))
tib_time = (
(ucts_t0_tib - tib_counter0) * 1e-9 + # secs
event.lst.tel[telescope_id].evt.tib_pps_counter +
event.lst.tel[telescope_id].evt.tib_tenMHz_counter
* 10**(-7))
if event.lst.tel[
telescope_id].evt.extdevices_presence & 2:
# UCTS presence flag is OK
ucts_time = event.lst.tel[
telescope_id].evt.ucts_timestamp * 1e-9 # secs
if first_valid_ucts is None:
first_valid_ucts = ucts_time
if first_valid_ucts_tib is None \
and event.lst.tel[telescope_id].evt.extdevices_presence & 1:
first_valid_ucts_tib = ucts_time
else:
ucts_time = math.nan
# FIXME: directly use unix_tai format whenever astropy v4.1 is out
ucts_time_utc = unix_tai_to_time(ucts_time)
dragon_time_utc = unix_tai_to_time(dragon_time)
tib_time_utc = unix_tai_to_time(tib_time)
dl1_container.ucts_time = ucts_time_utc.unix
dl1_container.dragon_time = dragon_time_utc.unix
dl1_container.tib_time = tib_time_utc.unix
# Until the TIB trigger_type is fully reliable, we also add
# the ucts_trigger_type to the data
dl1_container.ucts_trigger_type = event.lst.tel[
telescope_id].evt.ucts_trigger_type
# Due to a DAQ bug, sometimes there are 'jumps' in the
# UCTS info in the raw files. After one such jump,
# all the UCTS info attached to an event actually
# corresponds to the next event. This one-event
# shift stays like that until there is another jump
# (then it becomes a 2-event shift and so on). We will
# keep track of those jumps, by storing the UCTS info
# of the previously read events in the list
# previous_ucts_time_unix. The list has one element
# for each of the jumps, so if there has been just
# one jump we have the UCTS info of the previous
# event only (which truly corresponds to the
# current event). If there have been n jumps, we keep
# the past n events. The info to be used for
# the current event is always the first element of
# the array, previous_ucts_time_unix[0], whereas the
# current event's (wrong) ucts info is placed last in
# the array. Each time the first array element is
# used, it is removed and the rest move up in the
# list. We have another similar array for the trigger
# types, previous_ucts_trigger_type
#
if len(previous_ucts_time_unix) > 0:
# keep the time & trigger type read for this
# event (which really correspond to a later event):
current_ucts_time = dl1_container.ucts_time
current_ucts_trigger_type = dl1_container.ucts_trigger_type
# put in dl1_container the proper time for this
# event:
dl1_container.ucts_time = \
previous_ucts_time_unix.pop(0)
dl1_container.ucts_trigger_type = \
previous_ucts_trigger_type.pop(0)
# now put the current values last in the list,
# for later use:
previous_ucts_time_unix.append(current_ucts_time)
previous_ucts_trigger_type.\
append(current_ucts_trigger_type)
# Now check consistency of UCTS and Dragon times. If
# UCTS time is ahead of Dragon time by more than
# 1.e-6 s, most likely the UCTS info has been
# lost for this event (i.e. there has been another
# 'jump' of those described above), and the one we have
# actually corresponds to the next event. So we put it
# back first in the list, to assign it to the next
# event. We also move the other elements down in the
# list, which will now be one element longer.
# We leave the current event with the same time,
# which will be approximately correct (depending on
# event rate), and set its ucts_trigger_type to -1,
# which will tell us a jump happened and hence this
# event does not have proper UCTS info.
if dl1_container.ucts_time - dl1_container.dragon_time > 1.e-6:
previous_ucts_time_unix.\
insert( 0, dl1_container.ucts_time)
previous_ucts_trigger_type.\
insert(0, dl1_container.ucts_trigger_type)
dl1_container.ucts_trigger_type = -1
# Select the timestamps to be used for pointing interpolation
if config['timestamps_pointing'] == "ucts":
event_timestamps = dl1_container.ucts_time
elif config['timestamps_pointing'] == "dragon":
event_timestamps = dragon_time_utc.unix
elif config['timestamps_pointing'] == "tib":
event_timestamps = tib_time_utc.unix
else:
raise ValueError(
"The timestamps_pointing option is not a valid one. \
Try ucts (default), dragon or tib.")
if pointing_file_path and event_timestamps > 0:
azimuth, altitude = pointings.cal_pointingposition(
event_timestamps, drive_data)
event.pointing.tel[telescope_id].azimuth = azimuth
event.pointing.tel[telescope_id].altitude = altitude
dl1_container.az_tel = azimuth
dl1_container.alt_tel = altitude
else:
dl1_container.az_tel = u.Quantity(np.nan, u.rad)
dl1_container.alt_tel = u.Quantity(np.nan, u.rad)
dl1_container.trigger_time = event.r0.tel[
telescope_id].trigger_time
dl1_container.trigger_type = event.r0.tel[
telescope_id].trigger_type
# FIXME: no need to read telescope characteristics like foclen for every event!
foclen = subarray.tel[
telescope_id].optics.equivalent_focal_length
mirror_area = u.Quantity(
subarray.tel[telescope_id].optics.mirror_area, u.m**2)
dl1_container.prefix = tel.prefix
# extra info for the image table
extra_im.tel_id = telescope_id
extra_im.selected_gain_channel = event.r1.tel[
telescope_id].selected_gain_channel
for container in [extra_im, dl1_container, event.r0, tel]:
add_global_metadata(container, metadata)
event.r0.prefix = ''
writer.write(table_name=f'telescope/image/{tel_name}',
containers=[event.index, tel, extra_im])
writer.write(table_name=f'telescope/parameters/{tel_name}',
containers=[event.index, dl1_container])
# Muon ring analysis, for real data only (MC is done starting from DL1 files)
if not is_simu:
bad_pixels = event.mon.tel[
telescope_id].calibration.unusable_pixels[0]
# Set to 0 unreliable pixels:
image = tel.image * (~bad_pixels)
# process only promising events, in terms of # of pixels with large signals:
if tag_pix_thr(image):
# re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings
numsamples = event.r1.tel[telescope_id].waveform.shape[
2] # not necessarily the same as in r0!
bad_pixels_hg = event.mon.tel[
telescope_id].calibration.unusable_pixels[0]
bad_pixels_lg = event.mon.tel[
telescope_id].calibration.unusable_pixels[1]
# Now set to 0 all samples in unreliable pixels. Important for global peak
# integrator in case of crazy pixels! TBD: can this be done in a simpler
# way?
bad_waveform = np.array(
([
np.transpose(
np.array(numsamples * [bad_pixels_hg])),
np.transpose(
np.array(numsamples * [bad_pixels_lg]))
]))
# print('hg bad pixels:',np.where(bad_pixels_hg))
# print('lg bad pixels:',np.where(bad_pixels_lg))
event.r1.tel[telescope_id].waveform *= ~bad_waveform
r1_dl1_calibrator_for_muon_rings(event)
tel = event.dl1.tel[telescope_id]
image = tel.image * (~bad_pixels)
# Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum)
# perhaps the event is no longer promising (e.g. if it has a large time evolution)
if not tag_pix_thr(image):
good_ring = False
else:
# read geometry from event.inst. But not needed for every event. FIXME?
geom = subarray.tel[telescope_id].\
camera.geometry
muonintensityparam, dist_mask, \
ring_size, size_outside_ring, muonringparam, \
good_ring, radial_distribution, \
mean_pixel_charge_around_ring,\
muonpars = \
analyze_muon_event(subarray,
event.index.event_id,
image, geom, foclen,
mirror_area, False, '')
# mirror_area, True, './')
# (test) plot muon rings as png files
# Now we want to obtain the waveform sample (in HG and LG) at which the ring light peaks:
bright_pixels_waveforms = event.r1.tel[
telescope_id].waveform[:, image >
min_pe_for_muon_t_calc, :]
stacked_waveforms = np.sum(bright_pixels_waveforms,
axis=-2)
# stacked waveforms from all bright pixels; shape (ngains, nsamples)
hg_peak_sample = np.argmax(stacked_waveforms,
axis=-1)[0]
lg_peak_sample = np.argmax(stacked_waveforms,
axis=-1)[1]
if good_ring:
fill_muon_event(-1, muon_parameters, good_ring,
event.index.event_id, dragon_time,
muonintensityparam, dist_mask,
muonringparam, radial_distribution,
ring_size, size_outside_ring,
mean_pixel_charge_around_ring,
muonpars, hg_peak_sample,
lg_peak_sample)
# writes mc information per telescope, including photo electron image
if is_simu \
and (event.mc.tel[telescope_id].true_image > 0).any() \
and config['write_pe_image']:
event.mc.tel[telescope_id].prefix = ''
writer.write(
table_name=f'simulation/{tel_name}',
containers=[event.mc.tel[telescope_id], extra_im])
if not is_simu:
# at the end of event loop ask calculation of remaining interleaved statistics
new_ped, new_ff = calibration_calculator.output_interleaved_results(
event)
# write monitoring events
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=new_ped,
new_ff=new_ff)
if first_valid_ucts is None:
logger.warning("Not valid UCTS timestamp found")
if first_valid_ucts_tib is None:
logger.warning("Not valid TIB counter value found")
if is_simu:
# Reconstruct source position from disp for all events and write the result in the output file
for tel_name in ['LST_LSTCam']:
focal = OpticsDescription.from_name(
tel_name.split('_')[0]).equivalent_focal_length
add_disp_to_parameters_table(output_filename,
dl1_params_lstcam_key, focal)
# Write energy histogram from simtel file and extra metadata
# ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever
if source.max_events is None:
write_simtel_energy_histogram(source,
output_filename,
obs_id=event.index.obs_id,
metadata=metadata)
else:
dir, name = os.path.split(output_filename)
name = name.replace('dl1', 'muons').replace('LST-1.1', 'LST-1')
# Consider the possibilities of DL1 files with .fits.h5 & .h5 ending:
name = name.replace('.fits.h5', '.fits').replace('.h5', '.fits')
muon_output_filename = Path(dir, name)
table = Table(muon_parameters)
table.write(muon_output_filename, format='fits', overwrite=True)
# Produce the dl1 datacheck .h5 file:
check_dl1(output_filename,
Path(output_filename).parent,
max_cores=1,
create_pdf=False)
def get_events(filename,
storedata=False,
test=False,
concatenate=False,
storeimg=False,
outdir='./results/'):
"""
Depreciated, use r0_to_dl1.
Read a Simtelarray file, extract pixels charge, calculate image
parameters and timing parameters and store the result in an hdf5
file.
Parameters:
-----------
filename: str
Name of the simtelarray file.
storedata: boolean
True: store extracted data in a hdf5 file
concatenate: boolean
True: store the extracted data at the end of an existing file
storeimg: boolean
True: store also pixel data
outdir: srt
Output directory
Returns:
--------
pandas DataFrame: output
"""
from warnings import warn
warn("Deprecated: use r0_to_dl1")
#Particle type:
particle_type = utils.guess_type(filename)
#Create data frame where DL2 data will be stored:
features = [
'obs_id',
'event_id',
'mc_energy',
'mc_alt',
'mc_az',
'mc_core_x',
'mc_core_y',
'mc_h_first_int',
'mc_type',
'gps_time',
'width',
'length',
'wl',
'phi',
'psi',
'r',
'x',
'y',
'intensity',
'skewness',
'kurtosis',
'mc_alt_tel',
'mc_az_tel',
'mc_core_distance',
'mc_x_max',
'time_gradient',
'intercept',
'src_x',
'src_y',
'disp_norm',
]
output = pd.DataFrame(columns=features)
#Read LST1 events:
source = event_source(input_url=filename,
allowed_tels={1}) #Open Simtelarray file
#Cleaning levels:
level1 = {'LSTCam': 6.}
level2 = level1.copy()
# We use as second cleaning level just half of the first cleaning level
for key in level2:
level2[key] *= 0.5
log10pixelHGsignal = {}
survived = {}
imagedata = np.array([])
for key in level1:
log10pixelHGsignal[key] = []
survived[key] = []
i = 0
for event in source:
if i % 100 == 0:
print("EVENT_ID: ", event.r0.event_id, "TELS: ",
event.r0.tels_with_data, "MC Energy:", event.mc.energy)
i = i + 1
ntels = len(event.r0.tels_with_data)
if test == True and i > 1000: # for quick tests
break
for ii, tel_id in enumerate(event.r0.tels_with_data):
geom = event.inst.subarray.tel[tel_id].camera #Camera geometry
tel_coords = event.inst.subarray.tel_coords[
event.inst.subarray.tel_indices[tel_id]]
data = event.r0.tel[tel_id].waveform
ped = event.mc.tel[tel_id].pedestal # the pedestal is the
#average (for pedestal events) of the *sum* of all samples,
#from sim_telarray
nsamples = data.shape[2] # total number of samples
# Subtract pedestal baseline. atleast_3d converts 2D to 3D matrix
pedcorrectedsamples = data - np.atleast_3d(ped) / nsamples
integrator = LocalPeakWindowSum()
integration, pulse_time = integrator(
pedcorrectedsamples
) # these are 2D matrices num_gains * num_pixels
chan = 0 # high gain used for now...
signals = integration[chan].astype(float)
dc2pe = event.mc.tel[tel_id].dc_to_pe # numgains * numpixels
signals *= dc2pe[chan]
# Add all individual pixel signals to the numpy array of the
# corresponding camera inside the log10pixelsignal dictionary
log10pixelHGsignal[str(geom)].extend(np.log10(signals))
# Apply image cleaning
cleanmask = tailcuts_clean(geom,
signals,
picture_thresh=level1[str(geom)],
boundary_thresh=level2[str(geom)],
keep_isolated_pixels=False,
min_number_picture_neighbors=1)
survived[str(geom)].extend(cleanmask)
clean = signals.copy()
clean[~cleanmask] = 0.0 # set to 0 pixels which did not
# survive cleaning
if np.max(clean) < 1.e-6: # skip images with no pixels
continue
# Calculate image parameters
hillas = hillas_parameters(geom, clean)
foclen = event.inst.subarray.tel[
tel_id].optics.equivalent_focal_length
w = np.rad2deg(np.arctan2(hillas.width, foclen))
l = np.rad2deg(np.arctan2(hillas.length, foclen))
#Calculate Timing parameters
peak_time = units.Quantity(pulse_time[chan]) * units.Unit("ns")
timepars = time.timing_parameters(geom, clean, peak_time, hillas)
if w >= 0:
if storeimg == True:
if imagedata.size == 0:
imagedata = clean
else:
imagedata = np.vstack([imagedata,
clean]) #Pixel content
#Hillas parameters
width = w.value
length = l.value
phi = hillas.phi.value
psi = hillas.psi.value
r = hillas.r.value
x = hillas.x.value
y = hillas.y.value
intensity = np.log10(hillas.intensity)
skewness = hillas.skewness
kurtosis = hillas.kurtosis
#MC data:
obs_id = event.r0.obs_id
event_id = event.r0.event_id
mc_energy = np.log10(event.mc.energy.value *
1e3) #Log10(Energy) in GeV
mc_alt = event.mc.alt.value
mc_az = event.mc.az.value
mc_core_x = event.mc.core_x.value
mc_core_y = event.mc.core_y.value
mc_h_first_int = event.mc.h_first_int.value
mc_type = event.mc.shower_primary_id
mc_az_tel = event.mcheader.run_array_direction[0].value
mc_alt_tel = event.mcheader.run_array_direction[1].value
mc_x_max = event.mc.x_max.value
gps_time = event.trig.gps_time.value
#Calculate mc_core_distance parameters
mc_core_distance = np.sqrt(
(tel_coords.x.value - event.mc.core_x.value)**2 +
(tel_coords.y.value - event.mc.core_y.value)**2)
#Timing parameters
time_gradient = timepars['slope'].value
intercept = timepars['intercept']
#Calculate disp_ and Source position in camera coordinates
tel = OpticsDescription.from_name(
'LST') #Telescope description
focal_length = tel.equivalent_focal_length.value
sourcepos = utils.cal_cam_source_pos(mc_alt, mc_az, mc_alt_tel,
mc_az_tel, focal_length)
src_x = sourcepos[0]
src_y = sourcepos[1]
disp = utils.disp_norm(sourcepos[0], sourcepos[1], x, y)
eventdf = pd.DataFrame([[
obs_id, event_id, mc_energy, mc_alt, mc_az, mc_core_x,
mc_core_y, mc_h_first_int, mc_type, gps_time, width,
length, width / length, phi, psi, r, x, y, intensity,
skewness, kurtosis, mc_alt_tel, mc_az_tel,
mc_core_distance, mc_x_max, time_gradient, intercept,
src_x, src_y, disp, mc_type
]],
columns=features)
output = output.append(eventdf, ignore_index=True)
outfile = outdir + particle_type + '_events.hdf5'
if storedata == True:
if (concatenate == False
or (concatenate == True
and np.DataSource().exists(outfile) == False)):
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
if storeimg == True:
f = h5py.File(outfile, 'r+')
f.create_dataset('images', data=imagedata)
f.close()
else:
if storeimg == True:
f = h5py.File(outfile, 'r')
images = f['images']
del f['images']
images = np.vstack([images, imagedata])
f.close()
saved = pd.read_hdf(outfile, key=particle_type + '_events')
output = saved.append(output, ignore_index=True)
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
f = h5py.File(outfile, 'r+')
f.create_dataset('images', data=images)
f.close()
else:
saved = pd.read_hdf(outfile, key=particle_type + '_events')
output = saved.append(output, ignore_index=True)
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
del source
return output
def get_events(filename,
storedata=False,
concatenate=False,
storeimg=False,
outdir='./results/'):
"""
Read a Simtelarray file, extract pixels charge, calculate image parameters and timing
parameters and store the result in an hdf5 file.
Parameters:
filename: str
Name of the simtelarray file.
storedata: boolean
True: store extracted data in a hdf5 file
concatenate: boolean
True: store the extracted data at the end of an existing file
storeimg: boolean
True: store also pixel data
outdir: srt
Output directory
"""
#Particle type:
particle_type = guess_type(filename)
#Create data frame where DL2 data will be stored:
features = [
'ObsID', 'EvID', 'mcEnergy', 'mcAlt', 'mcAz', 'mcCore_x', 'mcCore_y',
'mcHfirst', 'mcType', 'GPStime', 'width', 'length', 'w/l', 'phi',
'psi', 'r', 'x', 'y', 'intensity', 'skewness', 'kurtosis', 'mcAlttel',
'mcAztel', 'impact', 'mcXmax', 'time_gradient', 'intercept', 'SrcX',
'SrcY', 'disp', 'hadroness'
]
output = pd.DataFrame(columns=features)
#Read LST1 events:
source = EventSourceFactory.produce(
input_url=filename, allowed_tels={1}) #Open Simtelarray file
#Cleaning levels:
level1 = {'LSTCam': 6.}
level2 = level1.copy()
# We use as second cleaning level just half of the first cleaning level
for key in level2:
level2[key] *= 0.5
log10pixelHGsignal = {}
survived = {}
imagedata = np.array([])
for key in level1:
log10pixelHGsignal[key] = []
survived[key] = []
i = 0
for event in source:
if i % 100 == 0:
print("EVENT_ID: ", event.r0.event_id, "TELS: ",
event.r0.tels_with_data, "MC Energy:", event.mc.energy)
i = i + 1
ntels = len(event.r0.tels_with_data)
'''
if i > 100: # for quick tests
break
'''
for ii, tel_id in enumerate(event.r0.tels_with_data):
geom = event.inst.subarray.tel[tel_id].camera #Camera geometry
tel_coords = event.inst.subarray.tel_coords[
event.inst.subarray.tel_indices[tel_id]]
data = event.r0.tel[tel_id].waveform
ped = event.mc.tel[tel_id].pedestal
# the pedestal is the average (for pedestal events) of the *sum* of all samples, from sim_telarray
nsamples = data.shape[2] # total number of samples
pedcorrectedsamples = data - np.atleast_3d(
ped
) / nsamples # Subtract pedestal baseline. atleast_3d converts 2D to 3D matrix
integrator = LocalPeakIntegrator(None, None)
integration, peakpos, window = integrator.extract_charge(
pedcorrectedsamples
) # these are 2D matrices num_gains * num_pixels
chan = 0 # high gain used for now...
signals = integration[chan].astype(float)
dc2pe = event.mc.tel[tel_id].dc_to_pe # numgains * numpixels
signals *= dc2pe[chan]
# Add all individual pixel signals to the numpy array of the corresponding camera inside the log10pixelsignal dictionary
log10pixelHGsignal[str(geom)].extend(
np.log10(signals)
) # This seems to be faster like this, with normal python lists
# Apply image cleaning
cleanmask = tailcuts_clean(geom,
signals,
picture_thresh=level1[str(geom)],
boundary_thresh=level2[str(geom)],
keep_isolated_pixels=False,
min_number_picture_neighbors=1)
survived[str(geom)].extend(
cleanmask
) # This seems to be faster like this, with normal python lists
clean = signals.copy()
clean[
~cleanmask] = 0.0 # set to 0 pixels which did not survive cleaning
if np.max(clean) < 1.e-6: # skip images with no pixels
continue
# Calculate image parameters
hillas = hillas_parameters(
geom,
clean) # this one gives some warnings invalid value in sqrt
foclen = event.inst.subarray.tel[
tel_id].optics.equivalent_focal_length
w = np.rad2deg(np.arctan2(hillas.width, foclen))
l = np.rad2deg(np.arctan2(hillas.length, foclen))
#Calculate Timing parameters
peak_time = units.Quantity(peakpos[chan]) * units.Unit("ns")
timepars = time.timing_parameters(geom.pix_x, geom.pix_y, clean,
peak_time, hillas.psi)
if w >= 0:
if storeimg == True:
if imagedata.size == 0:
imagedata = clean
else:
imagedata = np.vstack([imagedata,
clean]) #Pixel content
width = w.value
length = l.value
phi = hillas.phi.value
psi = hillas.psi.value
r = hillas.r.value
x = hillas.x.value
y = hillas.y.value
intensity = np.log10(hillas.intensity)
skewness = hillas.skewness
kurtosis = hillas.kurtosis
#Store parameters from event and MC:
ObsID = event.r0.obs_id
EvID = event.r0.event_id
mcEnergy = np.log10(event.mc.energy.value *
1e3) #Log10(Energy) in GeV
mcAlt = event.mc.alt.value
mcAz = event.mc.az.value
mcCore_x = event.mc.core_x.value
mcCore_y = event.mc.core_y.value
mcHfirst = event.mc.h_first_int.value
mcType = event.mc.shower_primary_id
mcAztel = event.mcheader.run_array_direction[0].value
mcAlttel = event.mcheader.run_array_direction[1].value
mcXmax = event.mc.x_max.value
GPStime = event.trig.gps_time.value
impact = np.sqrt(
(tel_coords.x.value - event.mc.core_x.value)**2 +
(tel_coords.y.value - event.mc.core_y.value)**2)
time_gradient = timepars[0].value
intercept = timepars[1].value
#Calculate Disp and Source position in camera coordinates
tel = OpticsDescription.from_name(
'LST') #Telescope description
focal_length = tel.equivalent_focal_length.value
sourcepos = transformations.calc_CamSourcePos(
mcAlt, mcAz, mcAlttel, mcAztel, focal_length)
SrcX = sourcepos[0]
SrcY = sourcepos[1]
disp = transformations.calc_DISP(sourcepos[0], sourcepos[1], x,
y)
hadroness = 0
if particle_type == 'proton':
hadroness = 1
eventdf = pd.DataFrame([[
ObsID, EvID, mcEnergy, mcAlt, mcAz, mcCore_x, mcCore_y,
mcHfirst, mcType, GPStime, width, length, width / length,
phi, psi, r, x, y, intensity, skewness, kurtosis, mcAlttel,
mcAztel, impact, mcXmax, time_gradient, intercept, SrcX,
SrcY, disp, hadroness
]],
columns=features)
output = output.append(eventdf, ignore_index=True)
outfile = outdir + particle_type + '_events.hdf5'
if storedata == True:
if concatenate == False or (concatenate == True and
np.DataSource().exists(outfile) == False):
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
if storeimg == True:
f = h5py.File(outfile, 'r+')
f.create_dataset('images', data=imagedata)
f.close()
else:
if storeimg == True:
f = h5py.File(outfile, 'r')
images = f['images']
del f['images']
images = np.vstack([images, imagedata])
f.close()
saved = pd.read_hdf(outfile, key=particle_type + '_events')
output = saved.append(output, ignore_index=True)
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
f = h5py.File(outfile, 'r+')
f.create_dataset('images', data=images)
f.close()
else:
saved = pd.read_hdf(outfile, key=particle_type + '_events')
output = saved.append(output, ignore_index=True)
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
del source
return output
def r0_to_dl1(input_filename=get_dataset_path('gamma_test_large.simtel.gz'),
output_filename=None,
custom_config={},
pedestal_path=None,
calibration_path=None,
time_calibration_path=None,
pointing_file_path=None,
ucts_t0_dragon=math.nan,
dragon_counter0=math.nan,
ucts_t0_tib=math.nan,
tib_counter0=math.nan):
"""
Chain r0 to dl1
Save the extracted dl1 parameters in output_filename
Parameters
----------
input_filename: str
path to input file, default: `gamma_test_large.simtel.gz`
output_filename: str
path to output file, default: `./` + basename(input_filename)
custom_config: path to a configuration file
pedestal_path: Path to the DRS4 pedestal file
calibration_path: Path to the file with calibration constants and
pedestals
time_calibration_path: Path to the DRS4 time correction file
pointing_file_path: path to the Drive log with the pointing information
Arguments below are just temporal and will be removed whenever UCTS+EvB
is proved to stably and reliably provide timestamps.
ucts_t0_dragon: first valid ucts_time
dragon_counter0: Dragon counter corresponding to ucts_t0_dragon
ucts_t0_tib: first valid ucts_time for the first valid TIB counter
tib_counter0: first valid TIB counter
Returns
-------
"""
if output_filename is None:
output_filename = ('dl1_' +
os.path.basename(input_filename).rsplit('.', 1)[0] +
'.h5')
if os.path.exists(output_filename):
raise AttributeError(output_filename + ' exists, exiting.')
config = replace_config(standard_config, custom_config)
custom_calibration = config["custom_calibration"]
try:
source = event_source(input_filename, back_seekable=True)
except:
# back_seekable might not be available for other sources that eventio
# TODO for real data: source with calibration file and pointing file
source = event_source(input_filename)
is_simu = source.metadata['is_simulation']
source.allowed_tels = config["allowed_tels"]
if config["max_events"] is not None:
source.max_events = config["max_events"] + 1
metadata = global_metadata(source)
write_metadata(metadata, output_filename)
cal_mc = load_calibrator_from_config(config)
# minimum number of pe in a pixel to include it in calculation of muon ring time (peak sample):
min_pe_for_muon_t_calc = 10.
# Dictionary to store muon ring parameters
muon_parameters = create_muon_table()
if not is_simu:
# TODO : add DRS4 calibration config in config file, read it and pass it here
r0_r1_calibrator = LSTR0Corrections(pedestal_path=pedestal_path,
tel_id=1)
# all this will be cleaned up in a next PR related to the configuration files
r1_dl1_calibrator = LSTCameraCalibrator(
calibration_path=calibration_path,
time_calibration_path=time_calibration_path,
extractor_product=config['image_extractor'],
gain_threshold=Config(config).gain_selector_config['threshold'],
config=Config(config),
allowed_tels=[1],
)
# Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but
# using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous
r1_dl1_calibrator_for_muon_rings = LSTCameraCalibrator(
calibration_path=calibration_path,
time_calibration_path=time_calibration_path,
extractor_product=config['image_extractor_for_muons'],
gain_threshold=Config(config).gain_selector_config['threshold'],
config=Config(config),
allowed_tels=[1],
)
dl1_container = DL1ParametersContainer()
if pointing_file_path:
# Open drive report
pointings = PointingPosition()
pointings.drive_path = pointing_file_path
drive_data = pointings._read_drive_report()
extra_im = ExtraImageInfo()
extra_im.prefix = '' # get rid of the prefix
event = next(iter(source))
write_array_info(event, output_filename)
### Write extra information to the DL1 file
if is_simu:
write_mcheader(event.mcheader,
output_filename,
obs_id=event.r0.obs_id,
filters=filters,
metadata=metadata)
subarray = event.inst.subarray
with HDF5TableWriter(
filename=output_filename,
group_name='dl1/event',
mode='a',
filters=filters,
add_prefix=True,
# overwrite = True,
) as writer:
print("USING FILTERS: ", writer._h5file.filters)
if is_simu:
# build a mapping of tel_id back to tel_index:
# (note this should be part of SubarrayDescription)
idx = np.zeros(max(subarray.tel_indices) + 1)
for key, val in subarray.tel_indices.items():
idx[key] = val
# the final transform then needs the mapping and the number of telescopes
tel_list_transform = partial(
utils.expand_tel_list,
max_tels=len(event.inst.subarray.tel) + 1,
)
writer.add_column_transform(table_name='subarray/trigger',
col_name='tels_with_trigger',
transform=tel_list_transform)
### EVENT LOOP ###
for i, event in enumerate(source):
if i % 100 == 0:
print(i)
event.dl0.prefix = ''
event.mc.prefix = 'mc'
event.trig.prefix = ''
# write sub tables
if is_simu:
write_subarray_tables(writer, event, metadata)
if not custom_calibration:
cal_mc(event)
else:
r0_r1_calibrator.calibrate(event)
r1_dl1_calibrator(event)
# Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten.
# When the last version of the method is implemented, vol. reduction will be done at dl0
check_and_apply_volume_reduction(event, config)
# FIXME? This should be eventually done after we evaluate whether the image is
# a candidate muon ring. In that case the full image could be kept, or reduced
# only after the ring analysis is complete.
for ii, telescope_id in enumerate(event.r0.tels_with_data):
tel = event.dl1.tel[telescope_id]
tel.prefix = '' # don't really need one
# remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST'
tel_name = str(event.inst.subarray.tel[telescope_id])[4:]
tel_name = tel_name.replace('-003', '')
if custom_calibration:
lst_calibration(event, telescope_id)
try:
dl1_filled = get_dl1(event,
telescope_id,
dl1_container=dl1_container,
custom_config=config,
use_main_island=True)
except HillasParameterizationError:
logging.exception(
'HillasParameterizationError in get_dl1()')
continue
if dl1_filled is not None:
# Some custom def
dl1_container.wl = dl1_container.width / dl1_container.length
# Log10(Energy) in GeV
if is_simu:
dl1_container.mc_energy = event.mc.energy.value
dl1_container.log_mc_energy = np.log10(
event.mc.energy.value * 1e3)
dl1_container.fill_mc(event)
dl1_container.log_intensity = np.log10(
dl1_container.intensity)
dl1_container.gps_time = event.trig.gps_time.value
if not is_simu:
# GPS + WRS + UCTS is now working in its nominal configuration.
# These TS are stored into ucts_time container.
# TS can be alternatively calculated from the TIB and
# Dragon modules counters based on the first valid UCTS TS
# as the reference point. For the time being, the three TS
# are stored in the DL1 files for checking purposes.
ucts_time = event.lst.tel[
telescope_id].evt.ucts_timestamp * 1e-9 # secs
module_id = 82 # Get counters from the central Dragon module
if math.isnan(ucts_t0_dragon) and math.isnan(dragon_counter0) \
and math.isnan(ucts_t0_tib) and math.isnan(tib_counter0):
# Dragon/TIB timestamps not based on a valid absolute reference timestamp
dragon_time = (
event.lst.tel[telescope_id].svc.date +
event.lst.tel[telescope_id].evt.
pps_counter[module_id] +
event.lst.tel[telescope_id].evt.
tenMHz_counter[module_id] * 10**(-7))
tib_time = (
event.lst.tel[telescope_id].svc.date +
event.lst.tel[telescope_id].evt.tib_pps_counter
+ event.lst.tel[telescope_id].evt.
tib_tenMHz_counter * 10**(-7))
else:
# Dragon/TIB timestamps based on a valid absolute reference timestamp
dragon_time = (
(ucts_t0_dragon - dragon_counter0) * 1e-9
+ # secs
event.lst.tel[telescope_id].evt.
pps_counter[module_id] +
event.lst.tel[telescope_id].evt.
tenMHz_counter[module_id] * 10**(-7))
tib_time = (
(ucts_t0_tib - tib_counter0) * 1e-9 + # secs
event.lst.tel[telescope_id].evt.tib_pps_counter
+ event.lst.tel[telescope_id].evt.
tib_tenMHz_counter * 10**(-7))
# FIXME: directly use unix_tai format whenever astropy v4.1 is out
ucts_time_utc = unix_tai_to_utc(ucts_time)
dragon_time_utc = unix_tai_to_utc(dragon_time)
tib_time_utc = unix_tai_to_utc(tib_time)
dl1_container.ucts_time = ucts_time_utc.unix
dl1_container.dragon_time = dragon_time_utc.unix
dl1_container.tib_time = tib_time_utc.unix
# Select the timestamps to be used for pointing interpolation
if config['timestamps_pointing'] == "ucts":
event_timestamps = ucts_time_utc.unix
elif config['timestamps_pointing'] == "dragon":
event_timestamps = dragon_time_utc.unix
elif config['timestamps_pointing'] == "tib":
event_timestamps = tib_time_utc.unix
else:
raise ValueError(
"The timestamps_pointing option is not a valid one. \
Try ucts (default), dragon or tib."
)
if pointing_file_path and event_timestamps > 0:
azimuth, altitude = pointings.cal_pointingposition(
event_timestamps, drive_data)
event.pointing[telescope_id].azimuth = azimuth
event.pointing[telescope_id].altitude = altitude
dl1_container.az_tel = azimuth
dl1_container.alt_tel = altitude
else:
dl1_container.az_tel = u.Quantity(np.nan, u.rad)
dl1_container.alt_tel = u.Quantity(np.nan, u.rad)
# Until the TIB trigger_type is fully reliable, we also add
# the ucts_trigger_type to the data
extra_im.ucts_trigger_type = event.lst.tel[
telescope_id].evt.ucts_trigger_type
# FIXME: no need to read telescope characteristics like foclen for every event!
foclen = event.inst.subarray.tel[
telescope_id].optics.equivalent_focal_length
mirror_area = u.Quantity(
event.inst.subarray.tel[telescope_id].optics.
mirror_area, u.m**2)
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(
np.arctan2(dl1_container.length, foclen))
dl1_container.width = width.value
dl1_container.length = length.value
dl1_container.prefix = tel.prefix
extra_im.tel_id = telescope_id
extra_im.num_trig_pix = event.r0.tel[
telescope_id].num_trig_pix
extra_im.trigger_time = event.r0.tel[
telescope_id].trigger_time
extra_im.trigger_type = event.r0.tel[
telescope_id].trigger_type
extra_im.trig_pix_id = event.r0.tel[
telescope_id].trig_pix_id
for container in [extra_im, dl1_container, event.r0, tel]:
add_global_metadata(container, metadata)
event.r0.prefix = ''
writer.write(table_name=f'telescope/image/{tel_name}',
containers=[event.r0, tel, extra_im])
writer.write(table_name=f'telescope/parameters/{tel_name}',
containers=[dl1_container, extra_im])
# Muon ring analysis, for real data only (MC is done starting from DL1 files)
if not is_simu:
bad_pixels = event.mon.tel[
telescope_id].calibration.unusable_pixels[0]
# Set to 0 unreliable pixels:
image = tel.image * (~bad_pixels)
# process only promising events, in terms of # of pixels with large signals:
if tag_pix_thr(image):
# re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings
numsamples = event.r1.tel[
telescope_id].waveform.shape[
2] # not necessarily the same as in r0!
bad_pixels_hg = event.mon.tel[
telescope_id].calibration.unusable_pixels[0]
bad_pixels_lg = event.mon.tel[
telescope_id].calibration.unusable_pixels[1]
# Now set to 0 all samples in unreliable pixels. Important for global peak
# integrator in case of crazy pixels! TBD: can this be done in a simpler
# way?
bad_waveform = np.array(([
np.transpose(
np.array(numsamples * [bad_pixels_hg])),
np.transpose(
np.array(numsamples * [bad_pixels_lg]))
]))
# print('hg bad pixels:',np.where(bad_pixels_hg))
# print('lg bad pixels:',np.where(bad_pixels_lg))
event.r1.tel[
telescope_id].waveform *= ~bad_waveform
r1_dl1_calibrator_for_muon_rings(event)
tel = event.dl1.tel[telescope_id]
image = tel.image * (~bad_pixels)
# Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum)
# perhaps the event is no longer promising (e.g. if it has a large time evolution)
if not tag_pix_thr(image):
good_ring = False
else:
# read geometry from event.inst. But not needed for every event. FIXME?
geom = event.inst.subarray.tel[
telescope_id].camera
muonintensityparam, size_outside_ring, muonringparam, good_ring, \
radial_distribution, mean_pixel_charge_around_ring = \
analyze_muon_event(event.r0.event_id, image, geom, foclen,
mirror_area, False, '')
# mirror_area, True, './') # (test) plot muon rings as png files
# Now we want to obtain the waveform sample (in HG and LG) at which the ring light peaks:
bright_pixels_waveforms = event.r1.tel[
telescope_id].waveform[:, image >
min_pe_for_muon_t_calc, :]
stacked_waveforms = np.sum(
bright_pixels_waveforms, axis=-2)
# stacked waveforms from all bright pixels; shape (ngains, nsamples)
hg_peak_sample = np.argmax(stacked_waveforms,
axis=-1)[0]
lg_peak_sample = np.argmax(stacked_waveforms,
axis=-1)[1]
if good_ring:
fill_muon_event(muon_parameters, good_ring,
event.r0.event_id, dragon_time,
muonintensityparam,
muonringparam,
radial_distribution,
size_outside_ring,
mean_pixel_charge_around_ring,
hg_peak_sample, lg_peak_sample)
# writes mc information per telescope, including photo electron image
if is_simu \
and (event.mc.tel[telescope_id].photo_electron_image > 0).any() \
and config['write_pe_image']:
event.mc.tel[telescope_id].prefix = ''
writer.write(
table_name=f'simulation/{tel_name}',
containers=[event.mc.tel[telescope_id], extra_im])
if is_simu:
### Reconstruct source position from disp for all events and write the result in the output file
for tel_name in ['LST_LSTCam']:
focal = OpticsDescription.from_name(
tel_name.split('_')[0]).equivalent_focal_length
dl1_params_key = f'dl1/event/telescope/parameters/{tel_name}'
add_disp_to_parameters_table(output_filename, dl1_params_key,
focal)
# Write energy histogram from simtel file and extra metadata
# ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever
if source.max_events is None:
write_simtel_energy_histogram(source,
output_filename,
obs_id=event.dl0.obs_id,
metadata=metadata)
else:
dir = os.path.dirname(output_filename)
name = os.path.basename(output_filename)
k = name.find('Run')
muon_output_filename = name[0:name.find('LST-')+5] + '.' + \
name[k:k+13] + '.fits'
muon_output_filename = dir + '/' + muon_output_filename.replace(
"dl1", "muons")
table = Table(muon_parameters)
table.write(muon_output_filename, format='fits', overwrite=True)
def main():
std_config = get_standard_config()
if args.config_file is not None:
config = replace_config(std_config,
read_configuration_file(args.config_file))
else:
config = std_config
print(config['tailcut'])
foclen = OpticsDescription.from_name('LST').equivalent_focal_length
cam_table = Table.read(args.input_file,
path="instrument/telescope/camera/LSTCam")
camera_geom = CameraGeometry.from_table(cam_table)
dl1_container = DL1ParametersContainer()
parameters_to_update = list(HillasParametersContainer().keys())
parameters_to_update.extend([
'concentration_cog',
'concentration_core',
'concentration_pixel',
'leakage_intensity_width_1',
'leakage_intensity_width_2',
'leakage_pixels_width_1',
'leakage_pixels_width_2',
'n_islands',
'intercept',
'time_gradient',
'n_pixels',
'wl',
'r',
])
nodes_keys = get_dataset_keys(args.input_file)
if args.noimage:
nodes_keys.remove(dl1_images_lstcam_key)
auto_merge_h5files([args.input_file],
args.output_file,
nodes_keys=nodes_keys)
with tables.open_file(args.input_file, mode='r') as input:
image_table = input.root[dl1_images_lstcam_key]
with tables.open_file(args.output_file, mode='a') as output:
params = output.root[dl1_params_lstcam_key].read()
for ii, row in enumerate(image_table):
if ii % 10000 == 0:
print(ii)
image = row['image']
peak_time = row['peak_time']
signal_pixels = tailcuts_clean(camera_geom, image,
**config['tailcut'])
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
num_islands, island_labels = number_of_islands(
camera_geom, signal_pixels)
n_pixels_on_island = np.bincount(
island_labels.astype(np.int))
n_pixels_on_island[
0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
signal_pixels[island_labels != max_island_label] = False
hillas = hillas_parameters(camera_geom[signal_pixels],
image[signal_pixels])
dl1_container.fill_hillas(hillas)
dl1_container.set_timing_features(
camera_geom[signal_pixels], image[signal_pixels],
peak_time[signal_pixels], hillas)
dl1_container.set_leakage(camera_geom, image,
signal_pixels)
dl1_container.set_concentration(camera_geom, image, hillas)
dl1_container.n_islands = num_islands
dl1_container.wl = dl1_container.width / dl1_container.length
dl1_container.n_pixels = n_pixels
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(
np.arctan2(dl1_container.length, foclen))
dl1_container.width = width
dl1_container.length = length
dl1_container.r = np.sqrt(dl1_container.x**2 +
dl1_container.y**2)
else:
# for consistency with r0_to_dl1.py:
for key in dl1_container.keys():
dl1_container[key] = \
u.Quantity(0, dl1_container.fields[key].unit)
dl1_container.width = u.Quantity(np.nan, u.m)
dl1_container.length = u.Quantity(np.nan, u.m)
dl1_container.wl = u.Quantity(np.nan, u.m)
for p in parameters_to_update:
params[ii][p] = u.Quantity(dl1_container[p]).value
output.root[dl1_params_lstcam_key][:] = params
def _build_subarray_info(self, run):
"""
constructs a SubarrayDescription object from the info in an
MCRun
Parameters
----------
run: MCRun object
Returns
-------
SubarrayDescription :
instrumental information
"""
subarray = SubarrayDescription("MonteCarloArray")
runHeader = run.root.RunHeader
tabFocalTel = runHeader.tabFocalTel.read()
tabPosTelX = runHeader.tabPosTelX.read()
tabPosTelY = runHeader.tabPosTelY.read()
tabPosTelZ = runHeader.tabPosTelZ.read()
tabPoslXYZ = np.ascontiguousarray(
np.vstack((tabPosTelX, tabPosTelY, tabPosTelZ)).T)
'''
# Correspance HiPeData.Telscope.Type and camera name
# 0 LSTCam, 1 NectarCam, 2 FlashCam, 3 SCTCam,
# 4 ASTRICam, 5 DigiCam, 6 CHEC
'''
mapping_camera = {
0: 'LSTCam',
1: 'NectarCam',
2: 'FlashCam',
3: 'SCTCam',
4: 'ASTRICam',
5: 'DigiCam',
6: 'CHEC'
}
mapping_telName = {
0: 'LST',
1: 'MST',
2: 'MST',
3: 'MST',
4: 'SST-ASTRI',
5: 'SST-1M',
6: 'SST-2M'
}
for telNode in self.run.walk_nodes('/Tel', 'Group'):
try:
telType = uint64(telNode.telType.read())
telIndex = uint64(telNode.telIndex.read())
telId = uint64(telNode.telId.read())
cameraName = mapping_camera[telType]
telName = mapping_telName[telType]
camera = CameraGeometry.from_name(cameraName)
camera.cam_id = cameraName
foclen = tabFocalTel[telIndex] * u.m
tel_pos = tabPoslXYZ[telIndex] * u.m
camera.pix_x = telNode.tabPixelX.read() * u.m
camera.pix_y = telNode.tabPixelY.read() * u.m
#camera.rotate(-90.0*u.deg)
optic = OpticsDescription.from_name(telName)
optic.equivalent_focal_length = foclen
telescope_description = TelescopeDescription(telName,
telName,
optics=optic,
camera=camera)
#tel.optics.mirror_area = mirror_area
#tel.optics.num_mirror_tiles = num_tiles
subarray.tels[telId] = telescope_description
subarray.positions[telId] = tel_pos
except tables.exceptions.NoSuchNodeError as e:
pass
return subarray
def __init__(self, **kwargs):
"""
Constructor
Parameters
----------
kwargs: dict
Parameters to be passed.
NOTE: The file mask of the data to read can be passed with
the 'input_url' parameter.
"""
try:
import uproot
except ImportError:
raise ImportError(
"The 'uproot' package is required for the DLMAGICEventSource class."
)
self.file_list = glob.glob(kwargs['input_url'])
self.file_list.sort()
# Since EventSource can not handle file wild cards as input_url
# We substitute the input_url with first file matching
# the specified file mask.
del kwargs['input_url']
super().__init__(input_url=self.file_list[0], **kwargs)
# get run number
mask = r".*_za\d+to\d+_\d_(\d+)_([A-Z]+)_.*"
parsed_info = re.findall(mask, self.file_list[0])
self.run_number = parsed_info[0][0]
# MAGIC telescope positions in m wrt. to the center of CTA simulations
self.magic_tel_positions = {
1: [-27.24, -146.66, 50.00] * u.m,
2: [-96.44, -96.77, 51.00] * u.m
}
self.magic_tel_positions = self.magic_tel_positions
# MAGIC telescope description
optics = OpticsDescription.from_name('MAGIC')
geom = CameraGeometry.from_name('MAGICCam')
# Camera Readout for NectarCam used as a placeholder
readout = CameraReadout(
'MAGICCam',
sampling_rate=u.Quantity(1, u.GHz),
reference_pulse_shape=np.array([norm.pdf(np.arange(96), 48, 6)]),
reference_pulse_sample_width=u.Quantity(1, u.ns))
camera = CameraDescription('MAGICCam', geom, readout)
self.magic_tel_description = TelescopeDescription(name='MAGIC',
tel_type='LST',
optics=optics,
camera=camera)
self.magic_tel_descriptions = {
1: self.magic_tel_description,
2: self.magic_tel_description
}
self.magic_subarray = SubarrayDescription('MAGIC',
self.magic_tel_positions,
self.magic_tel_descriptions)
# Open ROOT files
self.calib_M1, self.calib_M2, self.star_M1, self.star_M2, self.superstar = None, None, None, None, None
for file in self.file_list:
uproot_file = uproot.open(file)
if "_Y_" in file:
if "_M1_" in file:
self.calib_M1 = uproot_file["Events"]
self.meta = uproot_file["RunHeaders"]
elif "_M2_" in file:
self.calib_M2 = uproot_file["Events"]
if "_I_" in file:
if "_M1_" in file:
self.star_M1 = uproot_file["Events"]
elif "_M2_" in file:
self.star_M2 = uproot_file["Events"]
if "_S_" in file:
self.superstar = uproot_file["Events"]
self.meta = uproot_file["RunHeaders"]
self._mc_header = self._parse_mc_header()
def main():
std_config = get_standard_config()
log.setLevel(logging.INFO)
handler = logging.StreamHandler()
logging.getLogger().addHandler(handler)
if args.config_file is not None:
config = replace_config(std_config,
read_configuration_file(args.config_file))
else:
config = std_config
if args.pedestal_cleaning:
print("Pedestal cleaning")
clean_method_name = 'tailcuts_clean_with_pedestal_threshold'
sigma = config[clean_method_name]['sigma']
pedestal_thresh = get_threshold_from_dl1_file(args.input_file, sigma)
cleaning_params = get_cleaning_parameters(config, clean_method_name)
pic_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params
log.info(
f"Fraction of pixel cleaning thresholds above picture thr.:"
f"{np.sum(pedestal_thresh>pic_th) / len(pedestal_thresh):.3f}")
picture_th = np.clip(pedestal_thresh, pic_th, None)
log.info(f"Tailcut clean with pedestal threshold config used:"
f"{config['tailcuts_clean_with_pedestal_threshold']}")
else:
clean_method_name = 'tailcut'
cleaning_params = get_cleaning_parameters(config, clean_method_name)
picture_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params
log.info(f"Tailcut config used: {config['tailcut']}")
use_only_main_island = True
if "use_only_main_island" in config[clean_method_name]:
use_only_main_island = config[clean_method_name][
"use_only_main_island"]
delta_time = None
if "delta_time" in config[clean_method_name]:
delta_time = config[clean_method_name]["delta_time"]
foclen = OpticsDescription.from_name('LST').equivalent_focal_length
cam_table = Table.read(args.input_file,
path="instrument/telescope/camera/LSTCam")
camera_geom = CameraGeometry.from_table(cam_table)
dl1_container = DL1ParametersContainer()
parameters_to_update = [
'intensity', 'x', 'y', 'r', 'phi', 'length', 'width', 'psi',
'skewness', 'kurtosis', 'concentration_cog', 'concentration_core',
'concentration_pixel', 'leakage_intensity_width_1',
'leakage_intensity_width_2', 'leakage_pixels_width_1',
'leakage_pixels_width_2', 'n_islands', 'intercept', 'time_gradient',
'n_pixels', 'wl', 'log_intensity'
]
nodes_keys = get_dataset_keys(args.input_file)
if args.noimage:
nodes_keys.remove(dl1_images_lstcam_key)
auto_merge_h5files([args.input_file],
args.output_file,
nodes_keys=nodes_keys)
with tables.open_file(args.input_file, mode='r') as input:
image_table = input.root[dl1_images_lstcam_key]
dl1_params_input = input.root[dl1_params_lstcam_key].colnames
disp_params = {
'disp_dx', 'disp_dy', 'disp_norm', 'disp_angle', 'disp_sign'
}
if set(dl1_params_input).intersection(disp_params):
parameters_to_update.extend(disp_params)
with tables.open_file(args.output_file, mode='a') as output:
params = output.root[dl1_params_lstcam_key].read()
for ii, row in enumerate(image_table):
dl1_container.reset()
image = row['image']
peak_time = row['peak_time']
signal_pixels = tailcuts_clean(camera_geom, image, picture_th,
boundary_th, isolated_pixels,
min_n_neighbors)
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
num_islands, island_labels = number_of_islands(
camera_geom, signal_pixels)
n_pixels_on_island = np.bincount(
island_labels.astype(np.int64))
n_pixels_on_island[
0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
if use_only_main_island:
signal_pixels[
island_labels != max_island_label] = False
# if delta_time has been set, we require at least one
# neighbor within delta_time to accept a pixel in the image:
if delta_time is not None:
cleaned_pixel_times = peak_time
# makes sure only signal pixels are used in the time
# check:
cleaned_pixel_times[~signal_pixels] = np.nan
new_mask = apply_time_delta_cleaning(
camera_geom, signal_pixels, cleaned_pixel_times, 1,
delta_time)
signal_pixels = new_mask
# count the surviving pixels
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
hillas = hillas_parameters(camera_geom[signal_pixels],
image[signal_pixels])
dl1_container.fill_hillas(hillas)
dl1_container.set_timing_features(
camera_geom[signal_pixels], image[signal_pixels],
peak_time[signal_pixels], hillas)
dl1_container.set_leakage(camera_geom, image,
signal_pixels)
dl1_container.set_concentration(
camera_geom, image, hillas)
dl1_container.n_islands = num_islands
dl1_container.wl = dl1_container.width / dl1_container.length
dl1_container.n_pixels = n_pixels
width = np.rad2deg(
np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(
np.arctan2(dl1_container.length, foclen))
dl1_container.width = width
dl1_container.length = length
dl1_container.log_intensity = np.log10(
dl1_container.intensity)
if set(dl1_params_input).intersection(disp_params):
disp_dx, disp_dy, disp_norm, disp_angle, disp_sign = disp(
dl1_container['x'].to_value(u.m),
dl1_container['y'].to_value(u.m), params['src_x'][ii],
params['src_y'][ii])
dl1_container['disp_dx'] = disp_dx
dl1_container['disp_dy'] = disp_dy
dl1_container['disp_norm'] = disp_norm
dl1_container['disp_angle'] = disp_angle
dl1_container['disp_sign'] = disp_sign
for p in parameters_to_update:
params[ii][p] = u.Quantity(dl1_container[p]).value
output.root[dl1_params_lstcam_key][:] = params
def get_expected_source_pos(data, data_type, config):
"""Get expected source position for source-dependent analysis .
Parameters:
-----------
data: Pandas DataFrame
data_type: string ('mc_gamma','mc_proton','real_data')
config: dictionnary containing configuration
"""
#For gamma MC, expected source position is actual one for each event
if data_type == 'mc_gamma':
expected_src_pos_x_m = data['src_x'].values
expected_src_pos_y_m = data['src_y'].values
#For proton MC, nominal source position is one written in config file
if data_type == 'mc_proton':
focal_length = OpticsDescription.from_name(
'LST').equivalent_focal_length
expected_src_pos = utils.sky_to_camera(
u.Quantity(data['mc_alt_tel'].values +
config['mc_nominal_source_x_deg'],
u.deg,
copy=False),
u.Quantity(data['mc_az_tel'].values +
config['mc_nominal_source_y_deg'],
u.deg,
copy=False), focal_length,
u.Quantity(data['mc_alt_tel'].values, u.deg, copy=False),
u.Quantity(data['mc_az_tel'].values, u.deg, copy=False))
expected_src_pos_x_m = expected_src_pos.x.to_value(u.m)
expected_src_pos_y_m = expected_src_pos.y.to_value(u.m)
# For real data
if data_type == 'real_data':
# source is always at the ceter of camera for ON mode
if config.get('observation_mode') == 'on':
expected_src_pos_x_m = np.zeros(len(data))
expected_src_pos_y_m = np.zeros(len(data))
# compute source position in camera coordinate event by event for wobble mode
if config.get('observation_mode') == 'wobble':
if 'source_name' in config:
source_coord = SkyCoord.from_name(config.get('source_name'))
else:
source_coord = SkyCoord(config.get('source_ra'),
config.get('source_dec'),
frame="icrs",
unit="deg")
focal_length = OpticsDescription.from_name(
'LST').equivalent_focal_length
time = data['dragon_time']
obstime = Time(time, scale='utc', format='unix')
pointing_alt = u.Quantity(data['alt_tel'], u.rad, copy=False)
pointing_az = u.Quantity(data['az_tel'], u.rad, copy=False)
source_pos = utils.radec_to_camera(source_coord, obstime,
pointing_alt, pointing_az,
focal_length)
expected_src_pos_x_m = source_pos.x.to_value(u.m)
expected_src_pos_y_m = source_pos.y.to_value(u.m)
return expected_src_pos_x_m, expected_src_pos_y_m
def r0_to_dl1(input_filename=get_dataset_path('gamma_test_large.simtel.gz'),
output_filename=None,
custom_config={}):
"""
Chain r0 to dl1
Save the extracted dl1 parameters in output_filename
Parameters
----------
input_filename: str
path to input file, default: `gamma_test_large.simtel.gz`
output_filename: str
path to output file, default: `./` + basename(input_filename)
config_file: path to a configuration file
Returns
-------
"""
if output_filename is None:
output_filename = ('dl1_' +
os.path.basename(input_filename).split('.')[0] +
'.h5')
config = replace_config(standard_config, custom_config)
custom_calibration = config["custom_calibration"]
source = event_source(input_filename)
source.allowed_tels = config["allowed_tels"]
source.max_events = config["max_events"]
cal = load_calibrator_from_config(config)
dl1_container = DL1ParametersContainer()
with HDF5TableWriter(filename=output_filename,
group_name='events',
overwrite=True) as writer:
for i, event in enumerate(source):
if i % 100 == 0:
print(i)
if not custom_calibration:
cal(event)
# for telescope_id, dl1 in event.dl1.tel.items():
for ii, telescope_id in enumerate(event.r0.tels_with_data):
if custom_calibration:
lst_calibration(event, telescope_id)
try:
dl1_filled = get_dl1(event,
telescope_id,
dl1_container=dl1_container,
custom_config=config)
except HillasParameterizationError:
logging.exception(
'HillasParameterizationError in get_dl1()')
continue
if dl1_filled is not None:
# Some custom def
dl1_container.wl = dl1_container.width / dl1_container.length
# Log10(Energy) in GeV
dl1_container.mc_energy = np.log10(event.mc.energy.value *
1e3)
dl1_container.intensity = np.log10(dl1_container.intensity)
dl1_container.gps_time = event.trig.gps_time.value
foclen = (event.inst.subarray.tel[telescope_id].optics.
equivalent_focal_length)
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(
np.arctan2(dl1_container.length, foclen))
dl1_container.width = width.value
dl1_container.length = length.value
if width >= 0:
# Camera geometry
camera = event.inst.subarray.tel[telescope_id].camera
writer.write(camera.cam_id, [dl1_container])
lst_focal = OpticsDescription.from_name('LST').equivalent_focal_length
with pd.HDFStore(output_filename) as store:
df = store['events/LSTCam']
source_pos_in_camera = sky_to_camera(
df.mc_alt.values * u.rad,
df.mc_az.values * u.rad,
lst_focal,
df.mc_alt_tel.values * u.rad,
df.mc_az_tel.values * u.rad,
)
disp_parameters = disp.disp(df.x.values * u.m, df.y.values * u.m,
source_pos_in_camera.x,
source_pos_in_camera.y)
disp_df = pd.DataFrame(np.transpose(disp_parameters),
columns=[
'disp_dx', 'disp_dy', 'disp_norm',
'disp_angle', 'disp_sign'
])
disp_df['src_x'] = source_pos_in_camera.x.value
disp_df['src_y'] = source_pos_in_camera.y.value
store['events/LSTCam'] = pd.concat([store['events/LSTCam'], disp_df],
axis=1)
with HDF5TableWriter(filename=output_filename,
group_name="simulation",
mode="a") as writer:
writer.write("run_config", [event.mcheader])