def angular_distance(event, hillas_containers):
"""
find the angular distance between the true array that point to the origin of
the shower of protons and the predicted point.
Parameters
----------
event :
calibrated event
hillas_containers :
dictionary with telescope IDs as key and
HillasParametersContainer instances as values
Returns
-------
angular_distance:
angular distance(in degrees) or False if the reconstruction can't be done due
TooFewTelescopesException or InvalidWidthException
"""
horizon_frame = AltAz()
reco = HillasReconstructor()
# SkyCoord
array_pointing_event = SkyCoord(az=event.mcheader.run_array_direction[0],
alt=event.mcheader.run_array_direction[1],
frame=horizon_frame)
try:
reconstruction_cleaned = reco.predict(
hillas_containers,
event.inst,
array_pointing_event,
)
except TooFewTelescopesException:
return False
except InvalidWidthException:
return False
array_pointing_reconstruction = SkyCoord(az=reconstruction_cleaned.az,
alt=reconstruction_cleaned.alt,
frame=horizon_frame)
# angular distance between reconstructed and event image
ang_dist = array_pointing_event.separation(array_pointing_reconstruction)
return ang_dist
def reconstruct_direction(array_event_id, group, instrument):
reco = HillasReconstructor()
# monkey patch this huansohn. this is super slow otherwise. who needs max h anyways
reco.fit_h_max = dummy_function_h_max
params = {}
pointing_azimuth = {}
pointing_altitude = {}
for index, row in group.iterrows():
tel_id = row.telescope_id
# the data in each event has to be put inside these namedtuples to call reco.predict
moments = SubMomentParameters(size=row.intensity,
cen_x=row.x * u.m,
cen_y=row.y * u.m,
length=row.length * u.m,
width=row.width * u.m,
psi=row.psi * u.rad)
params[tel_id] = moments
pointing_azimuth[tel_id] = row.pointing_azimuth * u.rad
pointing_altitude[tel_id] = row.pointing_altitude * u.rad
try:
reconstruction = reco.predict(params, instrument, pointing_azimuth,
pointing_altitude)
except NameError:
return {
'alt_prediction': np.nan,
'az_prediction': np.nan,
'core_x_prediction': np.nan,
'core_y_prediction': np.nan,
'array_event_id': array_event_id,
}
if reconstruction.alt.si.value == np.nan:
print('Not reconstructed')
print(params)
return {
'alt_prediction':
((np.pi / 2) - reconstruction.alt.si.value), # TODO srsly now? FFS
'az_prediction': reconstruction.az.si.value,
'core_x_prediction': reconstruction.core_x.si.value,
'core_y_prediction': reconstruction.core_y.si.value,
'array_event_id': array_event_id,
# 'h_max_prediction': reconstruction.h_max.si.value
}
if params.width > 0:
hillas_params[tel_id] = params
array_pointing = SkyCoord(
az=event.mcheader.run_array_direction[0],
alt=event.mcheader.run_array_direction[1],
frame=horizon_frame
)
if len(hillas_params) < 2:
continue
reco_result = reco.predict(
hillas_params, source.subarray,
array_pointing, telescope_pointings
)
# get angular offset between reconstructed shower direction and MC
# generated shower direction
off_angle = angular_separation(
event.mc.az,
event.mc.alt,
reco_result.az,
reco_result.alt
)
# Appending all estimated off angles
off_angles.append(off_angle.to(u.deg).value)
# calculate theta square for angles which are not nan
def test_reconstructors(reconstructors):
"""
a test of the complete fit procedure on one event including:
• tailcut cleaning
• hillas parametrisation
• HillasPlane creation
• direction fit
• position fit
in the end, proper units in the output are asserted"""
filename = get_dataset_path(
"gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz")
source = EventSource(filename, max_events=10)
subarray = source.subarray
calib = CameraCalibrator(source.subarray)
horizon_frame = AltAz()
for event in source:
calib(event)
sim_shower = event.simulation.shower
array_pointing = SkyCoord(az=sim_shower.az,
alt=sim_shower.alt,
frame=horizon_frame)
telescope_pointings = {}
for tel_id, dl1 in event.dl1.tel.items():
geom = source.subarray.tel[tel_id].camera.geometry
telescope_pointings[tel_id] = SkyCoord(
alt=event.pointing.tel[tel_id].altitude,
az=event.pointing.tel[tel_id].azimuth,
frame=horizon_frame,
)
mask = tailcuts_clean(geom,
dl1.image,
picture_thresh=10.0,
boundary_thresh=5.0)
dl1.parameters = ImageParametersContainer()
try:
moments = hillas_parameters(geom[mask], dl1.image[mask])
except HillasParameterizationError:
dl1.parameters.hillas = HillasParametersContainer()
continue
# Make sure we provide only good images for the test
if np.isnan(moments.width.value) or (moments.width.value == 0):
dl1.parameters.hillas = HillasParametersContainer()
else:
dl1.parameters.hillas = moments
hillas_dict = {
tel_id: dl1.parameters.hillas
for tel_id, dl1 in event.dl1.tel.items()
if np.isfinite(dl1.parameters.hillas.intensity)
}
if len(hillas_dict) < 2:
continue
for count, reco_method in enumerate(reconstructors):
if reco_method is HillasReconstructor:
reconstructor = HillasReconstructor(subarray)
reconstructor(event)
event.dl2.stereo.geometry["HillasReconstructor"].alt.to(u.deg)
event.dl2.stereo.geometry["HillasReconstructor"].az.to(u.deg)
event.dl2.stereo.geometry["HillasReconstructor"].core_x.to(u.m)
assert event.dl2.stereo.geometry[
"HillasReconstructor"].is_valid
else:
reconstructor = reco_method()
try:
reconstructor_out = reconstructor.predict(
hillas_dict,
source.subarray,
array_pointing,
telescope_pointings,
)
except InvalidWidthException:
continue
reconstructor_out.alt.to(u.deg)
reconstructor_out.az.to(u.deg)
reconstructor_out.core_x.to(u.m)
assert reconstructor_out.is_valid
picture_thresh=10,
boundary_thresh=5)
# set all rejected pixels to zero
cleaned_image[~cleanmask] = 0
# Calulate hillas parameters
# It fails for empty pixels
try:
hillas_params[tel_id] = hillas_parameters(camgeom, cleaned_image)
except:
pass
if len(hillas_params) < 2:
continue
reco_result = reco.predict(hillas_params, event.inst, point_altitude,
point_azimuth)
# get angular offset between reconstructed shower direction and MC
# generated shower direction
off_angle = angular_separation(event.mc.az, event.mc.alt, reco_result.az,
reco_result.alt)
# Appending all estimated off angles
off_angles.append(off_angle.to(u.deg).value)
# calculate theta square for angles which are not nan
off_angles = np.array(off_angles)
thetasquare = off_angles[np.isfinite(off_angles)]**2
# To plot thetasquare The number of events in th data files for LSTCam is not
# significantly high to give a nice thetasquare plot for gammas One can use
time_gradients[telescope_id] = hillas_c.skewness
print(geom.camera_name, time_gradients[telescope_id])
# make sure each telescope get's an arrow
if abs(time_gradients[telescope_id]) < 0.2:
time_gradients[telescope_id] = 1
# ignore events with less than two telescopes
if len(hillas_containers) < 2:
continue
array_pointing = SkyCoord(az=event.mcheader.run_array_direction[0],
alt=event.mcheader.run_array_direction[1],
frame=horizon_frame)
stereo = reco.predict(
hillas_containers,
event_source.subarray,
array_pointing,
)
plt.figure()
angle_offset = event.mcheader.run_array_direction[0]
disp = ArrayDisplay(event_source.subarray)
disp.set_vector_hillas(hillas_containers,
time_gradient=time_gradients,
angle_offset=angle_offset,
length=500)
plt.scatter(
event.mc.core_x,
event.mc.core_y,
s=200,
time_gradients[telescope_id] = timing_c.slope.value
else:
time_gradients[telescope_id] = hillas_c.skewness
print(camera.cam_id, time_gradients[telescope_id])
# make sure each telescope get's an arrow
if abs(time_gradients[telescope_id]) < 0.2:
time_gradients[telescope_id] = 1
# ignore events with less than two telescopes
if len(hillas_containers) < 2:
continue
stereo = reco.predict(
hillas_containers,
event.inst,
pointing_alt=pointing_altitude,
pointing_az=pointing_azimuth
)
plt.figure()
angle_offset = event.mcheader.run_array_direction[0]
disp = ArrayDisplay(event.inst.subarray)
disp.set_vector_hillas(
hillas_containers,
time_gradient=time_gradients,
angle_offset=angle_offset,
length=500
)
plt.scatter(
event.mc.core_x, event.mc.core_y,
camgeom, image, picture_thresh=10, boundary_thresh=5
)
# set all rejected pixels to zero
cleaned_image[~cleanmask] = 0
# Calulate hillas parameters
# It fails for empty pixels
try:
hillas_params[tel_id] = hillas_parameters(camgeom, cleaned_image)
except:
pass
if len(hillas_params) < 2:
continue
reco_result = reco.predict(hillas_params, event.inst, point_altitude, point_azimuth)
# get angular offset between reconstructed shower direction and MC
# generated shower direction
off_angle = angular_separation(
event.mc.az,
event.mc.alt,
reco_result.az,
reco_result.alt
)
# Appending all estimated off angles
off_angles.append(off_angle.to(u.deg).value)
# calculate theta square for angles which are not nan
off_angles = np.array(off_angles)
def process_event(event, calibrator, config):
'''Processes one event.
Calls calculate_image_features and performs a stereo hillas reconstruction.
ReconstructedShowerContainer will be filled with nans (+units) if hillas failed.
Returns:
--------
ArrayEventContainer
list(telescope_event_containers.values())
'''
logging.info(f'processing event {event.dl0.event_id}')
calibrator(event)
telescope_types = []
hillas_reconstructor = HillasReconstructor()
telescope_event_containers = {}
horizon_frame = AltAz()
telescope_pointings = {}
array_pointing = SkyCoord(
az=event.mcheader.run_array_direction[0],
alt=event.mcheader.run_array_direction[1],
frame=horizon_frame,
)
for telescope_id, dl1 in event.dl1.tel.items():
cam_type = event.inst.subarray.tels[telescope_id].camera.cam_id
if cam_type not in config.cleaning_level.keys():
logging.info(f"No cleaning levels for camera {cam_type}. Skipping event")
continue
telescope_types.append(str(event.inst.subarray.tels[telescope_id].optics))
try:
telescope_event_containers[telescope_id] = calculate_image_features(
telescope_id, event, dl1, config
)
except HillasParameterizationError:
logging.info(
f'Error calculating hillas features for event {event.dl0.event_id, telescope_id}',
exc_info=True,
)
continue
except Exception:
logging.info(
f'Error calculating image features for event {event.dl0.event_id, telescope_id}',
exc_info=True,
)
continue
telescope_pointings[telescope_id] = SkyCoord(
alt=telescope_event_containers[telescope_id].pointing.altitude,
az=telescope_event_containers[telescope_id].pointing.azimuth,
frame=horizon_frame,
)
if len(telescope_event_containers) < 1:
raise Exception('None of the allowed telescopes triggered for event %s', event.dl0.event_id)
parameters = {tel_id: telescope_event_containers[tel_id].hillas for tel_id in telescope_event_containers}
try:
reconstruction_container = hillas_reconstructor.predict(
parameters, event.inst, array_pointing, telescopes_pointings=telescope_pointings
)
reconstruction_container.prefix = ''
except Exception:
logging.info(
'Not enough telescopes for which Hillas parameters could be reconstructed', exc_info=True
)
reconstruction_container = ReconstructedShowerContainer()
reconstruction_container.alt = u.Quantity(np.nan, u.rad)
reconstruction_container.alt_uncert = u.Quantity(np.nan, u.rad)
reconstruction_container.az = u.Quantity(np.nan, u.rad)
reconstruction_container.az_uncert = np.nan
reconstruction_container.core_x = u.Quantity(np.nan, u.m)
reconstruction_container.core_y = u.Quantity(np.nan, u.m)
reconstruction_container.core_uncert = np.nan
reconstruction_container.h_max = u.Quantity(np.nan, u.m)
reconstruction_container.h_max_uncert = np.nan
reconstruction_container.is_valid = False
reconstruction_container.tel_ids = []
reconstruction_container.average_intensity = np.nan
reconstruction_container.goodness_of_fit = np.nan
reconstruction_container.prefix = ''
calculate_distance_to_core(telescope_event_containers, event, reconstruction_container)
mc_container = copy.deepcopy(event.mc)
mc_container.tel = None
mc_container.prefix = 'mc'
counter = Counter(telescope_types)
array_event = ArrayEventContainer(
array_event_id=event.dl0.event_id,
run_id=event.r0.obs_id,
reco=reconstruction_container,
total_intensity=sum([t.hillas.intensity for t in telescope_event_containers.values()]),
num_triggered_lst=counter['LST'],
num_triggered_mst=counter['MST'],
num_triggered_sst=counter['SST'],
num_triggered_telescopes=len(telescope_types),
mc=mc_container,
)
return array_event, list(telescope_event_containers.values())
def process_event(
event,
telescopes,
subarray,
stereo,
calib,
apply_quality_cuts):
if stereo:
hillas_containers = {}
telescope_pointings = {}
horizon_frame = AltAz()
reco = HillasReconstructor()
# Calibrate the event
calib(event)
# Container to count what telescopes types were triggered in this event
event_tels = []
tel_data = []
for tel_id, dl1 in event.dl1.tel.items():
# If specific telescopes were specified, skip those that weren't
if telescopes is not None:
if (tel_id not in telescopes):
continue
tel = subarray.tels[tel_id]
# Process the telescope event
tel_type, tel_data_dict, hillas_c = process_telescope(tel, dl1, stereo)
# Add the telescope event data if it wasn't skipped
if tel_type is not None:
if apply_quality_cuts:
if tel_data_dict['intensity'] < 60 or tel_data_dict['nislands'] > 3 or tel_data_dict['n_survived_pixels'] < 6 or tel_data_dict['intensity_width_1'] > 0.1:
continue
# Calculate mc impact distance
x1 = event.mc.core_x.value
y1 = event.mc.core_y.value
x2 = subarray.positions[tel_id][0].value
y2 = subarray.positions[tel_id][1].value
v = [x2 - x1, y2 - y1]
mc_impact_distance = np.linalg.norm(v)
# Adding a few extras to the telescope event data
tel_data_dict.update({
'array_event_id': event.index.event_id,
'run_id': event.index.obs_id,
'telescope_id': tel_id,
'telescope_event_id': tel_id,
'mc_impact_distance': mc_impact_distance})
# Add data to the tables used for geometric reconstruction
if stereo:
tel_data_dict.update({'impact_distance': np.nan})
hillas_containers[tel_id] = hillas_c
telescope_pointings[tel_id] = SkyCoord(
alt=event.mc.tel[tel_id].altitude_raw * u.rad,
az=event.mc.tel[tel_id].azimuth_raw * u.rad,
frame=horizon_frame)
event_tels.append(tel_type)
tel_data.append(tel_data_dict)
# Skip event if no telescopes were processed
if len(event_tels) == 0:
return None, None
arr_data = {}
if stereo and len(event_tels) > 1:
array_pointing = SkyCoord(
az=event.mcheader.run_array_direction[0],
alt=event.mcheader.run_array_direction[1],
frame=horizon_frame)
# Do geometric direction reconstruction
reconst = reco.predict(
hillas_containers,
subarray,
array_pointing,
telescope_pointings)
# Calculate impact distance from reconstructed core position
for i in range(len(tel_data)):
x1 = reconst.core_x.value
y1 = reconst.core_y.value
x2 = subarray.positions[tel_data[i]['telescope_id']][0].value
y2 = subarray.positions[tel_data[i]['telescope_id']][1].value
v = [x2 - x1, y2 - y1]
impact_distance = np.linalg.norm(v)
tel_data[i]['impact_distance'] = impact_distance
# Grab the extra stereo info for array_events
reconst_az = reconst.az.value
if reconst_az < -np.pi:
reconst_az += 2*np.pi
if reconst_az > np.pi:
reconst_az -= 2*np.pi
arr_data.update({
'alt': reconst.alt.value,
'alt_uncert': reconst.alt_uncert.value,
'average_intensity': reconst.average_intensity,
'az': reconst_az,
'az_uncert': reconst.az_uncert,
'core_uncert': reconst.core_uncert,
'core_x': reconst.core_x.value,
'core_y': reconst.core_y.value,
'goodness_of_fit': reconst.goodness_of_fit,
'h_max': reconst.h_max.value,
'h_max_uncert': reconst.h_max_uncert,
'is_valid': reconst.is_valid,
'stereo_flag': True})
if stereo and len(event_tels) == 1:
# If only one telescope is triggered in a stereo array, replace
# stereo features with NaN
arr_data.update({
'alt': np.nan,
'alt_uncert': np.nan,
'average_intensity': np.nan,
'az': np.nan,
'az_uncert': np.nan,
'core_uncert': np.nan,
'core_x': np.nan,
'core_y': np.nan,
'goodness_of_fit': np.nan,
'h_max': np.nan,
'h_max_uncert': np.nan,
'is_valid': np.nan,
'stereo_flag': False})
# Grab info for array event data
arr_data.update({
'array_event_id': event.index.event_id,
'run_id': event.index.obs_id,
'azimuth_raw': event.mc.tel[tel_id].azimuth_raw,
'altitude_raw': event.mc.tel[tel_id].altitude_raw,
'azimuth_cor': event.mc.tel[tel_id].azimuth_cor,
'altitude_cor': event.mc.tel[tel_id].altitude_cor,
'num_triggered_lst': event_tels.count('LST'),
'num_triggered_mst': event_tels.count('MST'),
'num_triggered_sst': event_tels.count('SST'),
'mc_energy': event.mc.energy.value,
'true_source_alt': event.mc.alt.value,
'true_source_az': event.mc.az.value,
'mc_core_x': event.mc.core_x.value,
'mc_core_y': event.mc.core_y.value,
'mc_h_first_int': event.mc.h_first_int.value,
'mc_x_max': event.mc.x_max.value,
'mc_shower_primary_id': event.mc.shower_primary_id})
return tel_data, arr_data
# display.image = cleaned
# display.add_colorbar()
#
# display.overlay_moments(hillas_c, color='xkcd:red')
# plt.figure()
# plt.show()
# array pointing needed for the creation of the TiltedFrame to perform the impact point reconstruction
array_pointing = SkyCoord(az=event.mc.az, alt=event.mc.alt, frame=horizon_frame)
event_info_c.event_id = event.r0.event_id
event_info_c.obs_id = event.r0.obs_id
if len(hillas_dict) > 1:
stereo = reco.predict(
hillas_dict, event.inst, array_pointing, telescope_pointings
)
writer.write(table_name='reconstructed',
containers=[stereo, event_info_c])
writer.write(table_name='true', containers=[event.mc, event_info_c])
# save event for plotting later
# if event.count == 3:
# plotting_event = deepcopy(event)
# plotting_hillas = hillas_dict
# #plotting_timing = time_gradients
# plotting_stereo = stereo
# thrown the simtel energy histogram needs to loop over all the simtel file so we don't dump in case of tests
time_gradients[telescope_id] = hillas_c.skewness
print(camera.cam_id, time_gradients[telescope_id])
# make sure each telescope get's an arrow
if abs(time_gradients[telescope_id]) < 0.2:
time_gradients[telescope_id] = 1
# ignore events with less than two telescopes
if len(hillas_containers) < 2:
continue
array_pointing = SkyCoord(az=event.mcheader.run_array_direction[0],
alt=event.mcheader.run_array_direction[1],
frame=horizon_frame)
stereo = reco.predict(
hillas_containers,
event.inst,
array_pointing,
)
plt.figure()
angle_offset = event.mcheader.run_array_direction[0]
disp = ArrayDisplay(event.inst.subarray)
disp.set_vector_hillas(hillas_containers,
time_gradient=time_gradients,
angle_offset=angle_offset,
length=500)
plt.scatter(
event.mc.core_x,
event.mc.core_y,
s=200,
def main():
input_url = find_in_path(
'gamma_20deg_0deg_run101___cta-prod3-lapalma3-2147m-LaPalma.simtel.gz',
'/home/matteo/ctasoft/resources')
events = event_source(input_url, max_events=1)
calibrator = CameraCalibrator()
horizon_frame = AltAz()
reco = HillasReconstructor()
hillas_containers = {}
for event in events:
calibrator(event)
print("energy: ", event.mc.energy)
for telescope_id, dl1 in event.dl1.tel.items():
image = dl1.image
# peakpos = dl1.pulse_time
camera = event.inst.subarray.tels[telescope_id].camera
# display dl1 images
"""fig, axs = plt.subplots(1, 1, figsize=(6, 3))
d1 = CameraDisplay(camera, ax=axs)
axs.set_title('Image ' + camera.cam_id)
d1.image = dl1.image
plt.show()
plt.close(fig)"""
# cleaning
boundary, picture, min_neighbors = cleaning_level[camera.cam_id]
clean = tailcuts_clean(camera,
image,
boundary_thresh=boundary,
picture_thresh=picture,
min_number_picture_neighbors=min_neighbors)
if clean.sum() < 5:
continue
# SkyCoord
array_pointing_event = SkyCoord(
az=event.mcheader.run_array_direction[0],
alt=event.mcheader.run_array_direction[1],
frame=horizon_frame)
hillas_c = hillas_parameters(camera[clean], image[clean])
hillas_containers[telescope_id] = hillas_c
if len(hillas_containers) < 2:
continue
reconstruction_cleaned = reco.predict(
hillas_containers,
event.inst,
array_pointing_event,
)
array_pointing_reconstruction = SkyCoord(
az=reconstruction_cleaned.az,
alt=reconstruction_cleaned.alt,
frame=horizon_frame)
# angular distance between reconstructed and event image
print(
"angular distance: ",
array_pointing_event.separation(array_pointing_reconstruction))
# ground truth difference
cleaned_image = dl1.image.copy()
cleaned_image[~clean] = 0.0
ground_truth_image = event.mc.tel[
telescope_id].photo_electron_image
diff = np.linalg.norm(cleaned_image - ground_truth_image)
print("average ground truth difference: ", np.average(diff))
cl = closest_approach_distance(hillas_containers[telescope_id])
print("closest approach: ", cl)
"""# display ground truth image
# It fails for empty pixels
try:
hillas_params[tel_id] = hillas_parameters(camgeom, cleaned_image)
except:
pass
array_pointing = SkyCoord(
az=event.mcheader.run_array_direction[0],
alt=event.mcheader.run_array_direction[1],
frame=horizon_frame
)
if len(hillas_params) < 2:
continue
reco_result = reco.predict(hillas_params, event.inst, array_pointing, telescope_pointings)
# get angular offset between reconstructed shower direction and MC
# generated shower direction
off_angle = angular_separation(
event.mc.az,
event.mc.alt,
reco_result.az,
reco_result.alt
)
# Appending all estimated off angles
off_angles.append(off_angle.to(u.deg).value)
# calculate theta square for angles which are not nan
off_angles = np.array(off_angles)
class Reconstructor:
def __init__(self,
events_path: str,
telescopes_path: str,
replace_folder: str = None,
version="ML1",
telescopes: list = None):
if version == "ML2":
raise NotImplementedError(
"This reconstructor is not implemented to work with ML2 yet")
self.version = version
self.dataset = load_dataset(events_path,
telescopes_path,
replace_folder=replace_folder)
if telescopes is not None:
if isinstance(telescopes, str):
telescopes = [telescopes]
self.dataset = filter_dataset(self.dataset, telescopes)
self.reconstructor = HillasReconstructor()
self.array_directions = dict()
for hdf5_file in self.hdf5_files:
self.array_directions[hdf5_file] = load_array_direction(hdf5_file)
self.cameras_by_event = dict(
(event_id, []) for event_id in self.event_uids)
self.n_tels_by_event = {}
tel_ids = dict()
repeated_tel_ids = list()
for event_id, tel_type, tel_id, obs_id, folder, source, x, y in zip(
self.dataset["event_unique_id"], self.dataset["type"],
self.dataset["telescope_id"],
self.dataset["observation_indice"], self.dataset["folder"],
self.dataset["source"], self.dataset["x"], self.dataset["y"]):
if event_id not in tel_ids:
tel_ids[event_id] = list()
if event_id not in self.n_tels_by_event:
self.n_tels_by_event[event_id] = {}
if tel_type not in self.n_tels_by_event[event_id]:
self.n_tels_by_event[event_id][tel_type] = 1
else:
self.n_tels_by_event[event_id][tel_type] += 1
if tel_id in tel_ids[event_id]:
repeated_tel_ids.append(tel_id)
tel_ids[event_id].append(tel_id)
self.cameras_by_event[event_id].append(
(obs_id, tel_id, tel_type, folder, source, x, y))
@property
def event_uids(self) -> list:
return self.dataset["event_unique_id"].unique()
@property
def hdf5_files(self) -> np.ndarray:
return (self.dataset["folder"] + "/" + self.dataset["source"]).unique()
def get_event_hdf5_file(self, event_id: str, tel_id: str):
event_group = self.dataset.groupby("event_unique_id").get_group(
event_id)
tel_row: DataFrame = event_group.loc[event_group["telescope_id"] ==
tel_id]
tel_row = tel_row.loc[tel_row.index[0]]
return tel_row["folder"] + "/" + tel_row["source"]
@property
def camera_radius(self):
return get_camera_radius(self.dataset)
@property
def mc_values(self):
values = dict()
events = self.dataset[[
"event_unique_id", "alt", "az", "mc_energy", "core_x", "core_y"
]].drop_duplicates().set_index(["event_unique_id"])
for event_uid in self.event_uids:
event = events.loc[event_uid]
values[event_uid] = dict(alt=event["alt"],
az=event["az"],
mc_energy=event["mc_energy"],
core_x=event["core_x"],
core_y=event["core_y"])
return values
def reconstruct_event(self,
event_id: str,
event_cutflow: CutFlow,
obs_cutflow: CutFlow,
energy_regressor=None,
loose=False) -> Union[None, dict, Tuple[dict, dict]]:
run_array_direction = None
n_valid_tels = 0
hillas_containers = dict()
hillas_by_obs = dict()
time_gradients = dict()
leakages = dict()
types = dict()
n_islands = dict()
positions = dict()
meta = dict()
for obs_id, tel_id, tel_type, folder, source, x, y in self.cameras_by_event[
event_id]:
_, camera_name = split_tel_type(tel_type)
charge, peak = load_camera(source,
folder,
tel_type,
obs_id,
version=self.version)
params = get_observation_parameters(charge,
peak,
camera_name,
obs_cutflow,
cut=not loose)
if params is None:
continue
moments, leakage_c, _, time_gradient, obs_n_islands = params
# assert tel_id not in hillas_containers.keys()
hillas_containers[tel_id] = moments
assert (tel_type, obs_id) not in hillas_by_obs.keys()
meta[tel_id] = (tel_type, obs_id)
hillas_by_obs[(tel_type, obs_id)] = moments
positions[tel_id] = (x, y)
time_gradients[(tel_type, obs_id)] = time_gradient
leakages[(tel_type, obs_id)] = leakage_c
n_islands[(tel_type, obs_id)] = obs_n_islands
types[tel_id] = tel_type
hdf5_file = self.get_event_hdf5_file(event_id, tel_id)
telescope_alias = TELESCOPES_ALIAS[self.version][tel_type]
run_array_direction = self.array_directions[hdf5_file][
telescope_alias][tel_id]
n_valid_tels += 1
subarray = generate_subarray_description(self.dataset, event_id)
if event_cutflow.cut(CFE_MIN_TELS_RECO, len(hillas_containers)):
return
array_pointing = SkyCoord(az=run_array_direction[0] * u.rad,
alt=run_array_direction[1] * u.rad,
frame="altaz")
reco = self.reconstructor.predict(hillas_containers, subarray,
array_pointing)
mc = self.mc_values
if energy_regressor is None:
energy = None
else:
energy = energy_regressor.predict_event(positions, types,
hillas_containers, reco,
time_gradients, leakages,
n_islands, meta)
return dict(pred_az=2 * np.pi * u.rad + reco.az,
pred_alt=reco.alt,
pred_core_x=reco.core_x,
pred_core_y=reco.core_y,
h_max=reco.h_max,
alt=mc[event_id]["alt"] * u.rad,
az=mc[event_id]["az"] * u.rad,
core_x=mc[event_id]["core_x"],
core_y=mc[event_id]["core_y"],
mc_energy=mc[event_id]["mc_energy"],
energy=energy,
time_gradients=time_gradients,
leakages=leakages,
event_id=event_id,
n_islands=n_islands,
hillas=hillas_by_obs,
run_array_direction=run_array_direction)
def get_event_cams_and_foclens(self, event_id: str):
subarray = generate_subarray_description(self.dataset, event_id)
tel_types = subarray.telescope_types
return {
tel_types[i].camera.camera_name:
tel_types[i].optics.equivalent_focal_length.value
for i in range(len(tel_types))
}
def reconstruct_all(self,
max_events=None,
min_valid_observations=2,
energy_regressor=None,
npix_bounds: Tuple[float, float] = None,
charge_bounds: Tuple[float, float] = None,
ellipticity_bounds: Tuple[float, float] = None,
nominal_distance_bounds: Tuple[float, float] = None,
save_to: str = None,
save_hillas: str = None,
loose: bool = False) -> dict:
event_ids = self.event_uids
if max_events is not None and max_events < len(event_ids):
event_ids = event_ids[:max_events]
event_cutflow = generate_event_cutflow(min_valid_observations)
obs_cutflow = generate_observation_cutflow(self.camera_radius,
npix_bounds, charge_bounds,
ellipticity_bounds,
nominal_distance_bounds)
reconstructions = {}
for event_id in tqdm(event_ids):
reco = self.reconstruct_event(event_id,
event_cutflow,
obs_cutflow,
energy_regressor=energy_regressor,
loose=loose)
if reco is None:
continue
reconstructions[event_id] = reco
print()
print("==Observations Cutflow Summary==")
for cut in obs_cutflow.cuts.items():
print(cut[0], cut[1][1])
print()
print("==Event Cutflow Summary==")
for cut in event_cutflow.cuts.items():
print(cut[0], cut[1][1])
print()
perc_reconstructed = 100 * len(reconstructions) / len(event_ids)
print(
f"N. Events Reconstructed: {len(reconstructions)}/{len(event_ids)} ({perc_reconstructed}%)"
)
if save_to is not None:
self.save_predictions(reconstructions, save_to)
if save_hillas is not None:
self.save_hillas_params(reconstructions, save_hillas)
return reconstructions
def plot_metrics(self,
max_events: int = None,
min_valid_observations=2,
energy_regressor: EnergyModel = None,
plot_charges: bool = False,
save_to: str = None,
save_hillas: str = None,
save_plots: str = None):
import ctaplot
reco = self.reconstruct_all(
max_events,
min_valid_observations=min_valid_observations,
energy_regressor=energy_regressor)
preds = list(reco.values())
if save_plots is not None:
reco_alt = np.array(
[pred['pred_alt'] / (1 * u.rad) for pred in preds])
reco_az = np.array(
[pred['pred_az'] / (1 * u.rad) for pred in preds])
alt = np.array([pred['alt'] / (1 * u.rad) for pred in preds])
az = np.array([pred['az'] / (1 * u.rad) for pred in preds])
if energy_regressor is not None:
energy = np.array([pred['energy'] for pred in preds])
mc_energy = np.array([pred['mc_energy'] for pred in preds])
_, (ax1, ax2) = plt.subplots(1, 2)
ctaplot.plot_angular_resolution_per_energy(reco_alt,
reco_az,
alt,
az,
mc_energy,
ax=ax1)
if energy_regressor is not None:
ctaplot.plot_energy_resolution(mc_energy, energy, ax=ax2)
plt.savefig(save_plots)
@staticmethod
def plot_prediction_vs_real(pred, real, ax, title):
ax.scatter(real, pred)
ax.set_title(title)
ax.grid('on')
@staticmethod
def plot(results: DataFrame, save_to: str):
import os
from gerumo import plot_error_and_angular_resolution
results['true_alt'] = results['alt']
results['true_az'] = results['az']
results['true_mc_energy'] = results['mc_energy']
plot_error_and_angular_resolution(results,
save_to=os.path.join(
save_to,
"angular_resolution.png"),
ylim=[0, 2])
fig, axes = plt.subplots(1, 2)
ax1, ax2 = axes
Reconstructor.plot_prediction_vs_real(results['pred_alt'],
results['alt'], ax1, "Altitude")
Reconstructor.plot_prediction_vs_real(
results['pred_az'].apply(
lambda rad: np.arctan2(np.sin(rad), np.cos(rad))),
results['az'].apply(
lambda rad: np.arctan2(np.sin(rad), np.cos(rad))), ax2,
"Azimuth")
fig.savefig(os.path.join(save_to, "scatter.png"))
@staticmethod
def save_predictions(reco: dict, path: str):
reco = {
event_id: dict(pred_az=r["pred_az"].value,
pred_alt=r["pred_alt"].value,
pred_core_x=r["pred_core_x"].value,
pred_core_y=r["pred_core_y"].value,
alt=r["alt"].value,
az=r["az"].value,
core_x=r["core_x"],
core_y=r["core_y"],
mc_energy=r["mc_energy"],
energy=r["energy"],
h_max=r["h_max"].value)
for event_id, r in reco.items()
}
df = DataFrame.from_dict(reco, orient="index")
df.to_csv(path, sep=',', index_label="event_unique_id")
@staticmethod
def save_hillas_params(reco: dict, path: str):
columns = [
"event_unique_id", "observation_indice", "type", "intensity",
"kurtosis", "length", "phi", "psi", "r", "skewness", "width", "x",
"y", "wl", "time_gradient", "leakage_intensity_width_2",
"n_islands", "telescope_az", "telescope_alt"
]
params = list()
for event_id, r in reco.items():
run_array_direction = r["run_array_direction"]
for (tel_type, obs_id), moments in r["hillas"].items():
time_gradient = r["time_gradients"][(tel_type, obs_id)]
leakage_c = r["leakages"][(tel_type, obs_id)]
n_islands = r["n_islands"][(tel_type, obs_id)]
obs_params = (event_id, obs_id, tel_type, moments.intensity,
moments.kurtosis, moments.length.value,
moments.phi.value, moments.psi.value,
moments.r.value, moments.skewness,
moments.width.value, moments.x.value,
moments.y.value, moments.width.value /
moments.length.value, time_gradient,
leakage_c.intensity_width_2, n_islands,
run_array_direction[0], run_array_direction[1])
params.append(obs_params)
df = DataFrame(params, columns=columns).set_index(columns[:3])
df.to_csv(path, sep=',')