def test_reconstructors(reconstructors):
"""
a test of the complete fit procedure on one event including:
• tailcut cleaning
• hillas parametrisation
• HillasPlane creation
• direction fit
• position fit
in the end, proper units in the output are asserted"""
filename = get_dataset_path(
"gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz")
source = EventSource(filename, max_events=10)
subarray = source.subarray
calib = CameraCalibrator(source.subarray)
horizon_frame = AltAz()
for event in source:
calib(event)
sim_shower = event.simulation.shower
array_pointing = SkyCoord(az=sim_shower.az,
alt=sim_shower.alt,
frame=horizon_frame)
telescope_pointings = {}
for tel_id, dl1 in event.dl1.tel.items():
geom = source.subarray.tel[tel_id].camera.geometry
telescope_pointings[tel_id] = SkyCoord(
alt=event.pointing.tel[tel_id].altitude,
az=event.pointing.tel[tel_id].azimuth,
frame=horizon_frame,
)
mask = tailcuts_clean(geom,
dl1.image,
picture_thresh=10.0,
boundary_thresh=5.0)
dl1.parameters = ImageParametersContainer()
try:
moments = hillas_parameters(geom[mask], dl1.image[mask])
except HillasParameterizationError:
dl1.parameters.hillas = HillasParametersContainer()
continue
# Make sure we provide only good images for the test
if np.isnan(moments.width.value) or (moments.width.value == 0):
dl1.parameters.hillas = HillasParametersContainer()
else:
dl1.parameters.hillas = moments
hillas_dict = {
tel_id: dl1.parameters.hillas
for tel_id, dl1 in event.dl1.tel.items()
if np.isfinite(dl1.parameters.hillas.intensity)
}
if len(hillas_dict) < 2:
continue
for count, reco_method in enumerate(reconstructors):
if reco_method is HillasReconstructor:
reconstructor = HillasReconstructor(subarray)
reconstructor(event)
event.dl2.stereo.geometry["HillasReconstructor"].alt.to(u.deg)
event.dl2.stereo.geometry["HillasReconstructor"].az.to(u.deg)
event.dl2.stereo.geometry["HillasReconstructor"].core_x.to(u.m)
assert event.dl2.stereo.geometry[
"HillasReconstructor"].is_valid
else:
reconstructor = reco_method()
try:
reconstructor_out = reconstructor.predict(
hillas_dict,
source.subarray,
array_pointing,
telescope_pointings,
)
except InvalidWidthException:
continue
reconstructor_out.alt.to(u.deg)
reconstructor_out.az.to(u.deg)
reconstructor_out.core_x.to(u.m)
assert reconstructor_out.is_valid
# unoptimized cleaning levels, copied from
# https://github.com/tudo-astroparticlephysics/cta_preprocessing
cleaning_level = {
'LSTCam': (3.5, 7.5, 2), # ?? (3, 6) for Abelardo...
'FlashCam': (4, 8, 2), # there is some scaling missing?
'ASTRICam': (5, 7, 2),
}
input_url = get_dataset_path('gamma_test_large.simtel.gz')
event_source = event_source(input_url)
calibrator = CameraCalibrator(subarray=event_source.subarray)
horizon_frame = AltAz()
reco = HillasReconstructor()
for event in event_source:
print('Event', event.count)
calibrator(event)
# mapping of telescope_id to parameters for stereo reconstruction
hillas_containers = {}
time_gradients = {}
for telescope_id, dl1 in event.dl1.tel.items():
geom = event_source.subarray.tels[telescope_id].camera.geometry
image = dl1.image
peakpos = dl1.peak_time
from ctapipe.image import hillas_parameters
from ctapipe.image import tailcuts_clean
from ctapipe.io import event_source
from ctapipe.reco import HillasReconstructor
from ctapipe.utils import datasets
if len(sys.argv) >= 2:
filename = sys.argv[1]
else:
# importing data from avaiable datasets in ctapipe
filename = datasets.get_dataset_path("gamma_test_large.simtel.gz")
# reading the Monte Carlo file for LST
source = event_source(filename, allowed_tels={1, 2, 3, 4})
reco = HillasReconstructor()
calib = CameraCalibrator()
off_angles = []
for event in source:
# calibrating the event
calib(event)
hillas_params = {}
subarray = event.inst.subarray
# pointing direction of the telescopes
point_azimuth = {}
point_altitude = {}
for tel_id in event.dl0.tels_with_data:
from ctapipe.io import event_source
from ctapipe.reco import HillasReconstructor
from ctapipe.utils import datasets
if len(sys.argv) >= 2:
filename = sys.argv[1]
else:
# importing data from avaiable datasets in ctapipe
filename = datasets.get_dataset_path("gamma_test_large.simtel.gz")
# reading the Monte Carlo file for LST
source = event_source(filename, allowed_tels={1, 2, 3, 4})
reco = HillasReconstructor()
calib = CameraCalibrator()
off_angles = []
for event in source:
# calibrating the event
calib.calibrate(event)
hillas_params = {}
subarray = event.inst.subarray
# pointing direction of the telescopes
point_azimuth = {}
point_altitude = {}
for tel_id in event.dl0.tels_with_data:
# https://github.com/tudo-astroparticlephysics/cta_preprocessing
cleaning_level = {
'LSTCam': (3.5, 7.5, 2), # ?? (3, 6) for Abelardo...
'FlashCam': (4, 8, 2), # there is some scaling missing?
'ASTRICam': (5, 7, 2),
}
input_url = get_dataset_path('gamma_test_large.simtel.gz')
event_source = EventSourceFactory.produce(input_url=input_url)
calibrator = CameraCalibrator(
eventsource=event_source,
)
reco = HillasReconstructor()
for event in event_source:
print('Event', event.count)
calibrator.calibrate(event)
# mapping of telescope_id to parameters for stereo reconstruction
hillas_containers = {}
pointing_azimuth = {}
pointing_altitude = {}
time_gradients = {}
for telescope_id, dl1 in event.dl1.tel.items():
camera = event.inst.subarray.tels[telescope_id].camera
image = dl1.image[0]
center = (0, 0)
line = np.array([x, y])
line = line.T
return closest_distance(center, line)
input_url = find_in_path(
'gamma_20deg_0deg_run101___cta-prod3-lapalma3-2147m-LaPalma.simtel.gz',
'/home/matteo/ctasoft/resources')
events = event_source(input_url, max_events=1)
calibrator = CameraCalibrator()
horizon_frame = AltAz()
reco = HillasReconstructor()
hillas_containers = {}
array_events = []
for event in events:
calibrator(event)
array_events.append(event)
print("energy: ", event.mc.energy)
print(event.r0.tels_with_data)
telescope_id = 3
def process_event(event, calibrator, config):
'''Processes one event.
Calls calculate_image_features and performs a stereo hillas reconstruction.
ReconstructedShowerContainer will be filled with nans (+units) if hillas failed.
Returns:
--------
ArrayEventContainer
list(telescope_event_containers.values())
'''
logging.info(f'processing event {event.dl0.event_id}')
calibrator(event)
telescope_types = []
hillas_reconstructor = HillasReconstructor()
telescope_event_containers = {}
horizon_frame = AltAz()
telescope_pointings = {}
array_pointing = SkyCoord(
az=event.mcheader.run_array_direction[0],
alt=event.mcheader.run_array_direction[1],
frame=horizon_frame,
)
for telescope_id, dl1 in event.dl1.tel.items():
cam_type = event.inst.subarray.tels[telescope_id].camera.cam_id
if cam_type not in config.cleaning_level.keys():
logging.info(f"No cleaning levels for camera {cam_type}. Skipping event")
continue
telescope_types.append(str(event.inst.subarray.tels[telescope_id].optics))
try:
telescope_event_containers[telescope_id] = calculate_image_features(
telescope_id, event, dl1, config
)
except HillasParameterizationError:
logging.info(
f'Error calculating hillas features for event {event.dl0.event_id, telescope_id}',
exc_info=True,
)
continue
except Exception:
logging.info(
f'Error calculating image features for event {event.dl0.event_id, telescope_id}',
exc_info=True,
)
continue
telescope_pointings[telescope_id] = SkyCoord(
alt=telescope_event_containers[telescope_id].pointing.altitude,
az=telescope_event_containers[telescope_id].pointing.azimuth,
frame=horizon_frame,
)
if len(telescope_event_containers) < 1:
raise Exception('None of the allowed telescopes triggered for event %s', event.dl0.event_id)
parameters = {tel_id: telescope_event_containers[tel_id].hillas for tel_id in telescope_event_containers}
try:
reconstruction_container = hillas_reconstructor.predict(
parameters, event.inst, array_pointing, telescopes_pointings=telescope_pointings
)
reconstruction_container.prefix = ''
except Exception:
logging.info(
'Not enough telescopes for which Hillas parameters could be reconstructed', exc_info=True
)
reconstruction_container = ReconstructedShowerContainer()
reconstruction_container.alt = u.Quantity(np.nan, u.rad)
reconstruction_container.alt_uncert = u.Quantity(np.nan, u.rad)
reconstruction_container.az = u.Quantity(np.nan, u.rad)
reconstruction_container.az_uncert = np.nan
reconstruction_container.core_x = u.Quantity(np.nan, u.m)
reconstruction_container.core_y = u.Quantity(np.nan, u.m)
reconstruction_container.core_uncert = np.nan
reconstruction_container.h_max = u.Quantity(np.nan, u.m)
reconstruction_container.h_max_uncert = np.nan
reconstruction_container.is_valid = False
reconstruction_container.tel_ids = []
reconstruction_container.average_intensity = np.nan
reconstruction_container.goodness_of_fit = np.nan
reconstruction_container.prefix = ''
calculate_distance_to_core(telescope_event_containers, event, reconstruction_container)
mc_container = copy.deepcopy(event.mc)
mc_container.tel = None
mc_container.prefix = 'mc'
counter = Counter(telescope_types)
array_event = ArrayEventContainer(
array_event_id=event.dl0.event_id,
run_id=event.r0.obs_id,
reco=reconstruction_container,
total_intensity=sum([t.hillas.intensity for t in telescope_event_containers.values()]),
num_triggered_lst=counter['LST'],
num_triggered_mst=counter['MST'],
num_triggered_sst=counter['SST'],
num_triggered_telescopes=len(telescope_types),
mc=mc_container,
)
return array_event, list(telescope_event_containers.values())
# unoptimized cleaning levels, copied from
# https://github.com/tudo-astroparticlephysics/cta_preprocessing
cleaning_level = {
'LSTCam': (3.5, 7.5, 2), # ?? (3, 6) for Abelardo...
'FlashCam': (4, 8, 2), # there is some scaling missing?
'ASTRICam': (5, 7, 2),
}
input_url = get_dataset_path('gamma_test_large.simtel.gz')
event_source = event_source(input_url)
calibrator = CameraCalibrator()
horizon_frame = AltAz()
reco = HillasReconstructor()
for event in event_source:
print('Event', event.count)
calibrator(event)
# mapping of telescope_id to parameters for stereo reconstruction
hillas_containers = {}
time_gradients = {}
for telescope_id, dl1 in event.dl1.tel.items():
camera = event.inst.subarray.tels[telescope_id].camera
image = dl1.image
peakpos = dl1.pulse_time
iof.write_mcheader(event.mcheader, output_filename, obs_id=event.r0.obs_id, filters=filters, metadata=metadata)
extra_im_c = iof.ExtraImageInfo()
event_info_c = iof.EventInfo()
with HDF5TableWriter(filename=output_filename, group_name='events', mode='a') as writer:
for ii, event in enumerate(events):
if ii%1 == 0:
print(ii)
calibrator(event)
if len(event.r0.tels_with_data) < 2:
continue
reco = HillasReconstructor()
# mapping of telescope_id to parameters for stereo reconstruction
hillas_dict = {}
#time_gradients = {}
telescope_pointings = {}
for telescope_id, dl1 in event.dl1.tel.items():
tel = event.dl1.tel[telescope_id]
tel.prefix = '' # don't really need one for HDF5 writer
extra_im_c.tel_id = telescope_id
extra_im_c.event_id = event.r0.event_id
extra_im_c.obs_id = event.r0.obs_id
def process_event(event, reco_algorithm='planes'):
'''
Processes
'''
features = {}
params = {}
cleaning_mask = {}
pointing_azimuth = {}
pointing_altitude = {}
tel_x = {}
tel_y = {}
tel_focal_lengths = {}
for telescope_id, dl1 in event.dl1.tel.items():
camera = event.inst.subarray.tels[telescope_id].camera
if camera.cam_id not in allowed_cameras:
continue
telescope_type_name = event.inst.subarray.tels[
telescope_id].optics.tel_type
boundary_thresh, picture_thresh, min_number_picture_neighbors = cleaning_level[
camera.cam_id]
mask = tailcuts_clean(
camera,
dl1.image[0],
boundary_thresh=boundary_thresh,
picture_thresh=picture_thresh,
min_number_picture_neighbors=min_number_picture_neighbors)
cleaning_mask[telescope_id] = mask
try:
h = hillas_parameters_5(
camera[mask],
dl1.image[0, mask],
)
params[telescope_id] = h
except HillasParameterizationError:
continue
pointing_azimuth[
telescope_id] = event.mc.tel[telescope_id].azimuth_raw * u.rad
pointing_altitude[
telescope_id] = event.mc.tel[telescope_id].altitude_raw * u.rad
tel_x[telescope_id] = event.inst.subarray.positions[telescope_id][0]
tel_y[telescope_id] = event.inst.subarray.positions[telescope_id][1]
telescope_description = event.inst.subarray.tel[telescope_id]
tel_focal_lengths[
telescope_id] = telescope_description.optics.equivalent_focal_length
d = {
'array_event_id': event.dl0.event_id,
'telescope_id': int(telescope_id),
'camera_name': camera.cam_id,
'camera_id': names_to_id[camera.cam_id],
'run_id': event.r0.obs_id,
'telescope_type_name': telescope_type_name,
'telescope_type_id': types_to_id[telescope_type_name],
'pointing_azimuth': event.mc.tel[telescope_id].azimuth_raw,
'pointing_altitude': event.mc.tel[telescope_id].altitude_raw,
'mirror_area': telescope_description.optics.mirror_area,
'focal_length':
telescope_description.optics.equivalent_focal_length,
}
d.update(h.as_dict())
features[telescope_id] = ({k: strip_unit(v) for k, v in d.items()})
if reco_algorithm == 'intersection':
reco = HillasIntersection()
array_direction = SkyCoord(alt=event.mcheader.run_array_direction[1],
az=event.mcheader.run_array_direction[0],
frame='altaz')
reconstruction = reco.predict(params, tel_x, tel_y, tel_focal_lengths,
array_direction)
elif reco_algorithm == 'planes':
reco = HillasReconstructor()
reconstruction = reco.predict(params, event.inst, pointing_altitude,
pointing_azimuth)
for telescope_id in event.dl1.tel.keys():
if telescope_id not in params:
continue
camera = event.inst.subarray.tels[telescope_id].camera
if camera.cam_id not in allowed_cameras:
continue
pos = event.inst.subarray.positions[telescope_id]
x, y = pos[0], pos[1]
core_x = reconstruction.core_x
core_y = reconstruction.core_y
d = np.sqrt((core_x - x)**2 + (core_y - y)**2)
features[telescope_id]['distance_to_core'] = d.value
return pd.DataFrame(list(
features.values())), reconstruction, params, cleaning_mask
def main():
input_url = find_in_path(
'gamma_20deg_0deg_run101___cta-prod3-lapalma3-2147m-LaPalma.simtel.gz',
'/home/matteo/ctasoft/resources')
events = event_source(input_url, max_events=1)
calibrator = CameraCalibrator()
horizon_frame = AltAz()
reco = HillasReconstructor()
hillas_containers = {}
for event in events:
calibrator(event)
print("energy: ", event.mc.energy)
for telescope_id, dl1 in event.dl1.tel.items():
image = dl1.image
# peakpos = dl1.pulse_time
camera = event.inst.subarray.tels[telescope_id].camera
# display dl1 images
"""fig, axs = plt.subplots(1, 1, figsize=(6, 3))
d1 = CameraDisplay(camera, ax=axs)
axs.set_title('Image ' + camera.cam_id)
d1.image = dl1.image
plt.show()
plt.close(fig)"""
# cleaning
boundary, picture, min_neighbors = cleaning_level[camera.cam_id]
clean = tailcuts_clean(camera,
image,
boundary_thresh=boundary,
picture_thresh=picture,
min_number_picture_neighbors=min_neighbors)
if clean.sum() < 5:
continue
# SkyCoord
array_pointing_event = SkyCoord(
az=event.mcheader.run_array_direction[0],
alt=event.mcheader.run_array_direction[1],
frame=horizon_frame)
hillas_c = hillas_parameters(camera[clean], image[clean])
hillas_containers[telescope_id] = hillas_c
if len(hillas_containers) < 2:
continue
reconstruction_cleaned = reco.predict(
hillas_containers,
event.inst,
array_pointing_event,
)
array_pointing_reconstruction = SkyCoord(
az=reconstruction_cleaned.az,
alt=reconstruction_cleaned.alt,
frame=horizon_frame)
# angular distance between reconstructed and event image
print(
"angular distance: ",
array_pointing_event.separation(array_pointing_reconstruction))
# ground truth difference
cleaned_image = dl1.image.copy()
cleaned_image[~clean] = 0.0
ground_truth_image = event.mc.tel[
telescope_id].photo_electron_image
diff = np.linalg.norm(cleaned_image - ground_truth_image)
print("average ground truth difference: ", np.average(diff))
cl = closest_approach_distance(hillas_containers[telescope_id])
print("closest approach: ", cl)
"""# display ground truth image
from ctapipe.image import tailcuts_clean
from ctapipe.io import event_source
from ctapipe.reco import HillasReconstructor
from ctapipe.utils import datasets, linalg
# importing data from avaiable datasets in ctapipe
filename = datasets.get_dataset("gamma_test_large.simtel.gz")
# reading the Monte Carlo file for LST
source = event_source(filename, allowed_tels={1, 2, 3, 4})
# pointing direction of the telescopes
point_azimuth = {}
point_altitude = {}
reco = HillasReconstructor()
calib = CameraCalibrator(r1_product="HESSIOR1Calibrator")
off_angles = []
for event in source:
# The direction the incident particle. Converting Monte Carlo Shower
# parameter theta and phi to corresponding to 3 components (x,y,z) of a
# vector
shower_azimuth = event.mc.az # same as in Monte Carlo file i.e. phi
shower_altitude = np.pi * u.rad / 2 - event.mc.alt # altitude = 90 - theta
shower_direction = linalg.set_phi_theta(shower_azimuth, shower_altitude)
# calibrating the event
calib.calibrate(event)
hillas_params = {}
subarray = event.inst.subarray
from ctapipe.image import hillas_parameters
from ctapipe.image import tailcuts_clean
from ctapipe.io import EventSource
from ctapipe.reco import HillasReconstructor
from ctapipe.utils import datasets
if len(sys.argv) >= 2:
filename = sys.argv[1]
else:
# importing data from available datasets in ctapipe
filename = datasets.get_dataset_path("gamma_test_large.simtel.gz")
# reading the Monte Carlo file for LST
source = EventSource(filename, allowed_tels={1, 2, 3, 4})
reco = HillasReconstructor()
calib = CameraCalibrator(subarray=source.subarray)
horizon_frame = AltAz()
off_angles = []
for event in source:
# calibrating the event
calib(event)
hillas_params = {}
subarray = source.subarray
# dictionary for the pointing directions of the telescopes
telescope_pointings = {}
from ctapipe.io import event_source
from ctapipe.reco import HillasReconstructor
from ctapipe.utils import datasets
if len(sys.argv) >= 2:
filename = sys.argv[1]
else:
# importing data from available datasets in ctapipe
filename = datasets.get_dataset_path("gamma_test_large.simtel.gz")
# reading the Monte Carlo file for LST
source = event_source(filename, allowed_tels={1, 2, 3, 4})
reco = HillasReconstructor()
calib = CameraCalibrator(subarray=source.subarray)
horizon_frame = AltAz()
off_angles = []
for event in source:
# calibrating the event
calib(event)
hillas_params = {}
subarray = event.inst.subarray
# dictionary for the pointing directions of the telescopes
telescope_pointings = {}
def process_event(event, config):
'''
Processes
'''
reco_algorithm = config.reco_algorithm
features = {}
params = {}
pointing_azimuth = {}
pointing_altitude = {}
tel_x = {}
tel_y = {}
tel_focal_lengths = {}
cleaning_method = config.cleaning_method
valid_cleaning_methods = ['tailcuts_clean', 'fact_image_cleaning']
if cleaning_method not in valid_cleaning_methods:
print('Cleaning Method not implemented')
print('Please use one of ', valid_cleaning_methods)
return None
for telescope_id, dl1 in event.dl1.tel.items():
camera = event.inst.subarray.tels[telescope_id].camera
if camera.cam_id not in config.allowed_cameras:
continue
telescope_type_name = event.inst.subarray.tels[
telescope_id].optics.tel_type
if cleaning_method == 'tailcuts_clean':
boundary_thresh, picture_thresh, min_number_picture_neighbors = config.cleaning_level[
camera.cam_id]
mask = tailcuts_clean(camera, dl1.image[0],
*config.cleaning_level[camera.cam_id])
elif cleaning_method == 'fact_image_cleaning':
mask = fact_image_cleaning(
camera, dl1.image[0],
*config.cleaning_level_fact[camera.cam_id])
try:
cleaned = dl1.image[0].copy()
cleaned[~mask] = 0
hillas_container = hillas_parameters(
camera,
cleaned,
)
params[telescope_id] = hillas_container
except HillasParameterizationError:
continue
# probably wise to add try...except blocks here as well
# Add more Features here (look what ctapipe can do, timing?)
num_islands, island_labels = number_of_islands(camera, mask)
island_dict = {
'num_islands': num_islands,
'island_labels': island_labels
}
leakage_container = leakage(camera, dl1.image[0], mask)
timing_container = timing_parameters(camera, dl1.image[0],
dl1.peakpos[0], hillas_container)
pointing_azimuth[
telescope_id] = event.mc.tel[telescope_id].azimuth_raw * u.rad
pointing_altitude[
telescope_id] = event.mc.tel[telescope_id].altitude_raw * u.rad
tel_x[telescope_id] = event.inst.subarray.positions[telescope_id][0]
tel_y[telescope_id] = event.inst.subarray.positions[telescope_id][1]
telescope_description = event.inst.subarray.tel[telescope_id]
tel_focal_lengths[
telescope_id] = telescope_description.optics.equivalent_focal_length
d = {
'array_event_id': event.dl0.event_id,
'telescope_id': int(telescope_id),
'camera_name': camera.cam_id,
'camera_id': config.names_to_id[camera.cam_id],
'run_id': event.r0.obs_id,
'telescope_type_name': telescope_type_name,
'telescope_type_id': config.types_to_id[telescope_type_name],
'pointing_azimuth': event.mc.tel[telescope_id].azimuth_raw,
'pointing_altitude': event.mc.tel[telescope_id].altitude_raw,
'mirror_area': telescope_description.optics.mirror_area,
'focal_length':
telescope_description.optics.equivalent_focal_length,
}
d.update(hillas_container.as_dict())
d.update(leakage_container.as_dict())
d.update(island_dict)
d.update(timing_container.as_dict())
features[telescope_id] = ({k: strip_unit(v) for k, v in d.items()})
if reco_algorithm == 'intersection':
reco = HillasIntersection()
array_direction = SkyCoord(alt=event.mcheader.run_array_direction[1],
az=event.mcheader.run_array_direction[0],
frame='altaz')
reconstruction = reco.predict(params, tel_x, tel_y, tel_focal_lengths,
array_direction)
elif reco_algorithm == 'planes':
reco = HillasReconstructor()
reconstruction = reco.predict(params, event.inst, pointing_altitude,
pointing_azimuth)
for telescope_id in event.dl1.tel.keys():
if telescope_id not in params:
continue
camera = event.inst.subarray.tels[telescope_id].camera
if camera.cam_id not in config.allowed_cameras:
continue
pos = event.inst.subarray.positions[telescope_id]
x, y = pos[0], pos[1]
core_x = reconstruction.core_x
core_y = reconstruction.core_y
d = np.sqrt((core_x - x)**2 + (core_y - y)**2)
features[telescope_id]['distance_to_core'] = d.value
return pd.DataFrame(list(features.values())), reconstruction, params
def process_event(
event,
telescopes,
subarray,
stereo,
calib,
apply_quality_cuts):
if stereo:
hillas_containers = {}
telescope_pointings = {}
horizon_frame = AltAz()
reco = HillasReconstructor()
# Calibrate the event
calib(event)
# Container to count what telescopes types were triggered in this event
event_tels = []
tel_data = []
for tel_id, dl1 in event.dl1.tel.items():
# If specific telescopes were specified, skip those that weren't
if telescopes is not None:
if (tel_id not in telescopes):
continue
tel = subarray.tels[tel_id]
# Process the telescope event
tel_type, tel_data_dict, hillas_c = process_telescope(tel, dl1, stereo)
# Add the telescope event data if it wasn't skipped
if tel_type is not None:
if apply_quality_cuts:
if tel_data_dict['intensity'] < 60 or tel_data_dict['nislands'] > 3 or tel_data_dict['n_survived_pixels'] < 6 or tel_data_dict['intensity_width_1'] > 0.1:
continue
# Calculate mc impact distance
x1 = event.mc.core_x.value
y1 = event.mc.core_y.value
x2 = subarray.positions[tel_id][0].value
y2 = subarray.positions[tel_id][1].value
v = [x2 - x1, y2 - y1]
mc_impact_distance = np.linalg.norm(v)
# Adding a few extras to the telescope event data
tel_data_dict.update({
'array_event_id': event.index.event_id,
'run_id': event.index.obs_id,
'telescope_id': tel_id,
'telescope_event_id': tel_id,
'mc_impact_distance': mc_impact_distance})
# Add data to the tables used for geometric reconstruction
if stereo:
tel_data_dict.update({'impact_distance': np.nan})
hillas_containers[tel_id] = hillas_c
telescope_pointings[tel_id] = SkyCoord(
alt=event.mc.tel[tel_id].altitude_raw * u.rad,
az=event.mc.tel[tel_id].azimuth_raw * u.rad,
frame=horizon_frame)
event_tels.append(tel_type)
tel_data.append(tel_data_dict)
# Skip event if no telescopes were processed
if len(event_tels) == 0:
return None, None
arr_data = {}
if stereo and len(event_tels) > 1:
array_pointing = SkyCoord(
az=event.mcheader.run_array_direction[0],
alt=event.mcheader.run_array_direction[1],
frame=horizon_frame)
# Do geometric direction reconstruction
reconst = reco.predict(
hillas_containers,
subarray,
array_pointing,
telescope_pointings)
# Calculate impact distance from reconstructed core position
for i in range(len(tel_data)):
x1 = reconst.core_x.value
y1 = reconst.core_y.value
x2 = subarray.positions[tel_data[i]['telescope_id']][0].value
y2 = subarray.positions[tel_data[i]['telescope_id']][1].value
v = [x2 - x1, y2 - y1]
impact_distance = np.linalg.norm(v)
tel_data[i]['impact_distance'] = impact_distance
# Grab the extra stereo info for array_events
reconst_az = reconst.az.value
if reconst_az < -np.pi:
reconst_az += 2*np.pi
if reconst_az > np.pi:
reconst_az -= 2*np.pi
arr_data.update({
'alt': reconst.alt.value,
'alt_uncert': reconst.alt_uncert.value,
'average_intensity': reconst.average_intensity,
'az': reconst_az,
'az_uncert': reconst.az_uncert,
'core_uncert': reconst.core_uncert,
'core_x': reconst.core_x.value,
'core_y': reconst.core_y.value,
'goodness_of_fit': reconst.goodness_of_fit,
'h_max': reconst.h_max.value,
'h_max_uncert': reconst.h_max_uncert,
'is_valid': reconst.is_valid,
'stereo_flag': True})
if stereo and len(event_tels) == 1:
# If only one telescope is triggered in a stereo array, replace
# stereo features with NaN
arr_data.update({
'alt': np.nan,
'alt_uncert': np.nan,
'average_intensity': np.nan,
'az': np.nan,
'az_uncert': np.nan,
'core_uncert': np.nan,
'core_x': np.nan,
'core_y': np.nan,
'goodness_of_fit': np.nan,
'h_max': np.nan,
'h_max_uncert': np.nan,
'is_valid': np.nan,
'stereo_flag': False})
# Grab info for array event data
arr_data.update({
'array_event_id': event.index.event_id,
'run_id': event.index.obs_id,
'azimuth_raw': event.mc.tel[tel_id].azimuth_raw,
'altitude_raw': event.mc.tel[tel_id].altitude_raw,
'azimuth_cor': event.mc.tel[tel_id].azimuth_cor,
'altitude_cor': event.mc.tel[tel_id].altitude_cor,
'num_triggered_lst': event_tels.count('LST'),
'num_triggered_mst': event_tels.count('MST'),
'num_triggered_sst': event_tels.count('SST'),
'mc_energy': event.mc.energy.value,
'true_source_alt': event.mc.alt.value,
'true_source_az': event.mc.az.value,
'mc_core_x': event.mc.core_x.value,
'mc_core_y': event.mc.core_y.value,
'mc_h_first_int': event.mc.h_first_int.value,
'mc_x_max': event.mc.x_max.value,
'mc_shower_primary_id': event.mc.shower_primary_id})
return tel_data, arr_data
class DirectionBolt(TicklessBatchingBolt):
outputs = ['array_event_id', 'direction']
reco = HillasReconstructor()
instrument = []
# monkey patch this huansohn. this is super slow otherwise. who needs max h anyways
def _dummy_function_h_max(self, hillas_dict, subarray, tel_phi):
return -1
def initialize(self, conf, ctx):
self.reco.fit_h_max = self._dummy_function_h_max
instrument_description = '/Users/mackaiver/Development/wordcount/instrument_description.pkl'
self.instrument = pickle.load(open(instrument_description, 'rb'))
# do some horrible things to silencece astropy warnings in ctapipe
warnings.filterwarnings('ignore', category=AstropyDeprecationWarning, append=True)
warnings.filterwarnings('ignore', category=FutureWarning, append=True)
# def process(self, tup):
# row = tup.values[0]
# width = row.width
# self.emit([row, width])
def group_key(self, tup):
array_event_id = tup.values[0]
return array_event_id # collect batches of words
def process_batch(self, key, tups):
# emit the count of words we had per batch
group = pd.DataFrame(tups)
result = self.reconstruct_direction(key, group,)
print(result)
self.emit([key, result])
def reconstruct_direction(self, array_event_id, group):
params = {}
pointing_azimuth = {}
pointing_altitude = {}
for index, row in group.iterrows():
tel_id = row.telescope_id
# the data in each event has to be put inside these namedtuples to call reco.predict
moments = SubMomentParameters(size=row.intensity, cen_x=row.x * u.m, cen_y=row.y * u.m, length=row.length * u.m, width=row.width * u.m, psi=row.psi * u.rad)
params[tel_id] = moments
pointing_azimuth[tel_id] = row.pointing_azimuth * u.rad
pointing_altitude[tel_id] = row.pointing_altitude * u.rad
try:
reconstruction = reco.predict(params, instrument, pointing_azimuth, pointing_altitude)
except NameError:
return {'alt_prediction': np.nan,
'az_prediction': np.nan,
'core_x_prediction': np.nan,
'core_y_prediction': np.nan,
'array_event_id': array_event_id,
}
if reconstruction.alt.si.value == np.nan:
print('Not reconstructed')
print(params)
return {'alt_prediction': ((np.pi / 2) - reconstruction.alt.si.value), # TODO srsly now? FFS
'az_prediction': reconstruction.az.si.value,
'core_x_prediction': reconstruction.core_x.si.value,
'core_y_prediction': reconstruction.core_y.si.value,
'array_event_id': array_event_id,
# 'h_max_prediction': reconstruction.h_max.si.value
}
class Reconstructor:
def __init__(self,
events_path: str,
telescopes_path: str,
replace_folder: str = None,
version="ML1",
telescopes: list = None):
if version == "ML2":
raise NotImplementedError(
"This reconstructor is not implemented to work with ML2 yet")
self.version = version
self.dataset = load_dataset(events_path,
telescopes_path,
replace_folder=replace_folder)
if telescopes is not None:
if isinstance(telescopes, str):
telescopes = [telescopes]
self.dataset = filter_dataset(self.dataset, telescopes)
self.reconstructor = HillasReconstructor()
self.array_directions = dict()
for hdf5_file in self.hdf5_files:
self.array_directions[hdf5_file] = load_array_direction(hdf5_file)
self.cameras_by_event = dict(
(event_id, []) for event_id in self.event_uids)
self.n_tels_by_event = {}
tel_ids = dict()
repeated_tel_ids = list()
for event_id, tel_type, tel_id, obs_id, folder, source, x, y in zip(
self.dataset["event_unique_id"], self.dataset["type"],
self.dataset["telescope_id"],
self.dataset["observation_indice"], self.dataset["folder"],
self.dataset["source"], self.dataset["x"], self.dataset["y"]):
if event_id not in tel_ids:
tel_ids[event_id] = list()
if event_id not in self.n_tels_by_event:
self.n_tels_by_event[event_id] = {}
if tel_type not in self.n_tels_by_event[event_id]:
self.n_tels_by_event[event_id][tel_type] = 1
else:
self.n_tels_by_event[event_id][tel_type] += 1
if tel_id in tel_ids[event_id]:
repeated_tel_ids.append(tel_id)
tel_ids[event_id].append(tel_id)
self.cameras_by_event[event_id].append(
(obs_id, tel_id, tel_type, folder, source, x, y))
@property
def event_uids(self) -> list:
return self.dataset["event_unique_id"].unique()
@property
def hdf5_files(self) -> np.ndarray:
return (self.dataset["folder"] + "/" + self.dataset["source"]).unique()
def get_event_hdf5_file(self, event_id: str, tel_id: str):
event_group = self.dataset.groupby("event_unique_id").get_group(
event_id)
tel_row: DataFrame = event_group.loc[event_group["telescope_id"] ==
tel_id]
tel_row = tel_row.loc[tel_row.index[0]]
return tel_row["folder"] + "/" + tel_row["source"]
@property
def camera_radius(self):
return get_camera_radius(self.dataset)
@property
def mc_values(self):
values = dict()
events = self.dataset[[
"event_unique_id", "alt", "az", "mc_energy", "core_x", "core_y"
]].drop_duplicates().set_index(["event_unique_id"])
for event_uid in self.event_uids:
event = events.loc[event_uid]
values[event_uid] = dict(alt=event["alt"],
az=event["az"],
mc_energy=event["mc_energy"],
core_x=event["core_x"],
core_y=event["core_y"])
return values
def reconstruct_event(self,
event_id: str,
event_cutflow: CutFlow,
obs_cutflow: CutFlow,
energy_regressor=None,
loose=False) -> Union[None, dict, Tuple[dict, dict]]:
run_array_direction = None
n_valid_tels = 0
hillas_containers = dict()
hillas_by_obs = dict()
time_gradients = dict()
leakages = dict()
types = dict()
n_islands = dict()
positions = dict()
meta = dict()
for obs_id, tel_id, tel_type, folder, source, x, y in self.cameras_by_event[
event_id]:
_, camera_name = split_tel_type(tel_type)
charge, peak = load_camera(source,
folder,
tel_type,
obs_id,
version=self.version)
params = get_observation_parameters(charge,
peak,
camera_name,
obs_cutflow,
cut=not loose)
if params is None:
continue
moments, leakage_c, _, time_gradient, obs_n_islands = params
# assert tel_id not in hillas_containers.keys()
hillas_containers[tel_id] = moments
assert (tel_type, obs_id) not in hillas_by_obs.keys()
meta[tel_id] = (tel_type, obs_id)
hillas_by_obs[(tel_type, obs_id)] = moments
positions[tel_id] = (x, y)
time_gradients[(tel_type, obs_id)] = time_gradient
leakages[(tel_type, obs_id)] = leakage_c
n_islands[(tel_type, obs_id)] = obs_n_islands
types[tel_id] = tel_type
hdf5_file = self.get_event_hdf5_file(event_id, tel_id)
telescope_alias = TELESCOPES_ALIAS[self.version][tel_type]
run_array_direction = self.array_directions[hdf5_file][
telescope_alias][tel_id]
n_valid_tels += 1
subarray = generate_subarray_description(self.dataset, event_id)
if event_cutflow.cut(CFE_MIN_TELS_RECO, len(hillas_containers)):
return
array_pointing = SkyCoord(az=run_array_direction[0] * u.rad,
alt=run_array_direction[1] * u.rad,
frame="altaz")
reco = self.reconstructor.predict(hillas_containers, subarray,
array_pointing)
mc = self.mc_values
if energy_regressor is None:
energy = None
else:
energy = energy_regressor.predict_event(positions, types,
hillas_containers, reco,
time_gradients, leakages,
n_islands, meta)
return dict(pred_az=2 * np.pi * u.rad + reco.az,
pred_alt=reco.alt,
pred_core_x=reco.core_x,
pred_core_y=reco.core_y,
h_max=reco.h_max,
alt=mc[event_id]["alt"] * u.rad,
az=mc[event_id]["az"] * u.rad,
core_x=mc[event_id]["core_x"],
core_y=mc[event_id]["core_y"],
mc_energy=mc[event_id]["mc_energy"],
energy=energy,
time_gradients=time_gradients,
leakages=leakages,
event_id=event_id,
n_islands=n_islands,
hillas=hillas_by_obs,
run_array_direction=run_array_direction)
def get_event_cams_and_foclens(self, event_id: str):
subarray = generate_subarray_description(self.dataset, event_id)
tel_types = subarray.telescope_types
return {
tel_types[i].camera.camera_name:
tel_types[i].optics.equivalent_focal_length.value
for i in range(len(tel_types))
}
def reconstruct_all(self,
max_events=None,
min_valid_observations=2,
energy_regressor=None,
npix_bounds: Tuple[float, float] = None,
charge_bounds: Tuple[float, float] = None,
ellipticity_bounds: Tuple[float, float] = None,
nominal_distance_bounds: Tuple[float, float] = None,
save_to: str = None,
save_hillas: str = None,
loose: bool = False) -> dict:
event_ids = self.event_uids
if max_events is not None and max_events < len(event_ids):
event_ids = event_ids[:max_events]
event_cutflow = generate_event_cutflow(min_valid_observations)
obs_cutflow = generate_observation_cutflow(self.camera_radius,
npix_bounds, charge_bounds,
ellipticity_bounds,
nominal_distance_bounds)
reconstructions = {}
for event_id in tqdm(event_ids):
reco = self.reconstruct_event(event_id,
event_cutflow,
obs_cutflow,
energy_regressor=energy_regressor,
loose=loose)
if reco is None:
continue
reconstructions[event_id] = reco
print()
print("==Observations Cutflow Summary==")
for cut in obs_cutflow.cuts.items():
print(cut[0], cut[1][1])
print()
print("==Event Cutflow Summary==")
for cut in event_cutflow.cuts.items():
print(cut[0], cut[1][1])
print()
perc_reconstructed = 100 * len(reconstructions) / len(event_ids)
print(
f"N. Events Reconstructed: {len(reconstructions)}/{len(event_ids)} ({perc_reconstructed}%)"
)
if save_to is not None:
self.save_predictions(reconstructions, save_to)
if save_hillas is not None:
self.save_hillas_params(reconstructions, save_hillas)
return reconstructions
def plot_metrics(self,
max_events: int = None,
min_valid_observations=2,
energy_regressor: EnergyModel = None,
plot_charges: bool = False,
save_to: str = None,
save_hillas: str = None,
save_plots: str = None):
import ctaplot
reco = self.reconstruct_all(
max_events,
min_valid_observations=min_valid_observations,
energy_regressor=energy_regressor)
preds = list(reco.values())
if save_plots is not None:
reco_alt = np.array(
[pred['pred_alt'] / (1 * u.rad) for pred in preds])
reco_az = np.array(
[pred['pred_az'] / (1 * u.rad) for pred in preds])
alt = np.array([pred['alt'] / (1 * u.rad) for pred in preds])
az = np.array([pred['az'] / (1 * u.rad) for pred in preds])
if energy_regressor is not None:
energy = np.array([pred['energy'] for pred in preds])
mc_energy = np.array([pred['mc_energy'] for pred in preds])
_, (ax1, ax2) = plt.subplots(1, 2)
ctaplot.plot_angular_resolution_per_energy(reco_alt,
reco_az,
alt,
az,
mc_energy,
ax=ax1)
if energy_regressor is not None:
ctaplot.plot_energy_resolution(mc_energy, energy, ax=ax2)
plt.savefig(save_plots)
@staticmethod
def plot_prediction_vs_real(pred, real, ax, title):
ax.scatter(real, pred)
ax.set_title(title)
ax.grid('on')
@staticmethod
def plot(results: DataFrame, save_to: str):
import os
from gerumo import plot_error_and_angular_resolution
results['true_alt'] = results['alt']
results['true_az'] = results['az']
results['true_mc_energy'] = results['mc_energy']
plot_error_and_angular_resolution(results,
save_to=os.path.join(
save_to,
"angular_resolution.png"),
ylim=[0, 2])
fig, axes = plt.subplots(1, 2)
ax1, ax2 = axes
Reconstructor.plot_prediction_vs_real(results['pred_alt'],
results['alt'], ax1, "Altitude")
Reconstructor.plot_prediction_vs_real(
results['pred_az'].apply(
lambda rad: np.arctan2(np.sin(rad), np.cos(rad))),
results['az'].apply(
lambda rad: np.arctan2(np.sin(rad), np.cos(rad))), ax2,
"Azimuth")
fig.savefig(os.path.join(save_to, "scatter.png"))
@staticmethod
def save_predictions(reco: dict, path: str):
reco = {
event_id: dict(pred_az=r["pred_az"].value,
pred_alt=r["pred_alt"].value,
pred_core_x=r["pred_core_x"].value,
pred_core_y=r["pred_core_y"].value,
alt=r["alt"].value,
az=r["az"].value,
core_x=r["core_x"],
core_y=r["core_y"],
mc_energy=r["mc_energy"],
energy=r["energy"],
h_max=r["h_max"].value)
for event_id, r in reco.items()
}
df = DataFrame.from_dict(reco, orient="index")
df.to_csv(path, sep=',', index_label="event_unique_id")
@staticmethod
def save_hillas_params(reco: dict, path: str):
columns = [
"event_unique_id", "observation_indice", "type", "intensity",
"kurtosis", "length", "phi", "psi", "r", "skewness", "width", "x",
"y", "wl", "time_gradient", "leakage_intensity_width_2",
"n_islands", "telescope_az", "telescope_alt"
]
params = list()
for event_id, r in reco.items():
run_array_direction = r["run_array_direction"]
for (tel_type, obs_id), moments in r["hillas"].items():
time_gradient = r["time_gradients"][(tel_type, obs_id)]
leakage_c = r["leakages"][(tel_type, obs_id)]
n_islands = r["n_islands"][(tel_type, obs_id)]
obs_params = (event_id, obs_id, tel_type, moments.intensity,
moments.kurtosis, moments.length.value,
moments.phi.value, moments.psi.value,
moments.r.value, moments.skewness,
moments.width.value, moments.x.value,
moments.y.value, moments.width.value /
moments.length.value, time_gradient,
leakage_c.intensity_width_2, n_islands,
run_array_direction[0], run_array_direction[1])
params.append(obs_params)
df = DataFrame(params, columns=columns).set_index(columns[:3])
df.to_csv(path, sep=',')