def test_estimator_results():
"""
creating some planes pointing in different directions (two
north-south, two east-west) and that have a slight position errors (+-
0.1 m in one of the four cardinal directions """
horizon_frame = HorizonFrame()
p1 = SkyCoord(alt=43 * u.deg, az=45 * u.deg, frame=horizon_frame)
p2 = SkyCoord(alt=47 * u.deg, az=45 * u.deg, frame=horizon_frame)
circle1 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, 1, 0] * u.m)
p1 = SkyCoord(alt=44 * u.deg, az=90 * u.deg, frame=horizon_frame)
p2 = SkyCoord(alt=46 * u.deg, az=90 * u.deg, frame=horizon_frame)
circle2 = HillasPlane(p1=p1, p2=p2, telescope_position=[1, 0, 0] * u.m)
p1 = SkyCoord(alt=44.5 * u.deg, az=45 * u.deg, frame=horizon_frame)
p2 = SkyCoord(alt=46.5 * u.deg, az=45 * u.deg, frame=horizon_frame)
circle3 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, -1, 0] * u.m)
p1 = SkyCoord(alt=43.5 * u.deg, az=90 * u.deg, frame=horizon_frame)
p2 = SkyCoord(alt=45.5 * u.deg, az=90 * u.deg, frame=horizon_frame)
circle4 = HillasPlane(p1=p1, p2=p2, telescope_position=[-1, 0, 0] * u.m)
# creating the fit class and setting the the great circle member
fit = HillasReconstructor()
fit.hillas_planes = {1: circle1, 2: circle2, 3: circle3, 4: circle4}
# performing the direction fit with the minimisation algorithm
# and a seed that is perpendicular to the up direction
dir_fit_minimise, _ = fit.estimate_direction()
print("direction fit test minimise:", dir_fit_minimise)
print()
def test_FitGammaHillas():
'''
a test of the complete fit procedure on one event including:
• tailcut cleaning
• hillas parametrisation
• GreatCircle creation
• direction fit
• position fit
in the end, proper units in the output are asserted '''
filename = get_dataset("gamma_test.simtel.gz")
fit = HillasReconstructor()
cam_geom = {}
tel_phi = {}
tel_theta = {}
source = hessio_event_source(filename)
for event in source:
hillas_dict = {}
for tel_id in event.dl0.tels_with_data:
if tel_id not in cam_geom:
cam_geom[tel_id] = CameraGeometry.guess(
event.inst.pixel_pos[tel_id][0],
event.inst.pixel_pos[tel_id][1],
event.inst.optical_foclen[tel_id])
tel_phi[tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
tel_theta[tel_id] = (np.pi/2-event.mc.tel[tel_id].altitude_raw)*u.rad
pmt_signal = event.r0.tel[tel_id].adc_sums[0]
mask = tailcuts_clean(cam_geom[tel_id], pmt_signal,
picture_thresh=10., boundary_thresh=5.)
pmt_signal[mask == 0] = 0
try:
moments = hillas_parameters(event.inst.pixel_pos[tel_id][0],
event.inst.pixel_pos[tel_id][1],
pmt_signal)
hillas_dict[tel_id] = moments
except HillasParameterizationError as e:
print(e)
continue
if len(hillas_dict) < 2: continue
fit_result = fit.predict(hillas_dict, event.inst, tel_phi, tel_theta)
print(fit_result)
fit_result.alt.to(u.deg)
fit_result.az.to(u.deg)
fit_result.core_x.to(u.m)
assert fit_result.is_valid
return
def test_estimator_results():
"""
creating some planes pointing in different directions (two
north-south, two east-west) and that have a slight position errors (+-
0.1 m in one of the four cardinal directions """
p1 = SkyCoord(alt=43 * u.deg, az=45 * u.deg, frame='altaz')
p2 = SkyCoord(alt=47 * u.deg, az=45 * u.deg, frame='altaz')
circle1 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, 1, 0] * u.m)
p1 = SkyCoord(alt=44 * u.deg, az=90 * u.deg, frame='altaz')
p2 = SkyCoord(alt=46 * u.deg, az=90 * u.deg, frame='altaz')
circle2 = HillasPlane(p1=p1, p2=p2, telescope_position=[1, 0, 0] * u.m)
p1 = SkyCoord(alt=44.5 * u.deg, az=45 * u.deg, frame='altaz')
p2 = SkyCoord(alt=46.5 * u.deg, az=45 * u.deg, frame='altaz')
circle3 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, -1, 0] * u.m)
p1 = SkyCoord(alt=43.5 * u.deg, az=90 * u.deg, frame='altaz')
p2 = SkyCoord(alt=45.5 * u.deg, az=90 * u.deg, frame='altaz')
circle4 = HillasPlane(p1=p1, p2=p2, telescope_position=[-1, 0, 0] * u.m)
# creating the fit class and setting the the great circle member
fit = HillasReconstructor()
fit.hillas_planes = {1: circle1, 2: circle2, 3: circle3, 4: circle4}
# performing the direction fit with the minimisation algorithm
# and a seed that is perpendicular to the up direction
dir_fit_minimise, _ = fit.estimate_direction()
print("direction fit test minimise:", dir_fit_minimise)
print()
def test_h_max_results():
"""
creating some planes pointing in different directions (two
north-south, two east-west) and that have a slight position errors (+-
0.1 m in one of the four cardinal directions """
p1 = SkyCoord(alt=0 * u.deg, az=45 * u.deg, frame='altaz')
p2 = SkyCoord(alt=0 * u.deg, az=45 * u.deg, frame='altaz')
circle1 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, 1, 0] * u.m)
p1 = SkyCoord(alt=0 * u.deg, az=90 * u.deg, frame='altaz')
p2 = SkyCoord(alt=0 * u.deg, az=90 * u.deg, frame='altaz')
circle2 = HillasPlane(p1=p1, p2=p2, telescope_position=[1, 0, 0] * u.m)
p1 = SkyCoord(alt=0 * u.deg, az=45 * u.deg, frame='altaz')
p2 = SkyCoord(alt=0 * u.deg, az=45 * u.deg, frame='altaz')
circle3 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, -1, 0] * u.m)
p1 = SkyCoord(alt=0 * u.deg, az=90 * u.deg, frame='altaz')
p2 = SkyCoord(alt=0 * u.deg, az=90 * u.deg, frame='altaz')
circle4 = HillasPlane(p1=p1, p2=p2, telescope_position=[-1, 0, 0] * u.m)
# creating the fit class and setting the the great circle member
fit = HillasReconstructor()
fit.hillas_planes = {1: circle1, 2: circle2, 3: circle3, 4: circle4}
# performing the direction fit with the minimisation algorithm
# and a seed that is perpendicular to the up direction
h_max_reco = fit.estimate_h_max()
print("h max fit test minimise:", h_max_reco)
# the results should be close to the direction straight up
np.testing.assert_allclose(h_max_reco, 0, atol=1e-8)
def test_h_max_results():
"""
creating some planes pointing in different directions (two
north-south, two east-west) and that have a slight position errors (+-
0.1 m in one of the four cardinal directions """
horizon_frame = HorizonFrame()
p1 = SkyCoord(alt=0 * u.deg, az=45 * u.deg, frame=horizon_frame)
p2 = SkyCoord(alt=0 * u.deg, az=45 * u.deg, frame=horizon_frame)
circle1 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, 1, 0] * u.m)
p1 = SkyCoord(alt=0 * u.deg, az=90 * u.deg, frame=horizon_frame)
p2 = SkyCoord(alt=0 * u.deg, az=90 * u.deg, frame=horizon_frame)
circle2 = HillasPlane(p1=p1, p2=p2, telescope_position=[1, 0, 0] * u.m)
p1 = SkyCoord(alt=0 * u.deg, az=45 * u.deg, frame=horizon_frame)
p2 = SkyCoord(alt=0 * u.deg, az=45 * u.deg, frame=horizon_frame)
circle3 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, -1, 0] * u.m)
p1 = SkyCoord(alt=0 * u.deg, az=90 * u.deg, frame=horizon_frame)
p2 = SkyCoord(alt=0 * u.deg, az=90 * u.deg, frame=horizon_frame)
circle4 = HillasPlane(p1=p1, p2=p2, telescope_position=[-1, 0, 0] * u.m)
# creating the fit class and setting the the great circle member
fit = HillasReconstructor()
fit.hillas_planes = {1: circle1, 2: circle2, 3: circle3, 4: circle4}
# performing the direction fit with the minimisation algorithm
# and a seed that is perpendicular to the up direction
h_max_reco = fit.estimate_h_max()
print("h max fit test minimise:", h_max_reco)
# the results should be close to the direction straight up
np.testing.assert_allclose(h_max_reco.value, 0, atol=1e-8)
def test_reconstruction():
"""
a test of the complete fit procedure on one event including:
• tailcut cleaning
• hillas parametrisation
• HillasPlane creation
• direction fit
• position fit
in the end, proper units in the output are asserted """
filename = get_dataset_path("gamma_test.simtel.gz")
fit = HillasReconstructor()
tel_azimuth = {}
tel_altitude = {}
source = EventSourceFactory.produce(
input_url=filename,
product='HESSIOEventSource',
)
for event in source:
hillas_dict = {}
for tel_id in event.dl0.tels_with_data:
geom = event.inst.subarray.tel[tel_id].camera
tel_azimuth[tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
tel_altitude[tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad
pmt_signal = event.r0.tel[tel_id].image[0]
mask = tailcuts_clean(geom,
pmt_signal,
picture_thresh=10.,
boundary_thresh=5.)
pmt_signal[mask == 0] = 0
try:
moments = hillas_parameters(geom, pmt_signal)
hillas_dict[tel_id] = moments
except HillasParameterizationError as e:
print(e)
continue
if len(hillas_dict) < 2:
continue
fit_result = fit.predict(hillas_dict, event.inst, tel_azimuth,
tel_altitude)
print(fit_result)
fit_result.alt.to(u.deg)
fit_result.az.to(u.deg)
fit_result.core_x.to(u.m)
assert fit_result.is_valid
return
def __init__(self,
event,
telescope_list,
camera_types,
ChargeExtration,
pe_thresh,
min_neighbors,
tail_thresholds,
DirReco,
quality_cuts,
LUT=None):
super().__init__()
'''
Parmeters
---------
event : ctapipe event container
calibrator : ctapipe camera calibrator
reconstructor : ctapipe hillas reconstructor
telescope_list : list with telescope configuration or "all"
pe_thresh : dict with thresholds for gain selection
tail_thresholds : dict with thresholds for image cleaning
quality_cuts : dict containing quality cuts
canera_types : list with camera types to analyze
'''
self.event = event
self.telescope_list = telescope_list
self.pe_thresh = pe_thresh
self.min_neighbors = min_neighbors
self.tail_thresholds = tail_thresholds
self.quality_cuts = quality_cuts
self.camera_types = camera_types
self.dirreco = DirReco
self.LUTgenerator = LUT
if (self.dirreco["weights"] == "LUT") | (self.dirreco["weights"]
== "doublepass"):
self.weights = {}
else:
self.weights = None
self.hillas_dict = {}
self.camera_dict = {}
self.edge_pixels = {}
# configurations for calibrator
cfg = Config()
cfg["ChargeExtractorFactory"]["product"] = \
ChargeExtration["ChargeExtractorProduct"]
cfg['WaveformCleanerFactory']['product'] = \
ChargeExtration["WaveformCleanerProduct"]
self.calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator",
config=cfg) # calibration
self.reconstructor = HillasReconstructor() # direction
def test_FitGammaHillas():
'''
a test of the complete fit procedure on one event including:
• tailcut cleaning
• hillas parametrisation
• GreatCircle creation
• direction fit
• position fit
in the end, proper units in the output are asserted '''
filename = get_dataset("gamma_test.simtel.gz")
fit = HillasReconstructor()
tel_phi = {}
tel_theta = {}
source = hessio_event_source(filename)
for event in source:
hillas_dict = {}
for tel_id in event.dl0.tels_with_data:
geom = event.inst.subarray.tel[tel_id].camera
tel_phi[tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
tel_theta[tel_id] = (np.pi / 2 -
event.mc.tel[tel_id].altitude_raw) * u.rad
pmt_signal = event.r0.tel[tel_id].image[0]
mask = tailcuts_clean(geom,
pmt_signal,
picture_thresh=10.,
boundary_thresh=5.)
pmt_signal[mask == 0] = 0
try:
moments = hillas_parameters(geom, pmt_signal)
hillas_dict[tel_id] = moments
except HillasParameterizationError as e:
print(e)
continue
if len(hillas_dict) < 2: continue
fit_result = fit.predict(hillas_dict, event.inst, tel_phi, tel_theta)
print(fit_result)
fit_result.alt.to(u.deg)
fit_result.az.to(u.deg)
fit_result.core_x.to(u.m)
assert fit_result.is_valid
return
def test_reconstruction():
"""
a test of the complete fit procedure on one event including:
• tailcut cleaning
• hillas parametrisation
• HillasPlane creation
• direction fit
• position fit
in the end, proper units in the output are asserted """
filename = get_dataset_path("gamma_test.simtel.gz")
fit = HillasReconstructor()
tel_azimuth = {}
tel_altitude = {}
source = EventSourceFactory.produce(input_url=filename)
for event in source:
hillas_dict = {}
for tel_id in event.dl0.tels_with_data:
geom = event.inst.subarray.tel[tel_id].camera
tel_azimuth[tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
tel_altitude[tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad
pmt_signal = event.r0.tel[tel_id].image[0]
mask = tailcuts_clean(geom, pmt_signal,
picture_thresh=10., boundary_thresh=5.)
pmt_signal[mask == 0] = 0
try:
moments = hillas_parameters(geom, pmt_signal)
hillas_dict[tel_id] = moments
except HillasParameterizationError as e:
print(e)
continue
if len(hillas_dict) < 2:
continue
fit_result = fit.predict(hillas_dict, event.inst, tel_azimuth, tel_altitude)
print(fit_result)
fit_result.alt.to(u.deg)
fit_result.az.to(u.deg)
fit_result.core_x.to(u.m)
assert fit_result.is_valid
def test_fit_core():
'''
creating some great circles pointing in different directions (two
north-south,
two east-west) and that have a slight position errors (+- 0.1 m in one of
the four
cardinal directions '''
circle1 = GreatCircle([[1, 0, 0], [0, 0, 1]])
circle1.pos = [0, 0.1] * u.m
circle1.trace = [1, 0, 0]
circle2 = GreatCircle([[0, 1, 0], [0, 0, 1]])
circle2.pos = [0.1, 0] * u.m
circle2.trace = [0, 1, 0]
circle3 = GreatCircle([[1, 0, 0], [0, 0, 1]])
circle3.pos = [0, -.1] * u.m
circle3.trace = [1, 0, 0]
circle4 = GreatCircle([[0, 1, 0], [0, 0, 1]])
circle4.pos = [-.1, 0] * u.m
circle4.trace = [0, 1, 0]
# creating the fit class and setting the the great circle member
fit = HillasReconstructor()
fit.circles = {1: circle1, 2: circle2, 3: circle3, 4: circle4}
# performing the position fit with the minimisation algorithm
# and a seed that is quite far away
pos_fit_minimise = fit.fit_core_minimise([100, 1000] * u.m)
print("position fit test minimise:", pos_fit_minimise)
print()
# performing the position fit with the geometric algorithm
pos_fit_crosses, err_est_pos_fit_crosses = fit.fit_core_crosses()
print("position fit test crosses:", pos_fit_crosses)
print("error estimate:", err_est_pos_fit_crosses)
print()
# the results should be close to the origin of the coordinate system
np.testing.assert_allclose(pos_fit_minimise / u.m, [0, 0], atol=1e-3)
np.testing.assert_allclose(pos_fit_crosses / u.m, [0, 0], atol=1e-3)
def test_fit_core():
'''
creating some great circles pointing in different directions (two
north-south,
two east-west) and that have a slight position errors (+- 0.1 m in one of
the four
cardinal directions '''
circle1 = GreatCircle([[1, 0, 0], [0, 0, 1]])
circle1.pos = [0, 0.1] * u.m
circle1.trace = [1, 0, 0]
circle2 = GreatCircle([[0, 1, 0], [0, 0, 1]])
circle2.pos = [0.1, 0] * u.m
circle2.trace = [0, 1, 0]
circle3 = GreatCircle([[1, 0, 0], [0, 0, 1]])
circle3.pos = [0, -.1] * u.m
circle3.trace = [1, 0, 0]
circle4 = GreatCircle([[0, 1, 0], [0, 0, 1]])
circle4.pos = [-.1, 0] * u.m
circle4.trace = [0, 1, 0]
# creating the fit class and setting the the great circle member
fit = HillasReconstructor()
fit.circles = {1: circle1, 2: circle2, 3: circle3, 4: circle4}
# performing the position fit with the minimisation algorithm
# and a seed that is quite far away
pos_fit_minimise = fit.fit_core_minimise([100, 1000] * u.m)
print("position fit test minimise:", pos_fit_minimise)
print()
# performing the position fit with the geometric algorithm
pos_fit_crosses, err_est_pos_fit_crosses = fit.fit_core_crosses()
print("position fit test crosses:", pos_fit_crosses)
print("error estimate:", err_est_pos_fit_crosses)
print()
# the results should be close to the origin of the coordinate system
np.testing.assert_allclose(pos_fit_minimise / u.m, [0, 0], atol=1e-3)
np.testing.assert_allclose(pos_fit_crosses / u.m, [0, 0], atol=1e-3)
def test_fit_origin():
'''
creating some great circles pointing in different directions (two
north-south,
two east-west) and that have a slight position errors (+- 0.1 m in one of
the four
cardinal directions '''
circle1 = GreatCircle([[1, 0, 0], [0, 0, 1]])
circle1.pos = [0, 0.1] * u.m
circle1.trace = [1, 0, 0]
circle2 = GreatCircle([[0, 1, 0], [0, 0, 1]])
circle2.pos = [0.1, 0] * u.m
circle2.trace = [0, 1, 0]
circle3 = GreatCircle([[1, 0, 0], [0, 0, 1]])
circle3.pos = [0, -.1] * u.m
circle3.trace = [1, 0, 0]
circle4 = GreatCircle([[0, 1, 0], [0, 0, 1]])
circle4.pos = [-.1, 0] * u.m
circle4.trace = [0, 1, 0]
# creating the fit class and setting the the great circle member
fit = HillasReconstructor()
fit.circles = {1: circle1, 2: circle2, 3: circle3, 4: circle4}
# performing the direction fit with the minimisation algorithm
# and a seed that is perpendicular to the up direction
dir_fit_minimise = fit.fit_origin_minimise((0.1, 0.1, 1))
print("direction fit test minimise:", dir_fit_minimise)
print()
# performing the direction fit with the geometric algorithm
dir_fit_crosses = fit.fit_origin_crosses()[0]
print("direction fit test crosses:", dir_fit_crosses)
print()
# the results should be close to the direction straight up
# np.testing.assert_allclose(dir_fit_minimise, [0, 0, 1], atol=1e-1)
np.testing.assert_allclose(dir_fit_crosses, [0, 0, 1], atol=1e-3)
def test_fit_origin():
'''
creating some great circles pointing in different directions (two
north-south,
two east-west) and that have a slight position errors (+- 0.1 m in one of
the four
cardinal directions '''
circle1 = GreatCircle([[1, 0, 0], [0, 0, 1]])
circle1.pos = [0, 0.1] * u.m
circle1.trace = [1, 0, 0]
circle2 = GreatCircle([[0, 1, 0], [0, 0, 1]])
circle2.pos = [0.1, 0] * u.m
circle2.trace = [0, 1, 0]
circle3 = GreatCircle([[1, 0, 0], [0, 0, 1]])
circle3.pos = [0, -.1] * u.m
circle3.trace = [1, 0, 0]
circle4 = GreatCircle([[0, 1, 0], [0, 0, 1]])
circle4.pos = [-.1, 0] * u.m
circle4.trace = [0, 1, 0]
# creating the fit class and setting the the great circle member
fit = HillasReconstructor()
fit.circles = {1: circle1, 2: circle2, 3: circle3, 4: circle4}
# performing the direction fit with the minimisation algorithm
# and a seed that is perpendicular to the up direction
dir_fit_minimise = fit.fit_origin_minimise((0.1, 0.1, 1))
print("direction fit test minimise:", dir_fit_minimise)
print()
# performing the direction fit with the geometric algorithm
dir_fit_crosses = fit.fit_origin_crosses()[0]
print("direction fit test crosses:", dir_fit_crosses)
print()
# the results should be close to the direction straight up
# np.testing.assert_allclose(dir_fit_minimise, [0, 0, 1], atol=1e-1)
np.testing.assert_allclose(dir_fit_crosses, [0, 0, 1], atol=1e-3)
class EventPreparer:
"""
Class which loop on events and returns results stored in container
The Class has several purposes. First of all, it prepares the images of the event
that will be further use for reconstruction by applying calibration, cleaning and
selection. Then, it reconstructs the geometry of the event and then returns
image (e.g. Hillas parameters)and event information (e.g. reults od the
reconstruction).
Parameters
----------
config: dict
Configuration with analysis parameters
mode: str
Mode of the reconstruction, e.g. tail or wave
event_cutflow: ctapipe.utils.CutFlow
Statistic of events processed
image_cutflow: ctapipe.utils.CutFlow
Statistic of images processed
Returns: dict
Dictionnary of results
"""
def __init__(self, config, mode, event_cutflow=None, image_cutflow=None):
# Cleaning for reconstruction
self.cleaner_reco = ImageCleaner( # for reconstruction
config=config["ImageCleaning"]["biggest"],
mode=mode)
# Cleaning for energy/score estimation
# Add possibility to force energy/score cleaning with tailcut analysis
force_mode = mode
try:
if config["General"]["force_tailcut_for_extended_cleaning"] is True:
force_mode = config["General"]["force_mode"]
print("> Activate force-mode for cleaning!!!!")
except:
pass # force_mode = mode
self.cleaner_extended = ImageCleaner( # for energy/score estimation
config=config["ImageCleaning"]["extended"],
mode=force_mode)
# Image book keeping
self.image_cutflow = image_cutflow or CutFlow("ImageCutFlow")
# Add quality cuts on images
charge_bounds = config["ImageSelection"]["charge"]
npix_bounds = config["ImageSelection"]["pixel"]
ellipticity_bounds = config["ImageSelection"]["ellipticity"]
nominal_distance_bounds = config["ImageSelection"]["nominal_distance"]
self.camera_radius = {
"LSTCam": 1.126,
"NectarCam": 1.126,
} # Average between max(xpix) and max(ypix), in meter
self.image_cutflow.set_cuts(
OrderedDict([
("noCuts", None),
("min pixel", lambda s: np.count_nonzero(s) < npix_bounds[0]),
("min charge", lambda x: x < charge_bounds[0]),
# ("poor moments", lambda m: m.width <= 0 or m.length <= 0 or np.isnan(m.width) or np.isnan(m.length)), # TBC, maybe we loose events without nan conditions
("poor moments", lambda m: m.width <= 0 or m.length <= 0),
(
"bad ellipticity",
lambda m: (m.width / m.length) < ellipticity_bounds[0] or
(m.width / m.length) > ellipticity_bounds[-1],
),
# ("close to the edge", lambda m, cam_id: m.r.value > (nominal_distance_bounds[-1] * 1.12949101073069946)) # in meter
(
"close to the edge",
lambda m, cam_id: m.r.value >
(nominal_distance_bounds[-1] * self.camera_radius[cam_id]),
), # in meter
]))
# Reconstruction
self.shower_reco = HillasReconstructor()
# Event book keeping
self.event_cutflow = event_cutflow or CutFlow("EventCutFlow")
# Add cuts on events
min_ntel = config["Reconstruction"]["min_tel"]
self.event_cutflow.set_cuts(
OrderedDict([
("noCuts", None),
("min2Tels trig", lambda x: x < min_ntel),
("min2Tels reco", lambda x: x < min_ntel),
("direction nan", lambda x: x.is_valid == False),
]))
def prepare_event(self, source, return_stub=False):
# configuration for the camera calibrator
# modifies the integration window to be more like in MARS
# JLK, only for LST!!!!
# Option for integration correction is done above
cfg = Config()
cfg["ChargeExtractorFactory"]["window_width"] = 5
cfg["ChargeExtractorFactory"]["window_shift"] = 2
self.calib = CameraCalibrator(
config=cfg,
extractor_product="LocalPeakIntegrator",
eventsource=source,
tool=None,
)
for event in source:
self.event_cutflow.count("noCuts")
if self.event_cutflow.cut("min2Tels trig",
len(event.dl0.tels_with_data)):
if return_stub:
yield stub(event)
else:
continue
# calibrate the event
self.calib.calibrate(event)
# telescope loop
tot_signal = 0
max_signals = {}
n_pixel_dict = {}
hillas_dict_reco = {} # for geometry
hillas_dict = {} # for discrimination
n_tels = {
"tot": len(event.dl0.tels_with_data),
"LST": 0,
"MST": 0,
"SST": 0,
}
n_cluster_dict = {}
impact_dict_reco = {} # impact distance measured in tilt system
point_azimuth_dict = {}
point_altitude_dict = {}
# To compute impact parameter in tilt system
run_array_direction = event.mcheader.run_array_direction
az, alt = run_array_direction[0], run_array_direction[1]
ground_frame = GroundFrame()
for tel_id in event.dl0.tels_with_data:
self.image_cutflow.count("noCuts")
camera = event.inst.subarray.tel[tel_id].camera
# count the current telescope according to its size
tel_type = event.inst.subarray.tel[tel_id].optics.tel_type
# JLK, N telescopes before cut selection are not really interesting for
# discrimination, too much fluctuations
# n_tels[tel_type] += 1
# the camera image as a 1D array and stuff needed for calibration
# Choose gain according to pywicta's procedure
image_1d = simtel_event_to_images(event=event,
tel_id=tel_id,
ctapipe_format=True)
pmt_signal = image_1d.input_image # calibrated image
# clean the image
try:
with warnings.catch_warnings():
# Image with biggest cluster (reco cleaning)
image_biggest = self.cleaner_reco.clean_image(
pmt_signal, camera)
image_biggest2d = geometry_converter.image_1d_to_2d(
image_biggest, camera.cam_id)
image_biggest2d = filter_pixels_clusters(
image_biggest2d)
image_biggest = geometry_converter.image_2d_to_1d(
image_biggest2d, camera.cam_id)
# Image for score/energy estimation (with clusters)
image_extended = self.cleaner_extended.clean_image(
pmt_signal, camera)
except FileNotFoundError as e: # JLK, WHAT?
print(e)
continue
# Apply some selection
if self.image_cutflow.cut("min pixel", image_biggest):
continue
if self.image_cutflow.cut("min charge", np.sum(image_biggest)):
continue
# For cluster counts
image_2d = geometry_converter.image_1d_to_2d(
image_extended, camera.cam_id)
n_cluster_dict[
tel_id] = pixel_clusters.number_of_pixels_clusters(
array=image_2d, threshold=0)
# could this go into `hillas_parameters` ...?
max_signals[tel_id] = np.max(image_extended)
# do the hillas reconstruction of the images
# QUESTION should this change in numpy behaviour be done here
# or within `hillas_parameters` itself?
# JLK: make selection on biggest cluster
with np.errstate(invalid="raise", divide="raise"):
try:
moments_reco = hillas_parameters(
camera,
image_biggest) # for geometry (eg direction)
moments = hillas_parameters(
camera, image_extended
) # for discrimination and energy reconstruction
# if width and/or length are zero (e.g. when there is only only one
# pixel or when all pixel are exactly in one row), the
# parametrisation won't be very useful: skip
if self.image_cutflow.cut("poor moments",
moments_reco):
# print('poor moments')
continue
if self.image_cutflow.cut("close to the edge",
moments_reco, camera.cam_id):
# print('close to the edge')
continue
if self.image_cutflow.cut("bad ellipticity",
moments_reco):
# print('bad ellipticity: w={}, l={}'.format(moments_reco.width, moments_reco.length))
continue
except (FloatingPointError,
hillas.HillasParameterizationError):
continue
point_azimuth_dict[
tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
point_altitude_dict[
tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad
n_tels[tel_type] += 1
hillas_dict[tel_id] = moments
hillas_dict_reco[tel_id] = moments_reco
n_pixel_dict[tel_id] = len(np.where(image_extended > 0)[0])
tot_signal += moments.intensity
n_tels["reco"] = len(hillas_dict_reco)
n_tels["discri"] = len(hillas_dict)
if self.event_cutflow.cut("min2Tels reco", n_tels["reco"]):
if return_stub:
yield stub(event)
else:
continue
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# Reconstruction results
reco_result = self.shower_reco.predict(
hillas_dict_reco,
event.inst,
point_altitude_dict,
point_azimuth_dict,
)
# shower_sys = TiltedGroundFrame(pointing_direction=HorizonFrame(
# az=reco_result.az,
# alt=reco_result.alt
# ))
# Impact parameter for energy estimation (/ tel)
subarray = event.inst.subarray
for tel_id in hillas_dict.keys():
pos = subarray.positions[tel_id]
tel_ground = SkyCoord(pos[0],
pos[1],
pos[2],
frame=ground_frame)
# tel_tilt = tel_ground.transform_to(shower_sys)
core_ground = SkyCoord(
reco_result.core_x,
reco_result.core_y,
0 * u.m,
frame=ground_frame,
)
# core_tilt = core_ground.transform_to(shower_sys)
# Should be better handled (tilted frame)
impact_dict_reco[tel_id] = np.sqrt(
(core_ground.x - tel_ground.x)**2 +
(core_ground.y - tel_ground.y)**2)
except Exception as e:
print("exception in reconstruction:", e)
raise
if return_stub:
yield stub(event)
else:
continue
if self.event_cutflow.cut("direction nan", reco_result):
if return_stub:
yield stub(event)
else:
continue
yield PreparedEvent(
event=event,
n_pixel_dict=n_pixel_dict,
hillas_dict=hillas_dict,
hillas_dict_reco=hillas_dict_reco,
n_tels=n_tels,
tot_signal=tot_signal,
max_signals=max_signals,
n_cluster_dict=n_cluster_dict,
reco_result=reco_result,
impact_dict=impact_dict_reco,
)
class PrepareList(Cutter):
'''
Prepare a feature list to save to table. It takes an event, does the
calibration, image cleaning, parametrization and reconstruction.
From this some basic features will be extracted and written to the file
which later on can be used for training of the classifiers or energy
regressors.
test
'''
true_az = {}
true_alt = {}
max_signal = {}
tot_signal = 0
impact = {}
def __init__(self,
event,
telescope_list,
camera_types,
ChargeExtration,
pe_thresh,
min_neighbors,
tail_thresholds,
DirReco,
quality_cuts,
LUT=None):
super().__init__()
'''
Parmeters
---------
event : ctapipe event container
calibrator : ctapipe camera calibrator
reconstructor : ctapipe hillas reconstructor
telescope_list : list with telescope configuration or "all"
pe_thresh : dict with thresholds for gain selection
tail_thresholds : dict with thresholds for image cleaning
quality_cuts : dict containing quality cuts
canera_types : list with camera types to analyze
'''
self.event = event
self.telescope_list = telescope_list
self.pe_thresh = pe_thresh
self.min_neighbors = min_neighbors
self.tail_thresholds = tail_thresholds
self.quality_cuts = quality_cuts
self.camera_types = camera_types
self.dirreco = DirReco
self.LUTgenerator = LUT
if (self.dirreco["weights"] == "LUT") | (self.dirreco["weights"]
== "doublepass"):
self.weights = {}
else:
self.weights = None
self.hillas_dict = {}
self.camera_dict = {}
self.edge_pixels = {}
# configurations for calibrator
cfg = Config()
cfg["ChargeExtractorFactory"]["product"] = \
ChargeExtration["ChargeExtractorProduct"]
cfg['WaveformCleanerFactory']['product'] = \
ChargeExtration["WaveformCleanerProduct"]
self.calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator",
config=cfg) # calibration
self.reconstructor = HillasReconstructor() # direction
def get_impact(self, hillas_dict):
'''
calculate impact parameters for all telescopes that were
used for parametrization.
Paremeters
----------
hillas_dict : dict with hillas HillasParameterContainers
Returnes
--------
impact : impact parameter or NaN if calculation failed
'''
# check if event was prepared before
try:
assert self.reco_result
except AssertionError:
self.prepare()
impact = {}
for tel_id in hillas_dict.keys():
try:
pred_core = np.array([
self.reco_result.core_x.value,
self.reco_result.core_y.value
]) * u.m
# tel_coords start at 0 instead of 1...
tel_position = np.array([
self.event.inst.subarray.tel_coords[tel_id - 1].x.value,
self.event.inst.subarray.tel_coords[tel_id - 1].y.value
]) * u.m
impact[tel_id] = linalg.length(pred_core - tel_position)
except AttributeError:
impact[tel_id] = np.nan
return impact
def get_offangle(self, tel_id, direction="reco"):
'''
Get the angular offset between the reconstructed direction and the
pointing direction of the telescope.
Parameters
----------
tel_id : integer
Telecope ID
true_off : string
if "mc", the true MC direction is taken for calculation. Otherwise,
if "reco" the reconstructed value will be taken
Returns
-------
off_angles : dictionary
dictionary with tel_ids as keys and the offangle as entries.
'''
if direction == "reco":
off_angle = angular_separation(
self.event.mc.tel[tel_id].azimuth_raw * u.rad,
self.event.mc.tel[tel_id].altitude_raw * u.rad,
self.reco_result.az, self.reco_result.alt)
elif direction == "mc":
off_angle = angular_separation(
self.event.mc.tel[tel_id].azimuth_raw * u.rad,
self.event.mc.tel[tel_id].altitude_raw * u.rad,
self.event.mc.az, self.event.mc.alt)
return off_angle
def get_weight(self, method, camera, tel_id, hillas_par, offangle=None):
"""
Get the weighting for HillasReconustructor. Possible methods are
'default', which will fall back to the standard weighting applied
in capipe, 'LUT' which will take the weights from a LUT and
'doublepass' which might be used for diffuse simulations. In this
case it returns the weights for the first pass.
method : sting
method to get the weighting.
camera: CameraDescription
tel_id: integer
hillas_par: HillasParameterContainer
"""
if method == "default":
pass
elif method == "LUT":
if np.isnan(hillas_par.width) & (not np.isnan(hillas_par.length)):
hillas_par.width = 0 * u.m
self.weights[tel_id] = self.LUTgenerator.get_weight_from_LUT(
hillas_par,
camera.cam_id,
min_stat=self.dirreco["min_stat"],
ratio_cut=self.dirreco["wl_ratio_cut"][camera.cam_id])
elif method == "doublepass":
# first pass with default weighting
if np.isnan(hillas_par.width) & (not np.isnan(hillas_par.length)):
hillas_par.width = 0 * u.m
self.weights[tel_id] = hillas_par.intensity * (
1 * u.m + hillas_par.length) / (1 * u.m + hillas_par.width)
elif method == "second_pass":
# weights for second pass
self.weights[
tel_id] = self.LUTgenerator.get_weight_from_diffuse_LUT(
self.hillas_dict[tel_id],
offangle,
camera.cam_id,
min_stat=self.dirreco["min_stat"],
ratio_cut=self.dirreco["wl_ratio_cut"][camera.cam_id])
else:
raise KeyError("Weighting method {} not known.".format(method))
def prepare(self):
'''
Prepare event performimng calibration, image cleaning,
hillas parametrization, hillas intersection for the single
event. Additionally, the impact distance will be calculated.
'''
tels_per_type = {}
no_weight = []
# calibrate event
self.calibrator.calibrate(self.event)
# loop over all telescopeswith data in it
for tel_id in self.event.r0.tels_with_data:
# check if telescope is selected for analysis
# This also could be done already in event_source when reading th data
if (tel_id in self.telescope_list) | (self.telescope_list
== "all"):
pass
else:
continue
# get camera information
camera = self.event.inst.subarray.tel[tel_id].camera
self.camera_dict[tel_id] = camera
image = self.event.dl1.tel[tel_id].image
if camera.cam_id in self.pe_thresh.keys():
image, select = pick_gain_channel(
image, self.pe_thresh[camera.cam_id], True)
else:
image = np.squeeze(image)
# image cleaning
mask = tailcuts_clean(
camera,
image,
picture_thresh=self.tail_thresholds[camera.cam_id][1],
boundary_thresh=self.tail_thresholds[camera.cam_id][0],
min_number_picture_neighbors=self.min_neighbors)
# go to next telescope if no pixels survived cleaning
if not any(mask):
continue
cleaned_image = np.copy(image)
cleaned_image[~mask] = 0
# calculate the hillas parameters
hillas_par = hillas_parameters(camera, cleaned_image)
# quality cuts
leakage = leakage = self.leakage_cut(
camera=camera,
hillas_parameters=hillas_par,
radius=self.quality_cuts["leakage_cut"]["radius"],
max_dist=self.quality_cuts["leakage_cut"]["dist"],
image=cleaned_image,
rows=self.quality_cuts["leakage_cut"]["rows"],
fraction=self.quality_cuts["leakage_cut"]["frac"],
method=self.quality_cuts["leakage_cut"]["method"],
)
size = self.size_cut(hillas_par, self.quality_cuts["size"])
if not (leakage & size):
# size or leakage cuts not passed
continue
# get the weighting for HillasReconstructor
try:
self.get_weight(self.dirreco["weights"], camera, tel_id,
hillas_par)
except LookupFailedError:
# this telescope will be ignored, should only happen for method LUT here
no_weight.append(tel_id)
continue
self.hillas_dict[tel_id] = hillas_par
self.max_signal[tel_id] = np.max(cleaned_image) # brightest pix
try:
tels_per_type[camera.cam_id].append(tel_id)
except KeyError:
tels_per_type[camera.cam_id] = [tel_id]
try:
assert tels_per_type
except AssertionError:
raise TooFewTelescopesException("No image survived the leakage "
"or size cuts.")
# collect some additional information
for tel_id in self.hillas_dict:
self.tot_signal += self.hillas_dict[tel_id].intensity # total size
self.true_az[
tel_id] = self.event.mc.tel[tel_id].azimuth_raw * u.rad
self.true_alt[
tel_id] = self.event.mc.tel[tel_id].altitude_raw * u.rad
# wil raise exception if cut was not passed
# self.multiplicity_cut(self.quality_cuts["multiplicity"]["cuts"],
# tels_per_type, method=self.quality_cuts["multiplicity"]["method"])
if self.dirreco["weights"] == "LUT":
# remove telescopes withough weights
print("Removed {} of {} telescopes due LUT problems".format(
len(no_weight),
len(self.hillas_dict) + len(no_weight)))
# do Hillas reconstruction
self.reco_result = self.reconstructor.predict(self.hillas_dict,
self.event.inst,
self.true_alt,
self.true_az,
ext_weight=self.weights)
if self.dirreco["weights"] == "doublepass":
# take the reconstructed direction to get an estimate of the offangle and
# get weights from the second pass from the diffuse LUT.
self.weights = {} # reset the weights from earlier
no_weight = []
for tel_id in self.hillas_dict:
predicted_offangle = self.get_offangle(tel_id,
direction="reco")
predicted_offangle = predicted_offangle.to(u.deg).value
camera = self.camera_dict[tel_id] # reload camera_information
# get the weighting for HillasReconstructor
try:
self.get_weight("second_pass", camera, tel_id,
self.hillas_dict[tel_id],
predicted_offangle)
except LookupFailedError:
no_weight.append(tel_id)
print("Removed {} of {} telescopes due LUT problems".format(
len(no_weight), len(self.hillas_dict)))
# remove those types from tels_per_type
for tel_id in no_weight:
del self.hillas_dict[tel_id]
for cam_id in tels_per_type:
if tel_id in tels_per_type[cam_id]:
index = np.where(
np.array(tels_per_type[cam_id]) == tel_id)
tels_per_type[cam_id].pop(int(
index[0])) # remove from list
# redo the multiplicity cut to check if it still fulfilled
# self.multiplicity_cut(self.quality_cuts["multiplicity"]["cuts"], tels_per_type,
# method=self.quality_cuts["multiplicity"]["method"])
# do the second pass with new weights
self.reco_result = self.reconstructor.predict(
self.hillas_dict,
self.event.inst,
self.true_alt,
self.true_az,
ext_weight=self.weights)
# Number of telescopes triggered per type
self.n_tels_per_type = {
tel: len(tels_per_type[tel])
for tel in tels_per_type
}
for tel_id in self.hillas_dict:
try:
agree = self.mc_offset == self.get_offangle(tel_id,
direction="mc")
except AttributeError:
self.mc_offset = self.get_offangle(tel_id, direction="mc")
continue
if not agree:
raise ValueError(
"The pointing of the telescopes seems to be different.")
self.impact = self.get_impact(self.hillas_dict) # impact parameter
def get_reconstructed_parameters(self):
'''
Return the parameters for writing to table.
Returns
-------
prepared parameters :
impact
max_signal
tot_signal
n_tels_per_type
hillas_dict
mc_offangle
reco_result
'''
# check if event was prepared before
try:
assert self.impact
except AssertionError:
self.prepare()
return (self.impact, self.max_signal, self.tot_signal,
self.n_tels_per_type, self.hillas_dict, self.mc_offset,
self.reco_result)
def __init__(self, config, mode, event_cutflow=None, image_cutflow=None):
"""Initiliaze an EventPreparer object."""
# Cleaning for reconstruction
self.cleaner_reco = ImageCleaner( # for reconstruction
config=config["ImageCleaning"]["biggest"],
mode=mode)
# Cleaning for energy/score estimation
# Add possibility to force energy/score cleaning with tailcut analysis
force_mode = mode
try:
if config["General"]["force_tailcut_for_extended_cleaning"] is True:
force_mode = config["General"]["force_mode"]
print("> Activate force-mode for cleaning!!!!")
except:
pass # force_mode = mode
self.cleaner_extended = ImageCleaner( # for energy/score estimation
config=config["ImageCleaning"]["extended"],
mode=force_mode)
# Image book keeping
self.image_cutflow = image_cutflow or CutFlow("ImageCutFlow")
# Add quality cuts on images
charge_bounds = config["ImageSelection"]["charge"]
npix_bounds = config["ImageSelection"]["pixel"]
ellipticity_bounds = config["ImageSelection"]["ellipticity"]
nominal_distance_bounds = config["ImageSelection"]["nominal_distance"]
self.camera_radius = {
"LSTCam": 1.126,
"NectarCam": 1.126,
} # Average between max(xpix) and max(ypix), in meters
self.image_cutflow.set_cuts(
OrderedDict([
("noCuts", None),
("min pixel", lambda s: np.count_nonzero(s) < npix_bounds[0]),
("min charge", lambda x: x < charge_bounds[0]),
# ("poor moments", lambda m: m.width <= 0 or m.length <= 0 or np.isnan(m.width) or np.isnan(m.length)),
# TBC, maybe we loose events without nan conditions
("poor moments", lambda m: m.width <= 0 or m.length <= 0),
(
"bad ellipticity",
lambda m: (m.width / m.length) < ellipticity_bounds[0] or
(m.width / m.length) > ellipticity_bounds[-1],
),
# ("close to the edge", lambda m, cam_id: m.r.value > (nominal_distance_bounds[-1] * 1.12949101073069946))
# in meter
(
"close to the edge",
lambda m, cam_id: m.r.value >
(nominal_distance_bounds[-1] * self.camera_radius[cam_id]),
), # in meter
]))
# configuration for the camera calibrator
# modifies the integration window to be more like in MARS
# JLK, only for LST!!!!
cfg = Config()
cfg["ChargeExtractorFactory"]["window_width"] = 5
cfg["ChargeExtractorFactory"]["window_shift"] = 2
extractor = LocalPeakWindowSum(config=cfg)
self.calib = CameraCalibrator(config=cfg, image_extractor=extractor)
# Reconstruction
self.shower_reco = HillasReconstructor()
# Event book keeping
self.event_cutflow = event_cutflow or CutFlow("EventCutFlow")
# Add cuts on events
min_ntel = config["Reconstruction"]["min_tel"]
self.event_cutflow.set_cuts(
OrderedDict([
("noCuts", None),
("min2Tels trig", lambda x: x < min_ntel),
("min2Tels reco", lambda x: x < min_ntel),
("direction nan", lambda x: x.is_valid == False),
]))
class EventPreparer:
"""Class which loop on events and returns results stored in container.
The Class has several purposes. First of all, it prepares the images of the
event that will be further use for reconstruction by applying calibration,
cleaning and selection. Then, it reconstructs the geometry of the event and
then returns image (e.g. Hillas parameters)and event information
(e.g. results of the reconstruction).
Parameters
----------
config: dict
Configuration with analysis parameters
mode: str
Mode of the reconstruction, e.g. tail or wave
event_cutflow: ctapipe.utils.CutFlow
Statistic of events processed
image_cutflow: ctapipe.utils.CutFlow
Statistic of images processed
Returns: dict
Dictionnary of results
"""
def __init__(self, config, mode, event_cutflow=None, image_cutflow=None):
"""Initiliaze an EventPreparer object."""
# Cleaning for reconstruction
self.cleaner_reco = ImageCleaner( # for reconstruction
config=config["ImageCleaning"]["biggest"],
mode=mode)
# Cleaning for energy/score estimation
# Add possibility to force energy/score cleaning with tailcut analysis
force_mode = mode
try:
if config["General"]["force_tailcut_for_extended_cleaning"] is True:
force_mode = config["General"]["force_mode"]
print("> Activate force-mode for cleaning!!!!")
except:
pass # force_mode = mode
self.cleaner_extended = ImageCleaner( # for energy/score estimation
config=config["ImageCleaning"]["extended"],
mode=force_mode)
# Image book keeping
self.image_cutflow = image_cutflow or CutFlow("ImageCutFlow")
# Add quality cuts on images
charge_bounds = config["ImageSelection"]["charge"]
npix_bounds = config["ImageSelection"]["pixel"]
ellipticity_bounds = config["ImageSelection"]["ellipticity"]
nominal_distance_bounds = config["ImageSelection"]["nominal_distance"]
self.camera_radius = {
"LSTCam": 1.126,
"NectarCam": 1.126,
} # Average between max(xpix) and max(ypix), in meters
self.image_cutflow.set_cuts(
OrderedDict([
("noCuts", None),
("min pixel", lambda s: np.count_nonzero(s) < npix_bounds[0]),
("min charge", lambda x: x < charge_bounds[0]),
# ("poor moments", lambda m: m.width <= 0 or m.length <= 0 or np.isnan(m.width) or np.isnan(m.length)),
# TBC, maybe we loose events without nan conditions
("poor moments", lambda m: m.width <= 0 or m.length <= 0),
(
"bad ellipticity",
lambda m: (m.width / m.length) < ellipticity_bounds[0] or
(m.width / m.length) > ellipticity_bounds[-1],
),
# ("close to the edge", lambda m, cam_id: m.r.value > (nominal_distance_bounds[-1] * 1.12949101073069946))
# in meter
(
"close to the edge",
lambda m, cam_id: m.r.value >
(nominal_distance_bounds[-1] * self.camera_radius[cam_id]),
), # in meter
]))
# configuration for the camera calibrator
# modifies the integration window to be more like in MARS
# JLK, only for LST!!!!
cfg = Config()
cfg["ChargeExtractorFactory"]["window_width"] = 5
cfg["ChargeExtractorFactory"]["window_shift"] = 2
extractor = LocalPeakWindowSum(config=cfg)
self.calib = CameraCalibrator(config=cfg, image_extractor=extractor)
# Reconstruction
self.shower_reco = HillasReconstructor()
# Event book keeping
self.event_cutflow = event_cutflow or CutFlow("EventCutFlow")
# Add cuts on events
min_ntel = config["Reconstruction"]["min_tel"]
self.event_cutflow.set_cuts(
OrderedDict([
("noCuts", None),
("min2Tels trig", lambda x: x < min_ntel),
("min2Tels reco", lambda x: x < min_ntel),
("direction nan", lambda x: x.is_valid == False),
]))
def prepare_event(self, source, return_stub=False, save_images=False):
for event in source:
self.event_cutflow.count("noCuts")
if self.event_cutflow.cut("min2Tels trig",
len(event.dl0.tels_with_data)):
if return_stub:
yield stub(event)
else:
continue
self.calib(event)
# telescope loop
tot_signal = 0
dl1_phe_image = None
mc_phe_image = None
max_signals = {}
n_pixel_dict = {}
hillas_dict_reco = {} # for direction reconstruction
hillas_dict = {} # for discrimination
n_tels = {
"tot": len(event.dl0.tels_with_data),
"LST_LST_LSTCam": 0,
"MST_MST_NectarCam": 0,
"SST": 0, # add later correct names when testing on Paranal
}
n_cluster_dict = {}
impact_dict_reco = {} # impact distance measured in tilt system
point_azimuth_dict = {}
point_altitude_dict = {}
# Compute impact parameter in tilt system
run_array_direction = event.mcheader.run_array_direction
az, alt = run_array_direction[0], run_array_direction[1]
ground_frame = GroundFrame()
for tel_id in event.dl0.tels_with_data:
self.image_cutflow.count("noCuts")
camera = event.inst.subarray.tel[tel_id].camera
# count the current telescope according to its size
tel_type = str(event.inst.subarray.tel[tel_id])
# use ctapipe's functionality to get the calibrated image
pmt_signal = event.dl1.tel[tel_id].image
# Save the calibrated image after the gain has been chosen
# automatically by ctapipe, together with the simulated one
if save_images is True:
dl1_phe_image = pmt_signal
mc_phe_image = event.mc.tel[tel_id].photo_electron_image
if self.cleaner_reco.mode == "tail": # tail uses only ctapipe
# Cleaning used for direction reconstruction
image_biggest, mask_reco = self.cleaner_reco.clean_image(
pmt_signal, camera)
# find all islands using this cleaning
num_islands, labels = number_of_islands(camera, mask_reco)
if num_islands == 1: # if only ONE islands is left ...
# ...use directly the old mask and reduce dimensions
# to make Hillas parametrization faster
camera_biggest = camera[mask_reco]
image_biggest = image_biggest[mask_reco]
elif num_islands > 1: # if more islands survived..
# ...find the biggest one
mask_biggest = largest_island(labels)
# and also reduce dimensions
camera_biggest = camera[mask_biggest]
image_biggest = image_biggest[mask_biggest]
else: # if no islands survived use old camera and image
camera_biggest = camera
# Cleaning used for score/energy estimation
image_extended, mask_extended = self.cleaner_extended.clean_image(
pmt_signal, camera)
# find all islands with this cleaning
# we will also register how many have been found
n_cluster_dict[tel_id], labels = number_of_islands(
camera, mask_extended)
# NOTE: the next check shouldn't be necessary if we keep
# all the isolated pixel clusters, but for now the
# two cleanings are set the same in analysis.yml because
# the performance of the extended one has never been really
# studied in model estimation.
# (This is a nice way to ask for volunteers :P)
# if some islands survived
if num_islands > 0:
# keep all of them and reduce dimensions
camera_extended = camera[mask_extended]
image_extended = image_extended[mask_extended]
else: # otherwise continue with the old camera and image
camera_extended = camera
# could this go into `hillas_parameters` ...?
# this is basically the charge of ALL islands
# not calculated later by the Hillas parametrization!
max_signals[tel_id] = np.max(image_extended)
else: # for wavelets we stick to old pywi-cta code
try: # "try except FileNotFoundError" not clear to me, but for now it stays...
with warnings.catch_warnings():
# Image with biggest cluster (reco cleaning)
image_biggest, mask_reco = self.cleaner_reco.clean_image(
pmt_signal, camera)
image_biggest2d = geometry_converter.image_1d_to_2d(
image_biggest, camera.cam_id)
image_biggest2d = filter_pixels_clusters(
image_biggest2d)
image_biggest = geometry_converter.image_2d_to_1d(
image_biggest2d, camera.cam_id)
# Image for score/energy estimation (with clusters)
image_extended, mask_extended = self.cleaner_extended.clean_image(
pmt_signal, camera)
# This last part was outside the pywi-cta block
# before, but is indeed part of it because it uses
# pywi-cta functions in the "extended" case
# For cluster counts
image_2d = geometry_converter.image_1d_to_2d(
image_extended, camera.cam_id)
n_cluster_dict[
tel_id] = pixel_clusters.number_of_pixels_clusters(
array=image_2d, threshold=0)
# could this go into `hillas_parameters` ...?
max_signals[tel_id] = np.max(image_extended)
except FileNotFoundError as e: # JLK, WHAT?
print(e)
continue
# ==============================================================
# Apply some selection
if self.image_cutflow.cut("min pixel", image_biggest):
continue
if self.image_cutflow.cut("min charge", np.sum(image_biggest)):
continue
# do the hillas reconstruction of the images
# QUESTION should this change in numpy behaviour be done here
# or within `hillas_parameters` itself?
# JLK: make selection on biggest cluster
with np.errstate(invalid="raise", divide="raise"):
try:
moments_reco = hillas_parameters(
camera_biggest,
image_biggest) # for geometry (eg direction)
moments = hillas_parameters(
camera_extended, image_extended
) # for discrimination and energy reconstruction
# if width and/or length are zero (e.g. when there is
# only only one pixel or when all pixel are exactly
# in one row), the parametrisation
# won't be very useful: skip
if self.image_cutflow.cut("poor moments",
moments_reco):
continue
if self.image_cutflow.cut("close to the edge",
moments_reco, camera.cam_id):
continue
if self.image_cutflow.cut("bad ellipticity",
moments_reco):
continue
except (FloatingPointError,
hillas.HillasParameterizationError):
continue
point_azimuth_dict[
tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
point_altitude_dict[
tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad
n_tels[tel_type] += 1
hillas_dict[tel_id] = moments
hillas_dict_reco[tel_id] = moments_reco
n_pixel_dict[tel_id] = len(np.where(image_extended > 0)[0])
tot_signal += moments.intensity
n_tels["reco"] = len(hillas_dict_reco)
n_tels["discri"] = len(hillas_dict)
if self.event_cutflow.cut("min2Tels reco", n_tels["reco"]):
if return_stub:
yield stub(event)
else:
continue
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# Reconstruction results
reco_result = self.shower_reco.predict(
hillas_dict_reco,
event.inst,
SkyCoord(alt=alt, az=az, frame="altaz"),
{
tel_id: SkyCoord(
alt=point_altitude_dict[tel_id],
az=point_azimuth_dict[tel_id],
frame="altaz",
) # cycle only on tels which still have an image
for tel_id in point_altitude_dict.keys()
},
)
# Impact parameter for energy estimation (/ tel)
subarray = event.inst.subarray
for tel_id in hillas_dict.keys():
pos = subarray.positions[tel_id]
tel_ground = SkyCoord(pos[0],
pos[1],
pos[2],
frame=ground_frame)
core_ground = SkyCoord(
reco_result.core_x,
reco_result.core_y,
0 * u.m,
frame=ground_frame,
)
# Should be better handled (tilted frame)
impact_dict_reco[tel_id] = np.sqrt(
(core_ground.x - tel_ground.x)**2 +
(core_ground.y - tel_ground.y)**2)
except Exception as e:
print("exception in reconstruction:", e)
raise
if return_stub:
yield stub(event)
else:
continue
if self.event_cutflow.cut("direction nan", reco_result):
if return_stub:
yield stub(event)
else:
continue
yield PreparedEvent(
event=event,
dl1_phe_image=dl1_phe_image,
mc_phe_image=mc_phe_image,
n_pixel_dict=n_pixel_dict,
hillas_dict=hillas_dict,
hillas_dict_reco=hillas_dict_reco,
n_tels=n_tels,
tot_signal=tot_signal,
max_signals=max_signals,
n_cluster_dict=n_cluster_dict,
reco_result=reco_result,
impact_dict=impact_dict_reco,
)
def main():
# your favourite units here
energy_unit = u.TeV
angle_unit = u.deg
dist_unit = u.m
agree_threshold = .5
min_tel = 3
parser = make_argparser()
parser.add_argument('--classifier',
type=str,
default=expandvars(
"$CTA_SOFT/tino_cta/data/classifier_pickle/"
"classifier_{mode}_{cam_id}_{classifier}.pkl"))
parser.add_argument('--regressor',
type=str,
default=expandvars(
"$CTA_SOFT/tino_cta/data/classifier_pickle/"
"regressor_{mode}_{cam_id}_{regressor}.pkl"))
parser.add_argument('-o',
'--outfile',
type=str,
default="",
help="location to write the classified events to.")
parser.add_argument('--wave_dir',
type=str,
default=None,
help="directory where to find mr_filter. "
"if not set look in $PATH")
parser.add_argument(
'--wave_temp_dir',
type=str,
default='/dev/shm/',
help="directory where mr_filter to store the temporary fits "
"files")
group = parser.add_mutually_exclusive_group()
group.add_argument('--proton',
action='store_true',
help="do protons instead of gammas")
group.add_argument('--electron',
action='store_true',
help="do electrons instead of gammas")
args = parser.parse_args()
if args.infile_list:
filenamelist = []
for f in args.infile_list:
filenamelist += glob("{}/{}".format(args.indir, f))
filenamelist.sort()
elif args.proton:
filenamelist = sorted(glob("{}/proton/*gz".format(args.indir)))
elif args.electron:
filenamelist = glob("{}/electron/*gz".format(args.indir))
channel = "electron"
else:
filenamelist = sorted(glob("{}/gamma/*gz".format(args.indir)))
if not filenamelist:
print("no files found; check indir: {}".format(args.indir))
exit(-1)
# keeping track of events and where they were rejected
Eventcutflow = CutFlow("EventCutFlow")
Imagecutflow = CutFlow("ImageCutFlow")
# takes care of image cleaning
cleaner = ImageCleaner(mode=args.mode,
cutflow=Imagecutflow,
wavelet_options=args.raw,
tmp_files_directory=args.wave_temp_dir,
skip_edge_events=False,
island_cleaning=True)
# the class that does the shower reconstruction
shower_reco = HillasReconstructor()
preper = EventPreparer(
cleaner=cleaner,
hillas_parameters=hillas_parameters,
shower_reco=shower_reco,
event_cutflow=Eventcutflow,
image_cutflow=Imagecutflow,
# event/image cuts:
allowed_cam_ids=[],
min_ntel=2,
min_charge=args.min_charge,
min_pixel=3)
# wrapper for the scikit-learn classifier
classifier = EventClassifier.load(args.classifier.format(
**{
"mode": args.mode,
"wave_args": "mixed",
"classifier": 'RandomForestClassifier',
"cam_id": "{cam_id}"
}),
cam_id_list=args.cam_ids)
# wrapper for the scikit-learn regressor
regressor = EnergyRegressor.load(args.regressor.format(
**{
"mode": args.mode,
"wave_args": "mixed",
"regressor": "RandomForestRegressor",
"cam_id": "{cam_id}"
}),
cam_id_list=args.cam_ids)
ClassifierFeatures = namedtuple(
"ClassifierFeatures",
("impact_dist", "sum_signal_evt", "max_signal_cam", "sum_signal_cam",
"N_LST", "N_MST", "N_SST", "width", "length", "skewness", "kurtosis",
"h_max", "err_est_pos", "err_est_dir"))
EnergyFeatures = namedtuple(
"EnergyFeatures",
("impact_dist", "sum_signal_evt", "max_signal_cam", "sum_signal_cam",
"N_LST", "N_MST", "N_SST", "width", "length", "skewness", "kurtosis",
"h_max", "err_est_pos", "err_est_dir"))
# catch ctr-c signal to exit current loop and still display results
signal_handler = SignalHandler()
signal.signal(signal.SIGINT, signal_handler)
# this class defines the reconstruction parameters to keep track of
class RecoEvent(tb.IsDescription):
Run_ID = tb.Int16Col(dflt=-1, pos=0)
Event_ID = tb.Int16Col(dflt=-1, pos=1)
NTels_trig = tb.Int16Col(dflt=0, pos=0)
NTels_reco = tb.Int16Col(dflt=0, pos=1)
NTels_reco_lst = tb.Int16Col(dflt=0, pos=2)
NTels_reco_mst = tb.Int16Col(dflt=0, pos=3)
NTels_reco_sst = tb.Int16Col(dflt=0, pos=4)
MC_Energy = tb.Float32Col(dflt=np.nan, pos=5)
reco_Energy = tb.Float32Col(dflt=np.nan, pos=6)
reco_phi = tb.Float32Col(dflt=np.nan, pos=7)
reco_theta = tb.Float32Col(dflt=np.nan, pos=8)
off_angle = tb.Float32Col(dflt=np.nan, pos=9)
xi = tb.Float32Col(dflt=np.nan, pos=10)
DeltaR = tb.Float32Col(dflt=np.nan, pos=11)
ErrEstPos = tb.Float32Col(dflt=np.nan, pos=12)
ErrEstDir = tb.Float32Col(dflt=np.nan, pos=13)
gammaness = tb.Float32Col(dflt=np.nan, pos=14)
success = tb.BoolCol(dflt=False, pos=15)
channel = "gamma" if "gamma" in " ".join(filenamelist) else "proton"
reco_outfile = tb.open_file(
mode="w",
# if no outfile name is given (i.e. don't to write the event list to disk),
# need specify two "driver" arguments
**({
"filename": args.outfile
} if args.outfile else {
"filename": "no_outfile.h5",
"driver": "H5FD_CORE",
"driver_core_backing_store": False
}))
reco_table = reco_outfile.create_table("/", "reco_events", RecoEvent)
reco_event = reco_table.row
allowed_tels = None # all telescopes
allowed_tels = prod3b_tel_ids("L+N+D")
for i, filename in enumerate(filenamelist[:args.last]):
# print(f"file: {i} filename = {filename}")
source = hessio_event_source(filename,
allowed_tels=allowed_tels,
max_events=args.max_events)
# loop that cleans and parametrises the images and performs the reconstruction
for (event, hillas_dict, n_tels, tot_signal, max_signals, pos_fit,
dir_fit, h_max, err_est_pos,
err_est_dir) in preper.prepare_event(source, True):
# now prepare the features for the classifier
cls_features_evt = {}
reg_features_evt = {}
if hillas_dict is not None:
for tel_id in hillas_dict.keys():
Imagecutflow.count("pre-features")
tel_pos = np.array(event.inst.tel_pos[tel_id][:2]) * u.m
moments = hillas_dict[tel_id]
impact_dist = linalg.length(tel_pos - pos_fit)
cls_features_tel = ClassifierFeatures(
impact_dist=impact_dist / u.m,
sum_signal_evt=tot_signal,
max_signal_cam=max_signals[tel_id],
sum_signal_cam=moments.size,
N_LST=n_tels["LST"],
N_MST=n_tels["MST"],
N_SST=n_tels["SST"],
width=moments.width / u.m,
length=moments.length / u.m,
skewness=moments.skewness,
kurtosis=moments.kurtosis,
h_max=h_max / u.m,
err_est_pos=err_est_pos / u.m,
err_est_dir=err_est_dir / u.deg)
reg_features_tel = EnergyFeatures(
impact_dist=impact_dist / u.m,
sum_signal_evt=tot_signal,
max_signal_cam=max_signals[tel_id],
sum_signal_cam=moments.size,
N_LST=n_tels["LST"],
N_MST=n_tels["MST"],
N_SST=n_tels["SST"],
width=moments.width / u.m,
length=moments.length / u.m,
skewness=moments.skewness,
kurtosis=moments.kurtosis,
h_max=h_max / u.m,
err_est_pos=err_est_pos / u.m,
err_est_dir=err_est_dir / u.deg)
if np.isnan(cls_features_tel).any() or np.isnan(
reg_features_tel).any():
continue
Imagecutflow.count("features nan")
cam_id = event.inst.subarray.tel[tel_id].camera.cam_id
try:
reg_features_evt[cam_id] += [reg_features_tel]
cls_features_evt[cam_id] += [cls_features_tel]
except KeyError:
reg_features_evt[cam_id] = [reg_features_tel]
cls_features_evt[cam_id] = [cls_features_tel]
if cls_features_evt and reg_features_evt:
predict_energ = regressor.predict_by_event([reg_features_evt
])["mean"][0]
predict_proba = classifier.predict_proba_by_event(
[cls_features_evt])
gammaness = predict_proba[0, 0]
try:
# the MC direction of origin of the simulated particle
shower = event.mc
shower_core = np.array(
[shower.core_x / u.m, shower.core_y / u.m]) * u.m
shower_org = linalg.set_phi_theta(az_to_phi(shower.az),
alt_to_theta(shower.alt))
# and how the reconstructed direction compares to that
xi = linalg.angle(dir_fit, shower_org)
DeltaR = linalg.length(pos_fit[:2] - shower_core)
except Exception:
# naked exception catch, because I'm not sure where
# it would break in non-MC files
xi = np.nan
DeltaR = np.nan
phi, theta = linalg.get_phi_theta(dir_fit)
phi = (phi if phi > 0 else phi + 360 * u.deg)
# TODO: replace with actual array pointing direction
array_pointing = linalg.set_phi_theta(0 * u.deg, 20. * u.deg)
# angular offset between the reconstructed direction and the array
# pointing
off_angle = linalg.angle(dir_fit, array_pointing)
reco_event["NTels_trig"] = len(event.dl0.tels_with_data)
reco_event["NTels_reco"] = len(hillas_dict)
reco_event["NTels_reco_lst"] = n_tels["LST"]
reco_event["NTels_reco_mst"] = n_tels["MST"]
reco_event["NTels_reco_sst"] = n_tels["SST"]
reco_event["reco_Energy"] = predict_energ.to(energy_unit).value
reco_event["reco_phi"] = phi / angle_unit
reco_event["reco_theta"] = theta / angle_unit
reco_event["off_angle"] = off_angle / angle_unit
reco_event["xi"] = xi / angle_unit
reco_event["DeltaR"] = DeltaR / dist_unit
reco_event["ErrEstPos"] = err_est_pos / dist_unit
reco_event["ErrEstDir"] = err_est_dir / angle_unit
reco_event["gammaness"] = gammaness
reco_event["success"] = True
else:
reco_event["success"] = False
# save basic event infos
reco_event["MC_Energy"] = event.mc.energy.to(energy_unit).value
reco_event["Event_ID"] = event.r1.event_id
reco_event["Run_ID"] = event.r1.run_id
reco_table.flush()
reco_event.append()
if signal_handler.stop:
break
if signal_handler.stop:
break
# make sure everything gets written out nicely
reco_table.flush()
try:
print()
Eventcutflow()
print()
Imagecutflow()
# do some simple event selection
# and print the corresponding selection efficiency
N_selected = len([
x for x in reco_table.where(
"""(NTels_reco > min_tel) & (gammaness > agree_threshold)""")
])
N_total = len(reco_table)
print("\nfraction selected events:")
print("{} / {} = {} %".format(N_selected, N_total,
N_selected / N_total * 100))
except ZeroDivisionError:
pass
print("\nlength filenamelist:", len(filenamelist[:args.last]))
# do some plotting if so desired
if args.plot:
gammaness = [x['gammaness'] for x in reco_table]
NTels_rec = [x['NTels_reco'] for x in reco_table]
NTel_bins = np.arange(np.min(NTels_rec), np.max(NTels_rec) + 2) - .5
NTels_rec_lst = [x['NTels_reco_lst'] for x in reco_table]
NTels_rec_mst = [x['NTels_reco_mst'] for x in reco_table]
NTels_rec_sst = [x['NTels_reco_sst'] for x in reco_table]
reco_energy = np.array([x['reco_Energy'] for x in reco_table])
mc_energy = np.array([x['MC_Energy'] for x in reco_table])
fig = plt.figure(figsize=(15, 5))
plt.suptitle(" ** ".join(
[args.mode, "protons" if args.proton else "gamma"]))
plt.subplots_adjust(left=0.05, right=0.97, hspace=0.39, wspace=0.2)
ax = plt.subplot(131)
histo = np.histogram2d(NTels_rec,
gammaness,
bins=(NTel_bins, np.linspace(0, 1, 11)))[0].T
histo_normed = histo / histo.max(axis=0)
im = ax.imshow(
histo_normed,
interpolation='none',
origin='lower',
aspect='auto',
# extent=(*NTel_bins[[0, -1]], 0, 1),
cmap=plt.cm.inferno)
ax.set_xlabel("NTels")
ax.set_ylabel("drifted gammaness")
plt.title("Total Number of Telescopes")
# next subplot
ax = plt.subplot(132)
histo = np.histogram2d(NTels_rec_sst,
gammaness,
bins=(NTel_bins, np.linspace(0, 1, 11)))[0].T
histo_normed = histo / histo.max(axis=0)
im = ax.imshow(
histo_normed,
interpolation='none',
origin='lower',
aspect='auto',
# extent=(*NTel_bins[[0, -1]], 0, 1),
cmap=plt.cm.inferno)
ax.set_xlabel("NTels")
plt.setp(ax.get_yticklabels(), visible=False)
plt.title("Number of SSTs")
# next subplot
ax = plt.subplot(133)
histo = np.histogram2d(NTels_rec_mst,
gammaness,
bins=(NTel_bins, np.linspace(0, 1, 11)))[0].T
histo_normed = histo / histo.max(axis=0)
im = ax.imshow(
histo_normed,
interpolation='none',
origin='lower',
aspect='auto',
# extent=(*NTel_bins[[0, -1]], 0, 1),
cmap=plt.cm.inferno)
cb = fig.colorbar(im, ax=ax)
ax.set_xlabel("NTels")
plt.setp(ax.get_yticklabels(), visible=False)
plt.title("Number of MSTs")
plt.subplots_adjust(wspace=0.05)
# plot the energy migration matrix
plt.figure()
plt.hist2d(np.log10(reco_energy),
np.log10(mc_energy),
bins=20,
cmap=plt.cm.inferno)
plt.xlabel("E_MC / TeV")
plt.ylabel("E_rec / TeV")
plt.colorbar()
plt.show()
Cleaner = {
"w":
ImageCleaner(mode="wave",
cutflow=Imagecutflow,
skip_edge_events=skip_edge_events,
island_cleaning=island_cleaning,
wavelet_options=args.raw),
"t":
ImageCleaner(mode="tail",
cutflow=Imagecutflow,
skip_edge_events=skip_edge_events,
island_cleaning=island_cleaning)
}
# simple hillas-based shower reco
fit = HillasReconstructor()
signal_handler = SignalHandler()
if args.plot_c:
signal.signal(signal.SIGINT, signal_handler.stop_drawing)
else:
signal.signal(signal.SIGINT, signal_handler)
# keeping track of the hit distribution transverse to the shower axis on the camera
# for different energy bins
from modules.Histogram import nDHistogram
pe_vs_dp = {'p': {}, 'w': {}, 't': {}}
for k in pe_vs_dp.keys():
pe_vs_dp[k] = nDHistogram(
bin_edges=[np.arange(6),
np.linspace(-.1, .1, 42) * u.m],
def test_reconstruction():
"""
a test of the complete fit procedure on one event including:
• tailcut cleaning
• hillas parametrisation
• HillasPlane creation
• direction fit
• position fit
in the end, proper units in the output are asserted """
filename = get_dataset_path("gamma_test_large.simtel.gz")
source = EventSource(filename, max_events=10)
calib = CameraCalibrator(subarray=source.subarray)
horizon_frame = AltAz()
reconstructed_events = 0
for event in source:
calib(event)
mc = event.simulation.shower
array_pointing = SkyCoord(az=mc.az, alt=mc.alt, frame=horizon_frame)
hillas_dict = {}
telescope_pointings = {}
for tel_id, dl1 in event.dl1.tel.items():
geom = source.subarray.tel[tel_id].camera.geometry
telescope_pointings[tel_id] = SkyCoord(
alt=event.pointing.tel[tel_id].altitude,
az=event.pointing.tel[tel_id].azimuth,
frame=horizon_frame,
)
mask = tailcuts_clean(geom,
dl1.image,
picture_thresh=10.0,
boundary_thresh=5.0)
try:
moments = hillas_parameters(geom[mask], dl1.image[mask])
hillas_dict[tel_id] = moments
except HillasParameterizationError as e:
print(e)
continue
if len(hillas_dict) < 2:
continue
else:
reconstructed_events += 1
# The three reconstructions below gives the same results
fit = HillasReconstructor()
fit_result_parall = fit.predict(hillas_dict, source.subarray,
array_pointing)
fit = HillasReconstructor()
fit_result_tel_point = fit.predict(hillas_dict, source.subarray,
array_pointing, telescope_pointings)
for key in fit_result_parall.keys():
print(key, fit_result_parall[key], fit_result_tel_point[key])
fit_result_parall.alt.to(u.deg)
fit_result_parall.az.to(u.deg)
fit_result_parall.core_x.to(u.m)
assert fit_result_parall.is_valid
assert reconstructed_events > 0
# do some cross-validation now
if args.check:
# keeping track of events and where they were rejected
Eventcutflow = CutFlow("EventCutFlow")
Imagecutflow = CutFlow("ImageCutFlow")
# takes care of image cleaning
cleaner = ImageCleaner(mode=args.mode,
cutflow=Imagecutflow,
wavelet_options=args.raw,
skip_edge_events=False,
island_cleaning=True)
# the class that does the shower reconstruction
shower_reco = HillasReconstructor()
preper = EventPreparer(
cleaner=cleaner,
shower_reco=shower_reco,
event_cutflow=Eventcutflow,
image_cutflow=Imagecutflow,
# event/image cuts:
allowed_cam_ids=[], # [] or None means: all
min_ntel=2,
min_charge=args.min_charge,
min_pixel=3)
Imagecutflow.add_cut("features nan", lambda x: np.isnan(x).any())
energy_mc = []
energy_rec = []
def test_invalid_events():
"""
The HillasReconstructor is supposed to fail
in these cases:
- less than two teleskopes
- any width is NaN
- any width is 0
This test uses the same sample simtel file as
test_reconstruction(). As there are no invalid events in this
file, multiple hillas_dicts are constructed to make sure
Exceptions get thrown in the mentioned edge cases.
Test will fail if no Exception or another Exception gets thrown."""
filename = get_dataset_path("gamma_test_large.simtel.gz")
fit = HillasReconstructor()
tel_azimuth = {}
tel_altitude = {}
source = EventSource(filename, max_events=10)
subarray = source.subarray
calib = CameraCalibrator(subarray)
for event in source:
calib(event)
hillas_dict = {}
for tel_id, dl1 in event.dl1.tel.items():
geom = source.subarray.tel[tel_id].camera.geometry
tel_azimuth[tel_id] = event.pointing.tel[tel_id].azimuth
tel_altitude[tel_id] = event.pointing.tel[tel_id].altitude
mask = tailcuts_clean(geom,
dl1.image,
picture_thresh=10.0,
boundary_thresh=5.0)
try:
moments = hillas_parameters(geom[mask], dl1.image[mask])
hillas_dict[tel_id] = moments
except HillasParameterizationError as e:
continue
# construct a dict only containing the last telescope events
# (#telescopes < 2)
hillas_dict_only_one_tel = dict()
hillas_dict_only_one_tel[tel_id] = hillas_dict[tel_id]
with pytest.raises(TooFewTelescopesException):
fit.predict(hillas_dict_only_one_tel, subarray, tel_azimuth,
tel_altitude)
# construct a hillas dict with the width of the last event set to 0
# (any width == 0)
hillas_dict_zero_width = hillas_dict.copy()
hillas_dict_zero_width[tel_id]["width"] = 0 * u.m
with pytest.raises(InvalidWidthException):
fit.predict(hillas_dict_zero_width, subarray, tel_azimuth,
tel_altitude)
# construct a hillas dict with the width of the last event set to np.nan
# (any width == nan)
hillas_dict_nan_width = hillas_dict.copy()
hillas_dict_zero_width[tel_id]["width"] = np.nan * u.m
with pytest.raises(InvalidWidthException):
fit.predict(hillas_dict_nan_width, subarray, tel_azimuth,
tel_altitude)
def main():
# your favourite units here
energy_unit = u.TeV
angle_unit = u.deg
dist_unit = u.m
parser = make_argparser()
parser.add_argument(
'-o',
'--outfile',
type=str,
help="if given, write output file with reconstruction results")
parser.add_argument('--plot_c',
action='store_true',
help="plot camera-wise displays")
group = parser.add_mutually_exclusive_group()
group.add_argument('--proton',
action='store_true',
help="do protons instead of gammas")
group.add_argument('--electron',
action='store_true',
help="do electrons instead of gammas")
args = parser.parse_args()
if args.infile_list:
filenamelist = []
for f in args.infile_list:
filenamelist += glob("{}/{}".format(args.indir, f))
elif args.proton:
filenamelist = glob("{}/proton/*gz".format(args.indir))
channel = "proton"
elif args.electron:
filenamelist = glob("{}/electron/*gz".format(args.indir))
channel = "electron"
elif args.gamma:
filenamelist = glob("{}/gamma/*gz".format(args.indir))
channel = "gamma"
else:
raise ValueError("don't know which input to use...")
filenamelist.sort()
if not filenamelist:
print("no files found; check indir: {}".format(args.indir))
exit(-1)
else:
print("found {} files".format(len(filenamelist)))
tel_phi = {}
tel_theta = {}
# keeping track of events and where they were rejected
Eventcutflow = CutFlow("EventCutFlow")
Imagecutflow = CutFlow("ImageCutFlow")
# takes care of image cleaning
cleaner = ImageCleaner(mode=args.mode,
cutflow=Imagecutflow,
wavelet_options=args.raw,
skip_edge_events=args.skip_edge_events,
island_cleaning=True)
# the class that does the shower reconstruction
shower_reco = HillasReconstructor()
shower_max_estimator = ShowerMaxEstimator("paranal")
preper = EventPreparer(
cleaner=cleaner,
hillas_parameters=hillas_parameters,
shower_reco=shower_reco,
event_cutflow=Eventcutflow,
image_cutflow=Imagecutflow,
# event/image cuts:
allowed_cam_ids=[], # means: all
min_ntel=3,
min_charge=args.min_charge,
min_pixel=3)
# a signal handler to abort the event loop but still do the post-processing
signal_handler = SignalHandler()
signal.signal(signal.SIGINT, signal_handler)
try:
# this class defines the reconstruction parameters to keep track of
class RecoEvent(tb.IsDescription):
NTels_trigg = tb.Int16Col(dflt=1, pos=0)
NTels_clean = tb.Int16Col(dflt=1, pos=1)
EnMC = tb.Float32Col(dflt=1, pos=2)
xi = tb.Float32Col(dflt=1, pos=3)
DeltaR = tb.Float32Col(dflt=1, pos=4)
ErrEstPos = tb.Float32Col(dflt=1, pos=5)
ErrEstDir = tb.Float32Col(dflt=1, pos=6)
h_max = tb.Float32Col(dflt=1, pos=7)
reco_outfile = tb.open_file(
args.outfile,
mode="w",
# if we don't want to write the event list to disk, need to add more arguments
**({} if args.store else {
"driver": "H5FD_CORE",
"driver_core_backing_store": False
}))
reco_table = reco_outfile.create_table("/", "reco_event", RecoEvent)
reco_event = reco_table.row
except:
reco_event = RecoEvent()
print("no pytables installed?")
# ## ####### ####### ########
# ## ## ## ## ## ## ##
# ## ## ## ## ## ## ##
# ## ## ## ## ## ########
# ## ## ## ## ## ##
# ## ## ## ## ## ##
# ######## ####### ####### ##
cam_id_map = {}
# define here which telescopes to loop over
allowed_tels = None
# allowed_tels = prod3b_tel_ids("L+F+D")
for i, filename in enumerate(filenamelist[:args.last]):
print("file: {i} filename = {filename}".format(i=i, filename=filename))
source = hessio_event_source(filename,
allowed_tels=allowed_tels,
max_events=args.max_events)
# loop that cleans and parametrises the images and performs the reconstruction
for (event, hillas_dict, n_tels, tot_signal, max_signal, pos_fit,
dir_fit, h_max, err_est_pos,
err_est_dir) in preper.prepare_event(source):
shower = event.mc
org_alt = u.Quantity(shower.alt).to(u.deg)
org_az = u.Quantity(shower.az).to(u.deg)
if org_az > 180 * u.deg:
org_az -= 360 * u.deg
org_the = alt_to_theta(org_alt)
org_phi = az_to_phi(org_az)
if org_phi > 180 * u.deg:
org_phi -= 360 * u.deg
if org_phi < -180 * u.deg:
org_phi += 360 * u.deg
shower_org = linalg.set_phi_theta(org_phi, org_the)
shower_core = convert_astropy_array([shower.core_x, shower.core_y])
xi = linalg.angle(dir_fit, shower_org).to(angle_unit)
diff = linalg.length(pos_fit[:2] - shower_core)
# print some performance
print()
print("xi = {:4.3f}".format(xi))
print("pos = {:4.3f}".format(diff))
print("h_max reco: {:4.3f}".format(h_max.to(u.km)))
print("err_est_dir: {:4.3f}".format(err_est_dir.to(angle_unit)))
print("err_est_pos: {:4.3f}".format(err_est_pos))
try:
# store the reconstruction data in the PyTable
reco_event["NTels_trigg"] = n_tels["tot"]
reco_event["NTels_clean"] = len(shower_reco.circles)
reco_event["EnMC"] = event.mc.energy / energy_unit
reco_event["xi"] = xi / angle_unit
reco_event["DeltaR"] = diff / dist_unit
reco_event["ErrEstPos"] = err_est_pos / dist_unit
reco_event["ErrEstDir"] = err_est_dir / angle_unit
reco_event["h_max"] = h_max / dist_unit
reco_event.append()
reco_table.flush()
print()
print("xi res (68-percentile) = {:4.3f} {}".format(
np.percentile(reco_table.cols.xi, 68), angle_unit))
print("core res (68-percentile) = {:4.3f} {}".format(
np.percentile(reco_table.cols.DeltaR, 68), dist_unit))
print("h_max (median) = {:4.3f} {}".format(
np.percentile(reco_table.cols.h_max, 50), dist_unit))
except NoPyTables:
pass
if args.plot_c:
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.gca(projection='3d')
for c in shower_reco.circles.values():
points = [
c.pos + t * c.a * u.km for t in np.linspace(0, 15, 3)
]
ax.plot(*np.array(points).T,
linewidth=np.sqrt(c.weight) / 10)
ax.scatter(*c.pos[:, None].value, s=np.sqrt(c.weight))
plt.xlabel("x")
plt.ylabel("y")
plt.pause(.1)
# this plots
# • the MC shower core
# • the reconstructed shower core
# • the used telescopes
# • and the trace of the Hillas plane on the ground
plt.figure()
for tel_id, c in shower_reco.circles.items():
plt.scatter(c.pos[0], c.pos[1], s=np.sqrt(c.weight))
plt.gca().annotate(tel_id,
(c.pos[0].value, c.pos[1].value))
plt.plot([
c.pos[0].value - 500 * c.norm[1],
c.pos[0].value + 500 * c.norm[1]
], [
c.pos[1].value + 500 * c.norm[0],
c.pos[1].value - 500 * c.norm[0]
],
linewidth=np.sqrt(c.weight) / 10)
plt.scatter(*pos_fit[:2],
c="black",
marker="*",
label="fitted")
plt.scatter(*shower_core[:2],
c="black",
marker="P",
label="MC")
plt.legend()
plt.xlabel("x")
plt.ylabel("y")
plt.xlim(-1400, 1400)
plt.ylim(-1400, 1400)
plt.show()
if signal_handler.stop: break
if signal_handler.stop: break
print("\n" + "=" * 35 + "\n")
print("xi res (68-percentile) = {:4.3f} {}".format(
np.percentile(reco_table.cols.xi, 68), angle_unit))
print("core res (68-percentile) = {:4.3f} {}".format(
np.percentile(reco_table.cols.DeltaR, 68), dist_unit))
print("h_max (median) = {:4.3f} {}".format(
np.percentile(reco_table.cols.h_max, 50), dist_unit))
# print the cutflows for telescopes and camera images
print("\n\n")
Eventcutflow("min2Tels trig")
print()
Imagecutflow(sort_column=1)
# if we don't want to plot anything, we can exit now
if not args.plot:
return
# ######## ## ####### ######## ######
# ## ## ## ## ## ## ## ##
# ## ## ## ## ## ## ##
# ######## ## ## ## ## ######
# ## ## ## ## ## ##
# ## ## ## ## ## ## ##
# ## ######## ####### ## ######
plt.figure()
plt.hist(reco_table.cols.h_max, bins=np.linspace(000, 15000, 51, True))
plt.title(channel)
plt.xlabel("h_max reco")
plt.pause(.1)
figure = plt.figure()
xi_edges = np.linspace(0, 5, 20)
plt.hist(reco_table.cols.xi, bins=xi_edges, log=True)
plt.xlabel(r"$\xi$ / deg")
if args.write:
save_fig('{}/reco_xi_{}'.format(args.plots_dir, args.mode))
plt.pause(.1)
plt.figure()
plt.hist(reco_table.cols.ErrEstDir[:], bins=np.linspace(0, 20, 50))
plt.title(channel)
plt.xlabel("beta")
plt.pause(.1)
plt.figure()
plt.hist(np.log10(reco_table.cols.xi[:] / reco_table.cols.ErrEstDir[:]),
bins=50)
plt.title(channel)
plt.xlabel("log_10(xi / beta)")
plt.pause(.1)
# convert the xi-list into a dict with the number of used telescopes as keys
xi_vs_tel = {}
for xi, ntel in zip(reco_table.cols.xi, reco_table.cols.NTels_clean):
if ntel not in xi_vs_tel:
xi_vs_tel[ntel] = [xi]
else:
xi_vs_tel[ntel].append(xi)
print(args.mode)
for ntel, xis in sorted(xi_vs_tel.items()):
print("NTel: {} -- median xi: {}".format(ntel, np.median(xis)))
# print("histogram:", np.histogram(xis, bins=xi_edges))
# create a list of energy bin-edges and -centres for violin plots
Energy_edges = np.linspace(2, 8, 13)
Energy_centres = (Energy_edges[1:] + Energy_edges[:-1]) / 2.
# convert the xi-list in to an energy-binned dict with the bin centre as keys
xi_vs_energy = {}
for en, xi in zip(reco_table.cols.EnMC, reco_table.cols.xi):
# get the bin number this event belongs into
sbin = np.digitize(np.log10(en), Energy_edges) - 1
# the central value of the bin is the key for the dictionary
if Energy_centres[sbin] not in xi_vs_energy:
xi_vs_energy[Energy_centres[sbin]] = [xi]
else:
xi_vs_energy[Energy_centres[sbin]] += [xi]
# plotting the angular error as violin plots with binning in
# number of telescopes and shower energy
figure = plt.figure()
plt.subplot(211)
plt.violinplot([np.log10(a) for a in xi_vs_tel.values()],
[a for a in xi_vs_tel.keys()],
points=60,
widths=.75,
showextrema=False,
showmedians=True)
plt.xlabel("Number of Telescopes")
plt.ylabel(r"log($\xi$ / deg)")
plt.ylim(-3, 2)
plt.grid()
plt.subplot(212)
plt.violinplot([np.log10(a) for a in xi_vs_energy.values()],
[a for a in xi_vs_energy.keys()],
points=60,
widths=(Energy_edges[1] - Energy_edges[0]) / 1.5,
showextrema=False,
showmedians=True)
plt.xlabel(r"log(Energy / GeV)")
plt.ylabel(r"log($\xi$ / deg)")
plt.ylim(-3, 2)
plt.grid()
plt.tight_layout()
if args.write:
save_fig('{}/reco_xi_vs_E_NTel_{}'.format(args.plots_dir, args.mode))
plt.pause(.1)
# convert the diffs-list into a dict with the number of used telescopes as keys
diff_vs_tel = {}
for diff, ntel in zip(reco_table.cols.DeltaR, reco_table.cols.NTels_clean):
if ntel not in diff_vs_tel:
diff_vs_tel[ntel] = [diff]
else:
diff_vs_tel[ntel].append(diff)
# convert the diffs-list in to an energy-binned dict with the bin centre as keys
diff_vs_energy = {}
for en, diff in zip(reco_table.cols.EnMC, reco_table.cols.DeltaR):
# get the bin number this event belongs into
sbin = np.digitize(np.log10(en), Energy_edges) - 1
# the central value of the bin is the key for the dictionary
if Energy_centres[sbin] not in diff_vs_energy:
diff_vs_energy[Energy_centres[sbin]] = [diff]
else:
diff_vs_energy[Energy_centres[sbin]] += [diff]
# plotting the core position error as violin plots with binning in
# number of telescopes an shower energy
plt.figure()
plt.subplot(211)
plt.violinplot([np.log10(a) for a in diff_vs_tel.values()],
[a for a in diff_vs_tel.keys()],
points=60,
widths=.75,
showextrema=False,
showmedians=True)
plt.xlabel("Number of Telescopes")
plt.ylabel(r"log($\Delta R$ / m)")
plt.grid()
plt.subplot(212)
plt.violinplot([np.log10(a) for a in diff_vs_energy.values()],
[a for a in diff_vs_energy.keys()],
points=60,
widths=(Energy_edges[1] - Energy_edges[0]) / 1.5,
showextrema=False,
showmedians=True)
plt.xlabel(r"log(Energy / GeV)")
plt.ylabel(r"log($\Delta R$ / m)")
plt.grid()
plt.tight_layout()
if args.write:
save_fig('{}/reco_dist_vs_E_NTel_{}'.format(args.plots_dir, args.mode))
plt.show()
def test_invalid_events():
"""
The HillasReconstructor is supposed to fail
in these cases:
- less than two teleskopes
- any width is NaN
- any width is 0
This test uses the same sample simtel file as
test_reconstruction(). As there are no invalid events in this
file, multiple hillas_dicts are constructed to make sure
Exceptions get thrown in the mentioned edge cases.
Test will fail if no Exception or another Exception gets thrown."""
filename = get_dataset_path("gamma_test_large.simtel.gz")
fit = HillasReconstructor()
tel_azimuth = {}
tel_altitude = {}
source = event_source(filename, max_events=10)
for event in source:
hillas_dict = {}
for tel_id in event.dl0.tels_with_data:
geom = event.inst.subarray.tel[tel_id].camera
tel_azimuth[tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
tel_altitude[tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad
pmt_signal = event.r0.tel[tel_id].waveform[0].sum(axis=1)
mask = tailcuts_clean(geom, pmt_signal,
picture_thresh=10., boundary_thresh=5.)
pmt_signal[mask == 0] = 0
try:
moments = hillas_parameters(geom, pmt_signal)
hillas_dict[tel_id] = moments
except HillasParameterizationError as e:
continue
# construct a dict only containing the last telescope events
# (#telescopes < 2)
hillas_dict_only_one_tel = dict()
hillas_dict_only_one_tel[tel_id] = hillas_dict[tel_id]
with pytest.raises(TooFewTelescopesException):
fit.predict(hillas_dict_only_one_tel, event.inst, tel_azimuth, tel_altitude)
# construct a hillas dict with the width of the last event set to 0
# (any width == 0)
hillas_dict_zero_width = hillas_dict.copy()
hillas_dict_zero_width[tel_id]['width'] = 0 * u.m
with pytest.raises(InvalidWidthException):
fit.predict(hillas_dict_zero_width, event.inst, tel_azimuth, tel_altitude)
# construct a hillas dict with the width of the last event set to np.nan
# (any width == nan)
hillas_dict_nan_width = hillas_dict.copy()
hillas_dict_zero_width[tel_id]['width'] = np.nan * u.m
with pytest.raises(InvalidWidthException):
fit.predict(hillas_dict_nan_width, event.inst, tel_azimuth, tel_altitude)
# importing data from avaiable datasets in ctapipe
filename = datasets.get_dataset("gamma_test_large.simtel.gz")
# filename
# reading the Monte Carlo file for LST
source = event_source(filename, allowed_tels={1, 2, 3, 4})
# pointing direction of the telescopes
point_azimuth = {}
point_altitude = {}
reco = HillasReconstructor()
calib = CameraCalibrator(r1_product="HESSIOR1Calibrator")
off_angles = []
for event in source:
# The direction the incident particle.
# Converting Monte Carlo Shower parameter theta and phi to
# corresponding to 3 components (x,y,z) of a vector
shower_azimuth = event.mc.az # same as in Monte Carlo file i.e. phi
shower_altitude = np.pi * u.rad / 2 - event.mc.alt # altitude = 90 - theta
shower_direction = linalg.set_phi_theta(shower_azimuth, shower_altitude)
# calibrating the event
calib.calibrate(event)
hillas_params = {}
subarray = event.inst.subarray
def test_reconstruction():
"""
a test of the complete fit procedure on one event including:
• tailcut cleaning
• hillas parametrisation
• HillasPlane creation
• direction fit
• position fit
in the end, proper units in the output are asserted """
filename = get_dataset_path("gamma_test_large.simtel.gz")
source = event_source(filename, max_events=10)
horizon_frame = AltAz()
reconstructed_events = 0
for event in source:
array_pointing = SkyCoord(
az=event.mc.az,
alt=event.mc.alt,
frame=horizon_frame
)
hillas_dict = {}
telescope_pointings = {}
for tel_id in event.dl0.tels_with_data:
geom = event.inst.subarray.tel[tel_id].camera
telescope_pointings[tel_id] = SkyCoord(alt=event.mc.tel[tel_id].altitude_raw * u.rad,
az=event.mc.tel[tel_id].azimuth_raw * u.rad,
frame=horizon_frame)
pmt_signal = event.r0.tel[tel_id].waveform[0].sum(axis=1)
mask = tailcuts_clean(geom, pmt_signal,
picture_thresh=10., boundary_thresh=5.)
pmt_signal[mask == 0] = 0
try:
moments = hillas_parameters(geom, pmt_signal)
hillas_dict[tel_id] = moments
except HillasParameterizationError as e:
print(e)
continue
if len(hillas_dict) < 2:
continue
else:
reconstructed_events += 1
# The three reconstructions below gives the same results
fit = HillasReconstructor()
fit_result_parall = fit.predict(hillas_dict, event.inst, array_pointing)
fit = HillasReconstructor()
fit_result_tel_point = fit.predict(hillas_dict, event.inst, array_pointing, telescope_pointings)
for key in fit_result_parall.keys():
print(key, fit_result_parall[key], fit_result_tel_point[key])
fit_result_parall.alt.to(u.deg)
fit_result_parall.az.to(u.deg)
fit_result_parall.core_x.to(u.m)
assert fit_result_parall.is_valid
assert reconstructed_events > 0