def fit_origin_crosses(self):
"""calculates the origin of the gamma as the weighted average
direction of the intersections of all great circles
Returns
-------
gamma : shape (3) numpy array
direction of origin of the reconstructed shower as a 3D vector
crossings : shape (n,3) list
list of all the crossings of the `GreatCircle` list
"""
crossings = []
for perm in combinations(self.circles.values(), 2):
n1, n2 = perm[0].norm, perm[1].norm
# cross product automatically weighs in the angle between
# the two vectors: narrower angles have less impact,
# perpendicular vectors have the most
crossing = np.cross(n1, n2)
# two great circles cross each other twice (one would be
# the origin, the other one the direction of the gamma) it
# doesn't matter which we pick but it should at least be
# consistent: make sure to always take the "upper" solution
if crossing[2] < 0:
crossing *= -1
crossings.append(crossing * perm[0].weight * perm[1].weight)
result = linalg.normalise(np.sum(crossings, axis=0)) * u.dimless
off_angles = [linalg.angle(result, cross) / u.rad for cross in crossings]
err_est_dir = np.mean(off_angles) * u.rad
# averaging over the solutions of all permutations
return result, err_est_dir
def MEst(origin, circles, weights):
'''calculates the M-Estimator: a modified χ² that becomes
asymptotically linear for high values and is therefore less
sensitive to outliers
the test is performed to maximise the angles between the fit
direction and the all the normal vectors of the great circles
.. math::
M_{Est} = \sum_i{ 2*\sqrt{1 + \chi^2} - 2}
.. note::
seemingly inferior to negative sum of sin(angle)...
Parameters
-----------
origin : length-3 array
direction vector of the gamma's origin used as seed
circles : GreatCircle array
collection of great circles created from the camera images
weights : array
list of weights for each image/great circle
Returns
-------
MEstimator : float
'''
sin_ang = np.array([
linalg.length(np.cross(origin, circ.norm))
for circ in circles.values()
])
return -2 * np.sum(weights * np.sqrt((1 + np.square(sin_ang))) - 2)
return -np.sum(weights * np.square(sin_ang))
ang = np.array(
[linalg.angle(origin, circ.norm) for circ in circles.values()])
return np.sum(weights * np.sqrt(2. + (ang - np.pi / 2.)**2))
def _MEst(self, origin, circles, weights):
'''
calculates the M-Estimator:
a modified chi2 that becomes asymptotically linear for high values
and is therefore less sensitive to outliers
the test is performed to maximise the angles between the
fit direction and the all the normal vectors of the great circles
Parameters:
-----------
origin : length-3 array
direction vector of the gamma's origin used as seed
circles : GreatCircle array
collection of great circles created from the camera images
weights : array
list of weights for each image/great circle
Returns:
--------
MEstimator : float
Algorithm:
----------
M-Est = sum[ weight * sqrt( 2 * chi**2 ) ]
Note:
-----
seemingly inferior to negative sum of sin(angle)...
'''
sin_ang = np.array([
linalg.length(np.cross(origin, circ.norm))
for circ in circles.values()
])
return np.sum(weights * np.sqrt(2. + (sin_ang - np.pi / 2.)**2))
ang = np.array(
[linalg.angle(origin, circ.norm) for circ in circles.values()])
ang[ang > np.pi / 2.] = np.pi - ang[ang > np.pi / 2]
return np.sum(weights * np.sqrt(2. + (ang - np.pi / 2.)**2))
def MEst(origin, circles, weights):
'''calculates the M-Estimator: a modified χ² that becomes
asymptotically linear for high values and is therefore less
sensitive to outliers
the test is performed to maximise the angles between the fit
direction and the all the normal vectors of the great circles
.. math::
M_{Est} = \sum_i{ 2*\sqrt{1 + \chi^2} - 2}
.. note::
seemingly inferior to negative sum of sin(angle)...
Parameters
-----------
origin : length-3 array
direction vector of the gamma's origin used as seed
circles : GreatCircle array
collection of great circles created from the camera images
weights : array
list of weights for each image/great circle
Returns
-------
MEstimator : float
'''
sin_ang = np.array([linalg.length(np.cross(origin, circ.norm))
for circ in circles.values()])
return -2 * np.sum(weights * np.sqrt((1 + np.square(sin_ang))) - 2)
return -np.sum(weights * np.square(sin_ang))
ang = np.array([linalg.angle(origin, circ.norm)
for circ in circles.values()])
return np.sum(weights * np.sqrt(2. + (ang - np.pi / 2.) ** 2))
def _MEst(self, origin, circles, weights):
'''
calculates the M-Estimator:
a modified chi2 that becomes asymptotically linear for high values
and is therefore less sensitive to outliers
the test is performed to maximise the angles between the
fit direction and the all the normal vectors of the great circles
Parameters:
-----------
origin : length-3 array
direction vector of the gamma's origin used as seed
circles : GreatCircle array
collection of great circles created from the camera images
weights : array
list of weights for each image/great circle
Returns:
--------
MEstimator : float
Algorithm:
----------
M-Est = sum[ weight * sqrt( 2 * chi**2 ) ]
Note:
-----
seemingly inferior to negative sum of sin(angle)...
'''
sin_ang = np.array([linalg.length(np.cross(origin, circ.norm))
for circ in circles.values()])
return np.sum(weights*np.sqrt(2. + (sin_ang-np.pi/2.)**2))
ang = np.array([linalg.angle(origin, circ.norm)
for circ in circles.values()])
ang[ang > np.pi/2.] = np.pi-ang[ang > np.pi/2]
return np.sum(weights*np.sqrt(2. + (ang-np.pi/2.)**2))
def main():
# your favourite units here
energy_unit = u.TeV
angle_unit = u.deg
dist_unit = u.m
parser = make_argparser()
parser.add_argument('-o', '--outfile', type=str,
help="if given, write output file with reconstruction results")
parser.add_argument('--plot_c', action='store_true',
help="plot camera-wise displays")
group = parser.add_mutually_exclusive_group()
group.add_argument('--proton', action='store_true',
help="do protons instead of gammas")
group.add_argument('--electron', action='store_true',
help="do electrons instead of gammas")
args = parser.parse_args()
if args.infile_list:
filenamelist = []
for f in args.infile_list:
filenamelist += glob("{}/{}".format(args.indir, f))
elif args.proton:
filenamelist = glob("{}/proton/*gz".format(args.indir))
channel = "proton"
elif args.electron:
filenamelist = glob("{}/electron/*gz".format(args.indir))
channel = "electron"
elif args.gamma:
filenamelist = glob("{}/gamma/*gz".format(args.indir))
channel = "gamma"
else:
raise ValueError("don't know which input to use...")
filenamelist.sort()
if not filenamelist:
print("no files found; check indir: {}".format(args.indir))
exit(-1)
else:
print("found {} files".format(len(filenamelist)))
tel_phi = {}
tel_theta = {}
# keeping track of events and where they were rejected
Eventcutflow = CutFlow("EventCutFlow")
Imagecutflow = CutFlow("ImageCutFlow")
# takes care of image cleaning
cleaner = ImageCleaner(mode=args.mode, cutflow=Imagecutflow,
wavelet_options=args.raw,
skip_edge_events=args.skip_edge_events, island_cleaning=True)
# the class that does the shower reconstruction
shower_reco = HillasReconstructor()
shower_max_estimator = ShowerMaxEstimator("paranal")
preper = EventPreparer(cleaner=cleaner,
hillas_parameters=hillas_parameters, shower_reco=shower_reco,
event_cutflow=Eventcutflow, image_cutflow=Imagecutflow,
# event/image cuts:
allowed_cam_ids=[], # means: all
min_ntel=3,
min_charge=args.min_charge,
min_pixel=3)
# a signal handler to abort the event loop but still do the post-processing
signal_handler = SignalHandler()
signal.signal(signal.SIGINT, signal_handler)
try:
# this class defines the reconstruction parameters to keep track of
class RecoEvent(tb.IsDescription):
NTels_trigg = tb.Int16Col(dflt=1, pos=0)
NTels_clean = tb.Int16Col(dflt=1, pos=1)
EnMC = tb.Float32Col(dflt=1, pos=2)
xi = tb.Float32Col(dflt=1, pos=3)
DeltaR = tb.Float32Col(dflt=1, pos=4)
ErrEstPos = tb.Float32Col(dflt=1, pos=5)
ErrEstDir = tb.Float32Col(dflt=1, pos=6)
h_max = tb.Float32Col(dflt=1, pos=7)
reco_outfile = tb.open_file(
args.outfile, mode="w",
# if we don't want to write the event list to disk, need to add more arguments
**({} if args.store else {"driver": "H5FD_CORE",
"driver_core_backing_store": False}))
reco_table = reco_outfile.create_table("/", "reco_event", RecoEvent)
reco_event = reco_table.row
except:
reco_event = RecoEvent()
print("no pytables installed?")
# ## ####### ####### ########
# ## ## ## ## ## ## ##
# ## ## ## ## ## ## ##
# ## ## ## ## ## ########
# ## ## ## ## ## ##
# ## ## ## ## ## ##
# ######## ####### ####### ##
cam_id_map = {}
# define here which telescopes to loop over
allowed_tels = None
# allowed_tels = prod3b_tel_ids("L+F+D")
for i, filename in enumerate(filenamelist[:args.last]):
print("file: {i} filename = {filename}".format(i=i, filename=filename))
source = hessio_event_source(filename,
allowed_tels=allowed_tels,
max_events=args.max_events)
# loop that cleans and parametrises the images and performs the reconstruction
for (event, hillas_dict, n_tels,
tot_signal, max_signal, pos_fit, dir_fit, h_max,
err_est_pos, err_est_dir) in preper.prepare_event(source):
shower = event.mc
org_alt = u.Quantity(shower.alt).to(u.deg)
org_az = u.Quantity(shower.az).to(u.deg)
if org_az > 180 * u.deg:
org_az -= 360 * u.deg
org_the = alt_to_theta(org_alt)
org_phi = az_to_phi(org_az)
if org_phi > 180 * u.deg:
org_phi -= 360 * u.deg
if org_phi < -180 * u.deg:
org_phi += 360 * u.deg
shower_org = linalg.set_phi_theta(org_phi, org_the)
shower_core = convert_astropy_array([shower.core_x, shower.core_y])
xi = linalg.angle(dir_fit, shower_org).to(angle_unit)
diff = linalg.length(pos_fit[:2] - shower_core)
# print some performance
print()
print("xi = {:4.3f}".format(xi))
print("pos = {:4.3f}".format(diff))
print("h_max reco: {:4.3f}".format(h_max.to(u.km)))
print("err_est_dir: {:4.3f}".format(err_est_dir.to(angle_unit)))
print("err_est_pos: {:4.3f}".format(err_est_pos))
try:
# store the reconstruction data in the PyTable
reco_event["NTels_trigg"] = n_tels["tot"]
reco_event["NTels_clean"] = len(shower_reco.circles)
reco_event["EnMC"] = event.mc.energy / energy_unit
reco_event["xi"] = xi / angle_unit
reco_event["DeltaR"] = diff / dist_unit
reco_event["ErrEstPos"] = err_est_pos / dist_unit
reco_event["ErrEstDir"] = err_est_dir / angle_unit
reco_event["h_max"] = h_max / dist_unit
reco_event.append()
reco_table.flush()
print()
print("xi res (68-percentile) = {:4.3f} {}"
.format(np.percentile(reco_table.cols.xi, 68), angle_unit))
print("core res (68-percentile) = {:4.3f} {}"
.format(np.percentile(reco_table.cols.DeltaR, 68), dist_unit))
print("h_max (median) = {:4.3f} {}"
.format(np.percentile(reco_table.cols.h_max, 50), dist_unit))
except NoPyTables:
pass
if args.plot_c:
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.gca(projection='3d')
for c in shower_reco.circles.values():
points = [c.pos + t * c.a * u.km for t in np.linspace(0, 15, 3)]
ax.plot(*np.array(points).T, linewidth=np.sqrt(c.weight) / 10)
ax.scatter(*c.pos[:, None].value, s=np.sqrt(c.weight))
plt.xlabel("x")
plt.ylabel("y")
plt.pause(.1)
# this plots
# • the MC shower core
# • the reconstructed shower core
# • the used telescopes
# • and the trace of the Hillas plane on the ground
plt.figure()
for tel_id, c in shower_reco.circles.items():
plt.scatter(c.pos[0], c.pos[1], s=np.sqrt(c.weight))
plt.gca().annotate(tel_id, (c.pos[0].value, c.pos[1].value))
plt.plot([c.pos[0].value-500*c.norm[1], c.pos[0].value+500*c.norm[1]],
[c.pos[1].value+500*c.norm[0], c.pos[1].value-500*c.norm[0]],
linewidth=np.sqrt(c.weight)/10)
plt.scatter(*pos_fit[:2], c="black", marker="*", label="fitted")
plt.scatter(*shower_core[:2], c="black", marker="P", label="MC")
plt.legend()
plt.xlabel("x")
plt.ylabel("y")
plt.xlim(-1400, 1400)
plt.ylim(-1400, 1400)
plt.show()
if signal_handler.stop: break
if signal_handler.stop: break
print("\n" + "="*35 + "\n")
print("xi res (68-percentile) = {:4.3f} {}"
.format(np.percentile(reco_table.cols.xi, 68), angle_unit))
print("core res (68-percentile) = {:4.3f} {}"
.format(np.percentile(reco_table.cols.DeltaR, 68), dist_unit))
print("h_max (median) = {:4.3f} {}"
.format(np.percentile(reco_table.cols.h_max, 50), dist_unit))
# print the cutflows for telescopes and camera images
print("\n\n")
Eventcutflow("min2Tels trig")
print()
Imagecutflow(sort_column=1)
# if we don't want to plot anything, we can exit now
if not args.plot:
return
# ######## ## ####### ######## ######
# ## ## ## ## ## ## ## ##
# ## ## ## ## ## ## ##
# ######## ## ## ## ## ######
# ## ## ## ## ## ##
# ## ## ## ## ## ## ##
# ## ######## ####### ## ######
plt.figure()
plt.hist(reco_table.cols.h_max, bins=np.linspace(000, 15000, 51, True))
plt.title(channel)
plt.xlabel("h_max reco")
plt.pause(.1)
figure = plt.figure()
xi_edges = np.linspace(0, 5, 20)
plt.hist(reco_table.cols.xi, bins=xi_edges, log=True)
plt.xlabel(r"$\xi$ / deg")
if args.write:
save_fig('{}/reco_xi_{}'.format(args.plots_dir, args.mode))
plt.pause(.1)
plt.figure()
plt.hist(reco_table.cols.ErrEstDir[:],
bins=np.linspace(0, 20, 50))
plt.title(channel)
plt.xlabel("beta")
plt.pause(.1)
plt.figure()
plt.hist(np.log10(reco_table.cols.xi[:] / reco_table.cols.ErrEstDir[:]), bins=50)
plt.title(channel)
plt.xlabel("log_10(xi / beta)")
plt.pause(.1)
# convert the xi-list into a dict with the number of used telescopes as keys
xi_vs_tel = {}
for xi, ntel in zip(reco_table.cols.xi, reco_table.cols.NTels_clean):
if ntel not in xi_vs_tel:
xi_vs_tel[ntel] = [xi]
else:
xi_vs_tel[ntel].append(xi)
print(args.mode)
for ntel, xis in sorted(xi_vs_tel.items()):
print("NTel: {} -- median xi: {}".format(ntel, np.median(xis)))
# print("histogram:", np.histogram(xis, bins=xi_edges))
# create a list of energy bin-edges and -centres for violin plots
Energy_edges = np.linspace(2, 8, 13)
Energy_centres = (Energy_edges[1:]+Energy_edges[:-1])/2.
# convert the xi-list in to an energy-binned dict with the bin centre as keys
xi_vs_energy = {}
for en, xi in zip(reco_table.cols.EnMC, reco_table.cols.xi):
# get the bin number this event belongs into
sbin = np.digitize(np.log10(en), Energy_edges)-1
# the central value of the bin is the key for the dictionary
if Energy_centres[sbin] not in xi_vs_energy:
xi_vs_energy[Energy_centres[sbin]] = [xi]
else:
xi_vs_energy[Energy_centres[sbin]] += [xi]
# plotting the angular error as violin plots with binning in
# number of telescopes and shower energy
figure = plt.figure()
plt.subplot(211)
plt.violinplot([np.log10(a) for a in xi_vs_tel.values()],
[a for a in xi_vs_tel.keys()],
points=60, widths=.75, showextrema=False, showmedians=True)
plt.xlabel("Number of Telescopes")
plt.ylabel(r"log($\xi$ / deg)")
plt.ylim(-3, 2)
plt.grid()
plt.subplot(212)
plt.violinplot([np.log10(a) for a in xi_vs_energy.values()],
[a for a in xi_vs_energy.keys()],
points=60, widths=(Energy_edges[1] - Energy_edges[0]) / 1.5,
showextrema=False, showmedians=True)
plt.xlabel(r"log(Energy / GeV)")
plt.ylabel(r"log($\xi$ / deg)")
plt.ylim(-3, 2)
plt.grid()
plt.tight_layout()
if args.write:
save_fig('{}/reco_xi_vs_E_NTel_{}'.format(args.plots_dir, args.mode))
plt.pause(.1)
# convert the diffs-list into a dict with the number of used telescopes as keys
diff_vs_tel = {}
for diff, ntel in zip(reco_table.cols.DeltaR, reco_table.cols.NTels_clean):
if ntel not in diff_vs_tel:
diff_vs_tel[ntel] = [diff]
else:
diff_vs_tel[ntel].append(diff)
# convert the diffs-list in to an energy-binned dict with the bin centre as keys
diff_vs_energy = {}
for en, diff in zip(reco_table.cols.EnMC, reco_table.cols.DeltaR):
# get the bin number this event belongs into
sbin = np.digitize(np.log10(en), Energy_edges) - 1
# the central value of the bin is the key for the dictionary
if Energy_centres[sbin] not in diff_vs_energy:
diff_vs_energy[Energy_centres[sbin]] = [diff]
else:
diff_vs_energy[Energy_centres[sbin]] += [diff]
# plotting the core position error as violin plots with binning in
# number of telescopes an shower energy
plt.figure()
plt.subplot(211)
plt.violinplot([np.log10(a) for a in diff_vs_tel.values()],
[a for a in diff_vs_tel.keys()],
points=60, widths=.75, showextrema=False, showmedians=True)
plt.xlabel("Number of Telescopes")
plt.ylabel(r"log($\Delta R$ / m)")
plt.grid()
plt.subplot(212)
plt.violinplot([np.log10(a) for a in diff_vs_energy.values()],
[a for a in diff_vs_energy.keys()],
points=60, widths=(Energy_edges[1] - Energy_edges[0]) / 1.5,
showextrema=False, showmedians=True)
plt.xlabel(r"log(Energy / GeV)")
plt.ylabel(r"log($\Delta R$ / m)")
plt.grid()
plt.tight_layout()
if args.write:
save_fig('{}/reco_dist_vs_E_NTel_{}'.format(args.plots_dir, args.mode))
plt.show()
def main():
# your favourite units here
energy_unit = u.TeV
angle_unit = u.deg
dist_unit = u.m
agree_threshold = .5
min_tel = 3
parser = make_argparser()
parser.add_argument('--classifier',
type=str,
default=expandvars(
"$CTA_SOFT/tino_cta/data/classifier_pickle/"
"classifier_{mode}_{cam_id}_{classifier}.pkl"))
parser.add_argument('--regressor',
type=str,
default=expandvars(
"$CTA_SOFT/tino_cta/data/classifier_pickle/"
"regressor_{mode}_{cam_id}_{regressor}.pkl"))
parser.add_argument('-o',
'--outfile',
type=str,
default="",
help="location to write the classified events to.")
parser.add_argument('--wave_dir',
type=str,
default=None,
help="directory where to find mr_filter. "
"if not set look in $PATH")
parser.add_argument(
'--wave_temp_dir',
type=str,
default='/dev/shm/',
help="directory where mr_filter to store the temporary fits "
"files")
group = parser.add_mutually_exclusive_group()
group.add_argument('--proton',
action='store_true',
help="do protons instead of gammas")
group.add_argument('--electron',
action='store_true',
help="do electrons instead of gammas")
args = parser.parse_args()
if args.infile_list:
filenamelist = []
for f in args.infile_list:
filenamelist += glob("{}/{}".format(args.indir, f))
filenamelist.sort()
elif args.proton:
filenamelist = sorted(glob("{}/proton/*gz".format(args.indir)))
elif args.electron:
filenamelist = glob("{}/electron/*gz".format(args.indir))
channel = "electron"
else:
filenamelist = sorted(glob("{}/gamma/*gz".format(args.indir)))
if not filenamelist:
print("no files found; check indir: {}".format(args.indir))
exit(-1)
# keeping track of events and where they were rejected
Eventcutflow = CutFlow("EventCutFlow")
Imagecutflow = CutFlow("ImageCutFlow")
# takes care of image cleaning
cleaner = ImageCleaner(mode=args.mode,
cutflow=Imagecutflow,
wavelet_options=args.raw,
tmp_files_directory=args.wave_temp_dir,
skip_edge_events=False,
island_cleaning=True)
# the class that does the shower reconstruction
shower_reco = HillasReconstructor()
preper = EventPreparer(
cleaner=cleaner,
hillas_parameters=hillas_parameters,
shower_reco=shower_reco,
event_cutflow=Eventcutflow,
image_cutflow=Imagecutflow,
# event/image cuts:
allowed_cam_ids=[],
min_ntel=2,
min_charge=args.min_charge,
min_pixel=3)
# wrapper for the scikit-learn classifier
classifier = EventClassifier.load(args.classifier.format(
**{
"mode": args.mode,
"wave_args": "mixed",
"classifier": 'RandomForestClassifier',
"cam_id": "{cam_id}"
}),
cam_id_list=args.cam_ids)
# wrapper for the scikit-learn regressor
regressor = EnergyRegressor.load(args.regressor.format(
**{
"mode": args.mode,
"wave_args": "mixed",
"regressor": "RandomForestRegressor",
"cam_id": "{cam_id}"
}),
cam_id_list=args.cam_ids)
ClassifierFeatures = namedtuple(
"ClassifierFeatures",
("impact_dist", "sum_signal_evt", "max_signal_cam", "sum_signal_cam",
"N_LST", "N_MST", "N_SST", "width", "length", "skewness", "kurtosis",
"h_max", "err_est_pos", "err_est_dir"))
EnergyFeatures = namedtuple(
"EnergyFeatures",
("impact_dist", "sum_signal_evt", "max_signal_cam", "sum_signal_cam",
"N_LST", "N_MST", "N_SST", "width", "length", "skewness", "kurtosis",
"h_max", "err_est_pos", "err_est_dir"))
# catch ctr-c signal to exit current loop and still display results
signal_handler = SignalHandler()
signal.signal(signal.SIGINT, signal_handler)
# this class defines the reconstruction parameters to keep track of
class RecoEvent(tb.IsDescription):
Run_ID = tb.Int16Col(dflt=-1, pos=0)
Event_ID = tb.Int16Col(dflt=-1, pos=1)
NTels_trig = tb.Int16Col(dflt=0, pos=0)
NTels_reco = tb.Int16Col(dflt=0, pos=1)
NTels_reco_lst = tb.Int16Col(dflt=0, pos=2)
NTels_reco_mst = tb.Int16Col(dflt=0, pos=3)
NTels_reco_sst = tb.Int16Col(dflt=0, pos=4)
MC_Energy = tb.Float32Col(dflt=np.nan, pos=5)
reco_Energy = tb.Float32Col(dflt=np.nan, pos=6)
reco_phi = tb.Float32Col(dflt=np.nan, pos=7)
reco_theta = tb.Float32Col(dflt=np.nan, pos=8)
off_angle = tb.Float32Col(dflt=np.nan, pos=9)
xi = tb.Float32Col(dflt=np.nan, pos=10)
DeltaR = tb.Float32Col(dflt=np.nan, pos=11)
ErrEstPos = tb.Float32Col(dflt=np.nan, pos=12)
ErrEstDir = tb.Float32Col(dflt=np.nan, pos=13)
gammaness = tb.Float32Col(dflt=np.nan, pos=14)
success = tb.BoolCol(dflt=False, pos=15)
channel = "gamma" if "gamma" in " ".join(filenamelist) else "proton"
reco_outfile = tb.open_file(
mode="w",
# if no outfile name is given (i.e. don't to write the event list to disk),
# need specify two "driver" arguments
**({
"filename": args.outfile
} if args.outfile else {
"filename": "no_outfile.h5",
"driver": "H5FD_CORE",
"driver_core_backing_store": False
}))
reco_table = reco_outfile.create_table("/", "reco_events", RecoEvent)
reco_event = reco_table.row
allowed_tels = None # all telescopes
allowed_tels = prod3b_tel_ids("L+N+D")
for i, filename in enumerate(filenamelist[:args.last]):
# print(f"file: {i} filename = {filename}")
source = hessio_event_source(filename,
allowed_tels=allowed_tels,
max_events=args.max_events)
# loop that cleans and parametrises the images and performs the reconstruction
for (event, hillas_dict, n_tels, tot_signal, max_signals, pos_fit,
dir_fit, h_max, err_est_pos,
err_est_dir) in preper.prepare_event(source, True):
# now prepare the features for the classifier
cls_features_evt = {}
reg_features_evt = {}
if hillas_dict is not None:
for tel_id in hillas_dict.keys():
Imagecutflow.count("pre-features")
tel_pos = np.array(event.inst.tel_pos[tel_id][:2]) * u.m
moments = hillas_dict[tel_id]
impact_dist = linalg.length(tel_pos - pos_fit)
cls_features_tel = ClassifierFeatures(
impact_dist=impact_dist / u.m,
sum_signal_evt=tot_signal,
max_signal_cam=max_signals[tel_id],
sum_signal_cam=moments.size,
N_LST=n_tels["LST"],
N_MST=n_tels["MST"],
N_SST=n_tels["SST"],
width=moments.width / u.m,
length=moments.length / u.m,
skewness=moments.skewness,
kurtosis=moments.kurtosis,
h_max=h_max / u.m,
err_est_pos=err_est_pos / u.m,
err_est_dir=err_est_dir / u.deg)
reg_features_tel = EnergyFeatures(
impact_dist=impact_dist / u.m,
sum_signal_evt=tot_signal,
max_signal_cam=max_signals[tel_id],
sum_signal_cam=moments.size,
N_LST=n_tels["LST"],
N_MST=n_tels["MST"],
N_SST=n_tels["SST"],
width=moments.width / u.m,
length=moments.length / u.m,
skewness=moments.skewness,
kurtosis=moments.kurtosis,
h_max=h_max / u.m,
err_est_pos=err_est_pos / u.m,
err_est_dir=err_est_dir / u.deg)
if np.isnan(cls_features_tel).any() or np.isnan(
reg_features_tel).any():
continue
Imagecutflow.count("features nan")
cam_id = event.inst.subarray.tel[tel_id].camera.cam_id
try:
reg_features_evt[cam_id] += [reg_features_tel]
cls_features_evt[cam_id] += [cls_features_tel]
except KeyError:
reg_features_evt[cam_id] = [reg_features_tel]
cls_features_evt[cam_id] = [cls_features_tel]
if cls_features_evt and reg_features_evt:
predict_energ = regressor.predict_by_event([reg_features_evt
])["mean"][0]
predict_proba = classifier.predict_proba_by_event(
[cls_features_evt])
gammaness = predict_proba[0, 0]
try:
# the MC direction of origin of the simulated particle
shower = event.mc
shower_core = np.array(
[shower.core_x / u.m, shower.core_y / u.m]) * u.m
shower_org = linalg.set_phi_theta(az_to_phi(shower.az),
alt_to_theta(shower.alt))
# and how the reconstructed direction compares to that
xi = linalg.angle(dir_fit, shower_org)
DeltaR = linalg.length(pos_fit[:2] - shower_core)
except Exception:
# naked exception catch, because I'm not sure where
# it would break in non-MC files
xi = np.nan
DeltaR = np.nan
phi, theta = linalg.get_phi_theta(dir_fit)
phi = (phi if phi > 0 else phi + 360 * u.deg)
# TODO: replace with actual array pointing direction
array_pointing = linalg.set_phi_theta(0 * u.deg, 20. * u.deg)
# angular offset between the reconstructed direction and the array
# pointing
off_angle = linalg.angle(dir_fit, array_pointing)
reco_event["NTels_trig"] = len(event.dl0.tels_with_data)
reco_event["NTels_reco"] = len(hillas_dict)
reco_event["NTels_reco_lst"] = n_tels["LST"]
reco_event["NTels_reco_mst"] = n_tels["MST"]
reco_event["NTels_reco_sst"] = n_tels["SST"]
reco_event["reco_Energy"] = predict_energ.to(energy_unit).value
reco_event["reco_phi"] = phi / angle_unit
reco_event["reco_theta"] = theta / angle_unit
reco_event["off_angle"] = off_angle / angle_unit
reco_event["xi"] = xi / angle_unit
reco_event["DeltaR"] = DeltaR / dist_unit
reco_event["ErrEstPos"] = err_est_pos / dist_unit
reco_event["ErrEstDir"] = err_est_dir / angle_unit
reco_event["gammaness"] = gammaness
reco_event["success"] = True
else:
reco_event["success"] = False
# save basic event infos
reco_event["MC_Energy"] = event.mc.energy.to(energy_unit).value
reco_event["Event_ID"] = event.r1.event_id
reco_event["Run_ID"] = event.r1.run_id
reco_table.flush()
reco_event.append()
if signal_handler.stop:
break
if signal_handler.stop:
break
# make sure everything gets written out nicely
reco_table.flush()
try:
print()
Eventcutflow()
print()
Imagecutflow()
# do some simple event selection
# and print the corresponding selection efficiency
N_selected = len([
x for x in reco_table.where(
"""(NTels_reco > min_tel) & (gammaness > agree_threshold)""")
])
N_total = len(reco_table)
print("\nfraction selected events:")
print("{} / {} = {} %".format(N_selected, N_total,
N_selected / N_total * 100))
except ZeroDivisionError:
pass
print("\nlength filenamelist:", len(filenamelist[:args.last]))
# do some plotting if so desired
if args.plot:
gammaness = [x['gammaness'] for x in reco_table]
NTels_rec = [x['NTels_reco'] for x in reco_table]
NTel_bins = np.arange(np.min(NTels_rec), np.max(NTels_rec) + 2) - .5
NTels_rec_lst = [x['NTels_reco_lst'] for x in reco_table]
NTels_rec_mst = [x['NTels_reco_mst'] for x in reco_table]
NTels_rec_sst = [x['NTels_reco_sst'] for x in reco_table]
reco_energy = np.array([x['reco_Energy'] for x in reco_table])
mc_energy = np.array([x['MC_Energy'] for x in reco_table])
fig = plt.figure(figsize=(15, 5))
plt.suptitle(" ** ".join(
[args.mode, "protons" if args.proton else "gamma"]))
plt.subplots_adjust(left=0.05, right=0.97, hspace=0.39, wspace=0.2)
ax = plt.subplot(131)
histo = np.histogram2d(NTels_rec,
gammaness,
bins=(NTel_bins, np.linspace(0, 1, 11)))[0].T
histo_normed = histo / histo.max(axis=0)
im = ax.imshow(
histo_normed,
interpolation='none',
origin='lower',
aspect='auto',
# extent=(*NTel_bins[[0, -1]], 0, 1),
cmap=plt.cm.inferno)
ax.set_xlabel("NTels")
ax.set_ylabel("drifted gammaness")
plt.title("Total Number of Telescopes")
# next subplot
ax = plt.subplot(132)
histo = np.histogram2d(NTels_rec_sst,
gammaness,
bins=(NTel_bins, np.linspace(0, 1, 11)))[0].T
histo_normed = histo / histo.max(axis=0)
im = ax.imshow(
histo_normed,
interpolation='none',
origin='lower',
aspect='auto',
# extent=(*NTel_bins[[0, -1]], 0, 1),
cmap=plt.cm.inferno)
ax.set_xlabel("NTels")
plt.setp(ax.get_yticklabels(), visible=False)
plt.title("Number of SSTs")
# next subplot
ax = plt.subplot(133)
histo = np.histogram2d(NTels_rec_mst,
gammaness,
bins=(NTel_bins, np.linspace(0, 1, 11)))[0].T
histo_normed = histo / histo.max(axis=0)
im = ax.imshow(
histo_normed,
interpolation='none',
origin='lower',
aspect='auto',
# extent=(*NTel_bins[[0, -1]], 0, 1),
cmap=plt.cm.inferno)
cb = fig.colorbar(im, ax=ax)
ax.set_xlabel("NTels")
plt.setp(ax.get_yticklabels(), visible=False)
plt.title("Number of MSTs")
plt.subplots_adjust(wspace=0.05)
# plot the energy migration matrix
plt.figure()
plt.hist2d(np.log10(reco_energy),
np.log10(mc_energy),
bins=20,
cmap=plt.cm.inferno)
plt.xlabel("E_MC / TeV")
plt.ylabel("E_rec / TeV")
plt.colorbar()
plt.show()
def main():
# your favourite units here
energy_unit = u.TeV
angle_unit = u.deg
dist_unit = u.m
parser = make_argparser()
parser.add_argument(
'-o',
'--outfile',
type=str,
help="if given, write output file with reconstruction results")
parser.add_argument('--plot_c',
action='store_true',
help="plot camera-wise displays")
group = parser.add_mutually_exclusive_group()
group.add_argument('--proton',
action='store_true',
help="do protons instead of gammas")
group.add_argument('--electron',
action='store_true',
help="do electrons instead of gammas")
args = parser.parse_args()
if args.infile_list:
filenamelist = []
for f in args.infile_list:
filenamelist += glob("{}/{}".format(args.indir, f))
elif args.proton:
filenamelist = glob("{}/proton/*gz".format(args.indir))
channel = "proton"
elif args.electron:
filenamelist = glob("{}/electron/*gz".format(args.indir))
channel = "electron"
elif args.gamma:
filenamelist = glob("{}/gamma/*gz".format(args.indir))
channel = "gamma"
else:
raise ValueError("don't know which input to use...")
filenamelist.sort()
if not filenamelist:
print("no files found; check indir: {}".format(args.indir))
exit(-1)
else:
print("found {} files".format(len(filenamelist)))
tel_phi = {}
tel_theta = {}
# keeping track of events and where they were rejected
Eventcutflow = CutFlow("EventCutFlow")
Imagecutflow = CutFlow("ImageCutFlow")
# takes care of image cleaning
cleaner = ImageCleaner(mode=args.mode,
cutflow=Imagecutflow,
wavelet_options=args.raw,
skip_edge_events=args.skip_edge_events,
island_cleaning=True)
# the class that does the shower reconstruction
shower_reco = HillasReconstructor()
shower_max_estimator = ShowerMaxEstimator("paranal")
preper = EventPreparer(
cleaner=cleaner,
hillas_parameters=hillas_parameters,
shower_reco=shower_reco,
event_cutflow=Eventcutflow,
image_cutflow=Imagecutflow,
# event/image cuts:
allowed_cam_ids=[], # means: all
min_ntel=3,
min_charge=args.min_charge,
min_pixel=3)
# a signal handler to abort the event loop but still do the post-processing
signal_handler = SignalHandler()
signal.signal(signal.SIGINT, signal_handler)
try:
# this class defines the reconstruction parameters to keep track of
class RecoEvent(tb.IsDescription):
NTels_trigg = tb.Int16Col(dflt=1, pos=0)
NTels_clean = tb.Int16Col(dflt=1, pos=1)
EnMC = tb.Float32Col(dflt=1, pos=2)
xi = tb.Float32Col(dflt=1, pos=3)
DeltaR = tb.Float32Col(dflt=1, pos=4)
ErrEstPos = tb.Float32Col(dflt=1, pos=5)
ErrEstDir = tb.Float32Col(dflt=1, pos=6)
h_max = tb.Float32Col(dflt=1, pos=7)
reco_outfile = tb.open_file(
args.outfile,
mode="w",
# if we don't want to write the event list to disk, need to add more arguments
**({} if args.store else {
"driver": "H5FD_CORE",
"driver_core_backing_store": False
}))
reco_table = reco_outfile.create_table("/", "reco_event", RecoEvent)
reco_event = reco_table.row
except:
reco_event = RecoEvent()
print("no pytables installed?")
# ## ####### ####### ########
# ## ## ## ## ## ## ##
# ## ## ## ## ## ## ##
# ## ## ## ## ## ########
# ## ## ## ## ## ##
# ## ## ## ## ## ##
# ######## ####### ####### ##
cam_id_map = {}
# define here which telescopes to loop over
allowed_tels = None
# allowed_tels = prod3b_tel_ids("L+F+D")
for i, filename in enumerate(filenamelist[:args.last]):
print("file: {i} filename = {filename}".format(i=i, filename=filename))
source = hessio_event_source(filename,
allowed_tels=allowed_tels,
max_events=args.max_events)
# loop that cleans and parametrises the images and performs the reconstruction
for (event, hillas_dict, n_tels, tot_signal, max_signal, pos_fit,
dir_fit, h_max, err_est_pos,
err_est_dir) in preper.prepare_event(source):
shower = event.mc
org_alt = u.Quantity(shower.alt).to(u.deg)
org_az = u.Quantity(shower.az).to(u.deg)
if org_az > 180 * u.deg:
org_az -= 360 * u.deg
org_the = alt_to_theta(org_alt)
org_phi = az_to_phi(org_az)
if org_phi > 180 * u.deg:
org_phi -= 360 * u.deg
if org_phi < -180 * u.deg:
org_phi += 360 * u.deg
shower_org = linalg.set_phi_theta(org_phi, org_the)
shower_core = convert_astropy_array([shower.core_x, shower.core_y])
xi = linalg.angle(dir_fit, shower_org).to(angle_unit)
diff = linalg.length(pos_fit[:2] - shower_core)
# print some performance
print()
print("xi = {:4.3f}".format(xi))
print("pos = {:4.3f}".format(diff))
print("h_max reco: {:4.3f}".format(h_max.to(u.km)))
print("err_est_dir: {:4.3f}".format(err_est_dir.to(angle_unit)))
print("err_est_pos: {:4.3f}".format(err_est_pos))
try:
# store the reconstruction data in the PyTable
reco_event["NTels_trigg"] = n_tels["tot"]
reco_event["NTels_clean"] = len(shower_reco.circles)
reco_event["EnMC"] = event.mc.energy / energy_unit
reco_event["xi"] = xi / angle_unit
reco_event["DeltaR"] = diff / dist_unit
reco_event["ErrEstPos"] = err_est_pos / dist_unit
reco_event["ErrEstDir"] = err_est_dir / angle_unit
reco_event["h_max"] = h_max / dist_unit
reco_event.append()
reco_table.flush()
print()
print("xi res (68-percentile) = {:4.3f} {}".format(
np.percentile(reco_table.cols.xi, 68), angle_unit))
print("core res (68-percentile) = {:4.3f} {}".format(
np.percentile(reco_table.cols.DeltaR, 68), dist_unit))
print("h_max (median) = {:4.3f} {}".format(
np.percentile(reco_table.cols.h_max, 50), dist_unit))
except NoPyTables:
pass
if args.plot_c:
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.gca(projection='3d')
for c in shower_reco.circles.values():
points = [
c.pos + t * c.a * u.km for t in np.linspace(0, 15, 3)
]
ax.plot(*np.array(points).T,
linewidth=np.sqrt(c.weight) / 10)
ax.scatter(*c.pos[:, None].value, s=np.sqrt(c.weight))
plt.xlabel("x")
plt.ylabel("y")
plt.pause(.1)
# this plots
# • the MC shower core
# • the reconstructed shower core
# • the used telescopes
# • and the trace of the Hillas plane on the ground
plt.figure()
for tel_id, c in shower_reco.circles.items():
plt.scatter(c.pos[0], c.pos[1], s=np.sqrt(c.weight))
plt.gca().annotate(tel_id,
(c.pos[0].value, c.pos[1].value))
plt.plot([
c.pos[0].value - 500 * c.norm[1],
c.pos[0].value + 500 * c.norm[1]
], [
c.pos[1].value + 500 * c.norm[0],
c.pos[1].value - 500 * c.norm[0]
],
linewidth=np.sqrt(c.weight) / 10)
plt.scatter(*pos_fit[:2],
c="black",
marker="*",
label="fitted")
plt.scatter(*shower_core[:2],
c="black",
marker="P",
label="MC")
plt.legend()
plt.xlabel("x")
plt.ylabel("y")
plt.xlim(-1400, 1400)
plt.ylim(-1400, 1400)
plt.show()
if signal_handler.stop: break
if signal_handler.stop: break
print("\n" + "=" * 35 + "\n")
print("xi res (68-percentile) = {:4.3f} {}".format(
np.percentile(reco_table.cols.xi, 68), angle_unit))
print("core res (68-percentile) = {:4.3f} {}".format(
np.percentile(reco_table.cols.DeltaR, 68), dist_unit))
print("h_max (median) = {:4.3f} {}".format(
np.percentile(reco_table.cols.h_max, 50), dist_unit))
# print the cutflows for telescopes and camera images
print("\n\n")
Eventcutflow("min2Tels trig")
print()
Imagecutflow(sort_column=1)
# if we don't want to plot anything, we can exit now
if not args.plot:
return
# ######## ## ####### ######## ######
# ## ## ## ## ## ## ## ##
# ## ## ## ## ## ## ##
# ######## ## ## ## ## ######
# ## ## ## ## ## ##
# ## ## ## ## ## ## ##
# ## ######## ####### ## ######
plt.figure()
plt.hist(reco_table.cols.h_max, bins=np.linspace(000, 15000, 51, True))
plt.title(channel)
plt.xlabel("h_max reco")
plt.pause(.1)
figure = plt.figure()
xi_edges = np.linspace(0, 5, 20)
plt.hist(reco_table.cols.xi, bins=xi_edges, log=True)
plt.xlabel(r"$\xi$ / deg")
if args.write:
save_fig('{}/reco_xi_{}'.format(args.plots_dir, args.mode))
plt.pause(.1)
plt.figure()
plt.hist(reco_table.cols.ErrEstDir[:], bins=np.linspace(0, 20, 50))
plt.title(channel)
plt.xlabel("beta")
plt.pause(.1)
plt.figure()
plt.hist(np.log10(reco_table.cols.xi[:] / reco_table.cols.ErrEstDir[:]),
bins=50)
plt.title(channel)
plt.xlabel("log_10(xi / beta)")
plt.pause(.1)
# convert the xi-list into a dict with the number of used telescopes as keys
xi_vs_tel = {}
for xi, ntel in zip(reco_table.cols.xi, reco_table.cols.NTels_clean):
if ntel not in xi_vs_tel:
xi_vs_tel[ntel] = [xi]
else:
xi_vs_tel[ntel].append(xi)
print(args.mode)
for ntel, xis in sorted(xi_vs_tel.items()):
print("NTel: {} -- median xi: {}".format(ntel, np.median(xis)))
# print("histogram:", np.histogram(xis, bins=xi_edges))
# create a list of energy bin-edges and -centres for violin plots
Energy_edges = np.linspace(2, 8, 13)
Energy_centres = (Energy_edges[1:] + Energy_edges[:-1]) / 2.
# convert the xi-list in to an energy-binned dict with the bin centre as keys
xi_vs_energy = {}
for en, xi in zip(reco_table.cols.EnMC, reco_table.cols.xi):
# get the bin number this event belongs into
sbin = np.digitize(np.log10(en), Energy_edges) - 1
# the central value of the bin is the key for the dictionary
if Energy_centres[sbin] not in xi_vs_energy:
xi_vs_energy[Energy_centres[sbin]] = [xi]
else:
xi_vs_energy[Energy_centres[sbin]] += [xi]
# plotting the angular error as violin plots with binning in
# number of telescopes and shower energy
figure = plt.figure()
plt.subplot(211)
plt.violinplot([np.log10(a) for a in xi_vs_tel.values()],
[a for a in xi_vs_tel.keys()],
points=60,
widths=.75,
showextrema=False,
showmedians=True)
plt.xlabel("Number of Telescopes")
plt.ylabel(r"log($\xi$ / deg)")
plt.ylim(-3, 2)
plt.grid()
plt.subplot(212)
plt.violinplot([np.log10(a) for a in xi_vs_energy.values()],
[a for a in xi_vs_energy.keys()],
points=60,
widths=(Energy_edges[1] - Energy_edges[0]) / 1.5,
showextrema=False,
showmedians=True)
plt.xlabel(r"log(Energy / GeV)")
plt.ylabel(r"log($\xi$ / deg)")
plt.ylim(-3, 2)
plt.grid()
plt.tight_layout()
if args.write:
save_fig('{}/reco_xi_vs_E_NTel_{}'.format(args.plots_dir, args.mode))
plt.pause(.1)
# convert the diffs-list into a dict with the number of used telescopes as keys
diff_vs_tel = {}
for diff, ntel in zip(reco_table.cols.DeltaR, reco_table.cols.NTels_clean):
if ntel not in diff_vs_tel:
diff_vs_tel[ntel] = [diff]
else:
diff_vs_tel[ntel].append(diff)
# convert the diffs-list in to an energy-binned dict with the bin centre as keys
diff_vs_energy = {}
for en, diff in zip(reco_table.cols.EnMC, reco_table.cols.DeltaR):
# get the bin number this event belongs into
sbin = np.digitize(np.log10(en), Energy_edges) - 1
# the central value of the bin is the key for the dictionary
if Energy_centres[sbin] not in diff_vs_energy:
diff_vs_energy[Energy_centres[sbin]] = [diff]
else:
diff_vs_energy[Energy_centres[sbin]] += [diff]
# plotting the core position error as violin plots with binning in
# number of telescopes an shower energy
plt.figure()
plt.subplot(211)
plt.violinplot([np.log10(a) for a in diff_vs_tel.values()],
[a for a in diff_vs_tel.keys()],
points=60,
widths=.75,
showextrema=False,
showmedians=True)
plt.xlabel("Number of Telescopes")
plt.ylabel(r"log($\Delta R$ / m)")
plt.grid()
plt.subplot(212)
plt.violinplot([np.log10(a) for a in diff_vs_energy.values()],
[a for a in diff_vs_energy.keys()],
points=60,
widths=(Energy_edges[1] - Energy_edges[0]) / 1.5,
showextrema=False,
showmedians=True)
plt.xlabel(r"log(Energy / GeV)")
plt.ylabel(r"log($\Delta R$ / m)")
plt.grid()
plt.tight_layout()
if args.write:
save_fig('{}/reco_dist_vs_E_NTel_{}'.format(args.plots_dir, args.mode))
plt.show()
continue
reco.get_great_circles(hillas_params, event.inst.subarray,
point_azimuth, point_altitude)
# fit the gamma's direction of origin
# return reconstructed direction (3 components) with errors on the values
reco_direction, reco_dir_err = reco.fit_origin_crosses()
# In case fit fails to get any real value
if np.isnan(reco_direction).any():
continue
# get angular offset between reconstructed shower direction and MC
# generated shower direction
off_angle = linalg.angle(reco_direction, shower_direction)
# Appending all estimated off angles
off_angles.append(off_angle.to(u.deg).value)
# calculate theta square
thetasq = []
for i in off_angles:
thetasq.append(i**2)
# To plot thetasquare
# The number of events in th data files for LSTCam is not significantly high to give a nice
# thetasquare plot for gammas
# One can use deedicated MC file for LST get nice plot
plt.figure(figsize=(10, 8))
plt.hist(thetasq, bins=np.linspace(0, 10, 50))
sum_w = np.sum(pmt_signal_w)
sum_t = np.sum(pmt_signal_t)
Epsilon_intensity_w = abs(sum_w - sum_p) / sum_p
Epsilon_intensity_t = abs(sum_t - sum_p) / sum_p
alpha = {}
length = {}
width = {}
for k, h in hillas.items():
fit.get_great_circles({tel_id: h}, event.inst.subarray,
tel_phi, tel_theta)
c = fit.circles[tel_id]
alpha[k] = abs((angle(c.norm, shower_org) * u.rad) -
90 * u.deg).to(u.deg)
length[k] = h.length
width[k] = h.width
for k, signal in { # 'p': pmt_signal_p,
'w': pmt_signal_w
}.items():
h = hillas[k]
p1_x = h.cen_x
p1_y = h.cen_y
p2_x = p1_x + h.length * np.cos(h.psi)
p2_y = p1_y + h.length * np.sin(h.psi)
def main():
# your favourite units here
energy_unit = u.TeV
angle_unit = u.deg
dist_unit = u.m
agree_threshold = .5
min_tel = 3
parser = make_argparser()
parser.add_argument('--classifier', type=str,
default='data/classifier_pickle/classifier'
'_{mode}_{cam_id}_{classifier}.pkl')
parser.add_argument('--regressor', type=str,
default='data/classifier_pickle/regressor'
'_{mode}_{cam_id}_{regressor}.pkl')
parser.add_argument('-o', '--outfile', type=str, default="",
help="location to write the classified events to.")
parser.add_argument('--wave_dir', type=str, default=None,
help="directory where to find mr_filter. "
"if not set look in $PATH")
parser.add_argument('--wave_temp_dir', type=str, default='/tmp/', help="directory "
"where mr_filter to store the temporary fits files")
group = parser.add_mutually_exclusive_group()
group.add_argument('--proton', action='store_true',
help="do protons instead of gammas")
group.add_argument('--electron', action='store_true',
help="do electrons instead of gammas")
args = parser.parse_args()
if args.infile_list:
filenamelist = []
for f in args.infile_list:
filenamelist += glob("{}/{}".format(args.indir, f))
filenamelist.sort()
elif args.proton:
filenamelist = sorted(glob("{}/proton/*gz".format(args.indir)))
elif args.electron:
filenamelist = glob("{}/electron/*gz".format(args.indir))
channel = "electron"
else:
filenamelist = sorted(glob("{}/gamma/*gz".format(args.indir)))
if not filenamelist:
print("no files found; check indir: {}".format(args.indir))
exit(-1)
# keeping track of events and where they were rejected
Eventcutflow = CutFlow("EventCutFlow")
Imagecutflow = CutFlow("ImageCutFlow")
# takes care of image cleaning
cleaner = ImageCleaner(mode=args.mode, cutflow=Imagecutflow,
wavelet_options=args.raw,
skip_edge_events=False, island_cleaning=True)
# the class that does the shower reconstruction
shower_reco = HillasReconstructor()
preper = EventPreparer(
cleaner=cleaner, hillas_parameters=hillas_parameters,
shower_reco=shower_reco,
event_cutflow=Eventcutflow, image_cutflow=Imagecutflow,
# event/image cuts:
allowed_cam_ids=[],
min_ntel=2, min_charge=args.min_charge, min_pixel=3)
# wrapper for the scikit-learn classifier
classifier = EventClassifier.load(
args.classifier.format(**{
"mode": args.mode,
"wave_args": "mixed",
"classifier": 'RandomForestClassifier',
"cam_id": "{cam_id}"}),
cam_id_list=args.cam_ids)
# wrapper for the scikit-learn regressor
regressor = EnergyRegressor.load(
args.regressor.format(**{
"mode": args.mode,
"wave_args": "mixed",
"regressor": "RandomForestRegressor",
"cam_id": "{cam_id}"}),
cam_id_list=args.cam_ids)
ClassifierFeatures = namedtuple(
"ClassifierFeatures", (
"impact_dist",
"sum_signal_evt",
"max_signal_cam",
"sum_signal_cam",
"N_LST",
"N_MST",
"N_SST",
"width",
"length",
"skewness",
"kurtosis",
"h_max",
"err_est_pos",
"err_est_dir"))
EnergyFeatures = namedtuple(
"EnergyFeatures", (
"impact_dist",
"sum_signal_evt",
"max_signal_cam",
"sum_signal_cam",
"N_LST",
"N_MST",
"N_SST",
"width",
"length",
"skewness",
"kurtosis",
"h_max",
"err_est_pos",
"err_est_dir"))
# catch ctr-c signal to exit current loop and still display results
signal_handler = SignalHandler()
signal.signal(signal.SIGINT, signal_handler)
# this class defines the reconstruction parameters to keep track of
class RecoEvent(tb.IsDescription):
Run_ID = tb.Int16Col(dflt=-1, pos=0)
Event_ID = tb.Int16Col(dflt=-1, pos=1)
NTels_trig = tb.Int16Col(dflt=0, pos=0)
NTels_reco = tb.Int16Col(dflt=0, pos=1)
NTels_reco_lst = tb.Int16Col(dflt=0, pos=2)
NTels_reco_mst = tb.Int16Col(dflt=0, pos=3)
NTels_reco_sst = tb.Int16Col(dflt=0, pos=4)
MC_Energy = tb.Float32Col(dflt=np.nan, pos=5)
reco_Energy = tb.Float32Col(dflt=np.nan, pos=6)
reco_phi = tb.Float32Col(dflt=np.nan, pos=7)
reco_theta = tb.Float32Col(dflt=np.nan, pos=8)
off_angle = tb.Float32Col(dflt=np.nan, pos=9)
xi = tb.Float32Col(dflt=np.nan, pos=10)
DeltaR = tb.Float32Col(dflt=np.nan, pos=11)
ErrEstPos = tb.Float32Col(dflt=np.nan, pos=12)
ErrEstDir = tb.Float32Col(dflt=np.nan, pos=13)
gammaness = tb.Float32Col(dflt=np.nan, pos=14)
channel = "gamma" if "gamma" in " ".join(filenamelist) else "proton"
reco_outfile = tb.open_file(
mode="w",
# if no outfile name is given (i.e. don't to write the event list to disk),
# need specify two "driver" arguments
**({"filename": args.outfile} if args.outfile else
{"filename": "no_outfile.h5",
"driver": "H5FD_CORE", "driver_core_backing_store": False}))
reco_table = reco_outfile.create_table("/", "reco_events", RecoEvent)
reco_event = reco_table.row
allowed_tels = None # all telescopes
allowed_tels = prod3b_tel_ids("L+N+D")
for i, filename in enumerate(filenamelist[:args.last]):
# print(f"file: {i} filename = {filename}")
source = hessio_event_source(filename,
allowed_tels=allowed_tels,
max_events=args.max_events)
# loop that cleans and parametrises the images and performs the reconstruction
for (event, hillas_dict, n_tels,
tot_signal, max_signals, pos_fit, dir_fit, h_max,
err_est_pos, err_est_dir) in preper.prepare_event(source, True):
# now prepare the features for the classifier
cls_features_evt = {}
reg_features_evt = {}
if hillas_dict is not None:
for tel_id in hillas_dict.keys():
Imagecutflow.count("pre-features")
tel_pos = np.array(event.inst.tel_pos[tel_id][:2]) * u.m
moments = hillas_dict[tel_id]
impact_dist = linalg.length(tel_pos - pos_fit)
cls_features_tel = ClassifierFeatures(
impact_dist=impact_dist / u.m,
sum_signal_evt=tot_signal,
max_signal_cam=max_signals[tel_id],
sum_signal_cam=moments.size,
N_LST=n_tels["LST"],
N_MST=n_tels["MST"],
N_SST=n_tels["SST"],
width=moments.width / u.m,
length=moments.length / u.m,
skewness=moments.skewness,
kurtosis=moments.kurtosis,
h_max=h_max / u.m,
err_est_pos=err_est_pos / u.m,
err_est_dir=err_est_dir / u.deg
)
reg_features_tel = EnergyFeatures(
impact_dist=impact_dist / u.m,
sum_signal_evt=tot_signal,
max_signal_cam=max_signals[tel_id],
sum_signal_cam=moments.size,
N_LST=n_tels["LST"],
N_MST=n_tels["MST"],
N_SST=n_tels["SST"],
width=moments.width / u.m,
length=moments.length / u.m,
skewness=moments.skewness,
kurtosis=moments.kurtosis,
h_max=h_max / u.m,
err_est_pos=err_est_pos / u.m,
err_est_dir=err_est_dir / u.deg
)
if np.isnan(cls_features_tel).any() or np.isnan(reg_features_tel).any():
continue
Imagecutflow.count("features nan")
cam_id = event.inst.subarray.tel[tel_id].camera.cam_id
try:
reg_features_evt[cam_id] += [reg_features_tel]
cls_features_evt[cam_id] += [cls_features_tel]
except KeyError:
reg_features_evt[cam_id] = [reg_features_tel]
cls_features_evt[cam_id] = [cls_features_tel]
# save basic event infos
reco_event["MC_Energy"] = event.mc.energy.to(energy_unit).value
reco_event["Event_ID"] = event.r1.event_id
reco_event["Run_ID"] = event.r1.run_id
if cls_features_evt and reg_features_evt:
predict_energ = regressor.predict_by_event([reg_features_evt])["mean"][0]
predict_proba = classifier.predict_proba_by_event([cls_features_evt])
gammaness = predict_proba[0, 0]
# the MC direction of origin of the simulated particle
shower = event.mc
shower_core = np.array([shower.core_x / u.m, shower.core_y / u.m]) * u.m
shower_org = linalg.set_phi_theta(shower.az + 90 * u.deg,
90. * u.deg - shower.alt)
# and how the reconstructed direction compares to that
xi = linalg.angle(dir_fit, shower_org)
phi, theta = linalg.get_phi_theta(dir_fit)
phi = (phi if phi > 0 else phi + 360 * u.deg)
DeltaR = linalg.length(pos_fit[:2] - shower_core)
# TODO: replace with actual array pointing direction
array_pointing = linalg.set_phi_theta(0 * u.deg, 20. * u.deg)
# angular offset between the reconstructed direction and the array
# pointing
off_angle = linalg.angle(dir_fit, array_pointing)
reco_event["NTels_trig"] = len(event.dl0.tels_with_data)
reco_event["NTels_reco"] = len(hillas_dict)
reco_event["NTels_reco_lst"] = n_tels["LST"]
reco_event["NTels_reco_mst"] = n_tels["MST"]
reco_event["NTels_reco_sst"] = n_tels["SST"]
reco_event["reco_Energy"] = predict_energ.to(energy_unit).value
reco_event["reco_phi"] = phi / angle_unit
reco_event["reco_theta"] = theta / angle_unit
reco_event["off_angle"] = off_angle / angle_unit
reco_event["xi"] = xi / angle_unit
reco_event["DeltaR"] = DeltaR / dist_unit
reco_event["ErrEstPos"] = err_est_pos / dist_unit
reco_event["ErrEstDir"] = err_est_dir / angle_unit
reco_event["gammaness"] = gammaness
reco_event.append()
reco_table.flush()
if signal_handler.stop:
break
if signal_handler.stop:
break
try:
print()
Eventcutflow()
print()
Imagecutflow()
# do some simple event selection
# and print the corresponding selection efficiency
N_selected = len([x for x in reco_table.where(
"""(NTels_reco > min_tel) & (gammaness > agree_threshold)""")])
N_total = len(reco_table)
print("\nfraction selected events:")
print("{} / {} = {} %".format(N_selected, N_total, N_selected / N_total * 100))
except ZeroDivisionError:
pass
print("\nlength filenamelist:", len(filenamelist[:args.last]))
# do some plotting if so desired
if args.plot:
gammaness = [x['gammaness'] for x in reco_table]
NTels_rec = [x['NTels_reco'] for x in reco_table]
NTel_bins = np.arange(np.min(NTels_rec), np.max(NTels_rec) + 2) - .5
NTels_rec_lst = [x['NTels_reco_lst'] for x in reco_table]
NTels_rec_mst = [x['NTels_reco_mst'] for x in reco_table]
NTels_rec_sst = [x['NTels_reco_sst'] for x in reco_table]
reco_energy = np.array([x['reco_Energy'] for x in reco_table])
mc_energy = np.array([x['MC_Energy'] for x in reco_table])
fig = plt.figure(figsize=(15, 5))
plt.suptitle(" ** ".join([args.mode, "protons" if args.proton else "gamma"]))
plt.subplots_adjust(left=0.05, right=0.97, hspace=0.39, wspace=0.2)
ax = plt.subplot(131)
histo = np.histogram2d(NTels_rec, gammaness,
bins=(NTel_bins, np.linspace(0, 1, 11)))[0].T
histo_normed = histo / histo.max(axis=0)
im = ax.imshow(histo_normed, interpolation='none', origin='lower',
aspect='auto',
# extent=(*NTel_bins[[0, -1]], 0, 1),
cmap=plt.cm.inferno)
ax.set_xlabel("NTels")
ax.set_ylabel("drifted gammaness")
plt.title("Total Number of Telescopes")
# next subplot
ax = plt.subplot(132)
histo = np.histogram2d(NTels_rec_sst, gammaness,
bins=(NTel_bins, np.linspace(0, 1, 11)))[0].T
histo_normed = histo / histo.max(axis=0)
im = ax.imshow(histo_normed, interpolation='none', origin='lower',
aspect='auto',
# extent=(*NTel_bins[[0, -1]], 0, 1),
cmap=plt.cm.inferno)
ax.set_xlabel("NTels")
plt.setp(ax.get_yticklabels(), visible=False)
plt.title("Number of SSTs")
# next subplot
ax = plt.subplot(133)
histo = np.histogram2d(NTels_rec_mst, gammaness,
bins=(NTel_bins, np.linspace(0, 1, 11)))[0].T
histo_normed = histo / histo.max(axis=0)
im = ax.imshow(histo_normed, interpolation='none', origin='lower',
aspect='auto',
# extent=(*NTel_bins[[0, -1]], 0, 1),
cmap=plt.cm.inferno)
cb = fig.colorbar(im, ax=ax)
ax.set_xlabel("NTels")
plt.setp(ax.get_yticklabels(), visible=False)
plt.title("Number of MSTs")
plt.subplots_adjust(wspace=0.05)
# plot the energy migration matrix
plt.figure()
plt.hist2d(np.log10(reco_energy), np.log10(mc_energy), bins=20,
cmap=plt.cm.inferno)
plt.xlabel("E_MC / TeV")
plt.ylabel("E_rec / TeV")
plt.colorbar()
plt.show()