def fit_origin_minimise(self, seed=[0, 0, 1], test_function=None): ''' fits the origin of the gamma with a minimisation procedure this function expects that get_great_circles has been run already a seed should be given otherwise it defaults to "straight up" supperted functions to minimise are an M-estimator and the negative sum of the angles to all normal vectors of the great circles Parameters: ----------- seed : length-3 array starting point of the minimisation test_function : member function if this class either _n_angle_sum or _MEst (or own implementation...) defaults to _n_angle_sum if none is given _n_angle_sum seemingly superior to _MEst Returns: -------- direction : length-3 numpy array as dimensionless quantity best fit for the origin of the gamma from the minimisation process ''' if test_function is None: test_function = self._n_angle_sum ''' using the sum of the cosines of each direction with every otherdirection as weight; don't use the product -- with many steep angles, the product will become too small and the weight (and the whole fit) useless ''' weights = [np.sum([linalg.length(np.cross(A.norm, B.norm)) for A in self.circles.values()] ) * B.weight for B in self.circles.values()] ''' minimising the test function ''' self.fit_result_origin = minimize(test_function, seed, args=(self.circles, weights), method='BFGS', options={'disp': False} ) return np.array(linalg.normalise(self.fit_result_origin.x))*u.dimless
def fit_origin_minimise(self, seed=(0, 0, 1), test_function=neg_angle_sum): ''' Fits the origin of the gamma with a minimisation procedure this function expects that :func:`get_great_circles<ctapipe.reco.FitGammaHillas.get_great_circles>` has been run already. A seed should be given otherwise it defaults to "straight up" supperted functions to minimise are an M-estimator and the negative sum of the angles to all normal vectors of the great circles Parameters ----------- seed : length-3 array starting point of the minimisation test_function : function object, optional (default: neg_angle_sum) either neg_angle_sum or MEst (or own implementation...) neg_angle_sum seemingly superior to MEst Returns ------- direction : length-3 numpy array as dimensionless quantity best fit for the origin of the gamma from the minimisation process ''' # using the sum of the cosines of each direction with every # other direction as weight; don't use the product -- with many # steep angles, the product will become too small and the # weight (and the whole fit) useless weights = [np.sum([linalg.length(np.cross(A.norm, B.norm)) for A in self.circles.values()] ) * B.weight for B in self.circles.values()] # minimising the test function self.fit_result_origin = minimize(test_function, seed, args=(self.circles, weights), method='BFGS', options={'disp': False} ) return np.array(linalg.normalise(self.fit_result_origin.x)) * u.dimless
def MEst(origin, circles, weights): '''calculates the M-Estimator: a modified χ² that becomes asymptotically linear for high values and is therefore less sensitive to outliers the test is performed to maximise the angles between the fit direction and the all the normal vectors of the great circles .. math:: M_{Est} = \sum_i{ 2*\sqrt{1 + \chi^2} - 2} .. note:: seemingly inferior to negative sum of sin(angle)... Parameters ----------- origin : length-3 array direction vector of the gamma's origin used as seed circles : GreatCircle array collection of great circles created from the camera images weights : array list of weights for each image/great circle Returns ------- MEstimator : float ''' sin_ang = np.array([linalg.length(np.cross(origin, circ.norm)) for circ in circles.values()]) return -2 * np.sum(weights * np.sqrt((1 + np.square(sin_ang))) - 2) return -np.sum(weights * np.square(sin_ang)) ang = np.array([linalg.angle(origin, circ.norm) for circ in circles.values()]) return np.sum(weights * np.sqrt(2. + (ang - np.pi / 2.) ** 2))
def _MEst(self, origin, circles, weights): ''' calculates the M-Estimator: a modified chi2 that becomes asymptotically linear for high values and is therefore less sensitive to outliers the test is performed to maximise the angles between the fit direction and the all the normal vectors of the great circles Parameters: ----------- origin : length-3 array direction vector of the gamma's origin used as seed circles : GreatCircle array collection of great circles created from the camera images weights : array list of weights for each image/great circle Returns: -------- MEstimator : float Algorithm: ---------- M-Est = sum[ weight * sqrt( 2 * chi**2 ) ] Note: ----- seemingly inferior to negative sum of sin(angle)... ''' sin_ang = np.array([linalg.length(np.cross(origin, circ.norm)) for circ in circles.values()]) return np.sum(weights*np.sqrt(2. + (sin_ang-np.pi/2.)**2)) ang = np.array([linalg.angle(origin, circ.norm) for circ in circles.values()]) ang[ang > np.pi/2.] = np.pi-ang[ang > np.pi/2] return np.sum(weights*np.sqrt(2. + (ang-np.pi/2.)**2))
def n_angle_sum(origin, circles, weights): '''calculates the negative sum of the angle between the fit direction and all the normal vectors of the great circles Parameters ----------- origin : length-3 array direction vector of the gamma's origin used as seed circles : GreatCircle array collection of great circles created from the camera images weights : array list of weights for each image/great circle Returns -------- n_sum_angles : float negative of the sum of the angles between the test direction and all normal vectors of the given great circles ''' sin_ang = np.array([linalg.length(np.cross(origin, circ.norm)) for circ in circles.values()]) return -np.sum(weights * sin_ang)
source = hessio_event_source(filename, allowed_tels=allowed_tels, max_events=args.max_events) # loop that cleans and parametrises the images and performs the reconstruction for (event, hillas_dict, n_tels, tot_signal, max_signals, pos_fit, dir_fit, h_max, err_est_pos, err_est_dir) in preper.prepare_event(source): n_faint = 0 for tel_id in hillas_dict.keys(): cam_id = event.inst.subarray.tel[tel_id].camera.cam_id moments = hillas_dict[tel_id] tel_pos = np.array(event.inst.tel_pos[tel_id][:2]) * u.m impact_dist = linalg.length(tel_pos - pos_fit) if moments.size > pe_thersh: n_faint += 1 feature_events[cam_id]["impact_dist"] = impact_dist / dist_unit feature_events[cam_id]["sum_signal_evt"] = tot_signal feature_events[cam_id]["max_signal_cam"] = max_signals[tel_id] feature_events[cam_id]["sum_signal_cam"] = moments.size feature_events[cam_id]["N_LST"] = n_tels["LST"] feature_events[cam_id]["N_MST"] = n_tels["MST"] feature_events[cam_id]["N_SST"] = n_tels["SST"] feature_events[cam_id]["width"] = moments.width / dist_unit feature_events[cam_id]["length"] = moments.length / dist_unit feature_events[cam_id]["skewness"] = moments.skewness feature_events[cam_id]["kurtosis"] = moments.kurtosis
def main(): # your favourite units here energy_unit = u.TeV angle_unit = u.deg dist_unit = u.m parser = make_argparser() parser.add_argument('-o', '--outfile', type=str, help="if given, write output file with reconstruction results") parser.add_argument('--plot_c', action='store_true', help="plot camera-wise displays") group = parser.add_mutually_exclusive_group() group.add_argument('--proton', action='store_true', help="do protons instead of gammas") group.add_argument('--electron', action='store_true', help="do electrons instead of gammas") args = parser.parse_args() if args.infile_list: filenamelist = [] for f in args.infile_list: filenamelist += glob("{}/{}".format(args.indir, f)) elif args.proton: filenamelist = glob("{}/proton/*gz".format(args.indir)) channel = "proton" elif args.electron: filenamelist = glob("{}/electron/*gz".format(args.indir)) channel = "electron" elif args.gamma: filenamelist = glob("{}/gamma/*gz".format(args.indir)) channel = "gamma" else: raise ValueError("don't know which input to use...") filenamelist.sort() if not filenamelist: print("no files found; check indir: {}".format(args.indir)) exit(-1) else: print("found {} files".format(len(filenamelist))) tel_phi = {} tel_theta = {} # keeping track of events and where they were rejected Eventcutflow = CutFlow("EventCutFlow") Imagecutflow = CutFlow("ImageCutFlow") # takes care of image cleaning cleaner = ImageCleaner(mode=args.mode, cutflow=Imagecutflow, wavelet_options=args.raw, skip_edge_events=args.skip_edge_events, island_cleaning=True) # the class that does the shower reconstruction shower_reco = HillasReconstructor() shower_max_estimator = ShowerMaxEstimator("paranal") preper = EventPreparer(cleaner=cleaner, hillas_parameters=hillas_parameters, shower_reco=shower_reco, event_cutflow=Eventcutflow, image_cutflow=Imagecutflow, # event/image cuts: allowed_cam_ids=[], # means: all min_ntel=3, min_charge=args.min_charge, min_pixel=3) # a signal handler to abort the event loop but still do the post-processing signal_handler = SignalHandler() signal.signal(signal.SIGINT, signal_handler) try: # this class defines the reconstruction parameters to keep track of class RecoEvent(tb.IsDescription): NTels_trigg = tb.Int16Col(dflt=1, pos=0) NTels_clean = tb.Int16Col(dflt=1, pos=1) EnMC = tb.Float32Col(dflt=1, pos=2) xi = tb.Float32Col(dflt=1, pos=3) DeltaR = tb.Float32Col(dflt=1, pos=4) ErrEstPos = tb.Float32Col(dflt=1, pos=5) ErrEstDir = tb.Float32Col(dflt=1, pos=6) h_max = tb.Float32Col(dflt=1, pos=7) reco_outfile = tb.open_file( args.outfile, mode="w", # if we don't want to write the event list to disk, need to add more arguments **({} if args.store else {"driver": "H5FD_CORE", "driver_core_backing_store": False})) reco_table = reco_outfile.create_table("/", "reco_event", RecoEvent) reco_event = reco_table.row except: reco_event = RecoEvent() print("no pytables installed?") # ## ####### ####### ######## # ## ## ## ## ## ## ## # ## ## ## ## ## ## ## # ## ## ## ## ## ######## # ## ## ## ## ## ## # ## ## ## ## ## ## # ######## ####### ####### ## cam_id_map = {} # define here which telescopes to loop over allowed_tels = None # allowed_tels = prod3b_tel_ids("L+F+D") for i, filename in enumerate(filenamelist[:args.last]): print("file: {i} filename = {filename}".format(i=i, filename=filename)) source = hessio_event_source(filename, allowed_tels=allowed_tels, max_events=args.max_events) # loop that cleans and parametrises the images and performs the reconstruction for (event, hillas_dict, n_tels, tot_signal, max_signal, pos_fit, dir_fit, h_max, err_est_pos, err_est_dir) in preper.prepare_event(source): shower = event.mc org_alt = u.Quantity(shower.alt).to(u.deg) org_az = u.Quantity(shower.az).to(u.deg) if org_az > 180 * u.deg: org_az -= 360 * u.deg org_the = alt_to_theta(org_alt) org_phi = az_to_phi(org_az) if org_phi > 180 * u.deg: org_phi -= 360 * u.deg if org_phi < -180 * u.deg: org_phi += 360 * u.deg shower_org = linalg.set_phi_theta(org_phi, org_the) shower_core = convert_astropy_array([shower.core_x, shower.core_y]) xi = linalg.angle(dir_fit, shower_org).to(angle_unit) diff = linalg.length(pos_fit[:2] - shower_core) # print some performance print() print("xi = {:4.3f}".format(xi)) print("pos = {:4.3f}".format(diff)) print("h_max reco: {:4.3f}".format(h_max.to(u.km))) print("err_est_dir: {:4.3f}".format(err_est_dir.to(angle_unit))) print("err_est_pos: {:4.3f}".format(err_est_pos)) try: # store the reconstruction data in the PyTable reco_event["NTels_trigg"] = n_tels["tot"] reco_event["NTels_clean"] = len(shower_reco.circles) reco_event["EnMC"] = event.mc.energy / energy_unit reco_event["xi"] = xi / angle_unit reco_event["DeltaR"] = diff / dist_unit reco_event["ErrEstPos"] = err_est_pos / dist_unit reco_event["ErrEstDir"] = err_est_dir / angle_unit reco_event["h_max"] = h_max / dist_unit reco_event.append() reco_table.flush() print() print("xi res (68-percentile) = {:4.3f} {}" .format(np.percentile(reco_table.cols.xi, 68), angle_unit)) print("core res (68-percentile) = {:4.3f} {}" .format(np.percentile(reco_table.cols.DeltaR, 68), dist_unit)) print("h_max (median) = {:4.3f} {}" .format(np.percentile(reco_table.cols.h_max, 50), dist_unit)) except NoPyTables: pass if args.plot_c: from mpl_toolkits.mplot3d import Axes3D fig = plt.figure() ax = fig.gca(projection='3d') for c in shower_reco.circles.values(): points = [c.pos + t * c.a * u.km for t in np.linspace(0, 15, 3)] ax.plot(*np.array(points).T, linewidth=np.sqrt(c.weight) / 10) ax.scatter(*c.pos[:, None].value, s=np.sqrt(c.weight)) plt.xlabel("x") plt.ylabel("y") plt.pause(.1) # this plots # • the MC shower core # • the reconstructed shower core # • the used telescopes # • and the trace of the Hillas plane on the ground plt.figure() for tel_id, c in shower_reco.circles.items(): plt.scatter(c.pos[0], c.pos[1], s=np.sqrt(c.weight)) plt.gca().annotate(tel_id, (c.pos[0].value, c.pos[1].value)) plt.plot([c.pos[0].value-500*c.norm[1], c.pos[0].value+500*c.norm[1]], [c.pos[1].value+500*c.norm[0], c.pos[1].value-500*c.norm[0]], linewidth=np.sqrt(c.weight)/10) plt.scatter(*pos_fit[:2], c="black", marker="*", label="fitted") plt.scatter(*shower_core[:2], c="black", marker="P", label="MC") plt.legend() plt.xlabel("x") plt.ylabel("y") plt.xlim(-1400, 1400) plt.ylim(-1400, 1400) plt.show() if signal_handler.stop: break if signal_handler.stop: break print("\n" + "="*35 + "\n") print("xi res (68-percentile) = {:4.3f} {}" .format(np.percentile(reco_table.cols.xi, 68), angle_unit)) print("core res (68-percentile) = {:4.3f} {}" .format(np.percentile(reco_table.cols.DeltaR, 68), dist_unit)) print("h_max (median) = {:4.3f} {}" .format(np.percentile(reco_table.cols.h_max, 50), dist_unit)) # print the cutflows for telescopes and camera images print("\n\n") Eventcutflow("min2Tels trig") print() Imagecutflow(sort_column=1) # if we don't want to plot anything, we can exit now if not args.plot: return # ######## ## ####### ######## ###### # ## ## ## ## ## ## ## ## # ## ## ## ## ## ## ## # ######## ## ## ## ## ###### # ## ## ## ## ## ## # ## ## ## ## ## ## ## # ## ######## ####### ## ###### plt.figure() plt.hist(reco_table.cols.h_max, bins=np.linspace(000, 15000, 51, True)) plt.title(channel) plt.xlabel("h_max reco") plt.pause(.1) figure = plt.figure() xi_edges = np.linspace(0, 5, 20) plt.hist(reco_table.cols.xi, bins=xi_edges, log=True) plt.xlabel(r"$\xi$ / deg") if args.write: save_fig('{}/reco_xi_{}'.format(args.plots_dir, args.mode)) plt.pause(.1) plt.figure() plt.hist(reco_table.cols.ErrEstDir[:], bins=np.linspace(0, 20, 50)) plt.title(channel) plt.xlabel("beta") plt.pause(.1) plt.figure() plt.hist(np.log10(reco_table.cols.xi[:] / reco_table.cols.ErrEstDir[:]), bins=50) plt.title(channel) plt.xlabel("log_10(xi / beta)") plt.pause(.1) # convert the xi-list into a dict with the number of used telescopes as keys xi_vs_tel = {} for xi, ntel in zip(reco_table.cols.xi, reco_table.cols.NTels_clean): if ntel not in xi_vs_tel: xi_vs_tel[ntel] = [xi] else: xi_vs_tel[ntel].append(xi) print(args.mode) for ntel, xis in sorted(xi_vs_tel.items()): print("NTel: {} -- median xi: {}".format(ntel, np.median(xis))) # print("histogram:", np.histogram(xis, bins=xi_edges)) # create a list of energy bin-edges and -centres for violin plots Energy_edges = np.linspace(2, 8, 13) Energy_centres = (Energy_edges[1:]+Energy_edges[:-1])/2. # convert the xi-list in to an energy-binned dict with the bin centre as keys xi_vs_energy = {} for en, xi in zip(reco_table.cols.EnMC, reco_table.cols.xi): # get the bin number this event belongs into sbin = np.digitize(np.log10(en), Energy_edges)-1 # the central value of the bin is the key for the dictionary if Energy_centres[sbin] not in xi_vs_energy: xi_vs_energy[Energy_centres[sbin]] = [xi] else: xi_vs_energy[Energy_centres[sbin]] += [xi] # plotting the angular error as violin plots with binning in # number of telescopes and shower energy figure = plt.figure() plt.subplot(211) plt.violinplot([np.log10(a) for a in xi_vs_tel.values()], [a for a in xi_vs_tel.keys()], points=60, widths=.75, showextrema=False, showmedians=True) plt.xlabel("Number of Telescopes") plt.ylabel(r"log($\xi$ / deg)") plt.ylim(-3, 2) plt.grid() plt.subplot(212) plt.violinplot([np.log10(a) for a in xi_vs_energy.values()], [a for a in xi_vs_energy.keys()], points=60, widths=(Energy_edges[1] - Energy_edges[0]) / 1.5, showextrema=False, showmedians=True) plt.xlabel(r"log(Energy / GeV)") plt.ylabel(r"log($\xi$ / deg)") plt.ylim(-3, 2) plt.grid() plt.tight_layout() if args.write: save_fig('{}/reco_xi_vs_E_NTel_{}'.format(args.plots_dir, args.mode)) plt.pause(.1) # convert the diffs-list into a dict with the number of used telescopes as keys diff_vs_tel = {} for diff, ntel in zip(reco_table.cols.DeltaR, reco_table.cols.NTels_clean): if ntel not in diff_vs_tel: diff_vs_tel[ntel] = [diff] else: diff_vs_tel[ntel].append(diff) # convert the diffs-list in to an energy-binned dict with the bin centre as keys diff_vs_energy = {} for en, diff in zip(reco_table.cols.EnMC, reco_table.cols.DeltaR): # get the bin number this event belongs into sbin = np.digitize(np.log10(en), Energy_edges) - 1 # the central value of the bin is the key for the dictionary if Energy_centres[sbin] not in diff_vs_energy: diff_vs_energy[Energy_centres[sbin]] = [diff] else: diff_vs_energy[Energy_centres[sbin]] += [diff] # plotting the core position error as violin plots with binning in # number of telescopes an shower energy plt.figure() plt.subplot(211) plt.violinplot([np.log10(a) for a in diff_vs_tel.values()], [a for a in diff_vs_tel.keys()], points=60, widths=.75, showextrema=False, showmedians=True) plt.xlabel("Number of Telescopes") plt.ylabel(r"log($\Delta R$ / m)") plt.grid() plt.subplot(212) plt.violinplot([np.log10(a) for a in diff_vs_energy.values()], [a for a in diff_vs_energy.keys()], points=60, widths=(Energy_edges[1] - Energy_edges[0]) / 1.5, showextrema=False, showmedians=True) plt.xlabel(r"log(Energy / GeV)") plt.ylabel(r"log($\Delta R$ / m)") plt.grid() plt.tight_layout() if args.write: save_fig('{}/reco_dist_vs_E_NTel_{}'.format(args.plots_dir, args.mode)) plt.show()
def main(): # your favourite units here energy_unit = u.TeV angle_unit = u.deg dist_unit = u.m agree_threshold = .5 min_tel = 3 parser = make_argparser() parser.add_argument('--classifier', type=str, default='data/classifier_pickle/classifier' '_{mode}_{cam_id}_{classifier}.pkl') parser.add_argument('--regressor', type=str, default='data/classifier_pickle/regressor' '_{mode}_{cam_id}_{regressor}.pkl') parser.add_argument('-o', '--outfile', type=str, default="", help="location to write the classified events to.") parser.add_argument('--wave_dir', type=str, default=None, help="directory where to find mr_filter. " "if not set look in $PATH") parser.add_argument('--wave_temp_dir', type=str, default='/tmp/', help="directory " "where mr_filter to store the temporary fits files") group = parser.add_mutually_exclusive_group() group.add_argument('--proton', action='store_true', help="do protons instead of gammas") group.add_argument('--electron', action='store_true', help="do electrons instead of gammas") args = parser.parse_args() if args.infile_list: filenamelist = [] for f in args.infile_list: filenamelist += glob("{}/{}".format(args.indir, f)) filenamelist.sort() elif args.proton: filenamelist = sorted(glob("{}/proton/*gz".format(args.indir))) elif args.electron: filenamelist = glob("{}/electron/*gz".format(args.indir)) channel = "electron" else: filenamelist = sorted(glob("{}/gamma/*gz".format(args.indir))) if not filenamelist: print("no files found; check indir: {}".format(args.indir)) exit(-1) # keeping track of events and where they were rejected Eventcutflow = CutFlow("EventCutFlow") Imagecutflow = CutFlow("ImageCutFlow") # takes care of image cleaning cleaner = ImageCleaner(mode=args.mode, cutflow=Imagecutflow, wavelet_options=args.raw, skip_edge_events=False, island_cleaning=True) # the class that does the shower reconstruction shower_reco = HillasReconstructor() preper = EventPreparer( cleaner=cleaner, hillas_parameters=hillas_parameters, shower_reco=shower_reco, event_cutflow=Eventcutflow, image_cutflow=Imagecutflow, # event/image cuts: allowed_cam_ids=[], min_ntel=2, min_charge=args.min_charge, min_pixel=3) # wrapper for the scikit-learn classifier classifier = EventClassifier.load( args.classifier.format(**{ "mode": args.mode, "wave_args": "mixed", "classifier": 'RandomForestClassifier', "cam_id": "{cam_id}"}), cam_id_list=args.cam_ids) # wrapper for the scikit-learn regressor regressor = EnergyRegressor.load( args.regressor.format(**{ "mode": args.mode, "wave_args": "mixed", "regressor": "RandomForestRegressor", "cam_id": "{cam_id}"}), cam_id_list=args.cam_ids) ClassifierFeatures = namedtuple( "ClassifierFeatures", ( "impact_dist", "sum_signal_evt", "max_signal_cam", "sum_signal_cam", "N_LST", "N_MST", "N_SST", "width", "length", "skewness", "kurtosis", "h_max", "err_est_pos", "err_est_dir")) EnergyFeatures = namedtuple( "EnergyFeatures", ( "impact_dist", "sum_signal_evt", "max_signal_cam", "sum_signal_cam", "N_LST", "N_MST", "N_SST", "width", "length", "skewness", "kurtosis", "h_max", "err_est_pos", "err_est_dir")) # catch ctr-c signal to exit current loop and still display results signal_handler = SignalHandler() signal.signal(signal.SIGINT, signal_handler) # this class defines the reconstruction parameters to keep track of class RecoEvent(tb.IsDescription): Run_ID = tb.Int16Col(dflt=-1, pos=0) Event_ID = tb.Int16Col(dflt=-1, pos=1) NTels_trig = tb.Int16Col(dflt=0, pos=0) NTels_reco = tb.Int16Col(dflt=0, pos=1) NTels_reco_lst = tb.Int16Col(dflt=0, pos=2) NTels_reco_mst = tb.Int16Col(dflt=0, pos=3) NTels_reco_sst = tb.Int16Col(dflt=0, pos=4) MC_Energy = tb.Float32Col(dflt=np.nan, pos=5) reco_Energy = tb.Float32Col(dflt=np.nan, pos=6) reco_phi = tb.Float32Col(dflt=np.nan, pos=7) reco_theta = tb.Float32Col(dflt=np.nan, pos=8) off_angle = tb.Float32Col(dflt=np.nan, pos=9) xi = tb.Float32Col(dflt=np.nan, pos=10) DeltaR = tb.Float32Col(dflt=np.nan, pos=11) ErrEstPos = tb.Float32Col(dflt=np.nan, pos=12) ErrEstDir = tb.Float32Col(dflt=np.nan, pos=13) gammaness = tb.Float32Col(dflt=np.nan, pos=14) channel = "gamma" if "gamma" in " ".join(filenamelist) else "proton" reco_outfile = tb.open_file( mode="w", # if no outfile name is given (i.e. don't to write the event list to disk), # need specify two "driver" arguments **({"filename": args.outfile} if args.outfile else {"filename": "no_outfile.h5", "driver": "H5FD_CORE", "driver_core_backing_store": False})) reco_table = reco_outfile.create_table("/", "reco_events", RecoEvent) reco_event = reco_table.row allowed_tels = None # all telescopes allowed_tels = prod3b_tel_ids("L+N+D") for i, filename in enumerate(filenamelist[:args.last]): # print(f"file: {i} filename = {filename}") source = hessio_event_source(filename, allowed_tels=allowed_tels, max_events=args.max_events) # loop that cleans and parametrises the images and performs the reconstruction for (event, hillas_dict, n_tels, tot_signal, max_signals, pos_fit, dir_fit, h_max, err_est_pos, err_est_dir) in preper.prepare_event(source, True): # now prepare the features for the classifier cls_features_evt = {} reg_features_evt = {} if hillas_dict is not None: for tel_id in hillas_dict.keys(): Imagecutflow.count("pre-features") tel_pos = np.array(event.inst.tel_pos[tel_id][:2]) * u.m moments = hillas_dict[tel_id] impact_dist = linalg.length(tel_pos - pos_fit) cls_features_tel = ClassifierFeatures( impact_dist=impact_dist / u.m, sum_signal_evt=tot_signal, max_signal_cam=max_signals[tel_id], sum_signal_cam=moments.size, N_LST=n_tels["LST"], N_MST=n_tels["MST"], N_SST=n_tels["SST"], width=moments.width / u.m, length=moments.length / u.m, skewness=moments.skewness, kurtosis=moments.kurtosis, h_max=h_max / u.m, err_est_pos=err_est_pos / u.m, err_est_dir=err_est_dir / u.deg ) reg_features_tel = EnergyFeatures( impact_dist=impact_dist / u.m, sum_signal_evt=tot_signal, max_signal_cam=max_signals[tel_id], sum_signal_cam=moments.size, N_LST=n_tels["LST"], N_MST=n_tels["MST"], N_SST=n_tels["SST"], width=moments.width / u.m, length=moments.length / u.m, skewness=moments.skewness, kurtosis=moments.kurtosis, h_max=h_max / u.m, err_est_pos=err_est_pos / u.m, err_est_dir=err_est_dir / u.deg ) if np.isnan(cls_features_tel).any() or np.isnan(reg_features_tel).any(): continue Imagecutflow.count("features nan") cam_id = event.inst.subarray.tel[tel_id].camera.cam_id try: reg_features_evt[cam_id] += [reg_features_tel] cls_features_evt[cam_id] += [cls_features_tel] except KeyError: reg_features_evt[cam_id] = [reg_features_tel] cls_features_evt[cam_id] = [cls_features_tel] # save basic event infos reco_event["MC_Energy"] = event.mc.energy.to(energy_unit).value reco_event["Event_ID"] = event.r1.event_id reco_event["Run_ID"] = event.r1.run_id if cls_features_evt and reg_features_evt: predict_energ = regressor.predict_by_event([reg_features_evt])["mean"][0] predict_proba = classifier.predict_proba_by_event([cls_features_evt]) gammaness = predict_proba[0, 0] # the MC direction of origin of the simulated particle shower = event.mc shower_core = np.array([shower.core_x / u.m, shower.core_y / u.m]) * u.m shower_org = linalg.set_phi_theta(shower.az + 90 * u.deg, 90. * u.deg - shower.alt) # and how the reconstructed direction compares to that xi = linalg.angle(dir_fit, shower_org) phi, theta = linalg.get_phi_theta(dir_fit) phi = (phi if phi > 0 else phi + 360 * u.deg) DeltaR = linalg.length(pos_fit[:2] - shower_core) # TODO: replace with actual array pointing direction array_pointing = linalg.set_phi_theta(0 * u.deg, 20. * u.deg) # angular offset between the reconstructed direction and the array # pointing off_angle = linalg.angle(dir_fit, array_pointing) reco_event["NTels_trig"] = len(event.dl0.tels_with_data) reco_event["NTels_reco"] = len(hillas_dict) reco_event["NTels_reco_lst"] = n_tels["LST"] reco_event["NTels_reco_mst"] = n_tels["MST"] reco_event["NTels_reco_sst"] = n_tels["SST"] reco_event["reco_Energy"] = predict_energ.to(energy_unit).value reco_event["reco_phi"] = phi / angle_unit reco_event["reco_theta"] = theta / angle_unit reco_event["off_angle"] = off_angle / angle_unit reco_event["xi"] = xi / angle_unit reco_event["DeltaR"] = DeltaR / dist_unit reco_event["ErrEstPos"] = err_est_pos / dist_unit reco_event["ErrEstDir"] = err_est_dir / angle_unit reco_event["gammaness"] = gammaness reco_event.append() reco_table.flush() if signal_handler.stop: break if signal_handler.stop: break try: print() Eventcutflow() print() Imagecutflow() # do some simple event selection # and print the corresponding selection efficiency N_selected = len([x for x in reco_table.where( """(NTels_reco > min_tel) & (gammaness > agree_threshold)""")]) N_total = len(reco_table) print("\nfraction selected events:") print("{} / {} = {} %".format(N_selected, N_total, N_selected / N_total * 100)) except ZeroDivisionError: pass print("\nlength filenamelist:", len(filenamelist[:args.last])) # do some plotting if so desired if args.plot: gammaness = [x['gammaness'] for x in reco_table] NTels_rec = [x['NTels_reco'] for x in reco_table] NTel_bins = np.arange(np.min(NTels_rec), np.max(NTels_rec) + 2) - .5 NTels_rec_lst = [x['NTels_reco_lst'] for x in reco_table] NTels_rec_mst = [x['NTels_reco_mst'] for x in reco_table] NTels_rec_sst = [x['NTels_reco_sst'] for x in reco_table] reco_energy = np.array([x['reco_Energy'] for x in reco_table]) mc_energy = np.array([x['MC_Energy'] for x in reco_table]) fig = plt.figure(figsize=(15, 5)) plt.suptitle(" ** ".join([args.mode, "protons" if args.proton else "gamma"])) plt.subplots_adjust(left=0.05, right=0.97, hspace=0.39, wspace=0.2) ax = plt.subplot(131) histo = np.histogram2d(NTels_rec, gammaness, bins=(NTel_bins, np.linspace(0, 1, 11)))[0].T histo_normed = histo / histo.max(axis=0) im = ax.imshow(histo_normed, interpolation='none', origin='lower', aspect='auto', # extent=(*NTel_bins[[0, -1]], 0, 1), cmap=plt.cm.inferno) ax.set_xlabel("NTels") ax.set_ylabel("drifted gammaness") plt.title("Total Number of Telescopes") # next subplot ax = plt.subplot(132) histo = np.histogram2d(NTels_rec_sst, gammaness, bins=(NTel_bins, np.linspace(0, 1, 11)))[0].T histo_normed = histo / histo.max(axis=0) im = ax.imshow(histo_normed, interpolation='none', origin='lower', aspect='auto', # extent=(*NTel_bins[[0, -1]], 0, 1), cmap=plt.cm.inferno) ax.set_xlabel("NTels") plt.setp(ax.get_yticklabels(), visible=False) plt.title("Number of SSTs") # next subplot ax = plt.subplot(133) histo = np.histogram2d(NTels_rec_mst, gammaness, bins=(NTel_bins, np.linspace(0, 1, 11)))[0].T histo_normed = histo / histo.max(axis=0) im = ax.imshow(histo_normed, interpolation='none', origin='lower', aspect='auto', # extent=(*NTel_bins[[0, -1]], 0, 1), cmap=plt.cm.inferno) cb = fig.colorbar(im, ax=ax) ax.set_xlabel("NTels") plt.setp(ax.get_yticklabels(), visible=False) plt.title("Number of MSTs") plt.subplots_adjust(wspace=0.05) # plot the energy migration matrix plt.figure() plt.hist2d(np.log10(reco_energy), np.log10(mc_energy), bins=20, cmap=plt.cm.inferno) plt.xlabel("E_MC / TeV") plt.ylabel("E_rec / TeV") plt.colorbar() plt.show()
def fit_core_crosses(self): r"""calculates the core position as the least linear square solution of an (over-constrained) equation system Notes ----- The basis is the "trace" of each telescope's `GreatCircle` which can be determined by the telescope's position P=(Px, Py) and the circle's normal vector, projected to the ground n=(nx, ny), so that for every r=(x, y) on the trace :math:`\vec n \cdot \vec r = \vec n \cdot \vec P` , :math:`n_x \cdot x + n_y \cdot y = d` In a perfect world, the traces of all telescopes cross in the shower's point of impact. This means that there is one common point (x, y) for every telescope, so we can write in matrix form: .. math:: :label: fullmatrix \begin{pmatrix} nx_1 & ny_1 \\ \vdots & \vdots \\ nx_n & ny_n \end{pmatrix} \cdot (x, y) = \begin{pmatrix} d_1 \\ \vdots \\ d_n \end{pmatrix} or :math:`\boldsymbol{A} \cdot \vec r = \vec D` . Since we do not live in a perfect world and there probably is no point r that fulfils this equation system, it is solved by the method of least linear square: .. math:: :label: rchisqr \vec{r}_{\chi^2} = (\boldsymbol{A}^\text{T} \cdot \boldsymbol{A})^{-1} \boldsymbol{A}^\text{T} \cdot \vec D :math:`\vec{r}_{\chi^2}` minimises the squared difference of .. math:: \vec D - \boldsymbol{A} \cdot \vec r Weights are applied to every line of equation :eq:`fullmatrix` as stored in circle.weight (assuming they have been set in `get_great_circles` or elsewhere). Returns ------- r_chisqr: numpy.ndarray(2) the minimum :math:`\chi^2` solution for the shower impact position pos_uncert: astropy length quantity error estimate on the reconstructed core position """ A = np.zeros((len(self.circles), 2)) D = np.zeros(len(self.circles)) for i, circle in enumerate(self.circles.values()): # apply weight from circle and from the tilt of the circle # towards the horizontal plane: simply projecting # circle.norm to the ground gives higher weight to planes # perpendicular to the ground and less to those that have # a steeper angle A[i] = circle.weight * circle.norm[:2] # since A[i] is used in the dot-product, no need to multiply the # weight here D[i] = np.dot(A[i], circle.pos[:2]) # the math from equation (2) would look like this: # ATA = np.dot(A.T, A) # ATAinv = np.linalg.inv(ATA) # ATAinvAT = np.dot(ATAinv, A.T) # return np.dot(ATAinvAT, D) * unit # instead used directly the numpy implementation # speed is the same, just handles already "SingularMatrixError" pos = np.linalg.lstsq(A, D)[0] * u.m weighted_sum_dist = np.sum([np.dot(pos[:2] - c.pos[:2], c.norm[:2]) * c.weight for c in self.circles.values()]) * pos.unit norm_sum_dist = np.sum([c.weight * linalg.length(c.norm[:2]) for c in self.circles.values()]) pos_uncert = abs(weighted_sum_dist / norm_sum_dist) return pos, pos_uncert
# loop that cleans and parametrises the images and performs the reconstruction for (event, hillas_dict, n_tels, tot_signal, max_signals, pos_fit, dir_fit, h_max, err_est_pos, err_est_dir) in preper.prepare_event(source): # now prepare the features for the classifier cls_features_evt = {} reg_features_evt = {} for tel_id in hillas_dict.keys(): Imagecutflow.count("pre-features") tel_pos = np.array(event.inst.tel_pos[tel_id][:2]) * u.m moments = hillas_dict[tel_id] impact_dist = linalg.length(tel_pos - pos_fit) reg_features_tel = EnergyFeatures( impact_dist=(impact_dist / u.m).si, sum_signal_evt=tot_signal, max_signal_cam=max_signals[tel_id], sum_signal_cam=moments.size, N_LST=n_tels["LST"], N_MST=n_tels["MST"], N_SST=n_tels["SST"], width=(moments.width / u.m).si, length=(moments.length / u.m).si, skewness=moments.skewness, kurtosis=moments.kurtosis, h_max=(h_max / u.m).si, err_est_pos=(err_est_pos / u.m).si,