def test_build_models(simulated_dl1_file, rf_models): infile = simulated_dl1_file custom_config = { "n_training_events": { "gamma_regressors": 0.99, "gamma_tmp_regressors": 0.78, "gamma_classifier": 0.21, "proton_classifier": 0.98 } } reg_energy, reg_disp_norm, cls_disp_sign, cls_gh = build_models( infile, infile, save_models=False, free_model_memory=False, custom_config=custom_config) build_models(infile, infile, save_models=True, free_model_memory=True, custom_config=custom_config) import joblib joblib.dump(reg_energy, rf_models["energy"], compress=3) joblib.dump(cls_gh, rf_models["gh_sep"], compress=3) joblib.dump(reg_disp_norm, rf_models["disp_norm"], compress=3) joblib.dump(cls_disp_sign, rf_models["disp_sign"], compress=3)
def main(): args = parser.parse_args() #Train the models config = {} if args.config_file is not None: config = read_configuration_file(args.config_file) dl1_to_dl2.build_models( args.gammafile, args.protonfile, save_models=args.save_models, path_models=args.path_models, custom_config=config, )
def main(): #Train the models config = {} if args.config_file is not None: try: config = read_configuration_file(args.config_file) except("Custom configuration could not be loaded !!!"): pass dl1_to_dl2.build_models(args.gammafile, args.protonfile, save_models=args.storerf, path_models=args.path_models, custom_config=config, )
def test_build_models(): from lstchain.reco.dl1_to_dl2 import build_models infile = dl1_file reg_energy, reg_disp, cls_gh = build_models(infile, infile, custom_config=custom_config, save_models=False) import joblib joblib.dump(reg_energy, file_model_energy) joblib.dump(reg_disp, file_model_disp) joblib.dump(cls_gh, file_model_gh_sep)
def test_fail_build_models(simulated_dl1_file): custom_config = { "n_training_events": { "gamma_regressors": 0.99, "gamma_tmp_regressors": 0.78, "gamma_classifier": 0.31, "proton_classifier": 0.98 } } _, _, _, _ = build_models(simulated_dl1_file, simulated_dl1_file, save_models=False, custom_config=custom_config)
def test_build_models(): from lstchain.reco.dl1_to_dl2 import build_models infile = 'dl1_gamma_test_large.h5' features = ['intensity', 'width', 'length'] reg_energy, reg_disp, cls_gh = build_models(infile, infile, features, save_models=True) from sklearn.externals import joblib joblib.dump(reg_energy, 'rf_energy.pkl') joblib.dump(reg_disp, 'rf_disp.pkl') joblib.dump(cls_gh, 'rf_cls_gh.pkl')
def test_build_models(simulated_dl1_file, rf_models): from lstchain.reco.dl1_to_dl2 import build_models infile = simulated_dl1_file reg_energy, reg_disp, cls_gh = build_models(infile, infile, custom_config=standard_config, save_models=False) import joblib joblib.dump(reg_energy, rf_models["energy"]) joblib.dump(reg_disp, rf_models["disp"]) joblib.dump(cls_gh, rf_models["gh_sep"])
def main(): custom_config = {} if args.config_file is not None: try: custom_config = read_configuration_file(args.config_file) except ("Custom configuration could not be loaded !!!"): pass config = replace_config(standard_config, custom_config) reg_energy, reg_disp_vector, cls_gh = dl1_to_dl2.build_models( args.gammafile, args.protonfile, save_models=args.storerf, path_models=args.path_models, custom_config=config, ) gammas = filter_events( pd.read_hdf(args.gammatest, key=dl1_params_lstcam_key), config["events_filters"], ) proton = filter_events( pd.read_hdf(args.protontest, key=dl1_params_lstcam_key), config["events_filters"], ) data = pd.concat([gammas, proton], ignore_index=True) dl2 = dl1_to_dl2.apply_models(data, cls_gh, reg_energy, reg_disp_vector, custom_config=config) ####PLOT SOME RESULTS##### selected_gammas = dl2.query('reco_type==0 & mc_type==0') if (len(selected_gammas) == 0): log.warning('No gammas selected, I will not plot any output') sys.exit() plot_dl2.plot_features(dl2) if not args.batch: plt.show() plot_dl2.energy_results(selected_gammas) if not args.batch: plt.show() plot_dl2.direction_results(selected_gammas) if not args.batch: plt.show() plot_dl2.plot_disp_vector(selected_gammas) if not args.batch: plt.show() plot_dl2.plot_pos(dl2) if not args.batch: plt.show() plot_dl2.plot_roc_gamma(dl2) if not args.batch: plt.show() plot_dl2.plot_models_features_importances(args.path_models, args.config_file) if not args.batch: plt.show() plt.hist(dl2[dl2['mc_type'] == 101]['gammaness'], bins=100) plt.hist(dl2[dl2['mc_type'] == 0]['gammaness'], bins=100) if not args.batch: plt.show()
def main(): custom_config = {} if args.config_file is not None: try: custom_config = read_configuration_file(args.config_file) except ("Custom configuration could not be loaded !!!"): pass config = replace_config(standard_config, custom_config) reg_energy, reg_disp_vector, cls_gh = dl1_to_dl2.build_models( args.gammafile, args.protonfile, save_models=args.storerf, path_models=args.path_models, custom_config=config, ) gammas = filter_events( pd.read_hdf(args.gammatest, key=dl1_params_lstcam_key), config["events_filters"], ) proton = filter_events( pd.read_hdf(args.protontest, key=dl1_params_lstcam_key), config["events_filters"], ) data = pd.concat([gammas, proton], ignore_index=True) dl2 = dl1_to_dl2.apply_models(data, cls_gh, reg_energy, reg_disp_vector, custom_config=config) ####PLOT SOME RESULTS##### gammas = dl2[dl2.gammaness >= 0.5] protons = dl2[dl2.gammaness < 0.5] gammas.reco_type = 0 protons.reco_type = 1 focal_length = 28 * u.m src_pos_reco = utils.reco_source_position_sky( gammas.x.values * u.m, gammas.y.values * u.m, gammas.reco_disp_dx.values * u.m, gammas.reco_disp_dy.values * u.m, focal_length, gammas.mc_alt_tel.values * u.rad, gammas.mc_az_tel.values * u.rad) plot_dl2.plot_features(dl2) plt.show() plot_dl2.plot_e(gammas, 10, 1.5, 3.5) plt.show() plot_dl2.calc_resolution(gammas) plt.show() plot_dl2.plot_e_resolution(gammas, 10, 1.5, 3.5) plt.show() plot_dl2.plot_disp_vector(gammas) plt.show() try: ctaplot.plot_theta2( gammas.mc_alt, np.arctan(np.tan(gammas.mc_az)), src_pos_reco.alt.rad, np.arctan(np.tan(src_pos_reco.az.rad)), bins=50, range=(0, 1), ) plt.show() ctaplot.plot_angular_res_per_energy( src_pos_reco.alt.rad, np.arctan(np.tan(src_pos_reco.az.rad)), gammas.mc_alt, np.arctan(np.tan(gammas.mc_az)), gammas.mc_energy) plt.show() except: pass regression_features = config["regression_features"] classification_features = config["classification_features"] plt.show() plot_dl2.plot_pos(dl2) plt.show() plot_dl2.plot_ROC(cls_gh, dl2, classification_features, -1) plt.show() plot_dl2.plot_importances(cls_gh, classification_features) plt.show() plot_dl2.plot_importances(reg_energy, regression_features) plt.show() plot_dl2.plot_importances(reg_disp_vector, regression_features) plt.show() plt.hist(dl2[dl2['mc_type'] == 101]['gammaness'], bins=100) plt.hist(dl2[dl2['mc_type'] == 0]['gammaness'], bins=100) plt.show()
parser.add_argument( '--config_file', '-conf', action='store', type=str, dest='config_file', help= 'Path to a configuration file. If none is given, a standard configuration is applied', default=None) args = parser.parse_args() if __name__ == '__main__': #Train the models features = [ 'intensity', 'width', 'length', 'x', 'y', 'psi', 'phi', 'wl', 'skewness', 'kurtosis', 'r', 'intercept', 'time_gradient' ] intensity_min = np.log10(100) r_max = 0.8 dl1_to_dl2.build_models(args.gammafile, args.protonfile, features, intensity_min=intensity_min, save_models=args.storerf, path_models=args.path_models, config_file=args.config_file)
def main(): args = parser.parse_args() custom_config = {} if args.config_file is not None: custom_config = read_configuration_file(args.config_file) config = replace_config(standard_config, custom_config) subarray_info = SubarrayDescription.from_hdf(args.gammatest) tel_id = config["allowed_tels"][0] if "allowed_tels" in config else 1 focal_length = subarray_info.tel[tel_id].optics.equivalent_focal_length reg_energy, reg_disp_norm, cls_disp_sign, cls_gh = dl1_to_dl2.build_models( args.gammafile, args.protonfile, save_models=args.save_models, path_models=args.path_models, custom_config=config, ) gammas = filter_events( pd.read_hdf(args.gammatest, key=dl1_params_lstcam_key), config["events_filters"], ) proton = filter_events( pd.read_hdf(args.protontest, key=dl1_params_lstcam_key), config["events_filters"], ) data = pd.concat([gammas, proton], ignore_index=True) dl2 = dl1_to_dl2.apply_models(data, cls_gh, reg_energy, reg_disp_norm=reg_disp_norm, cls_disp_sign=cls_disp_sign, focal_length=focal_length, custom_config=config) ####PLOT SOME RESULTS##### selected_gammas = dl2.query('reco_type==0 & mc_type==0') if (len(selected_gammas) == 0): log.warning('No gammas selected, I will not plot any output') sys.exit() plot_dl2.plot_features(dl2) if not args.batch: plt.show() plot_dl2.energy_results(selected_gammas) if not args.batch: plt.show() plot_dl2.direction_results(selected_gammas) if not args.batch: plt.show() plot_dl2.plot_disp_vector(selected_gammas) if not args.batch: plt.show() plot_dl2.plot_pos(dl2) if not args.batch: plt.show() plot_dl2.plot_roc_gamma(dl2) if not args.batch: plt.show() plot_dl2.plot_models_features_importances(args.path_models, args.config_file) if not args.batch: plt.show() plt.hist(dl2[dl2['mc_type'] == 101]['gammaness'], bins=100) plt.hist(dl2[dl2['mc_type'] == 0]['gammaness'], bins=100) if not args.batch: plt.show()
default='./results/') args = parser.parse_args() if __name__ == '__main__': #Train the models features = [ 'intensity', 'time_gradient', 'width', 'length', 'wl', 'phi', 'psi' ] reg_energy, reg_disp, cls_gh = dl1_to_dl2.build_models( args.gammafile, args.protonfile, features, save_models=args.storerf, path_models=args.path_models, ) #Get out the data from the Simtelarray file: data = dl0_to_dl1.get_events(args.datafile, False) #Apply the models to the data dl2 = dl1_to_dl2.apply_models(data, features, cls_gh, reg_energy, reg_disp) if args.storeresults == True: #Store results if not os.path.exists(args.outdir): os.mkdir(args.outdir)
#Train the models features = [ 'intensity', 'width', 'length', 'x', 'y', 'psi', 'phi', 'wl', 'skewness', 'kurtosis', 'r', 'time_gradient', 'intercept', 'leakage', 'n_islands' ] intensity_min = np.log10(200) leakage_cut = 0.2 r_min = 0.15 reg_energy, reg_disp_vector, cls_gh = dl1_to_dl2.build_models( args.gammafile, args.protonfile, features, intensity_min=intensity_min, leakage_cut=leakage_cut, save_models=args.storerf, path_models=args.path_models, config_file=args.config_file) gammas = dl1_to_dl2.filter_events(pd.read_hdf(args.gammatest, key='events/LSTCam'), leakage_cut=leakage_cut, intensity_min=intensity_min, r_min=r_min) proton = dl1_to_dl2.filter_events(pd.read_hdf(args.protontest, key='events/LSTCam'), leakage_cut=leakage_cut, intensity_min=intensity_min, r_min=r_min)
parser.add_argument('--opath', '-o', action='store', type=str, dest='path_models', help='Path to store the resulting RF', default='./trained_models/') parser.add_argument('--config_file', '-conf', action='store', type=str, dest='config_file', help='Path to a configuration file. If none is given, a standard configuration is applied', default=None ) args = parser.parse_args() if __name__ == '__main__': #Train the models config = {} if args.config_file is not None: try: config = read_configuration_file(args.config_file) except("Custom configuration could not be loaded !!!"): pass dl1_to_dl2.build_models(args.gammafile, args.protonfile, save_models=args.storerf, path_models=args.path_models, custom_config=config, )
if __name__ == '__main__': custom_config = {} if args.config_file is not None: try: custom_config = read_configuration_file(args.config_file) except("Custom configuration could not be loaded !!!"): pass config = replace_config(standard_config, custom_config) reg_energy, reg_disp_vector, cls_gh = dl1_to_dl2.build_models( args.gammafile, args.protonfile, save_models=args.storerf, path_models=args.path_models, custom_config=config, ) gammas = filter_events(pd.read_hdf(args.gammatest, key='events/LSTCam'), config["events_filters"] ) proton = filter_events(pd.read_hdf(args.protontest, key='events/LSTCam'), config["events_filters"], ) data = pd.concat([gammas, proton], ignore_index=True) dl2 = dl1_to_dl2.apply_models(data, cls_gh, reg_energy, reg_disp_vector, custom_config=config)