def test_build_models(simulated_dl1_file, rf_models):
infile = simulated_dl1_file
custom_config = {
"n_training_events": {
"gamma_regressors": 0.99,
"gamma_tmp_regressors": 0.78,
"gamma_classifier": 0.21,
"proton_classifier": 0.98
}
}
reg_energy, reg_disp_norm, cls_disp_sign, cls_gh = build_models(
infile,
infile,
save_models=False,
free_model_memory=False,
custom_config=custom_config)
build_models(infile,
infile,
save_models=True,
free_model_memory=True,
custom_config=custom_config)
import joblib
joblib.dump(reg_energy, rf_models["energy"], compress=3)
joblib.dump(cls_gh, rf_models["gh_sep"], compress=3)
joblib.dump(reg_disp_norm, rf_models["disp_norm"], compress=3)
joblib.dump(cls_disp_sign, rf_models["disp_sign"], compress=3)
def main():
args = parser.parse_args()
#Train the models
config = {}
if args.config_file is not None:
config = read_configuration_file(args.config_file)
dl1_to_dl2.build_models(
args.gammafile,
args.protonfile,
save_models=args.save_models,
path_models=args.path_models,
custom_config=config,
)
def main():
#Train the models
config = {}
if args.config_file is not None:
try:
config = read_configuration_file(args.config_file)
except("Custom configuration could not be loaded !!!"):
pass
dl1_to_dl2.build_models(args.gammafile,
args.protonfile,
save_models=args.storerf,
path_models=args.path_models,
custom_config=config,
)
def test_build_models():
from lstchain.reco.dl1_to_dl2 import build_models
infile = dl1_file
reg_energy, reg_disp, cls_gh = build_models(infile, infile, custom_config=custom_config, save_models=False)
import joblib
joblib.dump(reg_energy, file_model_energy)
joblib.dump(reg_disp, file_model_disp)
joblib.dump(cls_gh, file_model_gh_sep)
def test_fail_build_models(simulated_dl1_file):
custom_config = {
"n_training_events": {
"gamma_regressors": 0.99,
"gamma_tmp_regressors": 0.78,
"gamma_classifier": 0.31,
"proton_classifier": 0.98
}
}
_, _, _, _ = build_models(simulated_dl1_file,
simulated_dl1_file,
save_models=False,
custom_config=custom_config)
def test_build_models():
from lstchain.reco.dl1_to_dl2 import build_models
infile = 'dl1_gamma_test_large.h5'
features = ['intensity', 'width', 'length']
reg_energy, reg_disp, cls_gh = build_models(infile,
infile,
features,
save_models=True)
from sklearn.externals import joblib
joblib.dump(reg_energy, 'rf_energy.pkl')
joblib.dump(reg_disp, 'rf_disp.pkl')
joblib.dump(cls_gh, 'rf_cls_gh.pkl')
def test_build_models(simulated_dl1_file, rf_models):
from lstchain.reco.dl1_to_dl2 import build_models
infile = simulated_dl1_file
reg_energy, reg_disp, cls_gh = build_models(infile,
infile,
custom_config=standard_config,
save_models=False)
import joblib
joblib.dump(reg_energy, rf_models["energy"])
joblib.dump(reg_disp, rf_models["disp"])
joblib.dump(cls_gh, rf_models["gh_sep"])
def main():
custom_config = {}
if args.config_file is not None:
try:
custom_config = read_configuration_file(args.config_file)
except ("Custom configuration could not be loaded !!!"):
pass
config = replace_config(standard_config, custom_config)
reg_energy, reg_disp_vector, cls_gh = dl1_to_dl2.build_models(
args.gammafile,
args.protonfile,
save_models=args.storerf,
path_models=args.path_models,
custom_config=config,
)
gammas = filter_events(
pd.read_hdf(args.gammatest, key=dl1_params_lstcam_key),
config["events_filters"],
)
proton = filter_events(
pd.read_hdf(args.protontest, key=dl1_params_lstcam_key),
config["events_filters"],
)
data = pd.concat([gammas, proton], ignore_index=True)
dl2 = dl1_to_dl2.apply_models(data,
cls_gh,
reg_energy,
reg_disp_vector,
custom_config=config)
####PLOT SOME RESULTS#####
selected_gammas = dl2.query('reco_type==0 & mc_type==0')
if (len(selected_gammas) == 0):
log.warning('No gammas selected, I will not plot any output')
sys.exit()
plot_dl2.plot_features(dl2)
if not args.batch:
plt.show()
plot_dl2.energy_results(selected_gammas)
if not args.batch:
plt.show()
plot_dl2.direction_results(selected_gammas)
if not args.batch:
plt.show()
plot_dl2.plot_disp_vector(selected_gammas)
if not args.batch:
plt.show()
plot_dl2.plot_pos(dl2)
if not args.batch:
plt.show()
plot_dl2.plot_roc_gamma(dl2)
if not args.batch:
plt.show()
plot_dl2.plot_models_features_importances(args.path_models,
args.config_file)
if not args.batch:
plt.show()
plt.hist(dl2[dl2['mc_type'] == 101]['gammaness'], bins=100)
plt.hist(dl2[dl2['mc_type'] == 0]['gammaness'], bins=100)
if not args.batch:
plt.show()
def main():
custom_config = {}
if args.config_file is not None:
try:
custom_config = read_configuration_file(args.config_file)
except ("Custom configuration could not be loaded !!!"):
pass
config = replace_config(standard_config, custom_config)
reg_energy, reg_disp_vector, cls_gh = dl1_to_dl2.build_models(
args.gammafile,
args.protonfile,
save_models=args.storerf,
path_models=args.path_models,
custom_config=config,
)
gammas = filter_events(
pd.read_hdf(args.gammatest, key=dl1_params_lstcam_key),
config["events_filters"],
)
proton = filter_events(
pd.read_hdf(args.protontest, key=dl1_params_lstcam_key),
config["events_filters"],
)
data = pd.concat([gammas, proton], ignore_index=True)
dl2 = dl1_to_dl2.apply_models(data,
cls_gh,
reg_energy,
reg_disp_vector,
custom_config=config)
####PLOT SOME RESULTS#####
gammas = dl2[dl2.gammaness >= 0.5]
protons = dl2[dl2.gammaness < 0.5]
gammas.reco_type = 0
protons.reco_type = 1
focal_length = 28 * u.m
src_pos_reco = utils.reco_source_position_sky(
gammas.x.values * u.m, gammas.y.values * u.m,
gammas.reco_disp_dx.values * u.m, gammas.reco_disp_dy.values * u.m,
focal_length, gammas.mc_alt_tel.values * u.rad,
gammas.mc_az_tel.values * u.rad)
plot_dl2.plot_features(dl2)
plt.show()
plot_dl2.plot_e(gammas, 10, 1.5, 3.5)
plt.show()
plot_dl2.calc_resolution(gammas)
plt.show()
plot_dl2.plot_e_resolution(gammas, 10, 1.5, 3.5)
plt.show()
plot_dl2.plot_disp_vector(gammas)
plt.show()
try:
ctaplot.plot_theta2(
gammas.mc_alt,
np.arctan(np.tan(gammas.mc_az)),
src_pos_reco.alt.rad,
np.arctan(np.tan(src_pos_reco.az.rad)),
bins=50,
range=(0, 1),
)
plt.show()
ctaplot.plot_angular_res_per_energy(
src_pos_reco.alt.rad, np.arctan(np.tan(src_pos_reco.az.rad)),
gammas.mc_alt, np.arctan(np.tan(gammas.mc_az)), gammas.mc_energy)
plt.show()
except:
pass
regression_features = config["regression_features"]
classification_features = config["classification_features"]
plt.show()
plot_dl2.plot_pos(dl2)
plt.show()
plot_dl2.plot_ROC(cls_gh, dl2, classification_features, -1)
plt.show()
plot_dl2.plot_importances(cls_gh, classification_features)
plt.show()
plot_dl2.plot_importances(reg_energy, regression_features)
plt.show()
plot_dl2.plot_importances(reg_disp_vector, regression_features)
plt.show()
plt.hist(dl2[dl2['mc_type'] == 101]['gammaness'], bins=100)
plt.hist(dl2[dl2['mc_type'] == 0]['gammaness'], bins=100)
plt.show()
parser.add_argument(
'--config_file',
'-conf',
action='store',
type=str,
dest='config_file',
help=
'Path to a configuration file. If none is given, a standard configuration is applied',
default=None)
args = parser.parse_args()
if __name__ == '__main__':
#Train the models
features = [
'intensity', 'width', 'length', 'x', 'y', 'psi', 'phi', 'wl',
'skewness', 'kurtosis', 'r', 'intercept', 'time_gradient'
]
intensity_min = np.log10(100)
r_max = 0.8
dl1_to_dl2.build_models(args.gammafile,
args.protonfile,
features,
intensity_min=intensity_min,
save_models=args.storerf,
path_models=args.path_models,
config_file=args.config_file)
def main():
args = parser.parse_args()
custom_config = {}
if args.config_file is not None:
custom_config = read_configuration_file(args.config_file)
config = replace_config(standard_config, custom_config)
subarray_info = SubarrayDescription.from_hdf(args.gammatest)
tel_id = config["allowed_tels"][0] if "allowed_tels" in config else 1
focal_length = subarray_info.tel[tel_id].optics.equivalent_focal_length
reg_energy, reg_disp_norm, cls_disp_sign, cls_gh = dl1_to_dl2.build_models(
args.gammafile,
args.protonfile,
save_models=args.save_models,
path_models=args.path_models,
custom_config=config,
)
gammas = filter_events(
pd.read_hdf(args.gammatest, key=dl1_params_lstcam_key),
config["events_filters"],
)
proton = filter_events(
pd.read_hdf(args.protontest, key=dl1_params_lstcam_key),
config["events_filters"],
)
data = pd.concat([gammas, proton], ignore_index=True)
dl2 = dl1_to_dl2.apply_models(data,
cls_gh,
reg_energy,
reg_disp_norm=reg_disp_norm,
cls_disp_sign=cls_disp_sign,
focal_length=focal_length,
custom_config=config)
####PLOT SOME RESULTS#####
selected_gammas = dl2.query('reco_type==0 & mc_type==0')
if (len(selected_gammas) == 0):
log.warning('No gammas selected, I will not plot any output')
sys.exit()
plot_dl2.plot_features(dl2)
if not args.batch:
plt.show()
plot_dl2.energy_results(selected_gammas)
if not args.batch:
plt.show()
plot_dl2.direction_results(selected_gammas)
if not args.batch:
plt.show()
plot_dl2.plot_disp_vector(selected_gammas)
if not args.batch:
plt.show()
plot_dl2.plot_pos(dl2)
if not args.batch:
plt.show()
plot_dl2.plot_roc_gamma(dl2)
if not args.batch:
plt.show()
plot_dl2.plot_models_features_importances(args.path_models,
args.config_file)
if not args.batch:
plt.show()
plt.hist(dl2[dl2['mc_type'] == 101]['gammaness'], bins=100)
plt.hist(dl2[dl2['mc_type'] == 0]['gammaness'], bins=100)
if not args.batch:
plt.show()
default='./results/')
args = parser.parse_args()
if __name__ == '__main__':
#Train the models
features = [
'intensity', 'time_gradient', 'width', 'length', 'wl', 'phi', 'psi'
]
reg_energy, reg_disp, cls_gh = dl1_to_dl2.build_models(
args.gammafile,
args.protonfile,
features,
save_models=args.storerf,
path_models=args.path_models,
)
#Get out the data from the Simtelarray file:
data = dl0_to_dl1.get_events(args.datafile, False)
#Apply the models to the data
dl2 = dl1_to_dl2.apply_models(data, features, cls_gh, reg_energy, reg_disp)
if args.storeresults == True:
#Store results
if not os.path.exists(args.outdir):
os.mkdir(args.outdir)
#Train the models
features = [
'intensity', 'width', 'length', 'x', 'y', 'psi', 'phi', 'wl',
'skewness', 'kurtosis', 'r', 'time_gradient', 'intercept', 'leakage',
'n_islands'
]
intensity_min = np.log10(200)
leakage_cut = 0.2
r_min = 0.15
reg_energy, reg_disp_vector, cls_gh = dl1_to_dl2.build_models(
args.gammafile,
args.protonfile,
features,
intensity_min=intensity_min,
leakage_cut=leakage_cut,
save_models=args.storerf,
path_models=args.path_models,
config_file=args.config_file)
gammas = dl1_to_dl2.filter_events(pd.read_hdf(args.gammatest,
key='events/LSTCam'),
leakage_cut=leakage_cut,
intensity_min=intensity_min,
r_min=r_min)
proton = dl1_to_dl2.filter_events(pd.read_hdf(args.protontest,
key='events/LSTCam'),
leakage_cut=leakage_cut,
intensity_min=intensity_min,
r_min=r_min)
parser.add_argument('--opath', '-o', action='store', type=str,
dest='path_models',
help='Path to store the resulting RF',
default='./trained_models/')
parser.add_argument('--config_file', '-conf', action='store', type=str,
dest='config_file',
help='Path to a configuration file. If none is given, a standard configuration is applied',
default=None
)
args = parser.parse_args()
if __name__ == '__main__':
#Train the models
config = {}
if args.config_file is not None:
try:
config = read_configuration_file(args.config_file)
except("Custom configuration could not be loaded !!!"):
pass
dl1_to_dl2.build_models(args.gammafile,
args.protonfile,
save_models=args.storerf,
path_models=args.path_models,
custom_config=config,
)
if __name__ == '__main__':
custom_config = {}
if args.config_file is not None:
try:
custom_config = read_configuration_file(args.config_file)
except("Custom configuration could not be loaded !!!"):
pass
config = replace_config(standard_config, custom_config)
reg_energy, reg_disp_vector, cls_gh = dl1_to_dl2.build_models(
args.gammafile,
args.protonfile,
save_models=args.storerf,
path_models=args.path_models,
custom_config=config,
)
gammas = filter_events(pd.read_hdf(args.gammatest, key='events/LSTCam'),
config["events_filters"]
)
proton = filter_events(pd.read_hdf(args.protontest, key='events/LSTCam'),
config["events_filters"],
)
data = pd.concat([gammas, proton], ignore_index=True)
dl2 = dl1_to_dl2.apply_models(data, cls_gh, reg_energy, reg_disp_vector, custom_config=config)
