def run_estimation_iter(model, result_row, i_iter, config, valid_generator, test_generator, n_bins=N_BINS): logger = logging.getLogger() logger.info('-'*45) logger.info(f'iter : {i_iter}') flush(logger) iter_directory = os.path.join(model.results_path, f'iter_{i_iter}') os.makedirs(iter_directory, exist_ok=True) result_row['i'] = i_iter result_row['n_test_samples'] = config.N_TESTING_SAMPLES suffix = f'-mu={config.TRUE.mu:1.2f}_r={config.TRUE.r}_lambda={config.TRUE.lam}' logger.info('Generate testing data') test_generator.reset() X_test, y_test, w_test = test_generator.generate(*config.TRUE, n_samples=config.N_TESTING_SAMPLES, no_grad=True) # PLOT SUMMARIES evaluate_summary_computer(model, X_test, y_test, w_test, n_bins=n_bins, prefix='', suffix=suffix, directory=iter_directory) logger.info('Set up NLL computer') compute_summaries = model.summary_computer(n_bins=n_bins) compute_nll = NLLComputer(compute_summaries, valid_generator, X_test, w_test, config=config) # NLL PLOTS plot_nll_around_min(compute_nll, config.TRUE, iter_directory, suffix) # MINIMIZE NLL logger.info('Prepare minuit minimizer') minimizer = get_minimizer(compute_nll, config.CALIBRATED, config.CALIBRATED_ERROR) result_row.update(evaluate_minuit(minimizer, config.TRUE, iter_directory, suffix=suffix)) return result_row.copy()
def run_iter(i_cv, i_iter, config, seed, directory): logger = logging.getLogger() logger.info('-' * 45) logger.info(f'iter : {i_iter}') result_row = dict(i_cv=i_cv, i=i_iter) iter_directory = os.path.join(directory, f'iter_{i_iter}') os.makedirs(iter_directory, exist_ok=True) logger.info(f"True Parameters = {config.TRUE}") suffix = f'-mu={config.TRUE.mu:1.2f}_r={config.TRUE.r}_lambda={config.TRUE.lam}' generator = Generator(seed) # test_generator data, label = generator.sample_event(*config.TRUE, size=config.N_TESTING_SAMPLES) result_row['n_test_samples'] = config.N_TESTING_SAMPLES debug_label(label) compute_nll = lambda r, lam, mu: generator.nll(data, r, lam, mu) plot_nll_around_min(compute_nll, config.TRUE, iter_directory, suffix) logger.info('Prepare minuit minimizer') minimizer = get_minimizer(compute_nll, config.CALIBRATED, config.CALIBRATED_ERROR) minimizer.precision = None result_row.update( evaluate_minuit(minimizer, config.TRUE, iter_directory, suffix=suffix)) return result_row
def run_iter(compute_summaries, i_cv, i_iter, config, valid_generator, test_generator, directory): logger = logging.getLogger() result_row = dict(i_cv=i_cv, i=i_iter) iter_directory = os.path.join(directory, f'iter_{i_iter}') os.makedirs(iter_directory, exist_ok=True) logger.info(f"True Parameters = {config.TRUE}") suffix = f'-mu={config.TRUE.mu:1.2f}_r={config.TRUE.r}_lambda={config.TRUE.lam}' X_test, y_test, w_test = test_generator.generate(*config.TRUE, n_samples=config.N_TESTING_SAMPLES) debug_label(y_test) compute_nll = NLLComputer(compute_summaries, valid_generator, X_test, w_test, config=config) plot_nll_around_min(compute_nll, config.TRUE, iter_directory, suffix) logger.info('Prepare minuit minimizer') minimizer = get_minimizer(compute_nll, config.CALIBRATED, config.CALIBRATED_ERROR) result_row.update(evaluate_minuit(minimizer, config.TRUE)) return result_row
def run_iter(model, result_row, i_iter, i_cv, args, config, test_generator, n_bins=10): logger = logging.getLogger() iter_directory = os.path.join(model.results_path, f'iter_{i_iter}') os.makedirs(iter_directory, exist_ok=True) result_row['i'] = i_iter suffix = f'-mu={config.TRUE.mu:1.2f}_r={config.TRUE.r}_lambda={config.TRUE.lam}' logger.info('Generate testing data') test_generator.reset() X_test, y_test, w_test = test_generator.generate(*config.TRUE, n_samples=config.N_TESTING_SAMPLES) # PLOT SUMMARIES # evaluate_summary_computer(model, X_test, y_test, w_test, n_bins=n_bins, prefix='', suffix=suffix, directory=iter_directory) # logger.info('Set up NLL computer') # compute_summaries = ClassifierSummaryComputer(model, n_bins=n_bins) # compute_nll = NLLComputer(compute_summaries, valid_generator, X_test, w_test, config=config) compute_nll = NLL(X_test, w_test, i_cv, args, config=config, n_bins=n_bins) # NLL PLOTS plot_nll_around_min(compute_nll, config.TRUE, iter_directory, suffix) # MINIMIZE NLL logger.info('Prepare minuit minimizer') minimizer = get_minimizer(compute_nll, config.CALIBRATED, config.CALIBRATED_ERROR) result_row.update(evaluate_minuit(minimizer, config.TRUE)) return result_row.copy()
def run_estimation_iter(model, result_row, i_iter, config, valid_generator, test_generator, calib_r, calib_lam, n_bins=10): logger = logging.getLogger() logger.info('-'*45) logger.info(f'iter : {i_iter}') flush(logger) iter_directory = os.path.join(model.results_path, f'iter_{i_iter}') os.makedirs(iter_directory, exist_ok=True) result_row['i'] = i_iter result_row['n_test_samples'] = config.N_TESTING_SAMPLES suffix = f'-mu={config.TRUE.mu:1.2f}_r={config.TRUE.r}_lambda={config.TRUE.lam}' logger.info('Generate testing data') test_generator.reset() X_test, y_test, w_test = test_generator.generate(*config.TRUE, n_samples=config.N_TESTING_SAMPLES) # PLOT SUMMARIES evaluate_summary_computer(model, X_test, y_test, w_test, n_bins=n_bins, prefix='', suffix=suffix, directory=iter_directory) # CALIBRATION config = calibrates(calib_r, calib_lam, config, X_test, w_test) for name, value in config.FITTED.items(): result_row[name+"_fitted"] = value for name, value in config.FITTED_ERROR.items(): result_row[name+"_fitted_error"] = value logger.info('Set up NLL computer') compute_summaries = ClassifierSummaryComputer(model, n_bins=n_bins) compute_nll = NLLComputer(compute_summaries, valid_generator, X_test, w_test, config=config) # NLL PLOTS plot_nll_around_min(compute_nll, config.TRUE, iter_directory, suffix) # MINIMIZE NLL logger.info('Prepare minuit minimizer') minimizer = get_minimizer(compute_nll, config.FITTED, config.FITTED_ERROR) result_row.update(evaluate_minuit(minimizer, config.TRUE, iter_directory, suffix=suffix)) return result_row.copy()
def run(args, i_cv): logger = logging.getLogger() print_line() logger.info('Running iter n°{}'.format(i_cv)) print_line() result_row = {'i_cv': i_cv} result_table = [] # LOAD/GENERATE DATA logger.info('Set up data generator') pb_config = Config() seed = config.SEED + i_cv * 5 train_generator = Synthetic3DGeneratorTorch(seed) valid_generator = S3D2(seed + 1) test_generator = S3D2(seed + 2) # SET MODEL logger.info('Set up inferno') model = build_model(args, i_cv) flush(logger) # TRAINING / LOADING train_or_load_inferno(model, train_generator, retrain=args.retrain) # CHECK TRAINING result_row.update(evaluate_neural_net(model)) logger.info('Generate validation data') X_valid, y_valid, w_valid = valid_generator.generate( pb_config.CALIBRATED_R, pb_config.CALIBRATED_LAMBDA, pb_config.CALIBRATED_MU, n_samples=pb_config.N_VALIDATION_SAMPLES) # MEASUREMENT N_BINS = args.n_bins compute_summaries = model.compute_summaries for mu in pb_config.TRUE_MU_RANGE: true_params = Parameter(pb_config.TRUE.r, pb_config.TRUE.lam, mu) suffix = f'-mu={true_params.mu:1.2f}_r={true_params.r}_lambda={true_params.lam}' logger.info('Generate testing data') X_test, y_test, w_test = test_generator.generate( *true_params, n_samples=pb_config.N_TESTING_SAMPLES) # PLOT SUMMARIES evaluate_summary_computer(model, X_valid, y_valid, w_valid, X_test, w_test, n_bins=N_BINS, prefix='', suffix=suffix) logger.info('Set up NLL computer') compute_nll = S3D2NLL(compute_summaries, valid_generator, X_test, w_test) # NLL PLOTS plot_nll_around_min(compute_nll, true_params, model.path, suffix) # MINIMIZE NLL logger.info('Prepare minuit minimizer') minimizer = get_minimizer(compute_nll, pb_config.CALIBRATED, pb_config.CALIBRATED_ERROR) fmin, params = estimate(minimizer) result_row.update(evaluate_minuit(minimizer, fmin, params, true_params)) result_table.append(result_row.copy()) result_table = pd.DataFrame(result_table) logger.info('Plot params') param_names = pb_config.PARAM_NAMES for name in param_names: plot_params(name, result_table, title=model.full_name, directory=model.path) logger.info('DONE') return result_table