def run_estimation_iter(model,
result_row,
i_iter,
config,
valid_generator,
test_generator,
calibs,
n_bins=N_BINS,
tolerance=10):
logger = logging.getLogger()
logger.info('-' * 45)
logger.info(f'iter : {i_iter}')
flush(logger)
iter_directory = os.path.join(model.results_path, f'iter_{i_iter}')
os.makedirs(iter_directory, exist_ok=True)
result_row['i'] = i_iter
result_row['n_test_samples'] = test_generator.n_samples
suffix = config.get_suffix()
logger.info('Generate testing data')
test_generator.reset()
X_test, y_test, w_test = test_generator.generate(
*config.TRUE, n_samples=config.N_TESTING_SAMPLES, no_grad=True)
# PLOT SUMMARIES
evaluate_summary_computer(model,
X_test,
y_test,
w_test,
n_bins=n_bins,
prefix='',
suffix=suffix,
directory=iter_directory)
# CALIBRATION
config = calibrates(calibs, config, X_test, w_test)
for name, value in config.CALIBRATED.items():
result_row[name + "_calib"] = value
for name, value in config.CALIBRATED_ERROR.items():
result_row[name + "_calib_error"] = value
logger.info('Set up NLL computer')
compute_summaries = model.summary_computer(n_bins=n_bins)
compute_nll = NLLComputer(compute_summaries,
valid_generator,
X_test,
w_test,
config=config)
# NLL PLOTS
plot_nll_around_min(compute_nll, config.TRUE, iter_directory, suffix)
# MINIMIZE NLL
logger.info('Prepare minuit minimizer')
minimizer = get_minimizer(compute_nll,
config.CALIBRATED,
config.CALIBRATED_ERROR,
tolerance=tolerance)
result_row.update(
evaluate_minuit(minimizer, config.TRUE, iter_directory, suffix=suffix))
return result_row.copy()
def run_iter(i_cv, i_iter, config, valid_generator, test_generator, directory):
logger = logging.getLogger()
result_row = dict(i_cv=i_cv, i=i_iter)
iter_directory = os.path.join(directory, f'iter_{i_iter}')
os.makedirs(iter_directory, exist_ok=True)
logger.info(f"True Parameters = {config.TRUE}")
suffix = f'-mu={config.TRUE.mu:1.2f}_tes={config.TRUE.tes}_jes={config.TRUE.jes}_les={config.TRUE.les}'
# suffix += f'_nasty_bkg={config.TRUE.nasty_bkg}_sigma_soft={config.TRUE.sigma_soft}'
# TODO : Remove
logger.info(f"Calib Parameters = {config.CALIBRATED}")
X_test, y_test, w_test = test_generator.generate(
*config.CALIBRATED, n_samples=config.N_TESTING_SAMPLES)
logger.info(
f" s = {w_test[y_test==1].sum()} || b = {w_test[y_test==0].sum()} ")
# TODO : END
X_test, y_test, w_test = test_generator.generate(
*config.TRUE, n_samples=config.N_TESTING_SAMPLES)
debug_label(y_test)
# TODO : Remove
logger.info(
f" s = {w_test[y_test==1].sum()} || b = {w_test[y_test==0].sum()} ")
# TODO : END
compute_nll = LabelNLL(valid_generator, y_test, w_test, config=config)
plot_nll_around_min(compute_nll, config.TRUE, iter_directory, suffix)
logger.info('Prepare minuit minimizer')
minimizer = get_minimizer(compute_nll, config.CALIBRATED,
config.CALIBRATED_ERROR)
result_row.update(evaluate_minuit(minimizer, config.TRUE))
return result_row