def run_conditional_estimation(args, i_cv):
logger = logging.getLogger()
print_line()
logger.info('Running iter n°{}'.format(i_cv))
print_line()
result_row = {'i_cv': i_cv}
# LOAD/GENERATE DATA
logger.info('Set up data generator')
config = Config()
seed = SEED + i_cv * 5
train_generator = GeneratorTorch(seed, cuda=args.cuda)
train_generator = TrainGenerator(train_generator, cuda=args.cuda)
valid_generator = Generator(seed + 1)
test_generator = Generator(seed + 2)
# SET MODEL
logger.info('Set up classifier')
model = build_model(args, i_cv)
os.makedirs(model.results_path, exist_ok=True)
flush(logger)
# TRAINING / LOADING
train_or_load_neural_net(model, train_generator, retrain=args.retrain)
# CHECK TRAINING
logger.info('Generate validation data')
X_valid, y_valid, w_valid = valid_generator.generate(
*config.CALIBRATED, n_samples=config.N_VALIDATION_SAMPLES)
result_row.update(evaluate_neural_net(model, prefix='valid'))
result_row.update(
evaluate_classifier(model, X_valid, y_valid, w_valid, prefix='valid'))
# MEASUREMENT
evaluate_summary_computer(model,
X_valid,
y_valid,
w_valid,
n_bins=N_BINS,
prefix='valid_',
suffix='')
iter_results = [
run_conditional_estimation_iter(model,
result_row,
i,
test_config,
valid_generator,
test_generator,
n_bins=N_BINS)
for i, test_config in enumerate(config.iter_test_config())
]
conditional_estimate = pd.concat(iter_results)
conditional_estimate['i_cv'] = i_cv
fname = os.path.join(model.results_path, "conditional_estimations.csv")
conditional_estimate.to_csv(fname)
logger.info('DONE')
return conditional_estimate
def run(args, i_cv):
logger = logging.getLogger()
print_line()
logger.info('Running iter n°{}'.format(i_cv))
print_line()
result_row = {'i_cv': i_cv}
# LOAD/GENERATE DATA
logger.info('Set up data generator')
config = Config()
seed = SEED + i_cv * 5
train_generator = Generator(seed)
valid_generator = Generator(seed + 1)
test_generator = Generator(seed + 2)
train_generator = TrainGenerator(param_generator, train_generator)
# SET MODEL
logger.info('Set up regressor')
model = build_model(args, i_cv)
os.makedirs(model.results_path, exist_ok=True)
flush(logger)
# TRAINING / LOADING
train_or_load_neural_net(model, train_generator, retrain=args.retrain)
# CHECK TRAINING
logger.info('Generate validation data')
X_valid, y_valid, w_valid = valid_generator.generate(
*config.CALIBRATED, n_samples=config.N_VALIDATION_SAMPLES)
result_row.update(evaluate_neural_net(model, prefix='valid'))
evaluate_regressor(model, prefix='valid')
# MEASUREMENT
result_row['nfcn'] = NCALL
result_table = [
run_iter(model, result_row, i, test_config, valid_generator,
test_generator)
for i, test_config in enumerate(config.iter_test_config())
]
result_table = pd.DataFrame(result_table)
result_table.to_csv(os.path.join(model.results_path, 'results.csv'))
logger.info('Plot params')
param_names = config.PARAM_NAMES
for name in param_names:
plot_params(name,
result_table,
title=model.full_name,
directory=model.results_path)
logger.info('DONE')
return result_table
def main():
# BASIC SETUP
logger = set_logger()
args = REG_parse_args(
main_description="Training launcher for Regressor on S3D2 benchmark")
logger.info(args)
flush(logger)
# Setup model
logger.info("Setup model")
model = build_model(args, 0)
os.makedirs(model.results_directory, exist_ok=True)
# Setup data
logger.info("Setup data")
config = Config()
config_table = evaluate_config(config)
config_table.to_csv(
os.path.join(model.results_directory, 'config_table.csv'))
seed = SEED + 99999
train_generator = TrainGenerator(param_generator, Generator(seed))
valid_generator = Generator(seed + 1)
test_generator = Generator(seed + 2)
i_cv = 0
result_row = {'i_cv': i_cv}
# TRAINING / LOADING
train_or_load_neural_net(model, train_generator, retrain=args.retrain)
# CHECK TRAINING
result_row.update(evaluate_neural_net(model, prefix='valid'))
evaluate_regressor(model, prefix='valid')
print_line()
result_table = [
run_iter(model, result_row, i, test_config, valid_generator,
test_generator)
for i, test_config in enumerate(config.iter_test_config())
]
result_table = pd.DataFrame(result_table)
result_table.to_csv(os.path.join(model.results_directory, 'results.csv'))
logger.info('Plot params')
param_names = [CALIB_PARAM_NAME]
for name in param_names:
plot_params(name,
result_table,
title=model.full_name,
directory=model.results_directory)
logger.info('DONE')
def run_estimation(args, i_cv):
logger = logging.getLogger()
print_line()
logger.info('Running iter n°{}'.format(i_cv))
print_line()
result_row = {'i_cv': i_cv}
# LOAD/GENERATE DATA
logger.info('Set up data generator')
config = Config()
seed = SEED + i_cv * 5
train_generator = Generator(seed)
train_generator = TrainGenerator(param_generator, train_generator)
valid_generator = Generator(seed+1)
test_generator = Generator(seed+2)
# SET MODEL
logger.info('Set up classifier')
model = build_model(args, i_cv)
os.makedirs(model.results_path, exist_ok=True)
flush(logger)
# TRAINING / LOADING
train_or_load_pivot(model, train_generator, config.N_TRAINING_SAMPLES*N_AUGMENT, retrain=args.retrain)
# CHECK TRAINING
logger.info('Generate validation data')
X_valid, y_valid, w_valid = valid_generator.generate(*config.CALIBRATED, n_samples=config.N_VALIDATION_SAMPLES)
result_row.update(evaluate_neural_net(model, prefix='valid'))
result_row.update(evaluate_classifier(model, X_valid, y_valid, w_valid, prefix='valid'))
# MEASUREMENT
calib_r = load_calib_r(DATA_NAME, BENCHMARK_NAME)
calib_lam = load_calib_lam(DATA_NAME, BENCHMARK_NAME)
evaluate_summary_computer(model, X_valid, y_valid, w_valid, n_bins=N_BINS, prefix='valid_', suffix='')
iter_results = [run_estimation_iter(model, result_row, i, test_config, valid_generator, test_generator, calib_r, calib_lam, n_bins=N_BINS)
for i, test_config in enumerate(config.iter_test_config())]
result_table = pd.DataFrame(iter_results)
result_table.to_csv(os.path.join(model.results_path, 'estimations.csv'))
logger.info('Plot params')
param_names = config.PARAM_NAMES
for name in param_names:
plot_params(name, result_table, title=model.full_name, directory=model.results_path)
logger.info('DONE')
return result_table
def run(args, i_cv):
logger = logging.getLogger()
print_line()
logger.info('Running iter n°{}'.format(i_cv))
print_line()
result_row = {'i_cv': i_cv}
# LOAD/GENERATE DATA
logger.info('Set up data generator')
config = Config()
seed = SEED + i_cv * 5
# train_generator = Generator(seed)
# valid_generator = Generator(seed+1)
test_generator = Generator(seed+2)
# SET MODEL
# logger.info('Set up classifier')
model = build_model(args, i_cv)
# flush(logger)
# TRAINING / LOADING
# train_or_load_classifier(model, train_generator, config.CALIBRATED, config.N_TRAINING_SAMPLES, retrain=args.retrain)
# CHECK TRAINING
logger.info('Generate validation data')
# X_valid, y_valid, w_valid = valid_generator.generate(*config.CALIBRATED, n_samples=config.N_VALIDATION_SAMPLES)
# result_row.update(evaluate_classifier(model, X_valid, y_valid, w_valid, prefix='valid'))
# MEASUREMENT
N_BINS = 10
# evaluate_summary_computer(model, X_valid, y_valid, w_valid, n_bins=N_BINS, prefix='valid_', suffix='')
result_table = [run_iter(model, result_row, i, i_cv, args, test_config, test_generator, n_bins=N_BINS)
for i, test_config in enumerate(config.iter_test_config())]
result_table = pd.DataFrame(result_table)
result_table.to_csv(os.path.join(model.results_path, 'results.csv'))
logger.info('Plot params')
param_names = config.PARAM_NAMES
for name in param_names:
plot_params(name, result_table, title=model.full_name, directory=model.path)
logger.info('DONE')
return result_table
def run(args, i_cv):
logger = logging.getLogger()
print_line()
logger.info('Running iter n°{}'.format(i_cv))
print_line()
directory = os.path.join(DIRECTORY, f'cv_{i_cv}')
os.makedirs(directory, exist_ok=True)
config = Config()
seed = SEED + i_cv * 5
train_generator = Generator(seed)
valid_generator = Generator(seed+1)
test_generator = Generator(seed+2)
N_BINS = 10
X_train, y_train, w_train = train_generator.generate(*config.CALIBRATED, n_samples=config.N_TRAINING_SAMPLES)
compute_summaries = HistogramSummaryComputer(n_bins=N_BINS).fit(X_train)
result_table = [run_iter(compute_summaries, i_cv, i, test_config, valid_generator, test_generator, directory) for i, test_config in enumerate(config.iter_test_config())]
result_table = pd.DataFrame(result_table)
result_table.to_csv(os.path.join(directory, 'results.csv'))
logger.info('Plot params')
param_names = config.PARAM_NAMES
for name in param_names:
plot_params(name, result_table, title='Likelihood fit', directory=directory)
return result_table
