def objective(args):
"""Return limit computed on test set with model trained on training set."""
global tried_points
global all_losses
global n_tries
n_tries += 1
args['fit']['epochs'] = int(args['fit']['epochs'])
print("\nTry {}: testing model for sampling point: {}".format(
n_tries, args))
model = create_nonresonant_model(**args['model'])
history = model.fit(dataset.training_dataset,
dataset.training_targets,
sample_weight=dataset.training_weights,
verbose=False,
validation_data=(dataset.testing_dataset,
dataset.testing_targets,
dataset.testing_weights),
callbacks=callbacks,
**args['fit'])
limit = get_test_limit(dataset, model)
if limit is None:
return {'status': STATUS_FAIL}
tried_points.append(args)
all_losses.append(limit)
save_results()
plotTools.draw_keras_history(history,
output_dir=output_folder,
output_name="loss_try_{}_limit_{}.pdf".format(
n_tries, limit))
best_so_far = np.min(all_losses)
print("Try {}: Expected limit: {} -- Best so far: {}".format(
n_tries, limit, best_so_far))
return {'loss': limit, 'status': STATUS_OK}
history = model.fit(training_dataset, training_targets, sample_weight=training_weights, batch_size=batch_size, nb_epoch=nb_epoch,
verbose=True, validation_data=(testing_dataset, testing_targets, testing_weights), callbacks=callbacks)
end_time = timer()
training_time = datetime.timedelta(seconds=(end_time - start_time))
save_training_parameters(output_folder, model,
batch_size=batch_size, nb_epoch=nb_epoch,
training_time=str(training_time),
masses=signal_masses,
with_mass_column=add_mass_column,
inputs=inputs,
cut=cut,
weights=resonant_weights)
plotTools.draw_keras_history(history, output_dir=output_folder, output_name="loss.pdf")
# Save history
print("Saving model training history...")
output_history_filename = 'hh_resonant_trained_model_history.pklz'
output_history_filename = os.path.join(output_folder, output_history_filename)
with gzip.open(output_history_filename, 'wb') as f:
pickle.dump(history.history, f)
print("Done.")
model = keras.models.load_model(output_model_filename)
export_for_lwtnn(model, output_model_filename)
draw_resonant_training_plots(model, dataset, output_folder, split_by_mass=add_mass_column)
print("All done. Training time: %s" % str(training_time))
# verbose=True,
# validation_data=([testing_features, testing_independent], testing_targets, testing_weights),
# callbacks=callbacks_lsmi)
# Save only after the LSMI training, since that modifies the base model's weights
discr.save(output_model_filename)
lsmi.save(output_model_lsmi_filename)
end_time = timer()
training_time = datetime.timedelta(seconds=(end_time - start_time))
# Save history
print("Saving model training history...")
for history in [history_discr, history_lsmi, history_discr_batched]:
plotTools.draw_keras_history(history,
output_dir=output_folder,
output_name="history_" + history.name +
".pdf")
########################################
##### Re-loading model and evaluate ####
########################################
model = keras.models.load_model(output_model_filename)
export_for_lwtnn(model, output_model_filename)
draw_non_resonant_training_plots(model,
dataset,
output_folder,
split_by_parameters=add_parameters_columns)
draw_nn_vs_independent(model, dataset, np.linspace(0, 400, 20), output_folder)
def objective_cv(args):
"""Return limit averaged over k-fold cross-validated models,
on training data only. Test data NOT used."""
global tried_points
global all_losses
global n_tries
n_tries += 1
args['fit']['epochs'] = int(args['fit']['epochs'])
print("\nTry {}: testing model for sampling point: {}".format(
n_tries, args))
k_folds = args['k_folds']
skf = StratifiedKFold(n_splits=k_folds)
limits = []
for i, (train, test) in enumerate(
skf.split(dataset.training_dataset, dataset.training_targets)):
print("Training on fold {} out of {}...".format(i + 1, k_folds))
model = create_nonresonant_model(**args['model'])
history = model.fit(dataset.training_dataset[train],
dataset.training_targets[train],
sample_weight=dataset.training_weights[train],
verbose=False,
validation_data=(dataset.training_dataset[test],
dataset.training_targets[test],
dataset.training_weights[test]),
callbacks=callbacks,
**args['fit'])
limit = get_test_limit(dataset, model, test)
if limit is not None:
print("... Limit is {}".format(limit))
limits.append(limit)
plotTools.draw_keras_history(
history,
output_dir=output_folder,
output_name="loss_try_{}_fold_{}_limit_{}.pdf".format(
n_tries, i, limit))
else:
print("Try {} failed: Could not compute a limit!".format(n_tries))
return {'status': STATUS_FAIL}
limit = np.mean(limits)
tried_points.append(args)
all_losses.append(limit)
save_results()
best_so_far = np.min(all_losses)
print(
"Try {}: K-fold limits: {} -- Average expected limit: {} -- Best so far: {}"
.format(n_tries, limits, limit, best_so_far))
return {'loss': limit, 'status': STATUS_OK}
batch_size = 5000
nb_epoch = 50
history_discr = discr.fit(training_features,
training_targets,
sample_weight=training_weights,
batch_size=batch_size,
epochs=nb_epoch,
verbose=True,
validation_data=(testing_features,
testing_targets,
testing_weights),
callbacks=callbacks)
plotTools.draw_keras_history(history_discr, output_folder,
'discr_loss.pdf')
discr.save_weights(discr_weight_file)
if (not load_advers) and train_advers:
batch_size = 2048
nb_epoch = 40
print("\nTraining adversary!\n")
scores_train_bkg = discr.predict(training_features[bkg_cut],
batch_size=20000)
scores_test_bkg = discr.predict(testing_features[bkg_cut_test],
batch_size=20000)
history_advers = advers.fit(
scores_train_bkg,
def training_resonant(variables, selection, masses, output_folder,
output_model_filename):
parametricDNN = False
if len(masses) > 1: parametricDNN = True
# Loading Signal and Backgrounds
dataset = DatasetManager(variables, selection, masses)
dataset.loadResonantSignal(treename)
dataset.loadBackgrounds(treename)
dataset.split()
training_dataset, training_targets = dataset.get_training_combined_dataset_and_targets(
)
training_weights = dataset.get_training_combined_weights()
testing_dataset, testing_targets = dataset.get_testing_combined_dataset_and_targets(
)
testing_weights = dataset.get_testing_combined_weights()
n_inputs = len(variables)
if len(masses) > 1: ## add mass column
n_inputs += 1
# You create here the real DNN structure
model = create_resonant_model(n_inputs)
# callback: set of functions to be applied at given stages of the training procedure. You can use callbacks to get a view on internal states/statistics of model during training
callbacks = []
callbacks.append(
ModelCheckpoint(output_model_filename,
monitor='val_loss',
verbose=False,
save_best_only=True,
mode='auto')) #Save model after every epoch
# output_logs_folder = os.path.join('hh_resonant_trained_models', 'logs', output_suffix)
# callbacks.append(keras.callbacks.TensorBoard(log_dir=output_logs_folder, histogram_freq=1, write_graph=True, write_images=False))
callbacks.append(
LearningRateScheduler(lr_scheduler)
) # Provide learnign rate per epoch. lr_scheduler = have to be a function of epoch.
#n_inputs = len(inputs)
#if add_mass_column:
# n_inputs += 1
# You create here the real DNN structure
#model = create_resonant_model(n_inputs)
start_time = timer()
# You do the training with the compiled model
history = model.fit(training_dataset,
training_targets,
sample_weight=training_weights,
batch_size=batch_size,
epochs=epochs,
verbose=True,
validation_data=(testing_dataset, testing_targets,
testing_weights),
callbacks=callbacks)
end_time = timer()
training_time = datetime.timedelta(seconds=(end_time - start_time))
save_training_parameters(output_folder,
model,
batch_size=batch_size,
epochs=epochs,
training_time=str(training_time),
masses=masses,
with_mass_column=parametricDNN,
inputs=variables,
cut=selection)
plotTools.draw_keras_history(history,
output_dir=output_folder,
output_name="loss.pdf")
# Save history
print("Saving model training history...")
output_history_filename = 'hh_resonant_trained_model_history.pklz'
output_history_filename = os.path.join(output_folder,
output_history_filename)
with gzip.open(output_history_filename, 'wb') as f:
pickle.dump(history.history, f)
print("Done.")
print("All done. Training time: %s" % str(training_time))
return dataset, model