class NeuralNetwork:
"""
Features
- Run a single config_num
- Run multiple config_nums
- Will automatically write and save results of model - see save_dir and write_dir for more information
Notes:
- Supports Tensorboard
- By default, will load all addons
"""
def __init__(self,
channel,
gen,
binary=True,
write_filename='NN_output',
show_graph=False):
print(f'Loaded in {channel}, binary={binary}, gen={gen}')
self.addons_config_reco = {
'neutrino': {
'load_alpha': False,
'termination': 1000,
'imputer_mode': 'remove',
'save_alpha': True,
},
'met': {},
'ip': {},
'sv': {}
}
self.addons_config_gen = {
'neutrino': {
'load_alpha': False,
'termination': 1000,
'imputer_mode': 'remove',
'save_alpha': True,
},
'met': {},
'ip': {},
'sv': {}
}
self.show_graph = show_graph
self.channel = channel
self.binary = binary
self.write_filename = write_filename
self.gen = gen
self.save_dir = 'NN_output'
self.write_dir = 'NN_output'
self.model = None
def run(self,
config_num,
read=True,
from_hdf=True,
epochs=50,
batch_size=1024,
patience=20,
strict=False):
if not self.gen:
df = self.initialize(self.addons_config_reco,
read=read,
from_hdf=from_hdf,
strict=strict)
else:
df = self.initialize(self.addons_config_gen,
read=read,
from_hdf=from_hdf)
X_train, X_test, y_train, y_test = self.configure(df, config_num)
print(
f'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Training config {config_num}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
)
# self.model = self.custom_model()
# print('training with custom model')
if self.model is None:
print(f'Training with DEFAULT - kristof_model')
model = self.train(X_train,
X_test,
y_train,
y_test,
epochs=epochs,
batch_size=batch_size,
patience=patience)
if self.binary:
auc = self.evaluateBinary(model, X_test, y_test, self.history)
else:
w_a = df.w_a
w_b = df.w_b
auc = self.evaluate(model, X_test, y_test, self.history, w_a, w_b)
self.write(auc, self.history, self.addons_config_reco)
def runMultiple(self,
configs,
read=True,
from_hdf=True,
epochs=50,
batch_size=1024,
patience=10):
if not self.gen:
df = self.initialize(self.addons_config_reco,
read=read,
from_hdf=from_hdf)
else:
df = self.initialize(self.addons_config_gen,
read=read,
from_hdf=from_hdf)
for config_num in configs:
print(
f'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Training config {config_num}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
)
X_train, X_test, y_train, y_test = self.configure(df, config_num)
model = self.train(X_train,
X_test,
y_train,
y_test,
epochs=epochs,
batch_size=batch_size,
patience=patience)
if self.binary:
auc = self.evaluateBinary(model, X_test, y_test, self.history)
else:
w_a = df.w_a
w_b = df.w_b
auc = self.evaluate(model, X_test, y_test, self.history, w_a,
w_b)
self.write(self.gen, auc, self.history, self.addons_config_reco)
def runTuning(self,
config_num,
tuning_mode='random_sk',
read=True,
from_hdf=True):
if not self.gen:
df = self.initialize(self.addons_config_reco,
read=read,
from_hdf=from_hdf)
else:
df = self.initialize(self.addons_config_gen,
read=read,
from_hdf=from_hdf)
X_train, X_test, y_train, y_test = self.configure(df, config_num)
print(
f'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Tuning on config {config_num}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
)
tuner = Tuner(mode=tuning_mode)
if tuning_mode in {'random_sk', 'grid_search_cv'}:
model_grid, grid_result, param_grid = tuner.tune(
X_train, y_train, X_test, y_test)
self.layers = grid_result.best_params_['layers']
self.batch_norm = grid_result.best_params_['batch_norm']
self.dropout = grid_result.best_params_['dropout']
self.epochs = grid_result.best_params_['epochs']
self.batchsize = grid_result.best_params_['batch_size']
self.learning_rate = grid_result.best_params_['learning_rate']
self.activation = grid_result.best_params_['activation']
self.initializer_std = grid_result.best_params_['initializer_std']
self.nodes = grid_result.best_params_[
'nodes'] # !!! Kristof's edit on 25 Feb trying to fix the random_sk with the extra hyperparameters
self.model_str = 'grid_model'
grid_best_score = grid_result.best_score_
self.model = tuner.gridModel(layers=self.layers,
batch_norm=self.batch_norm,
dropout=self.dropout)
print("Best: %f using %s" %
(grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
print("%f (%f) with: %r" % (mean, stdev, param))
elif tuning_mode in {'hyperband', 'bayesian', 'random_kt'}:
self.model, best_hps, param_grid = tuner.tune(
X_train, y_train, X_test, y_test)
# hard coded epochs, batchsize - KerasTuner doesn't search over this parameter space
self.epochs = 50
self.batchsize = 10000
self.layers = best_hps.get('num_layers')
self.batch_norm = best_hps.get('batch_norm')
self.dropout = best_hps.get('dropout')
self.learning_rate = best_hps.get('learning_rate')
self.activation = best_hps.get('activation')
self.initializer_std = best_hps.get('initializer_std')
self.model_str = 'hyper_model'
grid_best_score = None
elif tuning_mode in {'hyperopt'}:
self.model, best_params, param_grid = tuner.tune(
X_train, y_train, X_test, y_test)
self.nodes = int(best_params['nodes'])
self.layers = int(best_params['num_layers'])
self.batch_norm = best_params['batch_norm']
self.dropout = best_params['dropout']
self.epochs = int(best_params['epochs'])
self.batchsize = int(best_params['batch_size'])
self.learning_rate = best_params['learning_rate']
self.activation = best_params['activation']
self.initializer_std = best_params['initializer_std']
self.model_str = 'hyperopt_model'
grid_best_score = None
else:
raise ValueError('Tuning mode not understood')
model = self.train(X_train,
X_test,
y_train,
y_test,
epochs=self.epochs,
batch_size=self.batchsize,
verbose=0)
if self.binary:
auc = self.evaluateBinary(model, X_test, y_test, self.history)
else:
w_a = df.w_a
w_b = df.w_b
auc = self.evaluate(model, X_test, y_test, self.history, w_a, w_b)
if not self.gen:
file = f'{self.write_dir}/tuning_reco_{self.channel}.txt'
else:
file = f'{self.write_dir}/tuning_gen_{self.channel}.txt'
self.model.save(f'{self.write_dir}/tuning_model_{self.channel}.h5')
with open(file, 'a+') as f:
print(f'Writing HPs to {file}')
time_str = datetime.datetime.now().strftime('%Y/%m/%d|%H:%M:%S')
# message = f'{time_str},{auc},{self.config_num},{self.layers},{self.batch_norm},{self.dropout},{self.epochs},{self.batchsize},{tuning_mode},{grid_best_score},{param_grid}\n'
message = f'{time_str},{auc},{self.config_num},{self.nodes},{self.layers},{self.batch_norm},{self.dropout},{self.epochs},{self.batchsize},{tuning_mode},{grid_best_score},{self.learning_rate},{self.activation},{self.initializer_std},{param_grid}\n'
print(f"Message: {message}")
f.write(message)
model_save_str = f'./saved_models/{self.channel}/model_{config_num}'
model.save(model_save_str)
def runWithSmearing(self,
config_num,
features: list,
from_hdf=True,
sample=False):
"""
Need to update smearing_hp.txt to get hyperparameter for tuning
Trying for config 1.6 smearing first
"""
if not self.gen:
print('CHANGING GEN TO TRUE - REQUIRED FOR SMEARING')
self.gen = True
print('SETTING SAVE_ALPHA TO FALSE')
self.addons_config_gen['neutrino']['save_alpha'] = False
self.addons_config_gen['neutrino']['load_alpha'] = False
df = self.initializeWithSmear(features,
self.addons_config_gen,
from_hdf=from_hdf,
sample=sample)
# get config 3.9 model if not rho_rho, if rho_rho, get 3.2
with open(f'./NN_output/smearing_hp_{config_num}.txt', 'r') as fh:
num_list = [line for line in fh]
if self.channel == 'rho_rho':
nn_arch = num_list[0].split(',')
elif self.channel == 'rho_a1':
nn_arch = num_list[1].split(',')
else:
nn_arch = num_list[2].split(',')
optimal_auc = float(nn_arch[1])
nodes = int(nn_arch[3])
num_layers = int(nn_arch[4])
batch_norm = bool(nn_arch[5])
dropout = float(nn_arch[6])
epochs = int(nn_arch[7])
batch_size = int(nn_arch[8])
learning_rate = float(nn_arch[11])
activation = str(nn_arch[12])
initializer_std = float(nn_arch[13])
params = {
'nodes': nodes,
'num_layers': num_layers,
'batch_norm': batch_norm,
'dropout': dropout,
'epochs': epochs,
'batch_size': batch_size,
'learning_rate': learning_rate,
'activation': activation,
'initializer_std': initializer_std,
}
print(f'Training with {params}')
tuner = Tuner()
self.model, _ = tuner.hyperOptModelNN(params)
X_train, X_test, y_train, y_test = self.configure(df, config_num)
model = self.train(X_train,
X_test,
y_train,
y_test,
epochs=epochs,
batch_size=batch_size)
# model = self.train(X_train, X_test, y_train, y_test, epochs=50, batch_size=10000)
if self.binary:
auc = self.evaluateBinary(model,
X_test,
y_test,
self.history,
plot=False)
else:
w_a = df.w_a
w_b = df.w_b
auc = self.evaluate(model,
X_test,
y_test,
self.history,
w_a,
w_b,
plot=False)
return auc, optimal_auc
def runSingleSmearAnalysis(self, features_list, from_hdf=True):
"""
for kristof
stick with 1.6
- feature 'pi_1' flag
"""
config_num = 3.6
f = open(
self.save_dir + '/' + self.channel + '_' + str(config_num) +
'_smearing_aucs.txt', 'a')
for feature in tqdm(features_list):
# auc, optimal_auc = self.runWithSmearing(1.6, [feature], from_hdf=from_hdf)
if isinstance(feature, list):
auc, optimal_auc = self.runWithSmearing(
config_num, feature, from_hdf=from_hdf,
sample=True) # sample=True should it be turned on?
degradation_auc = optimal_auc - auc
f.write('-'.join(feature) + ',' + str(degradation_auc) + ',' +
str(optimal_auc) + '\n')
print('-'.join(feature) + ',' + str(degradation_auc) + ',' +
str(auc) + ',' + str(optimal_auc) + '\n')
else:
auc, optimal_auc = self.runWithSmearing(config_num, [feature],
from_hdf=from_hdf)
degradation_auc = optimal_auc - auc
f.write(feature + ',' + str(degradation_auc) + ',' +
str(optimal_auc) + '\n')
print(feature + ',' + str(degradation_auc) + ',' + str(auc) +
',' + str(optimal_auc) + '\n')
f.close()
# plotting bar chart
def runSmearAnalysis(self, features_list, from_hdf=True):
config_num = 1.6
f = open(
self.save_dir + '/' + self.channel + '_' + str(config_num) +
'_smearing_aucs.txt', 'a')
auc, optimal_auc = self.runWithSmearing(config_num,
features_list,
from_hdf=from_hdf)
degradation_auc = optimal_auc - auc
print('-'.join(features_list) + ',' + str(degradation_auc) + ',' +
str(optimal_auc) + '\n')
f.write('-'.join(features_list) + ',' + str(degradation_auc) + ',' +
str(optimal_auc) + '\n')
f.close()
def initialize(self,
addons_config={},
read=True,
from_hdf=True,
strict=False):
"""
Initialize NN by loading/ creating the input data for NN via DataLoader
Params:
addons(dict) - addon map each value being an addon configuration
read - will read df inputs instead of creating them
from_hdf - will read events from HDF5 instead of .root file
Returns: df of NN inputs (to be configured)
"""
if not addons_config:
addons = []
else:
addons = addons_config.keys()
if not self.gen:
if self.channel == 'rho_rho':
variables = config.variables_rho_rho
elif self.channel == 'rho_a1':
variables = config.variables_rho_a1
elif self.channel == 'a1_a1':
variables = config.variables_a1_a1
else:
raise ValueError('Incorrect channel inputted')
else:
if self.channel == 'rho_rho':
variables = config.variables_gen_rho_rho
elif self.channel == 'rho_a1':
variables = config.variables_gen_rho_a1
elif self.channel == 'a1_a1':
variables = config.variables_gen_a1_a1
else:
raise ValueError('Incorrect channel inputted')
self.DL = DataLoader(variables, self.channel, self.gen)
CC = ConfigChecker(self.channel, self.binary, self.gen)
CC.checkInitialize(self.DL, addons_config, read, from_hdf)
if read:
print("WARNING: skipping over creating new configs")
if not self.gen:
df = self.DL.loadRecoData(self.binary, addons)
else:
df = self.DL.loadGenData(self.binary, addons)
else:
if not self.gen:
df = self.DL.createRecoData(self.binary,
from_hdf,
addons,
addons_config,
strict=strict)
else:
df = self.DL.createGenData(self.binary, from_hdf, addons,
addons_config)
return df
def initializeWithSmear(self,
features,
addons_config={},
from_hdf=True,
sample=True):
""" NO READ FUNCTION - ALWAYS CREATE SMEARING DIST """
if not addons_config:
addons = []
else:
addons = addons_config.keys()
if not self.gen:
if self.channel == 'rho_rho':
variables = config.variables_rho_rho
variables_smear = config.variables_smearing_rho_rho
elif self.channel == 'rho_a1':
variables = config.variables_rho_a1
variables_smear = config.variables_smearing_rho_a1
elif self.channel == 'a1_a1':
variables = config.variables_a1_a1
variables_smear = config.variables_smearing_a1_a1
else:
raise ValueError('Incorrect channel inputted')
else:
if self.channel == 'rho_rho':
variables = config.variables_gen_rho_rho
variables_smear = config.variables_smearing_rho_rho
elif self.channel == 'rho_a1':
variables = config.variables_gen_rho_a1
variables_smear = config.variables_smearing_rho_a1
elif self.channel == 'a1_a1':
variables = config.variables_gen_a1_a1
variables_smear = config.variables_smearing_a1_a1
else:
raise ValueError('Incorrect channel inputted')
self.DL = DataLoader(variables, self.channel, self.gen)
CC = ConfigChecker(self.channel, self.binary, self.gen)
CC.checkInitialize(self.DL, addons_config, read, from_hdf)
print(f'Loading .root info with using HDF5 as {from_hdf}')
df_orig = self.DL.readGenData(from_hdf=from_hdf)
print('Cleaning data')
df_clean, df_ps_clean, df_sm_clean = self.DL.cleanGenData(df_orig)
# print(df_clean.pi_E_1)
SM = Smearer(variables_smear, self.channel, features)
df_smeared = SM.createSmearedData(df_clean,
from_hdf=from_hdf,
sample=sample)
# remove Nans
df_smeared = df_smeared.dropna()
df_ps_smeared, df_sm_smeared = SM.selectPSSMFromData(df_smeared)
# addons = []
# addons_config = {}
df_smeared_transformed = self.DL.createTrainTestData(df_smeared,
df_ps_smeared,
df_sm_smeared,
binary,
True,
addons,
addons_config,
save=False)
# df_orig_transformed = self.DL.createTrainTestData(df_clean, df_ps_clean, df_sm_clean, binary, True, addons, addons_config, save=False)
# deal with imaginary numbers from boosting
df_smeared_transformed = df_smeared_transformed.dropna()
# df_smeared_transformed = df_smeared_transformed.apply(np.real)
# m_features = [x for x in df_smeared_transformed.columns if x.startswith('m')]
# for m_feature in m_features:
# df_smeared_transformed = df_smeared_transformed[df_smeared_transformed[m_feature]!=0]
# debugging part
# df.to_hdf('smearing/df_smeared.h5', 'df')
# df_smeared.to_hdf('./smearing/df_smeared.h5', 'df')
# df_clean.to_hdf('./smearing/df_orig.h5', 'df')
# df_smeared_transformed.to_hdf('smearing/df_smeared_transformed.h5', 'df')
# df_orig_transformed.to_hdf('smearing/df_orig_transformed.h5', 'df')
# exit()
return df_smeared_transformed
def configure(self, df, config_num):
"""
Configures NN inputs - selects config_num and creates train/test split
"""
self.config_num = config_num
CL = ConfigLoader(df, self.channel, self.gen)
X_train, X_test, y_train, y_test = CL.configTrainTestData(
self.config_num, self.binary)
return X_train, X_test, y_train, y_test
def train(self,
X_train,
X_test,
y_train,
y_test,
epochs=50,
batch_size=1024,
patience=20,
save=False,
verbose=1):
self.epochs = epochs
self.batch_size = batch_size
if self.model is None:
print('Using Kristof model')
self.model = self.kristof_model(X_train.shape[1])
self.history = tf.keras.callbacks.History()
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_auc',
patience=patience)
if not self.gen:
log_dir = f"logs/fit/{self.channel}_reco/" + datetime.datetime.now(
).strftime(
"%Y%m%d-%H%M%S"
) + f'_{self.config_num}_{self.layers}_{self.epochs}_{self.batch_size}_{self.model_str}'
else:
log_dir = f"logs/fit/{self.channel}_gen/" + datetime.datetime.now(
).strftime("%Y%m%d-%H%M%S") + f'_{self.epochs}'
if self.binary:
log_dir += '_b'
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1)
self.model.fit(
X_train,
y_train,
batch_size=batch_size,
epochs=epochs,
callbacks=[self.history, tensorboard_callback, early_stop],
validation_data=(X_test, y_test),
verbose=verbose)
if save:
self.model.save(f'./saved_models/{self.save_dir}/NN')
return self.model
def evaluate(self, model, X_test, y_test, history, w_a, w_b, plot=True):
if plot:
config_str = self.createConfigStr()
E = Evaluator(model, self.binary, self.save_dir, config_str)
else:
E = Evaluator(model, self.binary, self.save_dir, None)
auc = E.evaluate(X_test,
y_test,
history,
plot,
show=self.show_graph,
w_a=w_a,
w_b=w_b)
return auc
def write(self, auc, history, addons_config):
if not addons_config:
addons = []
else:
addons = addons_config.keys()
addons_loaded = "None"
if addons:
addons_loaded = '_' + '_'.join(addons)
if not self.gen:
file = f'{self.write_dir}/{self.write_filename}_reco_{self.channel}.txt'
else:
file = f'{self.write_dir}/{self.write_filename}_gen_{self.channel}.txt'
with open(file, 'a+') as f:
print(f'Writing to {file}')
time_str = datetime.datetime.now().strftime('%Y/%m/%d|%H:%M:%S')
actual_epochs = len(history.history["loss"])
f.write(
f'{time_str},{auc},{self.config_num},{self.layers},{self.epochs},{actual_epochs},{self.batch_size},{self.binary},{self.model_str},{addons_loaded}\n'
)
print('Finish writing')
f.close()
def evaluateBinary(self, model, X_test, y_test, history, plot=True):
if plot:
config_str = self.createConfigStr()
E = Evaluator(model, self.binary, self.save_dir, config_str)
else:
E = Evaluator(model, self.binary, self.save_dir, None)
auc = E.evaluate(X_test, y_test, history, plot, show=self.show_graph)
return auc
def createConfigStr(self):
if self.binary:
config_str = f'config{self.config_num}_{self.layers}_{self.epochs}_{self.batch_size}_{self.model_str}_binary'
else:
config_str = f'config{self.config_num}_{self.layers}_{self.epochs}_{self.batch_size}_{self.model_str}'
return config_str
def seq_model(self, units=(300, 300), batch_norm=False, dropout=None):
self.model = tf.keras.models.Sequential()
self.layers = len(units)
for unit in units:
self.model.add(
tf.keras.layers.Dense(unit, kernel_initializer='normal'))
if batch_norm:
self.model.add(tf.keras.layers.BatchNormalization())
self.model.add(tf.keras.layers.Activation('relu'))
if dropout is not None:
self.model.add(tf.keras.layers.Dropout(dropout))
self.model.add(tf.keras.layers.Dense(1, activation="sigmoid"))
metrics = ['AUC', 'accuracy']
self.model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=metrics)
self.model_str = "seq_model"
return self.model
def kristof_model(self, dimensions):
# model by kristof
model = tf.keras.models.Sequential()
self.layers = 2
self.model_str = "kristof_model"
metrics = ['AUC', 'accuracy']
model.add(
tf.keras.layers.Dense(300,
input_dim=dimensions,
kernel_initializer='normal',
activation='relu'))
model.add(
tf.keras.layers.Dense(300,
kernel_initializer='normal',
activation='relu')
) # !!! originally there were just 2 hidden layers
#model.add(tf.keras.layers.Dense(300, kernel_initializer='normal', activation='relu'))
#model.add(tf.keras.layers.Dense(300, kernel_initializer='normal', activation='relu'))
#model.add(tf.keras.layers.Dense(300, kernel_initializer='normal', activation='relu'))
model.add(tf.keras.layers.Dense(1, activation="sigmoid"))
opt = tf.keras.optimizers.Adam(learning_rate=0.001)
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=metrics)
return model
def custom_model(self):
from tensorflow.keras.backend import sigmoid
def swish(x, beta=1):
return (x * sigmoid(beta * x))
model = tf.keras.models.Sequential()
from tensorflow.keras import utils
from tensorflow.keras.layers import Activation
utils.get_custom_objects().update({'swish': Activation(swish)})
self.layers = 2
self.model_str = "custom_model"
metrics = ['AUC', 'accuracy']
model.add(
tf.keras.layers.Dense(300,
kernel_initializer='normal',
activation='swish'))
model.add(
tf.keras.layers.Dense(300,
kernel_initializer='normal',
activation='swish'))
model.add(tf.keras.layers.Dense(1, activation="sigmoid"))
opt = tf.keras.optimizers.Adam(learning_rate=0.001)
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=metrics)
return model
class NeuralNetwork:
"""
Features
- Run a single config_num
- Run multiple config_nums
- Will automatically write and save results of model - see save_dir and write_dir for more information
Notes:
- Supports Tensorboard
"""
def __init__(self, channel, binary, write_filename, show_graph=False):
self.show_graph = show_graph
self.channel = channel
self.binary = binary
self.write_filename = write_filename
# variables are bare bones only for rho_rho
# TODO: make variables for all channels neatly in data loader
self.variables_rho_rho = [
"wt_cp_sm", "wt_cp_ps", "wt_cp_mm", "rand",
"aco_angle_1",
"mva_dm_1", "mva_dm_2",
"tau_decay_mode_1", "tau_decay_mode_2",
"pi_E_1", "pi_px_1", "pi_py_1", "pi_pz_1",
"pi_E_2", "pi_px_2", "pi_py_2", "pi_pz_2",
"pi0_E_1", "pi0_px_1", "pi0_py_1", "pi0_pz_1",
"pi0_E_2", "pi0_px_2", "pi0_py_2", "pi0_pz_2",
"y_1_1", "y_1_2",
'met', 'metx', 'mety',
'metcov00', 'metcov01', 'metcov10', 'metcov11',
"gen_nu_p_1", "gen_nu_phi_1", "gen_nu_eta_1", #leading neutrino, gen level
"gen_nu_p_2", "gen_nu_phi_2", "gen_nu_eta_2" #subleading neutrino, gen level
]
self.variables_rho_a1 = [
"wt_cp_sm", "wt_cp_ps", "wt_cp_mm", "rand",
"aco_angle_1",
"mva_dm_1", "mva_dm_2",
"tau_decay_mode_1", "tau_decay_mode_2",
"pi_E_1", "pi_px_1", "pi_py_1", "pi_pz_1",
"pi_E_2", "pi_px_2", "pi_py_2", "pi_pz_2",
"pi0_E_1", "pi0_px_1", "pi0_py_1", "pi0_pz_1",
"pi2_px_2", "pi2_py_2", "pi2_pz_2", "pi2_E_2",
"pi3_px_2", "pi3_py_2", "pi3_pz_2", "pi3_E_2",
"ip_x_1", "ip_y_1", "ip_z_1",
"sv_x_2", "sv_y_2", "sv_z_2",
"y_1_1", "y_1_2",
]
self.variables_a1_a1 = [
"wt_cp_sm", "wt_cp_ps", "wt_cp_mm", "rand",
"aco_angle_1",
"mva_dm_1", "mva_dm_2",
"tau_decay_mode_1", "tau_decay_mode_2",
"pi_E_1", "pi_px_1", "pi_py_1", "pi_pz_1",
"pi_E_2", "pi_px_2", "pi_py_2", "pi_pz_2",
"pi2_E_1", "pi2_px_1", "pi2_py_1", "pi2_pz_1",
"pi3_E_1", "pi3_px_1", "pi3_py_1", "pi3_pz_1",
"pi2_px_2", "pi2_py_2", "pi2_pz_2", "pi2_E_2",
"pi3_px_2", "pi3_py_2", "pi3_pz_2", "pi3_E_2",
"ip_x_1", "ip_y_1", "ip_z_1",
"sv_x_2", "sv_y_2", "sv_z_2",
"y_1_1", "y_1_2",
]
self.save_dir = 'NN_output'
self.write_dir = 'NN_output'
self.model = None
def run(self, config_num, read=True, from_pickle=True, epochs=50, batch_size=1024, patience=10, addons_config={}):
df = self.initialize(addons_config, read=read, from_pickle=from_pickle)
X_train, X_test, y_train, y_test = self.configure(df, config_num)
print(f'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Training config {config_num}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
model = self.train(X_train, X_test, y_train, y_test, epochs=epochs, batch_size=batch_size, patience=patience)
if self.binary:
auc = self.evaluateBinary(model, X_test, y_test, self.history)
else:
w_a = df.w_a
w_b = df.w_b
auc = self.evaluate(model, X_test, y_test, self.history, w_a, w_b)
self.write(auc, self.history)
def runMultiple(self, configs, read=True, from_pickle=True, epochs=50, batch_size=1024, patience=10, addons_config={}):
df = self.initialize(addons_config, read=read, from_pickle=from_pickle)
for config_num in configs:
print(f'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Training config {config_num}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
X_train, X_test, y_train, y_test = self.configure(df, config_num)
model = self.train(X_train, X_test, y_train, y_test, epochs=epochs, batch_size=batch_size, patience=patience)
if self.binary:
auc = self.evaluateBinary(model, X_test, y_test, self.history)
else:
w_a = df.w_a
w_b = df.w_b
auc = self.evaluate(model, X_test, y_test, self.history, w_a, w_b)
self.write(auc, self.history)
def runHPTuning(self, config_num, read=True, from_pickle=True, epochs=50, tuner_epochs=50, batch_size=10000, tuner_batch_size=10000, patience=10, tuner_mode=0, addons_config={}):
df = self.initialize(addons_config, read=read, from_pickle=from_pickle)
X_train, X_test, y_train, y_test = self.configure(df, config_num)
print(f'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Tuning on config {config_num}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
best_hps, tuner = self.tuneHP(self.hyperModel, X_train, X_test, y_train, y_test, tuner_epochs=tuner_epochs, tuner_batch_size=tuner_batch_size, tuner_mode=tuner_mode)
print(tuner.results_summary())
model = tuner.hypermodel.build(best_hps)
self.model = model
# model.fit(X_train, y_train, epochs=epochs, validation_data = (X_test, y_test), verbose=0)
# verbose is set 0 to prevent logs from spam
model = self.train(X_train, X_test, y_train, y_test, epochs=epochs, batch_size=batch_size, patience=patience, verbose=0)
if self.binary:
auc = self.evaluateBinary(model, X_test, y_test, self.history)
else:
w_a = df.w_a
w_b = df.w_b
auc = self.evaluate(model, X_test, y_test, self.history, w_a, w_b)
file = f'{self.write_dir}/best_hp_{self.channel}.txt'
with open(file, 'a+') as f:
# print(f'Writing HPs to {file}')
time_str = datetime.datetime.now().strftime('%Y/%m/%d|%H:%M:%S')
best_num_layers = best_hps.get('num_layers')
# best_batch_norm = best_hps.get('batch_norm')
# best_dropout = best_hps.get('dropout')
actual_epochs = len(self.history.history["loss"])
# message = f'{time_str},{auc},{self.config_num},{best_num_layers},{best_batch_norm},{best_dropout},{tuner_mode}\n'
message = f'{time_str},{auc},{self.config_num},{best_num_layers},{tuner_mode}\n'
print(f"Message: {message}")
# f.write(message)
# model.save('./hp_model_1/')
def runGridSearch(self, config_num, read=True, from_pickle=True, addons_config={}, search_mode=0):
"""
Runs grid search on NN with given config_num
search_mode = 0: GridSearch
search_mode = 1: RandomSearch
"""
df = self.initialize(addons_config, read=read, from_pickle=from_pickle)
X_train, X_test, y_train, y_test = self.configure(df, config_num)
print(f'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Grid searching on config {config_num}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
model = KerasClassifier(self.gridModel, verbose=0)
if search_mode == 0:
layers = [2,3,4,5,6]
batch_norms = [True, False]
dropouts = [None, 0.2]
epochs = [100, 200, 500]
batch_sizes = [8192, 16384, 65536, 131072]
param_grid = dict(
layers=layers,
batch_norm=batch_norms,
dropout=dropouts,
epochs=epochs,
batch_size=batch_sizes
)
grid = GridSearchCV(estimator=model, param_grid=param_grid, cv=2, verbose=2, scoring='roc_auc')
elif search_mode == 1:
layers = np.arange(1,11)
batch_norms = [True, False]
dropouts = [None, 0.1, 0.2, 0.3, 0.4, 0.5]
epochs = [50, 100, 200, 500]
batch_sizes = [2**i for i in range(8, 19)]
# layers = [2,3]
# batch_norms = [True, False]
# dropouts = [None, 0.2]
# epochs = [200, 500]
# batch_sizes = [8192, 16384, 65536, 131072]
param_grid = dict(
layers=layers,
batch_norm=batch_norms,
dropout=dropouts,
epochs=epochs,
batch_size=batch_sizes
)
# can increase the distributions of params
grid = RandomizedSearchCV(estimator=model, param_distributions=param_grid, cv=2, verbose=2, scoring='roc_auc', random_state=seed_value, n_iter=20)
else:
raise ValueError('Search mode not defined correctly')
grid_result = grid.fit(X_train, y_train)
print(grid_result)
best_num_layers = grid_result.best_params_['layers']
best_batch_norm = grid_result.best_params_['batch_norm']
best_dropout = grid_result.best_params_['dropout']
best_epochs = grid_result.best_params_['epochs']
best_batchsize = grid_result.best_params_['batch_size']
self.model = self.gridModel(layers=best_num_layers, batch_norm=best_batch_norm, dropout=best_dropout)
model = self.train(X_train, X_test, y_train, y_test, epochs=best_epochs, batch_size=best_batchsize, verbose=0)
if self.binary:
auc = self.evaluateBinary(model, X_test, y_test, self.history)
else:
w_a = df.w_a
w_b = df.w_b
auc = self.evaluate(model, X_test, y_test, self.history, w_a, w_b)
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
print("%f (%f) with: %r" % (mean, stdev, param))
file = f'{self.write_dir}/grid_search_{self.channel}.txt'
with open(file, 'a+') as f:
print(f'Writing HPs to {file}')
time_str = datetime.datetime.now().strftime('%Y/%m/%d|%H:%M:%S')
message = f'{time_str},{auc},{self.config_num},{best_num_layers},{best_batch_norm},{best_dropout},{best_epochs},{best_batchsize},{search_mode},{grid_result.best_score_},{param_grid}\n'
print(f"Message: {message}")
f.write(message)
model.save(f'./saved_models/grid_search_model_{config_num}_{self.channel}_{search_mode}/')
def tuneHP(self, hyperModel, X_train, X_test, y_train, y_test, tuner_epochs=50, tuner_batch_size=10000, tuner_mode=0):
if tuner_mode == 0:
tuner = kt.Hyperband(hyperModel,
objective=kt.Objective("auc", direction="max"), # ['loss', 'auc', 'accuracy', 'val_loss', 'val_auc', 'val_accuracy']
max_epochs=200,
hyperband_iterations=3,
factor=3,
seed=seed_value,
directory='tuning',
project_name='model_hyperband_1',
overwrite=True)
elif tuner_mode == 1:
tuner = kt.BayesianOptimization(hyperModel,
objective='val_loss',
max_trials=100,
seed=seed_value,
directory='tuning',
project_name='model_bayesian_1',
overwrite=True)
elif tuner_mode == 2:
tuner = kt.RandomSearch(hyperModel,
objective='val_loss',
max_trials=1000,
seed=seed_value,
directory='tuning',
project_name='model_random_1',
overwrite=True)
else:
raise ValueError('Invalid tuner mode')
tuner.search(X_train, y_train, epochs=tuner_epochs, batch_size=tuner_batch_size, validation_data=(X_test, y_test), verbose=0)
# tuner.search(X_train, y_train, epochs=tuner_epochs, validation_data=(X_test, y_test), verbose=1)
best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]
print(tuner.search_space_summary())
return best_hps, tuner
def initialize(self, addons_config={}, read=True, from_pickle=True):
"""
Initialize NN by loading/ creating the input data for NN via DataLoader
Params:
addons(dict) - addon map each value being an addon configuration
read - will read df inputs instead of creating them
from_pickle - will read events from pickle instead of .root file
Returns: df of NN inputs (to be configured)
"""
if not addons_config:
addons = []
else:
addons = addons_config.keys()
if self.channel == 'rho_rho':
self.DL = DataLoader(self.variables_rho_rho, self.channel)
elif self.channel == 'rho_a1':
self.DL = DataLoader(self.variables_rho_a1, self.channel)
elif self.channel == 'a1_a1':
self.DL = DataLoader(self.variables_a1_a1, self.channel)
else:
raise ValueError('Incorrect channel inputted')
if read:
df = self.DL.loadRecoData(self.binary, addons)
else:
df = self.DL.createRecoData(self.binary, from_pickle, addons, addons_config)
return df
def configure(self, df, config_num):
"""
Configures NN inputs - selects config_num and creates train/test split
"""
self.config_num = config_num
CL = ConfigLoader(df, self.channel)
X_train, X_test, y_train, y_test = CL.configTrainTestData(self.config_num, self.binary)
return X_train, X_test, y_train, y_test
def train(self, X_train, X_test, y_train, y_test, epochs=50, batch_size=1024, patience=10, save=False, verbose=1):
self.epochs = epochs
self.batch_size = batch_size
if self.model is None:
self.model = self.kristof_model(X_train.shape[1])
self.history = tf.keras.callbacks.History()
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=patience)
log_dir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
self.model.fit(X_train, y_train,
batch_size=batch_size,
epochs=epochs,
callbacks=[self.history, tensorboard_callback], #, early_stop],
validation_data=(X_test, y_test),
verbose=verbose)
if save:
self.model.save(f'./saved_models/{self.save_dir}/NN')
return self.model
def evaluate(self, model, X_test, y_test, history, w_a, w_b):
config_str = self.createConfigStr()
E = Evaluator(model, self.binary, self.save_dir, config_str)
auc = E.evaluate(X_test, y_test, history, show=self.show_graph, w_a=w_a, w_b=w_b)
return auc
def write(self, auc, history):
file = f'{self.write_dir}/{self.write_filename}.txt'
with open(file, 'a+') as f:
print(f'Writing to {file}')
time_str = datetime.datetime.now().strftime('%Y/%m/%d|%H:%M:%S')
actual_epochs = len(history.history["loss"])
f.write(f'{time_str},{auc},{self.config_num},{self.layers},{self.epochs},{actual_epochs},{self.batch_size},{self.binary},{self.model_str}\n')
print('Finish writing')
f.close()
def evaluateBinary(self, model, X_test, y_test, history):
config_str = self.createConfigStr()
E = Evaluator(model, self.binary, self.save_dir, config_str)
auc = E.evaluate(X_test, y_test, history, show=self.show_graph)
return auc
def createConfigStr(self):
if self.binary:
config_str = f'config{self.config_num}_{self.layers}_{self.epochs}_{self.batch_size}_{self.model_str}_binary'
else:
config_str = f'config{self.config_num}_{self.layers}_{self.epochs}_{self.batch_size}_{self.model_str}'
return config_str
def seq_model(self, units=(300, 300), batch_norm=False, dropout=None):
self.model = tf.keras.models.Sequential()
self.layers = len(units)
for unit in units:
self.model.add(tf.keras.layers.Dense(unit, kernel_initializer='normal'))
if batch_norm:
self.model.add(tf.keras.layers.BatchNormalization())
self.model.add(tf.keras.layers.Activation('relu'))
if dropout is not None:
self.model.add(tf.keras.layers.Dropout(dropout))
self.model.add(tf.keras.layers.Dense(1, activation="sigmoid"))
metrics = ['AUC', 'accuracy']
self.model.compile(loss='binary_crossentropy', optimizer='adam', metrics=metrics)
self.model_str = "seq_model"
return self.model
def hyperModel(self, hp):
self.model = tf.keras.models.Sequential()
num_layers = hp.Int('num_layers', 2, 3)
self.layers = num_layers
for i in range(num_layers):
self.model.add(tf.keras.layers.Dense(units=300, kernel_initializer='normal'))
# if hp.Boolean('batch_norm', default=False):
# self.model.add(tf.keras.layers.BatchNormalization())
# self.model.add(tf.keras.layers.Dropout(rate=hp.Float('dropout', min_value=0.0, max_value=0.2, default=0.0, step=0.2)))
self.model.add(tf.keras.layers.Dense(1, activation="sigmoid"))
metrics = ['AUC', 'accuracy']
self.model.compile(loss='binary_crossentropy', optimizer='adam', metrics=metrics)
self.model_str = "hyper_model"
return self.model
def gridModel(self, layers=2, batch_norm=False, dropout=None):
self.model = tf.keras.models.Sequential()
self.layers = layers
for i in range(layers):
self.model.add(tf.keras.layers.Dense(300, kernel_initializer='normal'))
if batch_norm:
self.model.add(tf.keras.layers.BatchNormalization())
self.model.add(tf.keras.layers.Activation('relu'))
if dropout is not None:
self.model.add(tf.keras.layers.Dropout(dropout))
self.model.add(tf.keras.layers.Dense(1, activation="sigmoid"))
metrics = ['AUC', 'accuracy']
self.model.compile(loss='binary_crossentropy', optimizer='adam', metrics=metrics)
self.model_str = "grid_model"
return self.model
def kristof_model(self, dimensions):
# model by kristof
model = tf.keras.models.Sequential()
self.layers = 2
self.model_str = "kristof_model"
metrics = ['AUC', 'accuracy']
model.add(tf.keras.layers.Dense(300, input_dim=dimensions, kernel_initializer='normal', activation='relu'))
model.add(tf.keras.layers.Dense(300, kernel_initializer='normal', activation='relu'))
model.add(tf.keras.layers.Dense(1, activation="sigmoid"))
opt = tf.keras.optimizers.Adam(learning_rate=0.001)
model.compile(loss='binary_crossentropy', optimizer=opt, metrics=metrics)
return model
