def build_model(X_train, Y_train, X_val, Y_val, params):
model = Sequential()
model.add(
Embedding(params['vocab_size'],
params['e_size'],
input_length=params['seq_len']))
model.add(Conv1D(32, 7, activation='relu'))
model.add(MaxPooling1D(5))
model.add(Conv1D(32, 7, activation='relu'))
model.add(GlobalAveragePooling1D())
model.add(Dropout(params['dropout']))
hidden_layers(model, params, 1)
model.add(Dense(1, activation=params['last_activation']))
## COMPILE
model.compile(optimizer=params['optimizer'](lr_normalizer(
params['lr'], params['optimizer'])),
loss='binary_crossentropy',
metrics=['acc'])
out = model.fit(X_train,
Y_train,
batch_size=params['batch_size'],
epochs=params['epochs'],
validation_data=[X_val, Y_val],
verbose=2)
return out, model
def higgs_nn(X_train, Y_train, X_valid, Y_valid, params):
model = Sequential()
model.add(
Dense(75,
input_dim=X_train.shape[1],
activation=params['activation'],
kernel_initializer='normal'))
model.add(Dropout(params['dropout']))
hidden_layers(model, params, 2)
model.add(
Dense(2,
activation=params['last_activation'],
kernel_initializer='normal'))
model.compile(loss=params['losses'],
optimizer=params['optimizer'](
lr=lr_normalizer(params['lr'], params['optimizer'])),
metrics=['acc'])
history = model.fit(X_train,
Y_train,
validation_data=[X_valid, Y_valid],
batch_size=params['batch_size'],
epochs=params['epochs'],
verbose=2)
# finally we have to make sure that history object and model are returned
return history, model
def scan_hyperparameter(x_train, y_train, x_val, y_val, params):
'''
Using talos library to scan for the best hyperparameter settings for the
Neural Network setup
'''
# Building the model
model = Sequential()
model.add(Dense(params['first_neuron'], input_dim=x_train.shape[1],
activation=params['activation'],
kernel_initializer=params['kernel_initializer']))
model.add(Dropout(params['dropout']))
hidden_layers(model, params, 1)
model.add(Dense(1, activation=params['last_activation'],
kernel_initializer=params['kernel_initializer']))
model.compile(loss=params['losses'],
optimizer=params['optimizer'](),
metrics=['acc'])
# Fit the model and record to tensorboard
history = model.fit(x_train, y_train,
validation_data=[x_val, y_val],
batch_size=params['batch_size'],
epochs=params['epochs'],
verbose=0)
return history, model
def hbb_model(train_data,train_y,valid_data,valid_y,param):
model=Sequential()
model.add(Dense(param['first_neuron'],input_dim=train_data.shape[1],activation=param['activation'],kernel_initializer='normal'))
model.add(Dropout(param['dropout']))
hidden_layers(model,param,1)
model.add(Dense(train_y.shape[1],activation=param['last_activation'],kernel_initializer='normal'))
model.compile(optimizer=param['optimizer'](lr=lr_normalizer(param['lr'],param['optimizer'])),loss=param['losses'],metrics=['acc'])
out=model.fit(train_data,train_y,batch_size=param['batch_size'],epochs=param['epochs'],verbose=0,validation_data=[valid_data,valid_y])
return out,model
def dae_model_hl(x_train, y_train, x_val, y_val, params):
#print(params['x_train_noise'].shape)
print(x_train.shape)
print("masking training")
x_train_noise = mask_function(dataframe=x_train,
noise=float(params['noise']),
batch_sizes=300) #masking training
print("masking validation")
x_val_noise = mask_function(dataframe=x_val,
noise=float(params['noise']),
batch_sizes=300) #masking validation
print("building autoencoder network")
model = Sequential()
model.add(
Dense(params['first_neuron'],
activation=params['activation'],
input_shape=(x_train.shape[1], )))
model.add(Dropout(params['dropout']))
#m.add(Dense(128, activation='elu'))
hidden_layers(model, params, 1)
model.add(
Dense(params['embedding_size'],
activation=params['activation'],
name="bottleneck"))
hidden_layers(model, params, 1)
model.add(Dense(params['first_neuron'], activation=params['activation']))
#m.add(Dense(512, activation='elu'))
model.add(Dropout(params['dropout']))
model.add(Dense(x_train.shape[1], activation=params['last_activation']))
#m.compile(loss='mean_squared_error', optimizer = params['optmizer'])
model.compile(optimizer=params['optimizer'](
lr=lr_normalizer(params['lr'], params['optimizer'])),
loss=params['loss'],
metrics=['accuracy'])
print("training neural network")
out = model.fit(
x_train,
x_train_noise, #x_train_noise,
batch_size=params['batch_size'],
epochs=params['epochs'],
verbose=0,
validation_data=[x_val, x_val_noise], #x_val_noise],
callbacks=early_stopper(params['epochs'], mode='moderate'))
#callbacks=early_stopper(params['epochs'], mode='strict'))#noisy_train, train, batch_size=128, epochs=params['epochs'], verbose=1,
return out, model
def hbb_model(x_train, y_train, x_val, y_val, param):
model = Sequential()
model.add(
Dense(param['first_neuron'],
input_dim=x_train.shape[1],
activation=param['activation'],
kernel_initializer='orthogonal'))
model.add(Dropout(param['dropout']))
hidden_layers(model, param, 1)
model.add(
Dense(y_train.shape[1],
activation='softmax',
kernel_initializer='orthogonal'))
opt = keras.optimizers.Adam(lr=param['lr'], decay=param['decay'])
model.compile(optimizer=opt, loss=param['losses'], metrics=['acc'])
out = model.fit(x_train,
y_train,
batch_size=param['batch_size'],
epochs=param['epochs'],
verbose=0,
validation_data=[x_val, y_val])
return out, model
def autoFNN(X_train, Y_train, X_val, Y_val, params):
model = Sequential()
model.add(
Embedding(params['vocab_size'],
params['e_size'],
input_length=params['seq_len']))
model.add(Flatten())
hidden_layers(model, params, 1)
model.add(Dense(1, activation=params['last_activation']))
model.compile(optimizer=params['optimizer'](lr_normalizer(
params['lr'], params['optimizer'])),
loss='binary_crossentropy',
metrics=['acc'])
out = model.fit(X_train,
Y_train,
batch_size=params['batch_size'],
epochs=params['epochs'],
validation_data=[X_val, Y_val],
verbose=2)
return out, model
def build_model(x_train, y_train, x_val, y_val, params):
model = keras.Sequential()
model.add(keras.layers.Dense(10, activation=params['activation'],
input_dim=x_train.shape[1],
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=keras.regularizers.l1_l2(l1=params['l1'], l2=params['l2']),
bias_regularizer=None))
model.add(keras.layers.Dropout(params['dropout']))
# If we want to also test for number of layers and shapes, that's possible
hidden_layers(model, params, 1)
# Then we finish again with completely standard Keras way
model.add(keras.layers.Dense(1, activation=params['activation'], use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=keras.regularizers.l1_l2(l1=params['l1'], l2=params['l2']),
bias_regularizer=None))
model.compile(optimizer=params['optimizer'](lr=lr_normalizer(params['lr'], params['optimizer'])),
loss=params['losses'],
metrics=['mse'])
history = model.fit(x_train, y_train,
validation_data=[x_val, y_val],
batch_size=params['batch_size'],
epochs=params['epochs'],
callbacks=[early_stopper(epochs=params['epochs'], mode='moderate')],
#callbacks=[early_stopper(epochs=params['epochs'], mode='strict')],
verbose=0)
# Finally we have to make sure that history object and model are returned
return history, model
def _create_input_model(self, x_train, y_train, x_val, y_val, params):
model = Sequential()
if params['network'] != 'dense':
x_train = array_reshape_conv1d(x_train)
x_val = array_reshape_conv1d(x_val)
if params['network'] == 'conv1d':
model.add(Conv1D(params['first_neuron'], x_train.shape[1]))
model.add(Flatten())
elif params['network'] == 'lstm':
model.add(LSTM(params['first_neuron']))
if params['network'] == 'bidirectional_lstm':
model.add(Bidirectional(LSTM(params['first_neuron'])))
elif params['network'] == 'simplernn':
model.add(SimpleRNN(params['first_neuron']))
elif params['network'] == 'dense':
model.add(Dense(params['first_neuron'],
input_dim=x_train.shape[1],
activation='relu'))
model.add(Dropout(params['dropout']))
# add hidden layers to the model
hidden_layers(model, params, 1)
# output layer (this is scetchy)
try:
last_neuron = y_train.shape[1]
except IndexError:
if len(np.unique(y_train)) == 2:
last_neuron = 1
else:
last_neuron = len(np.unique(y_train))
model.add(Dense(last_neuron,
activation=params['last_activation']))
# bundle the optimizer with learning rate changes
optimizer = params['optimizer'](lr=lr_normalizer(params['lr'],
params['optimizer']))
# compile the model
model.compile(optimizer=optimizer,
loss=params['losses'],
metrics=['acc'])
# fit the model
out = model.fit(x_train, y_train,
batch_size=params['batch_size'],
epochs=params['epochs'],
verbose=0,
validation_data=[x_val, y_val])
# pass the output to Talos
return out, model
def NeuralNetGeneratorModel(x_train, y_train, x_val, y_val, params):
"""
Keras model for the Neural Network, used to scan the hyperparameter space by Talos
Uses the generator rather than the input data (which are dummies)
"""
# Scaler #
with open(parameters.scaler_path,
'rb') as handle: # Import scaler that was created before
scaler = pickle.load(handle)
# Design network #
# Left branch : classic inputs -> Preprocess -> onehot
inputs_numeric = []
means = []
variances = []
inputs_all = []
encoded_all = []
for idx in range(x_train.shape[1]):
inpName = parameters.inputs[idx].replace('$', '').replace(' ',
'').replace(
'_', '')
input_layer = tf.keras.Input(shape=(1, ), name=inpName)
# Categorical inputs #
if parameters.mask_op[idx]:
operation = getattr(Operations, parameters.operations[idx])()
encoded_all.append(operation(input_layer))
# Numerical inputs #
else:
inputs_numeric.append(input_layer)
means.append(scaler.mean_[idx])
variances.append(scaler.var_[idx])
inputs_all.append(input_layer)
# Concatenate all numerical inputs #
if int(tf_version[1]) < 4:
normalizer = preprocessing.Normalization(name='Normalization')
x_dummy = np.ones((10, len(means)))
# Needs a dummy to call the adapt method before setting the weights
normalizer.adapt(x_dummy)
normalizer.set_weights([np.array(means), np.array(variances)])
else:
normalizer = preprocessing.Normalization(mean=means,
variance=variances,
name='Normalization')
encoded_all.append(
normalizer(tf.keras.layers.concatenate(inputs_numeric,
name='Numerics')))
if len(encoded_all) > 1:
all_features = tf.keras.layers.concatenate(encoded_all,
axis=-1,
name="Features")
else:
all_features = encoded_all[0]
# Right branch : LBN
lbn_input_shape = (len(parameters.LBN_inputs) // 4, 4)
input_lbn_Layer = Input(shape=lbn_input_shape, name='LBN_inputs')
lbn_layer = LBNLayer(
lbn_input_shape,
n_particles=max(params['n_particles'],
1), # Hack so that 0 does not trigger error
boost_mode=LBN.PAIRS,
features=["E", "px", "py", "pz", "pt", "p", "m", "pair_cos"],
name='LBN')(input_lbn_Layer)
batchnorm = tf.keras.layers.BatchNormalization(name='batchnorm')(lbn_layer)
# Concatenation of left and right #
concatenate = tf.keras.layers.Concatenate(axis=-1)(
[all_features, batchnorm])
L1 = Dense(params['first_neuron'],
activation=params['activation'],
kernel_regularizer=l2(params['l2']))(
concatenate if params['n_particles'] > 0 else all_features)
hidden = hidden_layers(params, 1, batch_normalization=True).API(L1)
out = Dense(y_train.shape[1],
activation=params['output_activation'],
name='out')(hidden)
# Tensorboard logs #
# path_board = os.path.join(parameters.main_path,"TensorBoard")
# suffix = 0
# while(os.path.exists(os.path.join(path_board,"Run_"+str(suffix)))):
# suffix += 1
# path_board = os.path.join(path_board,"Run_"+str(suffix))
# os.makedirs(path_board)
# logging.info("TensorBoard log dir is at %s"%path_board)
# Callbacks #
# Early stopping to stop learning if val_loss plateau for too long #
early_stopping = EarlyStopping(**parameters.early_stopping_params)
# Reduce learnign rate in case of plateau #
reduceLR = ReduceLROnPlateau(**parameters.reduceLR_params)
# Custom loss function plot for debugging #
loss_history = LossHistory()
# Tensorboard for checking live the loss curve #
# board = TensorBoard(log_dir=path_board,
# histogram_freq=1,
# batch_size=params['batch_size'],
# write_graph=True,
# write_grads=True,
# write_images=True)
# Callback_list = [loss_history,early_stopping,reduceLR,board]
Callback_list = [loss_history, early_stopping, reduceLR]
# Compile #
if 'resume' not in params: # Normal learning
# Define model #
model_inputs = [inputs_all]
if params['n_particles'] > 0:
model_inputs.append(input_lbn_Layer)
model = Model(inputs=model_inputs, outputs=[out])
initial_epoch = 0
else: # a model has to be imported and resumes training
#custom_objects = {'PreprocessLayer': PreprocessLayer,'OneHot': OneHot.OneHot}
logging.info("Loaded model %s" % params['resume'])
a = Restore(params['resume'],
custom_objects=custom_objects,
method='h5')
model = a.model
initial_epoch = params['initial_epoch']
model.compile(optimizer=Adam(lr=params['lr']),
loss=params['loss_function'],
metrics=[
tf.keras.metrics.CategoricalAccuracy(),
tf.keras.metrics.AUC(multi_label=True),
tf.keras.metrics.Precision(),
tf.keras.metrics.Recall()
])
model.summary()
# Generator #
training_generator = DataGenerator(
path=parameters.config,
inputs=parameters.inputs,
outputs=parameters.outputs,
inputsLBN=parameters.LBN_inputs if params['n_particles'] > 0 else None,
cut=parameters.cut,
weight=parameters.weight,
batch_size=params['batch_size'],
state_set='training',
model_idx=params['model_idx'] if parameters.crossvalidation else None)
validation_generator = DataGenerator(
path=parameters.config,
inputs=parameters.inputs,
outputs=parameters.outputs,
inputsLBN=parameters.LBN_inputs if params['n_particles'] > 0 else None,
cut=parameters.cut,
weight=parameters.weight,
batch_size=params['batch_size'],
state_set='validation',
model_idx=params['model_idx'] if parameters.crossvalidation else None)
# Some verbose logging #
logging.info("Will use %d workers" % parameters.workers)
logging.warning("Tensorflow location " + tf.__file__)
if len(tf.config.experimental.list_physical_devices('XLA_GPU')) > 0:
logging.info("GPU detected")
#logging.warning(K.tensorflow_backend._get_available_gpus())
# Fit #
history = model.fit_generator(
generator=training_generator, # Training data from generator instance
validation_data=
validation_generator, # Validation data from generator instance
epochs=params['epochs'], # Number of epochs
verbose=1,
max_queue_size=parameters.workers * 2, # Length of batch queue
callbacks=Callback_list, # Callbacks
initial_epoch=
initial_epoch, # In case of resumed training will be different from 0
workers=parameters.
workers, # Number of threads for batch generation (0 : all in same)
shuffle=True, # Shuffle order at each epoch
use_multiprocessing=True) # Needs to be turned on for queuing batches
# Plot history #
PlotHistory(loss_history)
return history, model
def NeuralNetModel(x_train, y_train, x_val, y_val, params):
"""
Keras model for the Neural Network, used to scan the hyperparameter space by Talos
Uses the data provided as inputs
"""
# Split y = [target,weight], Talos does not leave room for the weight so had to be included in one of the arrays
w_train = y_train[:, -1]
w_val = y_val[:, -1]
y_train = y_train[:, :-1]
y_val = y_val[:, :-1]
x_train_lbn = x_train[:, -len(parameters.LBN_inputs):].reshape(
-1, 4,
len(parameters.LBN_inputs) // 4)
x_train = x_train[:, :-len(parameters.LBN_inputs)]
x_val_lbn = x_val[:, -len(parameters.LBN_inputs):].reshape(
-1, 4,
len(parameters.LBN_inputs) // 4)
x_val = x_val[:, :-len(parameters.LBN_inputs)]
# Scaler #
with open(parameters.scaler_path,
'rb') as handle: # Import scaler that was created before
scaler = pickle.load(handle)
# Design network #
# Left branch : classic inputs -> Preprocess -> onehot
inputs_numeric = []
means = []
variances = []
inputs_all = []
encoded_all = []
for idx in range(x_train.shape[1]):
inpName = parameters.inputs[idx].replace('$', '')
input_layer = tf.keras.Input(shape=(1, ), name=inpName)
# Categorical inputs #
if parameters.mask_op[idx]:
operation = getattr(Operations, parameters.operations[idx])()
encoded_all.append(operation(input_layer))
# Numerical inputs #
else:
inputs_numeric.append(input_layer)
means.append(scaler.mean_[idx])
variances.append(scaler.var_[idx])
inputs_all.append(input_layer)
# Concatenate all numerical inputs #
if int(tf_version[1]) < 4:
normalizer = preprocessing.Normalization(name='Normalization')
x_dummy = np.ones((10, len(means)))
# Needs a dummy to call the adapt method before setting the weights
normalizer.adapt(x_dummy)
normalizer.set_weights([np.array(means), np.array(variances)])
else:
normalizer = preprocessing.Normalization(mean=means,
variance=variances,
name='Normalization')
encoded_all.append(
normalizer(tf.keras.layers.concatenate(inputs_numeric,
name='Numerics')))
if len(encoded_all) > 1:
all_features = tf.keras.layers.concatenate(encoded_all,
axis=-1,
name="Features")
else:
all_features = encoded_all[0]
# Right branch : LBN
input_lbn_Layer = Input(shape=x_train_lbn.shape[1:], name='LBN_inputs')
lbn_layer = LBNLayer(
x_train_lbn.shape[1:],
n_particles=max(params['n_particles'],
1), # Hack so that 0 does not trigger error
boost_mode=LBN.PAIRS,
features=["E", "px", "py", "pz", "pt", "p", "m", "pair_cos"],
name='LBN')(input_lbn_Layer)
batchnorm = tf.keras.layers.BatchNormalization(name='batchnorm')(lbn_layer)
# Concatenation of left and right #
concatenate = tf.keras.layers.Concatenate(axis=-1)(
[all_features, batchnorm])
L1 = Dense(params['first_neuron'],
activation=params['activation'],
kernel_regularizer=l2(params['l2']))(
concatenate if params['n_particles'] > 0 else all_features)
hidden = hidden_layers(params, 1, batch_normalization=True).API(L1)
out = Dense(y_train.shape[1],
activation=params['output_activation'],
name='out')(hidden)
# Check preprocessing #
preprocess = Model(inputs=inputs_numeric, outputs=encoded_all[-1])
x_numeric = x_train[:, [not m for m in parameters.mask_op]]
out_preprocess = preprocess.predict(np.hsplit(x_numeric,
x_numeric.shape[1]),
batch_size=params['batch_size'])
mean_scale = np.mean(out_preprocess)
std_scale = np.std(out_preprocess)
if abs(mean_scale) > 0.01 or abs(
(std_scale - 1) /
std_scale) > 0.1: # Check that scaling is correct to 1%
logging.warning(
"Something is wrong with the preprocessing layer (mean = %0.6f, std = %0.6f), maybe you loaded an incorrect scaler"
% (mean_scale, std_scale))
# Tensorboard logs #
#path_board = os.path.join(parameters.main_path,"TensorBoard")
#suffix = 0
#while(os.path.exists(os.path.join(path_board,"Run_"+str(suffix)))):
# suffix += 1
#path_board = os.path.join(path_board,"Run_"+str(suffix))
#os.makedirs(path_board)
#logging.info("TensorBoard log dir is at %s"%path_board)
# Callbacks #
# Early stopping to stop learning if val_loss plateau for too long #
early_stopping = EarlyStopping(**parameters.early_stopping_params)
# Reduce learnign rate in case of plateau #
reduceLR = ReduceLROnPlateau(**parameters.reduceLR_params)
# Custom loss function plot for debugging #
loss_history = LossHistory()
# Tensorboard for checking live the loss curve #
#board = TensorBoard(log_dir=path_board,
# histogram_freq=1,
# batch_size=params['batch_size'],
# write_graph=True,
# write_grads=True,
# write_images=True)
Callback_list = [loss_history, early_stopping, reduceLR]
# Compile #
if 'resume' not in params: # Normal learning
# Define model #
model_inputs = [inputs_all]
if params['n_particles'] > 0:
model_inputs.append(input_lbn_Layer)
model = Model(inputs=model_inputs, outputs=[out])
initial_epoch = 0
else: # a model has to be imported and resumes training
#custom_objects = {'PreprocessLayer': PreprocessLayer,'OneHot': OneHot.OneHot}
logging.info("Loaded model %s" % params['resume'])
a = Restore(params['resume'],
custom_objects=custom_objects,
method='h5')
model = a.model
initial_epoch = params['initial_epoch']
model.compile(optimizer=Adam(lr=params['lr']),
loss=params['loss_function'],
metrics=[
tf.keras.metrics.CategoricalAccuracy(),
tf.keras.metrics.AUC(multi_label=True),
tf.keras.metrics.Precision(),
tf.keras.metrics.Recall()
])
model.summary()
fit_inputs = np.hsplit(x_train, x_train.shape[1])
fit_val = (np.hsplit(x_val, x_val.shape[1]), y_val, w_val)
if params['n_particles'] > 0:
fit_inputs.append(x_train_lbn)
fit_val[0].append(x_val_lbn)
# Fit #
history = model.fit(x=fit_inputs,
y=y_train,
sample_weight=w_train,
epochs=params['epochs'],
batch_size=params['batch_size'],
verbose=1,
validation_data=fit_val,
callbacks=Callback_list)
# Plot history #
PlotHistory(loss_history, params)
return history, model