def build_model(X_train, Y_train, X_val, Y_val, params): model = Sequential() model.add( Embedding(params['vocab_size'], params['e_size'], input_length=params['seq_len'])) model.add(Conv1D(32, 7, activation='relu')) model.add(MaxPooling1D(5)) model.add(Conv1D(32, 7, activation='relu')) model.add(GlobalAveragePooling1D()) model.add(Dropout(params['dropout'])) hidden_layers(model, params, 1) model.add(Dense(1, activation=params['last_activation'])) ## COMPILE model.compile(optimizer=params['optimizer'](lr_normalizer( params['lr'], params['optimizer'])), loss='binary_crossentropy', metrics=['acc']) out = model.fit(X_train, Y_train, batch_size=params['batch_size'], epochs=params['epochs'], validation_data=[X_val, Y_val], verbose=2) return out, model
def higgs_nn(X_train, Y_train, X_valid, Y_valid, params): model = Sequential() model.add( Dense(75, input_dim=X_train.shape[1], activation=params['activation'], kernel_initializer='normal')) model.add(Dropout(params['dropout'])) hidden_layers(model, params, 2) model.add( Dense(2, activation=params['last_activation'], kernel_initializer='normal')) model.compile(loss=params['losses'], optimizer=params['optimizer']( lr=lr_normalizer(params['lr'], params['optimizer'])), metrics=['acc']) history = model.fit(X_train, Y_train, validation_data=[X_valid, Y_valid], batch_size=params['batch_size'], epochs=params['epochs'], verbose=2) # finally we have to make sure that history object and model are returned return history, model
def scan_hyperparameter(x_train, y_train, x_val, y_val, params): ''' Using talos library to scan for the best hyperparameter settings for the Neural Network setup ''' # Building the model model = Sequential() model.add(Dense(params['first_neuron'], input_dim=x_train.shape[1], activation=params['activation'], kernel_initializer=params['kernel_initializer'])) model.add(Dropout(params['dropout'])) hidden_layers(model, params, 1) model.add(Dense(1, activation=params['last_activation'], kernel_initializer=params['kernel_initializer'])) model.compile(loss=params['losses'], optimizer=params['optimizer'](), metrics=['acc']) # Fit the model and record to tensorboard history = model.fit(x_train, y_train, validation_data=[x_val, y_val], batch_size=params['batch_size'], epochs=params['epochs'], verbose=0) return history, model
def hbb_model(train_data,train_y,valid_data,valid_y,param): model=Sequential() model.add(Dense(param['first_neuron'],input_dim=train_data.shape[1],activation=param['activation'],kernel_initializer='normal')) model.add(Dropout(param['dropout'])) hidden_layers(model,param,1) model.add(Dense(train_y.shape[1],activation=param['last_activation'],kernel_initializer='normal')) model.compile(optimizer=param['optimizer'](lr=lr_normalizer(param['lr'],param['optimizer'])),loss=param['losses'],metrics=['acc']) out=model.fit(train_data,train_y,batch_size=param['batch_size'],epochs=param['epochs'],verbose=0,validation_data=[valid_data,valid_y]) return out,model
def dae_model_hl(x_train, y_train, x_val, y_val, params): #print(params['x_train_noise'].shape) print(x_train.shape) print("masking training") x_train_noise = mask_function(dataframe=x_train, noise=float(params['noise']), batch_sizes=300) #masking training print("masking validation") x_val_noise = mask_function(dataframe=x_val, noise=float(params['noise']), batch_sizes=300) #masking validation print("building autoencoder network") model = Sequential() model.add( Dense(params['first_neuron'], activation=params['activation'], input_shape=(x_train.shape[1], ))) model.add(Dropout(params['dropout'])) #m.add(Dense(128, activation='elu')) hidden_layers(model, params, 1) model.add( Dense(params['embedding_size'], activation=params['activation'], name="bottleneck")) hidden_layers(model, params, 1) model.add(Dense(params['first_neuron'], activation=params['activation'])) #m.add(Dense(512, activation='elu')) model.add(Dropout(params['dropout'])) model.add(Dense(x_train.shape[1], activation=params['last_activation'])) #m.compile(loss='mean_squared_error', optimizer = params['optmizer']) model.compile(optimizer=params['optimizer']( lr=lr_normalizer(params['lr'], params['optimizer'])), loss=params['loss'], metrics=['accuracy']) print("training neural network") out = model.fit( x_train, x_train_noise, #x_train_noise, batch_size=params['batch_size'], epochs=params['epochs'], verbose=0, validation_data=[x_val, x_val_noise], #x_val_noise], callbacks=early_stopper(params['epochs'], mode='moderate')) #callbacks=early_stopper(params['epochs'], mode='strict'))#noisy_train, train, batch_size=128, epochs=params['epochs'], verbose=1, return out, model
def hbb_model(x_train, y_train, x_val, y_val, param): model = Sequential() model.add( Dense(param['first_neuron'], input_dim=x_train.shape[1], activation=param['activation'], kernel_initializer='orthogonal')) model.add(Dropout(param['dropout'])) hidden_layers(model, param, 1) model.add( Dense(y_train.shape[1], activation='softmax', kernel_initializer='orthogonal')) opt = keras.optimizers.Adam(lr=param['lr'], decay=param['decay']) model.compile(optimizer=opt, loss=param['losses'], metrics=['acc']) out = model.fit(x_train, y_train, batch_size=param['batch_size'], epochs=param['epochs'], verbose=0, validation_data=[x_val, y_val]) return out, model
def autoFNN(X_train, Y_train, X_val, Y_val, params): model = Sequential() model.add( Embedding(params['vocab_size'], params['e_size'], input_length=params['seq_len'])) model.add(Flatten()) hidden_layers(model, params, 1) model.add(Dense(1, activation=params['last_activation'])) model.compile(optimizer=params['optimizer'](lr_normalizer( params['lr'], params['optimizer'])), loss='binary_crossentropy', metrics=['acc']) out = model.fit(X_train, Y_train, batch_size=params['batch_size'], epochs=params['epochs'], validation_data=[X_val, Y_val], verbose=2) return out, model
def build_model(x_train, y_train, x_val, y_val, params): model = keras.Sequential() model.add(keras.layers.Dense(10, activation=params['activation'], input_dim=x_train.shape[1], use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=keras.regularizers.l1_l2(l1=params['l1'], l2=params['l2']), bias_regularizer=None)) model.add(keras.layers.Dropout(params['dropout'])) # If we want to also test for number of layers and shapes, that's possible hidden_layers(model, params, 1) # Then we finish again with completely standard Keras way model.add(keras.layers.Dense(1, activation=params['activation'], use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=keras.regularizers.l1_l2(l1=params['l1'], l2=params['l2']), bias_regularizer=None)) model.compile(optimizer=params['optimizer'](lr=lr_normalizer(params['lr'], params['optimizer'])), loss=params['losses'], metrics=['mse']) history = model.fit(x_train, y_train, validation_data=[x_val, y_val], batch_size=params['batch_size'], epochs=params['epochs'], callbacks=[early_stopper(epochs=params['epochs'], mode='moderate')], #callbacks=[early_stopper(epochs=params['epochs'], mode='strict')], verbose=0) # Finally we have to make sure that history object and model are returned return history, model
def _create_input_model(self, x_train, y_train, x_val, y_val, params): model = Sequential() if params['network'] != 'dense': x_train = array_reshape_conv1d(x_train) x_val = array_reshape_conv1d(x_val) if params['network'] == 'conv1d': model.add(Conv1D(params['first_neuron'], x_train.shape[1])) model.add(Flatten()) elif params['network'] == 'lstm': model.add(LSTM(params['first_neuron'])) if params['network'] == 'bidirectional_lstm': model.add(Bidirectional(LSTM(params['first_neuron']))) elif params['network'] == 'simplernn': model.add(SimpleRNN(params['first_neuron'])) elif params['network'] == 'dense': model.add(Dense(params['first_neuron'], input_dim=x_train.shape[1], activation='relu')) model.add(Dropout(params['dropout'])) # add hidden layers to the model hidden_layers(model, params, 1) # output layer (this is scetchy) try: last_neuron = y_train.shape[1] except IndexError: if len(np.unique(y_train)) == 2: last_neuron = 1 else: last_neuron = len(np.unique(y_train)) model.add(Dense(last_neuron, activation=params['last_activation'])) # bundle the optimizer with learning rate changes optimizer = params['optimizer'](lr=lr_normalizer(params['lr'], params['optimizer'])) # compile the model model.compile(optimizer=optimizer, loss=params['losses'], metrics=['acc']) # fit the model out = model.fit(x_train, y_train, batch_size=params['batch_size'], epochs=params['epochs'], verbose=0, validation_data=[x_val, y_val]) # pass the output to Talos return out, model
def NeuralNetGeneratorModel(x_train, y_train, x_val, y_val, params): """ Keras model for the Neural Network, used to scan the hyperparameter space by Talos Uses the generator rather than the input data (which are dummies) """ # Scaler # with open(parameters.scaler_path, 'rb') as handle: # Import scaler that was created before scaler = pickle.load(handle) # Design network # # Left branch : classic inputs -> Preprocess -> onehot inputs_numeric = [] means = [] variances = [] inputs_all = [] encoded_all = [] for idx in range(x_train.shape[1]): inpName = parameters.inputs[idx].replace('$', '').replace(' ', '').replace( '_', '') input_layer = tf.keras.Input(shape=(1, ), name=inpName) # Categorical inputs # if parameters.mask_op[idx]: operation = getattr(Operations, parameters.operations[idx])() encoded_all.append(operation(input_layer)) # Numerical inputs # else: inputs_numeric.append(input_layer) means.append(scaler.mean_[idx]) variances.append(scaler.var_[idx]) inputs_all.append(input_layer) # Concatenate all numerical inputs # if int(tf_version[1]) < 4: normalizer = preprocessing.Normalization(name='Normalization') x_dummy = np.ones((10, len(means))) # Needs a dummy to call the adapt method before setting the weights normalizer.adapt(x_dummy) normalizer.set_weights([np.array(means), np.array(variances)]) else: normalizer = preprocessing.Normalization(mean=means, variance=variances, name='Normalization') encoded_all.append( normalizer(tf.keras.layers.concatenate(inputs_numeric, name='Numerics'))) if len(encoded_all) > 1: all_features = tf.keras.layers.concatenate(encoded_all, axis=-1, name="Features") else: all_features = encoded_all[0] # Right branch : LBN lbn_input_shape = (len(parameters.LBN_inputs) // 4, 4) input_lbn_Layer = Input(shape=lbn_input_shape, name='LBN_inputs') lbn_layer = LBNLayer( lbn_input_shape, n_particles=max(params['n_particles'], 1), # Hack so that 0 does not trigger error boost_mode=LBN.PAIRS, features=["E", "px", "py", "pz", "pt", "p", "m", "pair_cos"], name='LBN')(input_lbn_Layer) batchnorm = tf.keras.layers.BatchNormalization(name='batchnorm')(lbn_layer) # Concatenation of left and right # concatenate = tf.keras.layers.Concatenate(axis=-1)( [all_features, batchnorm]) L1 = Dense(params['first_neuron'], activation=params['activation'], kernel_regularizer=l2(params['l2']))( concatenate if params['n_particles'] > 0 else all_features) hidden = hidden_layers(params, 1, batch_normalization=True).API(L1) out = Dense(y_train.shape[1], activation=params['output_activation'], name='out')(hidden) # Tensorboard logs # # path_board = os.path.join(parameters.main_path,"TensorBoard") # suffix = 0 # while(os.path.exists(os.path.join(path_board,"Run_"+str(suffix)))): # suffix += 1 # path_board = os.path.join(path_board,"Run_"+str(suffix)) # os.makedirs(path_board) # logging.info("TensorBoard log dir is at %s"%path_board) # Callbacks # # Early stopping to stop learning if val_loss plateau for too long # early_stopping = EarlyStopping(**parameters.early_stopping_params) # Reduce learnign rate in case of plateau # reduceLR = ReduceLROnPlateau(**parameters.reduceLR_params) # Custom loss function plot for debugging # loss_history = LossHistory() # Tensorboard for checking live the loss curve # # board = TensorBoard(log_dir=path_board, # histogram_freq=1, # batch_size=params['batch_size'], # write_graph=True, # write_grads=True, # write_images=True) # Callback_list = [loss_history,early_stopping,reduceLR,board] Callback_list = [loss_history, early_stopping, reduceLR] # Compile # if 'resume' not in params: # Normal learning # Define model # model_inputs = [inputs_all] if params['n_particles'] > 0: model_inputs.append(input_lbn_Layer) model = Model(inputs=model_inputs, outputs=[out]) initial_epoch = 0 else: # a model has to be imported and resumes training #custom_objects = {'PreprocessLayer': PreprocessLayer,'OneHot': OneHot.OneHot} logging.info("Loaded model %s" % params['resume']) a = Restore(params['resume'], custom_objects=custom_objects, method='h5') model = a.model initial_epoch = params['initial_epoch'] model.compile(optimizer=Adam(lr=params['lr']), loss=params['loss_function'], metrics=[ tf.keras.metrics.CategoricalAccuracy(), tf.keras.metrics.AUC(multi_label=True), tf.keras.metrics.Precision(), tf.keras.metrics.Recall() ]) model.summary() # Generator # training_generator = DataGenerator( path=parameters.config, inputs=parameters.inputs, outputs=parameters.outputs, inputsLBN=parameters.LBN_inputs if params['n_particles'] > 0 else None, cut=parameters.cut, weight=parameters.weight, batch_size=params['batch_size'], state_set='training', model_idx=params['model_idx'] if parameters.crossvalidation else None) validation_generator = DataGenerator( path=parameters.config, inputs=parameters.inputs, outputs=parameters.outputs, inputsLBN=parameters.LBN_inputs if params['n_particles'] > 0 else None, cut=parameters.cut, weight=parameters.weight, batch_size=params['batch_size'], state_set='validation', model_idx=params['model_idx'] if parameters.crossvalidation else None) # Some verbose logging # logging.info("Will use %d workers" % parameters.workers) logging.warning("Tensorflow location " + tf.__file__) if len(tf.config.experimental.list_physical_devices('XLA_GPU')) > 0: logging.info("GPU detected") #logging.warning(K.tensorflow_backend._get_available_gpus()) # Fit # history = model.fit_generator( generator=training_generator, # Training data from generator instance validation_data= validation_generator, # Validation data from generator instance epochs=params['epochs'], # Number of epochs verbose=1, max_queue_size=parameters.workers * 2, # Length of batch queue callbacks=Callback_list, # Callbacks initial_epoch= initial_epoch, # In case of resumed training will be different from 0 workers=parameters. workers, # Number of threads for batch generation (0 : all in same) shuffle=True, # Shuffle order at each epoch use_multiprocessing=True) # Needs to be turned on for queuing batches # Plot history # PlotHistory(loss_history) return history, model
def NeuralNetModel(x_train, y_train, x_val, y_val, params): """ Keras model for the Neural Network, used to scan the hyperparameter space by Talos Uses the data provided as inputs """ # Split y = [target,weight], Talos does not leave room for the weight so had to be included in one of the arrays w_train = y_train[:, -1] w_val = y_val[:, -1] y_train = y_train[:, :-1] y_val = y_val[:, :-1] x_train_lbn = x_train[:, -len(parameters.LBN_inputs):].reshape( -1, 4, len(parameters.LBN_inputs) // 4) x_train = x_train[:, :-len(parameters.LBN_inputs)] x_val_lbn = x_val[:, -len(parameters.LBN_inputs):].reshape( -1, 4, len(parameters.LBN_inputs) // 4) x_val = x_val[:, :-len(parameters.LBN_inputs)] # Scaler # with open(parameters.scaler_path, 'rb') as handle: # Import scaler that was created before scaler = pickle.load(handle) # Design network # # Left branch : classic inputs -> Preprocess -> onehot inputs_numeric = [] means = [] variances = [] inputs_all = [] encoded_all = [] for idx in range(x_train.shape[1]): inpName = parameters.inputs[idx].replace('$', '') input_layer = tf.keras.Input(shape=(1, ), name=inpName) # Categorical inputs # if parameters.mask_op[idx]: operation = getattr(Operations, parameters.operations[idx])() encoded_all.append(operation(input_layer)) # Numerical inputs # else: inputs_numeric.append(input_layer) means.append(scaler.mean_[idx]) variances.append(scaler.var_[idx]) inputs_all.append(input_layer) # Concatenate all numerical inputs # if int(tf_version[1]) < 4: normalizer = preprocessing.Normalization(name='Normalization') x_dummy = np.ones((10, len(means))) # Needs a dummy to call the adapt method before setting the weights normalizer.adapt(x_dummy) normalizer.set_weights([np.array(means), np.array(variances)]) else: normalizer = preprocessing.Normalization(mean=means, variance=variances, name='Normalization') encoded_all.append( normalizer(tf.keras.layers.concatenate(inputs_numeric, name='Numerics'))) if len(encoded_all) > 1: all_features = tf.keras.layers.concatenate(encoded_all, axis=-1, name="Features") else: all_features = encoded_all[0] # Right branch : LBN input_lbn_Layer = Input(shape=x_train_lbn.shape[1:], name='LBN_inputs') lbn_layer = LBNLayer( x_train_lbn.shape[1:], n_particles=max(params['n_particles'], 1), # Hack so that 0 does not trigger error boost_mode=LBN.PAIRS, features=["E", "px", "py", "pz", "pt", "p", "m", "pair_cos"], name='LBN')(input_lbn_Layer) batchnorm = tf.keras.layers.BatchNormalization(name='batchnorm')(lbn_layer) # Concatenation of left and right # concatenate = tf.keras.layers.Concatenate(axis=-1)( [all_features, batchnorm]) L1 = Dense(params['first_neuron'], activation=params['activation'], kernel_regularizer=l2(params['l2']))( concatenate if params['n_particles'] > 0 else all_features) hidden = hidden_layers(params, 1, batch_normalization=True).API(L1) out = Dense(y_train.shape[1], activation=params['output_activation'], name='out')(hidden) # Check preprocessing # preprocess = Model(inputs=inputs_numeric, outputs=encoded_all[-1]) x_numeric = x_train[:, [not m for m in parameters.mask_op]] out_preprocess = preprocess.predict(np.hsplit(x_numeric, x_numeric.shape[1]), batch_size=params['batch_size']) mean_scale = np.mean(out_preprocess) std_scale = np.std(out_preprocess) if abs(mean_scale) > 0.01 or abs( (std_scale - 1) / std_scale) > 0.1: # Check that scaling is correct to 1% logging.warning( "Something is wrong with the preprocessing layer (mean = %0.6f, std = %0.6f), maybe you loaded an incorrect scaler" % (mean_scale, std_scale)) # Tensorboard logs # #path_board = os.path.join(parameters.main_path,"TensorBoard") #suffix = 0 #while(os.path.exists(os.path.join(path_board,"Run_"+str(suffix)))): # suffix += 1 #path_board = os.path.join(path_board,"Run_"+str(suffix)) #os.makedirs(path_board) #logging.info("TensorBoard log dir is at %s"%path_board) # Callbacks # # Early stopping to stop learning if val_loss plateau for too long # early_stopping = EarlyStopping(**parameters.early_stopping_params) # Reduce learnign rate in case of plateau # reduceLR = ReduceLROnPlateau(**parameters.reduceLR_params) # Custom loss function plot for debugging # loss_history = LossHistory() # Tensorboard for checking live the loss curve # #board = TensorBoard(log_dir=path_board, # histogram_freq=1, # batch_size=params['batch_size'], # write_graph=True, # write_grads=True, # write_images=True) Callback_list = [loss_history, early_stopping, reduceLR] # Compile # if 'resume' not in params: # Normal learning # Define model # model_inputs = [inputs_all] if params['n_particles'] > 0: model_inputs.append(input_lbn_Layer) model = Model(inputs=model_inputs, outputs=[out]) initial_epoch = 0 else: # a model has to be imported and resumes training #custom_objects = {'PreprocessLayer': PreprocessLayer,'OneHot': OneHot.OneHot} logging.info("Loaded model %s" % params['resume']) a = Restore(params['resume'], custom_objects=custom_objects, method='h5') model = a.model initial_epoch = params['initial_epoch'] model.compile(optimizer=Adam(lr=params['lr']), loss=params['loss_function'], metrics=[ tf.keras.metrics.CategoricalAccuracy(), tf.keras.metrics.AUC(multi_label=True), tf.keras.metrics.Precision(), tf.keras.metrics.Recall() ]) model.summary() fit_inputs = np.hsplit(x_train, x_train.shape[1]) fit_val = (np.hsplit(x_val, x_val.shape[1]), y_val, w_val) if params['n_particles'] > 0: fit_inputs.append(x_train_lbn) fit_val[0].append(x_val_lbn) # Fit # history = model.fit(x=fit_inputs, y=y_train, sample_weight=w_train, epochs=params['epochs'], batch_size=params['batch_size'], verbose=1, validation_data=fit_val, callbacks=Callback_list) # Plot history # PlotHistory(loss_history, params) return history, model