def fit_model(model, train_batches, epochs=1, lr=None, val_batches=None, save_path='', callbacks=None): fit_callbacks = [ TQDMNotebookCallback(), UpdateHistoryCallback(), LearningRateSetter(epochs, lr) ] if save_path != '': fit_callbacks.append(ModelSave(save_path)) if callbacks is not None: fit_callbacks.append(callbacks) if val_batches is None: model.fit_generator(train_batches, train_batches.num_batches, epochs=np.sum(epochs), verbose=0, callbacks=fit_callbacks) else: model.fit_generator(train_batches, train_batches.num_batches, epochs=np.sum(epochs), verbose=0, callbacks=fit_callbacks, validation_data=val_batches, validation_steps=val_batches.num_batches)
def get_callbacks(MODEL_PATH): """ Saves model if validation IoU increased, applies Cyclic Learning Rate, shows TQDM based progress bar. """ checkpoint = ModelCheckpoint(MODEL_PATH, monitor='val_iou_score', verbose=1, save_best_only=True, mode='max') clr = CyclicLR(base_lr=0.000001, max_lr=0.001, step_size=3000, mode='exp_range') callbacks = [checkpoint, clr, TQDMNotebookCallback(leave_inner=True, leave_outer=True)] return callbacks
def fit(self, train_data, batch_size, epochs, learning_rate=1e-3): ''' Function to fit the style transfer model. Args: train_data (ndarray): Array of symmetrical train images in (288 x 288) size. batch_size (int): Batch size for training epochs (int): Number of epochs to run Returns: None ''' # Dictionary to use TQDMN to visualise training params = { 'verbose': 0, 'callbacks': [TQDMNotebookCallback(leave_inner=True)] } # Our target is 0 loss of the MSE (final model output) target = np.zeros((train_data.shape[0], 1)) # Set learning rate K.set_value(self.m_style.optimizer.lr, learning_rate) # Fit model self.m_style.fit(x=[train_data, train_data], y=target, batch_size=batch_size, epochs=epochs, **params)
def getModel(n, num_epochs=Epochs): model = Sequential() model.add(Dense(n, activation='relu', input_dim=len(feature_columns))) model.add(Dropout(0.5)) model.add(Dense(int(n / 2), activation='relu')) model.add(Dropout(0.5)) model.add(Dense(nitems, activation='softmax')) sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) callbacks = [ EarlyStopping(monitor='acc', patience=10, verbose=0), TQDMNotebookCallback() ] model.fit(X_train, y_train, epochs=num_epochs, batch_size=128, verbose=1, callbacks=callbacks) return model
def train_model(input_to_softmax, pickle_path, save_model_path, train_json='train_corpus.json', valid_json='valid_corpus.json', minibatch_size=20, spectrogram=True, mfcc_dim=13, optimizer=SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5), epochs=20, verbose=1, sort_by_duration=False, max_duration=10.0): # create a class instance for obtaining batches of data audio_gen = AudioGenerator(minibatch_size=minibatch_size, spectrogram=spectrogram, mfcc_dim=mfcc_dim, max_duration=max_duration, sort_by_duration=sort_by_duration) # add the training data to the generator audio_gen.load_train_data(train_json) audio_gen.load_validation_data(valid_json) # calculate steps_per_epoch num_train_examples = len(audio_gen.train_audio_paths) steps_per_epoch = num_train_examples // minibatch_size # calculate validation_steps num_valid_samples = len(audio_gen.valid_audio_paths) validation_steps = num_valid_samples // minibatch_size # add CTC loss to the NN specified in input_to_softmax model = add_ctc_loss(input_to_softmax) # CTC loss is implemented elsewhere, so use a dummy lambda function for the loss model.compile(loss={ 'ctc': lambda y_true, y_pred: y_pred }, optimizer=optimizer) # make results directory, if necessary if not os.path.exists('results'): os.makedirs('results') # add checkpointer checkpointer = ModelCheckpoint(filepath=os.path.join( 'results', save_model_path), verbose=0) # train the model callbacks = [TQDMNotebookCallback(), checkpointer ] if verbose < 0 else [checkpointer] hist = model.fit_generator(generator=audio_gen.next_train(), steps_per_epoch=steps_per_epoch, epochs=epochs, validation_data=audio_gen.next_valid(), validation_steps=validation_steps, callbacks=callbacks, verbose=verbose) # save model loss with open(os.path.join('results', pickle_path), 'wb') as f: pickle.dump(hist.history, f)
def fit(self, batches, val_batches, nb_epoch=1): """ Fits the model on data yielded batch-by-batch by a Python generator. See Keras documentation: https://keras.io/models/model/ """ self.model.fit_generator(batches, samples_per_epoch=batches.nb_sample, nb_epoch=nb_epoch, validation_data=val_batches, nb_val_samples=val_batches.nb_sample, verbose=1, callbacks=[TQDMNotebookCallback(leave_inner=True, leave_outer=True)])
def train(self, epochs=1): """ Train the model and save afterwards. If the shared model is not yet frozen, the network corresponding the first problem statement will be trained. Note that it is trained on all data, which isn't how it should be! When done properly, one should ALWAYS separate the full dataset in a training and it is a test set. It is also highly encouraged to incorporate a validation set as well. But again, to simplify the model in its whole, we will ignore this best practice. If the shared model is indeed frozen, only manually curated samples will be used to train the model on. :param epochs: Number of epochs the model will be trained on """ prep('Predicting...', key='training', silent=True) # Create TensorBoard instantiation tensorboard = TensorBoard(log_dir='logs/{}'.format(time())) # Train the suitable network if self.is_frozen: self.network_2.fit( x=self.train_images, y=self.train_labels, batch_size=config.BATCH_SIZE, epochs=epochs, verbose=0, callbacks=[ TQDMNotebookCallback(leave_inner=True), tensorboard ], ) self.current_epoch += epochs # Epoch count only considered for second network else: self.network_1.fit( x=self.evaluation_images, y=self.evaluation_labels, batch_size=config.BATCH_SIZE, epochs=epochs, verbose=0, callbacks=[ TQDMNotebookCallback(leave_inner=True), tensorboard ], ) drop(key='training', silent=True) self.save_model()
def fit_model_nb(model, X_train, train_t, X_test, test_t): # for fitting the model in a jupyter notebook history = History() model.fit(X_train, train_t, epochs=EPOCHS, batch_size=BATCH, validation_data=[X_test, test_t], verbose=0, callbacks=[TQDMNotebookCallback(), history]) return history
def fit(self, train_x: Union[Dict, List, np.ndarray], train_y: Union[Dict, List, np.ndarray], test_x: Union[Dict, List, np.ndarray] = None, test_y: Union[Dict, List, np.ndarray] = None) -> pd.DataFrame: """Fit the model using given training parameters. Parameters -------------------------- train: Tuple[Union[Dict, List, np.ndarray]], Either a tuple of list, np.ndarrays or dictionaries, containing the training data. test: Tuple[Union[Dict, List, np.ndarray]] = None Either a tuple of list, np.ndarrays or dictionaries, containing the validation data data. These data are optional, but they are required if the given monitor metric starts with "val_" Raises -------------------------- ValueError, If no test data are given but the monitor metric starts with the word "val_", meaning it has to be computed on the test data. Returns --------------------------- The training history as a pandas dataframe. """ test = None if all(d is not None for d in (test_x, test_y)): test = (test_x, test_y) if test is None and self._monitor.startswith("val_"): raise ValueError( "No test set was given, " "but a validation metric was required for the early stopping.") return pd.DataFrame( self._model.fit( train_x, train_y, epochs=self._max_epochs, batch_size=self._batch_size, validation_data=test, verbose=False, shuffle=True, callbacks=[ EarlyStopping(self._monitor, patience=self._patience), # We show the correct kind of callback depending if this # is running in a CLI or jupyter notebook-like environment. TQDMNotebookCallback() if is_notebook() else TQDMCallback() ]).history)
def main(): print('Loading train') train = load_train() X_train, y_train = get_X_y(train) print('Oversampling') X_train, y_train = oversample(X_train, y_train) print('Creating an embedding matrix') embedding_matrix, x1, x2, word_index = get_embedding_matrix(X_train) print('Training') model = get_model(word_index, embedding_matrix, X_train) model.compile(loss='binary_crossentropy', optimizer=RMSprop(0.001), metrics=['accuracy']) X_train, X_val, x1_train, x1_val, x2_train, x2_val, y_train, y_val = train_test_split( X_train, x1, x2, y_train, test_size=0.1, random_state=SEED) train_data = [ x1_train, x2_train, x1_train, x2_train, x1_train, x2_train, x1_train, x2_train, X_train ] val_data = [ x1_val, x2_val, x1_val, x2_val, x1_val, x2_val, x1_val, x2_val, X_val ] model.fit(train_data, y_train, batch_size=64 * 16, epochs=4, verbose=0, validation_data=(val_data, y_val), callbacks=[TQDMNotebookCallback()]) print('Loading test') test = load_test() X_test = test[features] del test gc.collect() print('Generating predictions without correction') predictions = model.predict_proba(X_test) generate_submission(predictions, 'xgb3_yO_nC') print('Generating predictions with correction') predictions = prediction_correction(predictions) generate_submission(predictions, 'xgb3_yO_yC')
def make_steps(step, ampl): """ Perform training epochs @param step Number of epochs to perform @param ampl the K, the randomized component of the score matrix. """ global w2ts, t2i, steps, features, score, histories # shuffle the training pictures random.shuffle(train) # Map whale id to the list of associated training picture hash value w2ts = {} for w, hs in w2hs.items(): for h in hs: if h in train_set: if w not in w2ts: w2ts[w] = [] if h not in w2ts[w]: w2ts[w].append(h) for w, ts in w2ts.items(): w2ts[w] = np.array(ts) # Map training picture hash value to index in 'train' array t2i = {} for i, t in enumerate(train): t2i[t] = i # Compute the match score for each picture pair features, score = compute_score() # Train the model for 'step' epochs history = model.fit_generator( TrainingData(score + ampl * np.random.random_sample(size=score.shape), steps=step, batch_size=32), initial_epoch=steps, epochs=steps + step, max_queue_size=12, workers=6, verbose=0, callbacks=[ TQDMNotebookCallback(leave_inner=True, metric_format='{value:0.3f}') ]).history steps += step # Collect history data history['epochs'] = steps history['ms'] = np.mean(score) history['lr'] = get_lr(model) print(history['epochs'], history['lr'], history['ms']) histories.append(history)
def evaluate_model(runs=5): scores = [] for i in range(runs): print('Executing run %d' % (i+1)) model = mymodel() history = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, callbacks=[TQDMNotebookCallback()], verbose=0) scores.append(model.evaluate(x_test, y_test, verbose=0)) print(' * Test set Loss: %.4f, Accuracy: %.4f' % (scores[-1][0], scores[-1][1])) accuracies = [score[1] for score in scores] return np.mean(accuracies), np.std(accuracies)
def _default_callbacks(self): return [ ModelCheckpoint(self._checkpoint_path, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True), ReduceLROnPlateau(monitor='val_loss', patience=3, verbose=1, factor=0.65, min_lr=0.00001), EarlyStopping(monitor='val_loss', patience=8), TQDMNotebookCallback() ]
def lstm_trainer(train_tfrecord, train_lr, train_epochs): # Building the model lstm_model = Sequential() lstm_model.add(BatchNormalization(input_shape=(None, 128))) lstm_model.add(Dropout(0.5)) lstm_model.add( LSTM(128, activation='relu', kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l2(0.01))) lstm_model.add(Dense(1, activation='sigmoid')) # try using different optimizers and different optimizer configs lstm_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) # Fitting the LSTM model batch_size = 32 CV_frac = 0.1 train_gen = data_generator(batch_size, train_tfrecord, 0, 1 - CV_frac) val_gen = data_generator(20, train_tfrecord, 1 - CV_frac, 1) rec_len = 17662 lstm_h = lstm_model.fit_generator( train_gen, steps_per_epoch=int(rec_len * (1 - CV_frac)) // batch_size, epochs=train_epochs, validation_data=val_gen, validation_steps=int(rec_len * CV_frac) // 20, verbose=1, callbacks=[TQDMNotebookCallback()]) # Plot the model architecture plot_model( lstm_model, to_file='../results/LSTM/LSTM_Loss=BinCE_Epochs{}_lt={}.png'.format( train_epochs, train_lr)) # Save the lstm model lstm_model.save( '../models/1LayerLSTM__Loss=BinCE_lr={}_Epochs={}.h5'.format( train_lr, train_epochs)) return lstm_h
def trainer(train_tfrecord, train_lr, train_epochs): #Logistic Regression Model # Adam optimizer with Loss = Binary Cross Entropy lr_model = Sequential() lr_model.add(BatchNormalization(input_shape=(10, 128))) lr_model.add(Flatten()) lr_model.add(Dense(1, activation='sigmoid')) adam = optimizers.Adam(lr=train_lr) lr_model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy']) #Training the model # Hyperparameters : Epochs = train_epochs, batch_size = 40, learning rate = train_lr batch_size = 40 CV_frac = 0.1 train_gen = data_generator(batch_size, train_tfrecord, 0, 1 - CV_frac) val_gen = data_generator(20, train_tfrecord, 1 - CV_frac, 1) rec_len = 17662 lr_h = lr_model.fit_generator( train_gen, steps_per_epoch=int(rec_len * (1 - CV_frac)) // batch_size, epochs=train_epochs, validation_data=val_gen, validation_steps=int(rec_len * CV_frac) // 20, verbose=1, callbacks=[TQDMNotebookCallback()]) #Save the model architecture image always at same path plot_model( lr_model, to_file= '../results/LogisticReg/model_LogisticRegression_BinaryCE_Adam_lr={}_Epochs{}.png' .format(train_lr, train_epochs)) #Save the model at the same path lr_model.save( '../models/LogisticRegression_BinCE_Adam_lr={}_Epochs={}.h5'.format( train_lr, train_epochs)) return lr_h
def evaluate_model(runs=5): scores = [] for i in range(runs): print('Executing run %d' % (i + 1)) model = mymodel() model.fit_generator(train_generator, callbacks=[TQDMNotebookCallback()], steps_per_epoch=num_train_samples // batch_size, epochs=epochs, verbose=0) print(' * Evaluating model on test set') scores.append( model.evaluate_generator(test_generator, steps=num_test_samples // batch_size, verbose=0)) print(' * Test set Loss: %.4f, Accuracy: %.4f' % (scores[-1][0], scores[-1][1])) accuracies = [score[1] for score in scores] return np.mean(accuracies), np.std(accuracies)
def train_fixed(self, x_trn, x_val, y_trn, y_val): self.models.append(model1()) #self.models.append(model1()) self.models.append(model1()) self.models.append(model1()) self.models.append(model1()) self.models.append(model1()) self.models.append(model1()) self.models.append(model1()) self.models.append(model1()) self.models.append(model1()) losses = np.zeros(len(self.models)) for i_model, model in enumerate(self.models): es = EarlyStopping(monitor='val_loss', mode='min', patience=10) mc = ModelCheckpoint('best_model.h5', monitor='val_loss', mode='min', save_best_only=True) model.fit(x=x_trn, y=y_trn, validation_data=(x_val, y_val), callbacks=[es, mc, TQDMNotebookCallback()], batch_size=25, verbose=0, epochs=150) # model = load_model('best_model.h5') losses[i_model] = model.evaluate(x_val, y_val)[0] best_model = self.models[np.argmin(losses)] best_model.save('fixed_model.h5') # Add model to fixed models #K.clear_session() self.fixed_models.append(load_model('fixed_model.h5')) self.models = [] return losses, np.argmin(losses)
def model_training_logging(self,autoencoder, train_x, params): """ Model training and Logging. Below is where we set up the actual run including checkpoints and the tensorboard. """ nb_epoch = 50 batch_size = 128 input_dim = train_x.shape[1] # num of columns, 30 encoding_dim = 14 hidden_dim = int(encoding_dim / 2) # i.e. 7 learning_rate = 1e-7 path_to_pickle = 'gdrive/My Drive/DataSet_Colab_NoteBooks/pickle-machine/' file = path_to_pickle + "autoencoder_husky50.h5" autoencoder.compile(metrics=['accuracy'], loss='mean_squared_error', optimizer='adam') cp = ModelCheckpoint(filepath=file, save_best_only=True, verbose=0) logs = '/data/tensorflow_logs' tb = TensorBoard(log_dir=logs, histogram_freq=0, write_graph=True, write_images=True) history = autoencoder.fit(train_x, train_x, epochs=nb_epoch, batch_size=batch_size, shuffle=True, # validation_data=(test_x, test_x), validation_split=0.2, verbose=1, callbacks=[TQDMNotebookCallback(), cp, tb]).history return history
def train(model, dataset_arrays, labels, config): """All the training without any of the extra printing.""" earlystop = callbacks.EarlyStopping( monitor="val_loss", patience=config["patience"], restore_best_weights=True, min_delta=1.0e-4, ) callback_list = [earlystop] if config.get("progbar"): tqdmcb = TQDMNotebookCallback() tqdmcb.on_train_batch_begin = tqdmcb.on_batch_begin tqdmcb.on_train_batch_end = tqdmcb.on_batch_end tqdmcb.on_test_batch_begin = tqdmcb.on_batch_begin tqdmcb.on_test_batch_end = tqdmcb.on_batch_end setattr(tqdmcb, "on_test_begin", lambda x: None) setattr(tqdmcb, "on_test_end", lambda x: None) callback_list.append(tqdmcb) if config.get("logdir"): tensorboard_callback = callbacks.TensorBoard(log_dir=config["logdir"]) callback_list.append(tensorboard_callback) model.compile( optimizer=config["optimizer"], loss=config["loss"], metrics=config.get("metrics", ["accuracy"]), weighted_metrics=config.get("weighted_metrics"), ) history = model.fit( dataset_arrays["train"], labels["train"], class_weight=config.get("class_weights"), verbose=config["verbose"], batch_size=config["batch_size"], epochs=config["epochs"], callbacks=callback_list, validation_split=config["val_split"], ) return history
history = LossHistory() early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=2, verbose=0, mode='auto') fitted_model = model.fit_generator(train_generator, steps_per_epoch=int(7394 * 1 - 0.5) // batch, epochs=epochs, validation_data=valid_generator, validation_steps=int(7394 * 0.25) // batch, callbacks=[ TQDMNotebookCallback(leave_inner=True, leave_outer=True), early_stopping, history ]) model.save('./models/new_model_cla_mode.h5') # model_json = model.to_json() # with open('./data_model2.json', 'w') as jf: # jf.write(model_json) # losses, val_losses = history.losses, history.val_losses fig = plt.figure(figsize=(15, 5)) plt.plot(fitted_model.history['loss'], 'g', label="train losses") plt.plot(fitted_model.history['val_loss'], 'r', label="valid losses") plt.grid(True) plt.title('Training loss vs. Validation loss')
write_graph=True) modelCallback = ModelCheckpoint( 'model_checkpoint_advbig_sos_eos_unk.{epoch:03d}-{loss:.3f}.hdf5', monitor='val_loss', verbose=1, save_best_only=False, save_weights_only=True, mode='auto', period=1) # creating different sets of Params to easily import into the model at train time params = { 'verbose': 1, 'callbacks': [ TQDMNotebookCallback(), reduce_LR, early_stopping, tbCallBack, modelCallback ] } params2 = { 'verbose': 1, 'callbacks': [LRDrop, TQDMNotebookCallback(), reduce_LR, early_stopping] } params3 = { 'verbose': 1, 'callbacks': [LRDrop, TQDMNotebookCallback(), reduce_LR, early_stopping] } # In[ ]:
def train_tlnn(regions, start, end, deep): from keras.models import Sequential from keras.layers import Dense, Dropout from keras.optimizers import Adam from IPython.core.display import HTML from keras_tqdm import TQDMNotebookCallback display(HTML(""" <style> .p-Widget.jp-OutputPrompt.jp-OutputArea-prompt:empty { padding: 0; border: 0; } </style> """)) X_test, y_test, _ = gen_data(regions, start, end) mod = [Dense(128, kernel_initializer='uniform', input_dim=len(X_test[0]), activation='relu'), Dropout(0.1)] if deep: mod.extend([Dense(32, kernel_initializer='uniform', activation='relu'), Dropout(0.1)]) model = Sequential(mod + [Dense(1, kernel_initializer='uniform', activation='linear')]) model.compile(optimizer=Adam(), loss='mse') batch_size = int(len(X_test)/64) model.fit(X_test, y_test, epochs=100, batch_size=batch_size, verbose=0, callbacks=[TQDMNotebookCallback()]) return model
def train_neural_network(x_train, y_train, x_val, y_val, configs, partial_model_architecture, batch_size=32, nb_epoch=5, normalize=True, checkpoint_dir=None, neural_network_name='my_neural_network', training_log_file='training.log', early_stopping=False): """Train a neural network to classify crystal structures represented as two-dimensional diffraction fingerprints. This model was introduced in [1]_. x_train: np.array, [batch, width, height, channels] .. [1] A. Ziletti, D. Kumar, M. Scheffler, and L. M. Ghiringhelli, “Insightful classification of crystal structures using deep learning”, Nature Communications, vol. 9, pp. 2775 (2018) .. codeauthor:: Angelo Ziletti <*****@*****.**> """ if checkpoint_dir is None: checkpoint_dir = configs['io']['results_folder'] filename_no_ext = os.path.abspath( os.path.normpath(os.path.join(checkpoint_dir, neural_network_name))) training_log_file_path = os.path.abspath( os.path.normpath(os.path.join(checkpoint_dir, training_log_file))) # reshape to follow the image conventions # - TensorFlow backend: [batch, width, height, channels] # - Theano backend: [batch, channels, width, height] x_train = reshape_images_to_theano(x_train) x_val = reshape_images_to_theano(x_val) assert x_train.shape[1] == x_val.shape[1] assert x_train.shape[2] == x_val.shape[2] assert x_train.shape[3] == x_val.shape[3] img_channels = x_train.shape[1] img_width = x_train.shape[2] img_height = x_train.shape[3] logger.info('Loading datasets.') logger.debug('x_train shape: {0}'.format(x_train.shape)) logger.debug('y_train shape: {0}'.format(y_train.shape)) logger.debug('x_val shape: {0}'.format(x_val.shape)) logger.debug('y_val shape: {0}'.format(y_val.shape)) logger.debug('Training samples: {0}'.format(x_train.shape[0])) logger.debug('Validation samples: {0}'.format(x_val.shape[0])) logger.debug("Img channels: {}".format(x_train.shape[1])) logger.debug("Img width: {}".format(x_train.shape[2])) logger.debug("Img height: {}".format(x_train.shape[3])) x_train = x_train.astype('float32') x_val = x_val.astype('float32') # normalize each image separately if normalize: for idx in range(x_train.shape[0]): x_train[idx, :, :, :] = (x_train[idx, :, :, :] - np.amin( x_train[idx, :, :, :])) / (np.amax(x_train[idx, :, :, :]) - np.amin(x_train[idx, :, :, :])) for idx in range(x_val.shape[0]): x_val[idx, :, :, :] = (x_val[idx, :, :, :] - np.amin( x_val[idx, :, :, :])) / (np.amax(x_val[idx, :, :, :]) - np.amin(x_val[idx, :, :, :])) # check if the image is already normalized logger.info( 'Maximum value in x_train for the 1st image (to check normalization): {0}' .format(np.amax(x_train[0, :, :, :]))) logger.info( 'Maximum value in x_val for the 1st image (to check normalization): {0}' .format(np.amax(x_val[0, :, :, :]))) # convert class vectors to binary class matrices nb_classes = len(set(y_train)) nb_classes_val = len(set(y_val)) if nb_classes_val != nb_classes: raise ValueError( "Different number of unique classes in training and validation set: {} vs {}." "Training set unique classes: {}" "Validation set unique classes: {}".format(nb_classes, nb_classes_val, set(y_train), set(y_val))) y_train = np_utils.to_categorical(y_train, nb_classes) y_val = np_utils.to_categorical(y_val, nb_classes) logger.info('Loading and formatting of data completed.') # return the Keras model model = partial_model_architecture(n_rows=img_width, n_columns=img_height, img_channels=img_channels, nb_classes=nb_classes) model.summary() # serialize model to JSON model_json = model.to_json() with open(filename_no_ext + ".json", "w") as json_file: json_file.write(model_json) callbacks = [] csv_logger = CSVLogger(training_log_file_path, separator=',', append=False) save_model_per_epoch = ModelCheckpoint(filename_no_ext + ".h5", monitor='val_acc', verbose=1, save_best_only=True, mode='max', period=1) callbacks.append(csv_logger) callbacks.append(save_model_per_epoch) # if you are running on Notebook if configs['runtime']['isBeaker']: callbacks.append( TQDMNotebookCallback(leave_inner=True, leave_outer=True)) else: callbacks.append(TQDMCallback(leave_inner=True, leave_outer=True)) if early_stopping: EarlyStopping(monitor='val_loss', min_delta=0.001, patience=1, verbose=0, mode='auto') adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, decay=0.0) model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) model.fit(x_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, validation_data=(x_val, y_val), shuffle=True, verbose=0, callbacks=callbacks) # serialize weights to HDF5 model.save(filename_no_ext + ".h5") logger.info("Model saved to disk.") logger.info("Filename: {0}".format(filename_no_ext)) del model
model.summary() ## NO SE PUEDE USAR TENSORBOARD CON DATOS DE VALIDACION QUE SEAN GENERADORES tbCallBack = TensorBoard(log_dir='./gdrive/My Drive/TFM/notebooks/logs_lstmsimple/', update_freq='batch', batch_size=1, histogram_freq=0, write_graph=True, write_images=True, write_grads=False) checkpoints = ModelCheckpoint('./checkpoints_recurrente_fully/recurrente_weights.{epoch:02d}-{val_loss:.2f}.hdf5', monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=False, mode='auto', period=1) history = model.fit_generator(frame_generator_train(batch_size, path_imagenes, train_data), validation_data = frame_generator_validation(batch_size, path_imagenes, validation_data), validation_steps=int(len(validation_data)/batch_size), max_queue_size=1, verbose=2, use_multiprocessing=True, workers=1, epochs=10, steps_per_epoch=int(len(train_data)/batch_size), shuffle=False, callbacks=[TQDMNotebookCallback(), checkpoints])#, checkpoints, tbCallBack]) """# TESTEO""" prediction = model.predict_generator(frame_generator_test(batch_size, path_imagenes, test_data), steps = len(test_data)) mean_squared_error(ytest, prediction) alv = ytest.astype(float) alv2 = np.expand_dims(alv, 1) from scipy.stats.stats import pearsonr pearsonr(alv2, prediction) pyplot.figure(figsize=(20,10))
# Instantiate the model model = PConvUnet(vgg_weights='./data/pytorch_vgg16.h5') FOLDER = './data/logs/C2/' # Run training for certain amount of epochs model.fit_generator( train_generator, steps_per_epoch=500, validation_data=val_generator, validation_steps=50, epochs=50, verbose=0, callbacks=[ TensorBoard( log_dir=FOLDER, write_graph=True ), ModelCheckpoint( FOLDER+'weights.{epoch:02d}-{loss:.2f}.h5', monitor='val_loss', save_best_only=True, save_weights_only=True ), LambdaCallback( on_epoch_end=lambda epoch, logs: plot_callback(model) ), TQDMNotebookCallback() ] )
def train_model(dataset_directory: str, model_name: str, width: int, height: int, output_name: str = None): print("Loading configuration and data-readers...") start_time = time() number_of_classes = len( os.listdir(os.path.join(dataset_directory, "training"))) training_configuration = ConfigurationFactory.get_configuration_by_name( model_name, width, height, number_of_classes) train_generator = ImageDataGenerator( rotation_range=training_configuration.rotation_range, zoom_range=training_configuration.zoom_range) training_data_generator = DirectoryIterator( directory=os.path.join(dataset_directory, "training"), image_data_generator=train_generator, target_size=(training_configuration.input_image_rows, training_configuration.input_image_columns), batch_size=training_configuration.training_minibatch_size, ) training_steps_per_epoch = np.math.ceil(training_data_generator.samples / training_data_generator.batch_size) validation_generator = ImageDataGenerator() validation_data_generator = DirectoryIterator( directory=os.path.join(dataset_directory, "validation"), image_data_generator=validation_generator, target_size=(training_configuration.input_image_rows, training_configuration.input_image_columns), batch_size=training_configuration.training_minibatch_size) validation_steps_per_epoch = np.math.ceil( validation_data_generator.samples / validation_data_generator.batch_size) test_generator = ImageDataGenerator() test_data_generator = DirectoryIterator( directory=os.path.join(dataset_directory, "test"), image_data_generator=test_generator, target_size=(training_configuration.input_image_rows, training_configuration.input_image_columns), batch_size=training_configuration.training_minibatch_size, shuffle=False) test_steps_per_epoch = np.math.ceil(test_data_generator.samples / test_data_generator.batch_size) model = training_configuration.classifier() model.summary() print("Model {0} loaded.".format(training_configuration.name())) print(training_configuration.summary()) start_of_training = datetime.date.today() if output_name is None: best_model_path = "{0}_{1}.h5".format(start_of_training, training_configuration.name()) else: best_model_path = "{0}.h5".format(output_name) monitor_variable = 'val_acc' model_checkpoint = ModelCheckpoint(best_model_path, monitor=monitor_variable, save_best_only=True, verbose=1) early_stop = EarlyStopping( monitor=monitor_variable, patience=training_configuration.number_of_epochs_before_early_stopping, verbose=1) learning_rate_reduction = ReduceLROnPlateau( monitor=monitor_variable, patience=training_configuration. number_of_epochs_before_reducing_learning_rate, verbose=1, factor=training_configuration.learning_rate_reduction_factor, min_lr=training_configuration.minimum_learning_rate) if output_name is None: log_directory = "./logs/{0}_{1}/".format(start_of_training, training_configuration.name()) else: log_directory = "./logs/{0}/".format(output_name) tensorboard_callback = TensorBoard( log_dir=log_directory, batch_size=training_configuration.training_minibatch_size) if running_inside_jupyter_notebook(): callbacks = [ model_checkpoint, early_stop, tensorboard_callback, learning_rate_reduction, TQDMNotebookCallback() ] else: callbacks = [ model_checkpoint, early_stop, tensorboard_callback, learning_rate_reduction, TQDMCallback() ] class_weight_calculator = ClassWeightCalculator() balancing_method = "skBalance" class_weights = class_weight_calculator.calculate_class_weights( dataset_directory, method=balancing_method, class_indices=training_data_generator.class_indices) if balancing_method is not None: print( "Using {0} method for obtaining class weights to compensate for an unbalanced dataset." .format(balancing_method)) print("Training on dataset...") history = model.fit_generator( generator=training_data_generator, steps_per_epoch=training_steps_per_epoch, epochs=training_configuration.number_of_epochs, callbacks=callbacks, validation_data=validation_data_generator, validation_steps=validation_steps_per_epoch, class_weight=class_weights, verbose=0) print("Loading best model from check-point and testing...") best_model = keras.models.load_model(best_model_path) test_data_generator.reset() file_names = test_data_generator.filenames class_labels = os.listdir(os.path.join(dataset_directory, "test")) # Notice that some classes have so few elements, that they are not present in the test-set and do not # appear in the final report. To obtain the correct classes, we have to enumerate all non-empty class # directories inside the test-folder and use them as labels names_of_classes_with_test_data = [ class_name for class_name in class_labels if os.listdir(os.path.join(dataset_directory, "test", class_name)) ] true_classes = test_data_generator.classes predictions = best_model.predict_generator(test_data_generator, steps=test_steps_per_epoch) predicted_classes = numpy.argmax(predictions, axis=1) test_data_generator.reset() evaluation = best_model.evaluate_generator(test_data_generator, steps=test_steps_per_epoch) classification_accuracy = 0 print("Reporting classification statistics with micro average") report = sklearn_reporting.classification_report( true_classes, predicted_classes, digits=3, target_names=names_of_classes_with_test_data, average='micro') print(report) print("Reporting classification statistics with macro average") report = sklearn_reporting.classification_report( true_classes, predicted_classes, digits=3, target_names=names_of_classes_with_test_data, average='macro') print(report) print("Reporting classification statistics with weighted average") report = sklearn_reporting.classification_report( true_classes, predicted_classes, digits=3, target_names=names_of_classes_with_test_data, average='weighted') print(report) indices_of_misclassified_files = [ i for i, e in enumerate(true_classes - predicted_classes) if e != 0 ] misclassified_files = [ file_names[i] for i in indices_of_misclassified_files ] misclassified_files_actual_prediction_indices = [ predicted_classes[i] for i in indices_of_misclassified_files ] misclassified_files_actual_prediction_classes = [ class_labels[i] for i in misclassified_files_actual_prediction_indices ] print("Misclassified files:") for i in range(len(misclassified_files)): print("\t{0} is incorrectly classified as {1}".format( misclassified_files[i], misclassified_files_actual_prediction_classes[i])) for i in range(len(best_model.metrics_names)): current_metric = best_model.metrics_names[i] print("{0}: {1:.5f}".format(current_metric, evaluation[i])) if current_metric == 'acc' or current_metric == 'output_class_acc': classification_accuracy = evaluation[i] print("Total Accuracy: {0:0.5f}%".format(classification_accuracy * 100)) print("Total Error: {0:0.5f}%".format((1 - classification_accuracy) * 100)) end_time = time() print("Execution time: {0:.1f}s".format(end_time - start_time)) training_result_image = "{1}_{0}_{2:.1f}p.png".format( training_configuration.name(), start_of_training, classification_accuracy * 100) TrainingHistoryPlotter.plot_history(history, training_result_image)
def load_bi_gru_model(x_train, y_train, x_train_dev, y_train_dev, params): """ The callback method that will be used from the Talos API in order to generate a configurable Keras RNN Model with Bidirection GRUs. :return (tuple): A tuple with the history object of the model train and the , generated Keras model itself. """ inputs = Input((MAX_SEQUENCE_LENGTH, )) EMBEDDING_DIM = params.get('embedding_dim') GRU_SIZE = params.get('gru_size', 200) DENSE = params.get('dense', 300) N_CLASSES = y_train.shape[1] embeddings_matrix_path = os.path.join(DATA_DIR, params['embeddings_matrix_path']) with open(embeddings_matrix_path, 'rb') as embeddings_matrix_pickle: embeddings_matrix = pickle.load(embeddings_matrix_pickle) visualize_process = params.get('visualize_process', False) visualize_process = (visualize_process if isinstance( visualize_process, bool) else visualize_process == 'True') with_early_stoping = params.get('with_early_stoping', True) with_early_stoping = (with_early_stoping if isinstance( with_early_stoping, bool) else with_early_stoping == 'True') multistack_run = params.get('multistack_run', False) multistack_run = (multistack_run if isinstance(multistack_run, bool) else multistack_run == 'True') # Apply default Callback methods if with_early_stoping: stopper = EarlyStopping( monitor=params.get('early_stopping_config__monitor', 'val_f1'), min_delta=params.get('early_stopping_config__min_delta', 0), patience=params.get('early_stopping_config__patience', 5), mode=params.get('early_stopping_config__mode', 'auto'), ) default_callbacks = [stopper] else: default_callbacks = [] # Apply extra monitoring Callback methods if visualize_process: checkpoint = ModelCheckpoint(params['model_type'], monitor='val_accuracy', verbose=1, save_best_only=True, mode='max') checkpoint2 = ModelCheckpoint(params['model_type'], monitor='val_f1', verbose=1, save_best_only=True, mode='max') extra_callbacks = [checkpoint, TQDMNotebookCallback(), checkpoint2] else: extra_callbacks = [] # Add an embedding layer with 0.2 dropout probability embeddings = Embedding(MAX_WORDS + 2, EMBEDDING_DIM, weights=[embeddings_matrix], input_length=MAX_SEQUENCE_LENGTH, mask_zero=True, trainable=False)(inputs) drop_emb = Dropout(params['embeddings_dropout'])(embeddings) # add a bidirectional gru layer with variational (recurrent) dropout bi_gru = Bidirectional( GRU(GRU_SIZE, return_sequences=True, recurrent_dropout=params['var_dropout']))(drop_emb) if multistack_run: # In Case of multistack RNN deep_attention_entry = GRU( GRU_SIZE, return_sequences=True, recurrent_dropout=params['var_dropout'])(bi_gru) else: deep_attention_entry = bi_gru # add a deep self attention layer x, attn = DeepAttention(return_attention=True)(deep_attention_entry) # add a hidden MLP layer drop = Dropout(params["mlp_dropout"])(x) out = Dense(DENSE, activation=params['rnn_activation'])(x) # add the output MLP layer out = Dense(N_CLASSES, activation=params['mlp_activation'])(out) model = Model(inputs, out) if visualize_process: print(model.summary()) model.compile( loss='categorical_crossentropy', optimizer=params['optimizer'], metrics=[precision, recall, f1, accuracy, 'categorical_accuracy']) history = model.fit(x_train, y_train, batch_size=params.get('batch_size', 32), epochs=params.get('epochs', 10), verbose=0, callbacks=default_callbacks + extra_callbacks, validation_data=(x_train_dev, y_train_dev), shuffle=True) return history, model
# In[20]: modelCallback = ModelCheckpoint( 'model_checkpoint_advsmall_sos_eos_unk_att.{epoch:03d}-{loss:.3f}.hdf5', monitor='val_loss', verbose=1, save_best_only=False, save_weights_only=True, mode='auto', period=1) # creating different sets of Params to easily import into the model at train time params = { 'verbose': 1, 'callbacks': [TQDMNotebookCallback(), reduce_LR, early_stopping, modelCallback] } params2 = { 'verbose': 1, 'callbacks': [LRDrop, TQDMNotebookCallback(), reduce_LR, early_stopping] } params3 = { 'verbose': 1, 'callbacks': [LRDrop, TQDMNotebookCallback(), reduce_LR, early_stopping] } # #### Set some parameters for the model # In[13]:
from keras_tqdm import TQDMNotebookCallback from keras.optimizers import SGD, Adam, Nadam, RMSprop ### Start Here with pre-loaded images X = np.load('mdata_X.npy') y = np.load('mdata_y.npy') ### base VGG-16 model model_base = VGG16(include_top=False, weights='imagenet', input_shape=(224, 224, 3)) ### Add additional custom-designed top-layers x = model_base.layers[-1].output x = Flatten()(x) x = Dense(512, init='orthogonal', activation='relu')(x) x = Dense(4, init='orthogonal', activation='softmax')(x) ### Train, Validate, and Test the Model model2 = Model(model_base.inputs, x) model2.compile(loss='sparse_categorical_crossentropy', metrics=['accuracy'], optimizer=Adam(lr=0.0001)) model2.fit(X, y, batch_size=64, epochs=10, validation_split=0.2, verbose=0, callbacks=[TQDMNotebookCallback(leave_inner=True)]) y_pred = model2.predict(X)
# # * **ReduceLrOnPlateau** - If the model is near a minimum and the learning rate is too high, then the model will circle around that minimum without ever reaching it. This callback will allow us to reduce the learning rate when the validation loss stops improving, allowing us to reach the optimal point. # * **CsvLogger** - This lets us log the output of the model after each epoch, which will allow us to track the progress without needing to use the console. # * **ModelCheckpoint** - Generally, we will want to use the model that has the lowest loss on the validation set. This callback will save the model each time the validation loss improves. # * **EarlyStopping** - We will want to stop training when the validation loss stops improving. Otherwise, we risk overfitting. This monitor will detect when the validation loss stops improving, and will stop the training process when that occurs. # In[7]: plateau_callback = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, min_lr=0.0001, verbose=1) checkpoint_filepath = os.path.join(MODEL_OUTPUT_DIR, 'models', '{0}_model.{1}-{2}.h5'.format('model', '{epoch:02d}', '{val_loss:.7f}')) checkAndCreateDir(checkpoint_filepath) checkpoint_callback = ModelCheckpoint(checkpoint_filepath, save_best_only=True, verbose=1) csv_callback = CSVLogger(os.path.join(MODEL_OUTPUT_DIR, 'training_log.csv')) early_stopping_callback = EarlyStopping(monitor='val_loss', patience=10, verbose=1) callbacks=[plateau_callback, csv_callback, checkpoint_callback, early_stopping_callback, TQDMNotebookCallback()] # It's time to train the model! With the default setup, this model takes ~45 min to train on an NVidia GTX970 GPU. Note: sometimes the model will get stuck with a constant validation loss for up to 7 epochs. If left to run, the model should terminate with a validation loss of approximately .0003. # In[8]: history = model.fit_generator(train_generator, steps_per_epoch=num_train_examples//batch_size, epochs=500, callbacks=callbacks, validation_data=eval_generator, validation_steps=num_eval_examples//batch_size, verbose=2) # Let's do a quick sanity check. We'll load a few training images and compare the labels and the predictions. These should be very close in value if our model has learned properly. # In[9]: