def split_validation(sample_list, model, percentage=0.2, epochs=20, iterations=None, evaluation_path="evaluation", draw_figures=False, run_detailed_evaluation=False, callbacks=[], return_output=False): # Calculate the number of samples in the validation set validation_size = int(math.ceil(float(len(sample_list) * percentage))) # Randomly pick samples until %-split percentage validation = [] for i in range(validation_size): validation_sample = sample_list.pop(np.random.choice(len(sample_list))) validation.append(validation_sample) # Rename the remaining cases as training training = sample_list # Reset Neural Network model weights model.reset_weights() # Run training & validation history = model.evaluate(training, validation, epochs=epochs, iterations=iterations, callbacks=callbacks) # Initialize evaluation directory create_directories(evaluation_path) # Draw plots for the training & validation if draw_figures: plot_validation(history.history, model.metrics, evaluation_path) # Make a detailed validation if run_detailed_evaluation: detailed_validation(validation, model, evaluation_path) # Return or backup the validation results if return_output : return history.history else : backup_history(history.history, evaluation_path)
def run_fold(fold, model, epochs=20, iterations=None, evaluation_path="evaluation", draw_figures=True, callbacks=[], save_models=True): # Load sampling fold from disk fold_path = os.path.join(evaluation_path, "fold_" + str(fold), "sample_list.json") training, validation = load_disk2fold(fold_path) # Reset Neural Network model weights model.reset_weights() # Initialize evaluation subdirectory for current fold subdir = os.path.join(evaluation_path, "fold_" + str(fold)) # Save model for each fold cb_model = ModelCheckpoint(os.path.join(subdir, "model.hdf5"), monitor="val_loss", verbose=1, save_best_only=True, mode="min") if save_models == True: cb_list = callbacks + [cb_model] else: cb_list = callbacks # Run training & validation history = model.evaluate(training, validation, epochs=epochs, iterations=iterations, callbacks=cb_list) # Backup current history dictionary backup_history(history.history, subdir) # Draw plots for the training & validation if draw_figures: plot_validation(history.history, model.metrics, subdir)
def cross_validation(sample_list, model, k_fold=3, epochs=20, iterations=None, evaluation_path="evaluation", draw_figures=True, run_detailed_evaluation=True, callbacks=[], direct_output=False): # Initialize result cache if direct_output : validation_results = [] # Randomly permute the sample list samples_permuted = np.random.permutation(sample_list) # Split sample list into folds folds = np.array_split(samples_permuted, k_fold) fold_indices = list(range(len(folds))) # Start cross-validation for i in fold_indices: # Reset Neural Network model weights model.reset_weights() # Subset training and validation data set training = np.concatenate([folds[x] for x in fold_indices if x!=i], axis=0) validation = folds[i] # Initialize evaluation subdirectory for current fold subdir = create_directories(evaluation_path, "fold_" + str(i)) # Run training & validation history = model.evaluate(training, validation, epochs=epochs, iterations=iterations, callbacks=callbacks) # Backup current history dictionary if direct_output : validation_results.append(history.history) else : backup_history(history.history, subdir) # Draw plots for the training & validation if draw_figures: plot_validation(history.history, model.metrics, subdir) # Make a detailed validation of the current cv-fold if run_detailed_evaluation: detailed_validation(validation, model, subdir) # Return the validation results if direct_output : return validation_results
def cross_validation(sample_list, model, k_fold=3, epochs=20, iterations=None, evaluation_path="evaluation", draw_figures=False, run_detailed_evaluation=False, callbacks=[], save_models=True, return_output=False): # Initialize result cache if return_output: validation_results = [] # Randomly permute the sample list samples_permuted = np.random.permutation(sample_list) # Split sample list into folds folds = np.array_split(samples_permuted, k_fold) fold_indices = list(range(len(folds))) # Start cross-validation for i in fold_indices: # Reset Neural Network model weights model.reset_weights() # Subset training and validation data set training = np.concatenate([folds[x] for x in fold_indices if x != i], axis=0) validation = folds[i] # Initialize evaluation subdirectory for current fold subdir = create_directories(evaluation_path, "fold_" + str(i)) # Save model for each fold cb_model = ModelCheckpoint(os.path.join(subdir, "model.hdf5"), monitor="val_loss", verbose=1, save_best_only=True, mode="min") if save_models == True: cb_list = callbacks + [cb_model] else: cb_list = callbacks # Run training & validation history = model.evaluate(training, validation, epochs=epochs, iterations=iterations, callbacks=cb_list) # Reset Learning Rate if ReduceLROnPlateau is in callbacks if any(isinstance(cb, ReduceLROnPlateau) for cb in cb_list): model.model.compile(optimizer=Adam(lr=model.learninig_rate), loss=model.loss, metrics=model.metrics) # Backup current history dictionary if return_output: validation_results.append(history.history) else: backup_history(history.history, subdir) # Draw plots for the training & validation if draw_figures: plot_validation(history.history, model.metrics, subdir) # Make a detailed validation of the current cv-fold if run_detailed_evaluation: detailed_validation(validation, model, subdir) # Return the validation results if return_output: return validation_results