Beispiel #1
0
def split_validation(sample_list, model, percentage=0.2, epochs=20,
                     iterations=None, evaluation_path="evaluation",
                     draw_figures=False, run_detailed_evaluation=False,
                     callbacks=[], return_output=False):
    # Calculate the number of samples in the validation set
    validation_size = int(math.ceil(float(len(sample_list) * percentage)))
    # Randomly pick samples until %-split percentage
    validation = []
    for i in range(validation_size):
        validation_sample = sample_list.pop(np.random.choice(len(sample_list)))
        validation.append(validation_sample)
    # Rename the remaining cases as training
    training = sample_list
    # Reset Neural Network model weights
    model.reset_weights()
    # Run training & validation
    history = model.evaluate(training, validation, epochs=epochs,
                             iterations=iterations, callbacks=callbacks)
    # Initialize evaluation directory
    create_directories(evaluation_path)
    # Draw plots for the training & validation
    if draw_figures:
        plot_validation(history.history, model.metrics, evaluation_path)
    # Make a detailed validation
    if run_detailed_evaluation:
        detailed_validation(validation, model, evaluation_path)
    # Return or backup the validation results
    if return_output : return history.history
    else : backup_history(history.history, evaluation_path)
Beispiel #2
0
def run_fold(fold,
             model,
             epochs=20,
             iterations=None,
             evaluation_path="evaluation",
             draw_figures=True,
             callbacks=[],
             save_models=True):
    # Load sampling fold from disk
    fold_path = os.path.join(evaluation_path, "fold_" + str(fold),
                             "sample_list.json")
    training, validation = load_disk2fold(fold_path)
    # Reset Neural Network model weights
    model.reset_weights()
    # Initialize evaluation subdirectory for current fold
    subdir = os.path.join(evaluation_path, "fold_" + str(fold))
    # Save model for each fold
    cb_model = ModelCheckpoint(os.path.join(subdir, "model.hdf5"),
                               monitor="val_loss",
                               verbose=1,
                               save_best_only=True,
                               mode="min")
    if save_models == True: cb_list = callbacks + [cb_model]
    else: cb_list = callbacks
    # Run training & validation
    history = model.evaluate(training,
                             validation,
                             epochs=epochs,
                             iterations=iterations,
                             callbacks=cb_list)
    # Backup current history dictionary
    backup_history(history.history, subdir)
    # Draw plots for the training & validation
    if draw_figures:
        plot_validation(history.history, model.metrics, subdir)
Beispiel #3
0
def cross_validation(sample_list, model, k_fold=3, epochs=20,
                     iterations=None, evaluation_path="evaluation",
                     draw_figures=True, run_detailed_evaluation=True,
                     callbacks=[], direct_output=False):
    # Initialize result cache
    if direct_output : validation_results = []
    # Randomly permute the sample list
    samples_permuted = np.random.permutation(sample_list)
    # Split sample list into folds
    folds = np.array_split(samples_permuted, k_fold)
    fold_indices = list(range(len(folds)))
    # Start cross-validation
    for i in fold_indices:
        # Reset Neural Network model weights
        model.reset_weights()
        # Subset training and validation data set
        training = np.concatenate([folds[x] for x in fold_indices if x!=i],
                                  axis=0)
        validation = folds[i]
        # Initialize evaluation subdirectory for current fold
        subdir = create_directories(evaluation_path, "fold_" + str(i))
        # Run training & validation
        history = model.evaluate(training, validation, epochs=epochs,
                                 iterations=iterations, callbacks=callbacks)
        # Backup current history dictionary
        if direct_output : validation_results.append(history.history)
        else : backup_history(history.history, subdir)
        # Draw plots for the training & validation
        if draw_figures:
            plot_validation(history.history, model.metrics, subdir)
        # Make a detailed validation of the current cv-fold
        if run_detailed_evaluation:
            detailed_validation(validation, model, subdir)
    # Return the validation results
    if direct_output : return validation_results
Beispiel #4
0
def cross_validation(sample_list,
                     model,
                     k_fold=3,
                     epochs=20,
                     iterations=None,
                     evaluation_path="evaluation",
                     draw_figures=False,
                     run_detailed_evaluation=False,
                     callbacks=[],
                     save_models=True,
                     return_output=False):
    # Initialize result cache
    if return_output: validation_results = []
    # Randomly permute the sample list
    samples_permuted = np.random.permutation(sample_list)
    # Split sample list into folds
    folds = np.array_split(samples_permuted, k_fold)
    fold_indices = list(range(len(folds)))
    # Start cross-validation
    for i in fold_indices:
        # Reset Neural Network model weights
        model.reset_weights()
        # Subset training and validation data set
        training = np.concatenate([folds[x] for x in fold_indices if x != i],
                                  axis=0)
        validation = folds[i]
        # Initialize evaluation subdirectory for current fold
        subdir = create_directories(evaluation_path, "fold_" + str(i))
        # Save model for each fold
        cb_model = ModelCheckpoint(os.path.join(subdir, "model.hdf5"),
                                   monitor="val_loss",
                                   verbose=1,
                                   save_best_only=True,
                                   mode="min")
        if save_models == True: cb_list = callbacks + [cb_model]
        else: cb_list = callbacks
        # Run training & validation
        history = model.evaluate(training,
                                 validation,
                                 epochs=epochs,
                                 iterations=iterations,
                                 callbacks=cb_list)
        # Reset Learning Rate if ReduceLROnPlateau is in callbacks
        if any(isinstance(cb, ReduceLROnPlateau) for cb in cb_list):
            model.model.compile(optimizer=Adam(lr=model.learninig_rate),
                                loss=model.loss,
                                metrics=model.metrics)
        # Backup current history dictionary
        if return_output: validation_results.append(history.history)
        else: backup_history(history.history, subdir)
        # Draw plots for the training & validation
        if draw_figures:
            plot_validation(history.history, model.metrics, subdir)
        # Make a detailed validation of the current cv-fold
        if run_detailed_evaluation:
            detailed_validation(validation, model, subdir)
    # Return the validation results
    if return_output: return validation_results