def split_validation(sample_list, model, percentage=0.2, epochs=20,
iterations=None, evaluation_path="evaluation",
draw_figures=False, run_detailed_evaluation=False,
callbacks=[], return_output=False):
# Calculate the number of samples in the validation set
validation_size = int(math.ceil(float(len(sample_list) * percentage)))
# Randomly pick samples until %-split percentage
validation = []
for i in range(validation_size):
validation_sample = sample_list.pop(np.random.choice(len(sample_list)))
validation.append(validation_sample)
# Rename the remaining cases as training
training = sample_list
# Reset Neural Network model weights
model.reset_weights()
# Run training & validation
history = model.evaluate(training, validation, epochs=epochs,
iterations=iterations, callbacks=callbacks)
# Initialize evaluation directory
create_directories(evaluation_path)
# Draw plots for the training & validation
if draw_figures:
plot_validation(history.history, model.metrics, evaluation_path)
# Make a detailed validation
if run_detailed_evaluation:
detailed_validation(validation, model, evaluation_path)
# Return or backup the validation results
if return_output : return history.history
else : backup_history(history.history, evaluation_path)
def cross_validation(sample_list, model, k_fold=3, epochs=20,
iterations=None, evaluation_path="evaluation",
draw_figures=True, run_detailed_evaluation=True,
callbacks=[], direct_output=False):
# Initialize result cache
if direct_output : validation_results = []
# Randomly permute the sample list
samples_permuted = np.random.permutation(sample_list)
# Split sample list into folds
folds = np.array_split(samples_permuted, k_fold)
fold_indices = list(range(len(folds)))
# Start cross-validation
for i in fold_indices:
# Reset Neural Network model weights
model.reset_weights()
# Subset training and validation data set
training = np.concatenate([folds[x] for x in fold_indices if x!=i],
axis=0)
validation = folds[i]
# Initialize evaluation subdirectory for current fold
subdir = create_directories(evaluation_path, "fold_" + str(i))
# Run training & validation
history = model.evaluate(training, validation, epochs=epochs,
iterations=iterations, callbacks=callbacks)
# Backup current history dictionary
if direct_output : validation_results.append(history.history)
else : backup_history(history.history, subdir)
# Draw plots for the training & validation
if draw_figures:
plot_validation(history.history, model.metrics, subdir)
# Make a detailed validation of the current cv-fold
if run_detailed_evaluation:
detailed_validation(validation, model, subdir)
# Return the validation results
if direct_output : return validation_results
def cross_validation(sample_list,
model,
k_fold=3,
epochs=20,
iterations=None,
evaluation_path="evaluation",
draw_figures=False,
run_detailed_evaluation=False,
callbacks=[],
save_models=True,
return_output=False):
# Initialize result cache
if return_output: validation_results = []
# Randomly permute the sample list
samples_permuted = np.random.permutation(sample_list)
# Split sample list into folds
folds = np.array_split(samples_permuted, k_fold)
fold_indices = list(range(len(folds)))
# Start cross-validation
for i in fold_indices:
# Reset Neural Network model weights
model.reset_weights()
# Subset training and validation data set
training = np.concatenate([folds[x] for x in fold_indices if x != i],
axis=0)
validation = folds[i]
# Initialize evaluation subdirectory for current fold
subdir = create_directories(evaluation_path, "fold_" + str(i))
# Save model for each fold
cb_model = ModelCheckpoint(os.path.join(subdir, "model.hdf5"),
monitor="val_loss",
verbose=1,
save_best_only=True,
mode="min")
if save_models == True: cb_list = callbacks + [cb_model]
else: cb_list = callbacks
# Run training & validation
history = model.evaluate(training,
validation,
epochs=epochs,
iterations=iterations,
callbacks=cb_list)
# Reset Learning Rate if ReduceLROnPlateau is in callbacks
if any(isinstance(cb, ReduceLROnPlateau) for cb in cb_list):
model.model.compile(optimizer=Adam(lr=model.learninig_rate),
loss=model.loss,
metrics=model.metrics)
# Backup current history dictionary
if return_output: validation_results.append(history.history)
else: backup_history(history.history, subdir)
# Draw plots for the training & validation
if draw_figures:
plot_validation(history.history, model.metrics, subdir)
# Make a detailed validation of the current cv-fold
if run_detailed_evaluation:
detailed_validation(validation, model, subdir)
# Return the validation results
if return_output: return validation_results
def leave_one_out(sample_list,
model,
epochs=20,
iterations=None,
callbacks=[],
evaluation_path="evaluation"):
# Choose a random sample
loo = sample_list.pop(np.random.choice(len(sample_list)))
# Reset Neural Network model weights
model.reset_weights()
# Train the model with the remaining samples
model.train(sample_list,
epochs=epochs,
iterations=iterations,
callbacks=callbacks)
# Initialize evaluation directory
create_directories(evaluation_path)
# Make a detailed validation on the LOO sample
detailed_validation([loo], model, evaluation_path)