def save_model(self, keep_all_checkpoints=False):
"""Save model. If keep_all_checkpoints is set to True, then a copy of the entire model's weight and optimizer state is preserved for each checkpoint, along with the corresponding epoch in the file name. If set to False, then only the latest model is kept on disk, saving a lot of space, but potentially losing a good model due to overtraining."""
from keras import models
model_epoch_filename = "model_epoch_{0:0>4}.hdf5".format(self.epochs_completed)
if keep_all_checkpoints:
chkpoint_model_path = os.path.join(settings.CHECKPOINTS_DIR, model_epoch_filename)
print_positive("Saving model after epoch #{} to disk:\n{}.".format(self.epochs_completed, chkpoint_model_path))
self.keras_model.save(chkpoint_model_path)
force_symlink("../checkpoints/{}".format(model_epoch_filename), self.model_path)
else:
print_positive("Overwriting latest model after epoch #{} to disk:\n{}.".format(self.epochs_completed, self.model_path))
self.keras_model.save(self.model_path)
return True
def read_checkpoint_file(self):
"""Read the 'checkpoint.json' file and update the class variables accordingly."""
checkpoint = None
if os.path.isfile(self.path_checkpoint_file):
print_positive("Found checkpoint file: {}".format(
self.path_checkpoint_file))
print_info("Verifying integrity of checkpoint file...")
try:
with open(self.path_checkpoint_file, "r") as fp:
try:
checkpoint = json.load(fp)
except ValueError as e:
handle_error(
"Failed to open checkpoint file '{0}'. ".format(
self.path_checkpoint_file) +
"It does not appear to be a valid JSON file.", e)
checkpoint = None
except IOError as e:
handle_error(
"Unable to open checkpoint file '{}' for reading.".format(
self.path_checkpoint_file), e)
### Failed to find or open checkpoint file. Set some values to 0 and exit
if checkpoint != None:
### Succesfully loaded check point file, gather the data!
print_positive(
"Successfully loaded checkpoint! Reading its data...")
self.epochs_completed = checkpoint['epochs_completed']
if checkpoint['model'] != settings.MODEL:
print_warning(
"Inconsistency detected: the checkpoint model '{0}' does not match command line argument of '{1}'."
.format(checkpoint['model'], settings.MODEL))
print_info("Discarding checkpoint and starting from scratch.")
return None
if checkpoint['exp_name'] != settings.EXP_NAME:
print_warning(
"Inconsistency detected: the checkpoint experiment name '{0}' does not match command line argument of '{1}'."
.format(checkpoint['exp_name'], settings.EXP_NAME))
print_info("Discarding checkpoint and starting from scratch.")
return None
self.wall_time = checkpoint['wall_time']
self.process_time = checkpoint['process_time']
else:
self.epochs_completed = 0
self.wall_time = 0
self.process_time = 0
return checkpoint
def load_model(self):
"""Return True if a valid model was found and correctly loaded. Return False if no model was loaded."""
from shutil import copyfile
from keras.models import load_model
chosen_model_path = None
best_model_path = None
best_model_glob = glob.glob(settings.MODELS_DIR + "/best_model_*.hdf5")
if len(best_model_glob) > 0:
best_model_path = best_model_glob[0]
chosen_model_path = best_model_path
print_positive("Loading *best* model with the lowest validation score: {}"
.format(best_model_path))
if best_model_path == None:
model_epoch_filename = "model_epoch_{0:0>4}.hdf5".format(self.epochs_completed)
most_recent_model_path = os.path.join(settings.CHECKPOINTS_DIR, model_epoch_filename)
chosen_model_path = self.model_path
if not os.path.isfile(chosen_model_path):
if not os.path.isfile(most_recent_model_path):
print_warning("Unexpected problem: cannot find the model's HDF5 file anymore at path:\n'{}'".format(most_recent_model_path))
return False
else:
chosen_model_path = most_recent_model_path
print_positive("Loading last known valid model (this includes the complete architecture, all weights, optimizer's state and so on)!")
# Check if file is readable first
try:
open(chosen_model_path, "r").close()
except Exception as e:
handle_error("Lacking permission to *open for reading* the HDF5 model located at\n{}."
.format(chosen_model_path), e)
return False
# Load the actual HDF5 model file
try:
self.keras_model = load_model(chosen_model_path)
except Exception as e:
handle_error("Unfortunately, the model did not parse as a valid HDF5 Keras model and cannot be loaded for an unkown reason. A backup of the model will be created, after which training will restart from scratch.".format(chosen_model_path), e)
try:
copyfile(chosen_model_path, "{}.backup".format(chosen_model_path))
except Exception as e:
handle_error("Looks like you're having a bad day. The copy operation failed for an unknown reason. We will exit before causing some serious damage ;). Better luck next time. Please verify your directory permissions and your default umask!.", e)
sys.exit(-3)
return False
return True
def _load_jpgs_and_captions_npy(self):
print_positive(
"Found the project datasets encoded as a 4-tensor in '.npy' format. Attempting to load..."
)
prefix = ""
if settings.MAX_TRAINING_SAMPLES != -1:
prefix = "subset_"
try:
for i, filename in enumerate([
self._images_filename, self._test_images_filename,
self._captions_ids_filename, self._captions_dict_filename
]):
path = os.path.join(settings.MSCOCO_DIR, prefix + filename)
if i == 0:
self.images = np.load(path)
if settings.MAX_TRAINING_SAMPLES != -1 and self.images.shape[
0] != settings.MAX_TRAINING_SAMPLES:
raise Exception("Incorrect number of images")
print_info("Loaded: {}".format(path))
elif i == 1:
self.test_images = np.load(path)
print_info("Loaded: {}".format(path))
elif i == 2:
self.captions_ids = np.load(path)
print_info("Loaded: {}".format(path))
elif i == 3:
self.captions_dict = np.load(path)
print_info("Loaded: {}".format(path))
log("")
print_positive("Successfully loaded entire datasets!")
self._is_dataset_loaded = True
except Exception as e:
handle_warning(
"Unable to load some of the '.npy' dataset files. Going back to loading '.jpg' files one at a time.",
e)
self._is_dataset_loaded = False
def load_model(self):
"""Return True if a valid model was found and correctly loaded. Return False if no model was loaded."""
import numpy as np
from lasagne.layers import set_all_param_values
if os.path.isfile(self.full_gen_path) and os.path.isfile(
self.full_disc_path):
print_positive(
"Found latest '.npz' model's weights files saved to disk at paths:\n{}\n{}"
.format(self.full_gen_path, self.full_disc_path))
else:
print_info(
"Cannot resume from checkpoint. Could not find '.npz' weights files, either {} or {}."
.format(self.full_gen_path, self.full_disc_path))
return False
try:
### Load the generator model's weights
print_info("Attempting to load generator model: {}".format(
self.full_gen_path))
with np.load(self.full_gen_path) as fp:
param_values = [fp['arr_%d' % i] for i in range(len(fp.files))]
set_all_param_values(self.generator, param_values)
### Load the discriminator model's weights
print_info("Attempting to load generator model: {}".format(
self.full_disc_path))
with np.load(self.full_disc_path) as fp:
param_values = [fp['arr_%d' % i] for i in range(len(fp.files))]
set_all_param_values(self.discriminator, param_values)
except Exception as e:
handle_error(
"Failed to read or parse the '.npz' weights files, either {} or {}."
.format(self.full_gen_path, self.full_disc_path), e)
return False
return True
def train(self, Dataset):
from keras.callbacks import EarlyStopping, ModelCheckpoint, LambdaCallback
### Get the datasets
X_train, X_test, Y_train, Y_test, id_train, id_val = Dataset.return_train_data()
#### Print model summary
#print_info("Model summary:")
#self.keras_model.summary()
#### Compile the model (if necessary)
self._compile()
#### Fit the model
# We fit the model iteratively, typically in more than one pass, in order to
# create frequent checkpoints. For example, if EPOCHS_PER_CHECKPOINT is set to 5
# and NUM_EPOCHS is set to 33, then the loop will iterate 6 times, performing 5
# epochs of training via the 'fit' method and on the 7th iteration, it will
# perform 3 epochs only in order to reach NUM_EPOCHS. Finally, a checkpoint is
# always created once training is complete, even if the next EPOCHS_PER_CHECKPOINT
# multiple was not reached yet.
### Print the major params again, for convenience
print_info("Starting training from epoch {0} to epoch {1} {2}, creating checkpoints every {3} epochs."
.format(self.epochs_completed + 1,
self.epochs_completed + settings.NUM_EPOCHS,
"(i.e. training an extra {0} epochs)".format(settings.NUM_EPOCHS),
settings.EPOCHS_PER_CHECKPOINT))
# Define training callbacks
early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=1, mode='min')
best_model_path = os.path.join(settings.MODELS_DIR,
"best_model_epoch.{epoch:03d}_loss.{val_loss:.4f}.hdf5")
checkpointer = ModelCheckpoint(filepath=best_model_path,
verbose=1, save_best_only=True)
epoch_complete = LambdaCallback(on_epoch_end = self.increment_epochs_completed)
# Ready to train!
print_positive("Starting to train model!...")
epoch = 0
verbose = settings.VERBOSE
if verbose == 2:
verbose = 0 # If verbose == 2, the 'epoch_complete' callback will already be printing the same
self.keras_model.fit(X_train, Y_train,
validation_split=0.1,
epochs = settings.NUM_EPOCHS,
batch_size = settings.BATCH_SIZE,
verbose = verbose,
initial_epoch = self.epochs_completed,
callbacks=[early_stopping, checkpointer, epoch_complete])
### Training complete
print_positive("Training complete!")
# Checkpoint time (save hyper parameters, model and checkpoint file)
print_positive("CHECKPOINT AT EPOCH {}. Updating 'checkpoint.json' file...".format(self.epochs_completed))
self.update_checkpoint(False)
### Evaluate the model's performance
print_info("Evaluating model...")
train_scores = self.keras_model.evaluate(X_train, Y_train, batch_size = settings.BATCH_SIZE, verbose = 0)
test_scores = self.keras_model.evaluate(X_test, Y_test, batch_size = settings.BATCH_SIZE, verbose = 0)
metric = self.keras_model.metrics_names[1]
print_positive("Training score {0: >6}: {1:.5f}".format(metric, train_scores[1]))
print_positive("Testing score {0: >6}: {1:.5f}".format(metric, train_scores[1]))
### Save the model's performance to disk
path_model_score = os.path.join(settings.PERF_DIR, "score.txt")
print_info("Saving performance to file '{}'".format(path_model_score))
with open(path_model_score, "w") as fd:
fd.write("Performance statistics\n")
fd.write("----------------------\n")
fd.write("Model = {}\n".format(settings.MODEL))
fd.write("Cumulative number of training epochs = {0}\n".format(self.epochs_completed))
fd.write("Training score (metric: {0: >6}) = {1:.5f}\n".format(metric, train_scores[1]))
fd.write("Testing score (metric: {0: >6}) = {1:.5f}\n".format(metric, train_scores[1]))
def run_experiment():
log("Welcome! This is my final project for the course:")
log(' IFT6266-H2017 (a.k.a. "Deep Learning"')
log(" Prof. Aaron Courville")
log("")
log("This program is copyrighted 2017 Philippe Paradis. All Rights Reserved."
)
log("")
log("Enjoy!")
log("")
model = None
if settings.MODEL == "conv_mlp_vgg16":
settings.MODEL = "conv_mlp"
settings.USE_VGG16_LOSS = True
# Define model's specific settings
if settings.MODEL == "test":
from keras_models import Test_Model
model = Test_Model()
elif settings.MODEL == "mlp":
from keras_models import MLP_Model
model = MLP_Model()
elif settings.MODEL == "conv_mlp":
from keras_models import Conv_MLP
model = Conv_MLP()
elif settings.MODEL == "conv_deconv":
from keras_models import Conv_Deconv
model = Conv_Deconv()
elif settings.MODEL == "lasagne_conv_deconv":
from lasagne_models import Lasagne_Conv_Deconv
model = Lasagne_Conv_Deconv(use_dropout=False)
elif settings.MODEL == "lasagne_conv_deconv_dropout":
from lasagne_models import Lasagne_Conv_Deconv
model = Lasagne_Conv_Deconv(use_dropout=True)
settings.MODEL = "lasagne_conv_deconv"
elif settings.MODEL == "dcgan":
model = models.DCGAN_Model()
elif settings.MODEL == "wgan":
from wgan import WGAN_Model
model = WGAN_Model()
elif settings.MODEL == "lsgan":
from lsgan import LSGAN_Model
model = LSGAN_Model()
elif settings.MODEL == "vgg16":
if not os.path.isfile("vgg16.pkl"):
log("Could not find VGG-16 pre-trained weights file 'vgg16.pkl'. Downloading..."
)
download_vgg16_weights()
from vgg16 import VGG16_Model
model = VGG16_Model()
else:
raise NotImplementedError()
### Check if --force flag was passed
if settings.FORCE_RUN:
stopfile = os.path.join(settings.BASE_DIR, "STOP")
if os.path.isfile(stopfile):
os.remove(stopfile)
### Check for STOP file in BASE_DIR. Who knows, this experiment could
### be a baddy which we certainly don't want to waste precious GPU time on! Oh no!
if model.check_stop_file():
print_error(
"Oh dear, it looks like a STOP file is present in this experiment's base directory, located here:\n{}\nIf you think the STOP file was added by error and you would like to pursue this experiment further, simply feel absolute free to delete this file (which is empty, anyway)."
.format(os.path.join(settings.BASE_DIR, "STOP")))
sys.exit(-2)
### Load checkpoint (if any). This will also load the hyper parameters file.
### This will also load the model's architecture, weights, optimizer states,
### that is, everything necessary to resume training.
print_info(
"Checking for a valid checkpoint. If so, load hyper parameters and all data from the last known state..."
)
checkpoint, hyperparams, resume_from_checkpoint = model.resume_last_checkpoint(
)
if resume_from_checkpoint:
print_positive(
"Found checkpoint, hyper parameters and model data all passing the integrity tests!"
"Ready to resume training!")
log("")
print_info("State of last checkpoint:")
for key in checkpoint:
log(" * {0: <20} = {1}".format(str(key), str(checkpoint[key])))
log("")
else:
print_info(
"No valid checkpoint found for this experiment. Building and training model from scratch."
)
### Build model's architecture
model.build()
### Save hyperparameters to a file
model.save_hyperparams()
if settings.NUM_EPOCHS == 0 and not settings.PERFORM_PREDICT_ONLY:
log("Okay, we specified 0 epochs, so we only created the experiment directory:\m{}\mand the hyper parameters file within that directory 'hyperparameters.json'."
.format(settings.BASE_DIR))
sys.exit(0)
###
### Make sure the dataset has been downloaded and extracted correctly on disk
###
if check_mscoco_dir(settings.MSCOCO_DIR) == False:
log("(!) The project dataset based on MSCOCO was not found in its expected location '{}' or the symlink is broken."
.format(settings.MSCOCO_DIR))
log("Attempting to download the dataset...")
rc = download_dataset()
if rc != 0:
log("(!) Failed to download the project dataset, exiting...")
sys.exit(rc)
if settings.TINY_DATASET == True:
if check_mscoco_dir(os.path.join(settings.THIS_DIR,
"mscoco_small/")) == False:
create_tiny_dataset()
settings.MSCOCO_DIR = os.path.join(settings.THIS_DIR,
"mscoco_small/")
verbosity_level = "Low"
if settings.VERBOSE == 1:
verbosity_level = "High"
elif settings.VERBOSE == 2:
verbosity_level = "Medium"
# Print info about our settings
log("============================================================")
print_info("Experiment name = %s" % settings.EXP_NAME)
log("============================================================")
log("")
print_info("Experiment settings and options:")
log(" * Model type = " + str(settings.MODEL))
log(" * Training epochs = " + str(settings.NUM_EPOCHS))
log(" * Batch size = " + str(settings.BATCH_SIZE))
log(" * Learning rate = " + str(settings.LEARNING_RATE))
log(" * Epochs per checkpoint = " + str(settings.EPOCHS_PER_CHECKPOINT))
log(" * Epochs per samples = " + str(settings.EPOCHS_PER_SAMPLES))
log(" * Feature Matching Loss = " + str(settings.FEATURE_MATCHING))
log(" * Keep model's data for every checkpoint = " +
str(settings.KEEP_ALL_CHECKPOINTS))
log(" * Verbosity = " + str(settings.VERBOSE) +
" ({})".format(verbosity_level))
log(" * Data augmentation = " + str(settings.DATASET_AUGMENTATION))
log(" * Load greyscale images = " +
str(settings.LOAD_BLACK_AND_WHITE_IMAGES))
log("")
if settings.MODEL in ["dcgan", "wgan", "lsgan"]:
print_info("GAN-specific settings:")
log(" * Type of GAN used = " + str(settings.MODEL))
log(" * Generator/critic updates per epoch = " +
str(settings.UPDATES_PER_EPOCH))
log(" * GAN learning rate = " + str(settings.GAN_LEARNING_RATE))
if settings.MODEL == "lsgan":
log(" * LSGAN architecture # = " +
str(settings.LSGAN_ARCHITECTURE))
log("")
### Print hyperparameters, as loaded from existing file or as initialized for new experiment
print_info("Hyper parameters:")
for key in model.hyper:
log(" * {0: <20} = {1}".format(str(key), str(model.hyper[key])))
log("")
#######################################
# Info about the dataset
#######################################
# The data is already split into training and validation datasets
# The training dataset has:
# - 82782 items
# - 984 MB of data
# The validation dataset has:
# - 40504 items
# - 481 MB of data
#
# There is also a pickled dictionary that maps image filenames (minutes the
# .jpg extension) to a list of 5 strings (the 5 human-generated captions).
# This dictionary is an OrderedDict with 123286 entries.
import dataset
### Create and initialize an empty InpaintingDataset object
Dataset = dataset.ColorsFirstDataset(settings.IMAGE_WIDTH,
settings.IMAGE_HEIGHT)
### Load dataset
Dataset.load_dataset()
log("")
print_info("Summary of data within dataset:")
log(" * images.shape = " + str(Dataset.images.shape))
log(" * captions_ids.shape = " + str(Dataset.captions_ids.shape))
log(" * captions_dict.shape = " + str(Dataset.captions_dict.shape))
log("")
### Train the model (computation intensive)
if settings.MODEL == "mlp" or settings.MODEL == "test" or settings.MODEL == "conv_mlp":
Dataset.preprocess()
Dataset.normalize()
Dataset.preload()
model.train(Dataset)
Dataset.denormalize()
### Produce predictions
Y_test_pred = model.predict(Dataset.get_data(X=True, Test=True),
batch_size=model.hyper['batch_size'])
### Reshape predictions to a 2d image and denormalize data
Y_test_pred = dataset.denormalize_data(Y_test_pred)
num_rows = Y_test_pred.shape[0]
Y_test_pred_2d = unflatten_to_4tensor(Y_test_pred,
num_rows,
32,
32,
is_colors_channel_first=True)
Y_test_pred_2d = transpose_colors_channel(Y_test_pred_2d,
from_first_to_last=True)
### Create dataset with colors channel last
NewDataset = dataset.ColorsLastDataset(settings.IMAGE_WIDTH,
settings.IMAGE_HEIGHT)
NewDataset.load_dataset()
NewDataset.preprocess(model="conv_deconv")
NewDataset.preload(model="conv_deconv")
### Save predictions to disk
save_keras_predictions(Y_test_pred_2d,
Dataset.id_val,
NewDataset,
num_images=50)
create_html_results_page("results.html", "assets/", num_images=50)
elif settings.MODEL == "conv_deconv":
Dataset.preprocess()
Dataset.normalize()
Dataset.preload()
model.train(Dataset)
Dataset.denormalize()
### Produce predictions
Y_test_pred_2d = model.predict(Dataset.get_data(X=True, Test=True),
batch_size=model.hyper['batch_size'])
### Reshape predictions
Y_test_pred_2d = dataset.denormalize_data(Y_test_pred_2d)
Y_test_pred_2d = transpose_colors_channel(Y_test_pred_2d,
from_first_to_last=True)
### Create dataset with colors channel last
NewDataset = dataset.ColorsLastDataset(settings.IMAGE_WIDTH,
settings.IMAGE_HEIGHT)
NewDataset.load_dataset()
NewDataset.preprocess(model="conv_deconv")
NewDataset.preload(model="conv_deconv")
### Save predictions to disk
save_keras_predictions(Y_test_pred_2d,
Dataset.id_val,
NewDataset,
num_images=50)
create_html_results_page("results.html", "assets/", num_images=50)
elif settings.MODEL == "lasagne_conv_deconv" or settings.MODEL == "vgg16":
Dataset.preprocess()
Dataset.normalize()
Dataset.preload()
model.train(Dataset)
elif settings.MODEL == "dcgan":
from lasagne.utils import floatX
Dataset.preprocess()
Dataset.normalize()
Dataset.preload()
generator, discriminator, train_fn, gen_fn = model.train(
Dataset,
num_epochs=settings.NUM_EPOCHS,
epochsize=10,
batchsize=64,
initial_eta=8e-5)
Dataset.denormalize()
settings.touch_dir(settings.SAMPLES_DIR)
for i in range(100):
samples = gen_fn(floatX(np.random.rand(10 * 10, 100)))
path = os.path.join(settings.EPOCHS_DIR, 'samples_%i.png' % i)
samples = dataset.denormalize_data(samples)
Image.fromarray(
samples.reshape(10, 10, 3, 64,
64).transpose(0, 3, 1, 4,
2).reshape(10 * 64, 10 * 64,
3)).save(path)
sample = gen_fn(floatX(np.random.rand(1, 100)))
sample = dataset.denormalize_data(sample)
path = os.path.join(settings.SAMPLES_DIR, 'one_sample_%i.png' % i)
Image.fromarray(
sample.reshape(3, 64, 64).transpose(1, 2,
0).reshape(64, 64,
3)).save(path)
elif settings.MODEL == "wgan":
import wgan
from lasagne.utils import floatX
Dataset.preprocess()
Dataset.normalize()
Dataset.preload()
generator, critic, generator_train_fn, critic_train_fn, gen_fn = wgan.train(
Dataset, num_epochs=settings.NUM_EPOCHS)
Dataset.denormalize()
settings.touch_dir(settings.SAMPLES_DIR)
for i in range(100):
samples = gen_fn(floatX(np.random.rand(10 * 10, 100)))
path = os.path.join(settings.EPOCHS_DIR, 'samples_%i.png' % i)
samples = dataset.denormalize_data(samples)
Image.fromarray(
samples.reshape(10, 10, 3, 64,
64).transpose(0, 3, 1, 4,
2).reshape(10 * 64, 10 * 64,
3)).save(path)
sample = gen_fn(floatX(np.random.rand(1, 100)))
sample = dataset.denormalize_data(sample)
path = os.path.join(settings.SAMPLES_DIR, 'one_sample_%i.png' % i)
Image.fromarray(
sample.reshape(3, 64, 64).transpose(1, 2,
0).reshape(64, 64,
3)).save(path)
elif settings.MODEL == "lsgan":
from lasagne.utils import floatX
Dataset.preprocess()
Dataset.normalize()
Dataset.preload()
generator, critic, gen_fn = model.train(
Dataset,
num_epochs=settings.NUM_EPOCHS,
epochsize=settings.UPDATES_PER_EPOCH,
batchsize=settings.BATCH_SIZE,
initial_eta=settings.GAN_LEARNING_RATE,
architecture=settings.LSGAN_ARCHITECTURE)
Dataset.denormalize()
for i in range(100):
samples = gen_fn(floatX(np.random.rand(10 * 10, 100)))
path = os.path.join(settings.SAMPLES_DIR,
"samples_{:0>3}.png".format(i))
samples = dataset.denormalize_data(samples)
Image.fromarray(
samples.reshape(10, 10, 3, 64,
64).transpose(0, 3, 1, 4,
2).reshape(10 * 64, 10 * 64,
3)).save(path)
sample = gen_fn(floatX(np.random.rand(1, 100)))
sample = dataset.denormalize_data(sample)
path = os.path.join(settings.SAMPLES_DIR,
"one_sample_{:0>3}.png".format(i))
Image.fromarray(
sample.reshape(3, 64, 64).transpose(1, 2,
0).reshape(64, 64,
3)).save(path)
### Success...? Well, at least we didn't crash :P
log("Exiting normally. That's typically a good sign :-)")
sys.exit(0)
def train(self, dataset):
log("Fetching data...")
X_train, X_val, y_train, y_val, ind_train, ind_val = dataset.return_train_data(
)
X_test, y_test = dataset.return_test_data()
#Variance of the prediction can be maximized to obtain sharper images.
#If this coefficient is set to "0", the loss is just the L2 loss.
StdevCoef = 0
# Prepare Theano variables for inputs and targets
input_var = T.tensor4('inputs')
target_var = T.tensor4('targets')
# Create neural network model
log("Building model and compiling functions...")
self.build_network(input_var, target_var)
# See if we can resume from previously saved model
print_info("Looking for existing model to resume training from...")
model_path = os.path.join(settings.MODELS_DIR,
settings.EXP_NAME + ".npz")
if self.load_model(model_path):
print_positive(
"Loaded saved model weights found on disk at: {}!".format(
model_path))
print_positive("Resuming training from exisiting weights!")
else:
print_info(
"Unable to find exisiting or load valid model file. Starting training from scratch!"
)
# Build loss function
train_loss = self.build_loss(input_var, target_var)
# Update expressions
from theano import shared
eta = shared(lasagne.utils.floatX(settings.LEARNING_RATE))
params = lasagne.layers.get_all_params(self.network_out,
trainable=True)
updates = lasagne.updates.adam(train_loss, params, learning_rate=eta)
# Train loss function
train_fn = theano.function([input_var, target_var],
train_loss,
updates=updates)
# Test/validation Loss expression (disable dropout and so on...)
test_loss = self.build_loss(input_var, target_var, deterministic=True)
# Validation loss function
val_test_fn = theano.function([input_var, target_var], test_loss)
# Predict function
predict_fn = theano.function([input_var],
lasagne.layers.get_output(
self.network_out, deterministic=True))
##########################################
# Finally, launch the training loop.
##########################################
keyboard_interrupt = False
completed_epochs = 0
try:
log("Starting training...")
batch_size = settings.BATCH_SIZE
best_val_loss = 1.0e30
best_val_loss_epoch = -1
for epoch in range(settings.NUM_EPOCHS):
start_time = time.time()
train_losses = []
for batch in self.iterate_minibatches(X_train,
y_train,
batch_size,
shuffle=True):
inputs, targets = batch
train_losses.append(train_fn(inputs, targets))
val_losses = []
for batch in self.iterate_minibatches(X_val,
y_val,
batch_size,
shuffle=False):
inputs, targets = batch
val_losses.append(val_test_fn(inputs, targets))
completed_epochs += 1
# Print the results for this epoch
mean_train_loss = np.mean(train_losses)
mean_val_loss = np.mean(val_losses)
log("Epoch {} of {} took {:.3f}s".format(
epoch + 1, settings.NUM_EPOCHS,
time.time() - start_time))
log(" - training loss: {:.6f}".format(mean_train_loss))
log(" - validation loss: {:.6f}".format(mean_val_loss))
create_checkpoint = False
STOP_FILE = False
if self.check_stop_file():
STOP_FILE = True
create_checkpoint = True
if epoch >= 8 and mean_val_loss < best_val_loss:
best_val_loss_epoch = epoch + 1
best_val_loss = mean_val_loss
create_checkpoint = True
print_positive(
"New best val loss = {:.6f}!!! Creating model checkpoint!"
.format(best_val_loss))
elif (epoch + 1) % settings.EPOCHS_PER_CHECKPOINT == 0:
create_checkpoint = True
print_info(
"Time for model checkpoint (every {} epochs)...".
format(settings.EPOCHS_PER_CHECKPOINT))
if create_checkpoint:
# Save checkpoint
model_checkpoint_filename = "model_checkpoint-val_loss.{:.6f}-epoch.{:0>3}.npz".format(
best_val_loss, epoch + 1)
model_checkpoint_path = os.path.join(
settings.CHECKPOINTS_DIR, model_checkpoint_filename)
print_info("Saving model checkpoint: {}".format(
model_checkpoint_path))
# Save model
self.save_model(model_checkpoint_path)
# Save samples for this epoch
if (epoch + 1) % settings.EPOCHS_PER_SAMPLES == 0:
num_samples = 100
num_rows = 10
num_cols = 10
samples = self.create_samples(X_val, y_val, batch_size,
num_samples, predict_fn)
samples = denormalize_data(samples)
samples_path = os.path.join(
settings.EPOCHS_DIR,
'samples_epoch_{0:0>5}.png'.format(epoch + 1))
print_info(
"Time for saving sample images (every {} epochs)... ".
format(settings.EPOCHS_PER_SAMPLES) +
"Saving {} sample images predicted validation dataset input images here: {}"
.format(num_samples, samples_path))
try:
import PIL.Image as Image
Image.fromarray(
samples.reshape(
num_rows, num_cols, 3, 32, 32).transpose(
0, 3, 1, 4,
2).reshape(num_rows * 32, num_cols * 32,
3)).save(samples_path)
except ImportError as e:
print_warning(
"Cannot import module 'PIL.Image', which is necessary for the Lasagne model to output its sample images. You should really install it!"
)
if STOP_FILE:
print_critical(
"STOP file found. Ending training here! Still producing results..."
)
break
except KeyboardInterrupt:
print_critical("Training interrupted by KeyboardInterrupt (^C)!")
print_info(
"Before shutting down, attempt to saving valid checkpoint, produce preliminary results, and so on."
)
keyboard_interrupt = True
print_info("Training complete!")
# Save model
self.save_model(
os.path.join(settings.MODELS_DIR, settings.EXP_NAME + ".npz"))
# Compute 'num_images' predictions to visualize and print the test error
num_images = 100
test_losses = []
num_iter = 0
num_predictions = min(X_test.shape[0], num_images)
indices = np.arange(X_test.shape[0])
if num_images < X_test.shape[0]:
indices = indices[0:num_images]
X_test_small = X_test[indices, :, :, :]
y_test_small = y_test[indices, :, :, :]
preds = np.zeros((num_predictions, 3, 32, 32))
for batch in self.iterate_minibatches(X_test_small,
y_test_small,
batch_size,
shuffle=False):
inputs, targets = batch
preds[num_iter * batch_size:(num_iter + 1) *
batch_size] = predict_fn(inputs)
test_losses.append(val_test_fn(inputs, targets))
num_iter += 1
log("Final results:")
log(" - test loss: {:.6f}".format(np.mean(test_losses)))
# Save predictions and create HTML page to visualize them
test_images_original = np.copy(normalize_data(dataset.test_images))
denormalize_and_save_jpg_results(preds, X_test, y_test,
test_images_original, num_images)
create_html_results_page(num_images)
# Save model's performance
path_model_score = os.path.join(settings.PERF_DIR, "score.txt")
print_info("Saving performance to file '{}'".format(path_model_score))
metric = settings.LOSS_FUNCTION
log("")
log("Performance statistics")
log("----------------------")
log(" * Model = {}".format(settings.MODEL))
log(" * Number of training epochs = {0}".format(completed_epochs))
log(" * Final training score (metric: {0: >6}) = {1:.5f}".format(
metric, np.mean(train_losses)))
log(" * Final validation score (metric: {0: >6}) = {1:.5f}".format(
metric, np.mean(val_losses)))
log(" * Best validation score (metric: {0: >6}) = {1:.5f}".format(
metric, best_val_loss))
log(" * Epoch for best validation score = {}".format(
best_val_loss_epoch))
log(" * Testing dataset (metric: {0: >6}) = {1:.5f}".format(
metric, np.mean(test_losses)))
log("")
with open(path_model_score, "w") as fd:
fd.write("Performance statistics\n")
fd.write("----------------------\n")
fd.write("Model = {}\n".format(settings.MODEL))
fd.write(
"Number of training epochs = {0}\n".format(completed_epochs))
fd.write(
"Final training score (metric: {0: >6}) = {1:.5f}\n".format(
metric, np.mean(train_losses)))
fd.write(
"Final validation score (metric: {0: >6}) = {1:.5f}\n".format(
metric, np.mean(val_losses)))
fd.write(
"Best validation score (metric: {0: >6}) = {1:.5f}\n".format(
metric, best_val_loss))
fd.write("Epoch for best validation score = {}\n".format(
best_val_loss_epoch))
fd.write(
"Testing dataset (metric: {0: >6}) = {1:.5f}\n".format(
metric, np.mean(test_losses)))
if keyboard_interrupt:
raise KeyboardInterrupt