def _trainModelPatience(self, x, y, data, labels, validationData, validationLabels,
classificationCost, maxEpochs):
training_options = self.training_options
save_best_weights = training_options.save_best_weights
miniBatchSize = training_options.miniBatchSize
nrMiniBatchesTrain = max(data.shape.eval()[0] / miniBatchSize, 1)
miniBatchValidateSize = min(validationData.shape.eval()[0], miniBatchSize * 10)
nrMiniBatchesValidate = max(validationData.shape.eval()[0] / miniBatchValidateSize, 1)
trainModel = self._makeTrainFunction(x, y, data, labels)
validateModel = self._makeValidateModelFunction(
x, y, validationData, validationLabels, classificationCost, miniBatchValidateSize)
trainNoDropout = self._makeValidateModelFunction(
x, y, data, labels, classificationCost, miniBatchSize)
epoch = 0
doneTraining = False
patience = 10 * nrMiniBatchesTrain # do at least 10 passes trough the data no matter what
patienceIncrease = 2 # Increase our patience up to patience * patienceIncrease
bestValidationError = np.inf
bestWeights = None
bestBiases = None
bestEpoch = 0
validationErrors = []
trainingErrors = []
trainingErrorNoDropout = []
try:
while (epoch < maxEpochs) and not doneTraining:
# Train the net with all data
print "epoch " + str(epoch)
momentum = training_options.momentumForEpochFunction(training_options.momentumMax, epoch)
for batchNr in xrange(nrMiniBatchesTrain):
iteration = epoch * nrMiniBatchesTrain + batchNr
trainingErrorBatch = trainModel(batchNr, momentum) / training_options.miniBatchSize
meanValidations = map(validateModel, xrange(nrMiniBatchesValidate))
meanValidationError = sum(meanValidations) / len(meanValidations)
if meanValidationError < bestValidationError:
print "increasing patience, still improving during training..."
patience = max(patience, iteration * patienceIncrease)
bestValidationError = meanValidationError
if save_best_weights:
# Save the weights which are the best ones
bestWeights = self.weights
bestBiases = self.biases
bestEpoch = epoch
validationErrors += [meanValidationError]
trainingErrors += [trainingErrorBatch]
trainingErrorNoDropout += [trainNoDropout(batchNr)]
if patience <= iteration:
doneTraining = True
epoch += 1
except KeyboardInterrupt:
print "you have interrupted training"
print "we will continue testing with the state of the network as it is"
# TODO: double check
if save_best_weights:
if bestWeights is not None and bestBiases is not None:
self.weights = bestWeights
self.biases = bestBiases
common. plotTrainingAndValidationErros(trainingErrors, validationErrors)
common.plotTrainingAndValidationErros(trainingErrorNoDropout, validationErrors)
print "number of epochs"
print epoch
def _trainLoopWithValidation(self, x, y, data, labels, validationData, validationLabels,
classificationCost, maxEpochs):
lastValidationError = np.inf
consecutive_decrease_error_count = 0.0
epoch = 0
training_options = self.training_options
save_best_weights = training_options.save_best_weights
miniBatchSize = training_options.miniBatchSize
nrMiniBatchesTrain = max(data.shape.eval()[0] / miniBatchSize, 1)
miniBatchValidateSize = min(validationData.shape.eval()[0], miniBatchSize * 10)
nrMiniBatchesValidate = max(validationData.shape.eval()[0] / miniBatchValidateSize, 1)
trainModel = self._makeTrainFunction(x, y, data, labels)
validateModel = self._makeValidateModelFunction(
x, y, validationData, validationLabels, classificationCost, miniBatchValidateSize)
trainNoDropout = self._makeValidateModelFunction(
x, y, data, labels, classificationCost, miniBatchSize)
validationErrors = []
trainingErrors = []
trainingErrorsNoDropout = []
bestValidationError = np.inf
bestWeights = None
bestBiases = None
bestEpoch = 0
try:
while epoch < maxEpochs and consecutive_decrease_error_count < 8:
print "epoch " + str(epoch)
momentum = self.training_options.momentumForEpochFunction(training_options.momentumMax, epoch)
sumErrors = 0.0
sumErrorsNoDropout = 0.0
for batchNr in xrange(nrMiniBatchesTrain):
sumErrors += trainModel(batchNr, momentum) / miniBatchSize
sumErrorsNoDropout += trainNoDropout(batchNr) / miniBatchSize
trainingErrors += [sumErrors / nrMiniBatchesTrain]
trainingErrorsNoDropout += [sumErrorsNoDropout / nrMiniBatchesTrain]
meanValidations = map(validateModel, xrange(nrMiniBatchesValidate))
meanValidationError = sum(meanValidations) / len(meanValidations)
validationErrors += [meanValidationError]
if save_best_weights:
if meanValidationError < bestValidationError:
bestValidationError = meanValidationError
# Save the weights which are the best ones
bestWeights = self.weights
bestBiases = self.biases
bestEpoch = epoch
consecutive_decrease_error_count = consecutive_decrease_error_count + 1 if meanValidationError > lastValidationError else 0
lastValidationError = meanValidationError
epoch += 1
except KeyboardInterrupt:
print "you have interrupted training"
print "we will continue testing with the state of the network as it is"
# TODO: flag for plotting
common.plotTrainingAndValidationErros(trainingErrors, validationErrors)
common.plotTrainingAndValidationErros(trainingErrorsNoDropout, validationErrors)
print "number of epochs"
print epoch + 1