def test_copy_params(self):
numpy.random.seed(0)
self.verbose = 3
self.input_ndarray = numpy.random.rand(1, 1, 10, 10)
self.input_tensor = theano.shared(self.input_ndarray)
self.output_ndarray = numpy.zeros((1,1,10,10))
self.output_tensor = theano.shared(self.output_ndarray)
self.source = [self.input_tensor]
self.dest = [self.output_tensor]
copy_params(source=self.source, destination= self.dest, borrow= True, verbose= self.verbose)
self.assertTrue(numpy.allclose(self.dest[0].eval(),self.source[0].eval()))
def _new_era(self, new_learning_rate=0.01, verbose=2):
"""
This re-initializes the learning rate to the learning rate variable. This also reinitializes
the parameters of the network to best_params.
Args:
new_learning_rate: rate at which you want fine tuning to begin.
verbose: Just as the rest of the toolbox.
"""
if verbose >= 3:
print("... setting up new era")
if self.softmax_head is True:
self.softmax_learning_rate.set_value(
numpy.asarray(new_learning_rate, dtype=theano.config.floatX))
self.generator_learning_rate.set_value(
numpy.asarray(new_learning_rate, dtype=theano.config.floatX))
self.discriminator_learning_rate.set_value(
numpy.asarray(new_learning_rate, dtype=theano.config.floatX))
# copying and removing only active_params. Is that a porblem ?
copy_params(source=self.best_params,
destination=self.active_params,
borrow=self.borrow)
def train(self, verbose, **kwargs):
"""
Training function of the network. Calling this will begin training.
Args:
epochs: ``(num_epochs for each learning rate... )`` to train Default is ``(20, 20)``
validate_after_epochs: 1, after how many epochs do you want to validate ?
show_progress: default is ``True``, will display a clean progressbar.
If ``verbose`` is ``3`` or more - False
early_terminate: ``True`` will allow early termination.
k : how many discriminator updates for every generator update.
learning_rates: (annealing_rate, learning_rates ... ) length must be one more than
``epochs`` Default is ``(0.05, 0.01, 0.001)``
save_after_epochs: 1, Save network after that many epochs of training.
pre_train_discriminator: If you want to pre-train the discriminator to make it stay
ahead of the generator for making predictions. This will only
train the softmax layer loss and not the fake or real loss.
"""
start_time = time.clock()
if verbose >= 1:
print(". Training")
if self.cooked_datastream is None:
raise Exception("Cannot train without cooking the network first")
if not 'epochs' in kwargs.keys():
epochs = (20, 20)
else:
epochs = kwargs["epochs"]
if not 'k' in kwargs.keys():
k = 1
else:
k = kwargs["k"]
if not 'validate_after_epochs' in kwargs.keys():
self.validate_after_epochs = 1
else:
self.validate_after_epochs = kwargs["validate_after_epochs"]
if not 'visualize_after_epochs' in kwargs.keys():
self.visualize_after_epochs = self.validate_after_epochs
else:
self.visualize_after_epochs = kwargs['visualize_after_epochs']
if not 'save_after_epochs' in kwargs.keys():
self.save_after_epochs = self.validate_after_epochs
else:
self.save_after_epochs = kwargs['save_after_epochs']
if not 'show_progress' in kwargs.keys():
show_progress = True
else:
show_progress = kwargs["show_progress"]
if progressbar_installed is False:
show_progress = False
if verbose == 3:
show_progress = False
if not 'training_accuracy' in kwargs.keys():
training_accuracy = False
else:
training_accuracy = kwargs["training_accuracy"]
if not 'early_terminate' in kwargs.keys():
patience = 5
else:
if kwargs["early_terminate"] is True:
patience = numpy.inf
else:
patience = 5
if not 'pre_train_discriminator' in kwargs.keys():
pre_train_discriminator = 0
else:
pre_train_discriminator = kwargs['pre_train_discriminator']
# (initial_learning_rate, fine_tuning_learning_rate, annealing)
if not 'learning_rates' in kwargs.keys():
learning_rates = (0.05, 0.01, 0.001)
else:
learning_rates = kwargs["learning_rates"]
# Just save some backup parameters
nan_insurance = []
for param in self.active_params:
nan_insurance.append(
theano.shared(param.get_value(borrow=self.borrow)))
if self.softmax_head is True:
self.softmax_learning_rate.set_value(learning_rates[1])
patience_increase = 2
improvement_threshold = 0.995
best_iteration = 0
epoch_counter = 0
early_termination = False
iteration = 0
era = 0
while (epoch_counter < pre_train_discriminator) \
and (not early_termination) \
and (self.softmax_head is True):
nan_flag = False
# This printing below and the progressbar should move to visualizer ?
if verbose >= 1:
print(".")
if verbose >= 2:
print("\n")
print(".. Pre-Training Epoch: " + str(epoch_counter))
if show_progress is True:
total_mini_batches = self.batches2train * self.mini_batches_per_batch[
0]
bar = progressbar.ProgressBar(maxval=total_mini_batches, \
widgets=[progressbar.AnimatedMarker(), \
' training ', ' ', progressbar.Percentage(), ' ',progressbar.ETA(), ]).start()
# Go through all the large batches
total_mini_batches_done = 0
for batch in xrange(self.batches2train):
if nan_flag is True:
# If NAN, restart the epoch, forget the current epoch.
break
# do multiple cached mini-batches in one loaded batch
if self.cache is True:
self._cache_data(batch=batch,
type='train',
verbose=verbose)
else:
# If dataset is not cached but need to be loaded all at once, check if trianing.
if not self.current_data_type == 'train':
# If cache is False, then there is only one batch to load.
self._cache_data(batch=0,
type='train',
verbose=verbose)
# run through all mini-batches in new batch of data that was loaded.
for minibatch in xrange(self.mini_batches_per_batch[0]):
# All important part of the training function. Batch Train.
# This is the classifier for discriminator. I will run this later.
softmax_cost = self.mini_batch_train_softmax(
minibatch, epoch_counter)
if numpy.isnan(softmax_cost):
nan_flag = True
new_lr = self.softmax_learning_rate.get_value(
borrow=self.borrow) * 0.1
self._new_era(new_learning_rate=new_lr,
verbose=verbose)
if verbose >= 2:
print(
".. NAN! Slowing learning rate by 10 times and restarting epoch."
)
break
self.softmax_cost = self.softmax_cost + [softmax_cost]
total_mini_batches_done = total_mini_batches_done + 1
if show_progress is False and verbose >= 3:
print(".. Mini batch: " + str(total_mini_batches_done))
self.print_status(epoch=epoch_counter, verbose=verbose)
if show_progress is True:
bar.update(total_mini_batches_done)
if show_progress is True:
bar.finish()
# post training items for one loop of batches.
if nan_flag is False:
if verbose >= 2:
if len(
self.softmax_cost
) < self.batches2train * self.mini_batches_per_batch[0]:
print(".. Discriminator Softmax Cost : " +
str(self.softmax_cost[-1]))
else:
print(".. Discriminator Softmax Cost : " + str(
numpy.mean(self.softmax_cost[
-1 * self.batches2train *
self.mini_batches_per_batch[0]:])))
if verbose >= 3:
print("... Learning Rate : " + str(
self.learning_rate.get_value(borrow=self.borrow)))
print("... Momentum : " +
str(self.current_momentum(epoch)))
best, better = self.validate(
epoch=epoch_counter,
training_accuracy=training_accuracy,
show_progress=show_progress,
verbose=verbose)
self.visualize(epoch=epoch_counter, verbose=verbose)
if best is True:
copy_params(source=self.active_params,
destination=nan_insurance,
borrow=self.borrow,
verbose=verbose)
copy_params(source=self.active_params,
destination=self.best_params,
borrow=self.borrow,
verbose=verbose)
self.softmax_decay_learning_rate(learning_rates[0])
if patience < epoch_counter:
early_termination = True
if final_era is False:
if verbose >= 3:
print(
"... Patience ran out lowering learning rate.")
new_lr = self.generator_learning_rate.get_value(
borrow=self.borrow) * 0.1
self._new_era(new_learning_rate=new_lr,
verbose=verbose)
early_termination = False
else:
if verbose >= 2:
print(".. Early stopping")
break
epoch_counter = epoch_counter + 1
end_time = time.clock()
if verbose >= 2 and self.softmax_head is True:
print(".. Pre- Training complete.Took " +
str((end_time - start_time) / 60) + " minutes")
# Reset training for the main loop
self.discriminator_learning_rate.set_value(learning_rates[1])
self.generator_learning_rate.set_value(learning_rates[1])
if self.softmax_head is True:
self.softmax_learning_rate.set_value(learning_rates[1])
patience_increase = 2
patience = numpy.inf
improvement_threshold = 0.995
best_iteration = 0
epoch_counter = 0
early_termination = False
iteration = 0
era = 0
if isinstance(epochs, int):
total_epochs = epochs
change_era = epochs + 1
elif len(epochs) > 1:
total_epochs = sum(epochs)
change_era = epochs[era]
else:
total_epochs = epochs
change_era = epochs + 1
final_era = False
gen_cost = 0.
# main loop
while (epoch_counter < total_epochs) and (not early_termination):
nan_flag = False
# check if its time for a new era.
if (epoch_counter == change_era):
# if final_era, while loop would have terminated.
era = era + 1
if era == len(epochs) - 1:
final_era = True
if verbose >= 3:
print("... Begin era " + str(era))
change_era = epoch_counter + epochs[era]
if self.learning_rate.get_value(
borrow=self.borrow) < learning_rates[era + 1]:
if verbose >= 2:
print(
".. Learning rate was already lower than specified. Not changing it."
)
new_lr = self.generator_learning_rate.get_value(
borrow=self.borrow)
else:
new_lr = learning_rates[era + 1]
self._new_era(new_learning_rate=new_lr, verbose=verbose)
# This printing below and the progressbar should move to visualizer ?
if verbose >= 1:
print(".")
if verbose >= 2:
print("\n")
print(".. Epoch: " + str(epoch_counter) + " Era: " +
str(era))
if show_progress is True:
total_mini_batches = self.batches2train * self.mini_batches_per_batch[
0]
bar = progressbar.ProgressBar(maxval=total_mini_batches, \
widgets=[progressbar.AnimatedMarker(), \
' training ', ' ', progressbar.Percentage(), ' ',progressbar.ETA(), ]).start()
# Go through all the large batches
total_mini_batches_done = 0
for batch in xrange(self.batches2train):
if nan_flag is True:
# If NAN, restart the epoch, forget the current epoch.
break
# do multiple cached mini-batches in one loaded batch
if self.cache is True:
self._cache_data(batch=batch,
type='train',
verbose=verbose)
else:
# If dataset is not cached but need to be loaded all at once, check if trianing.
if not self.current_data_type == 'train':
# If cache is False, then there is only one batch to load.
self._cache_data(batch=0,
type='train',
verbose=verbose)
# run through all mini-batches in new batch of data that was loaded.
for minibatch in xrange(self.mini_batches_per_batch[0]):
# All important part of the training function. Batch Train.
# This is the classifier for discriminator. I will run this later.
if self.softmax_head is True:
softmax_cost = self.mini_batch_train_softmax(
minibatch, epoch_counter)
disc_cost = self.mini_batch_train_discriminator(
minibatch, epoch_counter)
if (minibatch + 1) * (batch + 1) * (epoch_counter +
1) % k == 0:
gen_cost = self.mini_batch_train_generator(
minibatch, epoch_counter)
if numpy.isnan(gen_cost) or \
numpy.isnan(disc_cost):
nan_flag = True
new_lr = self.generator_learning_rate.get_value(
borrow=self.borrow) * 0.1
self._new_era(new_learning_rate=new_lr,
verbose=verbose)
if verbose >= 2:
print(
".. NAN! Slowing learning rate by 10 times and restarting epoch."
)
break
self.disc_cost = self.disc_cost + [disc_cost]
self.gen_cost = self.gen_cost + [gen_cost]
if self.softmax_head is True:
self.softmax_cost = self.softmax_cost + [softmax_cost]
total_mini_batches_done = total_mini_batches_done + 1
if show_progress is False and verbose >= 3:
print(".. Mini batch: " + str(total_mini_batches_done))
self.print_status(epoch=epoch_counter, verbose=verbose)
if show_progress is True:
bar.update(total_mini_batches_done)
if show_progress is True:
bar.finish()
# post training items for one loop of batches.
if nan_flag is False:
if verbose >= 2:
self.print_status(epoch=epoch_counter, verbose=verbose)
best, better = self.validate(
epoch=epoch_counter,
training_accuracy=training_accuracy,
show_progress=show_progress,
verbose=verbose)
self.save_params(epoch=epoch_counter, verbose=verbose)
self.visualize(epoch=epoch_counter, verbose=verbose)
if best is True:
copy_params(source=self.active_params,
destination=nan_insurance,
borrow=self.borrow,
verbose=verbose)
copy_params(source=self.active_params,
destination=self.best_params,
borrow=self.borrow,
verbose=verbose)
self.discriminator_decay_learning_rate(learning_rates[0])
self.generator_decay_learning_rate(learning_rates[0])
if self.softmax_head is True:
self.softmax_decay_learning_rate(learning_rates[0])
if patience < epoch_counter:
early_termination = True
if final_era is False:
if verbose >= 3:
print(
"... Patience ran out lowering learning rate.")
new_lr = self.fake_learning_rate.get_value(
borrow=self.borrow) * 0.1
self._new_era(new_learning_rate=new_lr,
verbose=verbose)
early_termination = False
else:
if verbose >= 2:
print(".. Early stopping")
break
epoch_counter = epoch_counter + 1
end_time = time.clock()
if verbose >= 2:
print(".. Training complete.Took " +
str((end_time - start_time) / 60) + " minutes")