def _compute_over_subset(self, subset, inputs, targets,
monitors_dict, monitor_function, monitors_outservice_dict,
plot):
inputs = raise_to_list(inputs)
targets = raise_to_list(targets)
if inputs is not None and len(monitors_dict) > 0:
monitors = {key: [] for key in monitors_dict.keys()}
data = [minibatch(input, self.batch_size, self.min_batch_size) for input in inputs]
if targets is not None and not self.unsupervised:
data += [minibatch(target, self.batch_size, self.min_batch_size) for target in targets]
for batch in min_normalized_izip(*data):
_outs = raise_to_list(monitor_function(*batch))
current_monitors = zip(monitors_dict.keys(), _outs)
for name, val in current_monitors:
val = numpy.asarray(val)
monitors[name].append(val)
# get the mean values for the batches
current_mean_monitors = {key: numpy.mean(vals, 0) for key, vals in monitors.items()}
# log the mean values!
log.info('%s monitors: %s', subset, str(current_mean_monitors))
# send the values to their outservices
for name, service in monitors_outservice_dict.items():
if name in current_mean_monitors and service:
service.write(current_mean_monitors[name], subset)
# if there is a plot, also send them over!
if plot:
plot.update_plots(epoch=self.epoch_counter, monitors=current_mean_monitors)
def _compute_over_subset(self, subset, inputs, targets,
monitors_dict, monitor_function, monitors_outservice_dict,
plot):
inputs = raise_to_list(inputs)
targets = raise_to_list(targets)
if inputs is not None and len(monitors_dict) > 0:
monitors = {key: [] for key in monitors_dict.keys()}
data = [minibatch(input, self.batch_size, self.min_batch_size) for input in inputs]
if targets is not None and not self.unsupervised:
data += [minibatch(target, self.batch_size, self.min_batch_size) for target in targets]
for batch in min_normalized_izip(*data):
_outs = raise_to_list(monitor_function(*batch))
current_monitors = zip(monitors_dict.keys(), _outs)
for name, val in current_monitors:
val = numpy.asarray(val)
monitors[name].append(val)
# get the mean values for the batches
current_mean_monitors = {key: numpy.mean(vals, 0) for key, vals in monitors.items()}
# log the mean values!
log.info('%s monitors: %s', subset, str(current_mean_monitors))
# send the values to their outservices
for name, service in monitors_outservice_dict.items():
if name in current_mean_monitors and service:
service.write(current_mean_monitors[name], subset)
# if there is a plot, also send them over!
if plot:
plot.update_plots(epoch=self.epoch_counter, monitors=current_mean_monitors)
def _perform_one_epoch(self, f_learn, plot=None):
"""
Performs a single training iteration with the given learn function.
"""
self.epoch_counter += 1
t = time.time()
log.info('EPOCH %s', str(self.epoch_counter))
# set the noise switches on for training function! (this is where things like dropout happen)
if not self.model.switches_on:
self.model.turn_on_switches()
#########
# train #
#########
train_costs = []
train_monitors = {key: [] for key in self.train_monitors_dict.keys()}
train_data = [
minibatch(input_data, self.batch_size, self.min_batch_size)
for input_data in raise_to_list(self.dataset.train_inputs)
]
if self.dataset.train_targets is not None and not self.unsupervised:
train_data += [
minibatch(target, self.batch_size, self.min_batch_size)
for target in raise_to_list(self.dataset.train_targets)
]
for batch in min_normalized_izip(*train_data):
_outs = raise_to_list(f_learn(*batch))
train_costs.append(_outs[0])
# handle any user defined monitors
if len(train_monitors) > 0:
current_monitors = zip(self.train_monitors_dict.keys(), _outs[1:])
for name, val in current_monitors:
val = numpy.asarray(val)
train_monitors[name].append(val)
# get the mean values for the batches
mean_train = numpy.mean(train_costs, 0)
current_mean_monitors = {key: numpy.mean(vals, 0) for key, vals in train_monitors.items()}
# log the mean values!
log.info('Train cost: %s', trunc(mean_train))
if len(current_mean_monitors) > 0:
log.info('Train monitors: %s', str(current_mean_monitors))
# send the values to their outservices
if self.train_outservice:
self.train_outservice.write(mean_train, "train")
for name, service in self.train_monitors_outservice_dict.items():
if name in current_mean_monitors and service:
service.write(current_mean_monitors[name], "train")
# if there is a plot, also send them over!
if plot:
current_mean_monitors.update({TRAIN_COST_KEY: mean_train})
plot.update_plots(epoch=self.epoch_counter, monitors=current_mean_monitors)
# set the noise switches off for valid and test sets! we assume unseen data is noisy anyway :)
if self.model.switches_on:
self.model.turn_off_switches()
#########
# valid #
#########
self._compute_over_subset("valid", self.dataset.valid_inputs, self.dataset.valid_targets,
self.valid_monitors_dict, self.valid_monitor_function,
self.valid_monitors_outservice_dict, plot)
########
# test #
########
self._compute_over_subset("test", self.dataset.test_inputs, self.dataset.test_targets,
self.test_monitors_dict, self.test_monitor_function,
self.test_monitors_outservice_dict, plot)
###########
# cleanup #
###########
# check for early stopping on train costs
cost = numpy.sum(train_costs)
# if the cost improved, reset the patience and record the best cost.
if cost < self.best_cost * self.early_stop_threshold:
self.patience = 0
self.best_cost = cost
# save the parameters that made it the best
self.best_params = {key: param.get_value(borrow=False) for key, param in self.params.items()}
elif not numpy.isnan(cost):
self.patience += 1
# check for stopping either from n_epochs or from threshold/patience
stop = False
if self.epoch_counter >= self.n_epoch:
log.info("Stopping (reached max number of epochs)...")
stop = True
if self.patience >= self.early_stop_length:
log.info("Stopping early (reached stop threshold)...")
stop = True
timing = time.time() - t
self.times.append(timing)
log.info('time: ' + make_time_units_string(timing))
log.debug('remaining time: ' +
make_time_units_string((self.n_epoch - self.epoch_counter) * numpy.mean(self.times)))
if (self.epoch_counter % self.save_frequency) == 0:
#save params
self.model.save_params('trained_epoch_' + str(self.epoch_counter))
# ANNEAL!
if not stop:
# perform the appropriate decay on the decay functions/parameters for this optimizer and model
for decay_param in self.get_decay_params():
decay_param.decay()
# return whether or not to stop this epoch
return stop
def _perform_one_epoch(self, f_learn, plot=None):
"""
Performs a single training iteration with the given learn function.
"""
self.epoch_counter += 1
t = time.time()
log.info('EPOCH %s', str(self.epoch_counter))
# set the noise switches on for training function! (this is where things like dropout happen)
if not self.model.switches_on:
self.model.turn_on_switches()
#########
# train #
#########
train_costs = []
train_monitors = {key: [] for key in self.train_monitors_dict.keys()}
train_data = [
minibatch(input_data, self.batch_size, self.min_batch_size)
for input_data in raise_to_list(self.dataset.train_inputs)
]
if self.dataset.train_targets is not None and not self.unsupervised:
train_data += [
minibatch(target, self.batch_size, self.min_batch_size)
for target in raise_to_list(self.dataset.train_targets)
]
for batch in min_normalized_izip(*train_data):
_outs = raise_to_list(f_learn(*batch))
train_costs.append(_outs[0])
# handle any user defined monitors (if different from the train cost)
if len(train_monitors) > 0:
current_monitors = zip(self.train_monitors_dict.keys(), _outs[1:])
for name, val in current_monitors:
val = numpy.asarray(val)
train_monitors[name].append(val)
# get the mean values for the batches
mean_train = numpy.mean(train_costs, 0)
current_mean_monitors = {key: numpy.mean(vals, 0) for key, vals in train_monitors.items()}
# log the mean values!
log.info('Train cost: %s', trunc(mean_train))
if len(current_mean_monitors) > 0:
log.info('Train monitors: %s', str(current_mean_monitors))
# send the values to their outservices
for name, service in self.train_monitors_outservice_dict.items():
if name in current_mean_monitors and service:
service.write(current_mean_monitors[name], "train")
# if there is a plot, also send them over!
if plot:
plot.update_plots(epoch=self.epoch_counter, monitors=current_mean_monitors)
# set the noise switches off for valid and test sets! we assume unseen data is noisy anyway :)
if self.model.switches_on:
self.model.turn_off_switches()
#########
# valid #
#########
self._compute_over_subset("valid", self.dataset.valid_inputs, self.dataset.valid_targets,
self.valid_monitors_dict, self.valid_monitor_function,
self.valid_monitors_outservice_dict, plot)
########
# test #
########
self._compute_over_subset("test", self.dataset.test_inputs, self.dataset.test_targets,
self.test_monitors_dict, self.test_monitor_function,
self.test_monitors_outservice_dict, plot)
###########
# cleanup #
###########
# check for early stopping on train costs
cost = numpy.sum(train_costs)
# if the cost improved, reset the patience and record the best cost.
if cost < self.best_cost * self.early_stop_threshold:
self.patience = 0
self.best_cost = cost
# save the parameters that made it the best
self.best_params = self.model.get_param_values(borrow=False)
elif not numpy.isnan(cost):
self.patience += 1
# check for stopping either from n_epochs or from threshold/patience
stop = False
if self.epoch_counter >= self.n_epoch:
log.info("Stopping (reached max number of epochs)...")
stop = True
if self.patience >= self.early_stop_length:
log.info("Stopping early (reached stop threshold)...")
stop = True
timing = time.time() - t
self.times.append(timing)
log.info('time: ' + make_time_units_string(timing))
log.debug('remaining time: ' +
make_time_units_string((self.n_epoch - self.epoch_counter) * numpy.mean(self.times)))
if (self.epoch_counter % self.save_frequency) == 0:
#save params
self.model.save_params('trained_epoch_' + str(self.epoch_counter))
# ANNEAL!
if not stop:
# perform the appropriate decay on the decay functions/parameters for this optimizer and model
for decay_param in self.get_decay_params():
decay_param.decay()
# return whether or not to stop this epoch
return stop