def get_param_values(self, borrow=True):
"""
This returns a list of the parameter values for the model.
This method is useful when you want to save the model parameters, or are doing distributed programming
and want to train parallel models.
------------------
:param borrow: theano 'borrow' parameter for get_value() method on shared variables
:type borrow: Boolean
:return: list of theano/numpy arrays of values for the model parameters
:rtype: List(array)
"""
# try to use theano's get_value() on each parameter returned by get_params()
try:
params = get_shared_values(self.get_params(), borrow=borrow)
except NotImplementedError:
log.exception(
"%s cannot get parameters, is missing get_params() method!",
str(type(self)))
raise
except AttributeError as e:
log.exception(
"%s cannot get parameters, there was an AttributeError %s "
"when going through the get_params()", str(type(self)), str(e))
raise
return params
def get_param_values(self, borrow=True):
"""
This returns a list of the parameter values for the model.
This method is useful when you want to save the model parameters, or are doing distributed programming
and want to train parallel models.
------------------
:param borrow: theano 'borrow' parameter for get_value() method on shared variables
:type borrow: Boolean
:return: list of theano/numpy arrays of values for the model parameters
:rtype: List(array)
"""
# try to use theano's get_value() on each parameter returned by get_params()
try:
params = get_shared_values(self.get_params(), borrow=borrow)
except NotImplementedError:
log.exception("%s cannot get parameters, is missing get_params() method!", str(type(self)))
raise
except AttributeError as e:
log.exception("%s cannot get parameters, there was an AttributeError %s "
"when going through the get_params()",
str(type(self)), str(e))
raise
return params
def get_param_values(self, borrow=True):
"""
This returns a list of the parameter values for the model.
This method is useful when you want to save the model parameters, or are doing distributed programming
and want to train parallel models.
Parameters
----------
borrow : bool, optional
Theano 'borrow' parameter for get_value() method on shared variables. Defaults to True.
Returns
-------
list(array_like)
List of theano/numpy arrays of values for the model parameters.
Raises
------
NotImplementedError
If `get_params()` hasn't been implemented.
AttributeError
If a parameter isn't a SharedVariable.
"""
# try to use theano's get_value() on each parameter returned by get_params()
try:
params = get_shared_values(self.get_params(), borrow=borrow)
except NotImplementedError:
log.exception(
"%s cannot get parameters, is missing get_params() method!",
str(type(self)))
raise
except AttributeError as e:
log.exception(
"%s cannot get parameters, there was an AttributeError %s "
"when going through the get_params()", str(type(self)), str(e))
raise
return params
def get_param_values(self, borrow=True):
"""
This returns a list of the parameter values for the model.
This method is useful when you want to save the model parameters, or are doing distributed programming
and want to train parallel models.
Parameters
----------
borrow : bool, optional
Theano 'borrow' parameter for get_value() method on shared variables. Defaults to True.
Returns
-------
list(array_like)
List of theano/numpy arrays of values for the model parameters.
Raises
------
NotImplementedError
If `get_params()` hasn't been implemented.
AttributeError
If a parameter isn't a SharedVariable.
"""
# try to use theano's get_value() on each parameter returned by get_params()
try:
params = get_shared_values(self.get_params(), borrow=borrow)
except NotImplementedError:
log.exception("%s cannot get parameters, is missing get_params() method!", str(type(self)))
raise
except AttributeError as e:
log.exception("%s cannot get parameters, there was an AttributeError %s "
"when going through the get_params()",
str(type(self)), str(e))
raise
return params
def _perform_one_epoch(self, f_learn):
self.epoch_counter += 1
t = time.time()
log.info('EPOCH %s', str(self.epoch_counter))
# set the noise switches on for training function! (this is where things like dropout happen)
if len(self.noise_switches) > 0:
log.debug("Turning on %s noise switches", str(len(self.noise_switches)))
switch_vals = [switch.get_value() for switch in self.noise_switches]
[switch.set_value(0.) for switch in self.noise_switches]
# train
train_costs = []
train_monitors = {key: [] for key in self.monitors.keys()}
for batch_start, batch_end in self.train_batches:
train_costs.append(f_learn(batch_start, batch_end))
self.call_monitors(monitor_function=self.train_monitor_function,
monitors_dict=train_monitors,
inputs=[batch_start, batch_end])
log.info('Train cost: %s', trunc(numpy.mean(train_costs, 0)))
if len(self.monitors.keys()) > 0:
log.info('Train monitors: %s',
str({key: numpy.mean(value, 0) for key, value in train_monitors.items()}))
# set the noise switches off for valid and test sets! we assume unseen data is noisy anyway :)
if len(self.noise_switches) > 0:
log.debug("Turning off %s noise switches", str(len(self.noise_switches)))
[switch.set_value(0.) for switch in self.noise_switches]
# valid
if self.dataset.hasSubset(VALID) and len(self.monitors.keys()) > 0:
valid_monitors = {key: [] for key in self.monitors.keys()}
for batch_start, batch_end in self.valid_batches:
self.call_monitors(monitor_function=self.valid_monitor_function,
monitors_dict=valid_monitors,
inputs=[batch_start, batch_end])
log.info('Valid monitors: %s',
str({key: numpy.mean(value, 0) for key, value in valid_monitors.items()}))
#test
if self.dataset.hasSubset(TEST) and len(self.monitors.keys()) > 0:
test_monitors = {key: [] for key in self.monitors.keys()}
for batch_start, batch_end in self.test_batches:
self.call_monitors(monitor_function=self.test_monitor_function,
monitors_dict=test_monitors,
inputs=[batch_start, batch_end])
log.info('Test monitors: %s', str({key: numpy.mean(value, 0) for key, value in test_monitors.items()}))
# check for early stopping on train costs
cost = numpy.sum(train_costs)
if cost < self.best_cost * self.early_stop_threshold:
self.patience = 0
self.best_cost = cost
# save the parameters that made it the best
self.best_params = get_shared_values(self.params)
else:
self.patience += 1
# check for stopping either from n_epochs or from threshold/patience
stop = False
if self.epoch_counter >= self.n_epoch:
log.info("Stopping (reached max number of epochs)...")
stop = True
if self.patience >= self.early_stop_length:
log.info("Stopping early (reached stop threshold)...")
stop = True
timing = time.time() - t
self.times.append(timing)
log.info('time: ' + make_time_units_string(timing))
log.info('remaining time: ' +
make_time_units_string((self.n_epoch - self.epoch_counter) * numpy.mean(self.times)))
if (self.epoch_counter % self.save_frequency) == 0:
#save params
self.model.save_params('trained_epoch_' + str(self.epoch_counter) + '.pkl')
# ANNEAL!
if not stop:
if hasattr(self, 'learning_rate_decay') and self.learning_rate_decay:
self.learning_rate_decay.decay()
if hasattr(self, 'momentum_decay') and self.momentum_decay:
self.momentum_decay.decay()
for decay_param in self.model.get_decay_params():
decay_param.decay()
# reset the switches
if len(self.noise_switches) > 0:
[switch.set_value(val) for switch, val in zip(self.noise_switches, switch_vals)]
# return whether or not to stop this epoch
return stop
def _perform_one_epoch(self, f_learn, plot=None):
"""
Performs a single training iteration with the given learn function.
"""
self.epoch_counter += 1
t = time.time()
log.info('EPOCH %s', str(self.epoch_counter))
# set the noise switches on for training function! (this is where things like dropout happen)
switch_vals = []
if len(self.noise_switches) > 0 and (self.valid_flag or self.test_flag or self.epoch_counter == 1):
log.debug("Turning on %s noise switches", str(len(self.noise_switches)))
switch_vals = [switch.get_value() for switch in self.noise_switches]
[switch.set_value(1.) for switch in self.noise_switches]
#########
# train #
#########
train_costs = []
train_monitors = {key: [] for key in self.train_monitors_dict.keys()}
train_data = [
minibatch(input, self.batch_size, self.min_batch_size)
for input in raise_to_list(self.dataset.train_inputs)
]
if self.dataset.train_targets is not None and not self.unsupervised:
train_data += [
minibatch(target, self.batch_size, self.min_batch_size)
for target in raise_to_list(self.dataset.train_targets)
]
for batch in min_normalized_izip(*train_data):
_outs = raise_to_list(f_learn(*batch))
train_costs.append(_outs[0])
# handle any user defined monitors
if len(train_monitors) > 0:
current_monitors = zip(self.train_monitors_dict.keys(), _outs[1:])
for name, val in current_monitors:
val = numpy.asarray(val)
train_monitors[name].append(val)
# get the mean values for the batches
mean_train = numpy.mean(train_costs, 0)
current_mean_monitors = {key: numpy.mean(vals, 0) for key, vals in train_monitors.items()}
# log the mean values!
log.info('Train cost: %s', trunc(mean_train))
if len(current_mean_monitors) > 0:
log.info('Train monitors: %s', str(current_mean_monitors))
# send the values to their outservices
if self.train_outservice:
self.train_outservice.write(mean_train, "train")
for name, service in self.train_monitors_outservice_dict.items():
if name in current_mean_monitors and service:
service.write(current_mean_monitors[name], "train")
# if there is a plot, also send them over!
if plot:
current_mean_monitors.update({TRAIN_COST_KEY: mean_train})
plot.update_plots(epoch=self.epoch_counter, monitors=current_mean_monitors)
# set the noise switches off for valid and test sets! we assume unseen data is noisy anyway :)
if len(self.noise_switches) > 0 and (self.valid_flag or self.test_flag):
log.debug("Turning off %s noise switches", str(len(self.noise_switches)))
[switch.set_value(0.) for switch in self.noise_switches]
#########
# valid #
#########
self._compute_over_subset("valid", self.dataset.valid_inputs, self.dataset.valid_targets,
self.valid_monitors_dict, self.valid_monitor_function,
self.valid_monitors_outservice_dict, plot)
########
# test #
########
self._compute_over_subset("test", self.dataset.test_inputs, self.dataset.test_targets,
self.test_monitors_dict, self.test_monitor_function,
self.test_monitors_outservice_dict, plot)
###########
# cleanup #
###########
# check for early stopping on train costs
cost = numpy.sum(train_costs)
# if the cost improved, reset the patience and record the best cost.
if cost < self.best_cost * self.early_stop_threshold:
self.patience = 0
self.best_cost = cost
# save the parameters that made it the best
self.best_params = get_shared_values(self.params)
elif not numpy.isnan(cost):
self.patience += 1
# check for stopping either from n_epochs or from threshold/patience
stop = False
if self.epoch_counter >= self.n_epoch:
log.info("Stopping (reached max number of epochs)...")
stop = True
if self.patience >= self.early_stop_length:
log.info("Stopping early (reached stop threshold)...")
stop = True
timing = time.time() - t
self.times.append(timing)
log.info('time: ' + make_time_units_string(timing))
log.debug('remaining time: ' +
make_time_units_string((self.n_epoch - self.epoch_counter) * numpy.mean(self.times)))
if (self.epoch_counter % self.save_frequency) == 0:
#save params
self.model.save_params('trained_epoch_' + str(self.epoch_counter))
# ANNEAL!
if not stop:
# perform the appropriate decay on the decay functions/parameters for this optimizer and model
for decay_param in self.get_decay_params():
decay_param.decay()
# reset the switches
if len(self.noise_switches) > 0:
[switch.set_value(val) for switch, val in zip(self.noise_switches, switch_vals)]
# return whether or not to stop this epoch
return stop
def _perform_one_epoch(self):
self.epoch_counter += 1
t = time.time()
log.info('EPOCH %s', str(self.epoch_counter))
#train
train_costs = []
train_monitors = {key: [] for key in self.monitors.keys()}
for x, y in self.iterator(self.dataset, datasets.TRAIN,
self.batch_size, self.minimum_batch_size,
self.rng):
if self.unsupervised:
train_costs.append(self.f_learn(x))
for key in self.monitors.keys():
monitor_function = self.monitors[key]
train_monitors[key].append(monitor_function(x))
else:
train_costs.append(self.f_learn(x, y))
for key in self.monitors.keys():
monitor_function = self.monitors[key]
train_monitors[key].append(monitor_function(x, y))
log.info('Train cost: %s', trunc(numpy.mean(train_costs, 0)))
if len(self.monitors.keys()) > 0:
log.info(
'Train monitors: %s',
str({
key: numpy.mean(value, 0)
for key, value in train_monitors.items()
}))
#valid
if self.dataset.hasSubset(
datasets.VALID) and len(self.monitors.keys()) > 0:
valid_monitors = {key: [] for key in self.monitors.keys()}
for x, y in self.iterator(self.dataset, datasets.VALID,
self.batch_size, self.minimum_batch_size,
self.rng):
if self.unsupervised:
for key in self.monitors.keys():
monitor_function = self.monitors[key]
valid_monitors[key].append(monitor_function(x))
else:
for key in self.monitors.keys():
monitor_function = self.monitors[key]
valid_monitors[key].append(monitor_function(x, y))
log.info(
'Valid monitors: %s',
str({
key: numpy.mean(value, 0)
for key, value in valid_monitors.items()
}))
#test
if self.dataset.hasSubset(
datasets.TEST) and len(self.monitors.keys()) > 0:
test_monitors = {key: [] for key in self.monitors.keys()}
for x, y in self.iterator(self.dataset, datasets.TEST,
self.batch_size, self.minimum_batch_size,
self.rng):
if self.unsupervised:
for key in self.monitors.keys():
monitor_function = self.monitors[key]
test_monitors[key].append(monitor_function(x))
else:
for key in self.monitors.keys():
monitor_function = self.monitors[key]
test_monitors[key].append(monitor_function(x, y))
log.info(
'Test monitors: %s',
str({
key: numpy.mean(value, 0)
for key, value in test_monitors.items()
}))
# check for early stopping on train costs
cost = numpy.sum(train_costs)
if cost < self.best_cost * self.early_stop_threshold:
self.patience = 0
self.best_cost = cost
# save the parameters that made it the best
self.best_params = get_shared_values(self.params)
else:
self.patience += 1
if self.epoch_counter >= self.n_epoch or self.patience >= self.early_stop_length:
log.info("Stopping early...")
self.STOP = True
timing = time.time() - t
self.times.append(timing)
log.info('time: ' + make_time_units_string(timing))
log.info('remaining time: ' +
make_time_units_string((self.n_epoch - self.epoch_counter) *
numpy.mean(self.times)))
if (self.epoch_counter % self.save_frequency) == 0:
#save params
self.model.save_params('trained_epoch_' + str(self.epoch_counter) +
'.pkl')
# ANNEAL!
if hasattr(self, 'learning_rate_decay'):
self.learning_rate_decay.decay()
if hasattr(self, 'momentum_decay'):
self.momentum_decay.decay()
for decay_param in self.model.get_decay_params():
decay_param.decay()
def _perform_one_epoch(self, f_learn, plot=None):
"""
Performs a single training iteration with the given learn function.
"""
self.epoch_counter += 1
t = time.time()
log.info('EPOCH %s', str(self.epoch_counter))
# set the noise switches on for training function! (this is where things like dropout happen)
switch_vals = []
if len(self.noise_switches) > 0 and (self.valid_flag or self.test_flag or self.epoch_counter == 1):
log.debug("Turning on %s noise switches", str(len(self.noise_switches)))
switch_vals = [switch.get_value() for switch in self.noise_switches]
[switch.set_value(1.) for switch in self.noise_switches]
#########
# train #
#########
train_costs = []
train_monitors = {key: [] for key in self.train_monitors_dict.keys()}
train_data = [
minibatch(input, self.batch_size, self.min_batch_size)
for input in raise_to_list(self.dataset.train_inputs)
]
if self.dataset.train_targets is not None and not self.unsupervised:
train_data += [
minibatch(target, self.batch_size, self.min_batch_size)
for target in raise_to_list(self.dataset.train_targets)
]
for batch in min_normalized_izip(*train_data):
_outs = raise_to_list(f_learn(*batch))
train_costs.append(_outs[0])
# handle any user defined monitors
if len(train_monitors) > 0:
current_monitors = zip(self.train_monitors_dict.keys(), _outs[1:])
for name, val in current_monitors:
val = numpy.asarray(val)
train_monitors[name].append(val)
# get the mean values for the batches
mean_train = numpy.mean(train_costs, 0)
current_mean_monitors = {key: numpy.mean(vals, 0) for key, vals in train_monitors.items()}
# log the mean values!
log.info('Train cost: %s', trunc(mean_train))
if len(current_mean_monitors) > 0:
log.info('Train monitors: %s', str(current_mean_monitors))
# send the values to their outservices
if self.train_outservice:
self.train_outservice.write(mean_train, "train")
for name, service in self.train_monitors_outservice_dict.items():
if name in current_mean_monitors and service:
service.write(current_mean_monitors[name], "train")
# if there is a plot, also send them over!
if plot:
current_mean_monitors.update({TRAIN_COST_KEY: mean_train})
plot.update_plots(epoch=self.epoch_counter, monitors=current_mean_monitors)
# set the noise switches off for valid and test sets! we assume unseen data is noisy anyway :)
if len(self.noise_switches) > 0 and (self.valid_flag or self.test_flag):
log.debug("Turning off %s noise switches", str(len(self.noise_switches)))
[switch.set_value(0.) for switch in self.noise_switches]
#########
# valid #
#########
self._compute_over_subset("valid", self.dataset.valid_inputs, self.dataset.valid_targets,
self.valid_monitors_dict, self.valid_monitor_function,
self.valid_monitors_outservice_dict, plot)
########
# test #
########
self._compute_over_subset("test", self.dataset.test_inputs, self.dataset.test_targets,
self.test_monitors_dict, self.test_monitor_function,
self.test_monitors_outservice_dict, plot)
###########
# cleanup #
###########
# check for early stopping on train costs
cost = numpy.sum(train_costs)
# if the cost improved, reset the patience and record the best cost.
if cost < self.best_cost * self.early_stop_threshold:
self.patience = 0
self.best_cost = cost
# save the parameters that made it the best
self.best_params = get_shared_values(self.params)
elif not numpy.isnan(cost):
self.patience += 1
# check for stopping either from n_epochs or from threshold/patience
stop = False
if self.epoch_counter >= self.n_epoch:
log.info("Stopping (reached max number of epochs)...")
stop = True
if self.patience >= self.early_stop_length:
log.info("Stopping early (reached stop threshold)...")
stop = True
timing = time.time() - t
self.times.append(timing)
log.info('time: ' + make_time_units_string(timing))
log.debug('remaining time: ' +
make_time_units_string((self.n_epoch - self.epoch_counter) * numpy.mean(self.times)))
if (self.epoch_counter % self.save_frequency) == 0:
#save params
self.model.save_params('trained_epoch_' + str(self.epoch_counter) + '.pkl')
# ANNEAL!
if not stop:
# perform the appropriate decay on the decay functions/parameters for this optimizer and model
for decay_param in self.get_decay_params():
decay_param.decay()
# reset the switches
if len(self.noise_switches) > 0:
[switch.set_value(val) for switch, val in zip(self.noise_switches, switch_vals)]
# return whether or not to stop this epoch
return stop
