def cd_updates(self):
"""
Return a dictionary of shared variable updates that implements contrastive divergence
learning by stochastic gradient descent with an annealed learning rate.
"""
ups = {}
if self.persistent_chains:
grads = self.contrastive_grads()
ups.update(dict(self.sampler.updates()))
else:
cd1_sampler, final_p, cd1_updates = self.rbm.CD1_sampler(self.visible_batch,
self.batchsize)
self._last_cd1_sampler = cd1_sampler # hacked in here for the unit test
#ignore the cd1_sampler
grads = self.contrastive_grads(neg_v = final_p)
ups.update(dict(cd1_updates))
# contrastive divergence updates
# TODO: sgd_updates is a particular optization algo (others are possible)
# parametrize so that algo is plugin
# the normalization normVF might be sgd-specific though...
# TODO: when sgd has an annealing schedule, this should
# go through that mechanism.
lr = TT.clip(
self.learn_rate * TT.cast(self.lr_anneal_start / (self.iter+1), floatX),
0.0, #min
self.learn_rate) #max
ups.update(dict(sgd_updates(
self.rbm.params(),
grads,
stepsizes=[a*lr for a in self.learn_rate_multipliers])))
ups[self.iter] = self.iter + 1
# add trainer updates (replace CD update of U)
ups[self.rbm.U], ups[self.normVF] = self.normalize_U(ups[self.rbm.U])
#l1_updates:
if (self.l1_penalty_start > 0) and (self.l1_penalty != 0.0):
ups[self.effective_l1_penalty] = TT.switch(
self.iter >= self.l1_penalty_start,
self.l1_penalty,
0.0)
if getattr(self,'p_lr', None):
ups[self.p_lr] = TT.switch(self.iter > self.p_training_start,
self.p_training_lr,
0)
new_P = ups[self.rbm.P] * self.p_mask
no_pos_P = TT.switch(new_P<0, new_P, 0)
ups[self.rbm.P] = - no_pos_P / no_pos_P.sum(axis=0) #normalize to that columns sum 1
return ups
def cd_updates(self):
"""
Return a dictionary of shared variable updates that implements contrastive divergence
learning by stochastic gradient descent with an annealed learning rate.
"""
ups = {}
if self.persistent_chains:
grads = self.contrastive_grads()
ups.update(dict(self.sampler.updates()))
else:
cd1_sampler, final_p, cd1_updates = self.rbm.CD1_sampler(
self.visible_batch, self.batchsize)
self._last_cd1_sampler = cd1_sampler # hacked in here for the unit test
#ignore the cd1_sampler
grads = self.contrastive_grads(neg_v=final_p)
ups.update(dict(cd1_updates))
# contrastive divergence updates
# TODO: sgd_updates is a particular optization algo (others are possible)
# parametrize so that algo is plugin
# the normalization normVF might be sgd-specific though...
# TODO: when sgd has an annealing schedule, this should
# go through that mechanism.
lr = TT.clip(
self.learn_rate * TT.cast(self.lr_anneal_start /
(self.iter + 1), floatX),
0.0, #min
self.learn_rate) #max
ups.update(
dict(
sgd_updates(
self.rbm.params(),
grads,
stepsizes=[a * lr for a in self.learn_rate_multipliers])))
ups[self.iter] = self.iter + 1
# add trainer updates (replace CD update of U)
ups[self.rbm.U], ups[self.normVF] = self.normalize_U(ups[self.rbm.U])
#l1_updates:
if (self.l1_penalty_start > 0) and (self.l1_penalty != 0.0):
ups[self.effective_l1_penalty] = TT.switch(
self.iter >= self.l1_penalty_start, self.l1_penalty, 0.0)
if getattr(self, 'p_lr', None):
ups[self.p_lr] = TT.switch(self.iter > self.p_training_start,
self.p_training_lr, 0)
new_P = ups[self.rbm.P] * self.p_mask
no_pos_P = TT.switch(new_P < 0, new_P, 0)
ups[self.rbm.P] = -no_pos_P / no_pos_P.sum(
axis=0) #normalize to that columns sum 1
return ups
def updates(self, gradients):
"""
Return symbolic updates to apply given a set of gradients
on the parameters being optimized.
Parameters
----------
gradients : list of tensor_likes
List of symbolic gradients for the parameters contained
in self.params, in the same order as in self.params.
Returns
-------
updates : dict
A dictionary with the shared variables in self.params as keys
and a symbolic expression of how they are to be updated each
SGD step as values.
Notes
-----
`cost_updates` is a convenient helper function that takes all
necessary gradients with respect to a given symbolic cost.
"""
ups = {}
# Add the learning rate/iteration updates
l_ups, learn_rates = self.learning_rate_updates()
safe_update(ups, l_ups)
# Get the updates from sgd_updates, a PyLearn library function.
p_up = dict(sgd_updates(self.params, gradients, learn_rates))
# Add the things in p_up to ups
safe_update(ups, p_up)
# Clip the values if needed.
# We do not want the clipping values to force an upcast
# of the update: updates should have the same type as params
for param, (p_min, p_max) in self.clipping_values.iteritems():
p_min = tensor.as_tensor(p_min)
p_max = tensor.as_tensor(p_max)
dtype = param.dtype
if p_min.dtype != dtype:
p_min = tensor.cast(p_min, dtype)
if p_max.dtype != dtype:
p_max = tensor.cast(p_max, dtype)
ups[param] = tensor.clip(ups[param], p_min, p_max)
# Return the updates dictionary.
return ups
def updates(self, gradients):
"""
Return symbolic updates to apply given a set of gradients
on the parameters being optimized.
Parameters
----------
gradients : list of tensor_likes
List of symbolic gradients for the parameters contained
in self.params, in the same order as in self.params.
Returns
-------
updates : dict
A dictionary with the shared variables in self.params as keys
and a symbolic expression of how they are to be updated each
SGD step as values.
Notes
-----
`cost_updates` is a convenient helper function that takes all
necessary gradients with respect to a given symbolic cost.
"""
ups = {}
# Add the learning rate/iteration updates
l_ups, learn_rates = self.learning_rate_updates()
safe_update(ups, l_ups)
# Get the updates from sgd_updates, a PyLearn library function.
p_up = dict(sgd_updates(self.params, gradients, learn_rates))
# Add the things in p_up to ups
safe_update(ups, p_up)
# Clip the values if needed.
# We do not want the clipping values to force an upcast
# of the update: updates should have the same type as params
for param, (p_min, p_max) in self.clipping_values.iteritems():
p_min = tensor.as_tensor(p_min)
p_max = tensor.as_tensor(p_max)
dtype = param.dtype
if p_min.dtype != dtype:
p_min = tensor.cast(p_min, dtype)
if p_max.dtype != dtype:
p_max = tensor.cast(p_max, dtype)
ups[param] = tensor.clip(ups[param], p_min, p_max)
# Return the updates dictionary.
return ups
