def cd_updates(self): """ Return a dictionary of shared variable updates that implements contrastive divergence learning by stochastic gradient descent with an annealed learning rate. """ ups = {} if self.persistent_chains: grads = self.contrastive_grads() ups.update(dict(self.sampler.updates())) else: cd1_sampler, final_p, cd1_updates = self.rbm.CD1_sampler(self.visible_batch, self.batchsize) self._last_cd1_sampler = cd1_sampler # hacked in here for the unit test #ignore the cd1_sampler grads = self.contrastive_grads(neg_v = final_p) ups.update(dict(cd1_updates)) # contrastive divergence updates # TODO: sgd_updates is a particular optization algo (others are possible) # parametrize so that algo is plugin # the normalization normVF might be sgd-specific though... # TODO: when sgd has an annealing schedule, this should # go through that mechanism. lr = TT.clip( self.learn_rate * TT.cast(self.lr_anneal_start / (self.iter+1), floatX), 0.0, #min self.learn_rate) #max ups.update(dict(sgd_updates( self.rbm.params(), grads, stepsizes=[a*lr for a in self.learn_rate_multipliers]))) ups[self.iter] = self.iter + 1 # add trainer updates (replace CD update of U) ups[self.rbm.U], ups[self.normVF] = self.normalize_U(ups[self.rbm.U]) #l1_updates: if (self.l1_penalty_start > 0) and (self.l1_penalty != 0.0): ups[self.effective_l1_penalty] = TT.switch( self.iter >= self.l1_penalty_start, self.l1_penalty, 0.0) if getattr(self,'p_lr', None): ups[self.p_lr] = TT.switch(self.iter > self.p_training_start, self.p_training_lr, 0) new_P = ups[self.rbm.P] * self.p_mask no_pos_P = TT.switch(new_P<0, new_P, 0) ups[self.rbm.P] = - no_pos_P / no_pos_P.sum(axis=0) #normalize to that columns sum 1 return ups
def cd_updates(self): """ Return a dictionary of shared variable updates that implements contrastive divergence learning by stochastic gradient descent with an annealed learning rate. """ ups = {} if self.persistent_chains: grads = self.contrastive_grads() ups.update(dict(self.sampler.updates())) else: cd1_sampler, final_p, cd1_updates = self.rbm.CD1_sampler( self.visible_batch, self.batchsize) self._last_cd1_sampler = cd1_sampler # hacked in here for the unit test #ignore the cd1_sampler grads = self.contrastive_grads(neg_v=final_p) ups.update(dict(cd1_updates)) # contrastive divergence updates # TODO: sgd_updates is a particular optization algo (others are possible) # parametrize so that algo is plugin # the normalization normVF might be sgd-specific though... # TODO: when sgd has an annealing schedule, this should # go through that mechanism. lr = TT.clip( self.learn_rate * TT.cast(self.lr_anneal_start / (self.iter + 1), floatX), 0.0, #min self.learn_rate) #max ups.update( dict( sgd_updates( self.rbm.params(), grads, stepsizes=[a * lr for a in self.learn_rate_multipliers]))) ups[self.iter] = self.iter + 1 # add trainer updates (replace CD update of U) ups[self.rbm.U], ups[self.normVF] = self.normalize_U(ups[self.rbm.U]) #l1_updates: if (self.l1_penalty_start > 0) and (self.l1_penalty != 0.0): ups[self.effective_l1_penalty] = TT.switch( self.iter >= self.l1_penalty_start, self.l1_penalty, 0.0) if getattr(self, 'p_lr', None): ups[self.p_lr] = TT.switch(self.iter > self.p_training_start, self.p_training_lr, 0) new_P = ups[self.rbm.P] * self.p_mask no_pos_P = TT.switch(new_P < 0, new_P, 0) ups[self.rbm.P] = -no_pos_P / no_pos_P.sum( axis=0) #normalize to that columns sum 1 return ups
def updates(self, gradients): """ Return symbolic updates to apply given a set of gradients on the parameters being optimized. Parameters ---------- gradients : list of tensor_likes List of symbolic gradients for the parameters contained in self.params, in the same order as in self.params. Returns ------- updates : dict A dictionary with the shared variables in self.params as keys and a symbolic expression of how they are to be updated each SGD step as values. Notes ----- `cost_updates` is a convenient helper function that takes all necessary gradients with respect to a given symbolic cost. """ ups = {} # Add the learning rate/iteration updates l_ups, learn_rates = self.learning_rate_updates() safe_update(ups, l_ups) # Get the updates from sgd_updates, a PyLearn library function. p_up = dict(sgd_updates(self.params, gradients, learn_rates)) # Add the things in p_up to ups safe_update(ups, p_up) # Clip the values if needed. # We do not want the clipping values to force an upcast # of the update: updates should have the same type as params for param, (p_min, p_max) in self.clipping_values.iteritems(): p_min = tensor.as_tensor(p_min) p_max = tensor.as_tensor(p_max) dtype = param.dtype if p_min.dtype != dtype: p_min = tensor.cast(p_min, dtype) if p_max.dtype != dtype: p_max = tensor.cast(p_max, dtype) ups[param] = tensor.clip(ups[param], p_min, p_max) # Return the updates dictionary. return ups