def adadelta(self, lr, tparams, grads, model_input, cost, givens=None): """ An adaptive learning rate optimizer Parameters ---------- lr : Theano SharedVariable Initial learning rate tpramas: Theano SharedVariable Model parameters grads: Theano variable Gradients of cost w.r.t to parameres input: Theano variable of input, list. cost: Theano variable Objective fucntion to minimize Notes ----- For more information, see [ADADELTA]_. .. [ADADELTA] Matthew D. Zeiler, *ADADELTA: An Adaptive Learning Rate Method*, arXiv:1212.5701. """ zipped_grads = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_grad' % k) for k, p in tparams.iteritems()] running_up2 = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_rup2' % k) for k, p in tparams.iteritems()] running_grads2 = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_rgrad2' % k) for k, p in tparams.iteritems()] zgup = [(zg, g) for zg, g in zip(zipped_grads, grads)] rg2up = [(rg2, 0.95 * rg2 + 0.05 * (g ** 2)) for rg2, g in zip(running_grads2, grads)] f_grad_shared = theano.function(model_input, cost, updates=zgup + rg2up, name='adadelta_f_grad_shared', givens=givens) updir = [-T.sqrt(ru2 + 1e-6) / T.sqrt(rg2 + 1e-6) * zg for zg, ru2, rg2 in zip(zipped_grads, running_up2, running_grads2)] ru2up = [(ru2, 0.95 * ru2 + 0.05 * (ud ** 2)) for ru2, ud in zip(running_up2, updir)] param_up = [(p, p + ud) for p, ud in zip(tparams.values(), updir)] f_update = theano.function([lr], [], updates=ru2up + param_up, on_unused_input='ignore', name='adadelta_f_update', givens=givens) return f_grad_shared, f_update
def adadelta(self, lr, tparams, grads, model_input, cost, givens=None): """ An adaptive learning rate optimizer Parameters ---------- lr : tensorheano SharedVariable Initial learning rate tpramas: tensorheano SharedVariable Model parameters grads: tensorheano variable Gradients of cost w.r.t to parameres input: tensorheano variable of input, list. cost: tensorheano variable Objective fucntion to minimize Notes ----- For more information, see [ADADELtensorA]_. .. [ADADELtensorA] Matthew D. Zeiler, *ADADELtensorA: An Adaptive Learning Rate Method*, arXiv:1212.5701. """ zipped_grads = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_grad' % k) for k, p in tparams.iteritems()] running_up2 = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_rup2' % k) for k, p in tparams.iteritems()] running_grads2 = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_rgrad2' % k) for k, p in tparams.iteritems()] zgup = [(zg, g) for zg, g in zip(zipped_grads, grads)] rg2up = [(rg2, 0.95 * rg2 + 0.05 * (g ** 2)) for rg2, g in zip(running_grads2, grads)] f_grad_shared = theano.function(model_input, cost, updates=zgup + rg2up, name='adadelta_f_grad_shared', givens=givens) updir = [-tensor.sqrt(ru2 + 1e-6) / tensor.sqrt(rg2 + 1e-6) * zg for zg, ru2, rg2 in zip(zipped_grads, running_up2, running_grads2)] ru2up = [(ru2, 0.95 * ru2 + 0.05 * (ud ** 2)) for ru2, ud in zip(running_up2, updir)] param_up = [(p, p + ud) for p, ud in zip(tparams.values(), updir)] f_update = theano.function([lr], [], updates=ru2up + param_up, on_unused_input='ignore', name='adadelta_f_update', givens=givens) return f_grad_shared, f_update
def getOutput(self, inputs): """ Get outputs of encoder layer. Return all of the hidden status. """ (self.sentence, self.mask) = inputs assert self.mask is not None n_steps = self.sentence.shape[0] if self.sentence.ndim == 3: n_samples = self.sentence.shape[1] else: n_samples = 1 last_h = tensor.alloc(numpy_floatX(0.), n_samples, self.hidden_status_dim) state_below = tensor.dot(self.sentence, self.node.get_params_W()) results, _ = theano.scan(self.node.node_update, sequences=[self.mask, state_below], outputs_info=[last_h], name=self._p(self.prefix, '_scan'), n_steps=n_steps) hidden_status_outputs = results return hidden_status_outputs
def get_output(self, inputs): """ Get outputs of encoder layer. Return all of the hidden status. """ (self.sentence, self.mask, self.encoder_hidden_status, self.question_mask) = inputs assert self.mask is not None n_steps = self.sentence.shape[0] if self.sentence.ndim == 3: n_samples = self.sentence.shape[1] else: n_samples = 1 # last_s = tensor.dot(self.encoder_hidden_status[0, :, self.hidden_status_dim:], # self.params[self._p(self.prefix, 'Ws')]) last_s = tensor.alloc(numpy_floatX(0.), n_samples, self.hidden_status_dim) state_below = self.sentence def upd(am_, x_, s_, h_, qm_): c, alpha = self.attention_node.node_update(s_, h_, qm_) x_ = tensor.dot(tensor.concatenate([x_, c], axis=1), self.node.get_params_W()) s = self.node.node_update(am_, x_, s_) return s, c, alpha results, _ = theano.scan(upd, sequences=[self.mask, state_below], outputs_info=[last_s, None, None], non_sequences=[self.encoder_hidden_status, self.question_mask], name=self._p(self.prefix, '_scan'), n_steps=n_steps) hidden_status_outputs, context_outputs, alpha_outputs = results return hidden_status_outputs, context_outputs, alpha_outputs
def getOutput(self, inputs): """ Get outputs of encoder layer. Return all of the hidden status. """ (self.sentence, self.mask) = inputs assert self.mask is not None n_steps = self.sentence.shape[0] if self.sentence.ndim == 3: n_samples = self.sentence.shape[1] else: n_samples = 1 last_h = tensor.alloc(numpy_floatX(0.), n_samples, self.hidden_status_dim) state_below = tensor.dot(self.sentence, self.node.get_params_W()) results, _ = theano.scan(self.node.node_update, sequences=[self.mask, state_below], outputs_info=[last_h], name=self._p(self.prefix, '_scan'), n_steps=n_steps) hidden_status_outputs = results return hidden_status_outputs
def get_output(self, inputs): """ Get outputs of encoder layer. Return all of the hidden status. """ (self.sentence, self.mask) = inputs assert self.mask is not None n_steps = self.sentence.shape[0] if self.sentence.ndim == 3: n_samples = self.sentence.shape[1] else: n_samples = 1 hidden_states_list = [self.sentence] for idx in range(self.n_layers) : sentence = hidden_states_list[idx] state_below = tensor.dot(sentence, self.node_list[idx].get_params_W()) last_h = tensor.alloc(numpy_floatX(0.), n_samples, self.hidden_status_dim) results, _ = theano.scan(self.node_list[idx].node_update, sequences=[self.mask, state_below], outputs_info=[last_h], name=self._p(self.prefix, '_scan'+str(idx)), n_steps=n_steps) hidden_states_list.append(results) hidden_status_outputs = hidden_states_list return hidden_status_outputs
def get_output(self, inputs): """ Get outputs of encoder layer. Return all of the hidden status. """ (self.sentence, self.mask, self.question_children, self.question_children_mask, self.max_offset) = inputs assert self.mask is not None n_steps = self.sentence.shape[0] if self.sentence.ndim == 3: n_samples = self.sentence.shape[1] else: n_samples = 1 queue_buffer = tensor.alloc(numpy_floatX(0.), n_samples, self.max_offset, self.hidden_status_dim) state_below = tensor.dot(self.sentence, self.node.get_params_W()) non_seq = self.node.get_non_seq_parameter(n_samples) self.question_children_mask= self.question_children_mask.dimshuffle([0, 1, 2, 'x']) results, _ = theano.scan(self.node.node_update, sequences=[self.mask, state_below, self.question_children, self.question_children_mask], outputs_info=[queue_buffer, queue_buffer[:, -1, :]], non_sequences=non_seq, name=self._p(self.prefix, '_scan'), n_steps=n_steps) hidden_status_outputs = results[1] return hidden_status_outputs
def get_output(self, inputs): """ Get outputs of encoder layer. Return all of the hidden status. """ if len(inputs) == 4: (self.sentence, self.mask, self.forward_hidden_status, self.direction) = inputs if len(inputs) == 5: (self.sentence, self.mask, self.forward_hidden_status, self.direction, _) = inputs assert self.mask is not None n_steps = self.sentence.shape[0] if self.sentence.ndim == 3: n_samples = self.sentence.shape[1] else: n_samples = 1 if len(inputs) == 4: last_h = tensor.alloc(numpy_floatX(0.), n_samples, self.layer_number, self.hidden_status_dim) if len(inputs) == 5: last_h = inputs[4] last_h = tensor.alloc(last_h, self.layer_number, last_h.shape[0], last_h.shape[1]).dimshuffle([1, 0, 2]) state_below = tensor.dot( tensor.concatenate([ self.sentence, tensor.alloc(self.forward_hidden_status[-1, :, :], self.sentence.shape[0], self.forward_hidden_status.shape[1], self.hidden_status_dim) ], axis=2), self.node.get_params_W()) results, _ = theano.scan( self.node.node_update, sequences=[self.mask, state_below, self.direction], outputs_info=[last_h], name=self._p(self.prefix, '_scan'), n_steps=n_steps) hidden_status_outputs = results # p = printing.Print('hidden_status_outputs') # hidden_status_outputs = p(hidden_status_outputs) return hidden_status_outputs
def get_output(self, inputs): """ Get outputs of encoder layer. Return all of the hidden status. """ (self.sentence, self.mask, self.encoder_hidden_status, self.question_mask) = inputs assert self.mask is not None n_steps = self.sentence.shape[0] if self.sentence.ndim == 3: n_samples = self.sentence.shape[1] else: n_samples = 1 # last_s = tensor.dot(self.encoder_hidden_status[0, :, self.hidden_status_dim:], # self.params[self._p(self.prefix, 'Ws')]) last_s = tensor.alloc(numpy_floatX(0.), n_samples, self.hidden_status_dim) state_below = self.sentence def upd(am_, x_, s_, h_, qm_): c, alpha = self.attention_node.node_update(s_, h_, qm_) x_ = tensor.dot(tensor.concatenate([x_, c], axis=1), self.node.get_params_W()) s = self.node.node_update(am_, x_, s_) return s, c, alpha results, _ = theano.scan( upd, sequences=[self.mask, state_below], outputs_info=[last_s, None, None], non_sequences=[self.encoder_hidden_status, self.question_mask], name=self._p(self.prefix, '_scan'), n_steps=n_steps) hidden_status_outputs, context_outputs, alpha_outputs = results return hidden_status_outputs, context_outputs, alpha_outputs
def rmsprop(self, lr, tparams, grads, model_input, cost, givens=None): """ A variant of SGD that scales the step size by running average of the recent step norms. Parameters ---------- lr : Theano SharedVariable Initial learning rate tpramas: Theano SharedVariable Model parameters grads: Theano variable Gradients of cost w.r.t to parameres x: Theano variable Model inputs mask: Theano variable Sequence mask y: Theano variable Targets cost: Theano variable Objective fucntion to minimize Notes ----- For more information, see [Hint2014]_. .. [Hint2014] Geoff Hinton, *Neural Networks for Machine Learning*, lecture 6a, http://cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf """ zipped_grads = [ theano.shared(p.get_value() * numpy_floatX(0.), name='%s_grad' % k) for k, p in tparams.iteritems() ] running_grads = [ theano.shared(p.get_value() * numpy_floatX(0.), name='%s_rgrad' % k) for k, p in tparams.iteritems() ] running_grads2 = [ theano.shared(p.get_value() * numpy_floatX(0.), name='%s_rgrad2' % k) for k, p in tparams.iteritems() ] zgup = [(zg, g) for zg, g in zip(zipped_grads, grads)] rgup = [(rg, 0.95 * rg + 0.05 * g) for rg, g in zip(running_grads, grads)] rg2up = [(rg2, 0.95 * rg2 + 0.05 * (g**2)) for rg2, g in zip(running_grads2, grads)] f_grad_shared = theano.function(model_input, cost, updates=zgup + rgup + rg2up, name='rmsprop_f_grad_shared', givens=givens) updir = [ theano.shared(p.get_value() * numpy_floatX(0.), name='%s_updir' % k) for k, p in tparams.iteritems() ] updir_new = [(ud, 0.9 * ud - 1e-4 * zg / T.sqrt(rg2 - rg**2 + 1e-4)) for ud, zg, rg, rg2 in zip(updir, zipped_grads, running_grads, running_grads2)] param_up = [(p, p + udn[1]) for p, udn in zip(tparams.values(), updir_new)] f_update = theano.function([lr], [], updates=updir_new + param_up, on_unused_input='ignore', name='rmsprop_f_update', givens=givens) return f_grad_shared, f_update
def rmsprop(self, lr, tparams, grads, model_input, cost, givens=None): """ A variant of SGD that scales the step size by running average of the recent step norms. Parameters ---------- lr : tensorheano SharedVariable Initial learning rate tpramas: tensorheano SharedVariable Model parameters grads: tensorheano variable Gradients of cost w.r.t to parameres x: tensorheano variable Model inputs mask: tensorheano variable Sequence mask y: tensorheano variable tensorargets cost: tensorheano variable Objective fucntion to minimize Notes ----- For more information, see [Hint2014]_. .. [Hint2014] Geoff Hinton, *Neural Networks for Machine Learning*, lecture 6a, http://cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf """ zipped_grads = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_grad' % k) for k, p in tparams.iteritems()] running_grads = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_rgrad' % k) for k, p in tparams.iteritems()] running_grads2 = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_rgrad2' % k) for k, p in tparams.iteritems()] zgup = [(zg, g) for zg, g in zip(zipped_grads, grads)] rgup = [(rg, 0.95 * rg + 0.05 * g) for rg, g in zip(running_grads, grads)] rg2up = [(rg2, 0.95 * rg2 + 0.05 * (g ** 2)) for rg2, g in zip(running_grads2, grads)] f_grad_shared = theano.function(model_input, cost, updates=zgup + rgup + rg2up, name='rmsprop_f_grad_shared', givens=givens) updir = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_updir' % k) for k, p in tparams.iteritems()] updir_new = [(ud, 0.9 * ud - 1e-4 * zg / tensor.sqrt(rg2 - rg ** 2 + 1e-4)) for ud, zg, rg, rg2 in zip(updir, zipped_grads, running_grads, running_grads2)] param_up = [(p, p + udn[1]) for p, udn in zip(tparams.values(), updir_new)] f_update = theano.function([lr], [], updates=updir_new + param_up, on_unused_input='ignore', name='rmsprop_f_update', givens=givens) return f_grad_shared, f_update