def _apply_dense(self, grad, var, state): acc = state.get_slot(var, "accumulator") return training_ops.apply_adagrad( var, acc, state.get_hyper("learning_rate", var.dtype.base_dtype), grad, use_locking=self._use_locking)
def _apply_dense(self, grad, var): acc = self.get_slot(var, "accumulator") return training_ops.apply_adagrad( var, acc, math_ops.cast(self._learning_rate_tensor, var.dtype.base_dtype), grad, use_locking=self._use_locking)
def _apply_dense(self, grad, var): acc = self.get_slot(var, "accumulator") lr = self._learning_rate_tensor eps = 10e-8 sqdiv = tf.pow(lr, 2) - tf.pow(var, 2) #if sqdiv < 0 the next var absolutely become positive penalty = tf.to_float(tf.less(sqdiv, 0)) * 10e+12 g_max = (var * tf.sqrt(sqdiv)) / (sqdiv + eps) + penalty grad_clipped = tf.minimum(grad, g_max) return training_ops.apply_adagrad( var, acc, self._learning_rate_tensor, grad_clipped, use_locking=self._use_locking)
def _testTypesForAdagrad(self, x, y, lr, grad, use_gpu=None): self.setUp() with self.test_session(use_gpu=use_gpu): var = variables.Variable(x) accum = variables.Variable(y) variables.initialize_all_variables().run() self.assertAllEqual(x, var.eval()) apply_adagrad = training_ops.apply_adagrad(var, accum, lr, grad) out = apply_adagrad.eval() self.assertShapeEqual(out, apply_adagrad) self.assertAllClose(x - lr * grad * (y + grad * grad)**(-0.5), out) self.assertAllEqual(y + grad * grad, accum.eval())
def _testTypesForAdagrad(self, x, y, lr, grad, use_gpu=None): self.setUp() with self.test_session(use_gpu=use_gpu): var = variables.Variable(x) accum = variables.Variable(y) variables.global_variables_initializer().run() self.assertAllCloseAccordingToType(x, var.eval()) apply_adagrad = training_ops.apply_adagrad(var, accum, lr, grad) out = apply_adagrad.eval() self.assertShapeEqual(out, apply_adagrad) self.assertAllCloseAccordingToType(x - lr * grad * (y + grad * grad)** (-0.5), out) self.assertAllCloseAccordingToType(y + grad * grad, accum.eval())
def _testTypesForAdagrad(self, x, y, lr, grad, use_gpu=None): self.setUp() with self.session(use_gpu=use_gpu): var = variables.VariableV1(x) accum = variables.VariableV1(y) self.evaluate(variables.global_variables_initializer()) self.assertAllCloseAccordingToType(x, self.evaluate(var)) apply_adagrad = training_ops.apply_adagrad(var, accum, lr, grad) out = self.evaluate(apply_adagrad) self.assertShapeEqual(out, apply_adagrad) self.assertAllCloseAccordingToType(x - lr * grad * (y + grad * grad)** (-0.5), out) self.assertAllCloseAccordingToType(y + grad * grad, self.evaluate(accum))
def _apply_dense(self, grad, var): acc = self.get_slot(var, "accumulator") if self.quantizer is None: return training_ops.apply_adagrad( var, acc, math_ops.cast(self._learning_rate_tensor, var.dtype.base_dtype), grad, use_locking=self._use_locking) else: return self.apply_qadagrad( var, acc, math_ops.cast(self._learning_rate_tensor, var.dtype.base_dtype), grad)
def _apply_dense(self, grad, var): acc = self.get_slot(var, "accumulator") return training_ops.apply_adagrad( var, acc, self._learning_rate_tensor, grad, use_locking=self._use_locking)
def _apply_dense(self, grad, var): acc = self.get_slot(var, "accumulator") return training_ops.apply_adagrad( var, acc, self._learning_rate_tensor, grad, use_locking=self._use_locking)