Пример #1
0
  def _testTypesForFtrl(self, x, y, z, lr, grad, use_gpu=None, l1=0.0,
                        l2=0.0, lr_power=-0.5):
    self.setUp()
    with self.test_session(use_gpu=use_gpu):
      var = variables.Variable(x)
      accum = variables.Variable(y)
      linear = variables.Variable(z)
      variables.initialize_all_variables().run()

      self.assertAllCloseAccordingToType(x, var.eval())
      apply_ftrl = training_ops.apply_ftrl(var, accum, linear, grad, lr, l1, l2,
                                           lr_power)
      out = apply_ftrl.eval()
      self.assertShapeEqual(out, apply_ftrl)
      accum_update = y + grad * grad
      linear_update = z + grad - (accum_update ** (-lr_power) - y ** (
          -lr_power)) / lr * x
      quadratic = 1.0 / (accum_update ** (lr_power) * lr) + 2 * l2
      expected_out = np.array([(np.sign(
          linear_update[i]) * l1 - linear_update[i]) / (
              quadratic[i]) if np.abs(
                  linear_update[i]) > l1 else 0.0 for i in range(
                      linear_update.size)])
      self.assertAllCloseAccordingToType(accum_update, accum.eval())
      if x.dtype == np.float16:
        # The calculations here really are not very precise in float16.
        self.assertAllClose(linear_update, linear.eval(), rtol=2e-2, atol=2e-2)
        self.assertAllClose(expected_out, out, rtol=2e-2, atol=2e-2)
      else:
        self.assertAllClose(linear_update, linear.eval())
        self.assertAllClose(expected_out, out)
Пример #2
0
 def _apply_dense(self, grad, var):
   accum = self.get_slot(var, "accum")
   linear = self.get_slot(var, "linear")
   if self._l2_shrinkage_regularization_strength <= 0.0:
     return training_ops.apply_ftrl(
         var,
         accum,
         linear,
         grad,
         math_ops.cast(self._learning_rate_tensor, var.dtype.base_dtype),
         math_ops.cast(self._l1_regularization_strength_tensor,
                       var.dtype.base_dtype),
         math_ops.cast(self._l2_regularization_strength_tensor,
                       var.dtype.base_dtype),
         math_ops.cast(self._learning_rate_power_tensor, var.dtype.base_dtype),
         use_locking=self._use_locking)
   else:
     return training_ops.apply_ftrl_v2(
         var,
         accum,
         linear,
         grad,
         math_ops.cast(self._learning_rate_tensor, var.dtype.base_dtype),
         math_ops.cast(self._l1_regularization_strength_tensor,
                       var.dtype.base_dtype),
         math_ops.cast(self._l2_regularization_strength_tensor,
                       var.dtype.base_dtype),
         math_ops.cast(self._l2_shrinkage_regularization_strength_tensor,
                       var.dtype.base_dtype),
         math_ops.cast(self._learning_rate_power_tensor, var.dtype.base_dtype),
         use_locking=self._use_locking)
Пример #3
0
 def _apply_dense(self, grad, var):
   accum = self.get_slot(var, "accum")
   linear = self.get_slot(var, "linear")
   if self._l2_shrinkage_regularization_strength <= 0.0:
     return training_ops.apply_ftrl(
         var,
         accum,
         linear,
         grad,
         math_ops.cast(self._learning_rate_tensor, var.dtype.base_dtype),
         math_ops.cast(self._l1_regularization_strength_tensor,
                       var.dtype.base_dtype),
         math_ops.cast(self._l2_regularization_strength_tensor,
                       var.dtype.base_dtype),
         math_ops.cast(self._learning_rate_power_tensor, var.dtype.base_dtype),
         use_locking=self._use_locking)
   else:
     return training_ops.apply_ftrl_v2(
         var,
         accum,
         linear,
         grad,
         math_ops.cast(self._learning_rate_tensor, var.dtype.base_dtype),
         math_ops.cast(self._l1_regularization_strength_tensor,
                       var.dtype.base_dtype),
         math_ops.cast(self._l2_regularization_strength_tensor,
                       var.dtype.base_dtype),
         math_ops.cast(self._l2_shrinkage_regularization_strength_tensor,
                       var.dtype.base_dtype),
         math_ops.cast(self._learning_rate_power_tensor, var.dtype.base_dtype),
         use_locking=self._use_locking)
    def _testTypesForFtrl(self,
                          x,
                          y,
                          z,
                          lr,
                          grad,
                          use_gpu=None,
                          l1=0.0,
                          l2=0.0,
                          lr_power=-0.5):
        self.setUp()
        with self.test_session(use_gpu=use_gpu):
            var = variables.Variable(x)
            accum = variables.Variable(y)
            linear = variables.Variable(z)
            variables.initialize_all_variables().run()

            self.assertAllEqual(x, var.eval())
            apply_ftrl = training_ops.apply_ftrl(var, accum, linear, grad, lr,
                                                 l1, l2, lr_power)
            out = apply_ftrl.eval()
            self.assertShapeEqual(out, apply_ftrl)
            accum_update = y + grad * grad
            linear_update = z + grad - (accum_update**(-lr_power) - y**
                                        (-lr_power)) / lr * x
            quadratic = 1.0 / (accum_update**(lr_power) * lr) + 2 * l2
            expected_out = np.array([
                (np.sign(linear_update[i]) * l1 - linear_update[i]) /
                (quadratic[i]) if np.abs(linear_update[i]) > l1 else 0.0
                for i in range(linear_update.size)
            ])
            self.assertAllClose(accum_update, accum.eval())
            self.assertAllClose(linear_update, linear.eval())
            self.assertAllClose(expected_out, out)
Пример #5
0
    def _testTypesForFtrlMultiplyLinearByLr(self,
                                            x,
                                            y,
                                            z,
                                            lr,
                                            grad,
                                            use_gpu=None,
                                            l1=0.0,
                                            l2=0.0,
                                            lr_power=-0.5):
        self.setUp()
        with self.session(use_gpu=use_gpu):
            var = variables.VariableV1(x)
            accum = variables.VariableV1(y)
            linear = variables.VariableV1(z)
            self.evaluate(variables.global_variables_initializer())

            self.assertAllCloseAccordingToType(x, self.evaluate(var))
            apply_ftrl = (training_ops.apply_ftrl(var,
                                                  accum,
                                                  linear,
                                                  grad,
                                                  lr,
                                                  l1,
                                                  l2,
                                                  lr_power,
                                                  multiply_linear_by_lr=True))
            out = self.evaluate(apply_ftrl)
            self.assertShapeEqual(out, apply_ftrl)
            accum_update = y + grad * grad
            linear_update = z + grad * lr - (accum_update**(-lr_power) - y**
                                             (-lr_power)) * x
            quadratic = accum_update**(-lr_power) + 2 * l2 * lr
            expected_out = np.array([
                (np.sign(linear_update[i]) * l1 * lr - linear_update[i]) /
                (quadratic[i]) if np.abs(linear_update[i]) > l1 * lr else 0.0
                for i in range(linear_update.size)
            ])
            self.assertAllCloseAccordingToType(accum_update,
                                               self.evaluate(accum))
            if x.dtype == np.float16:
                # The calculations here really are not very precise in float16.
                self.assertAllClose(linear_update,
                                    self.evaluate(linear),
                                    rtol=2e-2,
                                    atol=2e-2)
                self.assertAllClose(expected_out, out, rtol=2e-2, atol=2e-2)
            elif x.dtype == np.float32:
                # The calculations here not sufficiently precise in float32.
                self.assertAllClose(linear_update,
                                    self.evaluate(linear),
                                    rtol=1e-5,
                                    atol=1e-5)
                self.assertAllClose(expected_out, out, rtol=1e-5, atol=1e-5)
            else:
                self.assertAllClose(linear_update, self.evaluate(linear))
                self.assertAllClose(expected_out, out)
Пример #6
0
 def _apply_dense(self, grad, var):
   accum = self.get_slot(var, "accum")
   linear = self.get_slot(var, "linear")
   return training_ops.apply_ftrl(
       var, accum, linear, grad, self._learning_rate_tensor,
       self._l1_regularization_strength_tensor,
       self._l2_regularization_strength_tensor,
       self._learning_rate_power_tensor,
       use_locking=self._use_locking)
Пример #7
0
    def _testTypesForFtrl(self,
                          x,
                          y,
                          z,
                          lr,
                          grad,
                          use_gpu=None,
                          l1=0.0,
                          l2=0.0,
                          lr_power=-0.5):
        self.setUp()
        with self.test_session(use_gpu=use_gpu):
            var = variables.Variable(x)
            accum = variables.Variable(y)
            linear = variables.Variable(z)
            variables.global_variables_initializer().run()

            self.assertAllCloseAccordingToType(x, var.eval())
            apply_ftrl = training_ops.apply_ftrl(var, accum, linear, grad, lr,
                                                 l1, l2, lr_power)
            out = apply_ftrl.eval()
            self.assertShapeEqual(out, apply_ftrl)
            accum_update = y + grad * grad
            linear_update = z + grad - (accum_update**(-lr_power) - y**
                                        (-lr_power)) / lr * x
            quadratic = 1.0 / (accum_update**(lr_power) * lr) + 2 * l2
            expected_out = np.array([
                (np.sign(linear_update[i]) * l1 - linear_update[i]) /
                (quadratic[i]) if np.abs(linear_update[i]) > l1 else 0.0
                for i in range(linear_update.size)
            ])
            self.assertAllCloseAccordingToType(accum_update, accum.eval())
            if x.dtype == np.float16:
                # The calculations here really are not very precise in float16.
                self.assertAllClose(linear_update,
                                    linear.eval(),
                                    rtol=2e-2,
                                    atol=2e-2)
                self.assertAllClose(expected_out, out, rtol=2e-2, atol=2e-2)
            elif x.dtype == np.float32:
                # The calculations here not sufficiently precise in float32.
                self.assertAllClose(linear_update,
                                    linear.eval(),
                                    rtol=1e-5,
                                    atol=1e-5)
                self.assertAllClose(expected_out, out, rtol=1e-5, atol=1e-5)
            else:
                self.assertAllClose(linear_update, linear.eval())
                self.assertAllClose(expected_out, out)