def _testTypesForFtrl(self, x, y, z, lr, grad, use_gpu=None, l1=0.0, l2=0.0, lr_power=-0.5): self.setUp() with self.test_session(use_gpu=use_gpu): var = variables.Variable(x) accum = variables.Variable(y) linear = variables.Variable(z) variables.initialize_all_variables().run() self.assertAllCloseAccordingToType(x, var.eval()) apply_ftrl = training_ops.apply_ftrl(var, accum, linear, grad, lr, l1, l2, lr_power) out = apply_ftrl.eval() self.assertShapeEqual(out, apply_ftrl) accum_update = y + grad * grad linear_update = z + grad - (accum_update ** (-lr_power) - y ** ( -lr_power)) / lr * x quadratic = 1.0 / (accum_update ** (lr_power) * lr) + 2 * l2 expected_out = np.array([(np.sign( linear_update[i]) * l1 - linear_update[i]) / ( quadratic[i]) if np.abs( linear_update[i]) > l1 else 0.0 for i in range( linear_update.size)]) self.assertAllCloseAccordingToType(accum_update, accum.eval()) if x.dtype == np.float16: # The calculations here really are not very precise in float16. self.assertAllClose(linear_update, linear.eval(), rtol=2e-2, atol=2e-2) self.assertAllClose(expected_out, out, rtol=2e-2, atol=2e-2) else: self.assertAllClose(linear_update, linear.eval()) self.assertAllClose(expected_out, out)
def _apply_dense(self, grad, var): accum = self.get_slot(var, "accum") linear = self.get_slot(var, "linear") if self._l2_shrinkage_regularization_strength <= 0.0: return training_ops.apply_ftrl( var, accum, linear, grad, math_ops.cast(self._learning_rate_tensor, var.dtype.base_dtype), math_ops.cast(self._l1_regularization_strength_tensor, var.dtype.base_dtype), math_ops.cast(self._l2_regularization_strength_tensor, var.dtype.base_dtype), math_ops.cast(self._learning_rate_power_tensor, var.dtype.base_dtype), use_locking=self._use_locking) else: return training_ops.apply_ftrl_v2( var, accum, linear, grad, math_ops.cast(self._learning_rate_tensor, var.dtype.base_dtype), math_ops.cast(self._l1_regularization_strength_tensor, var.dtype.base_dtype), math_ops.cast(self._l2_regularization_strength_tensor, var.dtype.base_dtype), math_ops.cast(self._l2_shrinkage_regularization_strength_tensor, var.dtype.base_dtype), math_ops.cast(self._learning_rate_power_tensor, var.dtype.base_dtype), use_locking=self._use_locking)
def _testTypesForFtrl(self, x, y, z, lr, grad, use_gpu=None, l1=0.0, l2=0.0, lr_power=-0.5): self.setUp() with self.test_session(use_gpu=use_gpu): var = variables.Variable(x) accum = variables.Variable(y) linear = variables.Variable(z) variables.initialize_all_variables().run() self.assertAllEqual(x, var.eval()) apply_ftrl = training_ops.apply_ftrl(var, accum, linear, grad, lr, l1, l2, lr_power) out = apply_ftrl.eval() self.assertShapeEqual(out, apply_ftrl) accum_update = y + grad * grad linear_update = z + grad - (accum_update**(-lr_power) - y** (-lr_power)) / lr * x quadratic = 1.0 / (accum_update**(lr_power) * lr) + 2 * l2 expected_out = np.array([ (np.sign(linear_update[i]) * l1 - linear_update[i]) / (quadratic[i]) if np.abs(linear_update[i]) > l1 else 0.0 for i in range(linear_update.size) ]) self.assertAllClose(accum_update, accum.eval()) self.assertAllClose(linear_update, linear.eval()) self.assertAllClose(expected_out, out)
def _testTypesForFtrlMultiplyLinearByLr(self, x, y, z, lr, grad, use_gpu=None, l1=0.0, l2=0.0, lr_power=-0.5): self.setUp() with self.session(use_gpu=use_gpu): var = variables.VariableV1(x) accum = variables.VariableV1(y) linear = variables.VariableV1(z) self.evaluate(variables.global_variables_initializer()) self.assertAllCloseAccordingToType(x, self.evaluate(var)) apply_ftrl = (training_ops.apply_ftrl(var, accum, linear, grad, lr, l1, l2, lr_power, multiply_linear_by_lr=True)) out = self.evaluate(apply_ftrl) self.assertShapeEqual(out, apply_ftrl) accum_update = y + grad * grad linear_update = z + grad * lr - (accum_update**(-lr_power) - y** (-lr_power)) * x quadratic = accum_update**(-lr_power) + 2 * l2 * lr expected_out = np.array([ (np.sign(linear_update[i]) * l1 * lr - linear_update[i]) / (quadratic[i]) if np.abs(linear_update[i]) > l1 * lr else 0.0 for i in range(linear_update.size) ]) self.assertAllCloseAccordingToType(accum_update, self.evaluate(accum)) if x.dtype == np.float16: # The calculations here really are not very precise in float16. self.assertAllClose(linear_update, self.evaluate(linear), rtol=2e-2, atol=2e-2) self.assertAllClose(expected_out, out, rtol=2e-2, atol=2e-2) elif x.dtype == np.float32: # The calculations here not sufficiently precise in float32. self.assertAllClose(linear_update, self.evaluate(linear), rtol=1e-5, atol=1e-5) self.assertAllClose(expected_out, out, rtol=1e-5, atol=1e-5) else: self.assertAllClose(linear_update, self.evaluate(linear)) self.assertAllClose(expected_out, out)
def _apply_dense(self, grad, var): accum = self.get_slot(var, "accum") linear = self.get_slot(var, "linear") return training_ops.apply_ftrl( var, accum, linear, grad, self._learning_rate_tensor, self._l1_regularization_strength_tensor, self._l2_regularization_strength_tensor, self._learning_rate_power_tensor, use_locking=self._use_locking)
def _testTypesForFtrl(self, x, y, z, lr, grad, use_gpu=None, l1=0.0, l2=0.0, lr_power=-0.5): self.setUp() with self.test_session(use_gpu=use_gpu): var = variables.Variable(x) accum = variables.Variable(y) linear = variables.Variable(z) variables.global_variables_initializer().run() self.assertAllCloseAccordingToType(x, var.eval()) apply_ftrl = training_ops.apply_ftrl(var, accum, linear, grad, lr, l1, l2, lr_power) out = apply_ftrl.eval() self.assertShapeEqual(out, apply_ftrl) accum_update = y + grad * grad linear_update = z + grad - (accum_update**(-lr_power) - y** (-lr_power)) / lr * x quadratic = 1.0 / (accum_update**(lr_power) * lr) + 2 * l2 expected_out = np.array([ (np.sign(linear_update[i]) * l1 - linear_update[i]) / (quadratic[i]) if np.abs(linear_update[i]) > l1 else 0.0 for i in range(linear_update.size) ]) self.assertAllCloseAccordingToType(accum_update, accum.eval()) if x.dtype == np.float16: # The calculations here really are not very precise in float16. self.assertAllClose(linear_update, linear.eval(), rtol=2e-2, atol=2e-2) self.assertAllClose(expected_out, out, rtol=2e-2, atol=2e-2) elif x.dtype == np.float32: # The calculations here not sufficiently precise in float32. self.assertAllClose(linear_update, linear.eval(), rtol=1e-5, atol=1e-5) self.assertAllClose(expected_out, out, rtol=1e-5, atol=1e-5) else: self.assertAllClose(linear_update, linear.eval()) self.assertAllClose(expected_out, out)