def _testDense(self, use_resource=False, learning_rate=0.1, sign_decay_fn=None, py_sign_decay_fn=None, base=math.e, beta=0.9): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.test_session(use_gpu=True): # Initialize variables for numpy implementation. m0, m1 = 0.0, 0.0 var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) if use_resource: var0 = resource_variable_ops.ResourceVariable(var0_np) var1 = resource_variable_ops.ResourceVariable(var1_np) global_step = resource_variable_ops.ResourceVariable( 0, trainable=False) else: var0 = variables.VariableV1(var0_np) var1 = variables.VariableV1(var1_np) global_step = variables.VariableV1(0, trainable=False) grads0 = constant_op.constant(grads0_np) grads1 = constant_op.constant(grads1_np) opt = powersign.PowerSignOptimizer( learning_rate=learning_rate, base=base, beta=beta, sign_decay_fn=sign_decay_fn, ) update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), global_step=global_step) neg_update = opt.apply_gradients(zip([-grads0, -grads1], [var0, var1]), global_step=global_step) if not context.executing_eagerly(): self.evaluate(variables.global_variables_initializer()) # Fetch params to validate initial values self.assertAllClose([1.0, 2.0], self.evaluate(var0)) self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run 7 steps of powersign # first 4 steps with positive gradient # last 3 steps with negative gradient (sign(gm) should be -1) for t in range(1, 8): if t < 5: if not context.executing_eagerly(): self.evaluate(update) elif t > 1: opt.apply_gradients(zip([grads0, grads1], [var0, var1]), global_step=global_step) else: if not context.executing_eagerly(): self.evaluate(neg_update) elif t > 1: opt.apply_gradients(zip([-grads0, -grads1], [var0, var1]), global_step=global_step) var0_np, m0 = powersign_update_numpy( var0_np, grads0_np if t < 5 else -grads0_np, m0, learning_rate, base=base, beta=beta, py_sign_decay_fn=py_sign_decay_fn, t=t, ) var1_np, m1 = powersign_update_numpy( var1_np, grads1_np if t < 5 else -grads1_np, m1, learning_rate, base=base, beta=beta, py_sign_decay_fn=py_sign_decay_fn, t=t, ) # Validate updated params self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
def _testDense(self, learning_rate=0.1, sign_decay_fn=None, py_sign_decay_fn=None, base=math.e, beta=0.9): for dtype in self.float_types: with self.cached_session(), self.test_scope(): # Initialize variables for numpy implementation. m0, m1 = 0.0, 0.0 var0_np = np.array([1.0, 2.0], dtype=dtype) grads0_np = np.array([0.1, 0.1], dtype=dtype) var1_np = np.array([3.0, 4.0], dtype=dtype) grads1_np = np.array([0.01, 0.01], dtype=dtype) var0 = resource_variable_ops.ResourceVariable(var0_np) var1 = resource_variable_ops.ResourceVariable(var1_np) global_step = resource_variable_ops.ResourceVariable(0, trainable=False) grads0 = constant_op.constant(grads0_np) grads1 = constant_op.constant(grads1_np) opt = powersign.PowerSignOptimizer( learning_rate=learning_rate, base=base, beta=beta, sign_decay_fn=sign_decay_fn, ) update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]), global_step=global_step) neg_update = opt.apply_gradients(zip([-grads0, -grads1], [var0, var1]), global_step=global_step) variables.global_variables_initializer().run() # Fetch params to validate initial values self.assertAllClose([1.0, 2.0], var0.eval()) self.assertAllClose([3.0, 4.0], var1.eval()) # Run 7 steps of powersign # first 4 steps with positive gradient # last 3 steps with negative gradient (sign(gm) should be -1) for t in range(1, 8): if t < 5: update.run() else: neg_update.run() var0_np, m0 = powersign_update_numpy( var0_np, grads0_np if t < 5 else -grads0_np, m0, learning_rate, base=base, beta=beta, py_sign_decay_fn=py_sign_decay_fn, t=t, ) var1_np, m1 = powersign_update_numpy( var1_np, grads1_np if t < 5 else -grads1_np, m1, learning_rate, base=base, beta=beta, py_sign_decay_fn=py_sign_decay_fn, t=t, ) # Validate updated params self.assertAllCloseAccordingToType(var0_np, var0.eval()) self.assertAllCloseAccordingToType(var1_np, var1.eval())