def test_resource():
for i, dtype in enumerate(_dtypes_to_test(use_gpu=is_gpu_available())):
# Initialize variables for numpy implementation.
m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
var0 = tf.Variable(var0_np, name="var0_%d" % i)
var1 = tf.Variable(var1_np, name="var1_%d" % i)
grads0 = tf.constant(grads0_np)
grads1 = tf.constant(grads1_np)
def learning_rate():
return 0.001
opt = AdaBeliefOptimizer(learning_rate=learning_rate)
# Run 3 steps of AdaBelief
for t in range(3):
beta_1_power, beta_2_power = get_beta_accumulators(opt, dtype)
assert_allclose_according_to_type(0.9**(t + 1), beta_1_power)
assert_allclose_according_to_type(0.999**(t + 1), beta_2_power)
opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
var0_np, m0, v0 = adabelief_update_numpy(var0_np, grads0_np, t, m0,
v0)
var1_np, m1, v1 = adabelief_update_numpy(var1_np, grads1_np, t, m1,
v1)
# Validate updated params
assert_allclose_according_to_type(var0_np, var0.numpy())
assert_allclose_according_to_type(var1_np, var1.numpy())
def test_sharing():
for dtype in _dtypes_to_test(use_gpu=is_gpu_available()):
# Initialize variables for numpy implementation.
m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
var0 = tf.Variable(var0_np)
var1 = tf.Variable(var1_np)
grads0 = tf.constant(grads0_np)
grads1 = tf.constant(grads1_np)
opt = AdaBeliefOptimizer()
# Fetch params to validate initial values
np.testing.assert_allclose(np.asanyarray([1.0, 2.0]), var0.numpy())
np.testing.assert_allclose(np.asanyarray([3.0, 4.0]), var1.numpy())
# Run 3 steps of AdaBelief
for t in range(3):
beta_1_power, beta_2_power = get_beta_accumulators(opt, dtype)
assert_allclose_according_to_type(0.9**(t + 1), beta_1_power)
assert_allclose_according_to_type(0.999**(t + 1), beta_2_power)
opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
var0_np, m0, v0 = adabelief_update_numpy(var0_np, grads0_np, t, m0,
v0)
var1_np, m1, v1 = adabelief_update_numpy(var1_np, grads1_np, t, m1,
v1)
# Validate updated params
assert_allclose_according_to_type(var0_np, var0.numpy())
assert_allclose_according_to_type(var1_np, var1.numpy())
def test_sparse():
for dtype in _dtypes_to_test(use_gpu=is_gpu_available()):
# Initialize tf for numpy implementation.
m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
var0_np = np.array([1.0, 1.0, 2.0], dtype=dtype.as_numpy_dtype)
grads0_np = np.array([0.1, 0.0, 0.1], dtype=dtype.as_numpy_dtype)
var1_np = np.array([3.0, 3.0, 4.0], dtype=dtype.as_numpy_dtype)
grads1_np = np.array([0.01, 0.0, 0.01], dtype=dtype.as_numpy_dtype)
var0 = tf.Variable(var0_np)
var1 = tf.Variable(var1_np)
grads0_np_indices = np.array([0, 2], dtype=np.int32)
grads0 = tf.IndexedSlices(
tf.constant(grads0_np[grads0_np_indices]),
tf.constant(grads0_np_indices),
tf.constant([3]),
)
grads1_np_indices = np.array([0, 2], dtype=np.int32)
grads1 = tf.IndexedSlices(
tf.constant(grads1_np[grads1_np_indices]),
tf.constant(grads1_np_indices),
tf.constant([3]),
)
epsilon = 1e-7
optimizer = AdaBeliefOptimizer(epsilon=epsilon)
# Fetch params to validate initial values
np.testing.assert_allclose(np.asanyarray([1.0, 1.0, 2.0]),
var0.numpy())
np.testing.assert_allclose(np.asanyarray([3.0, 3.0, 4.0]),
var1.numpy())
# Run 3 steps of AdaBelief
for t in range(3):
beta_1_power, beta_2_power = get_beta_accumulators(
optimizer, dtype)
assert_allclose_according_to_type(0.9**(t + 1), beta_1_power)
assert_allclose_according_to_type(0.999**(t + 1), beta_2_power)
optimizer.apply_gradients(zip([grads0, grads1], [var0, var1]))
var0_np, m0, v0 = adabelief_update_numpy(var0_np,
grads0_np,
t,
m0,
v0,
epsilon=epsilon)
var1_np, m1, v1 = adabelief_update_numpy(var1_np,
grads1_np,
t,
m1,
v1,
epsilon=epsilon)
# Validate updated params
assert_allclose_according_to_type(var0_np, var0.numpy(), atol=2e-4)
assert_allclose_according_to_type(var1_np, var1.numpy(), atol=2e-4)
def test_basic_with_learning_rate_decay():
for i, dtype in enumerate(_dtypes_to_test(use_gpu=is_gpu_available())):
# Initialize variables for numpy implementation.
m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
var0 = tf.Variable(var0_np, name="var0_%d" % i)
var1 = tf.Variable(var1_np, name="var1_%d" % i)
grads0 = tf.constant(grads0_np)
grads1 = tf.constant(grads1_np)
learning_rate = 0.001
beta_1 = 0.9
beta_2 = 0.999
epsilon = 1e-14
decay = 0.5
weight_decay = 0.01
opt = AdaBeliefOptimizer(
learning_rate=learning_rate,
beta_1=beta_1,
beta_2=beta_2,
epsilon=epsilon,
weight_decay=weight_decay,
decay=decay,
)
# Run 3 steps of AdaBelief
for t in range(3):
opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
lr_np = learning_rate / (1 + decay * t)
var0_np, m0, v0 = adabelief_update_numpy(var0_np,
grads0_np,
t,
m0,
v0,
lr=lr_np,
weight_decay=weight_decay)
var1_np, m1, v1 = adabelief_update_numpy(var1_np,
grads1_np,
t,
m1,
v1,
lr=lr_np,
weight_decay=weight_decay)
# Validate updated params
assert_allclose_according_to_type(var0_np, var0.numpy(), atol=2e-4)
assert_allclose_according_to_type(var1_np, var1.numpy(), atol=2e-4)