def test_adam8p_dense(self):
opt = adam8p.Adam8POptimizer(learning_rate=1e-2)
v = tf.get_variable(name='weights',
shape=[10],
dtype=tf.float32,
initializer=tf.initializers.random_normal())
loss = tf.reduce_mean(v**2)
global_step = tf.train.get_or_create_global_step()
train_op = opt.minimize(loss, var_list=[v], global_step=global_step)
with self.cached_session() as sess:
sess.run(tf.initializers.global_variables())
initial_loss = sess.run(loss)
for _ in range(100):
sess.run(train_op)
final_loss = sess.run(loss)
self.assertLess(final_loss, 0.20804106)
self.assertGreater(initial_loss, 0.50804106)
def test_adam8p_sparse(self):
opt = adam8p.Adam8POptimizer(learning_rate=1e-2)
dense_var = tf.get_variable(
name='weights',
shape=[20, 20],
dtype=tf.float32,
initializer=tf.initializers.random_normal())
loss = tf.reduce_mean(
tf.nn.embedding_lookup(dense_var, np.arange(5, dtype=np.int32))**2)
global_step = tf.train.get_or_create_global_step()
train_op = opt.minimize(loss,
var_list=[dense_var],
global_step=global_step)
with self.cached_session() as sess:
sess.run(tf.initializers.global_variables())
initial_loss = sess.run(loss)
for _ in range(100):
sess.run(train_op)
final_loss = sess.run(loss)
print(final_loss, initial_loss)
self.assertLess(final_loss, 0.4)
self.assertGreater(initial_loss, 1.0)