def test_exp():
from eight_mile.tf import optz
tf.compat.v1.reset_default_graph()
sess = tf.compat.v1.Session()
lr_sched = create_lr_scheduler(**EXP_LR_CONFIG)
bl_exp = ExponentialDecayScheduler(**EXP_LR_CONFIG)
decay_rate = EXP_LR_CONFIG["decay_rate"]
lr_var = tf.compat.v1.placeholder(tf.float32, shape=(), name="lr")
step_var = tf.compat.v1.placeholder(tf.int32, shape=(), name="step")
gph = lr_sched(lr_var, step_var)
sess.run(tf.compat.v1.global_variables_initializer())
lrs = []
lrs_bl = []
for step in range(NUM_STEPS):
lr = sess.run(gph, feed_dict={lr_var: INIT_LR, step_var: step})
lrs += [lr]
lr_bl = bl_exp(step)
lrs_bl += [lr_bl]
inv_times = [(INIT_LR * decay_rate**(t / 100.0)) for t in range(NUM_STEPS)]
assert np.allclose(inv_times, lrs)
assert np.allclose(inv_times, lrs_bl)
def test_staircase_value():
sd = ExponentialDecayScheduler(1000, 0.9, lr=1.0, staircase=True)
gold = 1.0
test = sd(100)
np.testing.assert_allclose(test, gold)
gold = 0.9
test = sd(1001)
np.testing.assert_allclose(test, gold)
def test_exp_values():
sd = ExponentialDecayScheduler(1000, 0.9, lr=1.0)
gold = 0.9895192582062144
test = sd(100)
np.testing.assert_allclose(test, gold)
gold = 0.8999051805311098
test = sd(1001)
np.testing.assert_allclose(test, gold)
def test_staircase_decay_flat():
steps = np.random.randint(900, 1001)
sd = ExponentialDecayScheduler(steps,
np.random.rand(),
lr=np.random.rand(),
staircase=True)
stair_one_one = sd(np.random.randint(steps - 100, steps))
stair_one_two = sd(np.random.randint(steps - 100, steps))
stair_two = sd(np.random.randint(steps + 1, steps + 10))
assert stair_one_one == stair_one_two
assert stair_one_two != stair_two
def test_exp():
from eight_mile.tf import optz
lr_sched = create_lr_scheduler(**EXP_LR_CONFIG)
bl_exp = ExponentialDecayScheduler(**EXP_LR_CONFIG)
decay_rate = EXP_LR_CONFIG["decay_rate"]
lrs = []
lrs_bl = []
for step in range(NUM_STEPS):
lr = lr_sched(step)
lrs += [lr]
lr_bl = bl_exp(step)
lrs_bl += [lr_bl]
inv_times = [(INIT_LR * decay_rate**(t / 100.0)) for t in range(NUM_STEPS)]
assert np.allclose(inv_times, lrs)
assert np.allclose(inv_times, lrs_bl)