def test_learning_policy_schedule(transformer_factory, drop_factor):
base_learning_rate = 1.0
schedule = [20, 100, 300, 750, 1000]
lr_params = {
'name': 'schedule',
'base_lr': base_learning_rate,
'gamma': drop_factor,
'schedule': schedule
}
iteration = ng.placeholder((), dtype=np.dtype(np.uint32))
lro = LearningRateOptimizer(learning_rate=lr_params, iteration=iteration)
schedule.append(np.inf)
np_schedule = np.array(schedule)
with ExecutorFactory() as ex:
scheduled_learning_rate = ex.transformer.computation(
lro.lrate, iteration)
for iter_input in np.random.randint(0, 1100, 5):
baseline_value = scheduled_learning_rate(iter_input)
max_step_ind = np.where(iter_input < np_schedule)[0][0]
if isinstance(drop_factor, list):
scale_factor = np.prod(drop_factor[:max_step_ind])
else:
scale_factor = drop_factor**max_step_ind
reference_value = base_learning_rate * scale_factor
assert ng.testing.allclose(baseline_value,
reference_value,
rtol=1e-5)
def test_learning_policy_step(transformer_factory):
base_learning_rate = 1.0
drop_factor = 0.1
step = 20
lr_params = {
'name': 'step',
'base_lr': base_learning_rate,
'gamma': drop_factor,
'step': step
}
iteration = ng.placeholder((), dtype=np.dtype(np.uint32))
lro = LearningRateOptimizer(learning_rate=lr_params, iteration=iteration)
with ExecutorFactory() as ex:
stepped_learning_rate = ex.transformer.computation(
lro.lrate, iteration)
for iter_input in [10, 50, 90, 6, 15]:
baseline_value = stepped_learning_rate(iter_input)
reference_value = base_learning_rate * (drop_factor
**(iter_input // step))
assert ng.testing.allclose(baseline_value,
reference_value,
rtol=1e-5)
def test_learning_policy_fixed_without_input():
base_learning_rate = 0.1
lro = LearningRateOptimizer(learning_rate=base_learning_rate)
with ExecutorFactory() as ex:
fixed_learning_rate = ex.transformer.computation(lro.lrate)
baseline_value = fixed_learning_rate()
ng.testing.assert_allclose(baseline_value, base_learning_rate, rtol=1e-6)
def test_learning_policy_fixed_with_input():
base_learning_rate = 0.1
iteration = ng.placeholder((), dtype=np.dtype(np.uint32))
lro = LearningRateOptimizer(learning_rate=base_learning_rate, iteration=iteration)
with ExecutorFactory() as ex:
fixed_learning_rate = ex.transformer.computation(lro.lrate, iteration)
for iter_input in [10, 50, 90, 6, 15]:
baseline_value = fixed_learning_rate(iter_input)
ng.testing.assert_allclose(baseline_value, base_learning_rate, rtol=1e-6)