Beispiel #1
0
def test_learning_policy_schedule(transformer_factory, drop_factor):
    base_learning_rate = 1.0
    schedule = [20, 100, 300, 750, 1000]

    lr_params = {
        'name': 'schedule',
        'base_lr': base_learning_rate,
        'gamma': drop_factor,
        'schedule': schedule
    }

    iteration = ng.placeholder((), dtype=np.dtype(np.uint32))
    lro = LearningRateOptimizer(learning_rate=lr_params, iteration=iteration)

    schedule.append(np.inf)
    np_schedule = np.array(schedule)

    with ExecutorFactory() as ex:
        scheduled_learning_rate = ex.transformer.computation(
            lro.lrate, iteration)

        for iter_input in np.random.randint(0, 1100, 5):
            baseline_value = scheduled_learning_rate(iter_input)
            max_step_ind = np.where(iter_input < np_schedule)[0][0]
            if isinstance(drop_factor, list):
                scale_factor = np.prod(drop_factor[:max_step_ind])
            else:
                scale_factor = drop_factor**max_step_ind
            reference_value = base_learning_rate * scale_factor
            assert ng.testing.allclose(baseline_value,
                                       reference_value,
                                       rtol=1e-5)
Beispiel #2
0
def test_learning_policy_step(transformer_factory):
    base_learning_rate = 1.0
    drop_factor = 0.1
    step = 20

    lr_params = {
        'name': 'step',
        'base_lr': base_learning_rate,
        'gamma': drop_factor,
        'step': step
    }

    iteration = ng.placeholder((), dtype=np.dtype(np.uint32))
    lro = LearningRateOptimizer(learning_rate=lr_params, iteration=iteration)

    with ExecutorFactory() as ex:
        stepped_learning_rate = ex.transformer.computation(
            lro.lrate, iteration)

        for iter_input in [10, 50, 90, 6, 15]:
            baseline_value = stepped_learning_rate(iter_input)
            reference_value = base_learning_rate * (drop_factor
                                                    **(iter_input // step))

            assert ng.testing.allclose(baseline_value,
                                       reference_value,
                                       rtol=1e-5)
def test_learning_policy_fixed_without_input():
    base_learning_rate = 0.1

    lro = LearningRateOptimizer(learning_rate=base_learning_rate)

    with ExecutorFactory() as ex:
        fixed_learning_rate = ex.transformer.computation(lro.lrate)
        baseline_value = fixed_learning_rate()
        ng.testing.assert_allclose(baseline_value, base_learning_rate, rtol=1e-6)
def test_learning_policy_fixed_with_input():
    base_learning_rate = 0.1

    iteration = ng.placeholder((), dtype=np.dtype(np.uint32))
    lro = LearningRateOptimizer(learning_rate=base_learning_rate, iteration=iteration)

    with ExecutorFactory() as ex:
        fixed_learning_rate = ex.transformer.computation(lro.lrate, iteration)

        for iter_input in [10, 50, 90, 6, 15]:
            baseline_value = fixed_learning_rate(iter_input)

            ng.testing.assert_allclose(baseline_value, base_learning_rate, rtol=1e-6)