def testStaircase(self):
    initial_lr = 0.1
    k = 10
    decay_rate = 0.96
    step = resource_variable_ops.ResourceVariable(0)
    decayed_lr = learning_rate_decay.natural_exp_decay(
        initial_lr, step, k, decay_rate, staircase=True)

    self.evaluate(variables.global_variables_initializer())
    for i in range(k + 1):
      expected = initial_lr * math.exp(-decay_rate * (i // k))
      self.assertAllClose(self.evaluate(decayed_lr), expected, 1e-6)
      self.evaluate(step.assign_add(1))
示例#2
0
    def testDecay(self):
        initial_lr = 0.1
        k = 10
        decay_rate = 0.96
        step = resource_variable_ops.ResourceVariable(0)
        decayed_lr = learning_rate_decay.natural_exp_decay(
            initial_lr, step, k, decay_rate)

        self.evaluate(variables.global_variables_initializer())
        for i in range(k + 1):
            expected = initial_lr * math.exp(-i / k * decay_rate)
            self.assertAllClose(self.evaluate(decayed_lr), expected, 1e-6)
            self.evaluate(step.assign_add(1))
 def testDecay(self):
   initial_lr = 0.1
   k = 10
   decay_rate = 0.96
   step = state_ops.variable_op([], dtypes.int32)
   assign_step = state_ops.assign(step, 0)
   increment_step = state_ops.assign_add(step, 1)
   decayed_lr = learning_rate_decay.natural_exp_decay(initial_lr, step,
                                                      k, decay_rate)
   with self.test_session():
     assign_step.op.run()
     for i in range(k+1):
       expected = initial_lr * math.exp(-i / k * decay_rate)
       self.assertAllClose(decayed_lr.eval(), expected, 1e-6)
       increment_step.op.run()
示例#4
0
 def testDecay(self):
     initial_lr = 0.1
     k = 10
     decay_rate = 0.96
     step = state_ops.variable_op([], dtypes.int32)
     assign_step = state_ops.assign(step, 0)
     increment_step = state_ops.assign_add(step, 1)
     decayed_lr = learning_rate_decay.natural_exp_decay(
         initial_lr, step, k, decay_rate)
     with self.test_session():
         assign_step.op.run()
         for i in range(k + 1):
             expected = initial_lr * math.exp(-i / k * decay_rate)
             self.assertAllClose(decayed_lr.eval(), expected, 1e-6)
             increment_step.op.run()
 def testStaircase(self):
   initial_lr = 0.1
   k = 10
   decay_rate = 0.96
   step = gen_state_ops._variable(shape=[], dtype=dtypes.int32,
       name="step", container="", shared_name="")
   assign_step = state_ops.assign(step, 0)
   increment_step = state_ops.assign_add(step, 1)
   decayed_lr = learning_rate_decay.natural_exp_decay(initial_lr,
                                                      step,
                                                      k,
                                                      decay_rate,
                                                      staircase=True)
   with self.test_session():
     assign_step.op.run()
     for i in range(k+1):
       expected = initial_lr * math.exp(-decay_rate * (i // k))
       self.assertAllClose(decayed_lr.eval(), expected, 1e-6)
       increment_step.op.run()
 def testStaircase(self):
   initial_lr = 0.1
   k = 10
   decay_rate = 0.96
   step = gen_state_ops._variable(shape=[], dtype=dtypes.int32,
       name="step", container="", shared_name="")
   assign_step = state_ops.assign(step, 0)
   increment_step = state_ops.assign_add(step, 1)
   decayed_lr = learning_rate_decay.natural_exp_decay(initial_lr,
                                                      step,
                                                      k,
                                                      decay_rate,
                                                      staircase=True)
   with self.test_session():
     assign_step.op.run()
     for i in range(k+1):
       expected = initial_lr * math.exp(-decay_rate * (i // k))
       self.assertAllClose(decayed_lr.eval(), expected, 1e-6)
       increment_step.op.run()
示例#7
0
def apply_lr_decay(cfg, global_step):
    # Learning rate schedule
    if cfg.lr_decay is None:
        lr = cfg.lr
    elif cfg.lr_decay == 'exp':
        lr = exponential_decay(cfg.lr,
                               global_step,
                               cfg.decay_steps,
                               cfg.decay_rate,
                               staircase=cfg.staircase)
    elif cfg.lr_decay == 'piecewise':
        lr = piecewise_constant(global_step, cfg.lr_boundaries, cfg.lr_values)
    elif cfg.lr_decay == 'polynomial':
        lr = polynomial_decay(cfg.lr,
                              global_step,
                              cfg.decay_steps,
                              end_learning_rate=cfg.end_lr,
                              power=cfg.power,
                              cycle=cfg.staircase)

    elif cfg.lr_decay == 'natural_exp':
        lr = natural_exp_decay(cfg.lr,
                               global_step,
                               cfg.decay_steps,
                               cfg.decay_rate,
                               staircase=cfg.staircase)
    elif cfg.lr_decay == 'inverse_time':
        lr = inverse_time_decay(cfg.lr,
                                global_step,
                                cfg.decay_steps,
                                cfg.decay_rate,
                                staircase=cfg.staircase)

    elif cfg.lr_decay == 'STN':
        epoch = tf.cast(global_step / cfg.decay_steps, tf.int32)
        lr = cfg.lr * tf.pow(0.5, tf.cast(epoch / 50, cfg._FLOATX))
    else:
        raise NotImplementedError()
    return lr