def get_drop_fraction(self, global_step, is_mask_update_iter_op): """Returns a constant or annealing drop_fraction op.""" if self._drop_fraction_anneal == 'constant': drop_frac = self._drop_fraction_initial_value elif self._drop_fraction_anneal == 'cosine': decay_steps = self._end_step - self._begin_step drop_frac = learning_rate_decay.cosine_decay( self._drop_fraction_initial_value, global_step, decay_steps, name='cosine_drop_fraction') elif self._drop_fraction_anneal.startswith('exponential'): exponent = extract_number(self._drop_fraction_anneal) div_dtype = self._drop_fraction_initial_value.dtype power = math_ops.divide( math_ops.cast(global_step - self._begin_step, div_dtype), math_ops.cast(self._end_step - self._begin_step, div_dtype), ) drop_frac = math_ops.multiply(self._drop_fraction_initial_value, math_ops.pow(1 - power, exponent), name='%s_drop_fraction' % self._drop_fraction_anneal) else: raise ValueError('drop_fraction_anneal: %s is not valid' % self._drop_fraction_anneal) return array_ops.where(is_mask_update_iter_op, drop_frac, array_ops.zeros_like(drop_frac))
def _nested_func(global_step): return learning_rate_decay.cosine_decay( learning_rate=learning_rate, global_step=global_step, decay_steps=decay_steps, alpha=alpha, )
def testDecay(self): num_training_steps = 1000 initial_lr = 1.0 for step in range(0, 1500, 250): decayed_lr = learning_rate_decay.cosine_decay(initial_lr, step, num_training_steps) expected = self.np_cosine_decay(step, num_training_steps) self.assertAllClose(self.evaluate(decayed_lr), expected, 1e-6)
def testDecay(self): num_training_steps = 1000 initial_lr = 1.0 for step in range(0, 1500, 250): decayed_lr = learning_rate_decay.cosine_decay( initial_lr, step, num_training_steps) expected = self.np_cosine_decay(step, num_training_steps) self.assertAllClose(self.evaluate(decayed_lr), expected, 1e-6)
def testAlpha(self): num_training_steps = 1000 initial_lr = 1.0 alpha = 0.1 for step in range(0, 1500, 250): with self.test_session(): decayed_lr = learning_rate_decay.cosine_decay( initial_lr, step, num_training_steps, alpha) expected = self.np_cosine_decay(step, num_training_steps, alpha) self.assertAllClose(decayed_lr.eval(), expected, 1e-6)