def _learning_rate_default(self, multiply_by_parameter_scale):
step_num = tf.cast(tf.train.get_or_create_global_step(), tf.float32)
learning_rate = tf.minimum(tf.math.rsqrt(step_num + 1.0), 0.01)
if (not multiply_by_parameter_scale
and not layers.unit_scaling_convention()):
learning_rate *= 0.05
return learning_rate
def _learning_rate_default(self, multiply_by_parameter_scale):
learning_rate = tf.minimum(tf.math.rsqrt(step_num() + 1.0), 0.01)
if (not multiply_by_parameter_scale
and not layers.unit_scaling_convention()):
learning_rate *= 0.05
return learning_rate