def adam_w(learning_rate, hparams):
return contrib.opt().AdamWOptimizer(
weight_decay=hparams.weight_decay,
learning_rate=learning_rate,
beta1=hparams.optimizer_adam_beta1,
beta2=hparams.optimizer_adam_beta2,
epsilon=hparams.optimizer_adam_epsilon)
def adam(learning_rate, hparams):
# We change the default epsilon for Adam.
# Using LazyAdam as it's much faster for large vocabulary embeddings.
return contrib.opt().LazyAdamOptimizer(
learning_rate,
beta1=hparams.optimizer_adam_beta1,
beta2=hparams.optimizer_adam_beta2,
epsilon=hparams.optimizer_adam_epsilon)
def adam(learning_rate, hparams):
"""Return adam optimizer for the given params."""
# We change the default epsilon for Adam.
# Using LazyAdam as it's much faster for large vocabulary embeddings.
if contrib.is_tf2:
# in TF2 beta1 -> beta_1 :/
return contrib.opt().LazyAdamOptimizer(
learning_rate,
beta_1=hparams.optimizer_adam_beta1,
beta_2=hparams.optimizer_adam_beta2,
epsilon=hparams.optimizer_adam_epsilon)
else:
return contrib.opt().LazyAdamOptimizer(
learning_rate,
beta1=hparams.optimizer_adam_beta1,
beta2=hparams.optimizer_adam_beta2,
epsilon=hparams.optimizer_adam_epsilon)