def get_optimizer(learning_rate, hparams):
"""Get the tf.train.Optimizer for this optimizer string.
Args:
learning_rate: The learning_rate tensor.
hparams: tf.contrib.training.HParams object with the optimizer and
momentum values.
Returns:
optimizer: The tf.train.Optimizer based on the optimizer string.
"""
return {
"rmsprop":
tf.RMSPropOptimizer(learning_rate,
decay=0.95,
momentum=hparams.momentum,
epsilon=1e-4),
"adam":
tf.AdamOptimizer(learning_rate, beta1=0.9, beta2=0.999, epsilon=1e-8),
"adagrad":
tf.AdagradOptimizer(learning_rate, initial_accumulator_value=1.0),
"mom":
tf.MomentumOptimizer(learning_rate, momentum=hparams.momentum),
"sgd":
tf.GradientDescentOptimizer(learning_rate)
}.get(hparams.optimizer)
def build_optimizer(name,
learning_rate,
momentum=0.9,
epsilon=0.001,
rmsprop_decay=0.9,
sync_replicas=False,
num_replicas=1,
moving_average=None,
variables_to_average=None):
"""Builds an optimizer."""
if name == 'momentum':
optimizer = tf.train.MomentumOptimizer(learning_rate,
momentum,
use_nesterov=True)
elif name == 'rmsprop':
optimizer = tf.RMSPropOptimizer(learning_rate,
rmsprop_decay,
momentum=momentum,
epsilon=epsilon)
else:
raise Exception('Unknown optimizer : {}'.format(name))
# If multiple workers are used for training, average the gradients from the
# workers before updating.
if sync_replicas:
optimizer = tf.train.SyncReplicasOptimizer(optimizer, num_replicas,
num_replicas,
moving_average,
variables_to_average)
return optimizer
def create_optimizer(config, learning_rate):
"""Creates an Optimizer for training.
Args:
config: Object containing optimizer configuration parameters.
learning_rate: Tensor; the learning rate.
Returns:
optimizer: An Optimizer for training.
Raises:
ValueError: If config.optimizer is unrecognized.
"""
if config.optimizer == "momentum":
optimizer = tf.train.MomentumOptimizer(learning_rate,
momentum=config.momentum)
elif config.optimizer == "sgd":
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
elif config.optimizer == "adagrad":
optimizer = tf.train.AdagradOptimizer(learning_rate)
elif config.optimizer == "adam":
optimizer = tf.train.AdamOptimizer(learning_rate,
beta1=config.adam_beta1,
beta2=config.adam_beta2,
epsilon=config.adam_epsilon)
elif config.optimizer == "rmsprop":
optimizer = tf.RMSPropOptimizer(learning_rate)
else:
raise ValueError("Unknown optimizer: %s" % config.optimizer)
return optimizer
def create_optimizer(optimizer_name,
learning_rate,
momentum=0.9,
adam_beta1=0.9,
adam_beta2=0.999,
epsilon=0.001,
rmsprop_decay=0.9,
replicas_to_aggregate=None,
ema_average_func=None,
variables_to_average=None):
"""Return an optimizer by name."""
if optimizer_name == 'sgd':
tf.logging.info('Using SGD optimizer')
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=learning_rate)
elif optimizer_name == 'momentum':
tf.logging.info('Using Momentum optimizer')
optimizer = tf.train.tf.train.MomentumOptimizer(learning_rate,
momentum,
use_nesterov=True)
elif optimizer_name == 'rms':
tf.logging.info('Using RMSPropOptimizer')
optimizer = tf.RMSPropOptimizer(learning_rate,
rmsprop_decay,
momentum=momentum,
epsilon=epsilon)
else:
assert False, 'Unknown optimizer: %s' % optimizer_name
if replicas_to_aggregate > 1:
### To use SyncReplicasOptimizer with an Estimator, you need to send sync_replicas_hook while calling the fit.
### https://www.tensorflow.org/api_docs/python/tf/train/SyncReplicasOptimizer
optimizer = tf.train.SyncReplicasOptimizer(optimizer,
replicas_to_aggregate,
replicas_to_aggregate,
ema_average_func,
variables_to_average)
return optimizer
def create_optimizer(hparams, learning_rate, use_tpu=False):
"""Creates a TensorFlow Optimizer.
Args:
hparams: ConfigDict containing the optimizer configuration.
learning_rate: A Python float or a scalar Tensor.
use_tpu: If True, the returned optimizer is wrapped in a
CrossShardOptimizer.
Returns:
A TensorFlow optimizer.
Raises:
ValueError: If hparams.optimizer is unrecognized.
"""
optimizer_name = hparams.optimizer.lower()
if optimizer_name == "momentum":
optimizer = tf.compat.v1.train.MomentumOptimizer(
learning_rate,
momentum=hparams.get("momentum", 0.9),
use_nesterov=hparams.get("use_nesterov", False))
elif optimizer_name == "sgd":
optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate)
elif optimizer_name == "adagrad":
optimizer = tf.compat.v1.train.AdagradOptimizer(learning_rate)
elif optimizer_name == "adam":
optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate)
elif optimizer_name == "rmsprop":
optimizer = tf.RMSPropOptimizer(learning_rate)
else:
raise ValueError("Unknown optimizer: %s" % hparams.optimizer)
if use_tpu:
optimizer = tf.compat.v1.tpu.CrossShardOptimizer(optimizer)
return optimizer
