def build_optimizers_tf_v1(optimizer_config, global_step=None):
"""Create a TF v1 compatible optimizer based on config.
Args:
optimizer_config: A Optimizer proto message.
global_step: A variable representing the current step.
If None, defaults to tf.train.get_or_create_global_step()
Returns:
An optimizer and a list of variables for summary.
Raises:
ValueError: when using an unsupported input data type.
"""
optimizer_type = optimizer_config.WhichOneof('optimizer')
optimizer = None
summary_vars = []
if optimizer_type == 'rms_prop_optimizer':
config = optimizer_config.rms_prop_optimizer
learning_rate = _create_learning_rate(config.learning_rate,
global_step=global_step)
summary_vars.append(learning_rate)
optimizer = tf.train.RMSPropOptimizer(
learning_rate,
decay=config.decay,
momentum=config.momentum_optimizer_value,
epsilon=config.epsilon)
if optimizer_type == 'momentum_optimizer':
config = optimizer_config.momentum_optimizer
learning_rate = _create_learning_rate(config.learning_rate,
global_step=global_step)
summary_vars.append(learning_rate)
optimizer = tf.train.MomentumOptimizer(
learning_rate,
momentum=config.momentum_optimizer_value)
if optimizer_type == 'adam_optimizer':
config = optimizer_config.adam_optimizer
learning_rate = _create_learning_rate(config.learning_rate,
global_step=global_step)
summary_vars.append(learning_rate)
optimizer = tf.train.AdamOptimizer(learning_rate, epsilon=config.epsilon)
if optimizer is None:
raise ValueError('Optimizer %s not supported.' % optimizer_type)
if optimizer_config.use_moving_average:
optimizer = tf_opt.MovingAverageOptimizer(
optimizer, average_decay=optimizer_config.moving_average_decay)
return optimizer, summary_vars
def create_moving_average_optimizer(optimizer, average_decay=0.999):
"""Creates a function that returns a configured MovingAverageOptimizer.
Args:
optimizer: The original tf.Optimizer.
average_decay: Exponentional decay factor for the variable averaging.
Returns:
A parameterless function that returns the configured Momentum Optimizer.
"""
return contrib_opt.MovingAverageOptimizer(optimizer,
average_decay=average_decay)
def get_real_optimizer(self, global_step=None):
"""Get real optimizer for faster-rcnn."""
if self.optimizer:
return self.optimizer, self.summary_vars
else:
if self.type == 'RMSPropOptimizer':
learning_rate = self._create_learning_rate(
self.lr, global_step=global_step)
self.summary_vars.append(learning_rate)
self.optimizer = tf.train.RMSPropOptimizer(
learning_rate,
decay=self.weight_decay,
momentum=self.momentum,
epsilon=self.epsilon)
if self.type == 'MomentumOptimizer':
learning_rate = self._create_learning_rate(
self.lr, global_step=global_step)
self.summary_vars.append(learning_rate)
self.optimizer = tf.train.MomentumOptimizer(
learning_rate, momentum=self.momentum)
if self.type == 'AdamOptimizer':
learning_rate = self._create_learning_rate(
self.lr, global_step=global_step)
self.summary_vars.append(learning_rate)
self.optimizer = tf.train.AdamOptimizer(learning_rate,
epsilon=self.epsilon)
if self.optimizer is None:
raise ValueError('Optimizer %s not supported.' % self.type)
if self.use_moving_average:
self.optimizer = tf_opt.MovingAverageOptimizer(
self.optimizer, average_decay=self.moving_average_decay)
return self.optimizer, self.summary_vars
def BuildOpt(hparams):
"""Constructs the optimizer.
Args:
hparams: An instance of tf.HParams, with these parameters:
- batch_size
- examples_per_epoch
- learning_rate
- learning_rate_decay_factor
- model_weights_averaging
- momentum
- num_epochs_per_decay
- optimizer
- rmsprop_decay
- use_avg_model_params
Returns:
opt: The optimizer.
"""
logging.info('Hyperparameters: %s', hparams)
batch_size = hparams.batch_size
examples_per_epoch = hparams.examples_per_epoch
learning_rate_decay_factor = hparams.learning_rate_decay_factor
learning_rate = hparams.learning_rate
model_weights_averaging = hparams.model_weights_averaging
momentum = hparams.momentum
num_epochs_per_decay = hparams.num_epochs_per_decay
optimizer = hparams.optimizer
rmsprop_decay = hparams.rmsprop_decay
rmsprop_epsilon = hparams.rmsprop_epsilon
adam_beta2 = hparams.get('adam_beta2', 0.999)
adam_epsilon = hparams.get('adam_epsilon', 1e-8)
use_avg_model_params = hparams.use_avg_model_params
global_step = tf.train.get_or_create_global_step()
# Configure the learning rate using an exponetial decay.
decay_steps = int(examples_per_epoch / batch_size *
num_epochs_per_decay)
learning_rate = tf.train.exponential_decay(
learning_rate,
global_step,
decay_steps,
learning_rate_decay_factor,
staircase=True)
if not tpu_function.get_tpu_context():
tf.summary.scalar('Learning Rate', learning_rate)
if optimizer == 'momentum':
opt = tf.train.MomentumOptimizer(learning_rate, momentum)
elif optimizer == 'rmsprop':
opt = tf.train.RMSPropOptimizer(
learning_rate,
decay=rmsprop_decay,
momentum=momentum,
epsilon=rmsprop_epsilon)
else:
opt = tf.train.AdamOptimizer(
learning_rate,
beta1=momentum,
beta2=adam_beta2,
epsilon=adam_epsilon)
if use_avg_model_params:
# Callers of BuildOpt() with use_avg_model_params=True expect the
# MovingAverageOptimizer to be the last optimizer returned by this function
# so that the swapping_saver can be constructed from it.
return contrib_opt.MovingAverageOptimizer(
opt, average_decay=model_weights_averaging)
return opt
