def build_optimizers_tf_v1(optimizer_config, global_step=None): """Create a TF v1 compatible optimizer based on config. Args: optimizer_config: A Optimizer proto message. global_step: A variable representing the current step. If None, defaults to tf.train.get_or_create_global_step() Returns: An optimizer and a list of variables for summary. Raises: ValueError: when using an unsupported input data type. """ optimizer_type = optimizer_config.WhichOneof('optimizer') optimizer = None summary_vars = [] if optimizer_type == 'rms_prop_optimizer': config = optimizer_config.rms_prop_optimizer learning_rate = _create_learning_rate(config.learning_rate, global_step=global_step) summary_vars.append(learning_rate) optimizer = tf.train.RMSPropOptimizer( learning_rate, decay=config.decay, momentum=config.momentum_optimizer_value, epsilon=config.epsilon) if optimizer_type == 'momentum_optimizer': config = optimizer_config.momentum_optimizer learning_rate = _create_learning_rate(config.learning_rate, global_step=global_step) summary_vars.append(learning_rate) optimizer = tf.train.MomentumOptimizer( learning_rate, momentum=config.momentum_optimizer_value) if optimizer_type == 'adam_optimizer': config = optimizer_config.adam_optimizer learning_rate = _create_learning_rate(config.learning_rate, global_step=global_step) summary_vars.append(learning_rate) optimizer = tf.train.AdamOptimizer(learning_rate, epsilon=config.epsilon) if optimizer is None: raise ValueError('Optimizer %s not supported.' % optimizer_type) if optimizer_config.use_moving_average: optimizer = tf_opt.MovingAverageOptimizer( optimizer, average_decay=optimizer_config.moving_average_decay) return optimizer, summary_vars
def create_moving_average_optimizer(optimizer, average_decay=0.999): """Creates a function that returns a configured MovingAverageOptimizer. Args: optimizer: The original tf.Optimizer. average_decay: Exponentional decay factor for the variable averaging. Returns: A parameterless function that returns the configured Momentum Optimizer. """ return contrib_opt.MovingAverageOptimizer(optimizer, average_decay=average_decay)
def get_real_optimizer(self, global_step=None): """Get real optimizer for faster-rcnn.""" if self.optimizer: return self.optimizer, self.summary_vars else: if self.type == 'RMSPropOptimizer': learning_rate = self._create_learning_rate( self.lr, global_step=global_step) self.summary_vars.append(learning_rate) self.optimizer = tf.train.RMSPropOptimizer( learning_rate, decay=self.weight_decay, momentum=self.momentum, epsilon=self.epsilon) if self.type == 'MomentumOptimizer': learning_rate = self._create_learning_rate( self.lr, global_step=global_step) self.summary_vars.append(learning_rate) self.optimizer = tf.train.MomentumOptimizer( learning_rate, momentum=self.momentum) if self.type == 'AdamOptimizer': learning_rate = self._create_learning_rate( self.lr, global_step=global_step) self.summary_vars.append(learning_rate) self.optimizer = tf.train.AdamOptimizer(learning_rate, epsilon=self.epsilon) if self.optimizer is None: raise ValueError('Optimizer %s not supported.' % self.type) if self.use_moving_average: self.optimizer = tf_opt.MovingAverageOptimizer( self.optimizer, average_decay=self.moving_average_decay) return self.optimizer, self.summary_vars
def BuildOpt(hparams): """Constructs the optimizer. Args: hparams: An instance of tf.HParams, with these parameters: - batch_size - examples_per_epoch - learning_rate - learning_rate_decay_factor - model_weights_averaging - momentum - num_epochs_per_decay - optimizer - rmsprop_decay - use_avg_model_params Returns: opt: The optimizer. """ logging.info('Hyperparameters: %s', hparams) batch_size = hparams.batch_size examples_per_epoch = hparams.examples_per_epoch learning_rate_decay_factor = hparams.learning_rate_decay_factor learning_rate = hparams.learning_rate model_weights_averaging = hparams.model_weights_averaging momentum = hparams.momentum num_epochs_per_decay = hparams.num_epochs_per_decay optimizer = hparams.optimizer rmsprop_decay = hparams.rmsprop_decay rmsprop_epsilon = hparams.rmsprop_epsilon adam_beta2 = hparams.get('adam_beta2', 0.999) adam_epsilon = hparams.get('adam_epsilon', 1e-8) use_avg_model_params = hparams.use_avg_model_params global_step = tf.train.get_or_create_global_step() # Configure the learning rate using an exponetial decay. decay_steps = int(examples_per_epoch / batch_size * num_epochs_per_decay) learning_rate = tf.train.exponential_decay( learning_rate, global_step, decay_steps, learning_rate_decay_factor, staircase=True) if not tpu_function.get_tpu_context(): tf.summary.scalar('Learning Rate', learning_rate) if optimizer == 'momentum': opt = tf.train.MomentumOptimizer(learning_rate, momentum) elif optimizer == 'rmsprop': opt = tf.train.RMSPropOptimizer( learning_rate, decay=rmsprop_decay, momentum=momentum, epsilon=rmsprop_epsilon) else: opt = tf.train.AdamOptimizer( learning_rate, beta1=momentum, beta2=adam_beta2, epsilon=adam_epsilon) if use_avg_model_params: # Callers of BuildOpt() with use_avg_model_params=True expect the # MovingAverageOptimizer to be the last optimizer returned by this function # so that the swapping_saver can be constructed from it. return contrib_opt.MovingAverageOptimizer( opt, average_decay=model_weights_averaging) return opt