def get_optimizer(learning_rate, hparams): """Get the tf.train.Optimizer for this optimizer string. Args: learning_rate: The learning_rate tensor. hparams: tf.contrib.training.HParams object with the optimizer and momentum values. Returns: optimizer: The tf.train.Optimizer based on the optimizer string. """ return { "rmsprop": tf.RMSPropOptimizer(learning_rate, decay=0.95, momentum=hparams.momentum, epsilon=1e-4), "adam": tf.AdamOptimizer(learning_rate, beta1=0.9, beta2=0.999, epsilon=1e-8), "adagrad": tf.AdagradOptimizer(learning_rate, initial_accumulator_value=1.0), "mom": tf.MomentumOptimizer(learning_rate, momentum=hparams.momentum), "sgd": tf.GradientDescentOptimizer(learning_rate) }.get(hparams.optimizer)
def build_optimizer(name, learning_rate, momentum=0.9, epsilon=0.001, rmsprop_decay=0.9, sync_replicas=False, num_replicas=1, moving_average=None, variables_to_average=None): """Builds an optimizer.""" if name == 'momentum': optimizer = tf.train.MomentumOptimizer(learning_rate, momentum, use_nesterov=True) elif name == 'rmsprop': optimizer = tf.RMSPropOptimizer(learning_rate, rmsprop_decay, momentum=momentum, epsilon=epsilon) else: raise Exception('Unknown optimizer : {}'.format(name)) # If multiple workers are used for training, average the gradients from the # workers before updating. if sync_replicas: optimizer = tf.train.SyncReplicasOptimizer(optimizer, num_replicas, num_replicas, moving_average, variables_to_average) return optimizer
def create_optimizer(config, learning_rate): """Creates an Optimizer for training. Args: config: Object containing optimizer configuration parameters. learning_rate: Tensor; the learning rate. Returns: optimizer: An Optimizer for training. Raises: ValueError: If config.optimizer is unrecognized. """ if config.optimizer == "momentum": optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=config.momentum) elif config.optimizer == "sgd": optimizer = tf.train.GradientDescentOptimizer(learning_rate) elif config.optimizer == "adagrad": optimizer = tf.train.AdagradOptimizer(learning_rate) elif config.optimizer == "adam": optimizer = tf.train.AdamOptimizer(learning_rate, beta1=config.adam_beta1, beta2=config.adam_beta2, epsilon=config.adam_epsilon) elif config.optimizer == "rmsprop": optimizer = tf.RMSPropOptimizer(learning_rate) else: raise ValueError("Unknown optimizer: %s" % config.optimizer) return optimizer
def create_optimizer(optimizer_name, learning_rate, momentum=0.9, adam_beta1=0.9, adam_beta2=0.999, epsilon=0.001, rmsprop_decay=0.9, replicas_to_aggregate=None, ema_average_func=None, variables_to_average=None): """Return an optimizer by name.""" if optimizer_name == 'sgd': tf.logging.info('Using SGD optimizer') optimizer = tf.train.GradientDescentOptimizer( learning_rate=learning_rate) elif optimizer_name == 'momentum': tf.logging.info('Using Momentum optimizer') optimizer = tf.train.tf.train.MomentumOptimizer(learning_rate, momentum, use_nesterov=True) elif optimizer_name == 'rms': tf.logging.info('Using RMSPropOptimizer') optimizer = tf.RMSPropOptimizer(learning_rate, rmsprop_decay, momentum=momentum, epsilon=epsilon) else: assert False, 'Unknown optimizer: %s' % optimizer_name if replicas_to_aggregate > 1: ### To use SyncReplicasOptimizer with an Estimator, you need to send sync_replicas_hook while calling the fit. ### https://www.tensorflow.org/api_docs/python/tf/train/SyncReplicasOptimizer optimizer = tf.train.SyncReplicasOptimizer(optimizer, replicas_to_aggregate, replicas_to_aggregate, ema_average_func, variables_to_average) return optimizer
def create_optimizer(hparams, learning_rate, use_tpu=False): """Creates a TensorFlow Optimizer. Args: hparams: ConfigDict containing the optimizer configuration. learning_rate: A Python float or a scalar Tensor. use_tpu: If True, the returned optimizer is wrapped in a CrossShardOptimizer. Returns: A TensorFlow optimizer. Raises: ValueError: If hparams.optimizer is unrecognized. """ optimizer_name = hparams.optimizer.lower() if optimizer_name == "momentum": optimizer = tf.compat.v1.train.MomentumOptimizer( learning_rate, momentum=hparams.get("momentum", 0.9), use_nesterov=hparams.get("use_nesterov", False)) elif optimizer_name == "sgd": optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate) elif optimizer_name == "adagrad": optimizer = tf.compat.v1.train.AdagradOptimizer(learning_rate) elif optimizer_name == "adam": optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate) elif optimizer_name == "rmsprop": optimizer = tf.RMSPropOptimizer(learning_rate) else: raise ValueError("Unknown optimizer: %s" % hparams.optimizer) if use_tpu: optimizer = tf.compat.v1.tpu.CrossShardOptimizer(optimizer) return optimizer