def _create_optimizer(self):
""" Creates the optimizer. """
learning_rate = tf.get_variable(
Constants.LEARNING_RATE_VAR_NAME,
shape=(), dtype=tf.float32,
initializer=tf.constant_initializer(
value=self.params["optimizer.learning_rate"],
dtype=tf.float32),
trainable=False)
name = self.params["optimizer.name"]
tf.logging.info("use %s optimizer with initial learning rate=%f"
% (name, self.params["optimizer.learning_rate"]))
global_step_tensor = tf.train.get_or_create_global_step()
# create decay fn
decay_fn = create_learning_rate_decay_fn(**self.params["optimizer.lr_decay"])
other_tensor_dict = {}
if decay_fn: # apply learning rate decay
learning_rate, other_tensor_dict = decay_fn(learning_rate, global_step_tensor)
# add to collections
other_tensor_dict[Constants.LEARNING_RATE_VAR_NAME] = learning_rate
add_dict_to_collection(Constants.LEARNING_RATE_VAR_NAME, other_tensor_dict)
tf.add_to_collection(Constants.DISPLAY_KEY_COLLECTION_NAME, "training_stats/learning_rate")
tf.add_to_collection(Constants.DISPLAY_VALUE_COLLECTION_NAME, learning_rate)
# create optimizer
optimizer = _get_optimizer(name, learning_rate=learning_rate,
**self.params["optimizer.params"])
return optimizer
def _create_optimizer(self):
""" Creates the optimizer. """
learning_rate = tf.get_variable(
Constants.LEARNING_RATE_VAR_NAME,
shape=(), dtype=tf.float32,
initializer=tf.constant_initializer(
value=self.params["optimizer.learning_rate"],
dtype=tf.float32),
trainable=False)
name = self.params["optimizer.name"]
tf.logging.info("use %s optimizer with initial learning rate=%f"
% (name, self.params["optimizer.learning_rate"]))
global_step_tensor = tf.train.get_or_create_global_step()
# create decay fn
decay_fn = create_learning_rate_decay_fn(**self.params["optimizer.lr_decay"])
other_tensor_dict = {}
if decay_fn: # apply learning rate decay
learning_rate, other_tensor_dict = decay_fn(learning_rate, global_step_tensor)
# add to collections
other_tensor_dict[Constants.LEARNING_RATE_VAR_NAME] = learning_rate
add_dict_to_collection(Constants.LEARNING_RATE_VAR_NAME, other_tensor_dict)
tf.add_to_collection(Constants.DISPLAY_KEY_COLLECTION_NAME, "training_stats/learning_rate")
tf.add_to_collection(Constants.DISPLAY_VALUE_COLLECTION_NAME, learning_rate)
# create optimizer
optimizer = _get_optimizer(name, learning_rate=learning_rate,
**self.params["optimizer.params"])
return optimizer
def optimize(self, loss, variables=None, colocate_gradients_with_ops=False):
""" Creates the optimizer with learning rate decaying, optimizes
loss and return a train_op.
Args:
loss: The loss Tensor.
variables: A list of variables to optimize or None to use all trainable variables.
colocate_gradients_with_ops: Argument passed to
`tf.contrib.layers.optimize_loss`
Returns: The train_op.
"""
learning_rate = tf.get_variable(
Constants.LEARNING_RATE_VAR_NAME,
shape=(), dtype=tf.float32,
initializer=tf.constant_initializer(
value=self.params["optimizer.learning_rate"],
dtype=tf.float32),
trainable=False)
name = self.params["optimizer.name"]
tf.logging.info("use %s optimizer with initial learning rate=%f"
% (name, self.params["optimizer.learning_rate"]))
global_step_tensor = tf.train.get_or_create_global_step()
# create decay fn
decay_fn = create_learning_rate_decay_fn(**self.params["optimizer.lr_decay"])
other_tensor_dict = {}
if decay_fn: # apply learning rate decay
learning_rate, other_tensor_dict = decay_fn(learning_rate, global_step_tensor)
# add to collections
other_tensor_dict[Constants.LEARNING_RATE_VAR_NAME] = learning_rate
add_dict_to_collection(Constants.LEARNING_RATE_VAR_NAME, other_tensor_dict)
tf.add_to_collection(Constants.DISPLAY_KEY_COLLECTION_NAME, "training_stats/learning_rate")
tf.add_to_collection(Constants.DISPLAY_VALUE_COLLECTION_NAME, learning_rate)
# create optimizer
optimizer = _get_optimizer(name, learning_rate=learning_rate,
**self.params["optimizer.params"])
def _clip_gradients(grads_and_vars):
"""Clips gradients by global norm."""
gradients, variables = zip(*grads_and_vars)
clipped_gradients, _ = tf.clip_by_global_norm(
gradients, self.params["optimizer.clip_gradients"])
return list(zip(clipped_gradients, variables))
train_op = tf.contrib.layers.optimize_loss(
loss=loss,
global_step=global_step_tensor,
learning_rate=None, # self.params["optimizer.learning_rate"],
learning_rate_decay_fn=None,
clip_gradients=_clip_gradients if self.params["optimizer.clip_gradients"] > 0. else None,
variables=variables,
optimizer=optimizer,
summaries=["learning_rate", "loss"],
colocate_gradients_with_ops=colocate_gradients_with_ops)
return train_op