def test_applies_norm(self): optimizer = extenders.clip_gradients_by_norm( training.GradientDescentOptimizer(1.0), clip_norm=3.) with ops.Graph().as_default(): w = variables.Variable(1., name='weight') x = constant_op.constant(5.) y = -x * w grads = optimizer.compute_gradients(y, var_list=[w])[0] opt_op = optimizer.minimize(y, var_list=[w]) with training.MonitoredSession() as sess: grads_value = sess.run(grads) self.assertEqual(-5., grads_value[0]) sess.run(opt_op) new_w = sess.run(w) self.assertEqual(4., new_w) # 1 + 1*3 (w - lr * clipped_grad)
def test_applies_norm(self): optimizer = extenders.clip_gradients_by_norm( training.GradientDescentOptimizer(1.0), clip_norm=3.) with ops.Graph().as_default(): w = variables.Variable(1., name='weight') x = constant_op.constant(5.) y = -x * w grads = optimizer.compute_gradients(y, var_list=[w])[0] opt_op = optimizer.minimize(y, var_list=[w]) with training.MonitoredSession() as sess: grads_value = sess.run(grads) self.assertEqual(-5., grads_value[0]) sess.run(opt_op) new_w = sess.run(w) self.assertEqual(4., new_w) # 1 + 1*3 (w - lr * clipped_grad)
def _rnn_model_fn(features, labels, mode, head, rnn_cell_fn, sequence_feature_columns, context_feature_columns, optimizer='Adagrad', input_layer_partitioner=None, config=None): """Recurrent Neural Net model_fn. Args: features: dict of `Tensor` and `SparseTensor` objects returned from `input_fn`. labels: `Tensor` of shape [batch_size, 1] or [batch_size] with labels. mode: Defines whether this is training, evaluation or prediction. See `ModeKeys`. head: A `head_lib._Head` instance. rnn_cell_fn: A function with one argument, a `tf.estimator.ModeKeys`, and returns an object of type `tf.nn.rnn_cell.RNNCell`. sequence_feature_columns: Iterable containing `FeatureColumn`s that represent sequential model inputs. context_feature_columns: Iterable containing `FeatureColumn`s that represent model inputs not associated with a specific timestep. optimizer: String, `tf.Optimizer` object, or callable that creates the optimizer to use for training. If not specified, will use the Adagrad optimizer with a default learning rate of 0.05 and gradient clip norm of 5.0. input_layer_partitioner: Partitioner for input layer. Defaults to `min_max_variable_partitioner` with `min_slice_size` 64 << 20. config: `RunConfig` object to configure the runtime settings. Returns: An `EstimatorSpec` instance. Raises: ValueError: If mode or optimizer is invalid, or features has the wrong type. """ if not isinstance(features, dict): raise ValueError('features should be a dictionary of `Tensor`s. ' 'Given type: {}'.format(type(features))) # If user does not provide an optimizer instance, use the optimizer specified # by the string with default learning rate and gradient clipping. if not isinstance(optimizer, optimizer_lib.Optimizer): optimizer = optimizers.get_optimizer_instance( optimizer, learning_rate=_DEFAULT_LEARNING_RATE) optimizer = extenders.clip_gradients_by_norm(optimizer, _DEFAULT_CLIP_NORM) num_ps_replicas = config.num_ps_replicas if config else 0 partitioner = partitioned_variables.min_max_variable_partitioner( max_partitions=num_ps_replicas) with variable_scope.variable_scope( 'rnn', values=tuple(six.itervalues(features)), partitioner=partitioner): input_layer_partitioner = input_layer_partitioner or ( partitioned_variables.min_max_variable_partitioner( max_partitions=num_ps_replicas, min_slice_size=64 << 20)) logit_fn = _rnn_logit_fn_builder( output_units=head.logits_dimension, rnn_cell_fn=rnn_cell_fn, sequence_feature_columns=sequence_feature_columns, context_feature_columns=context_feature_columns, input_layer_partitioner=input_layer_partitioner) logits = logit_fn(features=features, mode=mode) def _train_op_fn(loss): """Returns the op to optimize the loss.""" return optimizer.minimize( loss, global_step=training_util.get_global_step()) return head.create_estimator_spec( features=features, mode=mode, labels=labels, train_op_fn=_train_op_fn, logits=logits)
def test_name(self): optimizer = extenders.clip_gradients_by_norm( training.GradientDescentOptimizer(1.0), clip_norm=3.) self.assertEqual('ClipByNormGradientDescent', optimizer.get_name())
def test_name(self): optimizer = extenders.clip_gradients_by_norm( training.GradientDescentOptimizer(1.0), clip_norm=3.) self.assertEqual('ClipByNormGradientDescent', optimizer.get_name())
def _rnn_model_fn(features, labels, mode, head, rnn_cell_fn, sequence_feature_columns, context_feature_columns, optimizer='Adagrad', input_layer_partitioner=None, config=None): """Recurrent Neural Net model_fn. Args: features: dict of `Tensor` and `SparseTensor` objects returned from `input_fn`. labels: `Tensor` of shape [batch_size, 1] or [batch_size] with labels. mode: Defines whether this is training, evaluation or prediction. See `ModeKeys`. head: A `head_lib._Head` instance. rnn_cell_fn: A function with one argument, a `tf.estimator.ModeKeys`, and returns an object of type `tf.nn.rnn_cell.RNNCell`. sequence_feature_columns: Iterable containing `FeatureColumn`s that represent sequential model inputs. context_feature_columns: Iterable containing `FeatureColumn`s that represent model inputs not associated with a specific timestep. optimizer: String, `tf.Optimizer` object, or callable that creates the optimizer to use for training. If not specified, will use the Adagrad optimizer with a default learning rate of 0.05 and gradient clip norm of 5.0. input_layer_partitioner: Partitioner for input layer. Defaults to `min_max_variable_partitioner` with `min_slice_size` 64 << 20. config: `RunConfig` object to configure the runtime settings. Returns: An `EstimatorSpec` instance. Raises: ValueError: If mode or optimizer is invalid, or features has the wrong type. """ if not isinstance(features, dict): raise ValueError('features should be a dictionary of `Tensor`s. ' 'Given type: {}'.format(type(features))) # If user does not provide an optimizer instance, use the optimizer specified # by the string with default learning rate and gradient clipping. if not isinstance(optimizer, optimizer_lib.Optimizer): optimizer = optimizers.get_optimizer_instance( optimizer, learning_rate=_DEFAULT_LEARNING_RATE) optimizer = extenders.clip_gradients_by_norm(optimizer, _DEFAULT_CLIP_NORM) num_ps_replicas = config.num_ps_replicas if config else 0 partitioner = partitioned_variables.min_max_variable_partitioner( max_partitions=num_ps_replicas) with variable_scope.variable_scope('rnn', values=tuple(six.itervalues(features)), partitioner=partitioner): input_layer_partitioner = input_layer_partitioner or ( partitioned_variables.min_max_variable_partitioner( max_partitions=num_ps_replicas, min_slice_size=64 << 20)) logit_fn = _rnn_logit_fn_builder( output_units=head.logits_dimension, rnn_cell_fn=rnn_cell_fn, sequence_feature_columns=sequence_feature_columns, context_feature_columns=context_feature_columns, input_layer_partitioner=input_layer_partitioner) logits = logit_fn(features=features, mode=mode) def _train_op_fn(loss): """Returns the op to optimize the loss.""" return optimizer.minimize( loss, global_step=training_util.get_global_step()) return head.create_estimator_spec(features=features, mode=mode, labels=labels, train_op_fn=_train_op_fn, logits=logits)