def test_loss_args(self):
"""Tests that variables are flattened and passed to static head's method."""
logits = [[1, 2], [3, 4]]
labels = [[0, 1], [0, 2]]
features = {
'weights': [[0.3, 0.2], [0.5, 100]],
'mask': [[1, 1], [1, 0]]
}
head = seq_head_lib.SequentialHeadWrapper(_MockHead(), 'mask',
'weights')
expected_output = {
'logits': [[1], [2], [3]],
'labels': [[0], [1], [0]],
'features': {
'weights': [[0.3], [0.2], [0.5]]
},
'mode': 'my-mode',
'regularization_losses': 123
}
output = head.loss(logits=_convert_to_tensor(logits),
labels=_convert_to_tensor(labels),
features=_convert_to_tensor(features),
mode='my-mode',
regularization_losses=123)
with self.cached_session() as sess:
self._assert_equal(output, dref=expected_output, session=sess)
def test_wrong_feature_column_type_in_iterable(self):
"""Tests error raised when the feature column doesn't have proper type."""
with self.assertRaisesRegexp(TypeError,
'Column must a string. Given type: .*.'):
_ = seq_head_lib.SequentialHeadWrapper(None,
'mask',
feature_columns=[1])
def test_create_estimator_spec_args(self):
"""Tests that variables are flattened and passed to static head's method."""
logits = [[1, 2], [3, 4]]
labels = [[0, 1], [0, 2]]
features = {
'weights': [[0.3, 0.2], [0.5, 100]],
'mask': [[1, 1], [1, 0]]
}
head = seq_head_lib.SequentialHeadWrapper(_MockHead(), 'mask',
'weights')
expected_output = {
'logits': [[1], [2], [3]],
'labels': [[0], [1], [0]],
'features': {
'weights': [[0.3], [0.2], [0.5]]
},
'mode': ModeKeys.TRAIN,
'regularization_losses': 123,
'optimizer': 'my-opt',
'train_op_fn': 'my-train-op'
}
spec = head.create_estimator_spec(
logits=_convert_to_tensor(logits),
labels=_convert_to_tensor(labels),
features=_convert_to_tensor(features),
mode=ModeKeys.TRAIN,
optimizer='my-opt',
train_op_fn='my-train-op',
regularization_losses=123)
with self.cached_session() as sess:
self.assertItemsEqual(spec.kwargs.keys(), expected_output.keys())
self._assert_equal(spec.kwargs, dref=expected_output, session=sess)
def test_multi_head_provided(self):
"""Tests error raised when a multi-head is provided."""
with self.assertRaisesRegexp(
ValueError,
'`MultiHead` is not supported with `SequentialHeadWrapper`.'):
_ = seq_head_lib.SequentialHeadWrapper(
multi_head.MultiHead(
[binary_head_lib.BinaryClassHead(name='test-head')]))
def test_wrong_feature_column_type(self):
"""Tests error raised when the feature column doesn't have proper type."""
with self.assertRaisesRegexp(
TypeError,
'`feature_columns` must be either a string or an iterable'):
_ = seq_head_lib.SequentialHeadWrapper(None,
'mask',
feature_columns=1)
def test_loss_reduction(self):
"""Tests loss reduction.
Use `loss` method in eager execution, else `create_estimator_spec` in TRAIN
mode.
logits = [[[2., 3., 4.], [5., -0.5, 0.]],
[[-1.0, 2.0, 0.5], [_]]],
labels = [[0, 1],
[2, _]]
weights = [[0.5, 0.2],
[0.3, _]]
loss = [0.5*2.40 + 0.2*5.41 + 0.3*1.74] / 3 = 0.94
"""
static_head = multi_head_lib.MultiClassHead(
n_classes=3, weight_column='weights')
head = seq_head_lib.SequentialHeadWrapper(static_head, 'sequence_mask',
'weights')
expected_loss = 0.942783
features = {
'weights':
tf.sparse.SparseTensor(
indices=((0, 0), (0, 1), (1, 0)),
values=(0.5, 0.2, 0.3),
dense_shape=(2, 2)),
'sequence_mask':
ops.convert_to_tensor([[1, 1], [1, 0]])
}
logits = ops.convert_to_tensor([[[2., 3., 4.], [5., -0.5, 0.]],
[[-1.0, 2.0, 0.5], [1.0, 0.5, 2.0]]])
labels = tf.sparse.SparseTensor(
indices=((0, 0), (0, 1), (1, 0)), values=(0, 1, 2), dense_shape=(2, 2))
class _Optimizer(tf.keras.optimizers.Optimizer):
def get_updates(self, loss, params):
del params, loss
return [tf.constant('op')]
def get_config(self):
config = super(_Optimizer, self).get_config()
return config
if tf.executing_eagerly():
loss = head.loss(logits=logits, labels=labels, features=features)
else:
spec = head.create_estimator_spec(
features,
ModeKeys.TRAIN,
logits,
labels=labels,
optimizer=_Optimizer('my_optimizer'),
trainable_variables=[
tf.Variable([1.0, 2.0], dtype=tf.dtypes.float32)
])
with self.cached_session() as sess:
loss = sess.run(spec.loss)
self.assertAllClose(loss, expected_loss, atol=1e-4)
def test_head_properties(self):
"""Tests that the head's properties are correcly implemented."""
static_head = binary_head_lib.BinaryClassHead(
loss_reduction=losses_utils.ReductionV2.SUM, name='a_static_head')
head = seq_head_lib.SequentialHeadWrapper(static_head,
'a_sequence_mask_col')
self.assertEqual(head.name, 'a_static_head_sequential')
self.assertEqual(head.logits_dimension, 1)
self.assertEqual(head.loss_reduction, losses_utils.ReductionV2.SUM)
self.assertEqual(head.input_sequence_mask_key, 'a_sequence_mask_col')
self.assertEqual(head.static_head.name, 'a_static_head')
def test_metrics(self):
"""Tests the `metrics` method.
Tests that:
- Returned metrics match the returned metrics of the static head.
- `regularization_losses` argument is properly passed to the static head's
method.
"""
head = seq_head_lib.SequentialHeadWrapper(
binary_head_lib.BinaryClassHead(), 'mask')
metrics = head.metrics(regularization_losses=2.5)
keys = metric_keys.MetricKeys
self.assertIn(keys.ACCURACY, metrics)
self.assertIn(keys.LOSS_REGULARIZATION, metrics)
def test_loss_reduction(self):
"""Tests loss reduction.
Use `loss` method in eager execution, else `create_estimator_spec` in TRAIN
mode.
logits = [[[2., 3., 4.], [5., -0.5, 0.]],
[[-1.0, 2.0, 0.5], [_]]],
labels = [[0, 1],
[2, _]]
weights = [[0.5, 0.2],
[0.3, _]]
loss = [0.5*2.40 + 0.2*5.41 + 0.3*1.74] / 3 = 0.94
"""
static_head = multi_head_lib.MultiClassHead(n_classes=3,
weight_column='weights')
head = seq_head_lib.SequentialHeadWrapper(static_head, 'sequence_mask',
'weights')
expected_loss = 0.942783
features = {
'weights':
sparse_tensor.SparseTensor(indices=((0, 0), (0, 1), (1, 0)),
values=(0.5, 0.2, 0.3),
dense_shape=(2, 2)),
'sequence_mask':
ops.convert_to_tensor([[1, 1], [1, 0]])
}
logits = ops.convert_to_tensor([[[2., 3., 4.], [5., -0.5, 0.]],
[[-1.0, 2.0, 0.5], [1.0, 0.5, 2.0]]])
labels = sparse_tensor.SparseTensor(indices=((0, 0), (0, 1), (1, 0)),
values=(0, 1, 2),
dense_shape=(2, 2))
class _Optimizer(object):
def minimize(self, loss, global_step):
del global_step, loss
return constant_op.constant('op')
if context.executing_eagerly():
loss = head.loss(logits=logits, labels=labels, features=features)
else:
spec = head.create_estimator_spec(features,
ModeKeys.TRAIN,
logits,
labels,
optimizer=_Optimizer())
with self.cached_session() as sess:
loss = sess.run(spec.loss)
self.assertAllClose(loss, expected_loss, atol=1e-4)
def test_predictions(self):
"""Tests predictions output.
Use `predictions` method in eager execution, else `create_estimator_spec` in
PREDICT mode.
logits = [[0.3, -0.4], [0.2, 0.2]]
logistics = 1 / (1 + exp(-logits))
= [[0.57, 0.40], [0.55, 0.55]]
"""
head = seq_head_lib.SequentialHeadWrapper(
binary_head_lib.BinaryClassHead(), 'sequence_mask')
logits = [[[0.3], [-0.4]], [[0.2], [0.2]]]
expected_logistics = [[[0.574443], [0.401312]], [[0.549834],
[0.549834]]]
features = {
'sequence_mask': ops.convert_to_tensor(np.array([[1, 1], [1, 0]]))
}
keys = prediction_keys.PredictionKeys
if tf.executing_eagerly():
predictions = head.predictions(logits=logits,
keys=[keys.LOGITS, keys.LOGISTIC])
self.assertItemsEqual(predictions.keys(),
[keys.LOGITS, keys.LOGISTIC])
self.assertAllClose(logits, predictions[keys.LOGITS])
self.assertAllClose(expected_logistics, predictions[keys.LOGISTIC])
return
spec = head.create_estimator_spec(features=features,
mode=ModeKeys.PREDICT,
logits=logits,
trainable_variables=[
tf.Variable(
[1.0, 2.0],
dtype=tf.dtypes.float32)
])
self.assertIn('sequence_mask', spec.predictions)
with self.cached_session() as sess:
self.assertAllEqual(sess.run(spec.predictions['sequence_mask']),
features['sequence_mask'])
self.assertAllClose(logits,
sess.run(spec.predictions[keys.LOGITS]))
self.assertAllClose(expected_logistics,
sess.run(spec.predictions[keys.LOGISTIC]))
def _test_flatten_method(self, features, feature_columns):
"""Runs seq head's `_flatten` method and returns output for testing."""
head = seq_head_lib.SequentialHeadWrapper(
static_head=None,
sequence_length_mask='sequence_mask',
feature_columns=feature_columns)
labels = {
'indices': ((0, 0), (0, 1), (1, 0)),
'values': (1, 2, 3),
'dense_shape': (2, 2)
}
logits = np.array([[[10], [11]], [[12], [13]]])
features = _convert_to_tensor(features)
labels = sparse_tensor.SparseTensor(**labels)
logits = ops.convert_to_tensor(logits)
output = head._flatten(labels, logits, features)
if context.executing_eagerly():
return output
with self.cached_session() as sess:
return sess.run(output)
def test_wrong_mask_type(self):
"""Tests error raised when the mask doesn't have proper type."""
with self.assertRaisesRegexp(
TypeError, '`sequence_mask` column must be a string.'):
_ = seq_head_lib.SequentialHeadWrapper(None,
sequence_length_mask=1)
def test_metrics_computation(self):
"""Runs metrics computation tests.
Use `update_metrics` method in eager execution, else `create_estimator_spec`
in EVAL mode.
logits = [[-101, 102, -103], [104, _, _]]
predicted_labels = [[0, 1, 0], [1, _, _]]
labels = [[1, 1, 1], [1, _, _]]
weights = [[2, 5, 1], [2, _, _]]
loss = (101*2 + 103*1) / 10 = 30.5
accuracy = (0 + 5 + 0 + 2) / (2 + 5 + 1 + 2) = 0.7
prediction_mean = (0 + 5 + 0 + 2) / (2 + 5 + 1 + 2) = 0.7
precision = (5 + 2) / (5 + 2) = 1.0
recall = (5 + 2) / (2 + 5 + 1 + 2) = 0.7
"""
static_head = binary_head_lib.BinaryClassHead(weight_column='weights')
head = seq_head_lib.SequentialHeadWrapper(static_head, 'sequence_mask',
'weights')
features = {
'sequence_mask': np.array([[1, 1, 1], [1, 0, 0]]),
'weights': np.array([[2, 5, 1], [2, 100, 100]])
}
regularization_losses = [100.]
logits = _convert_to_tensor([[-101, 102, -103], [104, 100, 100]])
labels = sparse_tensor.SparseTensor(values=[1, 1, 1, 1],
indices=((0, 0), (0, 1), (0, 2),
(1, 0)),
dense_shape=(2, 3))
features = _convert_to_tensor(features)
expected_loss = 30.5
keys = metric_keys.MetricKeys
expected_metrics = {
keys.LOSS_MEAN: expected_loss,
keys.ACCURACY: 0.7,
keys.PREDICTION_MEAN: 0.7,
keys.LABEL_MEAN: 1.0,
keys.LOSS_REGULARIZATION: 100,
keys.PRECISION: 1.0,
keys.RECALL: 0.7,
keys.ACCURACY_BASELINE: 1.0,
keys.AUC: 0.,
keys.AUC_PR: 1.0
}
if context.executing_eagerly():
eval_metrics = head.metrics(
regularization_losses=regularization_losses)
updated_metrics = head.update_metrics(eval_metrics, features,
logits, labels,
regularization_losses)
self.assertItemsEqual(expected_metrics.keys(),
updated_metrics.keys())
self.assertAllClose(
expected_metrics,
{k: updated_metrics[k].result()
for k in updated_metrics})
return
spec = head.create_estimator_spec(
features=features,
mode=ModeKeys.EVAL,
logits=logits,
labels=labels,
regularization_losses=regularization_losses)
with self.cached_session() as sess:
head_utils._initialize_variables(self, spec.scaffold)
self.assertIsNone(spec.scaffold.summary_op)
value_ops = {
k: spec.eval_metric_ops[k][0]
for k in spec.eval_metric_ops
}
update_ops = {
k: spec.eval_metric_ops[k][1]
for k in spec.eval_metric_ops
}
_ = sess.run(update_ops)
self.assertAllClose(expected_metrics,
{k: value_ops[k].eval()
for k in value_ops})
def __init__(self,
sequence_feature_columns,
context_feature_columns=None,
units=None,
cell_type=USE_DEFAULT,
rnn_cell_fn=None,
return_sequences=False,
model_dir=None,
n_classes=2,
weight_column=None,
label_vocabulary=None,
optimizer='Adagrad',
loss_reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE,
sequence_mask='sequence_mask',
config=None):
"""Initializes a `RNNClassifier` instance.
Args:
sequence_feature_columns: An iterable containing the `FeatureColumn`s
that represent sequential input. All items in the set should either be
sequence columns (e.g. `sequence_numeric_column`) or constructed from
one (e.g. `embedding_column` with `sequence_categorical_column_*` as
input).
context_feature_columns: An iterable containing the `FeatureColumn`s
for contextual input. The data represented by these columns will be
replicated and given to the RNN at each timestep. These columns must be
instances of classes derived from `DenseColumn` such as
`numeric_column`, not the sequential variants.
units: Iterable of integer number of hidden units per RNN layer. If
set, `cell_type` must also be specified and `rnn_cell_fn` must be
`None`.
cell_type: A class producing a RNN cell or a string specifying the cell
type. Supported strings are: `'simple_rnn'`, `'lstm'`, and `'gru'`. If
set, `units` must also be specified and `rnn_cell_fn` must be `None`.
rnn_cell_fn: A function that returns a RNN cell instance that will be used
to construct the RNN. If set, `units` and `cell_type` cannot be set.
This is for advanced users who need additional customization beyond
`units` and `cell_type`. Note that `tf.keras.layers.StackedRNNCells` is
needed for stacked RNNs.
return_sequences: A boolean indicating whether to return the last output
in the output sequence, or the full sequence. Note that if True,
`weight_column` must be None or a string.
model_dir: Directory to save model parameters, graph and etc. This can
also be used to load checkpoints from the directory into a estimator to
continue training a previously saved model.
n_classes: Number of label classes. Defaults to 2, namely binary
classification. Must be > 1.
weight_column: A string or a `NumericColumn` created by
`tf.feature_column.numeric_column` defining feature column representing
weights. It is used to down weight or boost examples during training. It
will be multiplied by the loss of the example. If it is a string, it is
used as a key to fetch weight tensor from the `features`. If it is a
`NumericColumn`, raw tensor is fetched by key `weight_column.key`, then
weight_column.normalizer_fn is applied on it to get weight tensor.
label_vocabulary: A list of strings represents possible label values. If
given, labels must be string type and have any value in
`label_vocabulary`. If it is not given, that means labels are
already encoded as integer or float within [0, 1] for `n_classes=2` and
encoded as integer values in {0, 1,..., n_classes-1} for `n_classes`>2 .
Also there will be errors if vocabulary is not provided and labels are
string.
optimizer: An instance of `tf.Optimizer` or string specifying optimizer
type. Defaults to Adagrad optimizer.
loss_reduction: One of `tf.losses.Reduction` except `NONE`. Describes how
to reduce training loss over batch. Defaults to `SUM_OVER_BATCH_SIZE`.
sequence_mask: A string with the name of the sequence mask tensor. If
`sequence_mask` is in the features dictionary, the provided tensor is
used, otherwise the sequence mask is computed from the length of
sequential features. The sequence mask is used in evaluation and
training mode to aggregate loss and metrics computation while excluding
padding steps. It is also added to the predictions dictionary in
prediction mode to indicate which steps are padding.
config: `RunConfig` object to configure the runtime settings.
Note that a RNN cell has:
- a `call` method.
- a `state_size` attribute.
- a `output_size` attribute.
- a `get_initial_state` method.
See the documentation on `tf.keras.layers.RNN` for more details.
Raises:
ValueError: If `units`, `cell_type`, and `rnn_cell_fn` are not
compatible.
"""
if n_classes == 2:
head = binary_head_lib.BinaryClassHead(
weight_column=weight_column,
label_vocabulary=label_vocabulary,
loss_reduction=loss_reduction)
else:
head = multi_head_lib.MultiClassHead(
n_classes=n_classes,
weight_column=weight_column,
label_vocabulary=label_vocabulary,
loss_reduction=loss_reduction)
if return_sequences:
logging.info(
'Converting head to sequential head with '
'`SequentialHeadWrapper` to allow sequential predictions.')
head = seq_head_lib.SequentialHeadWrapper(
head,
sequence_length_mask=sequence_mask,
feature_columns=weight_column)
super(RNNClassifier,
self).__init__(head=head,
sequence_feature_columns=sequence_feature_columns,
context_feature_columns=context_feature_columns,
units=units,
cell_type=cell_type,
rnn_cell_fn=rnn_cell_fn,
return_sequences=return_sequences,
model_dir=model_dir,
optimizer=optimizer,
config=config)
