def test_loss_args(self): """Tests that variables are flattened and passed to static head's method.""" logits = [[1, 2], [3, 4]] labels = [[0, 1], [0, 2]] features = { 'weights': [[0.3, 0.2], [0.5, 100]], 'mask': [[1, 1], [1, 0]] } head = seq_head_lib.SequentialHeadWrapper(_MockHead(), 'mask', 'weights') expected_output = { 'logits': [[1], [2], [3]], 'labels': [[0], [1], [0]], 'features': { 'weights': [[0.3], [0.2], [0.5]] }, 'mode': 'my-mode', 'regularization_losses': 123 } output = head.loss(logits=_convert_to_tensor(logits), labels=_convert_to_tensor(labels), features=_convert_to_tensor(features), mode='my-mode', regularization_losses=123) with self.cached_session() as sess: self._assert_equal(output, dref=expected_output, session=sess)
def test_wrong_feature_column_type_in_iterable(self): """Tests error raised when the feature column doesn't have proper type.""" with self.assertRaisesRegexp(TypeError, 'Column must a string. Given type: .*.'): _ = seq_head_lib.SequentialHeadWrapper(None, 'mask', feature_columns=[1])
def test_create_estimator_spec_args(self): """Tests that variables are flattened and passed to static head's method.""" logits = [[1, 2], [3, 4]] labels = [[0, 1], [0, 2]] features = { 'weights': [[0.3, 0.2], [0.5, 100]], 'mask': [[1, 1], [1, 0]] } head = seq_head_lib.SequentialHeadWrapper(_MockHead(), 'mask', 'weights') expected_output = { 'logits': [[1], [2], [3]], 'labels': [[0], [1], [0]], 'features': { 'weights': [[0.3], [0.2], [0.5]] }, 'mode': ModeKeys.TRAIN, 'regularization_losses': 123, 'optimizer': 'my-opt', 'train_op_fn': 'my-train-op' } spec = head.create_estimator_spec( logits=_convert_to_tensor(logits), labels=_convert_to_tensor(labels), features=_convert_to_tensor(features), mode=ModeKeys.TRAIN, optimizer='my-opt', train_op_fn='my-train-op', regularization_losses=123) with self.cached_session() as sess: self.assertItemsEqual(spec.kwargs.keys(), expected_output.keys()) self._assert_equal(spec.kwargs, dref=expected_output, session=sess)
def test_multi_head_provided(self): """Tests error raised when a multi-head is provided.""" with self.assertRaisesRegexp( ValueError, '`MultiHead` is not supported with `SequentialHeadWrapper`.'): _ = seq_head_lib.SequentialHeadWrapper( multi_head.MultiHead( [binary_head_lib.BinaryClassHead(name='test-head')]))
def test_wrong_feature_column_type(self): """Tests error raised when the feature column doesn't have proper type.""" with self.assertRaisesRegexp( TypeError, '`feature_columns` must be either a string or an iterable'): _ = seq_head_lib.SequentialHeadWrapper(None, 'mask', feature_columns=1)
def test_loss_reduction(self): """Tests loss reduction. Use `loss` method in eager execution, else `create_estimator_spec` in TRAIN mode. logits = [[[2., 3., 4.], [5., -0.5, 0.]], [[-1.0, 2.0, 0.5], [_]]], labels = [[0, 1], [2, _]] weights = [[0.5, 0.2], [0.3, _]] loss = [0.5*2.40 + 0.2*5.41 + 0.3*1.74] / 3 = 0.94 """ static_head = multi_head_lib.MultiClassHead( n_classes=3, weight_column='weights') head = seq_head_lib.SequentialHeadWrapper(static_head, 'sequence_mask', 'weights') expected_loss = 0.942783 features = { 'weights': tf.sparse.SparseTensor( indices=((0, 0), (0, 1), (1, 0)), values=(0.5, 0.2, 0.3), dense_shape=(2, 2)), 'sequence_mask': ops.convert_to_tensor([[1, 1], [1, 0]]) } logits = ops.convert_to_tensor([[[2., 3., 4.], [5., -0.5, 0.]], [[-1.0, 2.0, 0.5], [1.0, 0.5, 2.0]]]) labels = tf.sparse.SparseTensor( indices=((0, 0), (0, 1), (1, 0)), values=(0, 1, 2), dense_shape=(2, 2)) class _Optimizer(tf.keras.optimizers.Optimizer): def get_updates(self, loss, params): del params, loss return [tf.constant('op')] def get_config(self): config = super(_Optimizer, self).get_config() return config if tf.executing_eagerly(): loss = head.loss(logits=logits, labels=labels, features=features) else: spec = head.create_estimator_spec( features, ModeKeys.TRAIN, logits, labels=labels, optimizer=_Optimizer('my_optimizer'), trainable_variables=[ tf.Variable([1.0, 2.0], dtype=tf.dtypes.float32) ]) with self.cached_session() as sess: loss = sess.run(spec.loss) self.assertAllClose(loss, expected_loss, atol=1e-4)
def test_head_properties(self): """Tests that the head's properties are correcly implemented.""" static_head = binary_head_lib.BinaryClassHead( loss_reduction=losses_utils.ReductionV2.SUM, name='a_static_head') head = seq_head_lib.SequentialHeadWrapper(static_head, 'a_sequence_mask_col') self.assertEqual(head.name, 'a_static_head_sequential') self.assertEqual(head.logits_dimension, 1) self.assertEqual(head.loss_reduction, losses_utils.ReductionV2.SUM) self.assertEqual(head.input_sequence_mask_key, 'a_sequence_mask_col') self.assertEqual(head.static_head.name, 'a_static_head')
def test_metrics(self): """Tests the `metrics` method. Tests that: - Returned metrics match the returned metrics of the static head. - `regularization_losses` argument is properly passed to the static head's method. """ head = seq_head_lib.SequentialHeadWrapper( binary_head_lib.BinaryClassHead(), 'mask') metrics = head.metrics(regularization_losses=2.5) keys = metric_keys.MetricKeys self.assertIn(keys.ACCURACY, metrics) self.assertIn(keys.LOSS_REGULARIZATION, metrics)
def test_loss_reduction(self): """Tests loss reduction. Use `loss` method in eager execution, else `create_estimator_spec` in TRAIN mode. logits = [[[2., 3., 4.], [5., -0.5, 0.]], [[-1.0, 2.0, 0.5], [_]]], labels = [[0, 1], [2, _]] weights = [[0.5, 0.2], [0.3, _]] loss = [0.5*2.40 + 0.2*5.41 + 0.3*1.74] / 3 = 0.94 """ static_head = multi_head_lib.MultiClassHead(n_classes=3, weight_column='weights') head = seq_head_lib.SequentialHeadWrapper(static_head, 'sequence_mask', 'weights') expected_loss = 0.942783 features = { 'weights': sparse_tensor.SparseTensor(indices=((0, 0), (0, 1), (1, 0)), values=(0.5, 0.2, 0.3), dense_shape=(2, 2)), 'sequence_mask': ops.convert_to_tensor([[1, 1], [1, 0]]) } logits = ops.convert_to_tensor([[[2., 3., 4.], [5., -0.5, 0.]], [[-1.0, 2.0, 0.5], [1.0, 0.5, 2.0]]]) labels = sparse_tensor.SparseTensor(indices=((0, 0), (0, 1), (1, 0)), values=(0, 1, 2), dense_shape=(2, 2)) class _Optimizer(object): def minimize(self, loss, global_step): del global_step, loss return constant_op.constant('op') if context.executing_eagerly(): loss = head.loss(logits=logits, labels=labels, features=features) else: spec = head.create_estimator_spec(features, ModeKeys.TRAIN, logits, labels, optimizer=_Optimizer()) with self.cached_session() as sess: loss = sess.run(spec.loss) self.assertAllClose(loss, expected_loss, atol=1e-4)
def test_predictions(self): """Tests predictions output. Use `predictions` method in eager execution, else `create_estimator_spec` in PREDICT mode. logits = [[0.3, -0.4], [0.2, 0.2]] logistics = 1 / (1 + exp(-logits)) = [[0.57, 0.40], [0.55, 0.55]] """ head = seq_head_lib.SequentialHeadWrapper( binary_head_lib.BinaryClassHead(), 'sequence_mask') logits = [[[0.3], [-0.4]], [[0.2], [0.2]]] expected_logistics = [[[0.574443], [0.401312]], [[0.549834], [0.549834]]] features = { 'sequence_mask': ops.convert_to_tensor(np.array([[1, 1], [1, 0]])) } keys = prediction_keys.PredictionKeys if tf.executing_eagerly(): predictions = head.predictions(logits=logits, keys=[keys.LOGITS, keys.LOGISTIC]) self.assertItemsEqual(predictions.keys(), [keys.LOGITS, keys.LOGISTIC]) self.assertAllClose(logits, predictions[keys.LOGITS]) self.assertAllClose(expected_logistics, predictions[keys.LOGISTIC]) return spec = head.create_estimator_spec(features=features, mode=ModeKeys.PREDICT, logits=logits, trainable_variables=[ tf.Variable( [1.0, 2.0], dtype=tf.dtypes.float32) ]) self.assertIn('sequence_mask', spec.predictions) with self.cached_session() as sess: self.assertAllEqual(sess.run(spec.predictions['sequence_mask']), features['sequence_mask']) self.assertAllClose(logits, sess.run(spec.predictions[keys.LOGITS])) self.assertAllClose(expected_logistics, sess.run(spec.predictions[keys.LOGISTIC]))
def _test_flatten_method(self, features, feature_columns): """Runs seq head's `_flatten` method and returns output for testing.""" head = seq_head_lib.SequentialHeadWrapper( static_head=None, sequence_length_mask='sequence_mask', feature_columns=feature_columns) labels = { 'indices': ((0, 0), (0, 1), (1, 0)), 'values': (1, 2, 3), 'dense_shape': (2, 2) } logits = np.array([[[10], [11]], [[12], [13]]]) features = _convert_to_tensor(features) labels = sparse_tensor.SparseTensor(**labels) logits = ops.convert_to_tensor(logits) output = head._flatten(labels, logits, features) if context.executing_eagerly(): return output with self.cached_session() as sess: return sess.run(output)
def test_wrong_mask_type(self): """Tests error raised when the mask doesn't have proper type.""" with self.assertRaisesRegexp( TypeError, '`sequence_mask` column must be a string.'): _ = seq_head_lib.SequentialHeadWrapper(None, sequence_length_mask=1)
def test_metrics_computation(self): """Runs metrics computation tests. Use `update_metrics` method in eager execution, else `create_estimator_spec` in EVAL mode. logits = [[-101, 102, -103], [104, _, _]] predicted_labels = [[0, 1, 0], [1, _, _]] labels = [[1, 1, 1], [1, _, _]] weights = [[2, 5, 1], [2, _, _]] loss = (101*2 + 103*1) / 10 = 30.5 accuracy = (0 + 5 + 0 + 2) / (2 + 5 + 1 + 2) = 0.7 prediction_mean = (0 + 5 + 0 + 2) / (2 + 5 + 1 + 2) = 0.7 precision = (5 + 2) / (5 + 2) = 1.0 recall = (5 + 2) / (2 + 5 + 1 + 2) = 0.7 """ static_head = binary_head_lib.BinaryClassHead(weight_column='weights') head = seq_head_lib.SequentialHeadWrapper(static_head, 'sequence_mask', 'weights') features = { 'sequence_mask': np.array([[1, 1, 1], [1, 0, 0]]), 'weights': np.array([[2, 5, 1], [2, 100, 100]]) } regularization_losses = [100.] logits = _convert_to_tensor([[-101, 102, -103], [104, 100, 100]]) labels = sparse_tensor.SparseTensor(values=[1, 1, 1, 1], indices=((0, 0), (0, 1), (0, 2), (1, 0)), dense_shape=(2, 3)) features = _convert_to_tensor(features) expected_loss = 30.5 keys = metric_keys.MetricKeys expected_metrics = { keys.LOSS_MEAN: expected_loss, keys.ACCURACY: 0.7, keys.PREDICTION_MEAN: 0.7, keys.LABEL_MEAN: 1.0, keys.LOSS_REGULARIZATION: 100, keys.PRECISION: 1.0, keys.RECALL: 0.7, keys.ACCURACY_BASELINE: 1.0, keys.AUC: 0., keys.AUC_PR: 1.0 } if context.executing_eagerly(): eval_metrics = head.metrics( regularization_losses=regularization_losses) updated_metrics = head.update_metrics(eval_metrics, features, logits, labels, regularization_losses) self.assertItemsEqual(expected_metrics.keys(), updated_metrics.keys()) self.assertAllClose( expected_metrics, {k: updated_metrics[k].result() for k in updated_metrics}) return spec = head.create_estimator_spec( features=features, mode=ModeKeys.EVAL, logits=logits, labels=labels, regularization_losses=regularization_losses) with self.cached_session() as sess: head_utils._initialize_variables(self, spec.scaffold) self.assertIsNone(spec.scaffold.summary_op) value_ops = { k: spec.eval_metric_ops[k][0] for k in spec.eval_metric_ops } update_ops = { k: spec.eval_metric_ops[k][1] for k in spec.eval_metric_ops } _ = sess.run(update_ops) self.assertAllClose(expected_metrics, {k: value_ops[k].eval() for k in value_ops})
def __init__(self, sequence_feature_columns, context_feature_columns=None, units=None, cell_type=USE_DEFAULT, rnn_cell_fn=None, return_sequences=False, model_dir=None, n_classes=2, weight_column=None, label_vocabulary=None, optimizer='Adagrad', loss_reduction=losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE, sequence_mask='sequence_mask', config=None): """Initializes a `RNNClassifier` instance. Args: sequence_feature_columns: An iterable containing the `FeatureColumn`s that represent sequential input. All items in the set should either be sequence columns (e.g. `sequence_numeric_column`) or constructed from one (e.g. `embedding_column` with `sequence_categorical_column_*` as input). context_feature_columns: An iterable containing the `FeatureColumn`s for contextual input. The data represented by these columns will be replicated and given to the RNN at each timestep. These columns must be instances of classes derived from `DenseColumn` such as `numeric_column`, not the sequential variants. units: Iterable of integer number of hidden units per RNN layer. If set, `cell_type` must also be specified and `rnn_cell_fn` must be `None`. cell_type: A class producing a RNN cell or a string specifying the cell type. Supported strings are: `'simple_rnn'`, `'lstm'`, and `'gru'`. If set, `units` must also be specified and `rnn_cell_fn` must be `None`. rnn_cell_fn: A function that returns a RNN cell instance that will be used to construct the RNN. If set, `units` and `cell_type` cannot be set. This is for advanced users who need additional customization beyond `units` and `cell_type`. Note that `tf.keras.layers.StackedRNNCells` is needed for stacked RNNs. return_sequences: A boolean indicating whether to return the last output in the output sequence, or the full sequence. Note that if True, `weight_column` must be None or a string. model_dir: Directory to save model parameters, graph and etc. This can also be used to load checkpoints from the directory into a estimator to continue training a previously saved model. n_classes: Number of label classes. Defaults to 2, namely binary classification. Must be > 1. weight_column: A string or a `NumericColumn` created by `tf.feature_column.numeric_column` defining feature column representing weights. It is used to down weight or boost examples during training. It will be multiplied by the loss of the example. If it is a string, it is used as a key to fetch weight tensor from the `features`. If it is a `NumericColumn`, raw tensor is fetched by key `weight_column.key`, then weight_column.normalizer_fn is applied on it to get weight tensor. label_vocabulary: A list of strings represents possible label values. If given, labels must be string type and have any value in `label_vocabulary`. If it is not given, that means labels are already encoded as integer or float within [0, 1] for `n_classes=2` and encoded as integer values in {0, 1,..., n_classes-1} for `n_classes`>2 . Also there will be errors if vocabulary is not provided and labels are string. optimizer: An instance of `tf.Optimizer` or string specifying optimizer type. Defaults to Adagrad optimizer. loss_reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to reduce training loss over batch. Defaults to `SUM_OVER_BATCH_SIZE`. sequence_mask: A string with the name of the sequence mask tensor. If `sequence_mask` is in the features dictionary, the provided tensor is used, otherwise the sequence mask is computed from the length of sequential features. The sequence mask is used in evaluation and training mode to aggregate loss and metrics computation while excluding padding steps. It is also added to the predictions dictionary in prediction mode to indicate which steps are padding. config: `RunConfig` object to configure the runtime settings. Note that a RNN cell has: - a `call` method. - a `state_size` attribute. - a `output_size` attribute. - a `get_initial_state` method. See the documentation on `tf.keras.layers.RNN` for more details. Raises: ValueError: If `units`, `cell_type`, and `rnn_cell_fn` are not compatible. """ if n_classes == 2: head = binary_head_lib.BinaryClassHead( weight_column=weight_column, label_vocabulary=label_vocabulary, loss_reduction=loss_reduction) else: head = multi_head_lib.MultiClassHead( n_classes=n_classes, weight_column=weight_column, label_vocabulary=label_vocabulary, loss_reduction=loss_reduction) if return_sequences: logging.info( 'Converting head to sequential head with ' '`SequentialHeadWrapper` to allow sequential predictions.') head = seq_head_lib.SequentialHeadWrapper( head, sequence_length_mask=sequence_mask, feature_columns=weight_column) super(RNNClassifier, self).__init__(head=head, sequence_feature_columns=sequence_feature_columns, context_feature_columns=context_feature_columns, units=units, cell_type=cell_type, rnn_cell_fn=rnn_cell_fn, return_sequences=return_sequences, model_dir=model_dir, optimizer=optimizer, config=config)