def _test_metric_fn(metric_fn): input_fn = get_input_fn(x=[[[0.]]], y=[[[1]]]) estimator = linear.LinearClassifierV2([fc.numeric_column('x')]) estimator = extenders.add_metrics(estimator, metric_fn) estimator.train(input_fn=input_fn) metrics = estimator.evaluate(input_fn=input_fn) self.assertEqual(2., metrics['two'])
def test_should_error_out_for_not_recognized_args(self): estimator = linear.LinearClassifierV2([fc.numeric_column('x')]) def metric_fn(features, not_recognized): _, _ = features, not_recognized return {} with self.assertRaisesRegexp(ValueError, 'not_recognized'): estimator = extenders.add_metrics(estimator, metric_fn)
def test_all_supported_args(self): input_fn = get_input_fn(x=[[[0.]]], y=[[[1]]]) estimator = linear.LinearClassifierV2([fc.numeric_column('x')]) def metric_fn(features, predictions, labels, config): self.assertIn('x', features) self.assertIsNotNone(labels) self.assertIn('logistic', predictions) self.assertTrue(isinstance(config, run_config.RunConfig)) return {} estimator = extenders.add_metrics(estimator, metric_fn) estimator.train(input_fn=input_fn) estimator.evaluate(input_fn=input_fn)
def _test_metric_fn(metric_fn): input_fn = get_input_fn(x=np.arange(4)[:, None, None], y=np.ones(4)[:, None]) config = run_config.RunConfig(log_step_count_steps=1) estimator = linear.LinearClassifierV2([fc.numeric_column('x')], config=config) estimator = extenders.add_metrics(estimator, metric_fn) estimator.train(input_fn=input_fn) metrics = estimator.evaluate(input_fn=input_fn) self.assertIn('mean_x', metrics) self.assertEqual(1.5, metrics['mean_x']) # assert that it keeps original estimators metrics self.assertIn('auc', metrics)
def test_custom_metrics(self): """Tests that the custom metrics can be applied to the estimator.""" model_dir = self.get_temp_dir() estimator = ts_estimators.LSTMAutoRegressor( periodicities=1, input_window_size=1, output_window_size=1, num_features=1, num_units=4, optimizer=tf.compat.v1.train.AdamOptimizer(0.001), config=estimator_lib.RunConfig(tf_random_seed=4), model_dir=model_dir) def input_fn(): return { feature_keys.TrainEvalFeatures.TIMES: [[1, 2, 3], [7, 8, 9]], feature_keys.TrainEvalFeatures.VALUES: numpy.array([[[0.], [1.], [0.]], [[2.], [3.], [2.]]]) } def metrics_fn(predictions, features): # checking that the inputs are properly passed. predict = predictions["mean"] target = features[feature_keys.TrainEvalFeatures.VALUES][:, -1, 0] return { "plain_boring_metric386": (tf.math.reduce_mean(tf.math.abs(predict - target)), tf.no_op()), "fun_metric101": (tf.math.reduce_sum(predict + target), tf.no_op()), } # Evaluation without training is enough for testing custom metrics. estimator = extenders.add_metrics(estimator, metrics_fn) evaluation = estimator.evaluate(input_fn, steps=1) self.assertIn("plain_boring_metric386", evaluation) self.assertIn("fun_metric101", evaluation) self.assertIn("average_loss", evaluation)
def test_canned_estimator(self, distribution, estimator_cls): label_dimension = 2 batch_size = 10 # Adding one extra row (+ label_dimension) to test the last partial batch # use case. data = np.linspace(0., 2., batch_size * label_dimension + label_dimension, dtype=np.float32) data = data.reshape(batch_size + 1, label_dimension) fc = tf.feature_column.numeric_column('x', shape=(2, )) # Set kwargs based on the current canned estimator class. estimator_kw_args = { 'model_dir': self._model_dir, 'label_dimension': 2, } cls_args = inspect.getargspec(estimator_cls.__init__).args if 'hidden_units' in cls_args: estimator_kw_args['hidden_units'] = [2, 2] elif 'dnn_hidden_units' in cls_args: estimator_kw_args['dnn_hidden_units'] = [2, 2] if 'optimizer' in cls_args: estimator_kw_args['optimizer'] = 'SGD' else: estimator_kw_args['linear_optimizer'] = 'SGD' estimator_kw_args['dnn_optimizer'] = 'SGD' if 'feature_columns' in cls_args: estimator_kw_args['feature_columns'] = [fc] else: estimator_kw_args['linear_feature_columns'] = [fc] estimator_kw_args['dnn_feature_columns'] = [fc] def my_metrics(features): metric = metrics.Mean() metric.update_state(features['x']) return {'mean_x': metric} # Create a canned estimator and train to save a checkpoint. input_fn = self.dataset_input_fn(x={'x': data}, y=data, batch_size=batch_size, shuffle=False) canned_est = estimator_cls(**estimator_kw_args) canned_est.train(input_fn=input_fn) # Create a second canned estimator, warm-started from the first. del estimator_kw_args['model_dir'] estimator_kw_args['warm_start_from'] = canned_est.model_dir warm_started_canned_est = estimator_cls(**estimator_kw_args) warm_started_canned_est.train(input_fn=input_fn) # Create a third canned estimator, warm-started from the first. input_fn = self.dataset_input_fn(x={'x': data}, y=data, batch_size=batch_size // distribution.num_replicas_in_sync, shuffle=False) estimator_kw_args['config'] = run_config.RunConfig( train_distribute=distribution, eval_distribute=distribution) warm_started_canned_est_with_ds = estimator_cls(**estimator_kw_args) warm_started_canned_est_with_ds.train(input_fn=input_fn) for variable_name in warm_started_canned_est.get_variable_names(): self.assertAllClose( warm_started_canned_est_with_ds.get_variable_value( variable_name), warm_started_canned_est.get_variable_value(variable_name)) warm_started_canned_est = add_metrics(warm_started_canned_est, my_metrics) warm_started_canned_est_with_ds = add_metrics( warm_started_canned_est_with_ds, my_metrics) scores = warm_started_canned_est.evaluate(input_fn) scores_with_ds = warm_started_canned_est_with_ds.evaluate(input_fn) self.assertAlmostEqual(scores['loss'], scores_with_ds['loss'], 5) self.assertAlmostEqual(scores['mean_x'], scores_with_ds['mean_x'], 5)