def test_named_args_no_weights(self): features = {"f1": "f1_value", "f2": "f2_value"} labels_ = {"l1": "l1_value", "l2": "l2_value"} predictions_ = {"p1": "p1_value", "p2": "p2_value"} def _fn0(predictions, labels): self.assertEqual("p1_value", predictions) self.assertEqual("l1_value", labels) return "metric_fn_result" def _fn1(predictions, targets): self.assertEqual("p1_value", predictions) self.assertEqual("l1_value", targets) return "metric_fn_result" def _fn2(prediction, label): self.assertEqual("p1_value", prediction) self.assertEqual("l1_value", label) return "metric_fn_result" def _fn3(prediction, target): self.assertEqual("p1_value", prediction) self.assertEqual("l1_value", target) return "metric_fn_result" for fn in (_fn0, _fn1, _fn2, _fn3): spec = MetricSpec(metric_fn=fn, prediction_key="p1", label_key="l1") self.assertEqual( "metric_fn_result", spec.create_metric_ops(features, labels_, predictions_))
def testCustomMetrics(self): """Tests custom evaluation metrics.""" def _input_fn(num_epochs=None): # Create 4 rows, one of them (y = x), three of them (y=Not(x)) labels = tf.constant([[1], [0], [0], [0]]) features = { 'x': tf.train.limit_epochs(tf.ones(shape=[4, 1], dtype=tf.float32), num_epochs=num_epochs), } return features, labels def _my_metric_op(predictions, labels): # For the case of binary classification, the 2nd column of "predictions" # denotes the model predictions. labels = tf.to_float(labels) predictions = tf.slice(predictions, [0, 1], [-1, 1]) labels = math_ops.cast(labels, predictions.dtype) return tf.reduce_sum(tf.mul(predictions, labels)) classifier = tf.contrib.learn.DNNClassifier( feature_columns=[tf.contrib.layers.real_valued_column('x')], hidden_units=[3, 3], config=tf.contrib.learn.RunConfig(tf_random_seed=1)) classifier.fit(input_fn=_input_fn, steps=5) scores = classifier.evaluate( input_fn=_input_fn, steps=5, metrics={ 'my_accuracy': MetricSpec(metric_fn=tf.contrib.metrics.streaming_accuracy, prediction_key='classes'), 'my_precision': MetricSpec(metric_fn=tf.contrib.metrics.streaming_precision, prediction_key='classes'), 'my_metric': MetricSpec(metric_fn=_my_metric_op, prediction_key='probabilities') }) self.assertTrue( set(['loss', 'my_accuracy', 'my_precision', 'my_metric']).issubset(set(scores.keys()))) predict_input_fn = functools.partial(_input_fn, num_epochs=1) predictions = np.array( list(classifier.predict(input_fn=predict_input_fn))) self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions), scores['my_accuracy']) # Test the case where the 2nd element of the key is neither "classes" nor # "probabilities". with self.assertRaisesRegexp(KeyError, 'bad_type'): classifier.evaluate( input_fn=_input_fn, steps=5, metrics={ 'bad_name': MetricSpec(metric_fn=tf.contrib.metrics.streaming_auc, prediction_key='bad_type') })
def fit(self , X_train,y_train , step = 2000): # Specify that all features have real-value data self.feature_columns = [tf.contrib.layers.real_valued_column("", dimension=X_train[0].shape[0])] #validation_metrics = {"accuracy": tf.contrib.metrics.streaming_accuracy} self.validation_metrics = { "accuracy" : MetricSpec(metric_fn=tf.contrib.metrics.streaming_accuracy,prediction_key="classes"), "precision" : MetricSpec(metric_fn=tf.contrib.metrics.streaming_precision,prediction_key="classes"), "recall" : MetricSpec(metric_fn=tf.contrib.metrics.streaming_recall,prediction_key="classes")} #instantiate a validaition monitor #validation_monitor = tf.contrib.learn.monitors.ValidationMonitor(X_test,y_test,every_n_steps=50) #self.validation_monitor = tf.contrib.learn.monitors.ValidationMonitor( # X_test, # y_test, # every_n_steps=50, # metrics=self.validation_metrics, # early_stopping_metric="loss", # early_stopping_metric_minimize=True, # early_stopping_rounds=200) # Linear Classifier self.classifier = tf.contrib.learn.LinearClassifier(feature_columns=self.feature_columns, model_dir="data/linear", config=tf.contrib.learn.RunConfig(save_checkpoints_secs=1)) #self.classifier.fit(x=X_train,y=y_train,steps=2000,monitors=[validation_monitor]) self.classifier.fit(x=X_train,y=y_train,steps=2000)
def test_partial(self): features = {"f1": "f1_value", "f2": "f2_value"} labels = {"l1": "l1_value"} predictions = {"p1": "p1_value", "p2": "p2_value"} def custom_metric(predictions, labels, stuff, weights=None): self.assertEqual("p1_value", predictions) self.assertEqual("l1_value", labels) self.assertEqual("f2_value", weights) if stuff: return "metric_fn_result" raise ValueError("No stuff.") spec = MetricSpec( metric_fn=functools.partial(custom_metric, stuff=5), label_key="l1", prediction_key="p1", weight_key="f2") self.assertEqual( "metric_fn_result", spec.create_metric_ops(features, labels, predictions)) spec = MetricSpec( metric_fn=functools.partial(custom_metric, stuff=None), prediction_key="p1", label_key="l1", weight_key="f2") with self.assertRaisesRegexp(ValueError, "No stuff."): spec.create_metric_ops(features, labels, predictions)
def test_partial(self): features = { "feature1": "feature1_tensor", "feature2": "feature2_tensor" } labels = {"label1": "label1_tensor"} predictions = {"pred1": "pred1_tensor", "pred2": "pred2_tensor"} def custom_metric(predictions, labels, stuff, weights=None): if stuff: return predictions, labels, weights else: raise ValueError("Nooooo") partial_metric = functools.partial(custom_metric, stuff=5) passed = MetricSpec(metric_fn=partial_metric, label_key="label1", prediction_key="pred1", weight_key="feature2").create_metric_ops( features, labels, predictions) self.assertEqual(passed[0], "pred1_tensor") self.assertEqual(passed[1], "label1_tensor") self.assertEqual(passed[2], "feature2_tensor") broken_partial_metric = functools.partial(custom_metric, stuff=0) self.assertRaisesRegexp( ValueError, "Nooooo", MetricSpec(metric_fn=broken_partial_metric, prediction_key="pred1", label_key="label1", weight_key="feature2").create_metric_ops, features, labels, predictions)
def _create_evaluation_metrics(): """Creates the evaluation metrics for the model. Returns: A dictionary with keys that are strings naming the evaluation metrics and values that are functions taking arguments of (predictions, targets), returning a tuple of a tensor of the metric's value together with an op to update the metric's value. """ eval_metrics = {} for k in [1]: eval_metrics["precision_at_%d" % k] = MetricSpec( metric_fn=functools.partial( tf.contrib.metrics.streaming_sparse_precision_at_k, k=k)) eval_metrics["recall_at_%d" % k] = MetricSpec(metric_fn=functools.partial( tf.contrib.metrics.streaming_sparse_recall_at_k, k=k)) for class_id, class_label in _get_eval_labels(): k = 1 eval_metrics["precision_at_%d_%s" % (k, class_label)] = MetricSpec( metric_fn=functools.partial( tf.contrib.metrics.streaming_sparse_precision_at_k, k=k, class_id=class_id)) eval_metrics["recall_at_%d_%s" % (k, class_label)] = MetricSpec( metric_fn=functools.partial( tf.contrib.metrics.streaming_sparse_recall_at_k, k=k, class_id=class_id)) return eval_metrics
def test_named_args_with_weights(self): features = {"f1": "f1_value", "f2": "f2_value"} labels_ = {"l1": "l1_value", "l2": "l2_value"} predictions_ = {"p1": "p1_value", "p2": "p2_value"} def _fn0(predictions, labels, weights=None): self.assertEqual("p1_value", predictions) self.assertEqual("l1_value", labels) self.assertEqual("f2_value", weights) return "metric_fn_result" def _fn1(predictions, targets, weights=None): self.assertEqual("p1_value", predictions) self.assertEqual("l1_value", targets) self.assertEqual("f2_value", weights) return "metric_fn_result" def _fn2(prediction, label, weight=None): self.assertEqual("p1_value", prediction) self.assertEqual("l1_value", label) self.assertEqual("f2_value", weight) return "metric_fn_result" def _fn3(prediction, target, weight=None): self.assertEqual("p1_value", prediction) self.assertEqual("l1_value", target) self.assertEqual("f2_value", weight) return "metric_fn_result" for fn in (_fn0, _fn1, _fn2, _fn3): spec = MetricSpec( metric_fn=fn, prediction_key="p1", label_key="l1", weight_key="f2") self.assertEqual( "metric_fn_result", spec.create_metric_ops(features, labels_, predictions_))
def train(model, dataset_dir, model_dir, batch_size, train_steps, is_evaluate_accuracy, valid_every_n_steps, early_stopping_rounds): def input_fn_test(): return dataset_dsb17.inputs(data_dir=dataset_dir, mode=dataset_dsb17.DATASET_MODE.TEST, batch_size=batch_size) def input_fn_train(): return dataset_dsb17.inputs(data_dir=dataset_dir, mode=dataset_dsb17.DATASET_MODE.TRAIN, batch_size=batch_size) def input_fn_valid(): return dataset_dsb17.inputs(data_dir=dataset_dir, mode=dataset_dsb17.DATASET_MODE.VALID, batch_size=batch_size) loginfo("Start") loginfo("Create validation monitor") # Monitors validation_metrics = { "accuracy": MetricSpec(metric_fn=tf.contrib.metrics.streaming_accuracy, prediction_key=PredictionKey.CLASSES), "precision": MetricSpec(metric_fn=tf.contrib.metrics.streaming_precision, prediction_key=PredictionKey.CLASSES), "recall": MetricSpec(metric_fn=tf.contrib.metrics.streaming_recall, prediction_key=PredictionKey.CLASSES) } validation_monitor = learn.monitors.ValidationMonitor( input_fn=input_fn_valid, every_n_steps=valid_every_n_steps, early_stopping_rounds=early_stopping_rounds, metrics=validation_metrics) loginfo("Create classifier") start = time.time() classifier = learn.Estimator(model_fn=model, model_dir=model_dir) loginfo("Fit model") # Fit model. classifier.fit(input_fn=input_fn_train, steps=train_steps, monitors=[validation_monitor]) loginfo("Fit model finished. Time: %.03f s" % (time.time() - start)) start = time.time() if is_evaluate_accuracy: loginfo("Evaluate accuracy") # Evaluate accuracy. result = classifier.evaluate(input_fn=input_fn_test, metrics=validation_metrics) loginfo('Accuracy: {0:f}'.format(result['accuracy'])) loginfo("Time: %.03f s" % (time.time() - start)) loginfo("done")
def test_no_args(self): def _fn(): self.fail("Expected failure before metric_fn.") spec = MetricSpec(metric_fn=_fn) with self.assertRaises(TypeError): spec.create_metric_ops( {"f1": "f1_value"}, "labels_value", "predictions_value")
def test_no_args(self): def _fn(): self.fail("Expected failure before metric_fn.") spec = MetricSpec(metric_fn=_fn) with self.assertRaises(TypeError): spec.create_metric_ops({"f1": "f1_value"}, "labels_value", "predictions_value")
def testCustomMetrics(self): """Tests custom evaluation metrics.""" def _input_fn_train(): # Create 4 rows, one of them (y = x), three of them (y=Not(x)) target = tf.constant([[1], [0], [0], [0]], dtype=tf.float32) features = {'x': tf.ones(shape=[4, 1], dtype=tf.float32)} return features, target def _my_metric_op(predictions, targets): # For the case of binary classification, the 2nd column of "predictions" # denotes the model predictions. predictions = tf.slice(predictions, [0, 1], [-1, 1]) return tf.reduce_sum(tf.mul(predictions, targets)) classifier = tf.contrib.learn.LinearClassifier( feature_columns=[tf.contrib.layers.real_valued_column('x')]) classifier.fit(input_fn=_input_fn_train, steps=100) scores = classifier.evaluate( input_fn=_input_fn_train, steps=100, metrics={ 'my_accuracy': MetricSpec(metric_fn=tf.contrib.metrics.streaming_accuracy, prediction_key='classes'), 'my_precision': MetricSpec(metric_fn=tf.contrib.metrics.streaming_precision, prediction_key='classes'), 'my_metric': MetricSpec(metric_fn=_my_metric_op, prediction_key='probabilities') }) self.assertTrue( set(['loss', 'my_accuracy', 'my_precision', 'my_metric']).issubset(set(scores.keys()))) predictions = classifier.predict(input_fn=_input_fn_train) self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions), scores['my_accuracy']) # Test the case where the 2nd element of the key is neither "classes" nor # "probabilities". with self.assertRaises(ValueError): classifier.evaluate(input_fn=_input_fn_train, steps=100, metrics={ ('bad_name', 'bad_type'): tf.contrib.metrics.streaming_auc }) # Test the case where the tuple of the key doesn't have 2 elements. with self.assertRaises(ValueError): classifier.evaluate( input_fn=_input_fn_train, steps=100, metrics={ ('bad_length_name', 'classes', 'bad_length'): tf.contrib.metrics.streaming_accuracy })
def _create_evaluation_metrics_regression(): eval_metrics = {} eval_metrics['MAE'] = MetricSpec( metric_fn=tf.contrib.metrics.streaming_mean_absolute_error, prediction_key="predictions") eval_metrics['MSE'] = MetricSpec( metric_fn=tf.contrib.metrics.streaming_mean_squared_error, prediction_key="predictions") return eval_metrics
def main(unused_argv): # Load datasets. training_set = tf.contrib.learn.datasets.base.load_csv_with_header( filename=IRIS_TRAINING, target_dtype=np.int, features_dtype=np.float) test_set = tf.contrib.learn.datasets.base.load_csv_with_header( filename=IRIS_TEST, target_dtype=np.int, features_dtype=np.float) # Specify that all features have real-value data feature_columns = [tf.contrib.layers.real_valued_column("", dimension=4)] validation_metrics = { "accuracy": MetricSpec(metric_fn=tf.contrib.metrics.streaming_accuracy, prediction_key="classes"), "recall": MetricSpec(metric_fn=tf.contrib.metrics.streaming_recall, prediction_key="classes"), "precision": MetricSpec(metric_fn=tf.contrib.metrics.streaming_precision, prediction_key="classes") } validation_monitor = tf.contrib.learn.monitors.ValidationMonitor( test_set.data, test_set.target, every_n_steps=50, metrics=validation_metrics, early_stopping_metric="loss", early_stopping_metric_minimize=True, early_stopping_rounds=200) # Build 3 layer DNN with 10, 20, 10 units respectively. classifier = tf.contrib.learn.DNNClassifier( feature_columns=feature_columns, hidden_units=[10, 20, 10], n_classes=3, model_dir="/tmp/iris_model", config=tf.contrib.learn.RunConfig(save_checkpoints_secs=1)) # Fit model. classifier.fit(x=training_set.data, y=training_set.target, steps=2000, monitors=[validation_monitor]) # Evaluate accuracy. accuracy_score = classifier.evaluate(x=test_set.data, y=test_set.target)["accuracy"] print(f"Accuracy: {accuracy_score}") # Classify two new flower samples. new_samples = np.array([[6.4, 3.2, 4.5, 1.5], [5.8, 3.1, 5.0, 1.7]], dtype=float) y = list(classifier.predict(new_samples)) print(f"Predictions: {y}")
def testCustomMetrics(self): """Tests custom evaluation metrics.""" def _input_fn(num_epochs=None): # Create 4 rows, one of them (y = x), three of them (y=Not(x)) labels = constant_op.constant([[1.], [0.], [0.], [0.]]) features = { 'x': input_lib.limit_epochs( array_ops.ones(shape=[4, 1], dtype=dtypes.float32), num_epochs=num_epochs), } return features, labels def _my_metric_op(predictions, labels): return math_ops.reduce_sum(math_ops.multiply(predictions, labels)) regressor = debug.DebugRegressor( config=run_config.RunConfig(tf_random_seed=1)) regressor.fit(input_fn=_input_fn, steps=5) scores = regressor.evaluate( input_fn=_input_fn, steps=1, metrics={ 'my_error': MetricSpec( metric_fn=metric_ops.streaming_mean_squared_error, prediction_key='scores'), 'my_metric': MetricSpec(metric_fn=_my_metric_op, prediction_key='scores') }) self.assertIn('loss', set(scores.keys())) self.assertIn('my_error', set(scores.keys())) self.assertIn('my_metric', set(scores.keys())) predict_input_fn = functools.partial(_input_fn, num_epochs=1) predictions = np.array( list(regressor.predict_scores(input_fn=predict_input_fn))) self.assertAlmostEqual( _sklearn.mean_squared_error(np.array([1, 0, 0, 0]), predictions), scores['my_error']) # Tests the case where the prediction_key is not "scores". with self.assertRaisesRegexp(KeyError, 'bad_type'): regressor.evaluate( input_fn=_input_fn, steps=1, metrics={ 'bad_name': MetricSpec( metric_fn=metric_ops.streaming_auc, prediction_key='bad_type') })
def test_named_labels_no_predictions(self): features = {"f1": "f1_value"} labels_ = "labels_value" predictions_ = "predictions_value" def _fn(labels): self.assertEqual(labels_, labels) return "metric_fn_result" spec = MetricSpec(metric_fn=_fn) with self.assertRaises(TypeError): spec.create_metric_ops(features, labels_, predictions_)
def test_no_named_labels_or_predictions_2args(self): features = {"f1": "f1_value"} labels_ = "labels_value" predictions_ = "predictions_value" def _fn(a, b): del a, b self.fail("Expected failure before metric_fn.") spec = MetricSpec(metric_fn=_fn) with self.assertRaises(TypeError): spec.create_metric_ops(features, labels_, predictions_)
def test_no_named_labels_or_predictions_1arg(self): features = {"f1": "f1_value"} labels_ = "labels_value" predictions_ = "predictions_value" def _fn(a): self.assertEqual(predictions_, a) return "metric_fn_result" spec = MetricSpec(metric_fn=_fn) self.assertEqual( "metric_fn_result", spec.create_metric_ops(features, labels_, predictions_))
def test_single_labels_with_key(self): features = {"f1": "f1_value", "f2": "f2_value"} labels = "l1_value" predictions = {"p1": "p1_value", "p2": "p2_value"} def _fn(predictions, labels, weights=None): del labels, predictions, weights self.fail("Expected failure before metric_fn.") spec = MetricSpec( metric_fn=_fn, prediction_key="p1", label_key="l1", weight_key="f2") with self.assertRaisesRegexp( ValueError, "MetricSpec with label_key specified requires labels dict"): spec.create_metric_ops(features, labels, predictions)
def test_no_named_predictions_named_labels_second_arg(self): features = {"f1": "f1_value"} labels_ = "labels_value" predictions_ = "predictions_value" def _fn(predictions_by_another_name, labels): self.assertEqual(predictions_, predictions_by_another_name) self.assertEqual(labels_, labels) return "metric_fn_result" spec = MetricSpec(metric_fn=_fn) self.assertEqual( "metric_fn_result", spec.create_metric_ops(features, labels_, predictions_))
def _create_evaluation_metrics_classify(): eval_metrics = {} eval_metrics['accuracy'] = MetricSpec( metric_fn=tf.contrib.metrics.streaming_accuracy, prediction_key="predictions") # eval_metrics['accuracy1'] = MetricSpec(metric_fn=evaluate_accuracy) eval_metrics['recall'] = MetricSpec( metric_fn=tf.contrib.metrics.streaming_recall, prediction_key="predictions") eval_metrics['precision'] = MetricSpec( metric_fn=tf.contrib.metrics.streaming_precision, prediction_key="predictions") return eval_metrics
def testCustomMetricsWithMetricSpec(self): """Tests custom evaluation metrics that use MetricSpec.""" def _input_fn(num_epochs=None): # Create 4 rows, one of them (y = x), three of them (y=Not(x)) labels = tf.constant([[1.], [0.], [0.], [0.]]) features = { 'x': tf.train.limit_epochs( tf.ones(shape=[4, 1], dtype=tf.float32), num_epochs=num_epochs), } return features, labels def _my_metric_op(predictions, labels): return tf.reduce_sum(tf.multiply(predictions, labels)) regressor = tf.contrib.learn.DNNRegressor( feature_columns=[tf.contrib.layers.real_valued_column('x')], hidden_units=[3, 3], config=tf.contrib.learn.RunConfig(tf_random_seed=1)) regressor.fit(input_fn=_input_fn, steps=5) scores = regressor.evaluate( input_fn=_input_fn, steps=1, metrics={ 'my_error': MetricSpec( metric_fn=tf.contrib.metrics.streaming_mean_squared_error, prediction_key='scores'), 'my_metric': MetricSpec( metric_fn=_my_metric_op, prediction_key='scores') }) self.assertIn('loss', set(scores.keys())) self.assertIn('my_error', set(scores.keys())) self.assertIn('my_metric', set(scores.keys())) predict_input_fn = functools.partial(_input_fn, num_epochs=1) predictions = np.array(list(regressor.predict(input_fn=predict_input_fn))) self.assertAlmostEqual( _sklearn.mean_squared_error(np.array([1, 0, 0, 0]), predictions), scores['my_error']) # Tests the case where the prediction_key is not "scores". with self.assertRaisesRegexp(KeyError, 'bad_type'): regressor.evaluate( input_fn=_input_fn, steps=1, metrics={ 'bad_name': MetricSpec( metric_fn=tf.contrib.metrics.streaming_auc, prediction_key='bad_type')})
def test_labels_dict_no_key(self): features = {"f1": "f1_value", "f2": "f2_value"} labels = {"l1": "l1_value", "l2": "l2_value"} predictions = {"p1": "p1_value", "p2": "p2_value"} def _fn(labels, predictions, weights=None): del labels, predictions, weights self.fail("Expected failure before metric_fn.") spec = MetricSpec(metric_fn=_fn, prediction_key="p1", weight_key="f2") with self.assertRaisesRegexp( ValueError, "MetricSpec without specified label_key requires labels tensor or" " single element dict"): spec.create_metric_ops(features, labels, predictions)
def test_single_label(self): features = {"f1": "f1_value", "f2": "f2_value"} labels_ = "l1_value" predictions_ = {"p1": "p1_value", "p2": "p2_value"} def _fn(predictions, labels, weights=None): self.assertEqual("p1_value", predictions) self.assertEqual(labels_, labels) self.assertEqual("f2_value", weights) return "metric_fn_result" spec = MetricSpec(metric_fn=_fn, prediction_key="p1", weight_key="f2") self.assertEqual( "metric_fn_result", spec.create_metric_ops(features, labels_, predictions_))
def contrib_learn_classifier_test(): """Test tf.contrib.learn.DNN_classifier.""" language_column = layers.sparse_column_with_hash_bucket( "language", hash_bucket_size=20) feature_columns = [ layers.embedding_column(language_column, dimension=3), layers.real_valued_column("age", dtype=tf.int64) ] classifier = learn.DNNClassifier( n_classes=3, feature_columns=feature_columns, hidden_units=[100, 100], config=learn.RunConfig(tf_random_seed=1, model_dir="../model_saver/estimators/" "DNN_classifier_01"), # optimizer=optimizer_exp_decay ) classifier.fit(input_fn=_input_fn, steps=10000) print("variables_names:\n", str(classifier.get_variable_names())) # scores = classifier.evaluate(input_fn=_input_fn, # steps=100) # print("scores:\n", str(scores)) scores = classifier.evaluate( input_fn=_input_fn, steps=100, metrics={ 'my_accuracy': MetricSpec(metric_fn=metrics.streaming_accuracy, prediction_key="classes"), 'my_precision': MetricSpec(metric_fn=metrics.streaming_precision, prediction_key="classes"), 'my_recall': MetricSpec(metric_fn=metrics.streaming_recall, prediction_key="classes"), 'my_metric': MetricSpec(metric_fn=my_metric_op, prediction_key="classes") }) print("scores:\n", str(scores)) predictions = classifier.predict(input_fn=_input_fn, outputs=["classes", "probabilities"]) print("predictions") for prediction in predictions: print(prediction)
def create_evaluation_metrics(): ''' Create a dictionary containing the eval metrics ''' def custom_streaming_sparse_recall_at_k(predictions, labels, k): ''' customize my evaluation metric. preditctions has the shape [batch_size, num_class] -> [16, 10] so we have to change the labels shape [160, 1] -> [16, 10] :param predictions: shaped probability for each class :param labels: unshape lebels :param k: to k classes :return: recall_at_k metrics for evaluation ''' split_labs = tf.split(labels, num_or_size_splits=10, axis=0) shaped_labels = tf.concat(split_labs, axis=1) return tf.contrib.metrics.streaming_sparse_recall_at_k( predictions, shaped_labels, k) eval_metrics = {} for k in [1, 2, 5, 10]: eval_metrics["recall_at_%d" % k] = MetricSpec(metric_fn=functools.partial( custom_streaming_sparse_recall_at_k, k=k)) return eval_metrics
def test_multiple_prediction_args(self): def _fn(predictions, prediction, labels): del predictions, prediction, labels self.fail("Expected failure before metric_fn.") with self.assertRaisesRegexp(ValueError, "provide only one of.*prediction"): MetricSpec(metric_fn=_fn)
def test_multiple_weight_args(self): def _fn(predictions, labels, weights=None, weight=None): del predictions, labels, weights, weight self.fail("Expected failure before metric_fn.") with self.assertRaisesRegexp(ValueError, "provide only one of.*weight"): MetricSpec(metric_fn=_fn)
def create_evaluation_metrics(): eval_metrics = {} for k in [1, 2, 5, 10]: eval_metrics["recall_at_%d" % k] = MetricSpec(metric_fn=functools.partial( tf.contrib.metrics.streaming_sparse_recall_at_k, k=k)) return eval_metrics
def test_partial(self): features = {"f1": "f1_value", "f2": "f2_value"} labels = {"l1": "l1_value"} predictions = {"p1": "p1_value", "p2": "p2_value"} def custom_metric(predictions, labels, stuff, weights=None): self.assertEqual("p1_value", predictions) self.assertEqual("l1_value", labels) self.assertEqual("f2_value", weights) if stuff: return "metric_fn_result" raise ValueError("No stuff.") spec = MetricSpec(metric_fn=functools.partial(custom_metric, stuff=5), label_key="l1", prediction_key="p1", weight_key="f2") self.assertEqual("metric_fn_result", spec.create_metric_ops(features, labels, predictions)) spec = MetricSpec(metric_fn=functools.partial(custom_metric, stuff=None), prediction_key="p1", label_key="l1", weight_key="f2") with self.assertRaisesRegexp(ValueError, "No stuff."): spec.create_metric_ops(features, labels, predictions)
def fit(self, X_train, y_train, step=2000): t_fit = 0 t_predict = 0 # Specify that all features have real-value data feature_columns = [ tf.contrib.layers.real_valued_column("", dimension=X_train[0].shape[0]) ] #validation_metrics = {"accuracy": tf.contrib.metrics.streaming_accuracy} validation_metrics = { "accuracy": MetricSpec(metric_fn=tf.contrib.metrics.streaming_accuracy, prediction_key="classes"), "precision": MetricSpec(metric_fn=tf.contrib.metrics.streaming_precision, prediction_key="classes"), "recall": MetricSpec(metric_fn=tf.contrib.metrics.streaming_recall, prediction_key="classes") } #instantiate a validaition monitor #validation_monitor = tf.contrib.learn.monitors.ValidationMonitor(X_test,y_test,every_n_steps=50) #validation_monitor = tf.contrib.learn.monitors.ValidationMonitor( # X_test, # y_test, # every_n_steps=50, # metrics=validation_metrics, # early_stopping_metric="loss", # early_stopping_metric_minimize=True, # early_stopping_rounds=200) # Build 3 layer DNN with 10, 20, 10 units respectively. self.classifier = tf.contrib.learn.DNNClassifier( feature_columns=feature_columns, hidden_units=[10, 20, 10], n_classes=3, model_dir=self.model_dir, config=tf.contrib.learn.RunConfig(save_checkpoints_secs=1)) self.classifier.fit(x=X_train, y=y_train, steps=step) #self.fit_time = Timer(lambda: classifier.fit(x=X_train,y=y_train,steps=2000)) self.fit_time = 0
def test_partial_str(self): def custom_metric(predictions, labels, stuff, weights=None): return predictions, labels, weights, stuff partial_metric = functools.partial(custom_metric, stuff=5) metric_spec = MetricSpec(metric_fn=partial_metric, label_key="label1", prediction_key="pred1", weight_key="feature2") self.assertIn("custom_metric", str(metric_spec))
def test_str(self): metric_spec = MetricSpec(metric_fn=test_metric, label_key="label1", prediction_key="pred1", weight_key="feature2") string = str(metric_spec) self.assertIn("test_metric", string) self.assertIn("label1", string) self.assertIn("pred1", string) self.assertIn("feature2", string)
def test_label_key_without_label_arg(self): def _fn0(predictions, weights=None): del predictions, weights self.fail("Expected failure before metric_fn.") def _fn1(prediction, weight=None): del prediction, weight self.fail("Expected failure before metric_fn.") for fn in (_fn0, _fn1): with self.assertRaisesRegexp(ValueError, "label.*missing"): MetricSpec(metric_fn=fn, label_key="l1")