def test_named_args_no_weights(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels_ = {"l1": "l1_value", "l2": "l2_value"}
predictions_ = {"p1": "p1_value", "p2": "p2_value"}
def _fn0(predictions, labels):
self.assertEqual("p1_value", predictions)
self.assertEqual("l1_value", labels)
return "metric_fn_result"
def _fn1(predictions, targets):
self.assertEqual("p1_value", predictions)
self.assertEqual("l1_value", targets)
return "metric_fn_result"
def _fn2(prediction, label):
self.assertEqual("p1_value", prediction)
self.assertEqual("l1_value", label)
return "metric_fn_result"
def _fn3(prediction, target):
self.assertEqual("p1_value", prediction)
self.assertEqual("l1_value", target)
return "metric_fn_result"
for fn in (_fn0, _fn1, _fn2, _fn3):
spec = MetricSpec(metric_fn=fn,
prediction_key="p1",
label_key="l1")
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels_, predictions_))
def testCustomMetrics(self):
"""Tests custom evaluation metrics."""
def _input_fn(num_epochs=None):
# Create 4 rows, one of them (y = x), three of them (y=Not(x))
labels = tf.constant([[1], [0], [0], [0]])
features = {
'x':
tf.train.limit_epochs(tf.ones(shape=[4, 1], dtype=tf.float32),
num_epochs=num_epochs),
}
return features, labels
def _my_metric_op(predictions, labels):
# For the case of binary classification, the 2nd column of "predictions"
# denotes the model predictions.
labels = tf.to_float(labels)
predictions = tf.slice(predictions, [0, 1], [-1, 1])
labels = math_ops.cast(labels, predictions.dtype)
return tf.reduce_sum(tf.mul(predictions, labels))
classifier = tf.contrib.learn.DNNClassifier(
feature_columns=[tf.contrib.layers.real_valued_column('x')],
hidden_units=[3, 3],
config=tf.contrib.learn.RunConfig(tf_random_seed=1))
classifier.fit(input_fn=_input_fn, steps=5)
scores = classifier.evaluate(
input_fn=_input_fn,
steps=5,
metrics={
'my_accuracy':
MetricSpec(metric_fn=tf.contrib.metrics.streaming_accuracy,
prediction_key='classes'),
'my_precision':
MetricSpec(metric_fn=tf.contrib.metrics.streaming_precision,
prediction_key='classes'),
'my_metric':
MetricSpec(metric_fn=_my_metric_op,
prediction_key='probabilities')
})
self.assertTrue(
set(['loss', 'my_accuracy', 'my_precision',
'my_metric']).issubset(set(scores.keys())))
predict_input_fn = functools.partial(_input_fn, num_epochs=1)
predictions = np.array(
list(classifier.predict(input_fn=predict_input_fn)))
self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions),
scores['my_accuracy'])
# Test the case where the 2nd element of the key is neither "classes" nor
# "probabilities".
with self.assertRaisesRegexp(KeyError, 'bad_type'):
classifier.evaluate(
input_fn=_input_fn,
steps=5,
metrics={
'bad_name':
MetricSpec(metric_fn=tf.contrib.metrics.streaming_auc,
prediction_key='bad_type')
})
def fit(self , X_train,y_train , step = 2000):
# Specify that all features have real-value data
self.feature_columns = [tf.contrib.layers.real_valued_column("", dimension=X_train[0].shape[0])]
#validation_metrics = {"accuracy": tf.contrib.metrics.streaming_accuracy}
self.validation_metrics = {
"accuracy" : MetricSpec(metric_fn=tf.contrib.metrics.streaming_accuracy,prediction_key="classes"),
"precision" : MetricSpec(metric_fn=tf.contrib.metrics.streaming_precision,prediction_key="classes"),
"recall" : MetricSpec(metric_fn=tf.contrib.metrics.streaming_recall,prediction_key="classes")}
#instantiate a validaition monitor
#validation_monitor = tf.contrib.learn.monitors.ValidationMonitor(X_test,y_test,every_n_steps=50)
#self.validation_monitor = tf.contrib.learn.monitors.ValidationMonitor(
# X_test,
# y_test,
# every_n_steps=50,
# metrics=self.validation_metrics,
# early_stopping_metric="loss",
# early_stopping_metric_minimize=True,
# early_stopping_rounds=200)
# Linear Classifier
self.classifier = tf.contrib.learn.LinearClassifier(feature_columns=self.feature_columns,
model_dir="data/linear",
config=tf.contrib.learn.RunConfig(save_checkpoints_secs=1))
#self.classifier.fit(x=X_train,y=y_train,steps=2000,monitors=[validation_monitor])
self.classifier.fit(x=X_train,y=y_train,steps=2000)
def test_partial(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels = {"l1": "l1_value"}
predictions = {"p1": "p1_value", "p2": "p2_value"}
def custom_metric(predictions, labels, stuff, weights=None):
self.assertEqual("p1_value", predictions)
self.assertEqual("l1_value", labels)
self.assertEqual("f2_value", weights)
if stuff:
return "metric_fn_result"
raise ValueError("No stuff.")
spec = MetricSpec(
metric_fn=functools.partial(custom_metric, stuff=5),
label_key="l1",
prediction_key="p1",
weight_key="f2")
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels, predictions))
spec = MetricSpec(
metric_fn=functools.partial(custom_metric, stuff=None),
prediction_key="p1", label_key="l1", weight_key="f2")
with self.assertRaisesRegexp(ValueError, "No stuff."):
spec.create_metric_ops(features, labels, predictions)
def test_partial(self):
features = {
"feature1": "feature1_tensor",
"feature2": "feature2_tensor"
}
labels = {"label1": "label1_tensor"}
predictions = {"pred1": "pred1_tensor", "pred2": "pred2_tensor"}
def custom_metric(predictions, labels, stuff, weights=None):
if stuff:
return predictions, labels, weights
else:
raise ValueError("Nooooo")
partial_metric = functools.partial(custom_metric, stuff=5)
passed = MetricSpec(metric_fn=partial_metric,
label_key="label1",
prediction_key="pred1",
weight_key="feature2").create_metric_ops(
features, labels, predictions)
self.assertEqual(passed[0], "pred1_tensor")
self.assertEqual(passed[1], "label1_tensor")
self.assertEqual(passed[2], "feature2_tensor")
broken_partial_metric = functools.partial(custom_metric, stuff=0)
self.assertRaisesRegexp(
ValueError, "Nooooo",
MetricSpec(metric_fn=broken_partial_metric,
prediction_key="pred1",
label_key="label1",
weight_key="feature2").create_metric_ops, features,
labels, predictions)
def _create_evaluation_metrics():
"""Creates the evaluation metrics for the model.
Returns:
A dictionary with keys that are strings naming the evaluation
metrics and values that are functions taking arguments of
(predictions, targets), returning a tuple of a tensor of the metric's
value together with an op to update the metric's value.
"""
eval_metrics = {}
for k in [1]:
eval_metrics["precision_at_%d" % k] = MetricSpec(
metric_fn=functools.partial(
tf.contrib.metrics.streaming_sparse_precision_at_k, k=k))
eval_metrics["recall_at_%d" %
k] = MetricSpec(metric_fn=functools.partial(
tf.contrib.metrics.streaming_sparse_recall_at_k, k=k))
for class_id, class_label in _get_eval_labels():
k = 1
eval_metrics["precision_at_%d_%s" % (k, class_label)] = MetricSpec(
metric_fn=functools.partial(
tf.contrib.metrics.streaming_sparse_precision_at_k,
k=k,
class_id=class_id))
eval_metrics["recall_at_%d_%s" % (k, class_label)] = MetricSpec(
metric_fn=functools.partial(
tf.contrib.metrics.streaming_sparse_recall_at_k,
k=k,
class_id=class_id))
return eval_metrics
def test_named_args_with_weights(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels_ = {"l1": "l1_value", "l2": "l2_value"}
predictions_ = {"p1": "p1_value", "p2": "p2_value"}
def _fn0(predictions, labels, weights=None):
self.assertEqual("p1_value", predictions)
self.assertEqual("l1_value", labels)
self.assertEqual("f2_value", weights)
return "metric_fn_result"
def _fn1(predictions, targets, weights=None):
self.assertEqual("p1_value", predictions)
self.assertEqual("l1_value", targets)
self.assertEqual("f2_value", weights)
return "metric_fn_result"
def _fn2(prediction, label, weight=None):
self.assertEqual("p1_value", prediction)
self.assertEqual("l1_value", label)
self.assertEqual("f2_value", weight)
return "metric_fn_result"
def _fn3(prediction, target, weight=None):
self.assertEqual("p1_value", prediction)
self.assertEqual("l1_value", target)
self.assertEqual("f2_value", weight)
return "metric_fn_result"
for fn in (_fn0, _fn1, _fn2, _fn3):
spec = MetricSpec(
metric_fn=fn, prediction_key="p1", label_key="l1", weight_key="f2")
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels_, predictions_))
def train(model, dataset_dir, model_dir, batch_size, train_steps,
is_evaluate_accuracy, valid_every_n_steps, early_stopping_rounds):
def input_fn_test():
return dataset_dsb17.inputs(data_dir=dataset_dir,
mode=dataset_dsb17.DATASET_MODE.TEST,
batch_size=batch_size)
def input_fn_train():
return dataset_dsb17.inputs(data_dir=dataset_dir,
mode=dataset_dsb17.DATASET_MODE.TRAIN,
batch_size=batch_size)
def input_fn_valid():
return dataset_dsb17.inputs(data_dir=dataset_dir,
mode=dataset_dsb17.DATASET_MODE.VALID,
batch_size=batch_size)
loginfo("Start")
loginfo("Create validation monitor")
# Monitors
validation_metrics = {
"accuracy":
MetricSpec(metric_fn=tf.contrib.metrics.streaming_accuracy,
prediction_key=PredictionKey.CLASSES),
"precision":
MetricSpec(metric_fn=tf.contrib.metrics.streaming_precision,
prediction_key=PredictionKey.CLASSES),
"recall":
MetricSpec(metric_fn=tf.contrib.metrics.streaming_recall,
prediction_key=PredictionKey.CLASSES)
}
validation_monitor = learn.monitors.ValidationMonitor(
input_fn=input_fn_valid,
every_n_steps=valid_every_n_steps,
early_stopping_rounds=early_stopping_rounds,
metrics=validation_metrics)
loginfo("Create classifier")
start = time.time()
classifier = learn.Estimator(model_fn=model, model_dir=model_dir)
loginfo("Fit model")
# Fit model.
classifier.fit(input_fn=input_fn_train,
steps=train_steps,
monitors=[validation_monitor])
loginfo("Fit model finished. Time: %.03f s" % (time.time() - start))
start = time.time()
if is_evaluate_accuracy:
loginfo("Evaluate accuracy")
# Evaluate accuracy.
result = classifier.evaluate(input_fn=input_fn_test,
metrics=validation_metrics)
loginfo('Accuracy: {0:f}'.format(result['accuracy']))
loginfo("Time: %.03f s" % (time.time() - start))
loginfo("done")
def test_no_args(self):
def _fn():
self.fail("Expected failure before metric_fn.")
spec = MetricSpec(metric_fn=_fn)
with self.assertRaises(TypeError):
spec.create_metric_ops(
{"f1": "f1_value"}, "labels_value", "predictions_value")
def test_no_args(self):
def _fn():
self.fail("Expected failure before metric_fn.")
spec = MetricSpec(metric_fn=_fn)
with self.assertRaises(TypeError):
spec.create_metric_ops({"f1": "f1_value"}, "labels_value",
"predictions_value")
def testCustomMetrics(self):
"""Tests custom evaluation metrics."""
def _input_fn_train():
# Create 4 rows, one of them (y = x), three of them (y=Not(x))
target = tf.constant([[1], [0], [0], [0]], dtype=tf.float32)
features = {'x': tf.ones(shape=[4, 1], dtype=tf.float32)}
return features, target
def _my_metric_op(predictions, targets):
# For the case of binary classification, the 2nd column of "predictions"
# denotes the model predictions.
predictions = tf.slice(predictions, [0, 1], [-1, 1])
return tf.reduce_sum(tf.mul(predictions, targets))
classifier = tf.contrib.learn.LinearClassifier(
feature_columns=[tf.contrib.layers.real_valued_column('x')])
classifier.fit(input_fn=_input_fn_train, steps=100)
scores = classifier.evaluate(
input_fn=_input_fn_train,
steps=100,
metrics={
'my_accuracy':
MetricSpec(metric_fn=tf.contrib.metrics.streaming_accuracy,
prediction_key='classes'),
'my_precision':
MetricSpec(metric_fn=tf.contrib.metrics.streaming_precision,
prediction_key='classes'),
'my_metric':
MetricSpec(metric_fn=_my_metric_op,
prediction_key='probabilities')
})
self.assertTrue(
set(['loss', 'my_accuracy', 'my_precision',
'my_metric']).issubset(set(scores.keys())))
predictions = classifier.predict(input_fn=_input_fn_train)
self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions),
scores['my_accuracy'])
# Test the case where the 2nd element of the key is neither "classes" nor
# "probabilities".
with self.assertRaises(ValueError):
classifier.evaluate(input_fn=_input_fn_train,
steps=100,
metrics={
('bad_name', 'bad_type'):
tf.contrib.metrics.streaming_auc
})
# Test the case where the tuple of the key doesn't have 2 elements.
with self.assertRaises(ValueError):
classifier.evaluate(
input_fn=_input_fn_train,
steps=100,
metrics={
('bad_length_name', 'classes', 'bad_length'):
tf.contrib.metrics.streaming_accuracy
})
def _create_evaluation_metrics_regression():
eval_metrics = {}
eval_metrics['MAE'] = MetricSpec(
metric_fn=tf.contrib.metrics.streaming_mean_absolute_error,
prediction_key="predictions")
eval_metrics['MSE'] = MetricSpec(
metric_fn=tf.contrib.metrics.streaming_mean_squared_error,
prediction_key="predictions")
return eval_metrics
def main(unused_argv):
# Load datasets.
training_set = tf.contrib.learn.datasets.base.load_csv_with_header(
filename=IRIS_TRAINING, target_dtype=np.int, features_dtype=np.float)
test_set = tf.contrib.learn.datasets.base.load_csv_with_header(
filename=IRIS_TEST, target_dtype=np.int, features_dtype=np.float)
# Specify that all features have real-value data
feature_columns = [tf.contrib.layers.real_valued_column("", dimension=4)]
validation_metrics = {
"accuracy":
MetricSpec(metric_fn=tf.contrib.metrics.streaming_accuracy,
prediction_key="classes"),
"recall":
MetricSpec(metric_fn=tf.contrib.metrics.streaming_recall,
prediction_key="classes"),
"precision":
MetricSpec(metric_fn=tf.contrib.metrics.streaming_precision,
prediction_key="classes")
}
validation_monitor = tf.contrib.learn.monitors.ValidationMonitor(
test_set.data,
test_set.target,
every_n_steps=50,
metrics=validation_metrics,
early_stopping_metric="loss",
early_stopping_metric_minimize=True,
early_stopping_rounds=200)
# Build 3 layer DNN with 10, 20, 10 units respectively.
classifier = tf.contrib.learn.DNNClassifier(
feature_columns=feature_columns,
hidden_units=[10, 20, 10],
n_classes=3,
model_dir="/tmp/iris_model",
config=tf.contrib.learn.RunConfig(save_checkpoints_secs=1))
# Fit model.
classifier.fit(x=training_set.data,
y=training_set.target,
steps=2000,
monitors=[validation_monitor])
# Evaluate accuracy.
accuracy_score = classifier.evaluate(x=test_set.data,
y=test_set.target)["accuracy"]
print(f"Accuracy: {accuracy_score}")
# Classify two new flower samples.
new_samples = np.array([[6.4, 3.2, 4.5, 1.5], [5.8, 3.1, 5.0, 1.7]],
dtype=float)
y = list(classifier.predict(new_samples))
print(f"Predictions: {y}")
def testCustomMetrics(self):
"""Tests custom evaluation metrics."""
def _input_fn(num_epochs=None):
# Create 4 rows, one of them (y = x), three of them (y=Not(x))
labels = constant_op.constant([[1.], [0.], [0.], [0.]])
features = {
'x':
input_lib.limit_epochs(
array_ops.ones(shape=[4, 1], dtype=dtypes.float32),
num_epochs=num_epochs),
}
return features, labels
def _my_metric_op(predictions, labels):
return math_ops.reduce_sum(math_ops.multiply(predictions, labels))
regressor = debug.DebugRegressor(
config=run_config.RunConfig(tf_random_seed=1))
regressor.fit(input_fn=_input_fn, steps=5)
scores = regressor.evaluate(
input_fn=_input_fn,
steps=1,
metrics={
'my_error':
MetricSpec(
metric_fn=metric_ops.streaming_mean_squared_error,
prediction_key='scores'),
'my_metric':
MetricSpec(metric_fn=_my_metric_op, prediction_key='scores')
})
self.assertIn('loss', set(scores.keys()))
self.assertIn('my_error', set(scores.keys()))
self.assertIn('my_metric', set(scores.keys()))
predict_input_fn = functools.partial(_input_fn, num_epochs=1)
predictions = np.array(
list(regressor.predict_scores(input_fn=predict_input_fn)))
self.assertAlmostEqual(
_sklearn.mean_squared_error(np.array([1, 0, 0, 0]), predictions),
scores['my_error'])
# Tests the case where the prediction_key is not "scores".
with self.assertRaisesRegexp(KeyError, 'bad_type'):
regressor.evaluate(
input_fn=_input_fn,
steps=1,
metrics={
'bad_name':
MetricSpec(
metric_fn=metric_ops.streaming_auc,
prediction_key='bad_type')
})
def test_named_labels_no_predictions(self):
features = {"f1": "f1_value"}
labels_ = "labels_value"
predictions_ = "predictions_value"
def _fn(labels):
self.assertEqual(labels_, labels)
return "metric_fn_result"
spec = MetricSpec(metric_fn=_fn)
with self.assertRaises(TypeError):
spec.create_metric_ops(features, labels_, predictions_)
def test_no_named_labels_or_predictions_2args(self):
features = {"f1": "f1_value"}
labels_ = "labels_value"
predictions_ = "predictions_value"
def _fn(a, b):
del a, b
self.fail("Expected failure before metric_fn.")
spec = MetricSpec(metric_fn=_fn)
with self.assertRaises(TypeError):
spec.create_metric_ops(features, labels_, predictions_)
def test_named_labels_no_predictions(self):
features = {"f1": "f1_value"}
labels_ = "labels_value"
predictions_ = "predictions_value"
def _fn(labels):
self.assertEqual(labels_, labels)
return "metric_fn_result"
spec = MetricSpec(metric_fn=_fn)
with self.assertRaises(TypeError):
spec.create_metric_ops(features, labels_, predictions_)
def test_no_named_labels_or_predictions_2args(self):
features = {"f1": "f1_value"}
labels_ = "labels_value"
predictions_ = "predictions_value"
def _fn(a, b):
del a, b
self.fail("Expected failure before metric_fn.")
spec = MetricSpec(metric_fn=_fn)
with self.assertRaises(TypeError):
spec.create_metric_ops(features, labels_, predictions_)
def test_no_named_labels_or_predictions_1arg(self):
features = {"f1": "f1_value"}
labels_ = "labels_value"
predictions_ = "predictions_value"
def _fn(a):
self.assertEqual(predictions_, a)
return "metric_fn_result"
spec = MetricSpec(metric_fn=_fn)
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels_, predictions_))
def test_no_named_labels_or_predictions_1arg(self):
features = {"f1": "f1_value"}
labels_ = "labels_value"
predictions_ = "predictions_value"
def _fn(a):
self.assertEqual(predictions_, a)
return "metric_fn_result"
spec = MetricSpec(metric_fn=_fn)
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels_, predictions_))
def test_single_labels_with_key(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels = "l1_value"
predictions = {"p1": "p1_value", "p2": "p2_value"}
def _fn(predictions, labels, weights=None):
del labels, predictions, weights
self.fail("Expected failure before metric_fn.")
spec = MetricSpec(
metric_fn=_fn, prediction_key="p1", label_key="l1", weight_key="f2")
with self.assertRaisesRegexp(
ValueError, "MetricSpec with label_key specified requires labels dict"):
spec.create_metric_ops(features, labels, predictions)
def test_no_named_predictions_named_labels_second_arg(self):
features = {"f1": "f1_value"}
labels_ = "labels_value"
predictions_ = "predictions_value"
def _fn(predictions_by_another_name, labels):
self.assertEqual(predictions_, predictions_by_another_name)
self.assertEqual(labels_, labels)
return "metric_fn_result"
spec = MetricSpec(metric_fn=_fn)
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels_, predictions_))
def _create_evaluation_metrics_classify():
eval_metrics = {}
eval_metrics['accuracy'] = MetricSpec(
metric_fn=tf.contrib.metrics.streaming_accuracy,
prediction_key="predictions")
# eval_metrics['accuracy1'] = MetricSpec(metric_fn=evaluate_accuracy)
eval_metrics['recall'] = MetricSpec(
metric_fn=tf.contrib.metrics.streaming_recall,
prediction_key="predictions")
eval_metrics['precision'] = MetricSpec(
metric_fn=tf.contrib.metrics.streaming_precision,
prediction_key="predictions")
return eval_metrics
def test_single_labels_with_key(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels = "l1_value"
predictions = {"p1": "p1_value", "p2": "p2_value"}
def _fn(predictions, labels, weights=None):
del labels, predictions, weights
self.fail("Expected failure before metric_fn.")
spec = MetricSpec(
metric_fn=_fn, prediction_key="p1", label_key="l1", weight_key="f2")
with self.assertRaisesRegexp(
ValueError, "MetricSpec with label_key specified requires labels dict"):
spec.create_metric_ops(features, labels, predictions)
def test_no_named_predictions_named_labels_second_arg(self):
features = {"f1": "f1_value"}
labels_ = "labels_value"
predictions_ = "predictions_value"
def _fn(predictions_by_another_name, labels):
self.assertEqual(predictions_, predictions_by_another_name)
self.assertEqual(labels_, labels)
return "metric_fn_result"
spec = MetricSpec(metric_fn=_fn)
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels_, predictions_))
def testCustomMetricsWithMetricSpec(self):
"""Tests custom evaluation metrics that use MetricSpec."""
def _input_fn(num_epochs=None):
# Create 4 rows, one of them (y = x), three of them (y=Not(x))
labels = tf.constant([[1.], [0.], [0.], [0.]])
features = {
'x': tf.train.limit_epochs(
tf.ones(shape=[4, 1], dtype=tf.float32), num_epochs=num_epochs),
}
return features, labels
def _my_metric_op(predictions, labels):
return tf.reduce_sum(tf.multiply(predictions, labels))
regressor = tf.contrib.learn.DNNRegressor(
feature_columns=[tf.contrib.layers.real_valued_column('x')],
hidden_units=[3, 3],
config=tf.contrib.learn.RunConfig(tf_random_seed=1))
regressor.fit(input_fn=_input_fn, steps=5)
scores = regressor.evaluate(
input_fn=_input_fn,
steps=1,
metrics={
'my_error': MetricSpec(
metric_fn=tf.contrib.metrics.streaming_mean_squared_error,
prediction_key='scores'),
'my_metric': MetricSpec(
metric_fn=_my_metric_op,
prediction_key='scores')
})
self.assertIn('loss', set(scores.keys()))
self.assertIn('my_error', set(scores.keys()))
self.assertIn('my_metric', set(scores.keys()))
predict_input_fn = functools.partial(_input_fn, num_epochs=1)
predictions = np.array(list(regressor.predict(input_fn=predict_input_fn)))
self.assertAlmostEqual(
_sklearn.mean_squared_error(np.array([1, 0, 0, 0]), predictions),
scores['my_error'])
# Tests the case where the prediction_key is not "scores".
with self.assertRaisesRegexp(KeyError, 'bad_type'):
regressor.evaluate(
input_fn=_input_fn,
steps=1,
metrics={
'bad_name': MetricSpec(
metric_fn=tf.contrib.metrics.streaming_auc,
prediction_key='bad_type')})
def test_labels_dict_no_key(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels = {"l1": "l1_value", "l2": "l2_value"}
predictions = {"p1": "p1_value", "p2": "p2_value"}
def _fn(labels, predictions, weights=None):
del labels, predictions, weights
self.fail("Expected failure before metric_fn.")
spec = MetricSpec(metric_fn=_fn, prediction_key="p1", weight_key="f2")
with self.assertRaisesRegexp(
ValueError,
"MetricSpec without specified label_key requires labels tensor or"
" single element dict"):
spec.create_metric_ops(features, labels, predictions)
def test_single_label(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels_ = "l1_value"
predictions_ = {"p1": "p1_value", "p2": "p2_value"}
def _fn(predictions, labels, weights=None):
self.assertEqual("p1_value", predictions)
self.assertEqual(labels_, labels)
self.assertEqual("f2_value", weights)
return "metric_fn_result"
spec = MetricSpec(metric_fn=_fn, prediction_key="p1", weight_key="f2")
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels_, predictions_))
def test_labels_dict_no_key(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels = {"l1": "l1_value", "l2": "l2_value"}
predictions = {"p1": "p1_value", "p2": "p2_value"}
def _fn(labels, predictions, weights=None):
del labels, predictions, weights
self.fail("Expected failure before metric_fn.")
spec = MetricSpec(metric_fn=_fn, prediction_key="p1", weight_key="f2")
with self.assertRaisesRegexp(
ValueError,
"MetricSpec without specified label_key requires labels tensor or"
" single element dict"):
spec.create_metric_ops(features, labels, predictions)
def test_single_label(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels_ = "l1_value"
predictions_ = {"p1": "p1_value", "p2": "p2_value"}
def _fn(predictions, labels, weights=None):
self.assertEqual("p1_value", predictions)
self.assertEqual(labels_, labels)
self.assertEqual("f2_value", weights)
return "metric_fn_result"
spec = MetricSpec(metric_fn=_fn, prediction_key="p1", weight_key="f2")
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels_, predictions_))
def contrib_learn_classifier_test():
"""Test tf.contrib.learn.DNN_classifier."""
language_column = layers.sparse_column_with_hash_bucket(
"language", hash_bucket_size=20)
feature_columns = [
layers.embedding_column(language_column, dimension=3),
layers.real_valued_column("age", dtype=tf.int64)
]
classifier = learn.DNNClassifier(
n_classes=3,
feature_columns=feature_columns,
hidden_units=[100, 100],
config=learn.RunConfig(tf_random_seed=1,
model_dir="../model_saver/estimators/"
"DNN_classifier_01"),
# optimizer=optimizer_exp_decay
)
classifier.fit(input_fn=_input_fn, steps=10000)
print("variables_names:\n", str(classifier.get_variable_names()))
# scores = classifier.evaluate(input_fn=_input_fn,
# steps=100)
# print("scores:\n", str(scores))
scores = classifier.evaluate(
input_fn=_input_fn,
steps=100,
metrics={
'my_accuracy':
MetricSpec(metric_fn=metrics.streaming_accuracy,
prediction_key="classes"),
'my_precision':
MetricSpec(metric_fn=metrics.streaming_precision,
prediction_key="classes"),
'my_recall':
MetricSpec(metric_fn=metrics.streaming_recall,
prediction_key="classes"),
'my_metric':
MetricSpec(metric_fn=my_metric_op, prediction_key="classes")
})
print("scores:\n", str(scores))
predictions = classifier.predict(input_fn=_input_fn,
outputs=["classes", "probabilities"])
print("predictions")
for prediction in predictions:
print(prediction)
def create_evaluation_metrics():
'''
Create a dictionary containing the eval metrics
'''
def custom_streaming_sparse_recall_at_k(predictions, labels, k):
'''
customize my evaluation metric. preditctions has the shape [batch_size, num_class] -> [16, 10]
so we have to change the labels shape [160, 1] -> [16, 10]
:param predictions: shaped probability for each class
:param labels: unshape lebels
:param k: to k classes
:return: recall_at_k metrics for evaluation
'''
split_labs = tf.split(labels, num_or_size_splits=10, axis=0)
shaped_labels = tf.concat(split_labs, axis=1)
return tf.contrib.metrics.streaming_sparse_recall_at_k(
predictions, shaped_labels, k)
eval_metrics = {}
for k in [1, 2, 5, 10]:
eval_metrics["recall_at_%d" %
k] = MetricSpec(metric_fn=functools.partial(
custom_streaming_sparse_recall_at_k, k=k))
return eval_metrics
def test_multiple_prediction_args(self):
def _fn(predictions, prediction, labels):
del predictions, prediction, labels
self.fail("Expected failure before metric_fn.")
with self.assertRaisesRegexp(ValueError, "provide only one of.*prediction"):
MetricSpec(metric_fn=_fn)
def test_multiple_weight_args(self):
def _fn(predictions, labels, weights=None, weight=None):
del predictions, labels, weights, weight
self.fail("Expected failure before metric_fn.")
with self.assertRaisesRegexp(ValueError, "provide only one of.*weight"):
MetricSpec(metric_fn=_fn)
def create_evaluation_metrics():
eval_metrics = {}
for k in [1, 2, 5, 10]:
eval_metrics["recall_at_%d" %
k] = MetricSpec(metric_fn=functools.partial(
tf.contrib.metrics.streaming_sparse_recall_at_k, k=k))
return eval_metrics
def test_partial(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels = {"l1": "l1_value"}
predictions = {"p1": "p1_value", "p2": "p2_value"}
def custom_metric(predictions, labels, stuff, weights=None):
self.assertEqual("p1_value", predictions)
self.assertEqual("l1_value", labels)
self.assertEqual("f2_value", weights)
if stuff:
return "metric_fn_result"
raise ValueError("No stuff.")
spec = MetricSpec(metric_fn=functools.partial(custom_metric, stuff=5),
label_key="l1",
prediction_key="p1",
weight_key="f2")
self.assertEqual("metric_fn_result",
spec.create_metric_ops(features, labels, predictions))
spec = MetricSpec(metric_fn=functools.partial(custom_metric,
stuff=None),
prediction_key="p1",
label_key="l1",
weight_key="f2")
with self.assertRaisesRegexp(ValueError, "No stuff."):
spec.create_metric_ops(features, labels, predictions)
def fit(self, X_train, y_train, step=2000):
t_fit = 0
t_predict = 0
# Specify that all features have real-value data
feature_columns = [
tf.contrib.layers.real_valued_column("",
dimension=X_train[0].shape[0])
]
#validation_metrics = {"accuracy": tf.contrib.metrics.streaming_accuracy}
validation_metrics = {
"accuracy":
MetricSpec(metric_fn=tf.contrib.metrics.streaming_accuracy,
prediction_key="classes"),
"precision":
MetricSpec(metric_fn=tf.contrib.metrics.streaming_precision,
prediction_key="classes"),
"recall":
MetricSpec(metric_fn=tf.contrib.metrics.streaming_recall,
prediction_key="classes")
}
#instantiate a validaition monitor
#validation_monitor = tf.contrib.learn.monitors.ValidationMonitor(X_test,y_test,every_n_steps=50)
#validation_monitor = tf.contrib.learn.monitors.ValidationMonitor(
# X_test,
# y_test,
# every_n_steps=50,
# metrics=validation_metrics,
# early_stopping_metric="loss",
# early_stopping_metric_minimize=True,
# early_stopping_rounds=200)
# Build 3 layer DNN with 10, 20, 10 units respectively.
self.classifier = tf.contrib.learn.DNNClassifier(
feature_columns=feature_columns,
hidden_units=[10, 20, 10],
n_classes=3,
model_dir=self.model_dir,
config=tf.contrib.learn.RunConfig(save_checkpoints_secs=1))
self.classifier.fit(x=X_train, y=y_train, steps=step)
#self.fit_time = Timer(lambda: classifier.fit(x=X_train,y=y_train,steps=2000))
self.fit_time = 0
def test_partial_str(self):
def custom_metric(predictions, labels, stuff, weights=None):
return predictions, labels, weights, stuff
partial_metric = functools.partial(custom_metric, stuff=5)
metric_spec = MetricSpec(metric_fn=partial_metric,
label_key="label1",
prediction_key="pred1",
weight_key="feature2")
self.assertIn("custom_metric", str(metric_spec))
def test_str(self):
metric_spec = MetricSpec(metric_fn=test_metric,
label_key="label1",
prediction_key="pred1",
weight_key="feature2")
string = str(metric_spec)
self.assertIn("test_metric", string)
self.assertIn("label1", string)
self.assertIn("pred1", string)
self.assertIn("feature2", string)
def test_label_key_without_label_arg(self):
def _fn0(predictions, weights=None):
del predictions, weights
self.fail("Expected failure before metric_fn.")
def _fn1(prediction, weight=None):
del prediction, weight
self.fail("Expected failure before metric_fn.")
for fn in (_fn0, _fn1):
with self.assertRaisesRegexp(ValueError, "label.*missing"):
MetricSpec(metric_fn=fn, label_key="l1")
