def test_partial(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels = {"l1": "l1_value"}
predictions = {"p1": "p1_value", "p2": "p2_value"}
def custom_metric(predictions, labels, stuff, weights=None):
self.assertEqual("p1_value", predictions)
self.assertEqual("l1_value", labels)
self.assertEqual("f2_value", weights)
if stuff:
return "metric_fn_result"
raise ValueError("No stuff.")
spec = MetricSpec(metric_fn=functools.partial(custom_metric, stuff=5),
label_key="l1",
prediction_key="p1",
weight_key="f2")
self.assertEqual("metric_fn_result",
spec.create_metric_ops(features, labels, predictions))
spec = MetricSpec(metric_fn=functools.partial(custom_metric,
stuff=None),
prediction_key="p1",
label_key="l1",
weight_key="f2")
with self.assertRaisesRegexp(ValueError, "No stuff."):
spec.create_metric_ops(features, labels, predictions)
def test_partial(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels = {"l1": "l1_value"}
predictions = {"p1": "p1_value", "p2": "p2_value"}
def custom_metric(predictions, labels, stuff, weights=None):
self.assertEqual("p1_value", predictions)
self.assertEqual("l1_value", labels)
self.assertEqual("f2_value", weights)
if stuff:
return "metric_fn_result"
raise ValueError("No stuff.")
spec = MetricSpec(
metric_fn=functools.partial(custom_metric, stuff=5),
label_key="l1",
prediction_key="p1",
weight_key="f2")
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels, predictions))
spec = MetricSpec(
metric_fn=functools.partial(custom_metric, stuff=None),
prediction_key="p1", label_key="l1", weight_key="f2")
with self.assertRaisesRegexp(ValueError, "No stuff."):
spec.create_metric_ops(features, labels, predictions)
def test_no_args(self):
def _fn():
self.fail("Expected failure before metric_fn.")
spec = MetricSpec(metric_fn=_fn)
with self.assertRaises(TypeError):
spec.create_metric_ops(
{"f1": "f1_value"}, "labels_value", "predictions_value")
def test_no_args(self):
def _fn():
self.fail("Expected failure before metric_fn.")
spec = MetricSpec(metric_fn=_fn)
with self.assertRaises(TypeError):
spec.create_metric_ops({"f1": "f1_value"}, "labels_value",
"predictions_value")
def test_named_labels_no_predictions(self):
features = {"f1": "f1_value"}
labels_ = "labels_value"
predictions_ = "predictions_value"
def _fn(labels):
self.assertEqual(labels_, labels)
return "metric_fn_result"
spec = MetricSpec(metric_fn=_fn)
with self.assertRaises(TypeError):
spec.create_metric_ops(features, labels_, predictions_)
def test_no_named_labels_or_predictions_2args(self):
features = {"f1": "f1_value"}
labels_ = "labels_value"
predictions_ = "predictions_value"
def _fn(a, b):
del a, b
self.fail("Expected failure before metric_fn.")
spec = MetricSpec(metric_fn=_fn)
with self.assertRaises(TypeError):
spec.create_metric_ops(features, labels_, predictions_)
def test_no_named_labels_or_predictions_2args(self):
features = {"f1": "f1_value"}
labels_ = "labels_value"
predictions_ = "predictions_value"
def _fn(a, b):
del a, b
self.fail("Expected failure before metric_fn.")
spec = MetricSpec(metric_fn=_fn)
with self.assertRaises(TypeError):
spec.create_metric_ops(features, labels_, predictions_)
def test_named_labels_no_predictions(self):
features = {"f1": "f1_value"}
labels_ = "labels_value"
predictions_ = "predictions_value"
def _fn(labels):
self.assertEqual(labels_, labels)
return "metric_fn_result"
spec = MetricSpec(metric_fn=_fn)
with self.assertRaises(TypeError):
spec.create_metric_ops(features, labels_, predictions_)
def test_single_labels_with_key(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels = "l1_value"
predictions = {"p1": "p1_value", "p2": "p2_value"}
def _fn(predictions, labels, weights=None):
del labels, predictions, weights
self.fail("Expected failure before metric_fn.")
spec = MetricSpec(
metric_fn=_fn, prediction_key="p1", label_key="l1", weight_key="f2")
with self.assertRaisesRegexp(
ValueError, "MetricSpec with label_key specified requires labels dict"):
spec.create_metric_ops(features, labels, predictions)
def test_single_labels_with_key(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels = "l1_value"
predictions = {"p1": "p1_value", "p2": "p2_value"}
def _fn(predictions, labels, weights=None):
del labels, predictions, weights
self.fail("Expected failure before metric_fn.")
spec = MetricSpec(
metric_fn=_fn, prediction_key="p1", label_key="l1", weight_key="f2")
with self.assertRaisesRegexp(
ValueError, "MetricSpec with label_key specified requires labels dict"):
spec.create_metric_ops(features, labels, predictions)
def test_labels_dict_no_key(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels = {"l1": "l1_value", "l2": "l2_value"}
predictions = {"p1": "p1_value", "p2": "p2_value"}
def _fn(labels, predictions, weights=None):
del labels, predictions, weights
self.fail("Expected failure before metric_fn.")
spec = MetricSpec(metric_fn=_fn, prediction_key="p1", weight_key="f2")
with self.assertRaisesRegexp(
ValueError,
"MetricSpec without specified label_key requires labels tensor or"
" single element dict"):
spec.create_metric_ops(features, labels, predictions)
def test_labels_dict_no_key(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels = {"l1": "l1_value", "l2": "l2_value"}
predictions = {"p1": "p1_value", "p2": "p2_value"}
def _fn(labels, predictions, weights=None):
del labels, predictions, weights
self.fail("Expected failure before metric_fn.")
spec = MetricSpec(metric_fn=_fn, prediction_key="p1", weight_key="f2")
with self.assertRaisesRegexp(
ValueError,
"MetricSpec without specified label_key requires labels tensor or"
" single element dict"):
spec.create_metric_ops(features, labels, predictions)
def test_named_args_with_weights(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels_ = {"l1": "l1_value", "l2": "l2_value"}
predictions_ = {"p1": "p1_value", "p2": "p2_value"}
def _fn0(predictions, labels, weights=None):
self.assertEqual("p1_value", predictions)
self.assertEqual("l1_value", labels)
self.assertEqual("f2_value", weights)
return "metric_fn_result"
def _fn1(predictions, targets, weights=None):
self.assertEqual("p1_value", predictions)
self.assertEqual("l1_value", targets)
self.assertEqual("f2_value", weights)
return "metric_fn_result"
def _fn2(prediction, label, weight=None):
self.assertEqual("p1_value", prediction)
self.assertEqual("l1_value", label)
self.assertEqual("f2_value", weight)
return "metric_fn_result"
def _fn3(prediction, target, weight=None):
self.assertEqual("p1_value", prediction)
self.assertEqual("l1_value", target)
self.assertEqual("f2_value", weight)
return "metric_fn_result"
for fn in (_fn0, _fn1, _fn2, _fn3):
spec = MetricSpec(
metric_fn=fn, prediction_key="p1", label_key="l1", weight_key="f2")
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels_, predictions_))
def test_named_args_no_weights(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels_ = {"l1": "l1_value", "l2": "l2_value"}
predictions_ = {"p1": "p1_value", "p2": "p2_value"}
def _fn0(predictions, labels):
self.assertEqual("p1_value", predictions)
self.assertEqual("l1_value", labels)
return "metric_fn_result"
def _fn1(predictions, targets):
self.assertEqual("p1_value", predictions)
self.assertEqual("l1_value", targets)
return "metric_fn_result"
def _fn2(prediction, label):
self.assertEqual("p1_value", prediction)
self.assertEqual("l1_value", label)
return "metric_fn_result"
def _fn3(prediction, target):
self.assertEqual("p1_value", prediction)
self.assertEqual("l1_value", target)
return "metric_fn_result"
for fn in (_fn0, _fn1, _fn2, _fn3):
spec = MetricSpec(metric_fn=fn,
prediction_key="p1",
label_key="l1")
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels_, predictions_))
def test_no_named_labels_or_predictions_1arg(self):
features = {"f1": "f1_value"}
labels_ = "labels_value"
predictions_ = "predictions_value"
def _fn(a):
self.assertEqual(predictions_, a)
return "metric_fn_result"
spec = MetricSpec(metric_fn=_fn)
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels_, predictions_))
def test_no_named_labels_or_predictions_1arg(self):
features = {"f1": "f1_value"}
labels_ = "labels_value"
predictions_ = "predictions_value"
def _fn(a):
self.assertEqual(predictions_, a)
return "metric_fn_result"
spec = MetricSpec(metric_fn=_fn)
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels_, predictions_))
def test_no_named_predictions_named_labels_second_arg(self):
features = {"f1": "f1_value"}
labels_ = "labels_value"
predictions_ = "predictions_value"
def _fn(predictions_by_another_name, labels):
self.assertEqual(predictions_, predictions_by_another_name)
self.assertEqual(labels_, labels)
return "metric_fn_result"
spec = MetricSpec(metric_fn=_fn)
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels_, predictions_))
def test_no_named_predictions_named_labels_second_arg(self):
features = {"f1": "f1_value"}
labels_ = "labels_value"
predictions_ = "predictions_value"
def _fn(predictions_by_another_name, labels):
self.assertEqual(predictions_, predictions_by_another_name)
self.assertEqual(labels_, labels)
return "metric_fn_result"
spec = MetricSpec(metric_fn=_fn)
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels_, predictions_))
def test_single_label(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels_ = "l1_value"
predictions_ = {"p1": "p1_value", "p2": "p2_value"}
def _fn(predictions, labels, weights=None):
self.assertEqual("p1_value", predictions)
self.assertEqual(labels_, labels)
self.assertEqual("f2_value", weights)
return "metric_fn_result"
spec = MetricSpec(metric_fn=_fn, prediction_key="p1", weight_key="f2")
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels_, predictions_))
def test_single_label(self):
features = {"f1": "f1_value", "f2": "f2_value"}
labels_ = "l1_value"
predictions_ = {"p1": "p1_value", "p2": "p2_value"}
def _fn(predictions, labels, weights=None):
self.assertEqual("p1_value", predictions)
self.assertEqual(labels_, labels)
self.assertEqual("f2_value", weights)
return "metric_fn_result"
spec = MetricSpec(metric_fn=_fn, prediction_key="p1", weight_key="f2")
self.assertEqual(
"metric_fn_result",
spec.create_metric_ops(features, labels_, predictions_))
