def testMultiClassWithInvalidNClass(self):
try:
head_lib._multi_class_head(n_classes=1)
self.fail("Softmax with no n_classes did not raise error.")
except ValueError:
# Expected
pass
def testEval(self):
head1 = head_lib._multi_class_head(
n_classes=3, label_name="label1", head_name="head1")
head2 = head_lib._multi_class_head(
n_classes=4, label_name="label2", head_name="head2")
head = head_lib._multi_head((head1, head2), (1, .5))
labels = {
"label1": (1,),
"label2": (1,)
}
model_fn_ops = head.create_model_fn_ops(
features={"weights": (2.0, 10.0)},
labels=labels,
mode=model_fn.ModeKeys.EVAL,
train_op_fn=_noop_train_op,
logits=((-0.7, 0.2, .1, .1, .1, .1, .1),))
self.assertIsNotNone(model_fn_ops.predictions)
self.assertIsNotNone(model_fn_ops.loss)
self.assertIsNone(model_fn_ops.train_op)
self.assertIsNotNone(model_fn_ops.eval_metric_ops)
self.assertIsNone(model_fn_ops.output_alternatives)
metric_ops = model_fn_ops.eval_metric_ops
# Tests eval keys.
self.assertIn("accuracy/head1", metric_ops.keys())
self.assertIn("accuracy/head2", metric_ops.keys())
def testTrain_withHeadWeights(self):
head1 = head_lib._multi_class_head(n_classes=3,
label_name="label1",
head_name="head1")
head2 = head_lib._multi_class_head(n_classes=4,
label_name="label2",
head_name="head2")
head = head_lib._multi_head([head1, head2], [1, .5])
logits = constant_op.constant([[-0.7, 0.2, .1, .1, .1, .1, .1]])
labels = {
"label1": constant_op.constant([1]),
"label2": constant_op.constant([1])
}
features = {"weights": constant_op.constant([2.0, 10.0])}
model_fn_ops = head.head_ops(features,
labels,
model_fn.ModeKeys.TRAIN,
_noop_train_op,
logits=logits)
self.assertEquals(None, model_fn_ops.predictions)
self.assertTrue(model_fn_ops.loss is not None)
self.assertTrue(model_fn_ops.train_op is not None)
self.assertFalse(model_fn_ops.eval_metric_ops)
self.assertEquals(None, model_fn_ops.signature_fn)
self.assertEquals(None, model_fn_ops.output_alternatives)
with session.Session() as sess:
self.assertAlmostEqual(1.531,
sess.run(model_fn_ops.loss),
places=3)
def testMultiClassWithInvalidNClass(self):
try:
head_lib._multi_class_head(n_classes=1)
self.fail("Softmax with no n_classes did not raise error.")
except ValueError:
# Expected
pass
def testEval(self):
head1 = head_lib._multi_class_head(n_classes=3,
label_name="label1",
head_name="head1")
head2 = head_lib._multi_class_head(n_classes=4,
label_name="label2",
head_name="head2")
head = head_lib._multi_head([head1, head2], [1, .5])
logits = constant_op.constant([[-0.7, 0.2, .1, .1, .1, .1, .1]])
labels = {
"label1": constant_op.constant([1]),
"label2": constant_op.constant([1])
}
features = {"weights": constant_op.constant([2.0, 10.0])}
model_fn_ops = head.head_ops(features,
labels,
model_fn.ModeKeys.EVAL,
_noop_train_op,
logits=logits)
self.assertTrue(model_fn_ops.predictions)
self.assertTrue(model_fn_ops.loss is not None)
self.assertEquals(None, model_fn_ops.train_op)
self.assertTrue(model_fn_ops.eval_metric_ops)
self.assertEquals(None, model_fn_ops.signature_fn)
self.assertEquals(None, model_fn_ops.output_alternatives)
metric_ops = model_fn_ops.eval_metric_ops
# Tests eval keys
self.assertTrue("accuracy/head1" in metric_ops.keys())
self.assertTrue("accuracy/head2" in metric_ops.keys())
def testTrain_withNoHeadWeights(self):
head1 = head_lib._multi_class_head(
n_classes=3, label_name="label1", head_name="head1")
head2 = head_lib._multi_class_head(
n_classes=4, label_name="label2", head_name="head2")
head = head_lib._multi_head((head1, head2))
labels = {
"label1": (1,),
"label2": (1,)
}
model_fn_ops = head.create_model_fn_ops(
features={"weights": (2.0, 10.0)},
labels=labels,
mode=model_fn.ModeKeys.TRAIN,
train_op_fn=_noop_train_op,
logits=((-0.7, 0.2, .1, .1, .1, .1, .1),))
self.assertIsNone(model_fn_ops.predictions)
self.assertIsNotNone(model_fn_ops.loss)
self.assertIsNotNone(model_fn_ops.train_op)
self.assertFalse(model_fn_ops.eval_metric_ops)
self.assertIsNone(model_fn_ops.output_alternatives)
with session.Session() as sess:
self.assertAlmostEqual(2.224, sess.run(model_fn_ops.loss), places=3)
def testEval(self):
head1 = head_lib._multi_class_head(
n_classes=3, label_name="label1", head_name="head1")
head2 = head_lib._multi_class_head(
n_classes=4, label_name="label2", head_name="head2")
head = head_lib._multi_head((head1, head2), (1, .5))
labels = {
"label1": (1,),
"label2": (1,)
}
model_fn_ops = head.create_model_fn_ops(
features={"weights": (2.0, 10.0)},
labels=labels,
mode=model_fn.ModeKeys.EVAL,
train_op_fn=_noop_train_op,
logits=((-0.7, 0.2, .1, .1, .1, .1, .1),))
self.assertIsNotNone(model_fn_ops.predictions)
self.assertIsNotNone(model_fn_ops.loss)
self.assertIsNone(model_fn_ops.train_op)
self.assertIsNotNone(model_fn_ops.eval_metric_ops)
self.assertIsNone(model_fn_ops.output_alternatives)
metric_ops = model_fn_ops.eval_metric_ops
# Tests eval keys.
self.assertIn("accuracy/head1", metric_ops.keys())
self.assertIn("accuracy/head2", metric_ops.keys())
def testTrain_withNoHeadWeights(self):
head1 = head_lib._multi_class_head(n_classes=3, label_name="label1",
head_name="head1")
head2 = head_lib._multi_class_head(n_classes=4, label_name="label2",
head_name="head2")
head = head_lib._multi_head([head1, head2])
logits = tf.constant([[-0.7, 0.2, .1, .1, .1, .1, .1]])
labels = {
"label1": tf.constant([1]),
"label2": tf.constant([1])
}
features = {"weights": tf.constant([2.0, 10.0])}
model_fn_ops = head.head_ops(features, labels,
tf.contrib.learn.ModeKeys.TRAIN,
_noop_train_op, logits=logits)
self.assertEquals(None, model_fn_ops.predictions)
self.assertTrue(model_fn_ops.loss is not None)
self.assertTrue(model_fn_ops.train_op is not None)
self.assertFalse(model_fn_ops.eval_metric_ops)
self.assertEquals(None, model_fn_ops.signature_fn)
self.assertEquals(None, model_fn_ops.output_alternatives)
with tf.Session() as sess:
self.assertAlmostEqual(2.224, sess.run(model_fn_ops.loss), places=3)
def testTrain_withNoHeadWeights(self):
head1 = head_lib._multi_class_head(
n_classes=3, label_name="label1", head_name="head1")
head2 = head_lib._multi_class_head(
n_classes=4, label_name="label2", head_name="head2")
head = head_lib._multi_head((head1, head2))
labels = {
"label1": (1,),
"label2": (1,)
}
model_fn_ops = head.create_model_fn_ops(
features={"weights": (2.0, 10.0)},
labels=labels,
mode=model_fn.ModeKeys.TRAIN,
train_op_fn=_noop_train_op,
logits=((-0.7, 0.2, .1, .1, .1, .1, .1),))
self.assertIsNone(model_fn_ops.predictions)
self.assertIsNotNone(model_fn_ops.loss)
self.assertIsNotNone(model_fn_ops.train_op)
self.assertFalse(model_fn_ops.eval_metric_ops)
self.assertIsNone(model_fn_ops.output_alternatives)
with session.Session() as sess:
self.assertAlmostEqual(2.224, sess.run(model_fn_ops.loss), places=3)
def testEval(self):
head1 = head_lib._multi_class_head(n_classes=3, label_name="label1",
head_name="head1")
head2 = head_lib._multi_class_head(n_classes=4, label_name="label2",
head_name="head2")
head = head_lib._multi_head([head1, head2], [1, .5])
logits = tf.constant([[-0.7, 0.2, .1, .1, .1, .1, .1]])
labels = {
"label1": tf.constant([1]),
"label2": tf.constant([1])
}
features = {"weights": tf.constant([2.0, 10.0])}
model_fn_ops = head.head_ops(features, labels,
tf.contrib.learn.ModeKeys.EVAL,
_noop_train_op, logits=logits)
self.assertTrue(model_fn_ops.predictions)
self.assertTrue(model_fn_ops.loss is not None)
self.assertEquals(None, model_fn_ops.train_op)
self.assertTrue(model_fn_ops.eval_metric_ops)
self.assertEquals(None, model_fn_ops.signature_fn)
self.assertEquals(None, model_fn_ops.output_alternatives)
metric_ops = model_fn_ops.eval_metric_ops
# Tests eval keys
self.assertTrue("accuracy/head1" in metric_ops.keys())
self.assertTrue("accuracy/head2" in metric_ops.keys())
def testInvalidHeads(self):
named_head = head_lib._multi_class_head(
n_classes=3, label_name="label", head_name="head1")
unnamed_head = head_lib._multi_class_head(
n_classes=4, label_name="label")
with self.assertRaisesRegexp(ValueError, "must have names"):
head_lib._multi_head((named_head, unnamed_head))
with self.assertRaisesRegexp(ValueError, "must be SingleHead"):
head_lib._multi_head((named_head, head_lib._multi_head((named_head,))))
def testInvalidHeads(self):
named_head = head_lib._multi_class_head(
n_classes=3, label_name="label", head_name="head1")
unnamed_head = head_lib._multi_class_head(
n_classes=4, label_name="label")
with self.assertRaisesRegexp(ValueError, "must have names"):
head_lib._multi_head((named_head, unnamed_head))
with self.assertRaisesRegexp(ValueError, "must be SingleHead"):
head_lib._multi_head((named_head, head_lib._multi_head((named_head,))))
def testInfer(self):
head1 = head_lib._multi_class_head(n_classes=3,
label_name="label1",
head_name="head1")
head2 = head_lib._multi_class_head(n_classes=4,
label_name="label2",
head_name="head2")
head = head_lib._multi_head([head1, head2], [1, .5])
logits = constant_op.constant([[-0.7, 0.2, .1, .1, .1, .1, .1]])
labels = {
"label1": constant_op.constant([1]),
"label2": constant_op.constant([1])
}
features = {"weights": constant_op.constant([2.0, 10.0])}
model_fn_ops = head.head_ops(features,
labels,
model_fn.ModeKeys.INFER,
_noop_train_op,
logits=logits)
self.assertTrue(model_fn_ops.predictions)
self.assertEquals(None, model_fn_ops.loss)
self.assertEquals(None, model_fn_ops.train_op)
self.assertFalse(model_fn_ops.eval_metric_ops)
self.assertEquals(None, model_fn_ops.signature_fn)
self.assertTrue(len(model_fn_ops.output_alternatives) == 2)
# Tests predictions keys
pred_keys = model_fn_ops.predictions.keys()
self.assertTrue(
("head1", prediction_key.PredictionKey.PROBABILITIES) in pred_keys)
self.assertTrue(("head1",
prediction_key.PredictionKey.CLASSES) in pred_keys)
self.assertTrue(
("head2", prediction_key.PredictionKey.PROBABILITIES) in pred_keys)
self.assertTrue(("head2",
prediction_key.PredictionKey.CLASSES) in pred_keys)
# Tests output alternative
out_alts = model_fn_ops.output_alternatives
self.assertEquals(constants.ProblemType.CLASSIFICATION,
out_alts["head1"][0])
self.assertTrue(prediction_key.PredictionKey.PROBABILITIES in
out_alts["head1"][1].keys())
self.assertTrue(prediction_key.PredictionKey.CLASSES in
out_alts["head1"][1].keys())
self.assertEquals(constants.ProblemType.CLASSIFICATION,
out_alts["head2"][0])
self.assertTrue(prediction_key.PredictionKey.PROBABILITIES in
out_alts["head2"][1].keys())
self.assertTrue(prediction_key.PredictionKey.CLASSES in
out_alts["head2"][1].keys())
def testInfer(self):
head1 = head_lib._multi_class_head(
n_classes=3, label_name="label1", head_name="head1")
head2 = head_lib._multi_class_head(
n_classes=4, label_name="label2", head_name="head2")
head = head_lib._multi_head([head1, head2], [1, .5])
logits = constant_op.constant([[-0.7, 0.2, .1, .1, .1, .1, .1]])
labels = {
"label1": constant_op.constant([1]),
"label2": constant_op.constant([1])
}
features = {"weights": constant_op.constant([2.0, 10.0])}
model_fn_ops = head.head_ops(
features,
labels,
model_fn.ModeKeys.INFER,
_noop_train_op,
logits=logits)
self.assertTrue(model_fn_ops.predictions)
self.assertEquals(None, model_fn_ops.loss)
self.assertEquals(None, model_fn_ops.train_op)
self.assertFalse(model_fn_ops.eval_metric_ops)
self.assertEquals(None, model_fn_ops.signature_fn)
self.assertTrue(len(model_fn_ops.output_alternatives) == 2)
# Tests predictions keys
pred_keys = model_fn_ops.predictions.keys()
self.assertTrue(
("head1", prediction_key.PredictionKey.PROBABILITIES) in pred_keys)
self.assertTrue(
("head1", prediction_key.PredictionKey.CLASSES) in pred_keys)
self.assertTrue(
("head2", prediction_key.PredictionKey.PROBABILITIES) in pred_keys)
self.assertTrue(
("head2", prediction_key.PredictionKey.CLASSES) in pred_keys)
# Tests output alternative
out_alts = model_fn_ops.output_alternatives
self.assertEquals(constants.ProblemType.CLASSIFICATION,
out_alts["head1"][0])
self.assertTrue(prediction_key.PredictionKey.PROBABILITIES in
out_alts["head1"][1].keys())
self.assertTrue(
prediction_key.PredictionKey.CLASSES in out_alts["head1"][1].keys())
self.assertEquals(constants.ProblemType.CLASSIFICATION,
out_alts["head2"][0])
self.assertTrue(prediction_key.PredictionKey.PROBABILITIES in
out_alts["head2"][1].keys())
self.assertTrue(
prediction_key.PredictionKey.CLASSES in out_alts["head2"][1].keys())
def testInfer(self):
head1 = head_lib._multi_class_head(
n_classes=3, label_name="label1", head_name="head1")
head2 = head_lib._multi_class_head(
n_classes=4, label_name="label2", head_name="head2")
head = head_lib._multi_head((head1, head2), (1, .5))
labels = {
"label1": (1,),
"label2": (1,)
}
model_fn_ops = head.create_model_fn_ops(
features={"weights": (2.0, 10.0)},
labels=labels,
mode=model_fn.ModeKeys.INFER,
train_op_fn=_noop_train_op,
logits=((-0.7, 0.2, .1, .1, .1, .1, .1),))
self.assertIsNotNone(model_fn_ops.predictions)
self.assertIsNone(model_fn_ops.loss)
self.assertIsNone(model_fn_ops.train_op)
self.assertFalse(model_fn_ops.eval_metric_ops)
# Tests predictions keys.
self.assertItemsEqual((
("head1", prediction_key.PredictionKey.LOGITS),
("head1", prediction_key.PredictionKey.PROBABILITIES),
("head1", prediction_key.PredictionKey.CLASSES),
("head2", prediction_key.PredictionKey.LOGITS),
("head2", prediction_key.PredictionKey.PROBABILITIES),
("head2", prediction_key.PredictionKey.CLASSES),
), model_fn_ops.predictions.keys())
# Tests output alternative.
self.assertEquals({
"head1": constants.ProblemType.CLASSIFICATION,
"head2": constants.ProblemType.CLASSIFICATION,
}, {
k: v[0] for k, v in six.iteritems(model_fn_ops.output_alternatives)
})
self.assertItemsEqual((
prediction_key.PredictionKey.LOGITS,
prediction_key.PredictionKey.PROBABILITIES,
prediction_key.PredictionKey.CLASSES,
), model_fn_ops.output_alternatives["head1"][1].keys())
self.assertItemsEqual((
prediction_key.PredictionKey.LOGITS,
prediction_key.PredictionKey.PROBABILITIES,
prediction_key.PredictionKey.CLASSES,
), model_fn_ops.output_alternatives["head2"][1].keys())
def testInfer(self):
head1 = head_lib._multi_class_head(
n_classes=3, label_name="label1", head_name="head1")
head2 = head_lib._multi_class_head(
n_classes=4, label_name="label2", head_name="head2")
head = head_lib._multi_head((head1, head2), (1, .5))
labels = {
"label1": (1,),
"label2": (1,)
}
model_fn_ops = head.create_model_fn_ops(
features={"weights": (2.0, 10.0)},
labels=labels,
mode=model_fn.ModeKeys.INFER,
train_op_fn=_noop_train_op,
logits=((-0.7, 0.2, .1, .1, .1, .1, .1),))
self.assertIsNotNone(model_fn_ops.predictions)
self.assertIsNone(model_fn_ops.loss)
self.assertIsNone(model_fn_ops.train_op)
self.assertFalse(model_fn_ops.eval_metric_ops)
self.assertEquals(2, len(model_fn_ops.output_alternatives))
# Tests predictions keys.
pred_keys = model_fn_ops.predictions.keys()
self.assertIn(
("head1", prediction_key.PredictionKey.PROBABILITIES), pred_keys)
self.assertIn(
("head1", prediction_key.PredictionKey.CLASSES), pred_keys)
self.assertIn(
("head2", prediction_key.PredictionKey.PROBABILITIES), pred_keys)
self.assertIn(
("head2", prediction_key.PredictionKey.CLASSES), pred_keys)
# Tests output alternative.
out_alts = model_fn_ops.output_alternatives
self.assertEquals(constants.ProblemType.CLASSIFICATION,
out_alts["head1"][0])
self.assertIn(prediction_key.PredictionKey.PROBABILITIES,
out_alts["head1"][1].keys())
self.assertIn(
prediction_key.PredictionKey.CLASSES, out_alts["head1"][1].keys())
self.assertEquals(constants.ProblemType.CLASSIFICATION,
out_alts["head2"][0])
self.assertIn(prediction_key.PredictionKey.PROBABILITIES,
out_alts["head2"][1].keys())
self.assertIn(
prediction_key.PredictionKey.CLASSES, out_alts["head2"][1].keys())
def testJointLinearModel(self):
"""Tests that loss goes down with training."""
def input_fn():
return {
'age':
sparse_tensor.SparseTensor(values=['1'],
indices=[[0, 0]],
dense_shape=[1, 1]),
'language':
sparse_tensor.SparseTensor(values=['english'],
indices=[[0, 0]],
dense_shape=[1, 1])
}, constant_op.constant([[1]])
language = feature_column.sparse_column_with_hash_bucket(
'language', 100)
age = feature_column.sparse_column_with_hash_bucket('age', 2)
head = head_lib._multi_class_head(n_classes=2)
classifier = _joint_linear_estimator(head,
feature_columns=[age, language])
classifier.fit(input_fn=input_fn, steps=1000)
loss1 = classifier.evaluate(input_fn=input_fn, steps=1)['loss']
classifier.fit(input_fn=input_fn, steps=2000)
loss2 = classifier.evaluate(input_fn=input_fn, steps=1)['loss']
self.assertLess(loss2, loss1)
self.assertLess(loss2, 0.01)
def testMultiClassWithInvalidLogits(self):
head = head_lib._multi_class_head(n_classes=len(self._logits[0]) + 1)
with ops.Graph().as_default(), session.Session():
with self.assertRaisesRegexp(ValueError, "Dimensions.*not compatible"):
head.create_model_fn_ops(
{}, model_fn.ModeKeys.TRAIN, self._labels, _noop_train_op,
logits=self._logits)
def testBinaryClassificationWithCenteredBias(self):
head = head_lib._multi_class_head(n_classes=2,
enable_centered_bias=True)
with ops.Graph().as_default(), session.Session():
logits = constant_op.constant(self._logits)
labels = constant_op.constant(self._labels)
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.head_ops({},
labels,
model_fn.ModeKeys.TRAIN,
_noop_train_op,
logits=logits)
_assert_variables(self,
expected_global=(
"centered_bias_weight:0",
"centered_bias_weight/Adagrad:0",
),
expected_trainable=("centered_bias_weight:0", ))
variables.global_variables_initializer().run()
_assert_summary_tags(self, ["loss", "centered_bias/bias_0"])
expected_loss = .81326175
_assert_metrics(self, expected_loss,
self._expected_eval_metrics(expected_loss),
model_fn_ops)
def testBinaryClassificationWithWeights(self):
n_classes = 2
head = head_lib._multi_class_head(n_classes=n_classes,
weight_column_name="label_weight")
with ops.Graph().as_default(), session.Session():
weights = ((1., ), (0., ))
features = {"label_weight": constant_op.constant(weights)}
logits = constant_op.constant(self._logits)
labels = constant_op.constant(self._labels)
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.head_ops(features,
labels,
model_fn.ModeKeys.TRAIN,
_noop_train_op,
logits=logits)
_assert_no_variables(self)
_assert_summary_tags(self, ["loss"])
expected_total_loss = .31326166
_assert_metrics(
self,
expected_total_loss / len(weights),
{
"accuracy": 1. / 1,
"accuracy/baseline_label_mean": 1. / 1,
"accuracy/threshold_0.500000_mean": 1. / 1,
"auc": 0. / 1,
"labels/actual_label_mean": 1. / 1,
"labels/prediction_mean": .731059, # softmax
# TODO(ptucker): Is this the correct eval loss, sum not average?
"loss": expected_total_loss,
"precision/positive_threshold_0.500000_mean": 1. / 1,
"recall/positive_threshold_0.500000_mean": 1. / 1,
},
model_fn_ops)
def testJointLinearModel(self):
"""Tests that loss goes down with training."""
def input_fn():
return {
'age':
sparse_tensor.SparseTensor(
values=['1'], indices=[[0, 0]], dense_shape=[1, 1]),
'language':
sparse_tensor.SparseTensor(
values=['english'], indices=[[0, 0]], dense_shape=[1, 1])
}, constant_op.constant([[1]])
language = feature_column.sparse_column_with_hash_bucket('language', 100)
age = feature_column.sparse_column_with_hash_bucket('age', 2)
head = head_lib._multi_class_head(n_classes=2)
classifier = _joint_linear_estimator(head, feature_columns=[age, language])
classifier.fit(input_fn=input_fn, steps=1000)
loss1 = classifier.evaluate(input_fn=input_fn, steps=1)['loss']
classifier.fit(input_fn=input_fn, steps=2000)
loss2 = classifier.evaluate(input_fn=input_fn, steps=1)['loss']
self.assertLess(loss2, loss1)
self.assertLess(loss2, 0.01)
def get_conv_classifier():
n_classes = 5
feature_columns = [layers.real_valued_column("", dimension=3)]
# learning_rate = 1.0
# optimizer = AdagradOptimizer(learning_rate)
#
# learning_rate = 1.0
# optimizer = AdadeltaOptimizer(learning_rate=learning_rate)
# ~ 62.55%
learning_rate = 0.01
optimizer = AdamOptimizer(learning_rate, epsilon=0.1)
# learning_rate = 0.05
# optimizer = GradientDescentOptimizer(learning_rate)
# learning_rate = 0.1
# optimizer = RMSPropOptimizer(learning_rate, momentum=0.1)
# learning_rate = 0.1
# optimizer = FtrlOptimizer(learning_rate)
return SKCompat(Estimator(
model_fn=get_conv_model,
params={
'head': head_lib._multi_class_head( # pylint: disable=protected-access
n_classes,
enable_centered_bias=False),
'feature_columns': feature_columns,
'activation_fn': tf.nn.relu,
'learning_rate': learning_rate,
'optimizer': optimizer
},
model_dir='saved_model'))
def testBinaryClassificationWithLogitsInput(self):
n_classes = 2
head = head_lib._multi_class_head(n_classes=n_classes)
with ops.Graph().as_default(), session.Session():
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.create_model_fn_ops(
{}, self._labels, model_fn.ModeKeys.TRAIN, _noop_train_op,
logits_input=((0., 0.), (0., 0.)))
w = ("logits/weights:0", "logits/biases:0")
_assert_variables(
self, expected_global=w, expected_model=w, expected_trainable=w)
variables.global_variables_initializer().run()
_assert_summary_tags(self, ["loss"])
expected_loss = .69314718
label_mean = np.mean(self._labels)
_assert_metrics(self, expected_loss, {
"accuracy": 1. / 2,
"accuracy/baseline_label_mean": label_mean,
"accuracy/threshold_0.500000_mean": 1. / 2,
"auc": 1. / 2,
"labels/actual_label_mean": label_mean,
"labels/prediction_mean": .5, # softmax
"loss": expected_loss,
"precision/positive_threshold_0.500000_mean": 0. / 2,
"recall/positive_threshold_0.500000_mean": 0. / 1,
}, model_fn_ops)
def testBinaryClassificationWithWeights(self):
n_classes = 2
head = head_lib._multi_class_head(
n_classes=n_classes, weight_column_name="label_weight")
with tf.Graph().as_default(), tf.Session():
weights = ((1.,), (0.,))
features = {"label_weight": tf.constant(weights)}
logits = tf.constant(self._logits)
labels = tf.constant(self._labels)
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.head_ops(features, labels,
tf.contrib.learn.ModeKeys.TRAIN,
_noop_train_op, logits=logits)
_assert_no_variables(self)
_assert_summary_tags(self, ["loss"])
expected_total_loss = .31326166
_assert_metrics(
self, expected_total_loss / len(weights), {
"accuracy": 1. / 1,
"accuracy/baseline_label_mean": 1. / 1,
"accuracy/threshold_0.500000_mean": 1. / 1,
"auc": 0. / 1,
"labels/actual_label_mean": 1. / 1,
"labels/prediction_mean": .731059, # softmax
# TODO(ptucker): Is this the correct eval loss, sum not average?
"loss": expected_total_loss,
"precision/positive_threshold_0.500000_mean": 1. / 1,
"recall/positive_threshold_0.500000_mean": 1. / 1,
}, model_fn_ops)
def testRaisesNonEmbeddingColumn(self):
one_hot_language = tf.contrib.layers.one_hot_column(
tf.contrib.layers.sparse_column_with_hash_bucket('language', 10))
params = {
'feature_columns': [one_hot_language],
'head': head_lib._multi_class_head(2),
'hidden_units': [1],
# Set lr mult to 0. to keep embeddings constant.
'embedding_lr_multipliers': {
one_hot_language: 0.0
},
}
features = {
'language':
tf.SparseTensor(
values=['en', 'fr', 'zh'],
indices=[[0, 0], [1, 0], [2, 0]],
dense_shape=[3, 1]),
}
labels = tf.constant([[0], [0], [0]], dtype=tf.int32)
with self.assertRaisesRegexp(
ValueError, 'can only be defined for embedding columns'):
dnn._dnn_model_fn(features, labels,
tf.contrib.learn.ModeKeys.TRAIN, params)
def testMultiClassWithInvalidLogits(self):
head = head_lib._multi_class_head(n_classes=len(self._logits[0]) + 1)
with ops.Graph().as_default(), session.Session():
with self.assertRaisesRegexp(ValueError, "Dimensions.*not compatible"):
head.create_model_fn_ops(
{}, self._labels, model_fn.ModeKeys.TRAIN, _noop_train_op,
logits=self._logits)
def testBinaryClassificationWithLogitsInput(self):
n_classes = 2
head = head_lib._multi_class_head(n_classes=n_classes)
with ops.Graph().as_default(), session.Session():
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.create_model_fn_ops(
{}, model_fn.ModeKeys.TRAIN, self._labels, _noop_train_op,
logits_input=((0., 0.), (0., 0.)))
self._assert_output_alternatives(model_fn_ops)
w = ("binary_logistic_head/logits/weights:0",
"binary_logistic_head/logits/biases:0")
_assert_variables(
self, expected_global=w, expected_model=w, expected_trainable=w)
variables.global_variables_initializer().run()
_assert_summary_tags(self, ["loss"])
expected_loss = .69314718
label_mean = np.mean(self._labels)
_assert_metrics(self, expected_loss, {
"accuracy": 1. / 2,
"accuracy/baseline_label_mean": label_mean,
"accuracy/threshold_0.500000_mean": 1. / 2,
"auc": 1. / 2,
"labels/actual_label_mean": label_mean,
"labels/prediction_mean": .5, # softmax
"loss": expected_loss,
"precision/positive_threshold_0.500000_mean": 0. / 2,
"recall/positive_threshold_0.500000_mean": 0. / 1,
}, model_fn_ops)
def testLinearModel(self):
"""Tests that loss goes down with training."""
def input_fn():
return {
'age':
tf.constant([1]),
'language':
tf.SparseTensor(values=['english'],
indices=[[0, 0]],
shape=[1, 1])
}, tf.constant([[1]])
language = tf.contrib.layers.sparse_column_with_hash_bucket(
'language', 100)
age = tf.contrib.layers.real_valued_column('age')
head = head_lib._multi_class_head(n_classes=2)
classifier = LinearEstimator(head, feature_columns=[age, language])
classifier.fit(input_fn=input_fn, steps=1000)
loss1 = classifier.evaluate(input_fn=input_fn, steps=1)['loss']
classifier.fit(input_fn=input_fn, steps=2000)
loss2 = classifier.evaluate(input_fn=input_fn, steps=1)['loss']
self.assertLess(loss2, loss1)
self.assertLess(loss2, 0.01)
def testBinaryClassificationWithLogitsAndLogitsInput(self):
head = head_lib._multi_class_head(n_classes=2)
with ops.Graph().as_default(), session.Session():
with self.assertRaisesRegexp(
ValueError, "Both logits and logits_input supplied"):
head.create_model_fn_ops(
{}, self._labels, model_fn.ModeKeys.TRAIN, _noop_train_op,
logits_input=((0., 0.), (0., 0.)), logits=self._logits)
def testBinaryClassificationWithLogitsAndLogitsInput(self):
head = head_lib._multi_class_head(n_classes=2)
with ops.Graph().as_default(), session.Session():
with self.assertRaisesRegexp(
ValueError, "Both logits and logits_input supplied"):
head.create_model_fn_ops(
{}, model_fn.ModeKeys.TRAIN, self._labels, _noop_train_op,
logits_input=((0., 0.), (0., 0.)), logits=self._logits)
def testBinaryClassification(self):
head = head_lib._multi_class_head(n_classes=2)
with tf.Graph().as_default(), tf.Session() as sess:
logits = tf.constant([[1.0], [1.0]])
labels = tf.constant([[1.0], [0.0]])
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.head_ops({}, labels, tf.contrib.learn.ModeKeys.TRAIN, _noop_train_op, logits=logits)
self.assertAlmostEqual(0.81326175, sess.run(model_fn_ops.loss), delta=1e-6)
def testMultiClass(self):
head = head_lib._multi_class_head(n_classes=3)
with tf.Graph().as_default(), tf.Session() as sess:
logits = tf.constant([[1.0, 0.0, 0.0]])
labels = tf.constant([2])
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.head_ops({}, labels, tf.contrib.learn.ModeKeys.TRAIN, _noop_train_op, logits=logits)
self.assertAlmostEqual(1.5514446, sess.run(model_fn_ops.loss))
def testDNNModel(self):
"""Tests multi-class classification using matrix data as input."""
cont_features = [tf.contrib.layers.real_valued_column("feature", dimension=4)]
head = head_lib._multi_class_head(n_classes=3)
classifier = DNNEstimator(head, feature_columns=cont_features, hidden_units=[3, 3])
classifier.fit(input_fn=_iris_input_fn, steps=1000)
classifier.evaluate(input_fn=_iris_input_fn, steps=100)
def testMultiClassWithLogitsAndLogitsInput(self):
n_classes = 3
head = head_lib._multi_class_head(
n_classes=n_classes, metric_class_ids=range(n_classes))
with ops.Graph().as_default(), session.Session():
with self.assertRaisesRegexp(
ValueError, "Both logits and logits_input supplied"):
head.create_model_fn_ops(
{}, self._labels, model_fn.ModeKeys.TRAIN, _noop_train_op,
logits_input=((0., 0.),), logits=self._logits)
def testMultiClassWithLogitsAndLogitsInput(self):
n_classes = 3
head = head_lib._multi_class_head(
n_classes=n_classes, metric_class_ids=range(n_classes))
with ops.Graph().as_default(), session.Session():
with self.assertRaisesRegexp(
ValueError, "Both logits and logits_input supplied"):
head.create_model_fn_ops(
{}, model_fn.ModeKeys.TRAIN, self._labels, _noop_train_op,
logits_input=((0., 0.),), logits=self._logits)
def testBinaryClassification(self):
head = head_lib._multi_class_head(n_classes=2)
with tf.Graph().as_default(), tf.Session() as sess:
logits = tf.constant([[1.], [1.]])
targets = tf.constant([[1.], [0.]])
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.head_ops({}, targets,
tf.contrib.learn.ModeKeys.TRAIN,
None, logits=logits)
self.assertAlmostEqual(.81326163, sess.run(model_fn_ops.loss))
def testBinaryClassification(self):
head = head_lib._multi_class_head(n_classes=2)
with tf.Graph().as_default(), tf.Session() as sess:
logits = tf.constant([[1.], [1.]])
labels = tf.constant([[1.], [0.]])
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.head_ops({}, labels,
tf.contrib.learn.ModeKeys.TRAIN,
_noop_train_op, logits=logits)
self.assertAlmostEqual(.81326163, sess.run(model_fn_ops.loss))
def testErrorInSparseTensorLabels(self):
head = head_lib._multi_class_head(n_classes=2)
with tf.Graph().as_default():
prediction = tf.constant([[1.0], [1.0], [3.0]])
labels = tf.SparseTensor(
indices=tf.constant([[0, 0], [1, 0], [2, 0]], dtype=tf.int64),
values=tf.constant([0, 1, 1]),
shape=[3, 1],
)
with self.assertRaisesRegexp(ValueError, "SparseTensor is not supported as labels."):
head.head_ops({}, labels, tf.contrib.learn.ModeKeys.TRAIN, _noop_train_op, logits=prediction)
def testMultiClass(self):
head = head_lib._multi_class_head(n_classes=3)
with tf.Graph().as_default(), tf.Session() as sess:
logits = tf.constant([[1., 0., 0.]])
targets = tf.constant([2])
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.head_ops({}, targets,
tf.contrib.learn.ModeKeys.TRAIN,
None, logits=logits)
self.assertAlmostEqual(1.5514446, sess.run(model_fn_ops.loss))
def testMultiClassWithWeight(self):
head = head_lib._multi_class_head(n_classes=3, weight_column_name="label_weight")
with tf.Graph().as_default(), tf.Session() as sess:
features = {"label_weight": tf.constant([0.1])}
logits = tf.constant([[1.0, 0.0, 0.0]])
labels = tf.constant([2])
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.head_ops(
features, labels, tf.contrib.learn.ModeKeys.TRAIN, _noop_train_op, logits=logits
)
self.assertAlmostEqual(0.15514446, sess.run(model_fn_ops.loss))
def testBinaryClassificationWithWeights(self):
head = head_lib._multi_class_head(n_classes=2, weight_column_name="label_weight")
with tf.Graph().as_default(), tf.Session() as sess:
features = {"label_weight": tf.constant([[1.0], [0.0]])}
logits = tf.constant([[1.0], [1.0]])
labels = tf.constant([[1.0], [0.0]])
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.head_ops(
features, labels, tf.contrib.learn.ModeKeys.TRAIN, _noop_train_op, logits=logits
)
self.assertAlmostEqual(0.31326166 / 2, sess.run(model_fn_ops.loss), delta=1e-6)
def testErrorInSparseTensorLabels(self):
head = head_lib._multi_class_head(n_classes=2)
with tf.Graph().as_default():
prediction = tf.constant([[1.], [1.], [3.]])
labels = tf.SparseTensor(
indices=tf.constant([[0, 0], [1, 0], [2, 0]], dtype=tf.int64),
values=tf.constant([0, 1, 1]),
shape=[3, 1])
with self.assertRaisesRegexp(
ValueError, "SparseTensor is not supported as labels."):
head.head_ops({}, labels, tf.contrib.learn.ModeKeys.TRAIN,
_noop_train_op, logits=prediction)
def testErrorInSparseTensorTarget(self):
head = head_lib._multi_class_head(n_classes=2)
with tf.Graph().as_default():
prediction = tf.constant([[1.], [1.], [3.]])
targets = tf.SparseTensor(
indices=tf.constant([[0, 0], [1, 0], [2, 0]], dtype=tf.int64),
values=tf.constant([0, 1, 1]),
shape=[3, 1])
with self.assertRaisesRegexp(
ValueError, "SparseTensor is not supported as a target"):
head.head_ops({}, targets, tf.contrib.learn.ModeKeys.TRAIN, None,
logits=prediction)
def testDNNModel(self):
"""Tests multi-class classification using matrix data as input."""
cont_features = [
tf.contrib.layers.real_valued_column('feature', dimension=4)]
head = head_lib._multi_class_head(n_classes=3)
classifier = DNNEstimator(head,
feature_columns=cont_features,
hidden_units=[3, 3])
classifier.fit(input_fn=_iris_input_fn, steps=1000)
classifier.evaluate(input_fn=_iris_input_fn, steps=100)
def testBinaryClassificationInferMode(self):
n_classes = 2
head = head_lib._multi_class_head(n_classes=n_classes)
with ops.Graph().as_default(), session.Session():
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.create_model_fn_ops(
{}, model_fn.ModeKeys.INFER, self._labels, _noop_train_op,
logits=self._logits)
self._assert_output_alternatives(model_fn_ops)
self.assertIsNone(model_fn_ops.train_op)
_assert_no_variables(self)
def testBinaryClassificationInferMode(self):
n_classes = 2
head = head_lib._multi_class_head(n_classes=n_classes)
with ops.Graph().as_default(), session.Session():
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.create_model_fn_ops(
{}, model_fn.ModeKeys.INFER, self._labels, _noop_train_op,
logits=self._logits)
self._assert_output_alternatives(model_fn_ops)
self.assertIsNone(model_fn_ops.train_op)
_assert_no_variables(self)
def testBinaryClassificationWithWeights(self):
head = head_lib._multi_class_head(
n_classes=2, weight_column_name="label_weight")
with tf.Graph().as_default(), tf.Session() as sess:
features = {"label_weight": tf.constant([[1.], [0.]])}
logits = tf.constant([[1.], [1.]])
labels = tf.constant([[1.], [0.]])
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.head_ops(features, labels,
tf.contrib.learn.ModeKeys.TRAIN,
_noop_train_op, logits=logits)
self.assertAlmostEqual(.31326166 / 2, sess.run(model_fn_ops.loss))
def testMultiClassWithWeight(self):
head = head_lib._multi_class_head(
n_classes=3, weight_column_name="label_weight")
with tf.Graph().as_default(), tf.Session() as sess:
features = {"label_weight": tf.constant([0.1])}
logits = tf.constant([[1., 0., 0.]])
targets = tf.constant([2])
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.head_ops(features, targets,
tf.contrib.learn.ModeKeys.TRAIN,
None, logits=logits)
self.assertAlmostEqual(.15514446, sess.run(model_fn_ops.loss))
def testBinaryClassificationInferMode(self):
n_classes = 2
head = head_lib._multi_class_head(n_classes=n_classes)
with ops.Graph().as_default(), session.Session():
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.create_model_fn_ops(
{}, self._labels, model_fn.ModeKeys.INFER, _noop_train_op,
logits=self._logits)
self.assertIsNone(model_fn_ops.train_op)
_assert_no_variables(self)
self.assertEquals(1, len(model_fn_ops.output_alternatives))
self.assertEquals(constants.ProblemType.LOGISTIC_REGRESSION,
model_fn_ops.output_alternatives[None][0])
def testBinaryClassificationWithLogits(self):
n_classes = 2
head = head_lib._multi_class_head(n_classes=n_classes)
with ops.Graph().as_default(), session.Session():
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.create_model_fn_ops(
{}, self._labels, model_fn.ModeKeys.TRAIN, _noop_train_op,
logits=self._logits)
_assert_no_variables(self)
_assert_summary_tags(self, ["loss"])
expected_loss = .81326175
_assert_metrics(self, expected_loss,
self._expected_eval_metrics(expected_loss), model_fn_ops)
def testBinaryClassificationInferMode_withWightColumn(self):
n_classes = 2
head = head_lib._multi_class_head(n_classes=n_classes,
weight_column_name="label_weight")
with ops.Graph().as_default(), session.Session():
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.create_model_fn_ops(
# This is what is being tested, features should not have weight for
# inference.
{}, model_fn.ModeKeys.INFER, self._labels, _noop_train_op,
logits=self._logits)
self._assert_output_alternatives(model_fn_ops)
self.assertIsNone(model_fn_ops.train_op)
_assert_no_variables(self)
def testBinaryClassificationInferMode_withWightColumn(self):
n_classes = 2
head = head_lib._multi_class_head(n_classes=n_classes,
weight_column_name="label_weight")
with ops.Graph().as_default(), session.Session():
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.create_model_fn_ops(
# This is what is being tested, features should not have weight for
# inference.
{}, model_fn.ModeKeys.INFER, self._labels, _noop_train_op,
logits=self._logits)
self._assert_output_alternatives(model_fn_ops)
self.assertIsNone(model_fn_ops.train_op)
_assert_no_variables(self)
def testBinaryClassificationWithLabelName(self):
label_name = "my_label"
head = head_lib._multi_class_head(n_classes=2, label_name=label_name)
with ops.Graph().as_default(), session.Session():
logits = constant_op.constant(self._logits)
labels = {label_name: constant_op.constant(self._labels)}
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.head_ops(
{}, labels, model_fn.ModeKeys.TRAIN, _noop_train_op, logits=logits)
_assert_no_variables(self)
_assert_summary_tags(self, ["loss"])
expected_loss = .81326175
_assert_metrics(self, expected_loss,
self._expected_eval_metrics(expected_loss), model_fn_ops)
def testMultiClassEvalMode(self):
n_classes = 3
head = head_lib._multi_class_head(
n_classes=n_classes, metric_class_ids=range(n_classes))
with ops.Graph().as_default(), session.Session():
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.create_model_fn_ops(
{}, self._labels, model_fn.ModeKeys.EVAL, _noop_train_op,
logits=self._logits)
self.assertIsNone(model_fn_ops.train_op)
_assert_no_variables(self)
_assert_summary_tags(self, ["loss"])
expected_loss = 1.5514446
_assert_metrics(self, expected_loss,
self._expected_eval_metrics(expected_loss), model_fn_ops)
def testMultiClassWithLogits(self):
n_classes = 3
head = head_lib._multi_class_head(
n_classes=n_classes, metric_class_ids=range(n_classes))
with ops.Graph().as_default(), session.Session():
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.create_model_fn_ops(
{}, model_fn.ModeKeys.TRAIN, self._labels, _noop_train_op,
logits=self._logits)
self._assert_output_alternatives(model_fn_ops)
_assert_no_variables(self)
_assert_summary_tags(self, ["loss"])
expected_loss = 1.5514446
_assert_metrics(self, expected_loss,
self._expected_eval_metrics(expected_loss), model_fn_ops)
def testBinaryClassificationWithLabelName(self):
label_name = "my_label"
head = head_lib._multi_class_head(n_classes=2, label_name=label_name)
with tf.Graph().as_default(), tf.Session():
logits = tf.constant(self._logits)
labels = {label_name: tf.constant(self._labels)}
# logloss: z:label, x:logit
# z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
model_fn_ops = head.head_ops({}, labels,
tf.contrib.learn.ModeKeys.TRAIN,
_noop_train_op, logits=logits)
_assert_no_variables(self)
expected_loss = .81326175
_assert_metrics(
self, expected_loss, self._expected_eval_metrics(expected_loss),
model_fn_ops)
def testErrorInSparseTensorLabels(self):
n_classes = 2
head = head_lib._multi_class_head(n_classes=n_classes)
with ops.Graph().as_default():
labels = sparse_tensor.SparseTensorValue(
indices=((0, 0), (1, 0), (2, 0)),
values=(0, 1, 1),
dense_shape=(3, 1))
with self.assertRaisesRegexp(ValueError,
"SparseTensor is not supported as labels."):
head.create_model_fn_ops(
{},
labels,
model_fn.ModeKeys.TRAIN,
_noop_train_op,
logits=((1.,), (1.,), (3.,)))