def testIrisStreaming(self):
iris = datasets.load_iris()
def iris_data():
while True:
for x in iris.data:
yield x
def iris_predict_data():
for x in iris.data:
yield x
def iris_target():
while True:
for y in iris.target:
yield y
classifier = learn.LinearClassifier(
feature_columns=learn.infer_real_valued_columns_from_input(iris.data),
n_classes=3)
classifier.fit(iris_data(), iris_target(), max_steps=500)
score1 = accuracy_score(iris.target, classifier.predict(iris.data))
score2 = accuracy_score(iris.target,
classifier.predict(iris_predict_data()))
self.assertGreater(score1, 0.5, "Failed with score = {0}".format(score1))
self.assertEqual(score2, score1, "Scores from {0} iterator doesn't "
"match score {1} from full "
"data.".format(score2, score1))
def testIrisES(self):
random.seed(42)
iris = datasets.load_iris()
x_train, x_test, y_train, y_test = train_test_split(iris.data,
iris.target,
test_size=0.2,
random_state=42)
x_train, x_val, y_train, y_val = train_test_split(x_train,
y_train,
test_size=0.2,
random_state=42)
val_monitor = learn.monitors.ValidationMonitor(
x_val,
y_val,
every_n_steps=50,
early_stopping_rounds=100,
early_stopping_metric='accuracy',
early_stopping_metric_minimize=False)
# classifier without early stopping - overfitting
classifier1 = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10],
n_classes=3,
steps=1000)
classifier1.fit(x_train, y_train)
_ = accuracy_score(y_test, classifier1.predict(x_test))
# Full 1000 steps, 11 summaries and no evaluation summary.
# 11 summaries = first + every 100 out of 1000 steps.
self.assertEqual(11, len(_get_summary_events(classifier1.model_dir)))
with self.assertRaises(ValueError):
_get_summary_events(classifier1.model_dir + '/eval')
# classifier with early stopping - improved accuracy on testing set
classifier2 = learn.TensorFlowDNNClassifier(
hidden_units=[10, 20, 10],
n_classes=3,
steps=2000,
config=tf.contrib.learn.RunConfig(save_checkpoints_secs=1))
classifier2.fit(x_train, y_train, monitors=[val_monitor])
_ = accuracy_score(y_val, classifier2.predict(x_val))
_ = accuracy_score(y_test, classifier2.predict(x_test))
# Note, this test is unstable, so not checking for equality.
# See stability_test for examples of stability issues.
if val_monitor.early_stopped:
self.assertLess(val_monitor.best_step, 2000)
# Note, due to validation monitor stopping after the best score occur,
# the accuracy at current checkpoint is less.
# TODO(ipolosukhin): Time machine for restoring old checkpoints?
# flaky, still not always best_value better then score2 value.
# self.assertGreater(val_monitor.best_value, score2_val)
# Early stopped, unstable so checking only < then max.
self.assertLess(len(_get_summary_events(classifier2.model_dir)),
21)
self.assertLess(
len(_get_summary_events(classifier2.model_dir + '/eval')), 4)
def testIrisStreaming(self):
iris = datasets.load_iris()
def iris_data():
while True:
for x in iris.data:
yield x
def iris_predict_data():
for x in iris.data:
yield x
def iris_target():
while True:
for y in iris.target:
yield y
classifier = learn.LinearClassifier(
feature_columns=learn.infer_real_valued_columns_from_input(
iris.data),
n_classes=3)
classifier.fit(iris_data(), iris_target(), max_steps=500)
score1 = accuracy_score(iris.target, classifier.predict(iris.data))
score2 = accuracy_score(iris.target,
classifier.predict(iris_predict_data()))
self.assertGreater(score1, 0.5,
"Failed with score = {0}".format(score1))
self.assertEqual(
score2, score1, "Scores from {0} iterator doesn't "
"match score {1} from full "
"data.".format(score2, score1))
def testIrisStreaming(self):
iris = datasets.load_iris()
def iris_data():
while True:
for x in iris.data:
yield x
def iris_predict_data():
for x in iris.data:
yield x
def iris_target():
while True:
for y in iris.target:
yield y
classifier = learn.TensorFlowLinearClassifier(n_classes=3, steps=100)
classifier.fit(iris_data(), iris_target())
score1 = accuracy_score(iris.target, classifier.predict(iris.data))
score2 = accuracy_score(iris.target,
classifier.predict(iris_predict_data()))
self.assertGreater(score1, 0.5, "Failed with score = {0}".format(score1))
self.assertEqual(score2, score1, "Scores from {0} iterator doesn't "
"match score {1} from full "
"data.".format(score2, score1))
def testIrisES(self):
random.seed(42)
iris = datasets.load_iris()
x_train, x_test, y_train, y_test = train_test_split(iris.data,
iris.target,
test_size=0.2,
random_state=42)
x_train, x_val, y_train, y_val = train_test_split(
x_train, y_train, test_size=0.2)
val_monitor = learn.monitors.ValidationMonitor(x_val, y_val,
early_stopping_rounds=100)
# classifier without early stopping - overfitting
classifier1 = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10],
n_classes=3,
steps=1000)
classifier1.fit(x_train, y_train)
accuracy_score(y_test, classifier1.predict(x_test))
# classifier with early stopping - improved accuracy on testing set
classifier2 = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10],
n_classes=3,
steps=1000)
classifier2.fit(x_train, y_train, monitors=[val_monitor])
accuracy_score(y_test, classifier2.predict(x_test))
def testIrisContinueTraining(self):
iris = datasets.load_iris()
classifier = learn.TensorFlowLinearClassifier(n_classes=3,
learning_rate=0.01, continue_training=True, steps=250)
classifier.fit(iris.data, iris.target)
score1 = accuracy_score(iris.target, classifier.predict(iris.data))
classifier.fit(iris.data, iris.target)
score2 = accuracy_score(iris.target, classifier.predict(iris.data))
self.assertGreater(score2, score1,
"Failed with score = {0}".format(score2))
def testIrisES(self):
random.seed(42)
iris = datasets.load_iris()
x_train, x_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.2, random_state=42)
x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=42)
val_monitor = learn.monitors.ValidationMonitor(
x_val,
y_val,
every_n_steps=50,
early_stopping_rounds=100,
early_stopping_metric="accuracy",
early_stopping_metric_minimize=False,
)
# classifier without early stopping - overfitting
classifier1 = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], n_classes=3, steps=1000)
classifier1.fit(x_train, y_train)
_ = accuracy_score(y_test, classifier1.predict(x_test))
# Full 1000 steps, 12 summaries and no evaluation summary.
# 12 summaries = global_step + first + every 100 out of 1000 steps.
self.assertEqual(12, len(_get_summary_events(classifier1.model_dir)))
with self.assertRaises(ValueError):
_get_summary_events(classifier1.model_dir + "/eval")
# classifier with early stopping - improved accuracy on testing set
classifier2 = learn.TensorFlowDNNClassifier(
hidden_units=[10, 20, 10],
n_classes=3,
steps=2000,
config=tf.contrib.learn.RunConfig(save_checkpoints_secs=1),
)
classifier2.fit(x_train, y_train, monitors=[val_monitor])
_ = accuracy_score(y_val, classifier2.predict(x_val))
_ = accuracy_score(y_test, classifier2.predict(x_test))
# Note, this test is unstable, so not checking for equality.
# See stability_test for examples of stability issues.
if val_monitor.early_stopped:
self.assertLess(val_monitor.best_step, 2000)
# Note, due to validation monitor stopping after the best score occur,
# the accuracy at current checkpoint is less.
# TODO(ipolosukhin): Time machine for restoring old checkpoints?
# flaky, still not always best_value better then score2 value.
# self.assertGreater(val_monitor.best_value, score2_val)
# Early stopped, unstable so checking only < then max.
self.assertLess(len(_get_summary_events(classifier2.model_dir)), 21)
# Eval typically has ~6 events, but it varies based on the run.
self.assertLess(len(_get_summary_events(classifier2.model_dir + "/eval")), 8)
def testIrisContinueTraining(self):
iris = datasets.load_iris()
classifier = learn.LinearClassifier(
feature_columns=learn.infer_real_valued_columns_from_input(iris.data),
n_classes=3)
classifier.fit(iris.data, iris.target, steps=100)
score1 = accuracy_score(iris.target, classifier.predict(iris.data))
classifier.fit(iris.data, iris.target, steps=500)
score2 = accuracy_score(iris.target, classifier.predict(iris.data))
self.assertGreater(
score2, score1,
"Failed with score2 {0} <= score1 {1}".format(score2, score1))
def testIrisContinueTraining(self):
iris = datasets.load_iris()
classifier = learn.LinearClassifier(
feature_columns=learn.infer_real_valued_columns_from_input(iris.data),
n_classes=3)
classifier.fit(iris.data, iris.target, steps=100)
score1 = accuracy_score(iris.target, list(classifier.predict(iris.data)))
classifier.fit(iris.data, iris.target, steps=500)
score2 = accuracy_score(iris.target, list(classifier.predict(iris.data)))
self.assertGreater(
score2, score1,
"Failed with score2 {0} <= score1 {1}".format(score2, score1))
def testIrisContinueTraining(self):
iris = datasets.load_iris()
classifier = learn.TensorFlowLinearClassifier(n_classes=3,
learning_rate=0.01,
continue_training=True,
steps=250)
classifier.fit(iris.data, iris.target)
score1 = accuracy_score(iris.target, classifier.predict(iris.data))
classifier.fit(iris.data, iris.target)
score2 = accuracy_score(iris.target, classifier.predict(iris.data))
self.assertGreater(score2, score1,
"Failed with score = {0}".format(score2))
def testIrisContinueTraining(self):
iris = datasets.load_iris()
classifier = learn.TensorFlowLinearClassifier(
feature_columns=learn.infer_real_valued_columns_from_input(iris.data),
n_classes=3,
learning_rate=0.01,
continue_training=True,
steps=250)
classifier.fit(iris.data, iris.target)
score1 = accuracy_score(iris.target, classifier.predict(iris.data))
classifier.fit(iris.data, iris.target, steps=500)
score2 = accuracy_score(iris.target, classifier.predict(iris.data))
self.assertGreater(
score2, score1,
"Failed with score2 {0} <= score1 {1}".format(score2, score1))
def testIrisMomentum(self):
random.seed(42)
iris = datasets.load_iris()
x_train, x_test, y_train, y_test = train_test_split(iris.data,
iris.target,
test_size=0.2,
random_state=42)
def custom_optimizer():
return momentum_lib.MomentumOptimizer(learning_rate=0.01,
momentum=0.9)
classifier = learn.DNNClassifier(
hidden_units=[10, 20, 10],
feature_columns=learn.infer_real_valued_columns_from_input(
x_train),
n_classes=3,
optimizer=custom_optimizer,
config=learn.RunConfig(tf_random_seed=1))
classifier.fit(x_train, y_train, steps=400)
predictions = np.array(list(classifier.predict_classes(x_test)))
score = accuracy_score(y_test, predictions)
self.assertGreater(score, 0.65,
"Failed with score = {0}".format(score))
def testIrisMomentum(self):
random.seed(42)
iris = datasets.load_iris()
x_train, x_test, y_train, y_test = train_test_split(iris.data,
iris.target,
test_size=0.2,
random_state=42)
def custom_optimizer(learning_rate):
return tf.train.MomentumOptimizer(learning_rate, 0.9)
cont_features = [
tf.contrib.layers.real_valued_column("", dimension=4)]
classifier = learn.TensorFlowDNNClassifier(
feature_columns=cont_features,
hidden_units=[10, 20, 10],
n_classes=3,
steps=400,
learning_rate=0.01,
optimizer=custom_optimizer)
classifier.fit(x_train, y_train)
score = accuracy_score(y_test, classifier.predict(x_test))
self.assertGreater(score, 0.65, "Failed with score = {0}".format(score))
def testIrisAllDictionaryInput(self):
iris = base.load_iris()
est = estimator.Estimator(model_fn=logistic_model_no_mode_fn)
iris_data = {'input': iris.data}
iris_target = {'labels': iris.target}
est.fit(iris_data, iris_target, steps=100)
scores = est.evaluate(
x=iris_data,
y=iris_target,
metrics={
('accuracy', 'class'): metric_ops.streaming_accuracy
})
predictions = list(est.predict(x=iris_data))
predictions_class = list(est.predict(x=iris_data, outputs=['class']))
self.assertEqual(len(predictions), iris.target.shape[0])
classes_batch = np.array([p['class'] for p in predictions])
self.assertAllClose(classes_batch,
np.array([p['class'] for p in predictions_class]))
self.assertAllClose(classes_batch,
np.argmax(
np.array([p['prob'] for p in predictions]), axis=1))
other_score = _sklearn.accuracy_score(iris.target, classes_batch)
self.assertAllClose(other_score, scores['accuracy'])
self.assertTrue('global_step' in scores)
self.assertEqual(scores['global_step'], 100)
def testCustomMetrics(self):
"""Tests weight column in evaluation."""
def _input_fn_train():
# Create 4 rows, one of them (y = x), three of them (y=Not(x))
target = tf.constant([[1], [0], [0], [0]])
features = {'x': tf.ones(shape=[4, 1], dtype=tf.float32),}
return features, target
classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
dnn_hidden_units=[3, 3])
classifier.train(input_fn=_input_fn_train, steps=100)
scores = classifier.evaluate(
input_fn=_input_fn_train,
steps=100,
metrics={
'my_accuracy': tf.contrib.metrics.streaming_accuracy,
'my_precision': tf.contrib.metrics.streaming_precision
})
self.assertTrue(set(['loss', 'my_accuracy', 'my_precision']).issubset(set(
scores.keys())))
predictions = classifier.predict(input_fn=_input_fn_train)
self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions),
scores['my_accuracy'])
def testDNNDropout0_1(self):
# Dropping only a little.
iris = datasets.load_iris()
classifier = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], n_classes=3, dropout=0.1)
classifier.fit(iris.data, iris.target)
score = accuracy_score(iris.target, classifier.predict(iris.data))
self.assertGreater(score, 0.9, "Failed with score = {0}".format(score))
def testIrisAllDictionaryInput(self):
iris = tf.contrib.learn.datasets.load_iris()
est = tf.contrib.learn.Estimator(model_fn=logistic_model_no_mode_fn)
iris_data = {'input': iris.data}
iris_target = {'labels': iris.target}
est.fit(iris_data, iris_target, steps=100)
scores = est.evaluate(x=iris_data,
y=iris_target,
metrics={
('accuracy', 'class'):
tf.contrib.metrics.streaming_accuracy
})
predictions = list(est.predict(x=iris_data))
predictions_class = list(est.predict(x=iris_data, outputs=['class']))
self.assertEqual(len(predictions), iris.target.shape[0])
classes_batch = np.array([p['class'] for p in predictions])
self.assertAllClose(classes_batch,
np.array([p['class'] for p in predictions_class]))
self.assertAllClose(
classes_batch,
np.argmax(np.array([p['prob'] for p in predictions]), axis=1))
other_score = _sklearn.accuracy_score(iris.target, classes_batch)
self.assertAllClose(other_score, scores['accuracy'])
self.assertTrue('global_step' in scores)
self.assertEqual(scores['global_step'], 100)
def testIrisClassWeight(self):
iris = datasets.load_iris()
classifier = learn.TensorFlowLinearClassifier(
n_classes=3, class_weight=[0.1, 0.8, 0.1])
classifier.fit(iris.data, iris.target)
score = accuracy_score(iris.target, classifier.predict(iris.data))
self.assertLess(score, 0.7, "Failed with score = {0}".format(score))
def testCustomMetrics(self):
"""Tests weight column in evaluation."""
def _input_fn_train():
# Create 4 rows, one of them (y = x), three of them (y=Not(x))
target = tf.constant([[1], [0], [0], [0]])
features = {
'x': tf.ones(shape=[4, 1], dtype=tf.float32),
}
return features, target
classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
dnn_hidden_units=[3, 3])
classifier.fit(input_fn=_input_fn_train, steps=100)
scores = classifier.evaluate(input_fn=_input_fn_train,
steps=100,
metrics={
'my_accuracy':
tf.contrib.metrics.streaming_accuracy,
'my_precision':
tf.contrib.metrics.streaming_precision
})
self.assertTrue(
set(['loss', 'my_accuracy',
'my_precision']).issubset(set(scores.keys())))
predictions = classifier.predict(input_fn=_input_fn_train)
self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions),
scores['my_accuracy'])
def testIrisClassWeight(self):
iris = datasets.load_iris()
classifier = learn.TensorFlowLinearClassifier(
n_classes=3, class_weight=[0.1, 0.8, 0.1])
classifier.fit(iris.data, iris.target)
score = accuracy_score(iris.target, classifier.predict(iris.data))
self.assertLess(score, 0.7, "Failed with score = {0}".format(score))
def testIrisMomentum(self):
random.seed(42)
iris = datasets.load_iris()
x_train, x_test, y_train, y_test = train_test_split(iris.data,
iris.target,
test_size=0.2,
random_state=42)
def custom_optimizer(learning_rate):
return tf.train.MomentumOptimizer(learning_rate, 0.9)
classifier = learn.TensorFlowDNNClassifier(
hidden_units=[10, 20, 10],
feature_columns=learn.infer_real_valued_columns_from_input(
x_train),
n_classes=3,
steps=400,
learning_rate=0.01,
optimizer=custom_optimizer)
classifier.fit(x_train, y_train)
score = accuracy_score(y_test, classifier.predict(x_test))
self.assertGreater(score, 0.65,
"Failed with score = {0}".format(score))
def testIrisMomentum(self):
random.seed(42)
iris = datasets.load_iris()
x_train, x_test, y_train, y_test = train_test_split(iris.data,
iris.target,
test_size=0.2,
random_state=42)
# setup exponential decay function
def exp_decay(global_step):
return tf.train.exponential_decay(learning_rate=0.1,
global_step=global_step,
decay_steps=100,
decay_rate=0.001)
def custom_optimizer(learning_rate):
return tf.train.MomentumOptimizer(learning_rate, 0.9)
classifier = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10],
n_classes=3,
steps=400,
learning_rate=exp_decay,
optimizer=custom_optimizer)
classifier.fit(x_train, y_train)
score = accuracy_score(y_test, classifier.predict(x_test))
self.assertGreater(score, 0.65, "Failed with score = {0}".format(score))
def testCustomMetrics(self):
"""Tests custom evaluation metrics."""
def _input_fn(num_epochs=None):
# Create 4 rows, one of them (y = x), three of them (y=Not(x))
labels = tf.constant([[1], [0], [0], [0]])
features = {
'x':
tf.train.limit_epochs(tf.ones(shape=[4, 1], dtype=tf.float32),
num_epochs=num_epochs),
}
return features, labels
def _my_metric_op(predictions, labels):
# For the case of binary classification, the 2nd column of "predictions"
# denotes the model predictions.
labels = tf.to_float(labels)
predictions = tf.slice(predictions, [0, 1], [-1, 1])
labels = math_ops.cast(labels, predictions.dtype)
return tf.reduce_sum(tf.mul(predictions, labels))
classifier = tf.contrib.learn.DNNClassifier(
feature_columns=[tf.contrib.layers.real_valued_column('x')],
hidden_units=[3, 3],
config=tf.contrib.learn.RunConfig(tf_random_seed=1))
classifier.fit(input_fn=_input_fn, steps=5)
scores = classifier.evaluate(
input_fn=_input_fn,
steps=5,
metrics={
'my_accuracy':
MetricSpec(metric_fn=tf.contrib.metrics.streaming_accuracy,
prediction_key='classes'),
'my_precision':
MetricSpec(metric_fn=tf.contrib.metrics.streaming_precision,
prediction_key='classes'),
'my_metric':
MetricSpec(metric_fn=_my_metric_op,
prediction_key='probabilities')
})
self.assertTrue(
set(['loss', 'my_accuracy', 'my_precision',
'my_metric']).issubset(set(scores.keys())))
predict_input_fn = functools.partial(_input_fn, num_epochs=1)
predictions = np.array(
list(classifier.predict(input_fn=predict_input_fn)))
self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions),
scores['my_accuracy'])
# Test the case where the 2nd element of the key is neither "classes" nor
# "probabilities".
with self.assertRaisesRegexp(KeyError, 'bad_type'):
classifier.evaluate(
input_fn=_input_fn,
steps=5,
metrics={
'bad_name':
MetricSpec(metric_fn=tf.contrib.metrics.streaming_auc,
prediction_key='bad_type')
})
def testIrisMomentum(self):
random.seed(42)
iris = datasets.load_iris()
X_train, X_test, y_train, y_test = train_test_split(iris.data,
iris.target,
test_size=0.2,
random_state=42)
# setup exponential decay function
def exp_decay(global_step):
return tf.train.exponential_decay(learning_rate=0.1,
global_step=global_step,
decay_steps=100,
decay_rate=0.001)
custom_optimizer = lambda learning_rate: tf.train.MomentumOptimizer(
learning_rate, 0.9)
classifier = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10],
n_classes=3,
steps=800,
learning_rate=exp_decay,
optimizer=custom_optimizer)
classifier.fit(X_train, y_train)
score = accuracy_score(y_test, classifier.predict(X_test))
self.assertGreater(score, 0.7, "Failed with score = {0}".format(score))
def testIris(self):
iris = datasets.load_iris()
classifier = learn.TensorFlowLinearClassifier(
feature_columns=learn.infer_real_valued_columns_from_input(iris.data),
n_classes=3)
classifier.fit(iris.data, [x for x in iris.target])
score = accuracy_score(iris.target, classifier.predict(iris.data))
self.assertGreater(score, 0.7, "Failed with score = {0}".format(score))
def testDNNDropout0(self):
# Dropout prob == 0.
iris = tf.contrib.learn.datasets.load_iris()
classifier = tf.contrib.learn.TensorFlowDNNClassifier(
hidden_units=[10, 20, 10], n_classes=3, dropout=0.0)
classifier.fit(iris.data, iris.target)
score = accuracy_score(iris.target, classifier.predict(iris.data))
self.assertGreater(score, 0.9, "Failed with score = {0}".format(score))
def testDNNDropout0(self):
# Dropout prob == 0.
iris = tf.contrib.learn.datasets.load_iris()
classifier = tf.contrib.learn.TensorFlowDNNClassifier(
hidden_units=[10, 20, 10], n_classes=3, dropout=0.0)
classifier.fit(iris.data, iris.target)
score = accuracy_score(iris.target, classifier.predict(iris.data))
self.assertGreater(score, 0.9, "Failed with score = {0}".format(score))
def testIrisSummaries(self):
iris = datasets.load_iris()
output_dir = tempfile.mkdtemp() + "learn_tests/"
classifier = learn.TensorFlowLinearClassifier(n_classes=3,
model_dir=output_dir)
classifier.fit(iris.data, iris.target)
score = accuracy_score(iris.target, classifier.predict(iris.data))
self.assertGreater(score, 0.5, "Failed with score = {0}".format(score))
def testCustomMetrics(self):
"""Tests custom evaluation metrics."""
def _input_fn(num_epochs=None):
# Create 4 rows, one of them (y = x), three of them (y=Not(x))
labels = tf.constant([[1], [0], [0], [0]])
features = {
'x': tf.train.limit_epochs(
tf.ones(shape=[4, 1], dtype=tf.float32), num_epochs=num_epochs),
}
return features, labels
def _my_metric_op(predictions, labels):
# For the case of binary classification, the 2nd column of "predictions"
# denotes the model predictions.
labels = tf.to_float(labels)
predictions = tf.strided_slice(
predictions, [0, 1], [-1, 2], end_mask=1)
labels = math_ops.cast(labels, predictions.dtype)
return tf.reduce_sum(tf.multiply(predictions, labels))
classifier = tf.contrib.learn.DNNClassifier(
feature_columns=[tf.contrib.layers.real_valued_column('x')],
hidden_units=[3, 3],
config=tf.contrib.learn.RunConfig(tf_random_seed=1))
classifier.fit(input_fn=_input_fn, steps=5)
scores = classifier.evaluate(
input_fn=_input_fn,
steps=5,
metrics={
'my_accuracy': MetricSpec(
metric_fn=tf.contrib.metrics.streaming_accuracy,
prediction_key='classes'),
'my_precision': MetricSpec(
metric_fn=tf.contrib.metrics.streaming_precision,
prediction_key='classes'),
'my_metric': MetricSpec(
metric_fn=_my_metric_op,
prediction_key='probabilities')
})
self.assertTrue(
set(['loss', 'my_accuracy', 'my_precision', 'my_metric'
]).issubset(set(scores.keys())))
predict_input_fn = functools.partial(_input_fn, num_epochs=1)
predictions = np.array(list(classifier.predict(input_fn=predict_input_fn)))
self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions),
scores['my_accuracy'])
# Test the case where the 2nd element of the key is neither "classes" nor
# "probabilities".
with self.assertRaisesRegexp(KeyError, 'bad_type'):
classifier.evaluate(
input_fn=_input_fn,
steps=5,
metrics={
'bad_name': MetricSpec(
metric_fn=tf.contrib.metrics.streaming_auc,
prediction_key='bad_type')})
def testCustomMetrics(self):
"""Tests custom evaluation metrics."""
def _input_fn_train():
# Create 4 rows, one of them (y = x), three of them (y=Not(x))
target = tf.constant([[1], [0], [0], [0]], dtype=tf.float32)
features = {'x': tf.ones(shape=[4, 1], dtype=tf.float32)}
return features, target
def _my_metric_op(predictions, targets):
# For the case of binary classification, the 2nd column of "predictions"
# denotes the model predictions.
predictions = tf.slice(predictions, [0, 1], [-1, 1])
return tf.reduce_sum(tf.mul(predictions, targets))
classifier = tf.contrib.learn.LinearClassifier(
feature_columns=[tf.contrib.layers.real_valued_column('x')])
classifier.fit(input_fn=_input_fn_train, steps=100)
scores = classifier.evaluate(
input_fn=_input_fn_train,
steps=100,
metrics={
'my_accuracy':
MetricSpec(metric_fn=tf.contrib.metrics.streaming_accuracy,
prediction_key='classes'),
'my_precision':
MetricSpec(metric_fn=tf.contrib.metrics.streaming_precision,
prediction_key='classes'),
'my_metric':
MetricSpec(metric_fn=_my_metric_op,
prediction_key='probabilities')
})
self.assertTrue(
set(['loss', 'my_accuracy', 'my_precision',
'my_metric']).issubset(set(scores.keys())))
predictions = classifier.predict(input_fn=_input_fn_train)
self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions),
scores['my_accuracy'])
# Test the case where the 2nd element of the key is neither "classes" nor
# "probabilities".
with self.assertRaises(ValueError):
classifier.evaluate(input_fn=_input_fn_train,
steps=100,
metrics={
('bad_name', 'bad_type'):
tf.contrib.metrics.streaming_auc
})
# Test the case where the tuple of the key doesn't have 2 elements.
with self.assertRaises(ValueError):
classifier.evaluate(
input_fn=_input_fn_train,
steps=100,
metrics={
('bad_length_name', 'classes', 'bad_length'):
tf.contrib.metrics.streaming_accuracy
})
def testCustomMetrics(self):
"""Tests custom evaluation metrics."""
def _input_fn(num_epochs=None):
# Create 4 rows, one of them (y = x), three of them (y=Not(x))
target = tf.constant([[1], [0], [0], [0]])
features = {
'x': tf.train.limit_epochs(
tf.ones(shape=[4, 1], dtype=tf.float32), num_epochs=num_epochs)}
return features, target
def _my_metric_op(predictions, targets):
# For the case of binary classification, the 2nd column of "predictions"
# denotes the model predictions.
targets = tf.to_float(targets)
predictions = tf.slice(predictions, [0, 1], [-1, 1])
return tf.reduce_sum(tf.mul(predictions, targets))
classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
dnn_hidden_units=[3, 3])
classifier.fit(input_fn=_input_fn, steps=100)
scores = classifier.evaluate(
input_fn=_input_fn,
steps=100,
metrics={
'my_accuracy': tf.contrib.metrics.streaming_accuracy,
('my_precision', 'classes'): tf.contrib.metrics.streaming_precision,
('my_metric', 'probabilities'): _my_metric_op
})
self.assertTrue(
set(['loss', 'my_accuracy', 'my_precision', 'my_metric'
]).issubset(set(scores.keys())))
predict_input_fn = functools.partial(_input_fn, num_epochs=1)
predictions = np.array(
list(classifier.predict(input_fn=predict_input_fn)))
self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions),
scores['my_accuracy'])
# Test the case where the 2nd element of the key is neither "classes" nor
# "probabilities".
with self.assertRaises(KeyError):
classifier.evaluate(
input_fn=_input_fn,
steps=100,
metrics={('bad_name', 'bad_type'): tf.contrib.metrics.streaming_auc})
# Test the case where the tuple of the key doesn't have 2 elements.
with self.assertRaises(ValueError):
classifier.evaluate(
input_fn=_input_fn,
steps=100,
metrics={
('bad_length_name', 'classes', 'bad_length'):
tf.contrib.metrics.streaming_accuracy
})
def testIrisSummaries(self):
iris = datasets.load_iris()
output_dir = tempfile.mkdtemp() + "learn_tests/"
classifier = learn.LinearClassifier(
feature_columns=learn.infer_real_valued_columns_from_input(iris.data),
n_classes=3, model_dir=output_dir)
classifier.fit(iris.data, iris.target, max_steps=100)
score = accuracy_score(iris.target, list(classifier.predict(iris.data)))
self.assertGreater(score, 0.5, "Failed with score = {0}".format(score))
def testIrisSummaries(self):
iris = datasets.load_iris()
output_dir = tempfile.mkdtemp() + "learn_tests/"
classifier = learn.LinearClassifier(
feature_columns=learn.infer_real_valued_columns_from_input(iris.data),
n_classes=3, model_dir=output_dir)
classifier.fit(iris.data, iris.target, max_steps=100)
score = accuracy_score(iris.target, classifier.predict(iris.data))
self.assertGreater(score, 0.5, "Failed with score = {0}".format(score))
def testDNNDropout0_1(self):
# Dropping only a little.
iris = tf.contrib.learn.datasets.load_iris()
classifier = tf.contrib.learn.TensorFlowDNNClassifier(
hidden_units=[10, 20, 10], n_classes=3, dropout=0.1)
classifier.fit(iris.data, iris.target)
score = accuracy_score(iris.target, classifier.predict(iris.data))
# If the quality is lower - dropout is not working.
self.assertGreater(score, 0.9, "Failed with score = {0}".format(score))
def testIrisClassWeight(self):
iris = datasets.load_iris()
# Note, class_weight are not supported anymore :( Use weight_column.
with self.assertRaises(ValueError):
classifier = learn.TensorFlowLinearClassifier(
n_classes=3, class_weight=[0.1, 0.8, 0.1])
classifier.fit(iris.data, iris.target)
score = accuracy_score(iris.target, classifier.predict(iris.data))
self.assertLess(score, 0.7, "Failed with score = {0}".format(score))
def testCustomMetrics(self):
"""Tests custom evaluation metrics."""
def _input_fn_train():
# Create 4 rows, one of them (y = x), three of them (y=Not(x))
target = tf.constant([[1], [0], [0], [0]], dtype=tf.float32)
features = {'x': tf.ones(shape=[4, 1], dtype=tf.float32)}
return features, target
def _my_metric_op(predictions, targets):
# For the case of binary classification, the 2nd column of "predictions"
# denotes the model predictions.
predictions = tf.slice(predictions, [0, 1], [-1, 1])
return tf.reduce_sum(tf.mul(predictions, targets))
classifier = tf.contrib.learn.LinearClassifier(
feature_columns=[tf.contrib.layers.real_valued_column('x')])
classifier.fit(input_fn=_input_fn_train, steps=100)
scores = classifier.evaluate(
input_fn=_input_fn_train,
steps=100,
metrics={
'my_accuracy': MetricSpec(
metric_fn=tf.contrib.metrics.streaming_accuracy,
prediction_key='classes'),
'my_precision': MetricSpec(
metric_fn=tf.contrib.metrics.streaming_precision,
prediction_key='classes'),
'my_metric': MetricSpec(metric_fn=_my_metric_op,
prediction_key='probabilities')
})
self.assertTrue(
set(['loss', 'my_accuracy', 'my_precision', 'my_metric'
]).issubset(set(scores.keys())))
predictions = classifier.predict(input_fn=_input_fn_train)
self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions),
scores['my_accuracy'])
# Test the case where the 2nd element of the key is neither "classes" nor
# "probabilities".
with self.assertRaises(ValueError):
classifier.evaluate(
input_fn=_input_fn_train,
steps=100,
metrics={('bad_name', 'bad_type'): tf.contrib.metrics.streaming_auc})
# Test the case where the tuple of the key doesn't have 2 elements.
with self.assertRaises(ValueError):
classifier.evaluate(
input_fn=_input_fn_train,
steps=100,
metrics={
('bad_length_name', 'classes', 'bad_length'):
tf.contrib.metrics.streaming_accuracy
})
def testDNNDropout0_9(self):
# Dropping out most of it.
iris = tf.contrib.learn.datasets.load_iris()
classifier = tf.contrib.learn.TensorFlowDNNClassifier(
hidden_units=[10, 20, 10], n_classes=3, dropout=0.9)
classifier.fit(iris.data, iris.target)
score = accuracy_score(iris.target, classifier.predict(iris.data))
self.assertGreater(score, 0.3, "Failed with score = {0}".format(score))
# If the quality is higher - dropout is not working.
self.assertLess(score, 0.5, "Failed with score = {0}".format(score))
def testDNNDropout0(self):
# Dropout prob == 0.
iris = tf.contrib.learn.datasets.load_iris()
feature_columns = [tf.contrib.layers.real_valued_column("", dimension=4)]
classifier = tf.contrib.learn.TensorFlowDNNClassifier(
feature_columns=feature_columns, hidden_units=[10, 20, 10], n_classes=3,
dropout=0.0)
classifier.fit(iris.data, iris.target)
score = accuracy_score(iris.target, classifier.predict(iris.data))
self.assertGreater(score, 0.9, "Failed with score = {0}".format(score))
def test_pandas_series(self):
if HAS_PANDAS:
random.seed(42)
iris = datasets.load_iris()
data = pd.DataFrame(iris.data)
labels = pd.Series(iris.target)
classifier = learn.TensorFlowLinearClassifier(n_classes=3)
classifier.fit(data, labels)
score = accuracy_score(labels, classifier.predict(data))
self.assertGreater(score, 0.5, "Failed with score = {0}".format(score))
def testDNNDropout0_1(self):
# Dropping only a little.
tf.set_random_seed(42)
iris = tf.contrib.learn.datasets.load_iris()
classifier = tf.contrib.learn.TensorFlowDNNClassifier(
hidden_units=[10, 20, 10], n_classes=3, dropout=0.1)
classifier.fit(iris.data, iris.target)
score = accuracy_score(iris.target, classifier.predict(iris.data))
# If the quality is lower - dropout is not working.
self.assertGreater(score, 0.9, "Failed with score = {0}".format(score))
def testDNNDropout0_9(self):
# Dropping out most of it.
iris = datasets.load_iris()
classifier = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10],
n_classes=3,
dropout=0.9)
classifier.fit(iris.data, iris.target)
score = accuracy_score(iris.target, classifier.predict(iris.data))
self.assertGreater(score, 0.3, "Failed with score = {0}".format(score))
self.assertLess(score, 0.4, "Failed with score = {0}".format(score))
def _runIrisAll(self, est): iris = tf.contrib.learn.datasets.load_iris() est.fit(iris.data, iris.target, steps=100) scores = est.evaluate(x=iris.data, y=iris.target, name='eval') predictions = est.predict(x=iris.data) predictions_proba = est.predict_proba(x=iris.data) self.assertEqual(predictions.shape[0], iris.target.shape[0]) self.assertAllEqual(predictions, np.argmax(predictions_proba, axis=1)) other_score = _sklearn.accuracy_score(iris.target, predictions) self.assertAllClose(other_score, scores['accuracy'])
def _runIrisAll(self, est): iris = tf.contrib.learn.datasets.load_iris() est.fit(iris.data, iris.target, steps=100) scores = est.evaluate(x=iris.data, y=iris.target, name='eval') predictions = est.predict(x=iris.data) predictions_proba = est.predict_proba(x=iris.data) self.assertEqual(predictions.shape[0], iris.target.shape[0]) self.assertAllEqual(predictions, np.argmax(predictions_proba, axis=1)) other_score = _sklearn.accuracy_score(iris.target, predictions) self.assertAllClose(other_score, scores['accuracy'])
def testIrisPredictAsIterable(self): iris = tf.contrib.learn.datasets.load_iris() est = tf.contrib.learn.Classifier(model_fn=logistic_model_fn, n_classes=3) est.fit(iris.data, iris.target, steps=100) scores = est.evaluate(x=iris.data, y=iris.target, name='eval') predictions = list(est.predict(x=iris.data, as_iterable=True)) predictions_proba = list(est.predict_proba(x=iris.data, as_iterable=True)) self.assertEqual(len(predictions), iris.target.shape[0]) self.assertAllEqual(predictions, np.argmax(predictions_proba, axis=1)) other_score = _sklearn.accuracy_score(iris.target, predictions) self.assertAllClose(other_score, scores['accuracy'])
def testDNN(self):
path = tf.test.get_temp_dir() + '/tmp_saver3'
random.seed(42)
iris = datasets.load_iris()
classifier = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], n_classes=3)
classifier.fit(iris.data, iris.target)
classifier.save(path)
new_classifier = learn.TensorFlowEstimator.restore(path)
self.assertEqual(type(new_classifier), type(classifier))
score = accuracy_score(iris.target, new_classifier.predict(iris.data))
self.assertGreater(score, 0.5, "Failed with score = {0}".format(score))
def testIrisPredictAsIterable(self): iris = tf.contrib.learn.datasets.load_iris() est = tf.contrib.learn.Classifier(model_fn=logistic_model_fn, n_classes=3) est.fit(iris.data, iris.target, steps=100) scores = est.evaluate(x=iris.data, y=iris.target, name='eval') predictions = list(est.predict(x=iris.data, as_iterable=True)) predictions_proba = list(est.predict_proba(x=iris.data, as_iterable=True)) self.assertEqual(len(predictions), iris.target.shape[0]) self.assertAllEqual(predictions, np.argmax(predictions_proba, axis=1)) other_score = _sklearn.accuracy_score(iris.target, predictions) self.assertAllClose(other_score, scores['accuracy'])
def test_pandas_series(self):
if HAS_PANDAS:
import pandas as pd
random.seed(42)
iris = datasets.load_iris()
data = pd.DataFrame(iris.data)
labels = pd.Series(iris.target)
classifier = learn.TensorFlowLinearClassifier(n_classes=3)
classifier.fit(data, labels)
score = accuracy_score(labels, classifier.predict(data))
self.assertGreater(score, 0.5, "Failed with score = {0}".format(score))
def testDNNDropout0_1(self):
# Dropping only a little.
iris = tf.contrib.learn.datasets.load_iris()
feature_columns = [tf.contrib.layers.real_valued_column("", dimension=4)]
classifier = tf.contrib.learn.TensorFlowDNNClassifier(
feature_columns=feature_columns, hidden_units=[10, 20, 10], n_classes=3,
dropout=0.1)
classifier.fit(iris.data, iris.target)
score = accuracy_score(iris.target, classifier.predict(iris.data))
# If the quality is lower - dropout is not working.
self.assertGreater(score, 0.9, "Failed with score = {0}".format(score))
def testDNNDropout0_9(self):
random.seed(42)
# Dropping out most of it.
iris = tf.contrib.learn.datasets.load_iris()
classifier = tf.contrib.learn.TensorFlowDNNClassifier(
hidden_units=[10, 20, 10], n_classes=3, dropout=0.9)
classifier.fit(iris.data, iris.target)
score = accuracy_score(iris.target, classifier.predict(iris.data))
self.assertGreater(score, 0.3, "Failed with score = {0}".format(score))
# If the quality is higher - dropout is not working.
self.assertLess(score, 0.6, "Failed with score = {0}".format(score))
def testIris(self):
path = tf.test.get_temp_dir() + '/tmp.saver'
random.seed(42)
iris = datasets.load_iris()
classifier = learn.TensorFlowLinearClassifier(n_classes=3)
classifier.fit(iris.data, iris.target)
classifier.save(path)
new_classifier = learn.TensorFlowEstimator.restore(path)
self.assertEqual(type(new_classifier), type(classifier))
score = accuracy_score(iris.target, new_classifier.predict(iris.data))
self.assertGreater(score, 0.5, 'Failed with score = {0}'.format(score))
def testIrisAll(self): iris = tf.contrib.learn.datasets.load_iris() est = tf.contrib.learn.Classifier(model_fn=logistic_model_fn, n_classes=3) est.fit(iris.data, iris.target, steps=100) scores = est.evaluate(x=iris.data, y=iris.target) predictions = est.predict(x=iris.data) predictions_proba = est.predict_proba(x=iris.data) self.assertEqual(predictions.shape[0], iris.target.shape[0]) self.assertAllClose(predictions, np.argmax(predictions_proba, axis=1)) other_score = _sklearn.accuracy_score(iris.target, predictions) self.assertAllClose(other_score, scores['accuracy'])
def testIris(self):
path = tf.test.get_temp_dir() + '/tmp.saver'
random.seed(42)
iris = datasets.load_iris()
classifier = learn.TensorFlowLinearClassifier(n_classes=3)
classifier.fit(iris.data, iris.target)
classifier.save(path)
new_classifier = learn.TensorFlowEstimator.restore(path)
self.assertEqual(type(new_classifier), type(classifier))
score = accuracy_score(iris.target, new_classifier.predict(iris.data))
self.assertGreater(score, 0.5, 'Failed with score = {0}'.format(score))
def testIrisAll(self):
iris = tf.contrib.learn.datasets.load_iris()
est = tf.contrib.learn.Classifier(model_fn=logistic_model_fn,
n_classes=3)
est.fit(iris.data, iris.target, steps=100)
scores = est.evaluate(x=iris.data, y=iris.target)
predictions = est.predict(x=iris.data)
predictions_proba = est.predict_proba(x=iris.data)
self.assertEqual(predictions.shape[0], iris.target.shape[0])
self.assertAllClose(predictions, np.argmax(predictions_proba, axis=1))
other_score = _sklearn.accuracy_score(iris.target, predictions)
self.assertAllClose(other_score, scores['accuracy'])
def test_pandas_series(self):
if HAS_PANDAS:
import pandas as pd # pylint: disable=g-import-not-at-top
random.seed(42)
iris = datasets.load_iris()
data = pd.DataFrame(iris.data)
labels = pd.Series(iris.target)
classifier = learn.TensorFlowLinearClassifier(
feature_columns=learn.infer_real_valued_columns_from_input(data),
n_classes=3)
classifier.fit(data, labels)
score = accuracy_score(labels, classifier.predict(data))
self.assertGreater(score, 0.5, "Failed with score = {0}".format(score))
