def testIrisStreaming(self): iris = datasets.load_iris() def iris_data(): while True: for x in iris.data: yield x def iris_predict_data(): for x in iris.data: yield x def iris_target(): while True: for y in iris.target: yield y classifier = learn.LinearClassifier( feature_columns=learn.infer_real_valued_columns_from_input(iris.data), n_classes=3) classifier.fit(iris_data(), iris_target(), max_steps=500) score1 = accuracy_score(iris.target, classifier.predict(iris.data)) score2 = accuracy_score(iris.target, classifier.predict(iris_predict_data())) self.assertGreater(score1, 0.5, "Failed with score = {0}".format(score1)) self.assertEqual(score2, score1, "Scores from {0} iterator doesn't " "match score {1} from full " "data.".format(score2, score1))
def testIrisES(self): random.seed(42) iris = datasets.load_iris() x_train, x_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.2, random_state=42) x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=42) val_monitor = learn.monitors.ValidationMonitor( x_val, y_val, every_n_steps=50, early_stopping_rounds=100, early_stopping_metric='accuracy', early_stopping_metric_minimize=False) # classifier without early stopping - overfitting classifier1 = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], n_classes=3, steps=1000) classifier1.fit(x_train, y_train) _ = accuracy_score(y_test, classifier1.predict(x_test)) # Full 1000 steps, 11 summaries and no evaluation summary. # 11 summaries = first + every 100 out of 1000 steps. self.assertEqual(11, len(_get_summary_events(classifier1.model_dir))) with self.assertRaises(ValueError): _get_summary_events(classifier1.model_dir + '/eval') # classifier with early stopping - improved accuracy on testing set classifier2 = learn.TensorFlowDNNClassifier( hidden_units=[10, 20, 10], n_classes=3, steps=2000, config=tf.contrib.learn.RunConfig(save_checkpoints_secs=1)) classifier2.fit(x_train, y_train, monitors=[val_monitor]) _ = accuracy_score(y_val, classifier2.predict(x_val)) _ = accuracy_score(y_test, classifier2.predict(x_test)) # Note, this test is unstable, so not checking for equality. # See stability_test for examples of stability issues. if val_monitor.early_stopped: self.assertLess(val_monitor.best_step, 2000) # Note, due to validation monitor stopping after the best score occur, # the accuracy at current checkpoint is less. # TODO(ipolosukhin): Time machine for restoring old checkpoints? # flaky, still not always best_value better then score2 value. # self.assertGreater(val_monitor.best_value, score2_val) # Early stopped, unstable so checking only < then max. self.assertLess(len(_get_summary_events(classifier2.model_dir)), 21) self.assertLess( len(_get_summary_events(classifier2.model_dir + '/eval')), 4)
def testIrisStreaming(self): iris = datasets.load_iris() def iris_data(): while True: for x in iris.data: yield x def iris_predict_data(): for x in iris.data: yield x def iris_target(): while True: for y in iris.target: yield y classifier = learn.LinearClassifier( feature_columns=learn.infer_real_valued_columns_from_input( iris.data), n_classes=3) classifier.fit(iris_data(), iris_target(), max_steps=500) score1 = accuracy_score(iris.target, classifier.predict(iris.data)) score2 = accuracy_score(iris.target, classifier.predict(iris_predict_data())) self.assertGreater(score1, 0.5, "Failed with score = {0}".format(score1)) self.assertEqual( score2, score1, "Scores from {0} iterator doesn't " "match score {1} from full " "data.".format(score2, score1))
def testIrisStreaming(self): iris = datasets.load_iris() def iris_data(): while True: for x in iris.data: yield x def iris_predict_data(): for x in iris.data: yield x def iris_target(): while True: for y in iris.target: yield y classifier = learn.TensorFlowLinearClassifier(n_classes=3, steps=100) classifier.fit(iris_data(), iris_target()) score1 = accuracy_score(iris.target, classifier.predict(iris.data)) score2 = accuracy_score(iris.target, classifier.predict(iris_predict_data())) self.assertGreater(score1, 0.5, "Failed with score = {0}".format(score1)) self.assertEqual(score2, score1, "Scores from {0} iterator doesn't " "match score {1} from full " "data.".format(score2, score1))
def testIrisES(self): random.seed(42) iris = datasets.load_iris() x_train, x_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.2, random_state=42) x_train, x_val, y_train, y_val = train_test_split( x_train, y_train, test_size=0.2) val_monitor = learn.monitors.ValidationMonitor(x_val, y_val, early_stopping_rounds=100) # classifier without early stopping - overfitting classifier1 = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], n_classes=3, steps=1000) classifier1.fit(x_train, y_train) accuracy_score(y_test, classifier1.predict(x_test)) # classifier with early stopping - improved accuracy on testing set classifier2 = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], n_classes=3, steps=1000) classifier2.fit(x_train, y_train, monitors=[val_monitor]) accuracy_score(y_test, classifier2.predict(x_test))
def testIrisContinueTraining(self): iris = datasets.load_iris() classifier = learn.TensorFlowLinearClassifier(n_classes=3, learning_rate=0.01, continue_training=True, steps=250) classifier.fit(iris.data, iris.target) score1 = accuracy_score(iris.target, classifier.predict(iris.data)) classifier.fit(iris.data, iris.target) score2 = accuracy_score(iris.target, classifier.predict(iris.data)) self.assertGreater(score2, score1, "Failed with score = {0}".format(score2))
def testIrisES(self): random.seed(42) iris = datasets.load_iris() x_train, x_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.2, random_state=42) x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=42) val_monitor = learn.monitors.ValidationMonitor( x_val, y_val, every_n_steps=50, early_stopping_rounds=100, early_stopping_metric="accuracy", early_stopping_metric_minimize=False, ) # classifier without early stopping - overfitting classifier1 = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], n_classes=3, steps=1000) classifier1.fit(x_train, y_train) _ = accuracy_score(y_test, classifier1.predict(x_test)) # Full 1000 steps, 12 summaries and no evaluation summary. # 12 summaries = global_step + first + every 100 out of 1000 steps. self.assertEqual(12, len(_get_summary_events(classifier1.model_dir))) with self.assertRaises(ValueError): _get_summary_events(classifier1.model_dir + "/eval") # classifier with early stopping - improved accuracy on testing set classifier2 = learn.TensorFlowDNNClassifier( hidden_units=[10, 20, 10], n_classes=3, steps=2000, config=tf.contrib.learn.RunConfig(save_checkpoints_secs=1), ) classifier2.fit(x_train, y_train, monitors=[val_monitor]) _ = accuracy_score(y_val, classifier2.predict(x_val)) _ = accuracy_score(y_test, classifier2.predict(x_test)) # Note, this test is unstable, so not checking for equality. # See stability_test for examples of stability issues. if val_monitor.early_stopped: self.assertLess(val_monitor.best_step, 2000) # Note, due to validation monitor stopping after the best score occur, # the accuracy at current checkpoint is less. # TODO(ipolosukhin): Time machine for restoring old checkpoints? # flaky, still not always best_value better then score2 value. # self.assertGreater(val_monitor.best_value, score2_val) # Early stopped, unstable so checking only < then max. self.assertLess(len(_get_summary_events(classifier2.model_dir)), 21) # Eval typically has ~6 events, but it varies based on the run. self.assertLess(len(_get_summary_events(classifier2.model_dir + "/eval")), 8)
def testIrisContinueTraining(self): iris = datasets.load_iris() classifier = learn.LinearClassifier( feature_columns=learn.infer_real_valued_columns_from_input(iris.data), n_classes=3) classifier.fit(iris.data, iris.target, steps=100) score1 = accuracy_score(iris.target, classifier.predict(iris.data)) classifier.fit(iris.data, iris.target, steps=500) score2 = accuracy_score(iris.target, classifier.predict(iris.data)) self.assertGreater( score2, score1, "Failed with score2 {0} <= score1 {1}".format(score2, score1))
def testIrisContinueTraining(self): iris = datasets.load_iris() classifier = learn.LinearClassifier( feature_columns=learn.infer_real_valued_columns_from_input(iris.data), n_classes=3) classifier.fit(iris.data, iris.target, steps=100) score1 = accuracy_score(iris.target, list(classifier.predict(iris.data))) classifier.fit(iris.data, iris.target, steps=500) score2 = accuracy_score(iris.target, list(classifier.predict(iris.data))) self.assertGreater( score2, score1, "Failed with score2 {0} <= score1 {1}".format(score2, score1))
def testIrisContinueTraining(self): iris = datasets.load_iris() classifier = learn.TensorFlowLinearClassifier(n_classes=3, learning_rate=0.01, continue_training=True, steps=250) classifier.fit(iris.data, iris.target) score1 = accuracy_score(iris.target, classifier.predict(iris.data)) classifier.fit(iris.data, iris.target) score2 = accuracy_score(iris.target, classifier.predict(iris.data)) self.assertGreater(score2, score1, "Failed with score = {0}".format(score2))
def testIrisContinueTraining(self): iris = datasets.load_iris() classifier = learn.TensorFlowLinearClassifier( feature_columns=learn.infer_real_valued_columns_from_input(iris.data), n_classes=3, learning_rate=0.01, continue_training=True, steps=250) classifier.fit(iris.data, iris.target) score1 = accuracy_score(iris.target, classifier.predict(iris.data)) classifier.fit(iris.data, iris.target, steps=500) score2 = accuracy_score(iris.target, classifier.predict(iris.data)) self.assertGreater( score2, score1, "Failed with score2 {0} <= score1 {1}".format(score2, score1))
def testIrisMomentum(self): random.seed(42) iris = datasets.load_iris() x_train, x_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.2, random_state=42) def custom_optimizer(): return momentum_lib.MomentumOptimizer(learning_rate=0.01, momentum=0.9) classifier = learn.DNNClassifier( hidden_units=[10, 20, 10], feature_columns=learn.infer_real_valued_columns_from_input( x_train), n_classes=3, optimizer=custom_optimizer, config=learn.RunConfig(tf_random_seed=1)) classifier.fit(x_train, y_train, steps=400) predictions = np.array(list(classifier.predict_classes(x_test))) score = accuracy_score(y_test, predictions) self.assertGreater(score, 0.65, "Failed with score = {0}".format(score))
def testIrisMomentum(self): random.seed(42) iris = datasets.load_iris() x_train, x_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.2, random_state=42) def custom_optimizer(learning_rate): return tf.train.MomentumOptimizer(learning_rate, 0.9) cont_features = [ tf.contrib.layers.real_valued_column("", dimension=4)] classifier = learn.TensorFlowDNNClassifier( feature_columns=cont_features, hidden_units=[10, 20, 10], n_classes=3, steps=400, learning_rate=0.01, optimizer=custom_optimizer) classifier.fit(x_train, y_train) score = accuracy_score(y_test, classifier.predict(x_test)) self.assertGreater(score, 0.65, "Failed with score = {0}".format(score))
def testIrisAllDictionaryInput(self): iris = base.load_iris() est = estimator.Estimator(model_fn=logistic_model_no_mode_fn) iris_data = {'input': iris.data} iris_target = {'labels': iris.target} est.fit(iris_data, iris_target, steps=100) scores = est.evaluate( x=iris_data, y=iris_target, metrics={ ('accuracy', 'class'): metric_ops.streaming_accuracy }) predictions = list(est.predict(x=iris_data)) predictions_class = list(est.predict(x=iris_data, outputs=['class'])) self.assertEqual(len(predictions), iris.target.shape[0]) classes_batch = np.array([p['class'] for p in predictions]) self.assertAllClose(classes_batch, np.array([p['class'] for p in predictions_class])) self.assertAllClose(classes_batch, np.argmax( np.array([p['prob'] for p in predictions]), axis=1)) other_score = _sklearn.accuracy_score(iris.target, classes_batch) self.assertAllClose(other_score, scores['accuracy']) self.assertTrue('global_step' in scores) self.assertEqual(scores['global_step'], 100)
def testCustomMetrics(self): """Tests weight column in evaluation.""" def _input_fn_train(): # Create 4 rows, one of them (y = x), three of them (y=Not(x)) target = tf.constant([[1], [0], [0], [0]]) features = {'x': tf.ones(shape=[4, 1], dtype=tf.float32),} return features, target classifier = tf.contrib.learn.DNNLinearCombinedClassifier( linear_feature_columns=[tf.contrib.layers.real_valued_column('x')], dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')], dnn_hidden_units=[3, 3]) classifier.train(input_fn=_input_fn_train, steps=100) scores = classifier.evaluate( input_fn=_input_fn_train, steps=100, metrics={ 'my_accuracy': tf.contrib.metrics.streaming_accuracy, 'my_precision': tf.contrib.metrics.streaming_precision }) self.assertTrue(set(['loss', 'my_accuracy', 'my_precision']).issubset(set( scores.keys()))) predictions = classifier.predict(input_fn=_input_fn_train) self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions), scores['my_accuracy'])
def testDNNDropout0_1(self): # Dropping only a little. iris = datasets.load_iris() classifier = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], n_classes=3, dropout=0.1) classifier.fit(iris.data, iris.target) score = accuracy_score(iris.target, classifier.predict(iris.data)) self.assertGreater(score, 0.9, "Failed with score = {0}".format(score))
def testIrisAllDictionaryInput(self): iris = tf.contrib.learn.datasets.load_iris() est = tf.contrib.learn.Estimator(model_fn=logistic_model_no_mode_fn) iris_data = {'input': iris.data} iris_target = {'labels': iris.target} est.fit(iris_data, iris_target, steps=100) scores = est.evaluate(x=iris_data, y=iris_target, metrics={ ('accuracy', 'class'): tf.contrib.metrics.streaming_accuracy }) predictions = list(est.predict(x=iris_data)) predictions_class = list(est.predict(x=iris_data, outputs=['class'])) self.assertEqual(len(predictions), iris.target.shape[0]) classes_batch = np.array([p['class'] for p in predictions]) self.assertAllClose(classes_batch, np.array([p['class'] for p in predictions_class])) self.assertAllClose( classes_batch, np.argmax(np.array([p['prob'] for p in predictions]), axis=1)) other_score = _sklearn.accuracy_score(iris.target, classes_batch) self.assertAllClose(other_score, scores['accuracy']) self.assertTrue('global_step' in scores) self.assertEqual(scores['global_step'], 100)
def testIrisClassWeight(self): iris = datasets.load_iris() classifier = learn.TensorFlowLinearClassifier( n_classes=3, class_weight=[0.1, 0.8, 0.1]) classifier.fit(iris.data, iris.target) score = accuracy_score(iris.target, classifier.predict(iris.data)) self.assertLess(score, 0.7, "Failed with score = {0}".format(score))
def testCustomMetrics(self): """Tests weight column in evaluation.""" def _input_fn_train(): # Create 4 rows, one of them (y = x), three of them (y=Not(x)) target = tf.constant([[1], [0], [0], [0]]) features = { 'x': tf.ones(shape=[4, 1], dtype=tf.float32), } return features, target classifier = tf.contrib.learn.DNNLinearCombinedClassifier( linear_feature_columns=[tf.contrib.layers.real_valued_column('x')], dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')], dnn_hidden_units=[3, 3]) classifier.fit(input_fn=_input_fn_train, steps=100) scores = classifier.evaluate(input_fn=_input_fn_train, steps=100, metrics={ 'my_accuracy': tf.contrib.metrics.streaming_accuracy, 'my_precision': tf.contrib.metrics.streaming_precision }) self.assertTrue( set(['loss', 'my_accuracy', 'my_precision']).issubset(set(scores.keys()))) predictions = classifier.predict(input_fn=_input_fn_train) self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions), scores['my_accuracy'])
def testIrisClassWeight(self): iris = datasets.load_iris() classifier = learn.TensorFlowLinearClassifier( n_classes=3, class_weight=[0.1, 0.8, 0.1]) classifier.fit(iris.data, iris.target) score = accuracy_score(iris.target, classifier.predict(iris.data)) self.assertLess(score, 0.7, "Failed with score = {0}".format(score))
def testIrisMomentum(self): random.seed(42) iris = datasets.load_iris() x_train, x_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.2, random_state=42) def custom_optimizer(learning_rate): return tf.train.MomentumOptimizer(learning_rate, 0.9) classifier = learn.TensorFlowDNNClassifier( hidden_units=[10, 20, 10], feature_columns=learn.infer_real_valued_columns_from_input( x_train), n_classes=3, steps=400, learning_rate=0.01, optimizer=custom_optimizer) classifier.fit(x_train, y_train) score = accuracy_score(y_test, classifier.predict(x_test)) self.assertGreater(score, 0.65, "Failed with score = {0}".format(score))
def testIrisMomentum(self): random.seed(42) iris = datasets.load_iris() x_train, x_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.2, random_state=42) # setup exponential decay function def exp_decay(global_step): return tf.train.exponential_decay(learning_rate=0.1, global_step=global_step, decay_steps=100, decay_rate=0.001) def custom_optimizer(learning_rate): return tf.train.MomentumOptimizer(learning_rate, 0.9) classifier = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], n_classes=3, steps=400, learning_rate=exp_decay, optimizer=custom_optimizer) classifier.fit(x_train, y_train) score = accuracy_score(y_test, classifier.predict(x_test)) self.assertGreater(score, 0.65, "Failed with score = {0}".format(score))
def testCustomMetrics(self): """Tests custom evaluation metrics.""" def _input_fn(num_epochs=None): # Create 4 rows, one of them (y = x), three of them (y=Not(x)) labels = tf.constant([[1], [0], [0], [0]]) features = { 'x': tf.train.limit_epochs(tf.ones(shape=[4, 1], dtype=tf.float32), num_epochs=num_epochs), } return features, labels def _my_metric_op(predictions, labels): # For the case of binary classification, the 2nd column of "predictions" # denotes the model predictions. labels = tf.to_float(labels) predictions = tf.slice(predictions, [0, 1], [-1, 1]) labels = math_ops.cast(labels, predictions.dtype) return tf.reduce_sum(tf.mul(predictions, labels)) classifier = tf.contrib.learn.DNNClassifier( feature_columns=[tf.contrib.layers.real_valued_column('x')], hidden_units=[3, 3], config=tf.contrib.learn.RunConfig(tf_random_seed=1)) classifier.fit(input_fn=_input_fn, steps=5) scores = classifier.evaluate( input_fn=_input_fn, steps=5, metrics={ 'my_accuracy': MetricSpec(metric_fn=tf.contrib.metrics.streaming_accuracy, prediction_key='classes'), 'my_precision': MetricSpec(metric_fn=tf.contrib.metrics.streaming_precision, prediction_key='classes'), 'my_metric': MetricSpec(metric_fn=_my_metric_op, prediction_key='probabilities') }) self.assertTrue( set(['loss', 'my_accuracy', 'my_precision', 'my_metric']).issubset(set(scores.keys()))) predict_input_fn = functools.partial(_input_fn, num_epochs=1) predictions = np.array( list(classifier.predict(input_fn=predict_input_fn))) self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions), scores['my_accuracy']) # Test the case where the 2nd element of the key is neither "classes" nor # "probabilities". with self.assertRaisesRegexp(KeyError, 'bad_type'): classifier.evaluate( input_fn=_input_fn, steps=5, metrics={ 'bad_name': MetricSpec(metric_fn=tf.contrib.metrics.streaming_auc, prediction_key='bad_type') })
def testIrisMomentum(self): random.seed(42) iris = datasets.load_iris() X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.2, random_state=42) # setup exponential decay function def exp_decay(global_step): return tf.train.exponential_decay(learning_rate=0.1, global_step=global_step, decay_steps=100, decay_rate=0.001) custom_optimizer = lambda learning_rate: tf.train.MomentumOptimizer( learning_rate, 0.9) classifier = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], n_classes=3, steps=800, learning_rate=exp_decay, optimizer=custom_optimizer) classifier.fit(X_train, y_train) score = accuracy_score(y_test, classifier.predict(X_test)) self.assertGreater(score, 0.7, "Failed with score = {0}".format(score))
def testIris(self): iris = datasets.load_iris() classifier = learn.TensorFlowLinearClassifier( feature_columns=learn.infer_real_valued_columns_from_input(iris.data), n_classes=3) classifier.fit(iris.data, [x for x in iris.target]) score = accuracy_score(iris.target, classifier.predict(iris.data)) self.assertGreater(score, 0.7, "Failed with score = {0}".format(score))
def testDNNDropout0(self): # Dropout prob == 0. iris = tf.contrib.learn.datasets.load_iris() classifier = tf.contrib.learn.TensorFlowDNNClassifier( hidden_units=[10, 20, 10], n_classes=3, dropout=0.0) classifier.fit(iris.data, iris.target) score = accuracy_score(iris.target, classifier.predict(iris.data)) self.assertGreater(score, 0.9, "Failed with score = {0}".format(score))
def testDNNDropout0(self): # Dropout prob == 0. iris = tf.contrib.learn.datasets.load_iris() classifier = tf.contrib.learn.TensorFlowDNNClassifier( hidden_units=[10, 20, 10], n_classes=3, dropout=0.0) classifier.fit(iris.data, iris.target) score = accuracy_score(iris.target, classifier.predict(iris.data)) self.assertGreater(score, 0.9, "Failed with score = {0}".format(score))
def testIrisSummaries(self): iris = datasets.load_iris() output_dir = tempfile.mkdtemp() + "learn_tests/" classifier = learn.TensorFlowLinearClassifier(n_classes=3, model_dir=output_dir) classifier.fit(iris.data, iris.target) score = accuracy_score(iris.target, classifier.predict(iris.data)) self.assertGreater(score, 0.5, "Failed with score = {0}".format(score))
def testCustomMetrics(self): """Tests custom evaluation metrics.""" def _input_fn(num_epochs=None): # Create 4 rows, one of them (y = x), three of them (y=Not(x)) labels = tf.constant([[1], [0], [0], [0]]) features = { 'x': tf.train.limit_epochs( tf.ones(shape=[4, 1], dtype=tf.float32), num_epochs=num_epochs), } return features, labels def _my_metric_op(predictions, labels): # For the case of binary classification, the 2nd column of "predictions" # denotes the model predictions. labels = tf.to_float(labels) predictions = tf.strided_slice( predictions, [0, 1], [-1, 2], end_mask=1) labels = math_ops.cast(labels, predictions.dtype) return tf.reduce_sum(tf.multiply(predictions, labels)) classifier = tf.contrib.learn.DNNClassifier( feature_columns=[tf.contrib.layers.real_valued_column('x')], hidden_units=[3, 3], config=tf.contrib.learn.RunConfig(tf_random_seed=1)) classifier.fit(input_fn=_input_fn, steps=5) scores = classifier.evaluate( input_fn=_input_fn, steps=5, metrics={ 'my_accuracy': MetricSpec( metric_fn=tf.contrib.metrics.streaming_accuracy, prediction_key='classes'), 'my_precision': MetricSpec( metric_fn=tf.contrib.metrics.streaming_precision, prediction_key='classes'), 'my_metric': MetricSpec( metric_fn=_my_metric_op, prediction_key='probabilities') }) self.assertTrue( set(['loss', 'my_accuracy', 'my_precision', 'my_metric' ]).issubset(set(scores.keys()))) predict_input_fn = functools.partial(_input_fn, num_epochs=1) predictions = np.array(list(classifier.predict(input_fn=predict_input_fn))) self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions), scores['my_accuracy']) # Test the case where the 2nd element of the key is neither "classes" nor # "probabilities". with self.assertRaisesRegexp(KeyError, 'bad_type'): classifier.evaluate( input_fn=_input_fn, steps=5, metrics={ 'bad_name': MetricSpec( metric_fn=tf.contrib.metrics.streaming_auc, prediction_key='bad_type')})
def testCustomMetrics(self): """Tests custom evaluation metrics.""" def _input_fn_train(): # Create 4 rows, one of them (y = x), three of them (y=Not(x)) target = tf.constant([[1], [0], [0], [0]], dtype=tf.float32) features = {'x': tf.ones(shape=[4, 1], dtype=tf.float32)} return features, target def _my_metric_op(predictions, targets): # For the case of binary classification, the 2nd column of "predictions" # denotes the model predictions. predictions = tf.slice(predictions, [0, 1], [-1, 1]) return tf.reduce_sum(tf.mul(predictions, targets)) classifier = tf.contrib.learn.LinearClassifier( feature_columns=[tf.contrib.layers.real_valued_column('x')]) classifier.fit(input_fn=_input_fn_train, steps=100) scores = classifier.evaluate( input_fn=_input_fn_train, steps=100, metrics={ 'my_accuracy': MetricSpec(metric_fn=tf.contrib.metrics.streaming_accuracy, prediction_key='classes'), 'my_precision': MetricSpec(metric_fn=tf.contrib.metrics.streaming_precision, prediction_key='classes'), 'my_metric': MetricSpec(metric_fn=_my_metric_op, prediction_key='probabilities') }) self.assertTrue( set(['loss', 'my_accuracy', 'my_precision', 'my_metric']).issubset(set(scores.keys()))) predictions = classifier.predict(input_fn=_input_fn_train) self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions), scores['my_accuracy']) # Test the case where the 2nd element of the key is neither "classes" nor # "probabilities". with self.assertRaises(ValueError): classifier.evaluate(input_fn=_input_fn_train, steps=100, metrics={ ('bad_name', 'bad_type'): tf.contrib.metrics.streaming_auc }) # Test the case where the tuple of the key doesn't have 2 elements. with self.assertRaises(ValueError): classifier.evaluate( input_fn=_input_fn_train, steps=100, metrics={ ('bad_length_name', 'classes', 'bad_length'): tf.contrib.metrics.streaming_accuracy })
def testCustomMetrics(self): """Tests custom evaluation metrics.""" def _input_fn(num_epochs=None): # Create 4 rows, one of them (y = x), three of them (y=Not(x)) target = tf.constant([[1], [0], [0], [0]]) features = { 'x': tf.train.limit_epochs( tf.ones(shape=[4, 1], dtype=tf.float32), num_epochs=num_epochs)} return features, target def _my_metric_op(predictions, targets): # For the case of binary classification, the 2nd column of "predictions" # denotes the model predictions. targets = tf.to_float(targets) predictions = tf.slice(predictions, [0, 1], [-1, 1]) return tf.reduce_sum(tf.mul(predictions, targets)) classifier = tf.contrib.learn.DNNLinearCombinedClassifier( linear_feature_columns=[tf.contrib.layers.real_valued_column('x')], dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')], dnn_hidden_units=[3, 3]) classifier.fit(input_fn=_input_fn, steps=100) scores = classifier.evaluate( input_fn=_input_fn, steps=100, metrics={ 'my_accuracy': tf.contrib.metrics.streaming_accuracy, ('my_precision', 'classes'): tf.contrib.metrics.streaming_precision, ('my_metric', 'probabilities'): _my_metric_op }) self.assertTrue( set(['loss', 'my_accuracy', 'my_precision', 'my_metric' ]).issubset(set(scores.keys()))) predict_input_fn = functools.partial(_input_fn, num_epochs=1) predictions = np.array( list(classifier.predict(input_fn=predict_input_fn))) self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions), scores['my_accuracy']) # Test the case where the 2nd element of the key is neither "classes" nor # "probabilities". with self.assertRaises(KeyError): classifier.evaluate( input_fn=_input_fn, steps=100, metrics={('bad_name', 'bad_type'): tf.contrib.metrics.streaming_auc}) # Test the case where the tuple of the key doesn't have 2 elements. with self.assertRaises(ValueError): classifier.evaluate( input_fn=_input_fn, steps=100, metrics={ ('bad_length_name', 'classes', 'bad_length'): tf.contrib.metrics.streaming_accuracy })
def testIrisSummaries(self): iris = datasets.load_iris() output_dir = tempfile.mkdtemp() + "learn_tests/" classifier = learn.LinearClassifier( feature_columns=learn.infer_real_valued_columns_from_input(iris.data), n_classes=3, model_dir=output_dir) classifier.fit(iris.data, iris.target, max_steps=100) score = accuracy_score(iris.target, list(classifier.predict(iris.data))) self.assertGreater(score, 0.5, "Failed with score = {0}".format(score))
def testIrisSummaries(self): iris = datasets.load_iris() output_dir = tempfile.mkdtemp() + "learn_tests/" classifier = learn.LinearClassifier( feature_columns=learn.infer_real_valued_columns_from_input(iris.data), n_classes=3, model_dir=output_dir) classifier.fit(iris.data, iris.target, max_steps=100) score = accuracy_score(iris.target, classifier.predict(iris.data)) self.assertGreater(score, 0.5, "Failed with score = {0}".format(score))
def testDNNDropout0_1(self): # Dropping only a little. iris = tf.contrib.learn.datasets.load_iris() classifier = tf.contrib.learn.TensorFlowDNNClassifier( hidden_units=[10, 20, 10], n_classes=3, dropout=0.1) classifier.fit(iris.data, iris.target) score = accuracy_score(iris.target, classifier.predict(iris.data)) # If the quality is lower - dropout is not working. self.assertGreater(score, 0.9, "Failed with score = {0}".format(score))
def testIrisClassWeight(self): iris = datasets.load_iris() # Note, class_weight are not supported anymore :( Use weight_column. with self.assertRaises(ValueError): classifier = learn.TensorFlowLinearClassifier( n_classes=3, class_weight=[0.1, 0.8, 0.1]) classifier.fit(iris.data, iris.target) score = accuracy_score(iris.target, classifier.predict(iris.data)) self.assertLess(score, 0.7, "Failed with score = {0}".format(score))
def testCustomMetrics(self): """Tests custom evaluation metrics.""" def _input_fn_train(): # Create 4 rows, one of them (y = x), three of them (y=Not(x)) target = tf.constant([[1], [0], [0], [0]], dtype=tf.float32) features = {'x': tf.ones(shape=[4, 1], dtype=tf.float32)} return features, target def _my_metric_op(predictions, targets): # For the case of binary classification, the 2nd column of "predictions" # denotes the model predictions. predictions = tf.slice(predictions, [0, 1], [-1, 1]) return tf.reduce_sum(tf.mul(predictions, targets)) classifier = tf.contrib.learn.LinearClassifier( feature_columns=[tf.contrib.layers.real_valued_column('x')]) classifier.fit(input_fn=_input_fn_train, steps=100) scores = classifier.evaluate( input_fn=_input_fn_train, steps=100, metrics={ 'my_accuracy': MetricSpec( metric_fn=tf.contrib.metrics.streaming_accuracy, prediction_key='classes'), 'my_precision': MetricSpec( metric_fn=tf.contrib.metrics.streaming_precision, prediction_key='classes'), 'my_metric': MetricSpec(metric_fn=_my_metric_op, prediction_key='probabilities') }) self.assertTrue( set(['loss', 'my_accuracy', 'my_precision', 'my_metric' ]).issubset(set(scores.keys()))) predictions = classifier.predict(input_fn=_input_fn_train) self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions), scores['my_accuracy']) # Test the case where the 2nd element of the key is neither "classes" nor # "probabilities". with self.assertRaises(ValueError): classifier.evaluate( input_fn=_input_fn_train, steps=100, metrics={('bad_name', 'bad_type'): tf.contrib.metrics.streaming_auc}) # Test the case where the tuple of the key doesn't have 2 elements. with self.assertRaises(ValueError): classifier.evaluate( input_fn=_input_fn_train, steps=100, metrics={ ('bad_length_name', 'classes', 'bad_length'): tf.contrib.metrics.streaming_accuracy })
def testDNNDropout0_9(self): # Dropping out most of it. iris = tf.contrib.learn.datasets.load_iris() classifier = tf.contrib.learn.TensorFlowDNNClassifier( hidden_units=[10, 20, 10], n_classes=3, dropout=0.9) classifier.fit(iris.data, iris.target) score = accuracy_score(iris.target, classifier.predict(iris.data)) self.assertGreater(score, 0.3, "Failed with score = {0}".format(score)) # If the quality is higher - dropout is not working. self.assertLess(score, 0.5, "Failed with score = {0}".format(score))
def testDNNDropout0(self): # Dropout prob == 0. iris = tf.contrib.learn.datasets.load_iris() feature_columns = [tf.contrib.layers.real_valued_column("", dimension=4)] classifier = tf.contrib.learn.TensorFlowDNNClassifier( feature_columns=feature_columns, hidden_units=[10, 20, 10], n_classes=3, dropout=0.0) classifier.fit(iris.data, iris.target) score = accuracy_score(iris.target, classifier.predict(iris.data)) self.assertGreater(score, 0.9, "Failed with score = {0}".format(score))
def test_pandas_series(self): if HAS_PANDAS: random.seed(42) iris = datasets.load_iris() data = pd.DataFrame(iris.data) labels = pd.Series(iris.target) classifier = learn.TensorFlowLinearClassifier(n_classes=3) classifier.fit(data, labels) score = accuracy_score(labels, classifier.predict(data)) self.assertGreater(score, 0.5, "Failed with score = {0}".format(score))
def testDNNDropout0_1(self): # Dropping only a little. tf.set_random_seed(42) iris = tf.contrib.learn.datasets.load_iris() classifier = tf.contrib.learn.TensorFlowDNNClassifier( hidden_units=[10, 20, 10], n_classes=3, dropout=0.1) classifier.fit(iris.data, iris.target) score = accuracy_score(iris.target, classifier.predict(iris.data)) # If the quality is lower - dropout is not working. self.assertGreater(score, 0.9, "Failed with score = {0}".format(score))
def testDNNDropout0_9(self): # Dropping out most of it. iris = datasets.load_iris() classifier = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], n_classes=3, dropout=0.9) classifier.fit(iris.data, iris.target) score = accuracy_score(iris.target, classifier.predict(iris.data)) self.assertGreater(score, 0.3, "Failed with score = {0}".format(score)) self.assertLess(score, 0.4, "Failed with score = {0}".format(score))
def _runIrisAll(self, est): iris = tf.contrib.learn.datasets.load_iris() est.fit(iris.data, iris.target, steps=100) scores = est.evaluate(x=iris.data, y=iris.target, name='eval') predictions = est.predict(x=iris.data) predictions_proba = est.predict_proba(x=iris.data) self.assertEqual(predictions.shape[0], iris.target.shape[0]) self.assertAllEqual(predictions, np.argmax(predictions_proba, axis=1)) other_score = _sklearn.accuracy_score(iris.target, predictions) self.assertAllClose(other_score, scores['accuracy'])
def _runIrisAll(self, est): iris = tf.contrib.learn.datasets.load_iris() est.fit(iris.data, iris.target, steps=100) scores = est.evaluate(x=iris.data, y=iris.target, name='eval') predictions = est.predict(x=iris.data) predictions_proba = est.predict_proba(x=iris.data) self.assertEqual(predictions.shape[0], iris.target.shape[0]) self.assertAllEqual(predictions, np.argmax(predictions_proba, axis=1)) other_score = _sklearn.accuracy_score(iris.target, predictions) self.assertAllClose(other_score, scores['accuracy'])
def testIrisPredictAsIterable(self): iris = tf.contrib.learn.datasets.load_iris() est = tf.contrib.learn.Classifier(model_fn=logistic_model_fn, n_classes=3) est.fit(iris.data, iris.target, steps=100) scores = est.evaluate(x=iris.data, y=iris.target, name='eval') predictions = list(est.predict(x=iris.data, as_iterable=True)) predictions_proba = list(est.predict_proba(x=iris.data, as_iterable=True)) self.assertEqual(len(predictions), iris.target.shape[0]) self.assertAllEqual(predictions, np.argmax(predictions_proba, axis=1)) other_score = _sklearn.accuracy_score(iris.target, predictions) self.assertAllClose(other_score, scores['accuracy'])
def testDNN(self): path = tf.test.get_temp_dir() + '/tmp_saver3' random.seed(42) iris = datasets.load_iris() classifier = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], n_classes=3) classifier.fit(iris.data, iris.target) classifier.save(path) new_classifier = learn.TensorFlowEstimator.restore(path) self.assertEqual(type(new_classifier), type(classifier)) score = accuracy_score(iris.target, new_classifier.predict(iris.data)) self.assertGreater(score, 0.5, "Failed with score = {0}".format(score))
def testIrisPredictAsIterable(self): iris = tf.contrib.learn.datasets.load_iris() est = tf.contrib.learn.Classifier(model_fn=logistic_model_fn, n_classes=3) est.fit(iris.data, iris.target, steps=100) scores = est.evaluate(x=iris.data, y=iris.target, name='eval') predictions = list(est.predict(x=iris.data, as_iterable=True)) predictions_proba = list(est.predict_proba(x=iris.data, as_iterable=True)) self.assertEqual(len(predictions), iris.target.shape[0]) self.assertAllEqual(predictions, np.argmax(predictions_proba, axis=1)) other_score = _sklearn.accuracy_score(iris.target, predictions) self.assertAllClose(other_score, scores['accuracy'])
def test_pandas_series(self): if HAS_PANDAS: import pandas as pd random.seed(42) iris = datasets.load_iris() data = pd.DataFrame(iris.data) labels = pd.Series(iris.target) classifier = learn.TensorFlowLinearClassifier(n_classes=3) classifier.fit(data, labels) score = accuracy_score(labels, classifier.predict(data)) self.assertGreater(score, 0.5, "Failed with score = {0}".format(score))
def testDNNDropout0_1(self): # Dropping only a little. iris = tf.contrib.learn.datasets.load_iris() feature_columns = [tf.contrib.layers.real_valued_column("", dimension=4)] classifier = tf.contrib.learn.TensorFlowDNNClassifier( feature_columns=feature_columns, hidden_units=[10, 20, 10], n_classes=3, dropout=0.1) classifier.fit(iris.data, iris.target) score = accuracy_score(iris.target, classifier.predict(iris.data)) # If the quality is lower - dropout is not working. self.assertGreater(score, 0.9, "Failed with score = {0}".format(score))
def testDNNDropout0_9(self): random.seed(42) # Dropping out most of it. iris = tf.contrib.learn.datasets.load_iris() classifier = tf.contrib.learn.TensorFlowDNNClassifier( hidden_units=[10, 20, 10], n_classes=3, dropout=0.9) classifier.fit(iris.data, iris.target) score = accuracy_score(iris.target, classifier.predict(iris.data)) self.assertGreater(score, 0.3, "Failed with score = {0}".format(score)) # If the quality is higher - dropout is not working. self.assertLess(score, 0.6, "Failed with score = {0}".format(score))
def testIris(self): path = tf.test.get_temp_dir() + '/tmp.saver' random.seed(42) iris = datasets.load_iris() classifier = learn.TensorFlowLinearClassifier(n_classes=3) classifier.fit(iris.data, iris.target) classifier.save(path) new_classifier = learn.TensorFlowEstimator.restore(path) self.assertEqual(type(new_classifier), type(classifier)) score = accuracy_score(iris.target, new_classifier.predict(iris.data)) self.assertGreater(score, 0.5, 'Failed with score = {0}'.format(score))
def testIrisAll(self): iris = tf.contrib.learn.datasets.load_iris() est = tf.contrib.learn.Classifier(model_fn=logistic_model_fn, n_classes=3) est.fit(iris.data, iris.target, steps=100) scores = est.evaluate(x=iris.data, y=iris.target) predictions = est.predict(x=iris.data) predictions_proba = est.predict_proba(x=iris.data) self.assertEqual(predictions.shape[0], iris.target.shape[0]) self.assertAllClose(predictions, np.argmax(predictions_proba, axis=1)) other_score = _sklearn.accuracy_score(iris.target, predictions) self.assertAllClose(other_score, scores['accuracy'])
def testIris(self): path = tf.test.get_temp_dir() + '/tmp.saver' random.seed(42) iris = datasets.load_iris() classifier = learn.TensorFlowLinearClassifier(n_classes=3) classifier.fit(iris.data, iris.target) classifier.save(path) new_classifier = learn.TensorFlowEstimator.restore(path) self.assertEqual(type(new_classifier), type(classifier)) score = accuracy_score(iris.target, new_classifier.predict(iris.data)) self.assertGreater(score, 0.5, 'Failed with score = {0}'.format(score))
def testIrisAll(self): iris = tf.contrib.learn.datasets.load_iris() est = tf.contrib.learn.Classifier(model_fn=logistic_model_fn, n_classes=3) est.fit(iris.data, iris.target, steps=100) scores = est.evaluate(x=iris.data, y=iris.target) predictions = est.predict(x=iris.data) predictions_proba = est.predict_proba(x=iris.data) self.assertEqual(predictions.shape[0], iris.target.shape[0]) self.assertAllClose(predictions, np.argmax(predictions_proba, axis=1)) other_score = _sklearn.accuracy_score(iris.target, predictions) self.assertAllClose(other_score, scores['accuracy'])
def test_pandas_series(self): if HAS_PANDAS: import pandas as pd # pylint: disable=g-import-not-at-top random.seed(42) iris = datasets.load_iris() data = pd.DataFrame(iris.data) labels = pd.Series(iris.target) classifier = learn.TensorFlowLinearClassifier( feature_columns=learn.infer_real_valued_columns_from_input(data), n_classes=3) classifier.fit(data, labels) score = accuracy_score(labels, classifier.predict(data)) self.assertGreater(score, 0.5, "Failed with score = {0}".format(score))