def test_ideal_situation(self): """ Ideal case with accuracy 1 """ for n in range(2, 100): y_true = np.identity(n) y_pred = np.identity(n) assert_equals(accuracy.hamming_accuracy(y_true, y_pred), 1)
def test_all_wrong(self): """ Wrong classification with zero accuracy """ for n in range(2, 100): y_true = np.ones((n, n)) y_pred = np.zeros((n, n)) assert_equals(accuracy.hamming_accuracy(y_true, y_pred), 0)
def multi_label_classification(): """ Case in point for computing the metrics: example on multi-label classification """ # Load modules import caffe import h5py import numpy as np from eval.inference import infer_to_h5_fixed_dims # Find paths to db/etc def path_to(path): return base_path + path base_path = '/mnt/scratch/pierre/caffe_sandbox_tryouts/' fpath_net = path_to('learning_curve/prototxt/net_hdf5_train.prototxt') fpath_weights = path_to('learning_curve/snapshots/net_hdf5_snapshot_iter_10000.caffemodel') fpath_db = path_to('inference/mnist_%s_train_lmdb') fpath_h5 = path_to('inference/db_2.h5') # Make n inferences and storing keys in an hdf5 if build_db activated build_db = True # change to True when needed if build_db: net = caffe.Net(fpath_net, fpath_weights, caffe.TRAIN) # caffe.TEST or caffe.TRAIN? keys = ["label", "score"] n = 1000 x = infer_to_h5_fixed_dims(net, keys, n, fpath_h5, preserve_batch=False) # Define y_true, y_score and y_pred f = h5py.File(fpath_h5, "r") y_true = f["label"][:] y_score = f["score"][:] y_pred = np.array([[int(prob>=0) for prob in preds] for preds in y_prob]) # Accuracy from accuracy import hamming_accuracy accuracy = hamming_accuracy(y_true, y_pred) print "Accuracy =", accuracy # ROC curve classes = range(10) print "ROC curve on scores:" plot_roc_curve(y_true, y_score, classes) plt.show() # Scikit-learn report from sklearn.metrics import classification_report report = classification_report(y_true, y_pred) print report # Confusion matrix from sklearn.metrics import confusion_matrix confusion = confusion_matrix(y_true, y_pred, labels=classes) print "Confusion matrix:\n", confusion pass
def test_compare_to_sklearn(self): """ Comparison using calculations by hand """ for n in range(2, 100): # Multi-label classification notation with binary label indicators y_true = np.random.randint(2, size=(n, n)) y_pred = np.random.randint(2, size=(n, n)) # Hamming accuracy by hand hamming_loss = 0. for i in range(n): for j in range(n): hamming_loss += (y_true[i, j] != y_pred[i, j]) # Compare both acc_sklearn = 1. - hamming_loss / n ** 2 acc_hamming = accuracy.hamming_accuracy(y_true, y_pred) assert_equals(acc_hamming, acc_sklearn)