Ejemplo n.º 1
0
 def test_ideal_situation(self):
     """
     Ideal case with accuracy 1
     """
     for n in range(2, 100):
         y_true = np.identity(n)
         y_pred = np.identity(n)
         assert_equals(accuracy.hamming_accuracy(y_true, y_pred), 1)
Ejemplo n.º 2
0
 def test_all_wrong(self):
     """
     Wrong classification with zero accuracy
     """
     for n in range(2, 100):
         y_true = np.ones((n, n))
         y_pred = np.zeros((n, n))
         assert_equals(accuracy.hamming_accuracy(y_true, y_pred), 0)
Ejemplo n.º 3
0
def multi_label_classification():
    """
    Case in point for computing the metrics:
    example on multi-label classification
    """
    # Load modules
    import caffe
    import h5py
    import numpy as np
    from eval.inference import infer_to_h5_fixed_dims

    # Find paths to db/etc
    def path_to(path):
        return base_path + path
    base_path = '/mnt/scratch/pierre/caffe_sandbox_tryouts/'
    fpath_net = path_to('learning_curve/prototxt/net_hdf5_train.prototxt')
    fpath_weights = path_to('learning_curve/snapshots/net_hdf5_snapshot_iter_10000.caffemodel')
    fpath_db = path_to('inference/mnist_%s_train_lmdb')
    fpath_h5 = path_to('inference/db_2.h5')

    # Make n inferences and storing keys in an hdf5 if build_db activated
    build_db = True # change to True when needed
    if build_db:
        net = caffe.Net(fpath_net, fpath_weights, caffe.TRAIN) # caffe.TEST or caffe.TRAIN?
        keys = ["label", "score"]
        n = 1000
        x = infer_to_h5_fixed_dims(net, keys, n, fpath_h5, preserve_batch=False)

    # Define y_true, y_score and y_pred
    f = h5py.File(fpath_h5, "r")
    y_true = f["label"][:]
    y_score = f["score"][:]
    y_pred = np.array([[int(prob>=0) for prob in preds] for preds in y_prob])

    # Accuracy
    from accuracy import hamming_accuracy
    accuracy = hamming_accuracy(y_true, y_pred)
    print "Accuracy =", accuracy

    # ROC curve
    classes = range(10)
    print "ROC curve on scores:"
    plot_roc_curve(y_true, y_score, classes)
    plt.show()

    # Scikit-learn report
    from sklearn.metrics import classification_report
    report = classification_report(y_true, y_pred)
    print report

    # Confusion matrix
    from sklearn.metrics import confusion_matrix
    confusion = confusion_matrix(y_true, y_pred, labels=classes)
    print "Confusion matrix:\n", confusion

    pass
Ejemplo n.º 4
0
 def test_compare_to_sklearn(self):
     """
     Comparison using calculations by hand
     """
     for n in range(2, 100):
         # Multi-label classification notation with binary label indicators
         y_true = np.random.randint(2, size=(n, n))
         y_pred = np.random.randint(2, size=(n, n))
         # Hamming accuracy by hand
         hamming_loss = 0.
         for i in range(n):
             for j in range(n):
                 hamming_loss += (y_true[i, j] != y_pred[i, j])
         # Compare both
         acc_sklearn = 1. - hamming_loss / n ** 2
         acc_hamming = accuracy.hamming_accuracy(y_true, y_pred)
         assert_equals(acc_hamming, acc_sklearn)