def test_symmetry():
"""Test the symmetry of score and loss functions"""
y_true, y_pred, _ = make_prediction(binary=True)
# We shouldn't forget any metrics
assert_equal(set(SYMETRIC_METRICS).union(NOT_SYMETRIC_METRICS,
THRESHOLDED_METRICS),
set(ALL_METRICS))
assert_equal(set(SYMETRIC_METRICS).intersection(set(NOT_SYMETRIC_METRICS)),
set([]))
# Symmetric metric
for name, metric in SYMETRIC_METRICS.items():
assert_equal(metric(y_true, y_pred),
metric(y_pred, y_true),
msg="%s is not symetric" % name)
# Not symmetric metrics
for name, metric in NOT_SYMETRIC_METRICS.items():
assert_true(metric(y_true, y_pred) != metric(y_pred, y_true),
msg="%s seems to be symetric" % name)
# Deprecated metrics
with warnings.catch_warnings(record=True):
# Throw deprecated warning
assert_equal(zero_one(y_true, y_pred),
zero_one(y_pred, y_true))
assert_equal(zero_one(y_true, y_pred, normalize=False),
zero_one(y_pred, y_true, normalize=False))
assert_equal(zero_one_score(y_true, y_pred),
zero_one_score(y_pred, y_true))
def test_symmetry():
"""Test the symmetry of score and loss functions"""
y_true, y_pred, _ = make_prediction(binary=True)
# We shouldn't forget any metrics
assert_equal(set(SYMETRIC_METRICS).union(set(NOT_SYMETRIC_METRICS)),
set(ALL_METRICS))
assert_equal(set(SYMETRIC_METRICS).intersection(set(NOT_SYMETRIC_METRICS)),
set([]))
# Symmetric metric
for name, metric in SYMETRIC_METRICS.items():
assert_equal(metric(y_true, y_pred),
metric(y_pred, y_true),
msg="%s is not symetric" % name)
# Not symmetric metrics
for name, metric in NOT_SYMETRIC_METRICS.items():
assert_true(metric(y_true, y_pred) != metric(y_pred, y_true),
msg="%s seems to be symetric" % name)
# Deprecated metrics
with warnings.catch_warnings(record=True):
# Throw deprecated warning
assert_equal(zero_one(y_true, y_pred),
zero_one(y_pred, y_true))
assert_equal(zero_one(y_true, y_pred, normalize=False),
zero_one(y_pred, y_true, normalize=False))
assert_equal(zero_one_score(y_true, y_pred),
zero_one_score(y_pred, y_true))
def test_losses():
"""Test loss functions"""
y_true, y_pred, _ = make_prediction(binary=True)
n_samples = y_true.shape[0]
n_classes = np.size(unique_labels(y_true))
# Classification
# --------------
with warnings.catch_warnings(True):
# Throw deprecated warning
assert_equal(zero_one(y_true, y_pred), 13)
assert_almost_equal(zero_one(y_true, y_pred, normalize=True),
13 / float(n_samples), 2)
assert_almost_equal(zero_one_loss(y_true, y_pred),
13 / float(n_samples), 2)
assert_equal(zero_one_loss(y_true, y_pred, normalize=False), 13)
assert_almost_equal(zero_one_loss(y_true, y_true), 0.0, 2)
assert_almost_equal(zero_one_loss(y_true, y_true, normalize=False), 0, 2)
assert_almost_equal(hamming_loss(y_true, y_pred),
2 * 13. / (n_samples * n_classes), 2)
assert_equal(accuracy_score(y_true, y_pred),
1 - zero_one_loss(y_true, y_pred))
assert_equal(accuracy_score(y_true, y_pred, normalize=False),
n_samples - zero_one_loss(y_true, y_pred, normalize=False))
with warnings.catch_warnings(True):
# Throw deprecated warning
assert_equal(zero_one_score(y_true, y_pred),
1 - zero_one_loss(y_true, y_pred))
# Regression
# ----------
assert_almost_equal(mean_squared_error(y_true, y_pred),
12.999 / n_samples, 2)
assert_almost_equal(mean_squared_error(y_true, y_true),
0.00, 2)
# mean_absolute_error and mean_squared_error are equal because
# it is a binary problem.
assert_almost_equal(mean_absolute_error(y_true, y_pred),
12.999 / n_samples, 2)
assert_almost_equal(mean_absolute_error(y_true, y_true), 0.00, 2)
assert_almost_equal(explained_variance_score(y_true, y_pred), -0.04, 2)
assert_almost_equal(explained_variance_score(y_true, y_true), 1.00, 2)
assert_equal(explained_variance_score([0, 0, 0], [0, 1, 1]), 0.0)
assert_almost_equal(r2_score(y_true, y_pred), -0.04, 2)
assert_almost_equal(r2_score(y_true, y_true), 1.00, 2)
assert_equal(r2_score([0, 0, 0], [0, 0, 0]), 1.0)
assert_equal(r2_score([0, 0, 0], [0, 1, 1]), 0.0)
def test_losses():
"""Test loss functions"""
y_true, y_pred, _ = make_prediction(binary=True)
n_samples = y_true.shape[0]
n_classes = np.size(unique_labels(y_true))
# Classification
# --------------
with warnings.catch_warnings(record=True):
# Throw deprecated warning
assert_equal(zero_one(y_true, y_pred), 11)
assert_almost_equal(zero_one(y_true, y_pred, normalize=True),
11 / float(n_samples), 2)
assert_almost_equal(zero_one_loss(y_true, y_pred), 11 / float(n_samples),
2)
assert_equal(zero_one_loss(y_true, y_pred, normalize=False), 11)
assert_almost_equal(zero_one_loss(y_true, y_true), 0.0, 2)
assert_almost_equal(zero_one_loss(y_true, y_true, normalize=False), 0, 2)
assert_almost_equal(hamming_loss(y_true, y_pred),
2 * 11. / (n_samples * n_classes), 2)
assert_equal(accuracy_score(y_true, y_pred),
1 - zero_one_loss(y_true, y_pred))
assert_equal(accuracy_score(y_true, y_pred, normalize=False),
n_samples - zero_one_loss(y_true, y_pred, normalize=False))
with warnings.catch_warnings(record=True):
# Throw deprecated warning
assert_equal(zero_one_score(y_true, y_pred),
1 - zero_one_loss(y_true, y_pred))
# Regression
# ----------
assert_almost_equal(mean_squared_error(y_true, y_pred), 10.999 / n_samples,
2)
assert_almost_equal(mean_squared_error(y_true, y_true), 0.00, 2)
# mean_absolute_error and mean_squared_error are equal because
# it is a binary problem.
assert_almost_equal(mean_absolute_error(y_true, y_pred),
10.999 / n_samples, 2)
assert_almost_equal(mean_absolute_error(y_true, y_true), 0.00, 2)
assert_almost_equal(explained_variance_score(y_true, y_pred), 0.16, 2)
assert_almost_equal(explained_variance_score(y_true, y_true), 1.00, 2)
assert_equal(explained_variance_score([0, 0, 0], [0, 1, 1]), 0.0)
assert_almost_equal(r2_score(y_true, y_pred), 0.12, 2)
assert_almost_equal(r2_score(y_true, y_true), 1.00, 2)
assert_equal(r2_score([0, 0, 0], [0, 0, 0]), 1.0)
assert_equal(r2_score([0, 0, 0], [0, 1, 1]), 0.0)
def test_symmetry():
"""Test the symmetry of score and loss functions"""
y_true, y_pred, _ = make_prediction(binary=True)
# symmetric
assert_equal(zero_one(y_true, y_pred), zero_one(y_pred, y_true))
assert_almost_equal(mean_squared_error(y_true, y_pred),
mean_squared_error(y_pred, y_true))
# not symmetric
assert_true(
explained_variance_score(y_true, y_pred) != explained_variance_score(
y_pred, y_true))
assert_true(r2_score(y_true, y_pred) != r2_score(y_pred, y_true))
def test_symmetry():
"""Test the symmetry of score and loss functions"""
y_true, y_pred, _ = make_prediction(binary=True)
# symmetric
assert_equal(zero_one(y_true, y_pred),
zero_one(y_pred, y_true))
assert_almost_equal(mean_squared_error(y_true, y_pred),
mean_squared_error(y_pred, y_true))
# not symmetric
assert_true(explained_variance_score(y_true, y_pred) !=
explained_variance_score(y_pred, y_true))
assert_true(r2_score(y_true, y_pred) !=
r2_score(y_pred, y_true))
def benchmark(clf):
t0 = time()
clf.fit(X_train, y_train)
train_time = time() - t0
t0 = time()
pred = clf.predict(X_test)
test_time = time() - t0
err = metrics.zero_one(y_test, pred) / float(pred.shape[0])
return err, train_time, test_time
def benchmark(clf):
t0 = time()
clf.fit(X_train, y_train)
train_time = time() - t0
t0 = time()
pred = clf.predict(X_test)
test_time = time() - t0
err = metrics.zero_one(y_test, pred) / float(pred.shape[0])
return err, train_time, test_time
def num_diff_w_perms(l1, l2):
"""Compute min_{p in perm} |p(l1)-l2|_1
This loops over all permutations so could be slow for many groups"""
label = list(set(l1))
min_diff = np.Inf
for p in permutations(label):
l1p = [p[label.index(l)] for l in l1]
min_diff = min(min_diff,metrics.zero_one(l1p, l2))
return min_diff
def num_diff_w_perms(l1, l2):
"""Compute min_{p in perm} |p(l1)-l2|_1
This loops over all permutations so could be slow for many groups"""
label = list(set(l1))
min_diff = np.Inf
for p in permutations(label):
l1p = [p[label.index(l)] for l in l1]
min_diff = min(min_diff, metrics.zero_one(l1p, l2))
return min_diff
def test_symmetry():
"""Test the symmetry of score and loss functions"""
y_true, y_pred, _ = make_prediction(binary=True)
# Symmetric metric
for metric in [accuracy_score,
lambda y1, y2: accuracy_score(y1, y2, normalize=False),
zero_one_loss,
lambda y1, y2: zero_one_loss(y1, y2, normalize=False),
hamming_loss,
f1_score,
matthews_corrcoef,
mean_squared_error,
mean_absolute_error]:
assert_equal(metric(y_true, y_pred),
metric(y_pred, y_true),
msg="%s is not symetric" % metric)
# Not symmetric metrics
for metric in [precision_score,
recall_score,
lambda y1, y2: fbeta_score(y1, y2, beta=0.5),
lambda y1, y2: fbeta_score(y1, y2, beta=2),
explained_variance_score,
r2_score]:
assert_true(metric(y_true, y_pred) != metric(y_pred, y_true),
msg="%s seems to be symetric" % metric)
# Deprecated metrics
with warnings.catch_warnings(True):
# Throw deprecated warning
assert_equal(zero_one(y_true, y_pred),
zero_one(y_pred, y_true))
assert_equal(zero_one(y_true, y_pred, normalize=False),
zero_one(y_pred, y_true, normalize=False))
assert_equal(zero_one_score(y_true, y_pred),
zero_one_score(y_pred, y_true))
def test_symmetry():
"""Test the symmetry of score and loss functions"""
y_true, y_pred, _ = make_prediction(binary=True)
# Symmetric metric
for metric in [
accuracy_score,
lambda y1, y2: accuracy_score(y1, y2, normalize=False),
zero_one_loss,
lambda y1, y2: zero_one_loss(y1, y2, normalize=False),
hamming_loss, f1_score, matthews_corrcoef, mean_squared_error,
mean_absolute_error
]:
assert_equal(metric(y_true, y_pred),
metric(y_pred, y_true),
msg="%s is not symetric" % metric)
# Not symmetric metrics
for metric in [
precision_score, recall_score,
lambda y1, y2: fbeta_score(y1, y2, beta=0.5),
lambda y1, y2: fbeta_score(y1, y2, beta=2),
explained_variance_score, r2_score
]:
assert_true(metric(y_true, y_pred) != metric(y_pred, y_true),
msg="%s seems to be symetric" % metric)
# Deprecated metrics
with warnings.catch_warnings(record=True):
# Throw deprecated warning
assert_equal(zero_one(y_true, y_pred), zero_one(y_pred, y_true))
assert_equal(zero_one(y_true, y_pred, normalize=False),
zero_one(y_pred, y_true, normalize=False))
assert_equal(zero_one_score(y_true, y_pred),
zero_one_score(y_pred, y_true))
def test_symmetry():
"""Test the symmetry of score and loss functions"""
y_true, y_pred, _ = make_prediction(binary=True)
# symmetric
assert_equal(accuracy_score(y_true, y_pred),
accuracy_score(y_pred, y_true))
with warnings.catch_warnings(True):
# Throw deprecated warning
assert_equal(zero_one(y_true, y_pred),
zero_one(y_pred, y_true))
assert_almost_equal(zero_one(y_true, y_pred, normalize=False),
zero_one(y_pred, y_true, normalize=False), 2)
assert_equal(zero_one_loss(y_true, y_pred),
zero_one_loss(y_pred, y_true))
assert_equal(zero_one_loss(y_true, y_pred, normalize=False),
zero_one_loss(y_pred, y_true, normalize=False))
with warnings.catch_warnings(True):
# Throw deprecated warning
assert_equal(zero_one_score(y_true, y_pred),
zero_one_score(y_pred, y_true))
assert_almost_equal(mean_squared_error(y_true, y_pred),
mean_squared_error(y_pred, y_true))
assert_almost_equal(mean_absolute_error(y_true, y_pred),
mean_absolute_error(y_pred, y_true))
# not symmetric
assert_true(explained_variance_score(y_true, y_pred) !=
explained_variance_score(y_pred, y_true))
assert_true(r2_score(y_true, y_pred) !=
r2_score(y_pred, y_true))
def test_symmetry():
"""Test the symmetry of score and loss functions"""
y_true, y_pred, _ = make_prediction(binary=True)
# symmetric
assert_equal(accuracy_score(y_true, y_pred),
accuracy_score(y_pred, y_true))
with warnings.catch_warnings(True):
# Throw deprecated warning
assert_equal(zero_one(y_true, y_pred),
zero_one(y_pred, y_true))
assert_almost_equal(zero_one(y_true, y_pred, normalize=False),
zero_one(y_pred, y_true, normalize=False), 2)
assert_equal(zero_one_loss(y_true, y_pred),
zero_one_loss(y_pred, y_true))
assert_equal(zero_one_loss(y_true, y_pred, normalize=False),
zero_one_loss(y_pred, y_true, normalize=False))
with warnings.catch_warnings(True):
# Throw deprecated warning
assert_equal(zero_one_score(y_true, y_pred),
zero_one_score(y_pred, y_true))
assert_almost_equal(mean_squared_error(y_true, y_pred),
mean_squared_error(y_pred, y_true))
assert_almost_equal(mean_absolute_error(y_true, y_pred),
mean_absolute_error(y_pred, y_true))
# not symmetric
assert_true(explained_variance_score(y_true, y_pred) !=
explained_variance_score(y_pred, y_true))
assert_true(r2_score(y_true, y_pred) !=
r2_score(y_pred, y_true))
def test_losses():
"""Test loss functions"""
y_true, y_pred, _ = make_prediction(binary=True)
n = y_true.shape[0]
assert_equal(zero_one(y_true, y_pred), 13)
assert_almost_equal(zero_one(y_true, y_pred, normalize=True),
13 / float(n), 2)
assert_almost_equal(mean_squared_error(y_true, y_pred), 12.999 / n, 2)
assert_almost_equal(mean_squared_error(y_true, y_true), 0.00, 2)
# mean_absolute_error and mean_squared_error are equal because of
# it is a binary problem.
assert_almost_equal(mean_absolute_error(y_true, y_pred), 12.999 / n, 2)
assert_almost_equal(mean_absolute_error(y_true, y_true), 0.00, 2)
assert_almost_equal(explained_variance_score(y_true, y_pred), -0.04, 2)
assert_almost_equal(explained_variance_score(y_true, y_true), 1.00, 2)
assert_equal(explained_variance_score([0, 0, 0], [0, 1, 1]), 0.0)
assert_almost_equal(r2_score(y_true, y_pred), -0.04, 2)
assert_almost_equal(r2_score(y_true, y_true), 1.00, 2)
assert_equal(r2_score([0, 0, 0], [0, 0, 0]), 1.0)
assert_equal(r2_score([0, 0, 0], [0, 1, 1]), 0.0)
def test_losses():
"""Test loss functions"""
y_true, y_pred, _ = make_prediction(binary=True)
n = y_true.shape[0]
assert_equal(zero_one(y_true, y_pred), 13)
assert_almost_equal(mean_squared_error(y_true, y_pred), 12.999 / n, 2)
assert_almost_equal(mean_squared_error(y_true, y_true), 0.00, 2)
assert_almost_equal(explained_variance_score(y_true, y_pred), -0.04, 2)
assert_almost_equal(explained_variance_score(y_true, y_true), 1.00, 2)
assert_equal(explained_variance_score([0, 0, 0], [0, 1, 1]), 0.0)
assert_almost_equal(r2_score(y_true, y_pred), -0.04, 2)
assert_almost_equal(r2_score(y_true, y_true), 1.00, 2)
assert_equal(r2_score([0, 0, 0], [0, 0, 0]), 1.0)
assert_equal(r2_score([0, 0, 0], [0, 1, 1]), 0.0)
def test_losses():
"""Test loss functions"""
y_true, y_pred, _ = make_prediction(binary=True)
n = y_true.shape[0]
assert_equal(zero_one(y_true, y_pred), 13)
assert_almost_equal(mean_squared_error(y_true, y_pred), 12.999 / n, 2)
assert_almost_equal(mean_squared_error(y_true, y_true), 0.00, 2)
assert_almost_equal(explained_variance_score(y_true, y_pred), -0.04, 2)
assert_almost_equal(explained_variance_score(y_true, y_true), 1.00, 2)
assert_equal(explained_variance_score([0, 0, 0], [0, 1, 1]), 0.0)
assert_almost_equal(r2_score(y_true, y_pred), -0.04, 2)
assert_almost_equal(r2_score(y_true, y_true), 1.00, 2)
assert_equal(r2_score([0, 0, 0], [0, 0, 0]), 1.0)
assert_equal(r2_score([0, 0, 0], [0, 1, 1]), 0.0)
def num_diff_w_perms(l1, l2):
label1 = list(set(l1))
label2 = list(set(l2))
label = label1
n = len(l1)
# cost = [[n-np.sum((l1==lab1)==(l2==lab2)) for lab1 in label1] for lab2 in label2]
#
# h = hungarian.Hungarian(cost)
# try:
# h.calculate()
# return h.getTotalPotential()/2.0
# except hungarian.HungarianError:
# print 'Hungary lost this round'
# return
min_diff = np.Inf
for p in permutations(label):
l1p = [p[label.index(l)] for l in l1]
min_diff = min(min_diff,metrics.zero_one(l1p, l2))
return min_diff
