def test_revaltraining(self):
data_tr = np.array([[0] * 10, [1] * 10] * 20)
misclass, model, labels = self.reval_cls.train(data_tr)
self.assertSequenceEqual([misclass] + [labels.tolist()], [
0.0,
relative_validation._kuhn_munkres_algorithm(
labels, np.array([1, 0] * 20)).tolist()
])
self.assertEqual(type(model), type(self.s))
nclust_range=[2, 7],
s=classifier,
c=clustering,
nrand=100)
metrics, nbest, _ = findbestclust.best_nclust(X_tr, y_tr)
out = findbestclust.evaluate(X_tr, X_ts, nbest)
plot_metrics(metrics,
"Reval performance for synthetic dataset with 10 features")
plt.scatter(X_ts[:, 0], X_ts[:, 1], c=out.test_cllab, cmap='rainbow_r')
plt.title("Predicted labels for 10-feature dataset")
# Compare Reval solution to true labels
print(f'AMI test set = {adjusted_mutual_info_score(y_ts, out.test_cllab)}')
relabeling = _kuhn_munkres_algorithm(y_ts, out.test_cllab)
print(f'ACC test set = {1 - zero_one_loss(y_ts, relabeling)}')
# Increase the number of features from 10 to 20
data2 = make_blobs(1000, 20, 5, cluster_std=5, random_state=42)
# Plot synthetic dataset
plt.scatter(data2[0][:, 0], data2[0][:, 1], c=data2[1], cmap='rainbow_r')
plt.title('True labels for 20-feature dataset')
X_tr, X_ts, y_tr, y_ts = train_test_split(data2[0],
data2[1],
test_size=0.30,
random_state=42,
stratify=data2[1])
random_state=42,
stratify=data[1])
# Apply relative clustering validation with KNN and Hierarchical clustering
classifier = KNeighborsClassifier()
clustering = AgglomerativeClustering()
findbestclust = FindBestClustCV(nfold=10,
nclust_range=[2, 7],
s=classifier,
c=clustering,
nrand=100)
metrics, nbest, _ = findbestclust.best_nclust(X_tr, y_tr)
out = findbestclust.evaluate(X_tr, X_ts, nbest)
perm_lab = _kuhn_munkres_algorithm(y_ts, out.test_cllab)
print(f"Best number of clusters: {nbest}")
print(f"Test set external ACC: " f"{1 - zero_one_loss(y_ts, perm_lab)}")
print(f'AMI = {adjusted_mutual_info_score(y_ts, out.test_cllab)}')
print(
f"Validation set normalized stability (misclassification): {metrics['val'][nbest]}"
)
print(f'Test set ACC = {out.test_acc}')
plot_metrics(metrics, title="Reval performance")
plt.scatter(X_ts[:, 0], X_ts[:, 1], c=y_ts, cmap='rainbow_r')
plt.title("True labels for test set")
plt.scatter(X_ts[:, 0], X_ts[:, 1], c=perm_lab, cmap='rainbow_r')
plt.scatter(mnist_ts[:, 0], mnist_ts[:, 1],
c=label_ts.astype(int), s=0.1, cmap='rainbow_r')
plt.title('UMAP-transformed test subsample of MNIST dataset (N=7,000)')
# Run relative clustering validation
classifier = KNeighborsClassifier()
clustering = AgglomerativeClustering()
findbestclust = FindBestClustCV(nfold=10, nclust_range=[2, 12],
s=classifier, c=clustering, nrand=100)
metrics, nbest, _ = findbestclust.best_nclust(mnist_tr, label_tr)
out = findbestclust.evaluate(mnist_tr, mnist_ts, nbest)
plot_metrics(metrics, "Relative clustering validation performance on MNIST dataset")
perm_lab = _kuhn_munkres_algorithm(label_ts.astype(int), out.test_cllab)
plt.scatter(mnist_ts[:, 0], mnist_ts[:, 1],
c=perm_lab, s=0.1, cmap='rainbow_r')
plt.title("Predicted labels for MNIST test set")
print(f"Best number of clusters: {nbest}")
print(f"Test set external ACC: "
f"{1 - zero_one_loss(label_ts.astype(int), perm_lab)}")
print(f'AMI = {adjusted_mutual_info_score(label_ts.astype(int), perm_lab)}')
print(f"Validation set normalized stability (misclassification): {metrics['val'][nbest]}")
print(f"Result accuracy (on test set): "
f"{out.test_acc}")
def test_khun_munkres_algorithm(self):
true_lab = np.array([1, 1, 1, 0, 0, 0])
pred_lab = np.array([0, 0, 0, 1, 1, 1])
new_lab = relative_validation._kuhn_munkres_algorithm(
true_lab, pred_lab)
self.assertSequenceEqual(new_lab.tolist(), [1, 1, 1, 0, 0, 0])