def test_revaltraining(self): data_tr = np.array([[0] * 10, [1] * 10] * 20) misclass, model, labels = self.reval_cls.train(data_tr) self.assertSequenceEqual([misclass] + [labels.tolist()], [ 0.0, relative_validation._kuhn_munkres_algorithm( labels, np.array([1, 0] * 20)).tolist() ]) self.assertEqual(type(model), type(self.s))
nclust_range=[2, 7], s=classifier, c=clustering, nrand=100) metrics, nbest, _ = findbestclust.best_nclust(X_tr, y_tr) out = findbestclust.evaluate(X_tr, X_ts, nbest) plot_metrics(metrics, "Reval performance for synthetic dataset with 10 features") plt.scatter(X_ts[:, 0], X_ts[:, 1], c=out.test_cllab, cmap='rainbow_r') plt.title("Predicted labels for 10-feature dataset") # Compare Reval solution to true labels print(f'AMI test set = {adjusted_mutual_info_score(y_ts, out.test_cllab)}') relabeling = _kuhn_munkres_algorithm(y_ts, out.test_cllab) print(f'ACC test set = {1 - zero_one_loss(y_ts, relabeling)}') # Increase the number of features from 10 to 20 data2 = make_blobs(1000, 20, 5, cluster_std=5, random_state=42) # Plot synthetic dataset plt.scatter(data2[0][:, 0], data2[0][:, 1], c=data2[1], cmap='rainbow_r') plt.title('True labels for 20-feature dataset') X_tr, X_ts, y_tr, y_ts = train_test_split(data2[0], data2[1], test_size=0.30, random_state=42, stratify=data2[1])
random_state=42, stratify=data[1]) # Apply relative clustering validation with KNN and Hierarchical clustering classifier = KNeighborsClassifier() clustering = AgglomerativeClustering() findbestclust = FindBestClustCV(nfold=10, nclust_range=[2, 7], s=classifier, c=clustering, nrand=100) metrics, nbest, _ = findbestclust.best_nclust(X_tr, y_tr) out = findbestclust.evaluate(X_tr, X_ts, nbest) perm_lab = _kuhn_munkres_algorithm(y_ts, out.test_cllab) print(f"Best number of clusters: {nbest}") print(f"Test set external ACC: " f"{1 - zero_one_loss(y_ts, perm_lab)}") print(f'AMI = {adjusted_mutual_info_score(y_ts, out.test_cllab)}') print( f"Validation set normalized stability (misclassification): {metrics['val'][nbest]}" ) print(f'Test set ACC = {out.test_acc}') plot_metrics(metrics, title="Reval performance") plt.scatter(X_ts[:, 0], X_ts[:, 1], c=y_ts, cmap='rainbow_r') plt.title("True labels for test set") plt.scatter(X_ts[:, 0], X_ts[:, 1], c=perm_lab, cmap='rainbow_r')
plt.scatter(mnist_ts[:, 0], mnist_ts[:, 1], c=label_ts.astype(int), s=0.1, cmap='rainbow_r') plt.title('UMAP-transformed test subsample of MNIST dataset (N=7,000)') # Run relative clustering validation classifier = KNeighborsClassifier() clustering = AgglomerativeClustering() findbestclust = FindBestClustCV(nfold=10, nclust_range=[2, 12], s=classifier, c=clustering, nrand=100) metrics, nbest, _ = findbestclust.best_nclust(mnist_tr, label_tr) out = findbestclust.evaluate(mnist_tr, mnist_ts, nbest) plot_metrics(metrics, "Relative clustering validation performance on MNIST dataset") perm_lab = _kuhn_munkres_algorithm(label_ts.astype(int), out.test_cllab) plt.scatter(mnist_ts[:, 0], mnist_ts[:, 1], c=perm_lab, s=0.1, cmap='rainbow_r') plt.title("Predicted labels for MNIST test set") print(f"Best number of clusters: {nbest}") print(f"Test set external ACC: " f"{1 - zero_one_loss(label_ts.astype(int), perm_lab)}") print(f'AMI = {adjusted_mutual_info_score(label_ts.astype(int), perm_lab)}') print(f"Validation set normalized stability (misclassification): {metrics['val'][nbest]}") print(f"Result accuracy (on test set): " f"{out.test_acc}")
def test_khun_munkres_algorithm(self): true_lab = np.array([1, 1, 1, 0, 0, 0]) pred_lab = np.array([0, 0, 0, 1, 1, 1]) new_lab = relative_validation._kuhn_munkres_algorithm( true_lab, pred_lab) self.assertSequenceEqual(new_lab.tolist(), [1, 1, 1, 0, 0, 0])