def compute_score(module_spec, data_spec): embeddings = compute_embeddings(module_spec=module_spec, data_spec=data_spec) distances = utils.compute_distance_matrix_loo(embeddings) labels = load_labels(data_spec=data_spec) error_rate = utils.knn_errorrate_loo(distances, labels, k=1) return np.array(error_rate)
def test_compute_distance_matrix_loo(self): np.random.seed(seed=self.random_seed) x_train = np.random.rand(self.train_samples, self.dim) d = utils.compute_distance_matrix_loo(x_train) self.assertEqual(d.shape, (self.train_samples, self.train_samples)) for i in range(self.train_samples): for j in range(self.train_samples): if i == j: self.assertEqual(float("inf"), d[i, j]) else: d_ij = np.linalg.norm(x_train[j, :] - x_train[i, :])**2 self.assertAlmostEqual(d_ij, d[i, j], places=5)
def test_compute_distance_matrix_loo_cosine(self): if not tf.executing_eagerly(): self.skipTest("Test requires eager mode.") np.random.seed(seed=self.random_seed) x_train = np.random.rand(self.train_samples, self.dim) d = utils.compute_distance_matrix_loo(x_train, measure="cosine") self.assertEqual(d.shape, (self.train_samples, self.train_samples)) for i in range(self.train_samples): for j in range(self.train_samples): if i == j: self.assertEqual(float("inf"), d[i, j]) else: d_ij = spdist.cosine(x_train[i, :], x_train[j, :]) self.assertAlmostEqual(d_ij, d[i, j], places=5)
def knn_errorrate_loo(self, k): x_train = np.random.rand(self.train_samples, self.dim) d = utils.compute_distance_matrix_loo(x_train) y_train = np.random.randint(self.classes, size=self.train_samples) err = utils.knn_errorrate_loo(d, y_train, k=k) cnt = 0.0 for i in range(self.train_samples): knn = KNeighborsClassifier(n_neighbors=k) mask = [True] * self.train_samples mask[i] = False knn.fit(x_train[mask], y_train[mask]) y_pred = knn.predict(x_train[i].reshape(-1, self.dim)) if y_pred != y_train[i]: cnt += 1 self.assertAlmostEqual(err, cnt / self.train_samples, places=5)
def test_knn_errorrate_loo_multik(self): if not tf.executing_eagerly(): self.skipTest("Test requires eager mode.") np.random.seed(seed=self.random_seed) x_train = np.random.rand(self.train_samples, self.dim) d = utils.compute_distance_matrix_loo(x_train) y_train = np.random.randint(self.classes, size=self.train_samples) ks_input = [5, 1, 5, 3] ks = [5, 3, 1] vals = [] for val in ks: err = utils.knn_errorrate_loo(d, y_train, k=val) vals.append(err) comp = utils.knn_errorrate_loo(d, y_train, k=ks_input) self.assertEqual(len(vals), len(comp)) for k, v in enumerate(comp): self.assertAlmostEqual(v, vals[k], places=5)