def compute_score(module_spec, data_spec):
embeddings = compute_embeddings(module_spec=module_spec,
data_spec=data_spec)
distances = utils.compute_distance_matrix_loo(embeddings)
labels = load_labels(data_spec=data_spec)
error_rate = utils.knn_errorrate_loo(distances, labels, k=1)
return np.array(error_rate)
def test_compute_distance_matrix_loo(self):
np.random.seed(seed=self.random_seed)
x_train = np.random.rand(self.train_samples, self.dim)
d = utils.compute_distance_matrix_loo(x_train)
self.assertEqual(d.shape, (self.train_samples, self.train_samples))
for i in range(self.train_samples):
for j in range(self.train_samples):
if i == j:
self.assertEqual(float("inf"), d[i, j])
else:
d_ij = np.linalg.norm(x_train[j, :] - x_train[i, :])**2
self.assertAlmostEqual(d_ij, d[i, j], places=5)
def test_compute_distance_matrix_loo_cosine(self):
if not tf.executing_eagerly():
self.skipTest("Test requires eager mode.")
np.random.seed(seed=self.random_seed)
x_train = np.random.rand(self.train_samples, self.dim)
d = utils.compute_distance_matrix_loo(x_train, measure="cosine")
self.assertEqual(d.shape, (self.train_samples, self.train_samples))
for i in range(self.train_samples):
for j in range(self.train_samples):
if i == j:
self.assertEqual(float("inf"), d[i, j])
else:
d_ij = spdist.cosine(x_train[i, :], x_train[j, :])
self.assertAlmostEqual(d_ij, d[i, j], places=5)
def knn_errorrate_loo(self, k):
x_train = np.random.rand(self.train_samples, self.dim)
d = utils.compute_distance_matrix_loo(x_train)
y_train = np.random.randint(self.classes, size=self.train_samples)
err = utils.knn_errorrate_loo(d, y_train, k=k)
cnt = 0.0
for i in range(self.train_samples):
knn = KNeighborsClassifier(n_neighbors=k)
mask = [True] * self.train_samples
mask[i] = False
knn.fit(x_train[mask], y_train[mask])
y_pred = knn.predict(x_train[i].reshape(-1, self.dim))
if y_pred != y_train[i]:
cnt += 1
self.assertAlmostEqual(err, cnt / self.train_samples, places=5)
def test_knn_errorrate_loo_multik(self):
if not tf.executing_eagerly():
self.skipTest("Test requires eager mode.")
np.random.seed(seed=self.random_seed)
x_train = np.random.rand(self.train_samples, self.dim)
d = utils.compute_distance_matrix_loo(x_train)
y_train = np.random.randint(self.classes, size=self.train_samples)
ks_input = [5, 1, 5, 3]
ks = [5, 3, 1]
vals = []
for val in ks:
err = utils.knn_errorrate_loo(d, y_train, k=val)
vals.append(err)
comp = utils.knn_errorrate_loo(d, y_train, k=ks_input)
self.assertEqual(len(vals), len(comp))
for k, v in enumerate(comp):
self.assertAlmostEqual(v, vals[k], places=5)