def test_lcb2(self):
qs_kwargs = {'model_change': False}
qs = QS.LeastConfidenceDynamicBias(**qs_kwargs)
al = ActiveLearningModel(self.clf, qs, **self.kwargs)
scores, choices = al.run(self.train_X, self.test_X, self.train_y,
self.test_y)
self.assertEqual(scores.shape, (self.ndraws, ))
def run_test(self, qs, eval_metric="accuracy"):
al = ActiveLearningModel(self.clf,
qs,
**self.kwargs,
eval_metric=eval_metric)
scores, choices = al.run(self.data.train_X, self.data.test_X,
self.data.train_y, self.data.test_y)
alc = compute_alc(scores, normalize=True)
L_init = self.data.train_X.shape[0] - scores.shape[0]
L_end = self.data.train_X.shape[0]
# Save the scores and learning curve plots.
outdir = os.path.join(LOGDIR, self.data.name)
if not os.path.exists(outdir):
os.makedirs(outdir)
score_file = os.path.join(outdir, "scores.jsonl")
with open(score_file, 'a') as outF:
outjson = {
"query_strategy": str(qs),
"dataset": self.data.name,
"draws": (L_init, L_end),
"alc": f"{alc:.4f}",
"max_accuracy": f"{np.max(scores):.4f}"
}
json.dump(outjson, outF)
outF.write('\n')
plt_title = f"{str(qs)} : {self.data.name}"
fig_file = os.path.join(outdir, f"{'_'.join(str(qs).split())}.png")
plot_learning_curve(scores,
L_init,
L_end,
eval_metric=eval_metric,
title=plt_title,
saveto=fig_file)
def test_entropy(self):
qs_kwargs = {'model_change': False}
qs = QS.Entropy(**qs_kwargs)
al = ActiveLearningModel(self.clf, qs, **self.kwargs)
scores, choices = al.run(self.train_X, self.test_X, self.train_y,
self.test_y)
self.assertEqual(scores.shape, (self.ndraws, ))
def test_minmax(self):
qs_kwargs = {'metric': 'mahalanobis'}
qs = QS.MinMax(**qs_kwargs)
al = ActiveLearningModel(self.clf, qs, **self.kwargs)
scores, choices = al.run(self.train_X, self.test_X, self.train_y,
self.test_y)
self.assertEqual(scores.shape, (self.ndraws, ))
self.assertEqual(al.query_strategy.distance_metric, 'mahalanobis')
def test_distdiv(self):
qs1 = QS.LeastConfidence(model_change=False)
qs2 = QS.MinMax(metric="euclidean")
qs = QS.DistDivSampler(qs1, qs2, lam=1)
al = ActiveLearningModel(self.clf, qs, **self.kwargs)
scores, choices = al.run(self.train_X, self.test_X, self.train_y,
self.test_y)
self.assertEqual(scores.shape, (self.ndraws, ))
def test_lc_mc(self):
qs_kwargs = {'model_change': True}
qs = QS.LeastConfidence(**qs_kwargs)
al = ActiveLearningModel(self.clf, qs, **self.kwargs)
scores, choices = al.run(self.train_X, self.test_X, self.train_y,
self.test_y)
self.assertEqual(scores.shape, (self.ndraws, ))
self.assertEqual(qs.model_change, True)
def test_limit_draws(self):
qs = QS.Random()
al = ActiveLearningModel(self.clf, qs, **self.kwargs)
scores, choices = al.run(self.train_X,
self.test_X,
self.train_y,
self.test_y,
ndraws=self.ndraws - 1)
self.assertEqual(scores.shape, (self.ndraws - 1, ))
def test_id(self):
qs1 = QS.LeastConfidence(model_change=False)
qs2 = QS.Density()
qs = QS.CombinedSampler(qs1, qs2, beta=3)
al = ActiveLearningModel(self.clf, qs, **self.kwargs)
scores, choices = al.run(self.train_X, self.test_X, self.train_y,
self.test_y)
self.assertEqual(scores.shape, (self.ndraws, ))
self.assertEqual(qs.beta, 3)
def test_margin(self):
qs = QS.Margin()
al = ActiveLearningModel(self.clf, qs, **self.kwargs)
scores, choices = al.run(self.train_X, self.test_X, self.train_y,
self.test_y)
self.assertEqual(scores.shape, (self.ndraws, ))