def _evaluate(self):
self.net.eval()
self.head.eval()
lfw_dataset = LFWRaw('datasets/eval/data/lfw',
transform=self.eval_transform)
evalloader = torch.utils.data.DataLoader(
lfw_dataset,
batch_size=self.params['batch_size'],
num_workers=self.params['workers'],
shuffle=False,
pin_memory=False)
features = {}
pbar = tqdm(
evalloader,
mininterval=2.0,
position=0,
leave=True,
# bar_format='{l_bar}{bar:5}{r_bar}{bar:-5b}'
dynamic_ncols=True)
for imgs, keys in pbar:
imgs = imgs.to(self.device)
output = self.net(imgs)
output = output.cpu().numpy()
for i, key in enumerate(keys):
features[str(key)] = output[i]
pbar.set_description("lfw eval", refresh=False)
pbar.close()
preds = np.array([])
trues = np.array([])
pairs_file = list(lfw_dataset.pairs_file)
for pair in pairs_file:
feat1 = features[str(pair[0])]
feat2 = features[str(pair[1])]
ground_truth = int(pair[2])
pred = calculate_cosin_sim(feat1, feat2)
preds = np.append(preds, pred)
trues = np.append(trues, ground_truth)
_, _, _, threshold = roc_auc_threshold(preds, trues)
acc = calculate_acc(preds, trues, threshold)
return acc
def _evaluate_kfold_(self, preds, trues):
if len(preds) > 50000:
return 0., 0. # don't do kfold on gigantic data
kfold = KFold(n_splits=10, shuffle=False)
accuracies = []
best_acc = 0
threshold = 0
for train, test in kfold.split(preds):
_, train_th = calculate_acc_and_thresh(preds[train], trues[train])
test_acc = calculate_acc(preds[test], trues[test], train_th)
accuracies.append(test_acc)
if test_acc > best_acc:
best_acc = test_acc
threshold = train_th
avg_acc = np.mean(accuracies)
return avg_acc, threshold
def _evaluate_roc_(self, preds, trues):
fpr, tpr, auc, threshold = roc_auc_threshold(preds, trues)
acc = calculate_acc(preds, trues, threshold)
return fpr, tpr, acc, auc, threshold