def evaluate_by_dataloader(self,
conf,
val_dataloader,
val_issame,
nrof_folds=5):
self.model.eval()
idx = 0
embeddings = np.zeros(
[len(val_dataloader.dataset), conf.embedding_size])
if conf.use_val_left_right_check:
lr_predicts = []
with torch.no_grad():
is_grid = False
for batch in tqdm(iter(val_dataloader)):
if conf.use_val_left_right_check:
imgs, im_arrays = batch
else:
imgs = batch
if not is_grid:
is_grid = True
grid = torchvision.utils.make_grid(imgs[:65])
grid = denormalize_image(grid)
self.writer.add_image('val_images',
grid,
self.step,
dataformats='HWC')
# tmp_embed, _ = self.model(imgs.to(conf.device))
tmp_embed = self.model(imgs.to(conf.device))
embeddings[idx:idx + len(imgs)] = tmp_embed.cpu()
if conf.use_val_left_right_check:
# import tensorflow as tf
lr_model = conf.lr_model
predicted = lr_model.predict_classes(
im_arrays.detach().cpu().numpy())
for i in range(0, len(predicted), 2):
lr_predicts.append(predicted[i] == predicted[i + 1])
idx += len(imgs)
if conf.use_val_left_right_check:
tpr, fpr, accuracy, best_thresholds = evaluate(
embeddings,
val_issame,
nrof_folds,
pos_thr=conf.pos_thr,
neg_thr=conf.neg_thr,
lr_predicts=lr_predicts)
else:
tpr, fpr, accuracy, best_thresholds = evaluate(
embeddings,
val_issame,
nrof_folds,
pos_thr=conf.pos_thr,
neg_thr=conf.neg_thr)
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve)
return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate(self, conf, carray, issame, nrof_folds=5, tta=False):
self.model.eval()
idx = 0
embeddings = np.zeros([len(carray), conf.embedding_size])
with torch.no_grad():
while idx + conf.batch_size <= len(carray):
batch = torch.tensor(carray[idx:idx + conf.batch_size])
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.to(conf.device)) + self.model(
fliped.to(conf.device))
embeddings[idx:idx + conf.batch_size] = l2_norm(emb_batch)
else:
embeddings[idx:idx + conf.batch_size] = self.model(
batch.to(conf.device)).cpu()
idx += conf.batch_size
if idx < len(carray):
batch = torch.tensor(carray[idx:])
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.to(conf.device)) + self.model(
fliped.to(conf.device))
embeddings[idx:] = l2_norm(emb_batch)
else:
embeddings[idx:] = self.model(batch.to(conf.device)).cpu()
tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame,
nrof_folds)
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve)
return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def learner_evaluate(model,
carray,
issame,
embedding_size=512,
batch_size=100,
nrof_folds=5,
tta=False):
idx = 0
embeddings = np.zeros([len(carray), embedding_size])
with torch.no_grad():
while idx + batch_size <= len(carray):
batch = torch.tensor(carray[idx:idx + batch_size])
#print(batch_size)
if tta:
fliped = hflip_batch(batch)
emb_batch = model(batch.to(device)) + model(fliped.to(device))
embeddings[idx:idx + batch_size] = l2_norm(emb_batch)
else:
embeddings[idx:idx + batch_size] = model(
batch.to(device)).cpu()
idx += batch_size
if idx < len(carray):
batch = torch.tensor(carray[idx:])
if tta:
fliped = hflip_batch(batch)
emb_batch = model(batch.to(device)) + model(fliped.to(device))
embeddings[idx:] = l2_norm(emb_batch)
else:
embeddings[idx:] = model(batch.to(device)).cpu()
tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame,
nrof_folds)
best_threshold_ret = recalculate(best_thresholds.mean())
return accuracy.mean(), best_threshold_ret
def evaluate(self, conf, carray, issame, nrof_folds=5, tta=False):
self.model.eval()
idx = 0
entry_num = carray.size()[0]
embeddings = np.zeros([entry_num, conf.embedding_size])
with torch.no_grad():
while idx + conf.batch_size <= entry_num:
batch = carray[idx:idx + conf.batch_size]
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.cuda()) + self.model(fliped.cuda())
embeddings[idx:idx + conf.batch_size] = l2_norm(emb_batch).cpu().detach().numpy()
else:
embeddings[idx:idx + conf.batch_size] = self.model(batch.cuda()).cpu().detach().numpy()
idx += conf.batch_size
if idx < entry_num:
batch = carray[idx:]
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.cuda()) + self.model(fliped.cuda())
embeddings[idx:] = l2_norm(emb_batch).cpu().detach().numpy()
else:
embeddings[idx:] = self.model(batch.cuda()).cpu().detach().numpy()
tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds)
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve)
return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate(self, conf, carray, issame, nrof_folds=5, tta=False):
embeddings = self.get_embeddings(conf, carray)
tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame,
nrof_folds)
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve)
return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def verify(model, carray, issame, nrof_folds=5):
idx = 0
embeddings = np.zeros([len(carray), 320])
with torch.no_grad():
while idx + batch_size <= len(carray):
batch = torch.tensor(carray[idx:idx + batch_size])
g = model[0](batch.to(device))
embs = model[2](g)
embeddings[idx:idx + batch_size] = embs.cpu()
idx += batch_size
if idx < len(carray):
batch = torch.tensor(carray[idx:])
g = model[0](batch.to(device))
embs = model[2](g)
embeddings[idx:] = embs.cpu()
tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame,
nrof_folds)
return accuracy.mean(), best_thresholds.mean()
def evaluate_ori(model, path, name, nrof_folds=10, tta=True):
from utils import ccrop_batch, hflip_batch
from models.model import l2_norm
from verifacation import evaluate
idx = 0
from data.data_pipe import get_val_pair
carray, issame = get_val_pair(path, name)
carray = carray[:, ::-1, :, :] # BGR 2 RGB!
if use_mxnet:
carray *= 0.5
carray += 0.5
carray *= 255.
embeddings = np.zeros([len(carray), 512])
if not use_mxnet:
with torch.no_grad():
while idx + bs <= len(carray):
batch = torch.tensor(carray[idx:idx + bs])
if tta:
# batch = ccrop_batch(batch)
fliped = hflip_batch(batch)
emb_batch = model(batch.cuda()) + model(fliped.cuda())
emb_batch = emb_batch.cpu()
embeddings[idx:idx + bs] = l2_norm(emb_batch)
else:
embeddings[idx:idx + bs] = model(batch.cuda()).cpu()
idx += bs
if idx < len(carray):
batch = torch.tensor(carray[idx:])
if tta:
# batch = ccrop_batch(batch)
fliped = hflip_batch(batch)
emb_batch = model(batch.cuda()) + model(fliped.cuda())
emb_batch = emb_batch.cpu()
embeddings[idx:] = l2_norm(emb_batch)
else:
embeddings[idx:] = model(batch.cuda()).cpu()
else:
from sklearn.preprocessing import normalize
while idx + bs <= len(carray):
batch = torch.tensor(carray[idx:idx + bs])
if tta:
fliped = hflip_batch(batch)
emb_batch = model(batch) + model(fliped)
embeddings[idx:idx + bs] = normalize(emb_batch)
else:
embeddings[idx:idx + bs] = model(batch)
idx += bs
if idx < len(carray):
batch = torch.tensor(carray[idx:])
if tta:
fliped = hflip_batch(batch)
emb_batch = model(batch) + model(fliped)
embeddings[idx:] = normalize(emb_batch)
else:
embeddings[idx:] = model(batch)
tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame,
nrof_folds)
roc_curve_tensor = None
# buf = gen_plot(fpr, tpr)
# roc_curve = Image.open(buf)
# roc_curve_tensor = trans.ToTensor()(roc_curve)
return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
