def evaluate_by_dataloader(self, conf, val_dataloader, val_issame, nrof_folds=5): self.model.eval() idx = 0 embeddings = np.zeros( [len(val_dataloader.dataset), conf.embedding_size]) if conf.use_val_left_right_check: lr_predicts = [] with torch.no_grad(): is_grid = False for batch in tqdm(iter(val_dataloader)): if conf.use_val_left_right_check: imgs, im_arrays = batch else: imgs = batch if not is_grid: is_grid = True grid = torchvision.utils.make_grid(imgs[:65]) grid = denormalize_image(grid) self.writer.add_image('val_images', grid, self.step, dataformats='HWC') # tmp_embed, _ = self.model(imgs.to(conf.device)) tmp_embed = self.model(imgs.to(conf.device)) embeddings[idx:idx + len(imgs)] = tmp_embed.cpu() if conf.use_val_left_right_check: # import tensorflow as tf lr_model = conf.lr_model predicted = lr_model.predict_classes( im_arrays.detach().cpu().numpy()) for i in range(0, len(predicted), 2): lr_predicts.append(predicted[i] == predicted[i + 1]) idx += len(imgs) if conf.use_val_left_right_check: tpr, fpr, accuracy, best_thresholds = evaluate( embeddings, val_issame, nrof_folds, pos_thr=conf.pos_thr, neg_thr=conf.neg_thr, lr_predicts=lr_predicts) else: tpr, fpr, accuracy, best_thresholds = evaluate( embeddings, val_issame, nrof_folds, pos_thr=conf.pos_thr, neg_thr=conf.neg_thr) buf = gen_plot(fpr, tpr) roc_curve = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve) return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate(self, conf, carray, issame, nrof_folds=5, tta=False): self.model.eval() idx = 0 embeddings = np.zeros([len(carray), conf.embedding_size]) with torch.no_grad(): while idx + conf.batch_size <= len(carray): batch = torch.tensor(carray[idx:idx + conf.batch_size]) if tta: fliped = hflip_batch(batch) emb_batch = self.model(batch.to(conf.device)) + self.model( fliped.to(conf.device)) embeddings[idx:idx + conf.batch_size] = l2_norm(emb_batch) else: embeddings[idx:idx + conf.batch_size] = self.model( batch.to(conf.device)).cpu() idx += conf.batch_size if idx < len(carray): batch = torch.tensor(carray[idx:]) if tta: fliped = hflip_batch(batch) emb_batch = self.model(batch.to(conf.device)) + self.model( fliped.to(conf.device)) embeddings[idx:] = l2_norm(emb_batch) else: embeddings[idx:] = self.model(batch.to(conf.device)).cpu() tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds) buf = gen_plot(fpr, tpr) roc_curve = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve) return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def learner_evaluate(model, carray, issame, embedding_size=512, batch_size=100, nrof_folds=5, tta=False): idx = 0 embeddings = np.zeros([len(carray), embedding_size]) with torch.no_grad(): while idx + batch_size <= len(carray): batch = torch.tensor(carray[idx:idx + batch_size]) #print(batch_size) if tta: fliped = hflip_batch(batch) emb_batch = model(batch.to(device)) + model(fliped.to(device)) embeddings[idx:idx + batch_size] = l2_norm(emb_batch) else: embeddings[idx:idx + batch_size] = model( batch.to(device)).cpu() idx += batch_size if idx < len(carray): batch = torch.tensor(carray[idx:]) if tta: fliped = hflip_batch(batch) emb_batch = model(batch.to(device)) + model(fliped.to(device)) embeddings[idx:] = l2_norm(emb_batch) else: embeddings[idx:] = model(batch.to(device)).cpu() tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds) best_threshold_ret = recalculate(best_thresholds.mean()) return accuracy.mean(), best_threshold_ret
def evaluate(self, conf, carray, issame, nrof_folds=5, tta=False): self.model.eval() idx = 0 entry_num = carray.size()[0] embeddings = np.zeros([entry_num, conf.embedding_size]) with torch.no_grad(): while idx + conf.batch_size <= entry_num: batch = carray[idx:idx + conf.batch_size] if tta: fliped = hflip_batch(batch) emb_batch = self.model(batch.cuda()) + self.model(fliped.cuda()) embeddings[idx:idx + conf.batch_size] = l2_norm(emb_batch).cpu().detach().numpy() else: embeddings[idx:idx + conf.batch_size] = self.model(batch.cuda()).cpu().detach().numpy() idx += conf.batch_size if idx < entry_num: batch = carray[idx:] if tta: fliped = hflip_batch(batch) emb_batch = self.model(batch.cuda()) + self.model(fliped.cuda()) embeddings[idx:] = l2_norm(emb_batch).cpu().detach().numpy() else: embeddings[idx:] = self.model(batch.cuda()).cpu().detach().numpy() tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds) buf = gen_plot(fpr, tpr) roc_curve = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve) return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate(self, conf, carray, issame, nrof_folds=5, tta=False): embeddings = self.get_embeddings(conf, carray) tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds) buf = gen_plot(fpr, tpr) roc_curve = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve) return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def verify(model, carray, issame, nrof_folds=5): idx = 0 embeddings = np.zeros([len(carray), 320]) with torch.no_grad(): while idx + batch_size <= len(carray): batch = torch.tensor(carray[idx:idx + batch_size]) g = model[0](batch.to(device)) embs = model[2](g) embeddings[idx:idx + batch_size] = embs.cpu() idx += batch_size if idx < len(carray): batch = torch.tensor(carray[idx:]) g = model[0](batch.to(device)) embs = model[2](g) embeddings[idx:] = embs.cpu() tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds) return accuracy.mean(), best_thresholds.mean()
def evaluate_ori(model, path, name, nrof_folds=10, tta=True): from utils import ccrop_batch, hflip_batch from models.model import l2_norm from verifacation import evaluate idx = 0 from data.data_pipe import get_val_pair carray, issame = get_val_pair(path, name) carray = carray[:, ::-1, :, :] # BGR 2 RGB! if use_mxnet: carray *= 0.5 carray += 0.5 carray *= 255. embeddings = np.zeros([len(carray), 512]) if not use_mxnet: with torch.no_grad(): while idx + bs <= len(carray): batch = torch.tensor(carray[idx:idx + bs]) if tta: # batch = ccrop_batch(batch) fliped = hflip_batch(batch) emb_batch = model(batch.cuda()) + model(fliped.cuda()) emb_batch = emb_batch.cpu() embeddings[idx:idx + bs] = l2_norm(emb_batch) else: embeddings[idx:idx + bs] = model(batch.cuda()).cpu() idx += bs if idx < len(carray): batch = torch.tensor(carray[idx:]) if tta: # batch = ccrop_batch(batch) fliped = hflip_batch(batch) emb_batch = model(batch.cuda()) + model(fliped.cuda()) emb_batch = emb_batch.cpu() embeddings[idx:] = l2_norm(emb_batch) else: embeddings[idx:] = model(batch.cuda()).cpu() else: from sklearn.preprocessing import normalize while idx + bs <= len(carray): batch = torch.tensor(carray[idx:idx + bs]) if tta: fliped = hflip_batch(batch) emb_batch = model(batch) + model(fliped) embeddings[idx:idx + bs] = normalize(emb_batch) else: embeddings[idx:idx + bs] = model(batch) idx += bs if idx < len(carray): batch = torch.tensor(carray[idx:]) if tta: fliped = hflip_batch(batch) emb_batch = model(batch) + model(fliped) embeddings[idx:] = normalize(emb_batch) else: embeddings[idx:] = model(batch) tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds) roc_curve_tensor = None # buf = gen_plot(fpr, tpr) # roc_curve = Image.open(buf) # roc_curve_tensor = trans.ToTensor()(roc_curve) return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor