def evaluate(self, cfg, carray, issame, nrof_folds=5, tta=False): self.model.eval() idx = 0 embeddings = np.zeros([len(carray), cfg.MODEL.HEADS.EMBEDDING_DIM]) batch_size = cfg.SOLVER.IMS_PER_BATCH with torch.no_grad(): while idx + batch_size <= len(carray): batch = torch.tensor(carray[idx:idx + batch_size]) if tta: fliped = hflip_batch(batch) emb_batch = self.model(batch.to(self.device)) + self.model( fliped.to(self.device)) embeddings[idx:idx + batch_size] = l2_norm(emb_batch) else: embeddings[idx:idx + batch_size] = self.model( batch.to(self.device)).cpu() idx += batch_size if idx < len(carray): batch = torch.tensor(carray[idx:]) if tta: fliped = hflip_batch(batch) emb_batch = self.model(batch.to(self.device)) + self.model( fliped.to(self.device)) embeddings[idx:] = l2_norm(emb_batch) else: embeddings[idx:] = self.model(batch.to(self.device)).cpu() tpr, fpr, accuracy, best_thresholds = scores(embeddings, issame, nrof_folds) buf = gen_plot(fpr, tpr) roc_curve = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve) return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate(self, conf, carray, issame, nrof_folds=5, tta=False): self.model.eval() idx = 0 embeddings = np.zeros([len(carray), conf.embedding_size]) with torch.no_grad(): while idx + conf.batch_size <= len(carray): batch = torch.tensor(carray[idx:idx + conf.batch_size]) if tta: fliped = batch.flip(-1) # I do not test in this case emb_batch = self.model(batch.to(conf.device)) + self.model(fliped.to(conf.device)) embeddings[idx:idx + conf.batch_size] = l2_norm(emb_batch) else: embeddings[idx:idx + conf.batch_size] = self.model(batch.to(conf.device)).cpu() idx += conf.batch_size if idx < len(carray): batch = torch.tensor(carray[idx:]) if tta: fliped = batch.flip(-1) emb_batch = self.model(batch.to(conf.device)) + self.model(fliped.to(conf.device)) embeddings[idx:] = l2_norm(emb_batch) else: embeddings[idx:] = self.model(batch.to(conf.device)).cpu() tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds) buf = gen_plot(fpr, tpr) roc_curve = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve) return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate(self, conf, val_loader, issame, nrof_folds=5, tta=False): self.model.model.eval() embeddings = torch.zeros( [len(val_loader.dataset), conf.embedding_size]) with torch.no_grad(): for idx, data in enumerate( val_loader): # data: batch_size * 3 * 112 * 112 batch_size = data.size(0) if tta: fliped = hflip_batch(data) emb_batch = self.model.model( data.to(conf.device)) + self.model.model( fliped.to(conf.device)) embeddings[idx:idx + batch_size] = l2_norm(emb_batch) else: embeddings[idx:idx + batch_size] = self.model.model( data.to(conf.device)) # embeddings: batch_size * 512 tpr, fpr, accuracy, best_thresholds = calculate_roc( self.thresholds, embeddings, issame, nrof_folds) buf = gen_plot(fpr, tpr) roc_curve = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve) return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate(self, conf, carray, issame, nrof_folds=5, tta=False): # todo accelerate eval logging.info('start eval') self.model.eval() idx = 0 embeddings = np.zeros([len(carray), conf.embedding_size]) with torch.no_grad(): while idx + conf.batch_size <= len(carray): batch = torch.tensor(carray[idx:idx + conf.batch_size]) if tta: fliped = hflip_batch(batch) emb_batch = self.model(batch.to(conf.device)) + self.model( fliped.to(conf.device)) embeddings[idx:idx + conf.batch_size] = l2_norm(emb_batch) else: embeddings[idx:idx + conf.batch_size] = self.model( batch.to(conf.device)).cpu() idx += conf.batch_size if idx < len(carray): batch = torch.tensor(carray[idx:]) if tta: fliped = hflip_batch(batch) emb_batch = self.model(batch.to(conf.device)) + self.model( fliped.to(conf.device)) embeddings[idx:] = l2_norm(emb_batch) else: embeddings[idx:] = self.model(batch.to(conf.device)).cpu() tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds) buf = gen_plot(fpr, tpr) roc_curve = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve) self.model.train() logging.info('eval end') return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate(self, conf, carray, issame, nrof_folds=5, tta=False): self.model.eval() idx = 0 entry_num = carray.size()[0] embeddings = np.zeros([entry_num, conf.embedding_size]) with torch.no_grad(): while idx + conf.batch_size <= entry_num: batch = carray[idx:idx + conf.batch_size] if tta: fliped = hflip_batch(batch) emb_batch = self.model(batch.cuda()) + self.model(fliped.cuda()) embeddings[idx:idx + conf.batch_size] = l2_norm(emb_batch).cpu().detach().numpy() else: embeddings[idx:idx + conf.batch_size] = self.model(batch.cuda()).cpu().detach().numpy() idx += conf.batch_size if idx < entry_num: batch = carray[idx:] if tta: fliped = hflip_batch(batch) emb_batch = self.model(batch.cuda()) + self.model(fliped.cuda()) embeddings[idx:] = l2_norm(emb_batch).cpu().detach().numpy() else: embeddings[idx:] = self.model(batch.cuda()).cpu().detach().numpy() tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds) buf = gen_plot(fpr, tpr) roc_curve = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve) return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate(self, conf, model_num, mode='test'): model = self.models[model_num] model.eval() predictions = [] prob = [] labels = [] loader = self.test_loader if mode == 'test' else self.train_loader with torch.no_grad(): # tqdm(loader, total=len(self.valid_loader), desc='valid', position=1) for imgs, label in loader: imgs = imgs.to(conf.device) self.optimizer.zero_grad() theta = model(imgs).detach() val, arg = torch.max(theta, dim=1) predictions.append(arg.cpu().numpy()) prob.append(theta.cpu().numpy()[:, 1]) labels.append(label.detach().cpu().numpy()) predictions = np.hstack(predictions) prob = np.hstack(prob) labels = np.hstack(labels) res = (predictions == labels) acc = sum(res) / len(res) fpr, tpr, _ = roc_curve(res, prob) buf = gen_plot(fpr, tpr) roc_curve_im = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve_im) return acc, roc_curve_tensor
def evaluate(self, conf, carray, issame, nrof_folds = 5, tta = False): self.model.eval() idx = 0 embeddings = np.zeros([len(carray), conf.embedding_size]) with torch.no_grad(): while idx + conf.batch_size <= len(carray): batch = torch.tensor(carray[idx:idx + conf.batch_size]) if tta: fliped = hflip_batch(batch) emb_batch = self.model(batch.to(conf.device)) + self.model(fliped.to(conf.device)) embeddings[idx:idx + conf.batch_size] = l2_norm(emb_batch) else: embeddings[idx:idx + conf.batch_size] = self.model(batch.to(conf.device)).cpu() idx += conf.batch_size if idx < len(carray): batch = torch.tensor(carray[idx:]) if tta: fliped = hflip_batch(batch) emb_batch = self.model(batch.to(conf.device)) + self.model(fliped.to(conf.device)) embeddings[idx:] = l2_norm(emb_batch) else: embeddings[idx:] = self.model(batch.to(conf.device)).cpu() tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds) try: tpr_val = tpr[np.less(fpr,0.0012)&np.greater(fpr,0.0008)][0] except: tpr_val = 0 buf = gen_plot(fpr, tpr) roc_curve = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve) return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor,tpr_val
def evaluate(self, conf, carray, issame, nrof_folds=5, tta=False): embeddings = self.get_embeddings(conf, carray) tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds) buf = gen_plot(fpr, tpr) roc_curve = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve) return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate_by_dataloader(self, conf, val_dataloader, val_issame, nrof_folds=5): self.model.eval() idx = 0 embeddings = np.zeros( [len(val_dataloader.dataset), conf.embedding_size]) if conf.use_val_left_right_check: lr_predicts = [] with torch.no_grad(): is_grid = False for batch in tqdm(iter(val_dataloader)): if conf.use_val_left_right_check: imgs, im_arrays = batch else: imgs = batch if not is_grid: is_grid = True grid = torchvision.utils.make_grid(imgs[:65]) grid = denormalize_image(grid) self.writer.add_image('val_images', grid, self.step, dataformats='HWC') # tmp_embed, _ = self.model(imgs.to(conf.device)) tmp_embed = self.model(imgs.to(conf.device)) embeddings[idx:idx + len(imgs)] = tmp_embed.cpu() if conf.use_val_left_right_check: # import tensorflow as tf lr_model = conf.lr_model predicted = lr_model.predict_classes( im_arrays.detach().cpu().numpy()) for i in range(0, len(predicted), 2): lr_predicts.append(predicted[i] == predicted[i + 1]) idx += len(imgs) if conf.use_val_left_right_check: tpr, fpr, accuracy, best_thresholds = evaluate( embeddings, val_issame, nrof_folds, pos_thr=conf.pos_thr, neg_thr=conf.neg_thr, lr_predicts=lr_predicts) else: tpr, fpr, accuracy, best_thresholds = evaluate( embeddings, val_issame, nrof_folds, pos_thr=conf.pos_thr, neg_thr=conf.neg_thr) buf = gen_plot(fpr, tpr) roc_curve = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve) return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate_alt(self, conf, mode='test'): label_names = [] if conf.type_only: label_names = ['calc', 'mass'] elif conf.cancer_only or conf.single_type: label_names = ['mal', 'ben'] else: label_names = ['calc_mal', 'mass_mal', 'calc_ben', 'mass_ben'] if not conf.no_bkg: label_names = ['bkg'] + label_names # todo experiment label_names = ['mass_mal', 'mass_ben', 'bkg', 'calc_ben', 'calc_mal'] for i in range(len(self.models)): self.models[i].eval() do_mean = -1 if len(self.models) > 1 else 0 ind_iter = range(do_mean, len(self.models)) predictions = dict(zip(ind_iter, [[] for i in ind_iter])) prob = dict(zip(ind_iter, [[] for i in ind_iter])) labels = [] loader = self.eval_train if mode == 'train' else self.eval_test pos = 2 if mode == 'train' else 1 with torch.no_grad(): for imgs, label in tqdm(loader, total=len(loader), desc=mode, position=pos): imgs = torch.cat(imgs).to(conf.device) self.optimizer.zero_grad() thetas = [model(imgs).detach() for model in self.models] if len(self.models) > 1: thetas = [torch.mean(torch.stack(thetas), 0)] + thetas for ind, theta in zip(range(do_mean, len(self.models)), thetas): val, arg = torch.max(theta, dim=1) predictions[ind].append(arg.cpu().numpy()) prob[ind].append(theta.cpu().numpy()) labels.append(torch.cat(label).detach().cpu().numpy()) labels = np.hstack(labels) results = [] for ind in range(do_mean, len(self.models)): curr_predictions = np.hstack(predictions[ind]) curr_prob = np.vstack(prob[ind]) # Compute ROC curve and ROC area for each class img_d_fig = plot_confusion_matrix(labels, curr_predictions, label_names, tensor_name='dev/cm_' + mode) res = (curr_predictions == labels) acc = sum(res) / len(res) fpr, tpr, _ = roc_curve(np.repeat(res, self.n_classes), curr_prob.ravel()) buf = gen_plot(fpr, tpr) roc_curve_im = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve_im) results.append((acc, roc_curve_tensor, img_d_fig)) return results
def evaluate_alt(self, conf, mode='test'): label_names = self.loader.dataset.label_names if not conf.no_bkg: label_names = ['null'] + label_names for i in range(len(self.models)): self.models[i].eval() do_mean = -1 if len(self.models) > 1 else 0 ind_iter = range(do_mean, len(self.models)) predictions = dict(zip(ind_iter, [[] for i in ind_iter])) prob = dict(zip(ind_iter, [[] for i in ind_iter])) labels = [] pos = 2 if mode == 'train' else 1 self.eval_loader.dataset.set_mode(mode) # todo check this works :) with torch.no_grad(): for imgs, label in tqdm(self.eval_loader, total=len(self.eval_loader), desc=mode, position=pos): imgs = imgs.to(conf.device) self.optimizer.zero_grad() thetas = [model(imgs).detach() for model in self.models] if len(self.models) > 1: thetas = [torch.mean(torch.stack(thetas), 0)] + thetas for ind, theta in zip(range(do_mean, len(self.models)), thetas): val, arg = torch.max(theta, dim=1) predictions[ind].append(arg.cpu().numpy()) prob[ind].append(theta.cpu().numpy()) labels.append(label.detach().cpu().numpy()) labels = np.hstack(labels) results = [] for ind in range(do_mean, len(self.models)): curr_predictions = np.hstack(predictions[ind]) curr_prob = np.vstack(prob[ind]) # Compute ROC curve and ROC area for each class img_d_fig = plot_confusion_matrix(labels, curr_predictions, label_names, tensor_name='dev/cm_' + mode) res = (curr_predictions == labels) acc = sum(res) / len(res) fpr, tpr, _ = roc_curve(np.repeat(res, self.n_classes), curr_prob.ravel()) buf = gen_plot(fpr, tpr) roc_curve_im = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve_im) results.append((acc, roc_curve_tensor, img_d_fig)) return results
def evaluate(self, conf, model_num, mode='test'): if model_num == -1: # means mean model for i in range(len(self.models)): self.models[i].eval() else: model = self.models[model_num] model.eval() # TODO look into this https://github.com/pytorch/pytorch/issues/11476 # batching is unstable... limit to less gpus or use sync label_names = [] if conf.type_only: label_names = ['calc', 'mass'] elif conf.cancer_only or conf.single_type: label_names = ['mal', 'ben'] else: label_names = ['calc_mal', 'calc_ben', 'mass_mal', 'mass_ben'] if not conf.no_bkg: label_names = ['bkg'] + label_names predictions = [] prob = [] labels = [] loader = self.eval_train if mode == 'train' else self.eval_test pos = 2 if mode == 'train' else 1 model_num_str = model_num if model_num > -1 else 'mean' with torch.no_grad(): for imgs, label in tqdm(loader, total=len(loader), desc=mode+'_'+str(model_num_str), position=pos): imgs = torch.cat(imgs).to(conf.device) self.optimizer.zero_grad() if model_num == -1: # means mean model theta = torch.mean(torch.stack([model(imgs).detach() for model in self.models]), 0) else: theta = model(imgs).detach() val, arg = torch.max(theta, dim=1) predictions.append(arg.cpu().numpy()) prob.append(theta.cpu().numpy()) labels.append(torch.cat(label).detach().cpu().numpy()) predictions = np.hstack(predictions) prob = np.vstack(prob) labels = np.hstack(labels) # Compute ROC curve and ROC area for each class img_d_fig = plot_confusion_matrix(labels, predictions, label_names, tensor_name='dev/cm_' + mode) res = (predictions == labels) acc = sum(res) / len(res) fpr, tpr, _ = roc_curve(np.repeat(res, self.n_classes), prob.ravel()) buf = gen_plot(fpr, tpr) roc_curve_im = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve_im) return acc, roc_curve_tensor, img_d_fig
def evaluate(self, conf, carray, issame, nrof_folds=10, tta=False, n=1): self.model.eval() idx = 0 embeddings = np.zeros([len(carray), conf.embedding_size // n]) i = 0 with torch.no_grad(): while idx + conf.batch_size <= len(carray): batch = torch.tensor(carray[idx:idx + conf.batch_size]) if tta: fliped = hflip_batch(batch) emb_batch = self.model(batch.to(conf.device)) + self.model( fliped.to(conf.device)) embeddings[idx:idx + conf.batch_size] = l2_norm( emb_batch).cpu()[:, i * conf.embedding_size // n:(i + 1) * conf.embedding_size // n] else: embeddings[idx:idx + conf.batch_size] = self.model( batch.to(conf.device)).cpu()[:, i * conf.embedding_size // n:(i + 1) * conf.embedding_size // n] idx += conf.batch_size if idx < len(carray): batch = torch.tensor(carray[idx:]) if tta: fliped = hflip_batch(batch) emb_batch = self.model(batch.to(conf.device)) + self.model( fliped.to(conf.device)) embeddings[idx:] = l2_norm( emb_batch).cpu()[:, i * conf.embedding_size // n:(i + 1) * conf.embedding_size // n] else: embeddings[idx:] = self.model(batch.to( conf.device)).cpu()[:, i * conf.embedding_size // n:(i + 1) * conf.embedding_size // n] tpr, fpr, accuracy, best_thresholds, angle_info = evaluate( embeddings, issame, nrof_folds) buf = gen_plot(fpr, tpr) roc_curve = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve) return accuracy.mean(), best_thresholds.mean( ), roc_curve_tensor, angle_info
def evaluate(self, conf, mode='test'): for i in range(len(self.models)): self.models[i].eval() do_mean = -1 if len(self.models) > 1 else 0 ind_iter = range(do_mean, len(self.models)) predictions = dict(zip(ind_iter, [[] for i in ind_iter])) prob = dict(zip(ind_iter, [[] for i in ind_iter])) with torch.no_grad(): self.optimizer.zero_grad() thetas = [model(self.X_test).detach() for model in self.models] if len(self.models) > 1: thetas = [torch.mean(torch.stack(thetas), 0)] + thetas for ind, theta in zip(range(do_mean, len(self.models)), thetas): val, arg = torch.max(theta, dim=1) predictions[ind].append(arg.cpu().numpy()) prob[ind].append(theta.cpu().numpy()) labels = self.y_test results = [] for ind in range(do_mean, len(self.models)): curr_predictions = np.hstack(predictions[ind]) curr_prob = np.vstack(prob[ind]) # Compute ROC curve and ROC area for each class img_d_fig = plot_confusion_matrix(labels, curr_predictions, label_names, tensor_name='dev/cm_' + mode) res = (curr_predictions == labels) acc = sum(res) / len(res) fpr, tpr, _ = roc_curve(np.repeat(res, self.n_classes), curr_prob.ravel()) buf = gen_plot(fpr, tpr) roc_curve_im = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve_im) results.append((acc, roc_curve_tensor, img_d_fig)) return results
def evaluate_custom(self, nrof_folds=5, tta=False): self.model.eval() idx = 0 embeddings1 = np.zeros( (len(self.val_loader.dataset), self.conf.embedding_size)) # print(embeddings1.shape) embeddings2 = np.zeros( (len(self.val_loader.dataset), self.conf.embedding_size)) # print(embeddings1.shape) actual_issame = np.zeros((len(self.val_loader.dataset))) with torch.no_grad(): samples_tqdm = tqdm(iter(self.val_loader)) for idx, (imgs1, imgs2, issame) in enumerate(samples_tqdm): embedding1 = self.model(imgs1.to( self.conf.device)).cpu().numpy() embedding2 = self.model(imgs2.to( self.conf.device)).cpu().numpy() # print(f"[INFO] shape embedding1: {embedding1.shape[0]}") embeddings1[idx * self.conf.batch_size:idx * self.conf.batch_size + embedding1.shape[0], :] = embedding1 embeddings2[idx * self.conf.batch_size:idx * self.conf.batch_size + embedding2.shape[0], :] = embedding2 actual_issame[idx * self.conf.batch_size:idx * self.conf.batch_size + issame.shape[0]] = issame # print(f"[INFO] actual_issame : {actual_issame.shape}") tpr, fpr, accuracy, best_thresholds = evaluate_custom( embeddings1, embeddings2, actual_issame, nrof_folds) buf = gen_plot(fpr, tpr) roc_curve = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve) # print(f"[INFO] Acc: {accuracy.mean()}"); # print(f"[INFO] TF: {best_thresholds.mean()}"); return accuracy.mean(), best_thresholds.mean( ), roc_curve_tensor, tpr, fpr
def evaluate(self, conf, model_num, mode='test'): model = self.models[model_num] model.eval() # TODO look into this https://github.com/pytorch/pytorch/issues/11476 # batching is unstable... limit to less gpus or use sync n_classes = 5 predictions = [] prob = [] labels = [] confidance = [] loader = self.eval_train if mode == 'train' else self.eval_test pos = 2 if mode == 'train' else 1 with torch.no_grad(): for imgs, label in tqdm(loader, total=len(loader), desc=mode+'_'+str(model_num), position=pos): imgs = torch.cat(imgs).to(conf.device) self.optimizer.zero_grad() theta = model(imgs).detach() val, arg = torch.max(theta, dim=1) confidance.append(val.cpu().numpy()) predictions.append(arg.cpu().numpy()) prob.append(theta.cpu().numpy()) labels.append(torch.cat(label).detach().cpu().numpy()) predictions = np.hstack(predictions) confidance = np.hstack(confidance) prob = np.vstack(prob) labels = np.hstack(labels) # Compute ROC curve and ROC area for each class res = (predictions == labels) acc = sum(res) / len(res) fpr, tpr, _ = roc_curve(res, confidance) buf = gen_plot(fpr, tpr) roc_curve_im = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve_im) return acc, roc_curve_tensor
def evaluate(self, conf, model_num, mode='test'): model = self.models[model_num] model.eval() n_classes = 5 predictions = [] prob = [] labels = [] sample_ratio = .25 loader = self.eval_loader.get_loader(mode, sample=sample_ratio) tot = len(self.eval_loader.train_table) if mode=='train' else len(self.eval_loader.test_table) tot = tot * sample_ratio // self.eval_loader.n_src_per_batch with torch.no_grad(): for imgs, label in tqdm(loader, total=tot, desc='valid_'+str(model_num), position=1): imgs = imgs.to(conf.device) self.optimizer.zero_grad() theta = model(imgs).detach() val, arg = torch.max(theta, dim=1) predictions.append(arg.cpu().numpy()) prob.append(theta.cpu().numpy()) labels.append(label.detach().cpu().numpy()) predictions = np.hstack(predictions) prob = np.vstack(prob) labels = np.hstack(labels) # Compute ROC curve and ROC area for each class res = (predictions == labels) acc = sum(res) / len(res) fpr, tpr, _ = roc_curve(np.repeat(res, n_classes), prob.ravel()) buf = gen_plot(fpr, tpr) roc_curve_im = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve_im) return acc, roc_curve_tensor
def evaluate(self, conf, carray, issame, nrof_folds=10, tta=True): self.model.eval() self.growup.eval() self.discriminator.eval() idx = 0 embeddings = np.zeros([len(carray), conf.embedding_size]) with torch.no_grad(): while idx + conf.batch_size <= len(carray): batch = torch.tensor(carray[idx:idx + conf.batch_size]) if tta: fliped = hflip_batch(batch) emb_batch = self.model( batch.to(conf.device)).cpu() + self.model( fliped.to(conf.device)).cpu() embeddings[idx:idx + conf.batch_size] = l2_norm(emb_batch).cpu() else: embeddings[idx:idx + conf.batch_size] = self.model( batch.to(conf.device)).cpu() idx += conf.batch_size if idx < len(carray): batch = torch.tensor(carray[idx:]) if tta: fliped = hflip_batch(batch) emb_batch = self.model( batch.to(conf.device)).cpu() + self.model( fliped.to(conf.device)).cpu() embeddings[idx:] = l2_norm(emb_batch).cpu() else: embeddings[idx:] = self.model(batch.to(conf.device)).cpu() tpr, fpr, accuracy, best_thresholds, dist = evaluate_dist( embeddings, issame, nrof_folds) buf = gen_plot(fpr, tpr) roc_curve = Image.open(buf) roc_curve_tensor = transforms.ToTensor()(roc_curve) return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor, dist
def evaluate(self, conf, mode='test'): for i in range(len(self.models)): self.models[i].eval() do_mean = -1 if len(self.models) > 1 else 0 ind_iter = range(do_mean, len(self.models)) prob = dict(zip(ind_iter, [[] for i in ind_iter])) rank_prob = dict(zip(ind_iter, [[] for i in ind_iter])) rank_predictions = dict(zip(ind_iter, [[] for i in ind_iter])) labels = [] rank_labels = [] pos = 2 if mode == 'train' else 1 self.loader.dataset.train = False with torch.no_grad(): for imgs, label in tqdm(self.eval_loader, total=len(self.eval_loader), desc=mode, position=pos): imgs = imgs.to(conf.device) if conf.rank: label, rank_label = label rank_labels.append(rank_label.detach().cpu().numpy()) labels.append(label.detach().cpu().numpy()) bs, n_crops, c, h, w = imgs.size() imgs = imgs.view(-1, c, h, w).cuda() self.optimizer.zero_grad() #thetas = [model(imgs).view(bs, n_crops, -1).mean(1).detach() for model in self.models] thetas = [] rank_thetas = [] for model_num in range(conf.n_models): if conf.rank: theta, rank_theta = self.models[model_num](imgs) if mode != 'train': rank_theta = rank_theta.view(bs, n_crops, -1).mean(1).detach() rank_thetas.append(rank_theta.detach()) else: theta = self.models[model_num](imgs) if mode != 'train': theta = theta.view(bs, n_crops, -1).mean(1).detach() thetas.append(theta.detach()) if len(self.models) > 1: thetas = [torch.mean(torch.stack(thetas), 0)] + thetas for ind, theta in zip(range(do_mean, len(self.models)), thetas): prob[ind].append(theta.cpu().numpy()) if conf.rank: if len(self.models) > 1: rank_thetas = [ torch.mean(torch.stack(rank_thetas), 0) ] + rank_thetas for ind, theta in zip(range(do_mean, len(self.models)), rank_thetas): val, arg = torch.max(theta, dim=1) rank_predictions[ind].append(arg.cpu().numpy()) rank_prob[ind].append(theta.cpu().numpy()) labels = np.vstack(labels) if conf.rank: rank_labels = np.hstack(rank_labels) results = [] for ind in range(do_mean, len(self.models)): cur_res = [] curr_prob = np.vstack(prob[ind]) AUROCs = [] for i in range(self.n_classes): AUROCs.append(roc_auc_score(labels[:, i], curr_prob[:, i])) AUROC_avg = np.array(AUROCs).mean() img_d_fig = plot_auc_vector(AUROCs, self.ds_test.label_names) cur_res.append((AUROC_avg, img_d_fig)) if conf.rank: mask = (rank_labels != -1) curr_predictions = np.hstack(rank_predictions[ind])[mask] curr_prob = np.vstack(rank_prob[ind])[mask] img_d_fig = plot_confusion_matrix( rank_labels, curr_predictions, self.ds_test.rank_label_names, tensor_name='dev/cm_' + mode) res = (curr_predictions == rank_labels) acc = sum(res) / len(res) fpr, tpr, _ = roc_curve( np.repeat(res, self.ds_test.n_rank_labels), curr_prob.ravel()) buf = gen_plot(fpr, tpr) roc_curve_im = Image.open(buf) roc_curve_tensor = trans.ToTensor()(roc_curve_im) cur_res.append((acc, roc_curve_tensor, img_d_fig)) results.append(cur_res) return results