def evaluate(self, cfg, carray, issame, nrof_folds=5, tta=False):
self.model.eval()
idx = 0
embeddings = np.zeros([len(carray), cfg.MODEL.HEADS.EMBEDDING_DIM])
batch_size = cfg.SOLVER.IMS_PER_BATCH
with torch.no_grad():
while idx + batch_size <= len(carray):
batch = torch.tensor(carray[idx:idx + batch_size])
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.to(self.device)) + self.model(
fliped.to(self.device))
embeddings[idx:idx + batch_size] = l2_norm(emb_batch)
else:
embeddings[idx:idx + batch_size] = self.model(
batch.to(self.device)).cpu()
idx += batch_size
if idx < len(carray):
batch = torch.tensor(carray[idx:])
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.to(self.device)) + self.model(
fliped.to(self.device))
embeddings[idx:] = l2_norm(emb_batch)
else:
embeddings[idx:] = self.model(batch.to(self.device)).cpu()
tpr, fpr, accuracy, best_thresholds = scores(embeddings, issame,
nrof_folds)
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve)
return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate(self, conf, carray, issame, nrof_folds=5, tta=False):
self.model.eval()
idx = 0
embeddings = np.zeros([len(carray), conf.embedding_size])
with torch.no_grad():
while idx + conf.batch_size <= len(carray):
batch = torch.tensor(carray[idx:idx + conf.batch_size])
if tta:
fliped = batch.flip(-1) # I do not test in this case
emb_batch = self.model(batch.to(conf.device)) + self.model(fliped.to(conf.device))
embeddings[idx:idx + conf.batch_size] = l2_norm(emb_batch)
else:
embeddings[idx:idx + conf.batch_size] = self.model(batch.to(conf.device)).cpu()
idx += conf.batch_size
if idx < len(carray):
batch = torch.tensor(carray[idx:])
if tta:
fliped = batch.flip(-1)
emb_batch = self.model(batch.to(conf.device)) + self.model(fliped.to(conf.device))
embeddings[idx:] = l2_norm(emb_batch)
else:
embeddings[idx:] = self.model(batch.to(conf.device)).cpu()
tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds)
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve)
return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate(self, conf, val_loader, issame, nrof_folds=5, tta=False):
self.model.model.eval()
embeddings = torch.zeros(
[len(val_loader.dataset), conf.embedding_size])
with torch.no_grad():
for idx, data in enumerate(
val_loader): # data: batch_size * 3 * 112 * 112
batch_size = data.size(0)
if tta:
fliped = hflip_batch(data)
emb_batch = self.model.model(
data.to(conf.device)) + self.model.model(
fliped.to(conf.device))
embeddings[idx:idx + batch_size] = l2_norm(emb_batch)
else:
embeddings[idx:idx + batch_size] = self.model.model(
data.to(conf.device)) # embeddings: batch_size * 512
tpr, fpr, accuracy, best_thresholds = calculate_roc(
self.thresholds, embeddings, issame, nrof_folds)
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve)
return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate(self, conf, carray, issame, nrof_folds=5, tta=False):
# todo accelerate eval
logging.info('start eval')
self.model.eval()
idx = 0
embeddings = np.zeros([len(carray), conf.embedding_size])
with torch.no_grad():
while idx + conf.batch_size <= len(carray):
batch = torch.tensor(carray[idx:idx + conf.batch_size])
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.to(conf.device)) + self.model(
fliped.to(conf.device))
embeddings[idx:idx + conf.batch_size] = l2_norm(emb_batch)
else:
embeddings[idx:idx + conf.batch_size] = self.model(
batch.to(conf.device)).cpu()
idx += conf.batch_size
if idx < len(carray):
batch = torch.tensor(carray[idx:])
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.to(conf.device)) + self.model(
fliped.to(conf.device))
embeddings[idx:] = l2_norm(emb_batch)
else:
embeddings[idx:] = self.model(batch.to(conf.device)).cpu()
tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame,
nrof_folds)
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve)
self.model.train()
logging.info('eval end')
return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate(self, conf, carray, issame, nrof_folds=5, tta=False):
self.model.eval()
idx = 0
entry_num = carray.size()[0]
embeddings = np.zeros([entry_num, conf.embedding_size])
with torch.no_grad():
while idx + conf.batch_size <= entry_num:
batch = carray[idx:idx + conf.batch_size]
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.cuda()) + self.model(fliped.cuda())
embeddings[idx:idx + conf.batch_size] = l2_norm(emb_batch).cpu().detach().numpy()
else:
embeddings[idx:idx + conf.batch_size] = self.model(batch.cuda()).cpu().detach().numpy()
idx += conf.batch_size
if idx < entry_num:
batch = carray[idx:]
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.cuda()) + self.model(fliped.cuda())
embeddings[idx:] = l2_norm(emb_batch).cpu().detach().numpy()
else:
embeddings[idx:] = self.model(batch.cuda()).cpu().detach().numpy()
tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds)
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve)
return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate(self, conf, model_num, mode='test'):
model = self.models[model_num]
model.eval()
predictions = []
prob = []
labels = []
loader = self.test_loader if mode == 'test' else self.train_loader
with torch.no_grad():
# tqdm(loader, total=len(self.valid_loader), desc='valid', position=1)
for imgs, label in loader:
imgs = imgs.to(conf.device)
self.optimizer.zero_grad()
theta = model(imgs).detach()
val, arg = torch.max(theta, dim=1)
predictions.append(arg.cpu().numpy())
prob.append(theta.cpu().numpy()[:, 1])
labels.append(label.detach().cpu().numpy())
predictions = np.hstack(predictions)
prob = np.hstack(prob)
labels = np.hstack(labels)
res = (predictions == labels)
acc = sum(res) / len(res)
fpr, tpr, _ = roc_curve(res, prob)
buf = gen_plot(fpr, tpr)
roc_curve_im = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve_im)
return acc, roc_curve_tensor
def evaluate(self, conf, carray, issame, nrof_folds = 5, tta = False):
self.model.eval()
idx = 0
embeddings = np.zeros([len(carray), conf.embedding_size])
with torch.no_grad():
while idx + conf.batch_size <= len(carray):
batch = torch.tensor(carray[idx:idx + conf.batch_size])
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.to(conf.device)) + self.model(fliped.to(conf.device))
embeddings[idx:idx + conf.batch_size] = l2_norm(emb_batch)
else:
embeddings[idx:idx + conf.batch_size] = self.model(batch.to(conf.device)).cpu()
idx += conf.batch_size
if idx < len(carray):
batch = torch.tensor(carray[idx:])
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.to(conf.device)) + self.model(fliped.to(conf.device))
embeddings[idx:] = l2_norm(emb_batch)
else:
embeddings[idx:] = self.model(batch.to(conf.device)).cpu()
tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds)
try:
tpr_val = tpr[np.less(fpr,0.0012)&np.greater(fpr,0.0008)][0]
except:
tpr_val = 0
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve)
return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor,tpr_val
def evaluate(self, conf, carray, issame, nrof_folds=5, tta=False):
embeddings = self.get_embeddings(conf, carray)
tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame,
nrof_folds)
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve)
return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate_by_dataloader(self,
conf,
val_dataloader,
val_issame,
nrof_folds=5):
self.model.eval()
idx = 0
embeddings = np.zeros(
[len(val_dataloader.dataset), conf.embedding_size])
if conf.use_val_left_right_check:
lr_predicts = []
with torch.no_grad():
is_grid = False
for batch in tqdm(iter(val_dataloader)):
if conf.use_val_left_right_check:
imgs, im_arrays = batch
else:
imgs = batch
if not is_grid:
is_grid = True
grid = torchvision.utils.make_grid(imgs[:65])
grid = denormalize_image(grid)
self.writer.add_image('val_images',
grid,
self.step,
dataformats='HWC')
# tmp_embed, _ = self.model(imgs.to(conf.device))
tmp_embed = self.model(imgs.to(conf.device))
embeddings[idx:idx + len(imgs)] = tmp_embed.cpu()
if conf.use_val_left_right_check:
# import tensorflow as tf
lr_model = conf.lr_model
predicted = lr_model.predict_classes(
im_arrays.detach().cpu().numpy())
for i in range(0, len(predicted), 2):
lr_predicts.append(predicted[i] == predicted[i + 1])
idx += len(imgs)
if conf.use_val_left_right_check:
tpr, fpr, accuracy, best_thresholds = evaluate(
embeddings,
val_issame,
nrof_folds,
pos_thr=conf.pos_thr,
neg_thr=conf.neg_thr,
lr_predicts=lr_predicts)
else:
tpr, fpr, accuracy, best_thresholds = evaluate(
embeddings,
val_issame,
nrof_folds,
pos_thr=conf.pos_thr,
neg_thr=conf.neg_thr)
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve)
return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def evaluate_alt(self, conf, mode='test'):
label_names = []
if conf.type_only:
label_names = ['calc', 'mass']
elif conf.cancer_only or conf.single_type:
label_names = ['mal', 'ben']
else:
label_names = ['calc_mal', 'mass_mal', 'calc_ben', 'mass_ben']
if not conf.no_bkg:
label_names = ['bkg'] + label_names
# todo experiment
label_names = ['mass_mal', 'mass_ben', 'bkg', 'calc_ben', 'calc_mal']
for i in range(len(self.models)):
self.models[i].eval()
do_mean = -1 if len(self.models) > 1 else 0
ind_iter = range(do_mean, len(self.models))
predictions = dict(zip(ind_iter, [[] for i in ind_iter]))
prob = dict(zip(ind_iter, [[] for i in ind_iter]))
labels = []
loader = self.eval_train if mode == 'train' else self.eval_test
pos = 2 if mode == 'train' else 1
with torch.no_grad():
for imgs, label in tqdm(loader, total=len(loader), desc=mode, position=pos):
imgs = torch.cat(imgs).to(conf.device)
self.optimizer.zero_grad()
thetas = [model(imgs).detach() for model in self.models]
if len(self.models) > 1: thetas = [torch.mean(torch.stack(thetas), 0)] + thetas
for ind, theta in zip(range(do_mean, len(self.models)), thetas):
val, arg = torch.max(theta, dim=1)
predictions[ind].append(arg.cpu().numpy())
prob[ind].append(theta.cpu().numpy())
labels.append(torch.cat(label).detach().cpu().numpy())
labels = np.hstack(labels)
results = []
for ind in range(do_mean, len(self.models)):
curr_predictions = np.hstack(predictions[ind])
curr_prob = np.vstack(prob[ind])
# Compute ROC curve and ROC area for each class
img_d_fig = plot_confusion_matrix(labels, curr_predictions, label_names, tensor_name='dev/cm_' + mode)
res = (curr_predictions == labels)
acc = sum(res) / len(res)
fpr, tpr, _ = roc_curve(np.repeat(res, self.n_classes), curr_prob.ravel())
buf = gen_plot(fpr, tpr)
roc_curve_im = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve_im)
results.append((acc, roc_curve_tensor, img_d_fig))
return results
def evaluate_alt(self, conf, mode='test'):
label_names = self.loader.dataset.label_names
if not conf.no_bkg:
label_names = ['null'] + label_names
for i in range(len(self.models)):
self.models[i].eval()
do_mean = -1 if len(self.models) > 1 else 0
ind_iter = range(do_mean, len(self.models))
predictions = dict(zip(ind_iter, [[] for i in ind_iter]))
prob = dict(zip(ind_iter, [[] for i in ind_iter]))
labels = []
pos = 2 if mode == 'train' else 1
self.eval_loader.dataset.set_mode(mode) # todo check this works :)
with torch.no_grad():
for imgs, label in tqdm(self.eval_loader,
total=len(self.eval_loader),
desc=mode,
position=pos):
imgs = imgs.to(conf.device)
self.optimizer.zero_grad()
thetas = [model(imgs).detach() for model in self.models]
if len(self.models) > 1:
thetas = [torch.mean(torch.stack(thetas), 0)] + thetas
for ind, theta in zip(range(do_mean, len(self.models)),
thetas):
val, arg = torch.max(theta, dim=1)
predictions[ind].append(arg.cpu().numpy())
prob[ind].append(theta.cpu().numpy())
labels.append(label.detach().cpu().numpy())
labels = np.hstack(labels)
results = []
for ind in range(do_mean, len(self.models)):
curr_predictions = np.hstack(predictions[ind])
curr_prob = np.vstack(prob[ind])
# Compute ROC curve and ROC area for each class
img_d_fig = plot_confusion_matrix(labels,
curr_predictions,
label_names,
tensor_name='dev/cm_' + mode)
res = (curr_predictions == labels)
acc = sum(res) / len(res)
fpr, tpr, _ = roc_curve(np.repeat(res, self.n_classes),
curr_prob.ravel())
buf = gen_plot(fpr, tpr)
roc_curve_im = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve_im)
results.append((acc, roc_curve_tensor, img_d_fig))
return results
def evaluate(self, conf, model_num, mode='test'):
if model_num == -1: # means mean model
for i in range(len(self.models)):
self.models[i].eval()
else:
model = self.models[model_num]
model.eval()
# TODO look into this https://github.com/pytorch/pytorch/issues/11476
# batching is unstable... limit to less gpus or use sync
label_names = []
if conf.type_only:
label_names = ['calc', 'mass']
elif conf.cancer_only or conf.single_type:
label_names = ['mal', 'ben']
else:
label_names = ['calc_mal', 'calc_ben', 'mass_mal', 'mass_ben']
if not conf.no_bkg:
label_names = ['bkg'] + label_names
predictions = []
prob = []
labels = []
loader = self.eval_train if mode == 'train' else self.eval_test
pos = 2 if mode == 'train' else 1
model_num_str = model_num if model_num > -1 else 'mean'
with torch.no_grad():
for imgs, label in tqdm(loader, total=len(loader), desc=mode+'_'+str(model_num_str), position=pos):
imgs = torch.cat(imgs).to(conf.device)
self.optimizer.zero_grad()
if model_num == -1: # means mean model
theta = torch.mean(torch.stack([model(imgs).detach() for model in self.models]), 0)
else:
theta = model(imgs).detach()
val, arg = torch.max(theta, dim=1)
predictions.append(arg.cpu().numpy())
prob.append(theta.cpu().numpy())
labels.append(torch.cat(label).detach().cpu().numpy())
predictions = np.hstack(predictions)
prob = np.vstack(prob)
labels = np.hstack(labels)
# Compute ROC curve and ROC area for each class
img_d_fig = plot_confusion_matrix(labels, predictions, label_names, tensor_name='dev/cm_' + mode)
res = (predictions == labels)
acc = sum(res) / len(res)
fpr, tpr, _ = roc_curve(np.repeat(res, self.n_classes), prob.ravel())
buf = gen_plot(fpr, tpr)
roc_curve_im = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve_im)
return acc, roc_curve_tensor, img_d_fig
def evaluate(self, conf, carray, issame, nrof_folds=10, tta=False, n=1):
self.model.eval()
idx = 0
embeddings = np.zeros([len(carray), conf.embedding_size // n])
i = 0
with torch.no_grad():
while idx + conf.batch_size <= len(carray):
batch = torch.tensor(carray[idx:idx + conf.batch_size])
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.to(conf.device)) + self.model(
fliped.to(conf.device))
embeddings[idx:idx + conf.batch_size] = l2_norm(
emb_batch).cpu()[:, i * conf.embedding_size //
n:(i + 1) * conf.embedding_size // n]
else:
embeddings[idx:idx + conf.batch_size] = self.model(
batch.to(conf.device)).cpu()[:,
i * conf.embedding_size //
n:(i + 1) *
conf.embedding_size // n]
idx += conf.batch_size
if idx < len(carray):
batch = torch.tensor(carray[idx:])
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.to(conf.device)) + self.model(
fliped.to(conf.device))
embeddings[idx:] = l2_norm(
emb_batch).cpu()[:, i * conf.embedding_size //
n:(i + 1) * conf.embedding_size // n]
else:
embeddings[idx:] = self.model(batch.to(
conf.device)).cpu()[:, i * conf.embedding_size //
n:(i + 1) * conf.embedding_size //
n]
tpr, fpr, accuracy, best_thresholds, angle_info = evaluate(
embeddings, issame, nrof_folds)
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve)
return accuracy.mean(), best_thresholds.mean(
), roc_curve_tensor, angle_info
def evaluate(self, conf, mode='test'):
for i in range(len(self.models)):
self.models[i].eval()
do_mean = -1 if len(self.models) > 1 else 0
ind_iter = range(do_mean, len(self.models))
predictions = dict(zip(ind_iter, [[] for i in ind_iter]))
prob = dict(zip(ind_iter, [[] for i in ind_iter]))
with torch.no_grad():
self.optimizer.zero_grad()
thetas = [model(self.X_test).detach() for model in self.models]
if len(self.models) > 1:
thetas = [torch.mean(torch.stack(thetas), 0)] + thetas
for ind, theta in zip(range(do_mean, len(self.models)), thetas):
val, arg = torch.max(theta, dim=1)
predictions[ind].append(arg.cpu().numpy())
prob[ind].append(theta.cpu().numpy())
labels = self.y_test
results = []
for ind in range(do_mean, len(self.models)):
curr_predictions = np.hstack(predictions[ind])
curr_prob = np.vstack(prob[ind])
# Compute ROC curve and ROC area for each class
img_d_fig = plot_confusion_matrix(labels,
curr_predictions,
label_names,
tensor_name='dev/cm_' + mode)
res = (curr_predictions == labels)
acc = sum(res) / len(res)
fpr, tpr, _ = roc_curve(np.repeat(res, self.n_classes),
curr_prob.ravel())
buf = gen_plot(fpr, tpr)
roc_curve_im = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve_im)
results.append((acc, roc_curve_tensor, img_d_fig))
return results
def evaluate_custom(self, nrof_folds=5, tta=False):
self.model.eval()
idx = 0
embeddings1 = np.zeros(
(len(self.val_loader.dataset), self.conf.embedding_size))
# print(embeddings1.shape)
embeddings2 = np.zeros(
(len(self.val_loader.dataset), self.conf.embedding_size))
# print(embeddings1.shape)
actual_issame = np.zeros((len(self.val_loader.dataset)))
with torch.no_grad():
samples_tqdm = tqdm(iter(self.val_loader))
for idx, (imgs1, imgs2, issame) in enumerate(samples_tqdm):
embedding1 = self.model(imgs1.to(
self.conf.device)).cpu().numpy()
embedding2 = self.model(imgs2.to(
self.conf.device)).cpu().numpy()
# print(f"[INFO] shape embedding1: {embedding1.shape[0]}")
embeddings1[idx *
self.conf.batch_size:idx * self.conf.batch_size +
embedding1.shape[0], :] = embedding1
embeddings2[idx *
self.conf.batch_size:idx * self.conf.batch_size +
embedding2.shape[0], :] = embedding2
actual_issame[idx *
self.conf.batch_size:idx * self.conf.batch_size +
issame.shape[0]] = issame
# print(f"[INFO] actual_issame : {actual_issame.shape}")
tpr, fpr, accuracy, best_thresholds = evaluate_custom(
embeddings1, embeddings2, actual_issame, nrof_folds)
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve)
# print(f"[INFO] Acc: {accuracy.mean()}");
# print(f"[INFO] TF: {best_thresholds.mean()}");
return accuracy.mean(), best_thresholds.mean(
), roc_curve_tensor, tpr, fpr
def evaluate(self, conf, model_num, mode='test'):
model = self.models[model_num]
model.eval()
# TODO look into this https://github.com/pytorch/pytorch/issues/11476
# batching is unstable... limit to less gpus or use sync
n_classes = 5
predictions = []
prob = []
labels = []
confidance = []
loader = self.eval_train if mode == 'train' else self.eval_test
pos = 2 if mode == 'train' else 1
with torch.no_grad():
for imgs, label in tqdm(loader, total=len(loader), desc=mode+'_'+str(model_num), position=pos):
imgs = torch.cat(imgs).to(conf.device)
self.optimizer.zero_grad()
theta = model(imgs).detach()
val, arg = torch.max(theta, dim=1)
confidance.append(val.cpu().numpy())
predictions.append(arg.cpu().numpy())
prob.append(theta.cpu().numpy())
labels.append(torch.cat(label).detach().cpu().numpy())
predictions = np.hstack(predictions)
confidance = np.hstack(confidance)
prob = np.vstack(prob)
labels = np.hstack(labels)
# Compute ROC curve and ROC area for each class
res = (predictions == labels)
acc = sum(res) / len(res)
fpr, tpr, _ = roc_curve(res, confidance)
buf = gen_plot(fpr, tpr)
roc_curve_im = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve_im)
return acc, roc_curve_tensor
def evaluate(self, conf, model_num, mode='test'):
model = self.models[model_num]
model.eval()
n_classes = 5
predictions = []
prob = []
labels = []
sample_ratio = .25
loader = self.eval_loader.get_loader(mode, sample=sample_ratio)
tot = len(self.eval_loader.train_table) if mode=='train' else len(self.eval_loader.test_table)
tot = tot * sample_ratio // self.eval_loader.n_src_per_batch
with torch.no_grad():
for imgs, label in tqdm(loader, total=tot, desc='valid_'+str(model_num), position=1):
imgs = imgs.to(conf.device)
self.optimizer.zero_grad()
theta = model(imgs).detach()
val, arg = torch.max(theta, dim=1)
predictions.append(arg.cpu().numpy())
prob.append(theta.cpu().numpy())
labels.append(label.detach().cpu().numpy())
predictions = np.hstack(predictions)
prob = np.vstack(prob)
labels = np.hstack(labels)
# Compute ROC curve and ROC area for each class
res = (predictions == labels)
acc = sum(res) / len(res)
fpr, tpr, _ = roc_curve(np.repeat(res, n_classes), prob.ravel())
buf = gen_plot(fpr, tpr)
roc_curve_im = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve_im)
return acc, roc_curve_tensor
def evaluate(self, conf, carray, issame, nrof_folds=10, tta=True):
self.model.eval()
self.growup.eval()
self.discriminator.eval()
idx = 0
embeddings = np.zeros([len(carray), conf.embedding_size])
with torch.no_grad():
while idx + conf.batch_size <= len(carray):
batch = torch.tensor(carray[idx:idx + conf.batch_size])
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(
batch.to(conf.device)).cpu() + self.model(
fliped.to(conf.device)).cpu()
embeddings[idx:idx +
conf.batch_size] = l2_norm(emb_batch).cpu()
else:
embeddings[idx:idx + conf.batch_size] = self.model(
batch.to(conf.device)).cpu()
idx += conf.batch_size
if idx < len(carray):
batch = torch.tensor(carray[idx:])
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(
batch.to(conf.device)).cpu() + self.model(
fliped.to(conf.device)).cpu()
embeddings[idx:] = l2_norm(emb_batch).cpu()
else:
embeddings[idx:] = self.model(batch.to(conf.device)).cpu()
tpr, fpr, accuracy, best_thresholds, dist = evaluate_dist(
embeddings, issame, nrof_folds)
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = transforms.ToTensor()(roc_curve)
return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor, dist
def evaluate(self, conf, mode='test'):
for i in range(len(self.models)):
self.models[i].eval()
do_mean = -1 if len(self.models) > 1 else 0
ind_iter = range(do_mean, len(self.models))
prob = dict(zip(ind_iter, [[] for i in ind_iter]))
rank_prob = dict(zip(ind_iter, [[] for i in ind_iter]))
rank_predictions = dict(zip(ind_iter, [[] for i in ind_iter]))
labels = []
rank_labels = []
pos = 2 if mode == 'train' else 1
self.loader.dataset.train = False
with torch.no_grad():
for imgs, label in tqdm(self.eval_loader,
total=len(self.eval_loader),
desc=mode,
position=pos):
imgs = imgs.to(conf.device)
if conf.rank:
label, rank_label = label
rank_labels.append(rank_label.detach().cpu().numpy())
labels.append(label.detach().cpu().numpy())
bs, n_crops, c, h, w = imgs.size()
imgs = imgs.view(-1, c, h, w).cuda()
self.optimizer.zero_grad()
#thetas = [model(imgs).view(bs, n_crops, -1).mean(1).detach() for model in self.models]
thetas = []
rank_thetas = []
for model_num in range(conf.n_models):
if conf.rank:
theta, rank_theta = self.models[model_num](imgs)
if mode != 'train':
rank_theta = rank_theta.view(bs, n_crops,
-1).mean(1).detach()
rank_thetas.append(rank_theta.detach())
else:
theta = self.models[model_num](imgs)
if mode != 'train':
theta = theta.view(bs, n_crops, -1).mean(1).detach()
thetas.append(theta.detach())
if len(self.models) > 1:
thetas = [torch.mean(torch.stack(thetas), 0)] + thetas
for ind, theta in zip(range(do_mean, len(self.models)),
thetas):
prob[ind].append(theta.cpu().numpy())
if conf.rank:
if len(self.models) > 1:
rank_thetas = [
torch.mean(torch.stack(rank_thetas), 0)
] + rank_thetas
for ind, theta in zip(range(do_mean, len(self.models)),
rank_thetas):
val, arg = torch.max(theta, dim=1)
rank_predictions[ind].append(arg.cpu().numpy())
rank_prob[ind].append(theta.cpu().numpy())
labels = np.vstack(labels)
if conf.rank:
rank_labels = np.hstack(rank_labels)
results = []
for ind in range(do_mean, len(self.models)):
cur_res = []
curr_prob = np.vstack(prob[ind])
AUROCs = []
for i in range(self.n_classes):
AUROCs.append(roc_auc_score(labels[:, i], curr_prob[:, i]))
AUROC_avg = np.array(AUROCs).mean()
img_d_fig = plot_auc_vector(AUROCs, self.ds_test.label_names)
cur_res.append((AUROC_avg, img_d_fig))
if conf.rank:
mask = (rank_labels != -1)
curr_predictions = np.hstack(rank_predictions[ind])[mask]
curr_prob = np.vstack(rank_prob[ind])[mask]
img_d_fig = plot_confusion_matrix(
rank_labels,
curr_predictions,
self.ds_test.rank_label_names,
tensor_name='dev/cm_' + mode)
res = (curr_predictions == rank_labels)
acc = sum(res) / len(res)
fpr, tpr, _ = roc_curve(
np.repeat(res, self.ds_test.n_rank_labels),
curr_prob.ravel())
buf = gen_plot(fpr, tpr)
roc_curve_im = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve_im)
cur_res.append((acc, roc_curve_tensor, img_d_fig))
results.append(cur_res)
return results
