def test(epoch, net,trainloader, testloader,criterion, device):
net.eval()
test_loss = 0
correct = 0
total = 0
scores, labels = [], []
with torch.no_grad():
for batch_idx, (inputs, targets) in enumerate(testloader):
inputs, targets = inputs.to(device), targets.to(device)
outputs = net(inputs)
# loss = criterion(outputs, targets)
# test_loss += loss.item()
# _, predicted = outputs.max(1)
scores.append(outputs)
labels.append(targets)
# total += targets.size(0)
# correct += predicted.eq(targets).sum().item()
progress_bar(batch_idx, len(testloader))
# Get the prdict results.
scores = torch.cat(scores,dim=0).cpu().numpy()
labels = torch.cat(labels,dim=0).cpu().numpy()
scores = np.array(scores)[:, np.newaxis, :]
labels = np.array(labels)
# Fit the weibull distribution from training data.
print("Fittting Weibull distribution...")
_, mavs, dists = compute_train_score_and_mavs_and_dists(args.train_class_num, trainloader, device, net)
categories = list(range(0, args.train_class_num))
weibull_model = fit_weibull(mavs, dists, categories, args.weibull_tail, "euclidean")
pred_softmax, pred_softmax_threshold, pred_openmax = [], [], []
for score in scores:
so, ss = openmax(weibull_model, categories, score,
0.5, args.weibull_alpha, "euclidean") # openmax_prob, softmax_prob
pred_softmax.append(np.argmax(ss))
pred_softmax_threshold.append(np.argmax(ss) if np.max(ss) >= args.weibull_threshold else args.train_class_num)
pred_openmax.append(np.argmax(so) if np.max(so) >= args.weibull_threshold else args.train_class_num)
print("Evaluation...")
eval_softmax = Evaluation(pred_softmax, labels)
eval_softmax_threshold = Evaluation(pred_softmax_threshold, labels)
eval_openmax = Evaluation(pred_openmax, labels)
print(f"Softmax accuracy is %.3f"%(eval_softmax.accuracy))
print(f"Softmax-with-threshold accuracy is %.3f"%(eval_softmax_threshold.accuracy))
print(f"Openmax accuracy is %.3f"%(eval_openmax.accuracy))
def test(net, trainloader, testloader):
net.eval()
scores, labels = [], []
with torch.no_grad():
for batch_idx, (inputs, targets) in enumerate(testloader):
inputs, targets = inputs.cuda(), targets.cuda()
outputs = net(inputs)
scores.append(outputs)
labels.append(targets)
# Get the prdict results.
scores = torch.cat(scores, dim=0).cpu().numpy()
labels = torch.cat(labels, dim=0).cpu().numpy()
scores = np.array(scores)[:, np.newaxis, :]
labels = np.array(labels)
# Fit the weibull distribution from training data.
_, mavs, dists = compute_train_score_and_mavs_and_dists(
args.train_class_num, trainloader, net, args.eu_weight)
print("Fittting Weibull distribution...")
categories = list(range(0, args.train_class_num))
weibull_model = fit_weibull(mavs, dists, categories, args.weibull_tail,
args.distance)
pred_softmax, pred_softmax_threshold, pred_openmax = [], [], []
li_softmax, li_openmax = [], []
li_softlogit, li_openlogit = [], []
for score in scores:
so, ss = openmax(weibull_model, categories, score, args.eu_weight,
args.weibull_alpha,
args.distance) # openmax_prob, softmax_prob
li_softlogit.append((ss))
li_openlogit.append((so))
li_softmax.append(np.max(ss))
li_openmax.append(so[args.train_class_num])
pred_softmax.append(np.argmax(ss))
pred_softmax_threshold.append(
np.argmax(ss) if np.max(ss) >= args.weibull_threshold else args.
train_class_num)
pred_openmax.append(
np.argmax(so) if np.max(so) >= args.weibull_threshold else args.
train_class_num)
correct = (labels == pred_openmax)
return np.array(
li_softmax
), li_openmax, li_softlogit, li_openlogit, pred_openmax, correct, labels
def validate(val_loader, train_loader, model):
# switch to evaluate mode
model.eval()
if args.local_rank == 0:
print("start evaluating...")
scores, labels = [], []
for i, data in enumerate(val_loader):
input = data[0]["data"]
target = data[0]["label"].squeeze().cuda().long()
val_loader_len = int(val_loader._size / args.batch_size)
if args.local_rank == 0 and i % 200 == 0:
print(f"evaluating {i}\t/{val_loader_len}...")
target = target.cuda(non_blocking=True)
input_var = Variable(input)
target_var = Variable(target)
# compute output
with torch.no_grad():
_, output = model(input_var)
# print(f"output shape is : {output.shape}, max target is {max(target_var)}, min target is {min(target_var)}")
scores.append(output)
labels.append(target)
# Get the prdict results.
scores = torch.cat(scores, dim=0).cpu().numpy()
labels = torch.cat(labels, dim=0).cpu().numpy()
scores = np.array(scores)[:, np.newaxis, :]
labels = np.array(labels)
# Fit the weibull distribution from training data.
print("Fittting Weibull distribution...")
_, mavs, dists = compute_train_score_and_mavs_and_dists(
args.train_class_num, train_loader, model)
print("Finish fittting Weibull distribution...")
categories = list(range(0, args.train_class_num))
weibull_model = fit_weibull(mavs, dists, categories, args.weibull_tail,
"euclidean")
pred_softmax, pred_softmax_threshold, pred_openmax = [], [], []
score_softmax, score_openmax = [], []
for score in scores:
so, ss = openmax(weibull_model, categories, score, 0.5,
args.weibull_alpha, "euclidean")
# print(f"so {so} \n ss {ss}")# openmax_prob, softmax_prob
pred_softmax.append(np.argmax(ss))
pred_softmax_threshold.append(
np.argmax(ss) if np.max(ss) >= args.weibull_threshold else args.
train_class_num)
pred_openmax.append(
np.argmax(so) if np.max(so) >= args.weibull_threshold else args.
train_class_num)
score_softmax.append(ss)
score_openmax.append(so)
print("Evaluation...")
eval_softmax = Evaluation(pred_softmax, labels)
eval_softmax_threshold = Evaluation(pred_softmax_threshold, labels)
eval_openmax = Evaluation(pred_openmax, labels)
# torch.save(eval_softmax, os.path.join(args.checkpoint, 'eval_softmax.pkl'))
# torch.save(eval_softmax_threshold, os.path.join(args.checkpoint, 'eval_softmax_threshold.pkl'))
# torch.save(eval_openmax, os.path.join(args.checkpoint, 'eval_openmax.pkl'))
softmax_results = torch.Tensor([
eval_softmax.accuracy, eval_softmax.f1_measure, eval_softmax.f1_macro,
eval_softmax.f1_macro_weighted
])
threshold_results = torch.Tensor([
eval_softmax_threshold.accuracy, eval_softmax_threshold.f1_measure,
eval_softmax_threshold.f1_macro,
eval_softmax_threshold.f1_macro_weighted
])
openmax_results = torch.Tensor([
eval_openmax.accuracy, eval_openmax.f1_measure, eval_openmax.f1_macro,
eval_openmax.f1_macro_weighted
])
softmax_results = reduce_tensor(softmax_results.to(input_var.device))
threshold_results = reduce_tensor(threshold_results.to(input_var.device))
openmax_results = reduce_tensor(openmax_results.to(input_var.device))
if args.local_rank == 0:
print(f"the result for three : Acc, F1, macro, w-marco")
print(f"the result for softmax : {softmax_results}")
print(f"the result for threshold : {threshold_results}")
print(f"the result for openmax : {openmax_results}")