def test(epoch, net,trainloader, testloader,criterion, device): net.eval() test_loss = 0 correct = 0 total = 0 scores, labels = [], [] with torch.no_grad(): for batch_idx, (inputs, targets) in enumerate(testloader): inputs, targets = inputs.to(device), targets.to(device) outputs = net(inputs) # loss = criterion(outputs, targets) # test_loss += loss.item() # _, predicted = outputs.max(1) scores.append(outputs) labels.append(targets) # total += targets.size(0) # correct += predicted.eq(targets).sum().item() progress_bar(batch_idx, len(testloader)) # Get the prdict results. scores = torch.cat(scores,dim=0).cpu().numpy() labels = torch.cat(labels,dim=0).cpu().numpy() scores = np.array(scores)[:, np.newaxis, :] labels = np.array(labels) # Fit the weibull distribution from training data. print("Fittting Weibull distribution...") _, mavs, dists = compute_train_score_and_mavs_and_dists(args.train_class_num, trainloader, device, net) categories = list(range(0, args.train_class_num)) weibull_model = fit_weibull(mavs, dists, categories, args.weibull_tail, "euclidean") pred_softmax, pred_softmax_threshold, pred_openmax = [], [], [] for score in scores: so, ss = openmax(weibull_model, categories, score, 0.5, args.weibull_alpha, "euclidean") # openmax_prob, softmax_prob pred_softmax.append(np.argmax(ss)) pred_softmax_threshold.append(np.argmax(ss) if np.max(ss) >= args.weibull_threshold else args.train_class_num) pred_openmax.append(np.argmax(so) if np.max(so) >= args.weibull_threshold else args.train_class_num) print("Evaluation...") eval_softmax = Evaluation(pred_softmax, labels) eval_softmax_threshold = Evaluation(pred_softmax_threshold, labels) eval_openmax = Evaluation(pred_openmax, labels) print(f"Softmax accuracy is %.3f"%(eval_softmax.accuracy)) print(f"Softmax-with-threshold accuracy is %.3f"%(eval_softmax_threshold.accuracy)) print(f"Openmax accuracy is %.3f"%(eval_openmax.accuracy))
def test(net, trainloader, testloader): net.eval() scores, labels = [], [] with torch.no_grad(): for batch_idx, (inputs, targets) in enumerate(testloader): inputs, targets = inputs.cuda(), targets.cuda() outputs = net(inputs) scores.append(outputs) labels.append(targets) # Get the prdict results. scores = torch.cat(scores, dim=0).cpu().numpy() labels = torch.cat(labels, dim=0).cpu().numpy() scores = np.array(scores)[:, np.newaxis, :] labels = np.array(labels) # Fit the weibull distribution from training data. _, mavs, dists = compute_train_score_and_mavs_and_dists( args.train_class_num, trainloader, net, args.eu_weight) print("Fittting Weibull distribution...") categories = list(range(0, args.train_class_num)) weibull_model = fit_weibull(mavs, dists, categories, args.weibull_tail, args.distance) pred_softmax, pred_softmax_threshold, pred_openmax = [], [], [] li_softmax, li_openmax = [], [] li_softlogit, li_openlogit = [], [] for score in scores: so, ss = openmax(weibull_model, categories, score, args.eu_weight, args.weibull_alpha, args.distance) # openmax_prob, softmax_prob li_softlogit.append((ss)) li_openlogit.append((so)) li_softmax.append(np.max(ss)) li_openmax.append(so[args.train_class_num]) pred_softmax.append(np.argmax(ss)) pred_softmax_threshold.append( np.argmax(ss) if np.max(ss) >= args.weibull_threshold else args. train_class_num) pred_openmax.append( np.argmax(so) if np.max(so) >= args.weibull_threshold else args. train_class_num) correct = (labels == pred_openmax) return np.array( li_softmax ), li_openmax, li_softlogit, li_openlogit, pred_openmax, correct, labels
def validate(val_loader, train_loader, model): # switch to evaluate mode model.eval() if args.local_rank == 0: print("start evaluating...") scores, labels = [], [] for i, data in enumerate(val_loader): input = data[0]["data"] target = data[0]["label"].squeeze().cuda().long() val_loader_len = int(val_loader._size / args.batch_size) if args.local_rank == 0 and i % 200 == 0: print(f"evaluating {i}\t/{val_loader_len}...") target = target.cuda(non_blocking=True) input_var = Variable(input) target_var = Variable(target) # compute output with torch.no_grad(): _, output = model(input_var) # print(f"output shape is : {output.shape}, max target is {max(target_var)}, min target is {min(target_var)}") scores.append(output) labels.append(target) # Get the prdict results. scores = torch.cat(scores, dim=0).cpu().numpy() labels = torch.cat(labels, dim=0).cpu().numpy() scores = np.array(scores)[:, np.newaxis, :] labels = np.array(labels) # Fit the weibull distribution from training data. print("Fittting Weibull distribution...") _, mavs, dists = compute_train_score_and_mavs_and_dists( args.train_class_num, train_loader, model) print("Finish fittting Weibull distribution...") categories = list(range(0, args.train_class_num)) weibull_model = fit_weibull(mavs, dists, categories, args.weibull_tail, "euclidean") pred_softmax, pred_softmax_threshold, pred_openmax = [], [], [] score_softmax, score_openmax = [], [] for score in scores: so, ss = openmax(weibull_model, categories, score, 0.5, args.weibull_alpha, "euclidean") # print(f"so {so} \n ss {ss}")# openmax_prob, softmax_prob pred_softmax.append(np.argmax(ss)) pred_softmax_threshold.append( np.argmax(ss) if np.max(ss) >= args.weibull_threshold else args. train_class_num) pred_openmax.append( np.argmax(so) if np.max(so) >= args.weibull_threshold else args. train_class_num) score_softmax.append(ss) score_openmax.append(so) print("Evaluation...") eval_softmax = Evaluation(pred_softmax, labels) eval_softmax_threshold = Evaluation(pred_softmax_threshold, labels) eval_openmax = Evaluation(pred_openmax, labels) # torch.save(eval_softmax, os.path.join(args.checkpoint, 'eval_softmax.pkl')) # torch.save(eval_softmax_threshold, os.path.join(args.checkpoint, 'eval_softmax_threshold.pkl')) # torch.save(eval_openmax, os.path.join(args.checkpoint, 'eval_openmax.pkl')) softmax_results = torch.Tensor([ eval_softmax.accuracy, eval_softmax.f1_measure, eval_softmax.f1_macro, eval_softmax.f1_macro_weighted ]) threshold_results = torch.Tensor([ eval_softmax_threshold.accuracy, eval_softmax_threshold.f1_measure, eval_softmax_threshold.f1_macro, eval_softmax_threshold.f1_macro_weighted ]) openmax_results = torch.Tensor([ eval_openmax.accuracy, eval_openmax.f1_measure, eval_openmax.f1_macro, eval_openmax.f1_macro_weighted ]) softmax_results = reduce_tensor(softmax_results.to(input_var.device)) threshold_results = reduce_tensor(threshold_results.to(input_var.device)) openmax_results = reduce_tensor(openmax_results.to(input_var.device)) if args.local_rank == 0: print(f"the result for three : Acc, F1, macro, w-marco") print(f"the result for softmax : {softmax_results}") print(f"the result for threshold : {threshold_results}") print(f"the result for openmax : {openmax_results}")