def test(epoch, net,trainloader, testloader,criterion, device):
net.eval()
test_loss = 0
correct = 0
total = 0
scores, labels = [], []
with torch.no_grad():
for batch_idx, (inputs, targets) in enumerate(testloader):
inputs, targets = inputs.to(device), targets.to(device)
outputs = net(inputs)
# loss = criterion(outputs, targets)
# test_loss += loss.item()
# _, predicted = outputs.max(1)
scores.append(outputs)
labels.append(targets)
# total += targets.size(0)
# correct += predicted.eq(targets).sum().item()
progress_bar(batch_idx, len(testloader))
# Get the prdict results.
scores = torch.cat(scores,dim=0).cpu().numpy()
labels = torch.cat(labels,dim=0).cpu().numpy()
scores = np.array(scores)[:, np.newaxis, :]
labels = np.array(labels)
# Fit the weibull distribution from training data.
print("Fittting Weibull distribution...")
_, mavs, dists = compute_train_score_and_mavs_and_dists(args.train_class_num, trainloader, device, net)
categories = list(range(0, args.train_class_num))
weibull_model = fit_weibull(mavs, dists, categories, args.weibull_tail, "euclidean")
pred_softmax, pred_softmax_threshold, pred_openmax = [], [], []
for score in scores:
so, ss = openmax(weibull_model, categories, score,
0.5, args.weibull_alpha, "euclidean") # openmax_prob, softmax_prob
pred_softmax.append(np.argmax(ss))
pred_softmax_threshold.append(np.argmax(ss) if np.max(ss) >= args.weibull_threshold else args.train_class_num)
pred_openmax.append(np.argmax(so) if np.max(so) >= args.weibull_threshold else args.train_class_num)
print("Evaluation...")
eval_softmax = Evaluation(pred_softmax, labels)
eval_softmax_threshold = Evaluation(pred_softmax_threshold, labels)
eval_openmax = Evaluation(pred_openmax, labels)
print(f"Softmax accuracy is %.3f"%(eval_softmax.accuracy))
print(f"Softmax-with-threshold accuracy is %.3f"%(eval_softmax_threshold.accuracy))
print(f"Openmax accuracy is %.3f"%(eval_openmax.accuracy))
def stage_evaluate(net,testloader,t_min, t_max, feature='energy'):
Feature_list = []
Predict_list = []
Target_list = []
with torch.no_grad():
for batch_idx, (inputs, targets) in enumerate(testloader):
inputs, targets = inputs.to(device), targets.to(device)
out = net(inputs) # shape [batch,class]
Feature_list.append(out[feature])
Target_list.append(targets)
_, predicted = (out["normweight_fea2cen"]).max(1)
Predict_list.append(predicted)
progress_bar(batch_idx, len(testloader), '| ')
Feature_list = torch.cat(Feature_list, dim=0)
if feature == "normweight_fea2cen":
# return the max propobility.
Feature_list = torch.softmax(Feature_list, dim=1).max(dim=1, keepdim=False)[0]
Target_list = torch.cat(Target_list, dim=0)
Predict_list = torch.cat(Predict_list, dim=0)
best_thres = 0.
best_eval = None
best_f1_measure = 0.
for thres in np.linspace(t_min,t_max,20):
Predict_list[Feature_list<thres] = args.train_class_num
eval = Evaluation(Predict_list.cpu().numpy(),Target_list.cpu().numpy())
if eval.f1_measure > best_f1_measure:
best_f1_measure = eval.f1_measure
best_thres = thres
best_eval = eval
print("===> Finial Evaluation...")
print(f"threshold is: {best_thres}")
print(f"F1: {best_eval.f1_measure}\nmacro-F1: {best_eval.f1_macro}")
def test(net, testloader, device):
net.eval()
scores, labels = [], []
with torch.no_grad():
for batch_idx, (inputs, targets) in enumerate(testloader):
inputs, targets = inputs.to(device), targets.to(device)
_, outputs = net(inputs)
scores.append(outputs)
labels.append(targets)
progress_bar(batch_idx, len(testloader))
# Get the prdict results.
scores = torch.cat(scores, dim=0)
scores = scores.softmax(dim=1)
scores = scores.cpu().numpy()
labels = torch.cat(labels, dim=0).cpu().numpy()
pred = []
for score in scores:
pred.append(np.argmax(score) if np.max(score) >= args.threshold else args.train_class_num)
print("Evaluation...")
eval = Evaluation(pred, labels, scores)
torch.save(eval, os.path.join(args.checkpoint, 'eval.pkl'))
print(f"Center-Loss accuracy is %.3f" % (eval.accuracy))
print(f"Center-Loss F1 is %.3f" % (eval.f1_measure))
print(f"Center-Loss f1_macro is %.3f" % (eval.f1_macro))
print(f"Center-Loss f1_macro_weighted is %.3f" % (eval.f1_macro_weighted))
print(f"Center-Loss area_under_roc is %.3f" % (eval.area_under_roc))
def test(net, testloader, device):
net.eval()
scores, labels = [], []
with torch.no_grad():
for batch_idx, (inputs, targets) in enumerate(testloader):
inputs, targets = inputs.to(device), targets.to(device)
_, outputs = net(inputs)
scores.append(outputs)
labels.append(targets)
progress_bar(batch_idx, len(testloader))
# Get the prdict results.
scores = torch.cat(scores,dim=0)
scores = scores.softmax(dim=1)
scores = scores.cpu().numpy()
labels = torch.cat(labels,dim=0).cpu().numpy()
pred = []
for score in scores:
pred.append(np.argmax(score) if np.max(score) >= args.threshold else args.train_class_num)
print("Evaluation...")
eval = Evaluation(pred, labels)
print(f"Center-Loss accuracy is %.3f"%(eval.accuracy))
def test_with_hist(net, dataloader, device, intervals=20, name="stage1_test"):
energy_list = [] # energy value
Target_list = []
Predict_list = []
with torch.no_grad():
for batch_idx, (inputs, targets) in enumerate(dataloader):
inputs, targets = inputs.to(device), targets.to(device)
out = net(inputs) # shape [batch,class]
energy_list.append(out["energy"])
Target_list.append(targets)
_, predicted = (out['normweight_fea2cen']).max(1)
Predict_list.append(predicted)
progress_bar(batch_idx, len(dataloader), "|||")
energy_list = torch.cat(energy_list, dim=0)
Target_list = torch.cat(Target_list, dim=0)
Predict_list = torch.cat(Predict_list, dim=0)
unknown_label = Target_list.max()
unknown_energy_list = energy_list[Target_list == unknown_label]
known_energy_list = energy_list[Target_list != unknown_label]
plot_listhist([known_energy_list, unknown_energy_list],
args,
labels=["known", "unknown"],
name=name + "_energy")
best_F1 = 0
best_thres = 0
best_eval = None
# for these unbounded metric, we explore more intervals by *5 to achieve a relatively fair comparison.
expand_factor = 5
openmetric_list = energy_list
threshold_min = openmetric_list.min().item()
threshold_max = openmetric_list.max().item()
for thres in np.linspace(threshold_min, threshold_max,
expand_factor * intervals):
Predict_list[openmetric_list < thres] = args.train_class_num
eval = Evaluation(Predict_list.cpu().numpy(),
Target_list.cpu().numpy())
if eval.f1_measure > best_F1:
best_F1 = eval.f1_measure
best_thres = thres
best_eval = eval
print(f"The energy range is [{threshold_min}, {threshold_max}] ")
print(f"Best F1 is: {best_F1} [in best threshold: {best_thres} ]")
return {
"best_F1": best_F1,
"best_thres": best_thres,
"best_eval": best_eval
}
def stage2_test(net, testloader, device):
correct = 0
total = 0
pred_list, target_list, score_list = [], [], []
with torch.no_grad():
for batch_idx, (inputs, targets) in enumerate(testloader):
inputs, targets = inputs.to(device), targets.to(device)
out = net(inputs)
threshold = out["thresholds"] # [class]
sim_fea2cen= out["sim_fea2cen"] # [batch,class]
dis_fea2cen= out["dis_fea2cen"] # [batch,class]
b,c = dis_fea2cen.shape
dis_predicted, predicted = (dis_fea2cen).min(1) #[b]
compare_threshold = 1*threshold[predicted]
predicted[(dis_predicted-compare_threshold)>0] = c
pred_list.extend(predicted.tolist())
target_list.extend(targets.tolist())
score_list.extend(sim_fea2cen.tolist())
total += targets.size(0)
correct += predicted.eq(targets).sum().item()
progress_bar(batch_idx, len(testloader), '| Acc: %.3f%% (%d/%d)'
% (100. * correct / total, correct, total))
eval_result = Evaluation(pred_list, target_list, score_list)
torch.save(eval_result, os.path.join(args.checkpoint, 'eval.pkl'))
print(f"accuracy is %.3f" % (eval_result.accuracy))
print(f"F1 is %.3f" % (eval_result.f1_measure))
print(f"f1_macro is %.3f" % (eval_result.f1_macro))
print(f"f1_macro_weighted is %.3f" % (eval_result.f1_macro_weighted))
print(f"area_under_roc is %.3f" % (eval_result.area_under_roc))
def detail_evalate(sim_list, dis_list, target_list, threshold):
predicts = []
labels = []
c = sim_list.shape[1]
# print(c)
for i in range(target_list.shape[0]):
sim, dis, target = sim_list[i], dis_list[i], target_list[i]
sim_value, sim_ind = sim.max(0)
dis_value, dis_ind = dis.min(0)
if sim_value < args.sim_threshold or dis_value > args.amplier * threshold[
dis_ind]:
# if sim_value < args.sim_threshold:
predict = c
else:
predict = sim_ind.item()
predicts.append(predict)
labels.append(target.item())
# print(f"sim_value{sim_value}\t predict{predict}\t target{target}\t dis_value{dis_value}\t")
eval_result = Evaluation(predicts, labels, sim_list.tolist())
print(f"accuracy is %.3f" % (eval_result.accuracy))
print(f"F1 is %.3f" % (eval_result.f1_measure))
print(f"f1_macro is %.3f" % (eval_result.f1_macro))
print(f"f1_macro_weighted is %.3f" % (eval_result.f1_macro_weighted))
print(f"area_under_roc is %.3f" % (eval_result.area_under_roc))
def test(net, testloader, criterion, device, intervals=20):
normfea_list = [] # extracted feature norm
cosine_list = [] # extracted cosine similarity
energy_list = [] # energy value
embed_fea_list = []
dotproduct_fea2cen_list = []
normweight_fea2cen_list = []
Target_list = []
Predict_list = []
with torch.no_grad():
for batch_idx, (inputs, targets) in enumerate(testloader):
inputs, targets = inputs.to(device), targets.to(device)
out = net(inputs) # shape [batch,class]
# Test cannot calculate loss beacuse of class number dismatch.
# loss_dict = criterion(out, targets)
# loss = loss_dict['loss']
# test_loss += loss.item()
normfea_list.append(out["norm_fea"])
cosine_list.append(out["cosine_fea2cen"])
energy_list.append(out["energy"])
embed_fea_list.append(out["embed_fea"])
dotproduct_fea2cen_list.append(out["dotproduct_fea2cen"])
normweight_fea2cen_list.append(out["normweight_fea2cen"])
Target_list.append(targets)
# "CenterLoss", "SoftmaxLoss", "ArcFaceLoss", "NormFaceLoss", "PSoftmaxLoss"
if args.loss in [
"CenterLoss",
"SoftmaxLoss",
]:
_, predicted = (out['dotproduct_fea2cen']).max(1)
elif args.loss in [
"ArcFaceLoss",
"NormFaceLoss",
]:
_, predicted = (out['cosine_fea2cen']).max(1)
else: # PSoftmaxLoss
_, predicted = (out['normweight_fea2cen']).max(1)
Predict_list.append(predicted)
progress_bar(batch_idx, len(testloader), "|||")
normfea_list = torch.cat(normfea_list, dim=0)
cosine_list = torch.cat(cosine_list, dim=0)
energy_list = torch.cat(energy_list, dim=0)
embed_fea_list = torch.cat(embed_fea_list, dim=0)
dotproduct_fea2cen_list = torch.cat(dotproduct_fea2cen_list, dim=0)
normweight_fea2cen_list = torch.cat(normweight_fea2cen_list, dim=0)
Target_list = torch.cat(Target_list, dim=0)
Predict_list = torch.cat(Predict_list, dim=0)
# "CenterLoss", "SoftmaxLoss", "ArcFaceLoss", "NormFaceLoss", "PSoftmaxLoss"
# "possibility", "distance",'norm','energy','cosine'
best_F1_possibility = 0
best_F1_norm = 0
best_F1_energy = 0
# for these unbounded metric, we explore more intervals by *5 to achieve a relatively fair comparison.
expand_factor = 5
Predict_list_possibility = Predict_list.clone().detach()
Predict_list_norm = Predict_list.clone().detach()
Predict_list_energy = Predict_list.clone().detach()
# possibility
if args.loss in ["SoftmaxLoss"]:
openmetric_possibility = dotproduct_fea2cen_list
if args.loss in ["PSoftmaxLoss"]:
openmetric_possibility = normweight_fea2cen_list
openmetric_possibility, _ = torch.softmax(openmetric_possibility,
dim=1).max(dim=1)
for thres in np.linspace(0.0, 1.0, intervals):
Predict_list_possibility[
openmetric_possibility < thres] = args.train_class_num
eval = Evaluation(Predict_list_possibility.cpu().numpy(),
Target_list.cpu().numpy())
if eval.f1_measure > best_F1_possibility:
best_F1_possibility = eval.f1_measure
# norm
openmetric_norm = normfea_list.squeeze(dim=1)
threshold_min_norm = openmetric_norm.min().item()
threshold_max_norm = openmetric_norm.max().item()
for thres in np.linspace(threshold_min_norm, threshold_max_norm,
expand_factor * intervals):
Predict_list_norm[openmetric_norm < thres] = args.train_class_num
eval = Evaluation(Predict_list_norm.cpu().numpy(),
Target_list.cpu().numpy())
if eval.f1_measure > best_F1_norm:
best_F1_norm = eval.f1_measure
# energy
openmetric_energy = energy_list
threshold_min_energy = openmetric_energy.min().item()
threshold_max_energy = openmetric_energy.max().item()
for thres in np.linspace(threshold_min_energy, threshold_max_energy,
expand_factor * intervals):
Predict_list_energy[openmetric_energy < thres] = args.train_class_num
eval = Evaluation(Predict_list_energy.cpu().numpy(),
Target_list.cpu().numpy())
if eval.f1_measure > best_F1_energy:
best_F1_energy = eval.f1_measure
print(
f"Best Possibility F1 is: {best_F1_possibility} | Norm F1 is :{best_F1_norm} | Energy F1 is: {best_F1_energy}"
)
return {
"best_F1_possibility": best_F1_possibility,
"best_F1_norm": best_F1_norm,
"best_F1_energy": best_F1_energy
}
def validate(val_loader, model,intervals=20):
# switch to evaluate mode
model.eval()
if args.local_rank == 0:
print("start evaluating...")
normfea_list = [] # extracted feature norm
energy_list = [] # energy value
normweight_fea2cen_list = []
Target_list = []
Predict_list = []
for i, data in enumerate(val_loader):
input = data[0]["data"]
target = data[0]["label"].squeeze().cuda().long()
val_loader_len = int(val_loader._size / args.batch_size)
if args.local_rank == 0 and i%200 ==0:
print(f"evaluating {i}\t/{val_loader_len}...")
target = target.cuda(non_blocking=True)
input_var = Variable(input)
target_var = Variable(target)
# compute output
with torch.no_grad():
out = model(input_var)
normfea_list.append(out["norm_fea"])
energy_list.append(out["energy"])
normweight_fea2cen_list.append(out["normweight_fea2cen"])
Target_list.append(target_var)
_, predicted = (out['normweight_fea2cen']).max(1)
Predict_list.append(predicted)
normfea_list = torch.cat(normfea_list, dim=0)
energy_list = torch.cat(energy_list, dim=0)
normweight_fea2cen_list = torch.cat(normweight_fea2cen_list, dim=0)
Target_list = torch.cat(Target_list, dim=0)
Predict_list = torch.cat(Predict_list, dim=0)
best_F1_possibility = 0
best_possibility_thr = 0
best_F1_norm = 0
best_norm_thr = 0
best_F1_energy = 0
best_energy_thr = 0
# for these unbounded metric, we explore more intervals by *5 to achieve a relatively fair comparison.
expand_factor = 5
Predict_list_possibility = Predict_list.clone().detach()
Predict_list_norm = Predict_list.clone().detach()
Predict_list_energy = Predict_list.clone().detach()
# possibility
openmetric_possibility = normweight_fea2cen_list
openmetric_possibility, _ = torch.softmax(openmetric_possibility, dim=1).max(dim=1)
default_thr_possibility = 0.5
Predict_list_possibility[openmetric_possibility < default_thr_possibility] = args.train_class_num
eval_possibility = Evaluation(Predict_list_possibility.cpu().numpy(), Target_list.cpu().numpy())
# norm
openmetric_norm = normfea_list.squeeze(dim=1)
threshold_min_norm = openmetric_norm.min().item()
threshold_max_norm = openmetric_norm.max().item()
default_thr_norm = threshold_min_norm*1.6
Predict_list_norm[openmetric_norm < default_thr_norm] = args.train_class_num
eval_norm = Evaluation(Predict_list_norm.cpu().numpy(), Target_list.cpu().numpy())
# energy
openmetric_energy = energy_list
threshold_min_energy = openmetric_energy.min().item()
threshold_max_energy = openmetric_energy.max().item()
default_thr_energy = threshold_min_energy*1.6
Predict_list_energy[openmetric_energy < default_thr_energy] = args.train_class_num
eval_energy = Evaluation(Predict_list_energy.cpu().numpy(), Target_list.cpu().numpy())
if args.local_rank == 0:
print(f"possibility: f1:{eval_possibility.f1_measure} | macro-F1: {eval_possibility.f1_macro_weighted}")
print(f"Norm: f1:{eval_norm.f1_measure} | macro-F1: {eval_norm.f1_macro_weighted}")
print(f"Energy: f1:{eval_energy.f1_measure} | macro-F1: {eval_energy.f1_macro_weighted}")
return {
"best_F1_possibility": best_F1_possibility,
"best_F1_norm": best_F1_norm,
"best_F1_energy": best_F1_energy
}
def validate(val_loader, train_loader, model):
# switch to evaluate mode
model.eval()
if args.local_rank == 0:
print("start evaluating...")
scores, labels = [], []
for i, data in enumerate(val_loader):
input = data[0]["data"]
target = data[0]["label"].squeeze().cuda().long()
val_loader_len = int(val_loader._size / args.batch_size)
if args.local_rank == 0 and i % 200 == 0:
print(f"evaluating {i}\t/{val_loader_len}...")
target = target.cuda(non_blocking=True)
input_var = Variable(input)
target_var = Variable(target)
# compute output
with torch.no_grad():
_, output = model(input_var)
# print(f"output shape is : {output.shape}, max target is {max(target_var)}, min target is {min(target_var)}")
scores.append(output)
labels.append(target)
# Get the prdict results.
scores = torch.cat(scores, dim=0).cpu().numpy()
labels = torch.cat(labels, dim=0).cpu().numpy()
scores = np.array(scores)[:, np.newaxis, :]
labels = np.array(labels)
# Fit the weibull distribution from training data.
print("Fittting Weibull distribution...")
_, mavs, dists = compute_train_score_and_mavs_and_dists(
args.train_class_num, train_loader, model)
print("Finish fittting Weibull distribution...")
categories = list(range(0, args.train_class_num))
weibull_model = fit_weibull(mavs, dists, categories, args.weibull_tail,
"euclidean")
pred_softmax, pred_softmax_threshold, pred_openmax = [], [], []
score_softmax, score_openmax = [], []
for score in scores:
so, ss = openmax(weibull_model, categories, score, 0.5,
args.weibull_alpha, "euclidean")
# print(f"so {so} \n ss {ss}")# openmax_prob, softmax_prob
pred_softmax.append(np.argmax(ss))
pred_softmax_threshold.append(
np.argmax(ss) if np.max(ss) >= args.weibull_threshold else args.
train_class_num)
pred_openmax.append(
np.argmax(so) if np.max(so) >= args.weibull_threshold else args.
train_class_num)
score_softmax.append(ss)
score_openmax.append(so)
print("Evaluation...")
eval_softmax = Evaluation(pred_softmax, labels)
eval_softmax_threshold = Evaluation(pred_softmax_threshold, labels)
eval_openmax = Evaluation(pred_openmax, labels)
# torch.save(eval_softmax, os.path.join(args.checkpoint, 'eval_softmax.pkl'))
# torch.save(eval_softmax_threshold, os.path.join(args.checkpoint, 'eval_softmax_threshold.pkl'))
# torch.save(eval_openmax, os.path.join(args.checkpoint, 'eval_openmax.pkl'))
softmax_results = torch.Tensor([
eval_softmax.accuracy, eval_softmax.f1_measure, eval_softmax.f1_macro,
eval_softmax.f1_macro_weighted
])
threshold_results = torch.Tensor([
eval_softmax_threshold.accuracy, eval_softmax_threshold.f1_measure,
eval_softmax_threshold.f1_macro,
eval_softmax_threshold.f1_macro_weighted
])
openmax_results = torch.Tensor([
eval_openmax.accuracy, eval_openmax.f1_measure, eval_openmax.f1_macro,
eval_openmax.f1_macro_weighted
])
softmax_results = reduce_tensor(softmax_results.to(input_var.device))
threshold_results = reduce_tensor(threshold_results.to(input_var.device))
openmax_results = reduce_tensor(openmax_results.to(input_var.device))
if args.local_rank == 0:
print(f"the result for three : Acc, F1, macro, w-marco")
print(f"the result for softmax : {softmax_results}")
print(f"the result for threshold : {threshold_results}")
print(f"the result for openmax : {openmax_results}")
def test(net, testloader, criterion, device, intervals=20):
normfea_list = [] # extracted feature norm
cosine_list = [] # extracted cosine similarity
energy_list = [] # energy value
embed_fea_list = []
dotproduct_fea2cen_list = []
normweight_fea2cen_list = []
Target_list = []
Predict_list = []
with torch.no_grad():
for batch_idx, (inputs, targets) in enumerate(testloader):
inputs, targets = inputs.to(device), targets.to(device)
out = net(inputs) # shape [batch,class]
# Test cannot calculate loss beacuse of class number dismatch.
# loss_dict = criterion(out, targets)
# loss = loss_dict['loss']
# test_loss += loss.item()
normfea_list.append(out["norm_fea"])
cosine_list.append(out["cosine_fea2cen"])
energy_list.append(out["energy"])
embed_fea_list.append(out["embed_fea"])
dotproduct_fea2cen_list.append(out["dotproduct_fea2cen"])
normweight_fea2cen_list.append(out["normweight_fea2cen"])
Target_list.append(targets)
# "CenterLoss", "SoftmaxLoss", "ArcFaceLoss", "NormFaceLoss", "PSoftmaxLoss"
if args.loss in [
"CenterLoss",
"SoftmaxLoss",
]:
_, predicted = (out['dotproduct_fea2cen']).max(1)
elif args.loss in [
"ArcFaceLoss",
"NormFaceLoss",
]:
_, predicted = (out['cosine_fea2cen']).max(1)
else: # PSoftmaxLoss
_, predicted = (out['normweight_fea2cen']).max(1)
Predict_list.append(predicted)
progress_bar(batch_idx, len(testloader), "|||")
normfea_list = torch.cat(normfea_list, dim=0)
cosine_list = torch.cat(cosine_list, dim=0)
energy_list = torch.cat(energy_list, dim=0)
embed_fea_list = torch.cat(embed_fea_list, dim=0)
dotproduct_fea2cen_list = torch.cat(dotproduct_fea2cen_list, dim=0)
normweight_fea2cen_list = torch.cat(normweight_fea2cen_list, dim=0)
Target_list = torch.cat(Target_list, dim=0)
Predict_list = torch.cat(Predict_list, dim=0)
# "CenterLoss", "SoftmaxLoss", "ArcFaceLoss", "NormFaceLoss", "PSoftmaxLoss"
# "possibility", "distance",'norm','energy','cosine'
best_F1 = 0
best_thres = 0
best_eval = None
# for these unbounded metric, we explore more intervals by *5 to achieve a relatively fair comparison.
expand_factor = 5
if args.openmetric == "possibility" and args.loss in [
"CenterLoss", "SoftmaxLoss"
]:
threshold_min = 0.0
threshold_max = 1.0
openmetric_list, _ = torch.softmax(dotproduct_fea2cen_list,
dim=1).max(dim=1)
for thres in np.linspace(threshold_min, threshold_max, intervals):
Predict_list[openmetric_list < thres] = args.train_class_num
eval = Evaluation(Predict_list.cpu().numpy(),
Target_list.cpu().numpy())
if eval.f1_measure > best_F1:
best_F1 = eval.f1_measure
best_thres = thres
best_eval = eval
if args.openmetric == "possibility" and args.loss in [
"ArcFaceLoss", "NormFaceLoss"
]:
threshold_min = 0.0
threshold_max = 1.0
fake_Target_list = Target_list
fake_Target_list[fake_Target_list ==
args.train_class_num] = args.train_class_num - 1
openmetric_list = criterion({"cosine_fea2cen": cosine_list},
fake_Target_list)["output"]
openmetric_list, _ = torch.softmax(openmetric_list, dim=1).max(dim=1)
print(f"openmetric_list shape is {openmetric_list.shape}")
print(
f"Threshold range is {openmetric_list.min()} | {openmetric_list.max()}"
)
for thres in np.linspace(threshold_min, threshold_max, intervals):
Predict_list[openmetric_list < thres] = args.train_class_num
eval = Evaluation(Predict_list.cpu().numpy(),
Target_list.cpu().numpy())
if eval.f1_measure > best_F1:
best_F1 = eval.f1_measure
best_thres = thres
best_eval = eval
if args.openmetric == "possibility" and args.loss in ["PSoftmaxLoss"]:
threshold_min = 0.0
threshold_max = 1.0
openmetric_list, _ = torch.softmax(normweight_fea2cen_list,
dim=1).max(dim=1)
for thres in np.linspace(threshold_min, threshold_max, intervals):
Predict_list[openmetric_list < thres] = args.train_class_num
eval = Evaluation(Predict_list.cpu().numpy(),
Target_list.cpu().numpy())
if eval.f1_measure > best_F1:
best_F1 = eval.f1_measure
best_thres = thres
best_eval = eval
if args.openmetric == "distance" and args.loss in ["CenterLoss"]:
distance = Distance()
openmetric_list = distance.l2(embed_fea_list, out["centroids"])
openmetric_list, _ = openmetric_list.min(dim=1)
threshold_min = openmetric_list.min().item()
threshold_max = openmetric_list.max().item()
print(f"Threshold range is {threshold_min} - {threshold_max}")
for thres in np.linspace(threshold_min, threshold_max,
expand_factor * intervals):
Predict_list[openmetric_list > thres] = args.train_class_num
eval = Evaluation(Predict_list.cpu().numpy(),
Target_list.cpu().numpy())
if eval.f1_measure > best_F1:
best_F1 = eval.f1_measure
best_thres = thres
best_eval = eval
if args.openmetric == "cosine" and args.loss in [
"ArcFaceLoss", "NormFaceLoss"
]:
openmetric_list, _ = cosine_list.max(dim=1)
threshold_min = openmetric_list.min().item()
threshold_max = openmetric_list.max().item()
print(
f"openmetric_list range cosine is {openmetric_list.min()} | {openmetric_list.max()} "
)
for thres in np.linspace(threshold_min, threshold_max, intervals):
Predict_list[openmetric_list < thres] = args.train_class_num
eval = Evaluation(Predict_list.cpu().numpy(),
Target_list.cpu().numpy())
if eval.f1_measure > best_F1:
best_F1 = eval.f1_measure
best_thres = thres
best_eval = eval
if args.openmetric == "norm" and args.loss in ["PSoftmaxLoss"]:
openmetric_list = normfea_list.squeeze(dim=1)
threshold_min = openmetric_list.min().item()
threshold_max = openmetric_list.max().item()
for thres in np.linspace(threshold_min, threshold_max,
expand_factor * intervals):
Predict_list[openmetric_list < thres] = args.train_class_num
eval = Evaluation(Predict_list.cpu().numpy(),
Target_list.cpu().numpy())
if eval.f1_measure > best_F1:
best_F1 = eval.f1_measure
best_thres = thres
best_eval = eval
if args.openmetric == "energy" and args.loss in ["PSoftmaxLoss"]:
openmetric_list = energy_list
threshold_min = openmetric_list.min().item()
threshold_max = openmetric_list.max().item()
for thres in np.linspace(threshold_min, threshold_max,
expand_factor * intervals):
Predict_list[openmetric_list < thres] = args.train_class_num
eval = Evaluation(Predict_list.cpu().numpy(),
Target_list.cpu().numpy())
if eval.f1_measure > best_F1:
best_F1 = eval.f1_measure
best_thres = thres
best_eval = eval
print(f"Best F1 is: {best_F1} [in best threshold: {best_thres} ]")
return {
"best_F1": best_F1,
"best_thres": best_thres,
"best_eval": best_eval
}
#outputTensor = loaded_1.forward(fourChannelHeatmap[0].unsqueeze(0).unsqueeze(0).to(device)) #
output = outputTensor[0].cpu().clone().detach().numpy()
compl = np.zeros([3, imageSize, int(imageSize * 5)])
compl[:, :,
0:imageSize] = (image.transpose(2, 0, 1) - 127.5) / 127.5
compl[:, :,
imageSize:imageSize * 2] = fourChannelHeatmap[0:3, :, :]
compl[:, :, imageSize * 2:imageSize * 3] = output[0:3, :, :]
compl[0, :, imageSize * 3:imageSize * 4] = output[3, :, :]
compl[1, :, imageSize * 3:imageSize * 4] = output[3, :, :]
compl[2, :, imageSize * 3:imageSize * 4] = output[3, :, :]
if ssim_counter % ssim_modulo == 0:
_, diff = ev.ssim(image, output[0:3, :, :].transpose(1, 2, 0))
ssim_counter += 1
compl[0, :, imageSize * 4:imageSize * 5] = diff / 255 * 2 - 1
compl[1, :, imageSize * 4:imageSize * 5] = diff / 255 * 2 - 1
compl[2, :, imageSize * 4:imageSize * 5] = diff / 255 * 2 - 1
output = compl.transpose(1, 2, 0)
output = output * 127.5 + 127.5
output = output.astype('uint8')
#live3DPlot(outputTensor[0])
if cv2.waitKey(1) & 0xFF == ord('s'):
hm = fourChannelHeatmap[0:3].cpu().clone().detach().numpy()
hm = hm * 127.5 + 127.5
hm = hm.astype('uint8')
def test(net, criterion, testloader, outloader, epoch=None, **options):
net.eval()
correct, total = 0, 0
torch.cuda.empty_cache()
_pred_k, _pred_u, _labels = [], [], []
with torch.no_grad():
for data, labels in testloader:
if options['use_gpu']:
data, labels = data.cuda(), labels.cuda()
with torch.set_grad_enabled(False):
x, y = net(data, True)
# print(f"labels is: {labels}")
logits, _ = criterion(x, y)
predictions = logits.data.max(1)[1]
total += labels.size(0)
correct += (predictions == labels.data).sum()
_pred_k.append(logits.data.cpu().numpy())
_labels.append(labels.data.cpu().numpy())
for batch_idx, (data, labels) in enumerate(outloader):
if options['use_gpu']:
data, labels = data.cuda(), labels.cuda()
with torch.set_grad_enabled(False):
x, y = net(data, True)
# x, y = net(data, return_feature=True)
logits, _ = criterion(x, y)
_pred_u.append(logits.data.cpu().numpy())
# Accuracy
acc = float(correct) * 100. / float(total)
# print('Acc: {:.5f}'.format(acc))
_pred_k = np.concatenate(_pred_k, 0)
_pred_u = np.concatenate(_pred_u, 0)
_labels = np.concatenate(_labels, 0)
# Out-of-Distribution detction evaluation
x1, x2 = np.max(_pred_k, axis=1), np.max(_pred_u, axis=1)
# print(f"x1: {x1}")
# print(f"x2: {x2}")
# print(f"len x1: {len(x1)}")
# print(f"len x2: {len(x2)}")
# print(f"predict_k: {_pred_k}")
# print(f"max predict_k: {np.max(_pred_k)}")
# print(f"min predict_k: {np.min(_pred_k)}")
# print(f"max _pred_u: {np.max(_pred_u)}")
# print(f"min _pred_u: {np.min(_pred_u)}")
tensor_predict_k = torch.Tensor(_pred_k)
tensor_predict_u = torch.Tensor(_pred_u)
tensor_lables = torch.Tensor(_labels)
tensor_lables_unknown = tensor_predict_u.shape[-1] * torch.ones(
tensor_predict_u.shape[0])
tensor_predicts = torch.cat([tensor_predict_k, tensor_predict_u], dim=0)
trensor_labels = torch.cat([tensor_lables, tensor_lables_unknown], dim=0)
openmetric_list, Predict_list = tensor_predicts.max(dim=1)
thres = options['thres']
Predict_list[openmetric_list < thres] = tensor_predict_u.shape[-1]
# print(f"tensor_predicts: {tensor_predicts}")
# print(f"tensor_predicts shape: {tensor_predicts.shape}")
# print(f"trensor_labels shape: {trensor_labels.shape}")
prediction_scores = openmetric_list.numpy().reshape(-1, 1)
eval = Evaluation(Predict_list.numpy(),
trensor_labels.numpy(),
prediction_scores=prediction_scores)
# print(f"my f1_measure is{eval.f1_measure}")
results = evaluation.metric_ood(x1, x2)['Bas']
# OSCR
_oscr_socre = evaluation.compute_oscr(_pred_k, _pred_u, _labels)
results['ACC'] = acc
results['OSCR'] = _oscr_socre * 100.
results["eval"] = eval
return results
def validate(val_loader, model,intervals=20):
# switch to evaluate mode
model.eval()
if args.local_rank == 0:
print("start evaluating...")
normfea_list = [] # extracted feature norm
energy_list = [] # energy value
normweight_fea2cen_list = []
Target_list = []
Predict_list = []
for i, data in enumerate(val_loader):
input = data[0]["data"]
target = data[0]["label"].squeeze().cuda().long()
val_loader_len = int(val_loader._size / args.batch_size)
if args.local_rank == 0 and i%200 ==0:
print(f"evaluating {i}\t/{val_loader_len}...")
target = target.cuda(non_blocking=True)
input_var = Variable(input)
target_var = Variable(target)
# compute output
with torch.no_grad():
out = model(input_var)
normfea_list.append(out["norm_fea"])
energy_list.append(out["energy"])
normweight_fea2cen_list.append(out["normweight_fea2cen"])
Target_list.append(target_var)
_, predicted = (out['normweight_fea2cen']).max(1)
Predict_list.append(predicted)
normfea_list = torch.cat(normfea_list, dim=0)
energy_list = torch.cat(energy_list, dim=0)
normweight_fea2cen_list = torch.cat(normweight_fea2cen_list, dim=0)
Target_list = torch.cat(Target_list, dim=0)
Predict_list = torch.cat(Predict_list, dim=0)
best_F1_possibility = 0
best_possibility_thr = 0
best_F1_norm = 0
best_norm_thr = 0
best_F1_energy = 0
best_energy_thr = 0
# for these unbounded metric, we explore more intervals by *5 to achieve a relatively fair comparison.
expand_factor = 5
Predict_list_possibility = Predict_list.clone().detach()
Predict_list_norm = Predict_list.clone().detach()
Predict_list_energy = Predict_list.clone().detach()
# possibility
openmetric_possibility = normweight_fea2cen_list
openmetric_possibility, _ = torch.softmax(openmetric_possibility, dim=1).max(dim=1)
for thres in np.linspace(0.0, 1.0, intervals):
Predict_list_possibility[openmetric_possibility < thres] = args.train_class_num
eval = Evaluation(Predict_list_possibility.cpu().numpy(), Target_list.cpu().numpy())
if eval.f1_measure > best_F1_possibility:
best_F1_possibility = eval.f1_measure
best_possibility_thr = thres
# norm
openmetric_norm = normfea_list.squeeze(dim=1)
threshold_min_norm = openmetric_norm.min().item()
threshold_max_norm = openmetric_norm.max().item()
for thres in np.linspace(threshold_min_norm, threshold_max_norm, expand_factor * intervals):
Predict_list_norm[openmetric_norm < thres] = args.train_class_num
eval = Evaluation(Predict_list_norm.cpu().numpy(), Target_list.cpu().numpy())
if eval.f1_measure > best_F1_norm:
best_F1_norm = eval.f1_measure
best_norm_thr = thres
# energy
openmetric_energy = energy_list
threshold_min_energy = openmetric_energy.min().item()
threshold_max_energy = openmetric_energy.max().item()
for thres in np.linspace(threshold_min_energy, threshold_max_energy, expand_factor * intervals):
Predict_list_energy[openmetric_energy < thres] = args.train_class_num
eval = Evaluation(Predict_list_energy.cpu().numpy(), Target_list.cpu().numpy())
if eval.f1_measure > best_F1_energy:
best_F1_energy = eval.f1_measure
best_energy_thr = thres
if args.local_rank == 0:
print(f"Energy range is {threshold_min_energy} to {threshold_max_energy}")
print(f"Norm range is {threshold_min_norm} to {threshold_max_norm}")
print(f"Best Possibility F1 is: {best_F1_possibility} | Norm F1 is :{best_F1_norm} | Energy F1 is: {best_F1_energy}")
print(
f"Best Possibility thre is: {best_possibility_thr} | Norm thr is :{best_norm_thr} | Energy thr is: {best_energy_thr}")
print(f"Max predict is {Predict_list.max()}")
print(f"Max target is {Target_list.max()}")
if args.local_rank == 1:
print(f"Max predict is {Predict_list.max()}")
print(f"Max target is {Target_list.max()}")
return {
"best_F1_possibility": best_F1_possibility,
"best_F1_norm": best_F1_norm,
"best_F1_energy": best_F1_energy
}