def stage_valmixup(net, dataloader, device, name="stage1_mixup"): print("validating mixup and trainloader ...") energy_loader_list = [] energy_mixup_list = [] target_list = [] with torch.no_grad(): for batch_idx, (inputs, targets) in enumerate(dataloader): inputs, targets = inputs.to(device), targets.to(device) mixed = mixup(inputs, targets, args) out_loader = net(inputs) out_mixed = net(mixed) energy_loader_list.append(out_loader["energy"]) energy_mixup_list.append(out_mixed["energy"]) target_list.append(targets) progress_bar(batch_idx, len(trainloader)) energy_loader_list = torch.cat(energy_loader_list, dim=0) energy_mixup_list = torch.cat(energy_mixup_list, dim=0) plot_listhist([energy_loader_list, energy_mixup_list], args, labels=["loader", "mixup"], name=name + "_energy") print("_______________Validate statistics:____________") print(f"train mid:{energy_loader_list.median()} | mixup mid:{energy_mixup_list.median()}") print(f"min energy:{min(energy_loader_list.min(), energy_mixup_list.min())} " f"| max energy:{max(energy_loader_list.max(), energy_mixup_list.max())}") return{ "mid_known": energy_loader_list.median(), "mid_unknown": energy_mixup_list.median() }
def stage1_valvae(net, testloader, device): print("validating vae and net ...") # loading vae model vae = VanillaVAE(in_channels=1, latent_dim=args.latent_dim) vae = vae.to(device) if device == 'cuda': vae = torch.nn.DataParallel(vae) cudnn.benchmark = True if os.path.isfile(args.vae_resume): vae_checkpoint = torch.load(args.vae_resume) vae.load_state_dict(vae_checkpoint['net']) print('==> Resuming vae from checkpoint, loaded..') normfea_test_list = [] normfea_sample_list = [] normfea_mix_list = [] target_list = [] with torch.no_grad(): for batch_idx, (inputs, targets) in enumerate(testloader): inputs, targets = inputs.to(device), targets.to(device) sampled = sampler(vae, device, args) out_test = net(inputs) out_sample = net(sampled) normfea_test_list.append(out_test["norm_fea"]) normfea_sample_list.append(out_sample["norm_fea"]) target_list.append(targets) progress_bar(batch_idx, len(trainloader)) for batch_idx, (inputs, targets) in enumerate(mixuploader): inputs, targets = inputs.to(device), targets.to(device) mixed = mixup(inputs, targets, args) out_mixed = net(mixed) normfea_mix_list.append(out_mixed["norm_fea"]) progress_bar(batch_idx, len(trainloader)) normfea_test_list = torch.cat(normfea_test_list, dim=0) normfea_sample_list = torch.cat(normfea_sample_list, dim=0) normfea_mix_list = torch.cat(normfea_mix_list, dim=0) target_list = torch.cat(target_list, dim=0) unknown_label = target_list.max() normfea_test_unknown_list = normfea_test_list[target_list == unknown_label] normfea_test_known_list = normfea_test_list[target_list != unknown_label] plot_listhist([ normfea_test_known_list, normfea_test_unknown_list, normfea_sample_list, normfea_mix_list ], args, labels=["test_known", "test_unknown", "sampled", "mixed"], name="stage1_valvaemix_normfea_result")
def stage_valvae(net, dataloader, device, name="stage1_valtrain&sample_normfea_result"): print("validating vae and net ...") # loading vae model vae = VanillaVAE(in_channels=1, latent_dim=args.latent_dim) vae = vae.to(device) if device == 'cuda': vae = torch.nn.DataParallel(vae) cudnn.benchmark = True if os.path.isfile(args.vae_resume): vae_checkpoint = torch.load(args.vae_resume) vae.load_state_dict(vae_checkpoint['net']) print('==> Resuming vae from checkpoint, loaded..') normfea_loader_list = [] normfea_sample_list = [] target_list = [] with torch.no_grad(): for batch_idx, (inputs, targets) in enumerate(dataloader): inputs, targets = inputs.to(device), targets.to(device) sampled = sampler(vae, device, args) out_test = net(inputs) out_sample = net(sampled) normfea_loader_list.append(out_test["norm_fea"]) normfea_sample_list.append(out_sample["norm_fea"]) target_list.append(targets) progress_bar(batch_idx, len(trainloader)) normfea_loader_list = torch.cat(normfea_loader_list, dim=0) normfea_sample_list = torch.cat(normfea_sample_list, dim=0) plot_listhist([normfea_loader_list, normfea_sample_list], args, labels=["train data", "sampled data"], name=name) print( f"train mid:{normfea_loader_list.median()} | sampled mid:{normfea_sample_list.median()}" ) print( f"min norm:{min(normfea_loader_list.min(), normfea_sample_list.min())} " f"| max norm:{max(normfea_loader_list.max(), normfea_sample_list.max())}" ) return { "vae": vae, "mid_known": normfea_loader_list.median(), "mid_unknown": normfea_sample_list.median() }
def test_with_hist(net, dataloader, device, intervals=20, name="stage1_test"): energy_list = [] # energy value Target_list = [] Predict_list = [] with torch.no_grad(): for batch_idx, (inputs, targets) in enumerate(dataloader): inputs, targets = inputs.to(device), targets.to(device) out = net(inputs) # shape [batch,class] energy_list.append(out["energy"]) Target_list.append(targets) _, predicted = (out['normweight_fea2cen']).max(1) Predict_list.append(predicted) progress_bar(batch_idx, len(dataloader), "|||") energy_list = torch.cat(energy_list, dim=0) Target_list = torch.cat(Target_list, dim=0) Predict_list = torch.cat(Predict_list, dim=0) unknown_label = Target_list.max() unknown_energy_list = energy_list[Target_list == unknown_label] known_energy_list = energy_list[Target_list != unknown_label] plot_listhist([known_energy_list, unknown_energy_list], args, labels=["known", "unknown"], name=name + "_energy") best_F1 = 0 best_thres = 0 best_eval = None # for these unbounded metric, we explore more intervals by *5 to achieve a relatively fair comparison. expand_factor = 5 openmetric_list = energy_list threshold_min = openmetric_list.min().item() threshold_max = openmetric_list.max().item() for thres in np.linspace(threshold_min, threshold_max, expand_factor * intervals): Predict_list[openmetric_list < thres] = args.train_class_num eval = Evaluation(Predict_list.cpu().numpy(), Target_list.cpu().numpy()) if eval.f1_measure > best_F1: best_F1 = eval.f1_measure best_thres = thres best_eval = eval print(f"The energy range is [{threshold_min}, {threshold_max}] ") print(f"Best F1 is: {best_F1} [in best threshold: {best_thres} ]") return { "best_F1": best_F1, "best_thres": best_thres, "best_eval": best_eval }
def stage_valmixup(net, dataloader, device, name="stage1_valtrain&sample_normfea_result"): print("validating mixup and trainloader ...") normfea_loader_list = [] normfea_mixup_list = [] target_list = [] with torch.no_grad(): for batch_idx, (inputs, targets) in enumerate(dataloader): inputs, targets = inputs.to(device), targets.to(device) mixed = mixup(inputs, targets, args) out_loader = net(inputs) out_mixed = net(mixed) normfea_loader_list.append(out_loader["norm_fea"]) normfea_mixup_list.append(out_mixed["norm_fea"]) target_list.append(targets) progress_bar(batch_idx, len(trainloader)) normfea_loader_list = torch.cat(normfea_loader_list, dim=0) normfea_mixup_list = torch.cat(normfea_mixup_list, dim=0) plot_listhist([normfea_loader_list, normfea_mixup_list], args, labels=["train data", "sampled data"], name=name) print("_______________Validate statistics:____________") print( f"train mid:{normfea_loader_list.median()} | mixup mid:{normfea_mixup_list.median()}" ) print( f"min norm:{min(normfea_loader_list.min(), normfea_mixup_list.min())} " f"| max norm:{max(normfea_loader_list.max(), normfea_mixup_list.max())}" ) return { "mid_known": normfea_loader_list.median(), "mid_unknown": normfea_mixup_list.median() }
def stage_test(net, testloader, device, name="stage1_test_normfea_doublebar"): correct = 0 total = 0 normfea_list = [] pnorm_list = [] energy_list = [] Target_list = [] with torch.no_grad(): for batch_idx, (inputs, targets) in enumerate(testloader): inputs, targets = inputs.to(device), targets.to(device) out = net(inputs) # shape [batch,class] normfea_list.append(out["norm_fea"]) pnorm_list.append(out["pnorm"]) energy_list.append(out["energy"]) Target_list.append(targets) _, predicted = (out["normweight_fea2cen"]).max(1) total += targets.size(0) correct += predicted.eq(targets).sum().item() progress_bar( batch_idx, len(testloader), '| Acc: %.3f%% (%d/%d)' % (100. * correct / total, correct, total)) print("\nTesting results is {:.2f}%".format(100. * correct / total)) normfea_list = torch.cat(normfea_list, dim=0) pnorm_list = torch.cat(pnorm_list, dim=0) energy_list = torch.cat(energy_list, dim=0) Target_list = torch.cat(Target_list, dim=0) unknown_label = Target_list.max() unknown_normfea_list = normfea_list[Target_list == unknown_label] known_normfea_list = normfea_list[Target_list != unknown_label] unknown_pnorm_list = pnorm_list[Target_list == unknown_label] known_pnorm_list = pnorm_list[Target_list != unknown_label] unknown_energy_list = energy_list[Target_list == unknown_label] known_energy_list = energy_list[Target_list != unknown_label] print("_______________Testing statistics:____________") print( f"test known mid:{known_normfea_list.median()} | unknown mid:{unknown_normfea_list.median()}" ) print( f"min norm:{min(known_normfea_list.min(), unknown_normfea_list.min())} " f"| max norm:{max(known_normfea_list.max(), unknown_normfea_list.max())}" ) plot_listhist([known_normfea_list, unknown_normfea_list], args, labels=["known", "unknown"], name=name) plot_listhist([known_pnorm_list, unknown_pnorm_list], args, labels=["known", "unknown"], name=name + "_pnorm") plot_listhist([known_energy_list, unknown_energy_list], args, labels=["known", "unknown"], name=name + "_energy")
def stage_valmixup(net, dataloader, device, name="stage1_mixup_doublebar"): print("validating mixup and trainloader ...") normfea_loader_list = [] normfea_mixup_list = [] pnorm_loader_list = [] pnorm_mixup_list = [] energy_loader_list = [] energy_mixup_list = [] normweight_loader_list = [] normweight_mixup_list = [] target_list = [] with torch.no_grad(): for batch_idx, (inputs, targets) in enumerate(dataloader): inputs, targets = inputs.to(device), targets.to(device) mixed = mixup(inputs, targets, args) out_loader = net(inputs) out_mixed = net(mixed) normfea_loader_list.append(out_loader["norm_fea"]) normfea_mixup_list.append(out_mixed["norm_fea"]) pnorm_loader_list.append(out_loader["pnorm"]) pnorm_mixup_list.append(out_mixed["pnorm"]) energy_loader_list.append(out_loader["energy"]) energy_mixup_list.append(out_mixed["energy"]) normweight_loader_list.append(out_loader["normweight_fea2cen"]) normweight_mixup_list.append(out_mixed["normweight_fea2cen"]) target_list.append(targets) progress_bar(batch_idx, len(trainloader)) normfea_loader_list = torch.cat(normfea_loader_list, dim=0) normfea_mixup_list = torch.cat(normfea_mixup_list, dim=0) pnorm_loader_list = torch.cat(pnorm_loader_list, dim=0) pnorm_mixup_list = torch.cat(pnorm_mixup_list, dim=0) energy_loader_list = torch.cat(energy_loader_list, dim=0) energy_mixup_list = torch.cat(energy_mixup_list, dim=0) normweight_loader_list = torch.cat(normweight_loader_list, dim=0) softmax_loader_list = torch.softmax(normweight_loader_list, dim=1).max(dim=1, keepdim=False)[0] normweight_mixup_list = torch.cat(normweight_mixup_list, dim=0) softmax_mixup_list = torch.softmax(normweight_mixup_list, dim=1).max(dim=1, keepdim=False)[0] plot_listhist([pnorm_loader_list, pnorm_mixup_list], args, labels=["loader", "mixup"], name=name + "_pnorm") plot_listhist([normfea_loader_list, normfea_mixup_list], args, labels=["loader", "mixup"], name=name + "_normfea") plot_listhist([energy_loader_list, energy_mixup_list], args, labels=["loader", "mixup"], name=name + "_energy") plot_listhist([softmax_loader_list, softmax_mixup_list], args, labels=["loader", "mixup"], name=name + "_softmax") print("_______________Validate statistics:____________") print( f"train mid:{energy_loader_list.median()} | mixup mid:{energy_mixup_list.median()}" ) print( f"min energy:{min(energy_loader_list.min(), energy_mixup_list.min())} " f"| max energy:{max(energy_loader_list.max(), energy_mixup_list.max())}" ) return { "mid_known": energy_loader_list.median(), "mid_unknown": energy_mixup_list.median() }