optimizerC = optim.Adam(netC.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) ###### summary writer writer = save_log(opt.log_path,mode) # train a classifier on seen classes, obtain teacher #print('train_feature', data.train_feature.shape) if opt.need_teacher: if opt.teacher_type =='seen_classes': print('pretrain classifier for seen classes!') pretrain_cls = classifier.CLASSIFIER(data.train_feature, util.map_label(data.train_label, data.seenclasses), data.test_seen_feature,data.test_seen_label,data.seenclasses,data,data.seenclasses.size(0), opt.resSize, opt.cuda, 0.001, 0.5, 100, 100, opt.pretrain_classifier,opt.teacher_type) elif opt.teacher_type =='all_classes': print('pretrain classifier for all classes!') ############# train/test split 4/1 feature_all = torch.cat((data.train_feature,data.test_seen_feature,data.test_unseen_feature),0) label_all = torch.cat((data.train_label,data.test_seen_label,data.test_unseen_label),0) data_num = label_all.shape[0] print('all_data_num',data_num) indices = list(range(data_num)) np.random.shuffle(indices) train_indices, val_indices = indices[int(data_num*opt.val_split):], indices[:int(data_num*opt.val_split)] #print('train_indices',train_indices) #print('val_indeces',val_indeces)
netDec.eval() netF.eval() syn_feature, syn_label = generate_syn_feature(netG, data.unseenclasses, data.attribute, opt.syn_num, netF=netF, netDec=netDec) # Generalized zero-shot learning if opt.gzsl: # Concatenate real seen features with synthesized unseen features train_X = torch.cat((data.train_feature, syn_feature), 0) train_Y = torch.cat((data.train_label, syn_label), 0) nclass = opt.nclass_all # Train GZSL classifier gzsl_cls = classifier.CLASSIFIER(train_X, train_Y, data, nclass, opt.cuda, opt.classifier_lr, 0.5, \ 25, opt.syn_num, generalized=True, netDec=netDec, dec_size=opt.attSize, dec_hidden_size=4096) if best_gzsl_acc < gzsl_cls.H: best_acc_seen, best_acc_unseen, best_gzsl_acc = gzsl_cls.acc_seen, gzsl_cls.acc_unseen, gzsl_cls.H print('GZSL: seen=%.4f, unseen=%.4f, h=%.4f' % (gzsl_cls.acc_seen, gzsl_cls.acc_unseen, gzsl_cls.H), end=" ") # Zero-shot learning # Train ZSL classifier zsl_cls = classifier.CLASSIFIER(syn_feature, util.map_label(syn_label, data.unseenclasses), \ data, data.unseenclasses.size(0), opt.cuda, opt.classifier_lr, 0.5, 25, opt.syn_num, \ generalized=False, netDec=netDec, dec_size=opt.attSize, dec_hidden_size=4096) acc = zsl_cls.acc if best_zsl_acc < acc: best_zsl_acc = acc print('ZSL: unseen accuracy=%.4f' % (acc))
print("Generator {}th finished time taken {}".format( epoch, time.time() - tic)) netG.eval() gzsl_syn_feature, gzsl_syn_label = generate_syn_feature( netG, data.GZSL_fake_test_labels, opt.fake_batch_size) if opt.gzsl: nclass = opt.nclass_all train_X = gzsl_syn_feature train_Y = gzsl_syn_label print(train_Y.shape) tic = time.time() gzsl_cls = classifier.CLASSIFIER(train_X, train_Y, data, nclass, opt.cuda, opt, opt.classifier_lr, 0.5, opt.classifier_epoch, opt.classifier_batch_size, True) sum_GZSL_F1_5 = gzsl_cls.sum_F1_scores_seen_unseen[ 4] * 100 + gzsl_cls.sum_F1_scores_seen_unseen[0] * 100 if sum_f1_best_GZSL_F1 < sum_GZSL_F1_5: gzsl_best_epoch = epoch sum_f1_best_GZSL_F1 = sum_GZSL_F1_5 sum_f1_best_GZSL_AP = gzsl_cls.sum_F1_scores_seen_unseen[0] sum_f1_best_GZSL_F1_3 = gzsl_cls.sum_F1_scores_seen_unseen[1] sum_f1_best_GZSL_P_3 = gzsl_cls.sum_F1_scores_seen_unseen[2] sum_f1_best_GZSL_R_3 = gzsl_cls.sum_F1_scores_seen_unseen[3] sum_f1_best_GZSL_F1_5 = gzsl_cls.sum_F1_scores_seen_unseen[4] sum_f1_best_GZSL_P_5 = gzsl_cls.sum_F1_scores_seen_unseen[5] sum_f1_best_GZSL_R_5 = gzsl_cls.sum_F1_scores_seen_unseen[6]
def train(): dataset = DATA_LOADER(opt) opt.C_dim = dataset.att_dim opt.X_dim = dataset.feature_dim opt.y_dim = dataset.ntrain_class data_layer = FeatDataLayer(dataset.train_label.numpy(), dataset.train_feature.numpy(), opt) result_zsl_knn = Result() result_gzsl_soft = Result() netG = Glow(classes=opt.y_dim, condition_dim=opt.C_dim).cuda() out_dir = 'out/{}/shuffle'.format(opt.dataset) os.makedirs(out_dir, exist_ok=True) print("The output dictionary is {}".format(out_dir)) log_dir = out_dir + '/log_{}.txt'.format(opt.dataset) with open(log_dir, 'w') as f: f.write('Training Start:') f.write(strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()) + '\n') start_step = 0 if opt.resume: if os.pathls\ .isfile(opt.resume): print("=> loading checkpoint '{}'".format(opt.resume)) checkpoint = torch.load(opt.resume) netG.load_state_dict(checkpoint['state_dict_G']) train_z = checkpoint['latent_z'].cuda() start_step = checkpoint['it'] print(checkpoint['log']) else: print("=> no checkpoint found at '{}'".format(opt.resume)) initial = True optimizerG = optim.Adam(netG.parameters(), lr=opt.lr) for it in range(start_step, opt.niter + 1): blobs = data_layer.forward() feat_data = blobs['data'] # image data labels = blobs['labels'].astype(int) # class labels idx = blobs['idx'].astype(int) C = np.array([dataset.train_att[i, :] for i in labels]) L = torch.from_numpy(labels).cuda() C = torch.from_numpy(C.astype('float32')).cuda() X = torch.from_numpy(feat_data).cuda() X = X.view(*X.shape, 1, 1) if initial is True: netG(x=X, y_onehot=C, reverse=False) initial = False z, nll, vaeloss, y_logit = netG(x=X, y_onehot=C, reverse=False) loss_generative = Glow.loss_generative(nll) loss_classes = Glow.loss_class(y_logit, L) loss = loss_generative + vaeloss + loss_classes * 0.01 netG.zero_grad() optimizerG.zero_grad() loss.backward() optimizerG.step() if it % opt.disp_interval == 0 and it: log_text = 'Iter-[{}/{}]; epoch: {} Gloss: {:.3f} vaeloss: {:.3f} clsloss: {:.3f}'.format( it, opt.niter, it // opt.evl_interval, float(loss_generative), float(vaeloss), float(loss_classes)) log_print(log_text, log_dir) if it % opt.evl_interval == 0 and it: netG.eval() gen_feat, gen_label = synthesize_feature_test( netG, dataset, 300, 0.5, opt) """ ZSL""" acc = eval_zsl_knn(gen_feat.numpy(), gen_label.numpy(), dataset) result_zsl_knn.update(it, acc) log_print("{}nn Classifer: ".format(opt.Knn), log_dir) log_print( "Accuracy is {:.2f}%, Best_acc [{:.2f}% | Iter-{}]".format( acc, result_zsl_knn.best_acc, result_zsl_knn.best_iter), log_dir) gen_feat, gen_label = synthesize_feature_test( netG, dataset, opt.nSample, 1.0, opt) """ GZSL""" # note test label need be shift with offset ntrain_class train_X = torch.cat((dataset.train_feature, gen_feat), 0) train_Y = torch.cat( (dataset.train_label, gen_label + dataset.ntrain_class), 0) cls = classifier.CLASSIFIER( train_X, train_Y, dataset, dataset.ntrain_class + dataset.ntest_class, True, opt.classifier_lr, 0.5, 25, opt.nSample, True) result_gzsl_soft.update_gzsl(it, cls.acc_unseen, cls.acc_seen, cls.H) log_print("GZSL Softmax:", log_dir) log_print( "U->T {:.2f}% S->T {:.2f}% H {:.2f}% Best_H [{:.2f}% {:.2f}% {:.2f}% | Iter-{}]" .format(cls.acc_unseen, cls.acc_seen, cls.H, result_gzsl_soft.best_acc_U_T, result_gzsl_soft.best_acc_S_T, result_gzsl_soft.best_acc, result_gzsl_soft.best_iter), log_dir) if result_zsl_knn.save_model: files2remove = glob.glob(out_dir + '/Best_model_ZSL_*') for _i in files2remove: os.remove(_i) save_model( it, netG, opt.manualSeed, log_text, out_dir + '/Best_model_ZSL_Acc_{:.2f}.tar'.format( result_zsl_knn.acc_list[-1])) if result_gzsl_soft.save_model: files2remove = glob.glob(out_dir + '/Best_model_GZSL_*') for _i in files2remove: os.remove(_i) save_model( it, netG, opt.manualSeed, log_text, out_dir + '/Best_model_GZSL_H_{:.2f}_S_{:.2f}_U_{:.2f}.tar'.format( result_gzsl_soft.best_acc, result_gzsl_soft.best_acc_S_T, result_gzsl_soft.best_acc_U_T)) netG.train() if it % opt.save_interval == 0 and it: save_model(it, netG, opt.manualSeed, log_text, out_dir + '/Iter_{:d}.tar'.format(it)) print('Save model to ' + out_dir + '/Iter_{:d}.tar'.format(it))
def MI_loss(mus, sigmas, i_c, alpha=1e-8): kl_divergence = (0.5 * torch.sum((mus ** 2) + (sigmas ** 2) - torch.log((sigmas ** 2) + alpha) - 1, dim=1)) MI_loss = (torch.mean(kl_divergence) - i_c) return MI_loss def optimize_beta(beta, MI_loss,alpha2=1e-6): beta_new = max(0, beta + (alpha2 * MI_loss)) # return the updated beta value: return beta_new # train a classifier on seen classes, obtain \theta of Equation (4) pretrain_cls = classifier.CLASSIFIER(data.train_feature, util.map_label(data.train_label, data.seenclasses), data.seenclasses.size(0), opt.resSize, opt.cuda, 0.001, 0.5, 50, 100) for p in pretrain_cls.model.parameters(): # set requires_grad to False p.requires_grad = False for epoch in range(opt.nepoch): FP = 0 mean_lossD = 0 mean_lossG = 0 for i in range(0, data.ntrain, opt.batch_size): for p in mapping.parameters(): # reset requires_grad p.requires_grad = True # they are set to False below in netG update for iter_d in range(opt.critic_iter): sample()
(epoch, opt.nepoch, D_cost.data[0], G_cost.data[0], Wasserstein_D.data[0],vae_loss_seen.data[0]),end=" ") # Evaluation netG.eval() netDec.eval() netF.eval() syn_feature, syn_label = generate_syn_feature(netG, data.unseenclasses, data.attribute, opt.syn_num, netF=netF, netDec=netDec) # Generalized zero-shot learning if opt.gzsl_od: # OD based GZSL seen_class = data.seenclasses.size(0) clsu = classifier.CLASSIFIER(syn_feature, util.map_label(syn_label, data.unseenclasses), data, data.unseenclasses.size(0), \ opt.cuda, _nepoch=25, _batch_size=opt.syn_num, netDec=netDec, dec_size=opt.attSize, dec_hidden_size=4096) clss = classifier.CLASSIFIER(data.train_feature, util.map_label(data.train_label,data.seenclasses), data, seen_class, opt.cuda, \ _nepoch=25, _batch_size=opt.syn_num, netDec=netDec, dec_size=opt.attSize, dec_hidden_size=4096) clsg = classifier_entropy.CLASSIFIER(data.train_feature, util.map_label(data.train_label,data.seenclasses), data, seen_class, \ syn_feature, syn_label, opt.cuda, clss, clsu, _batch_size=128, \ netDec=netDec, dec_size=opt.attSize, dec_hidden_size=4096) if best_gzsl_acc < clsg.H: best_acc_seen, best_acc_unseen, best_gzsl_acc = clsg.acc_seen, clsg.acc_unseen, clsg.H print('GZSL-OD: Acc seen=%.4f, Acc unseen=%.4f, h=%.4f' % (clsg.acc_seen, clsg.acc_unseen, clsg.H)) # Zero-shot learning # Train ZSL classifier zsl_cls = classifier.CLASSIFIER(syn_feature, util.map_label(syn_label, data.unseenclasses), data, data.unseenclasses.size(0), \ opt.cuda, opt.classifier_lr, 0.5, 25, opt.syn_num, generalized=False, netDec=netDec, \ dec_size=opt.attSize, dec_hidden_size=4096)
def train(): dataset = DATA_LOADER(opt) opt.C_dim = dataset.att_dim opt.X_dim = dataset.feature_dim opt.Z_dim = opt.latent_dim opt.y_dim = dataset.ntrain_class opt.niter = int(dataset.ntrain / opt.batchsize) * opt.nepoch data_layer = FeatDataLayer(dataset.train_label.numpy(), dataset.train_feature.numpy(), opt) result_zsl_knn = Result() result_gzsl_soft = Result() netG = Conditional_Generator(opt).cuda() netG.apply(weights_init) print(netG) train_z = torch.FloatTensor(len(dataset.train_feature), opt.Z_dim).normal_(0, opt.latent_var).cuda() out_dir = 'out/{}/nSample-{}_nZ-{}_sigma-{}_langevin_s-{}_step-{}'.format( opt.dataset, opt.nSample, opt.Z_dim, opt.sigma, opt.langevin_s, opt.langevin_step) os.makedirs(out_dir, exist_ok=True) print("The output dictionary is {}".format(out_dir)) log_dir = out_dir + '/log_{}.txt'.format(opt.dataset) with open(log_dir, 'w') as f: f.write('Training Start:') f.write(strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()) + '\n') start_step = 0 if opt.resume: if os.path.isfile(opt.resume): print("=> loading checkpoint '{}'".format(opt.resume)) checkpoint = torch.load(opt.resume) netG.load_state_dict(checkpoint['state_dict_G']) train_z = checkpoint['latent_z'].cuda() start_step = checkpoint['it'] print(checkpoint['log']) else: print("=> no checkpoint found at '{}'".format(opt.resume)) optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, weight_decay=opt.weight_decay) # range(start_step, opt.niter+1) for it in range(start_step, opt.niter + 1): blobs = data_layer.forward() feat_data = blobs['data'] # image data labels = blobs['labels'].astype(int) # class labels idx = blobs['idx'].astype(int) C = np.array([dataset.train_att[i, :] for i in labels]) C = torch.from_numpy(C.astype('float32')).cuda() X = torch.from_numpy(feat_data).cuda() Z = train_z[idx].cuda() optimizer_z = torch.optim.Adam([Z], lr=opt.lr, weight_decay=opt.weight_decay) # Alternatingly update weights w and infer latent_batch z for em_step in range(2): # EM_STEP # update w for _ in range(1): pred = netG(Z, C) loss = getloss(pred, X, Z, opt) loss.backward() torch.nn.utils.clip_grad_norm_(netG.parameters(), 1) optimizerG.step() optimizerG.zero_grad() # infer z for _ in range(opt.langevin_step): U_tau = torch.FloatTensor(Z.shape).normal_(0, opt.sigma_U).cuda() pred = netG(Z, C) loss = getloss(pred, X, Z, opt) loss = opt.langevin_s * 2 / 2 * loss loss.backward() torch.nn.utils.clip_grad_norm_([Z], 1) optimizer_z.step() optimizer_z.zero_grad() if it < opt.niter / 3: Z.data += opt.langevin_s * U_tau # update Z train_z[idx, ] = Z.data if it % opt.disp_interval == 0 and it: log_text = 'Iter-[{}/{}]; loss: {:.3f}'.format( it, opt.niter, loss.item()) log_print(log_text, log_dir) if it % opt.evl_interval == 0 and it: netG.eval() gen_feat, gen_label = synthesize_feature_test(netG, dataset, opt) """ ZSL""" acc = eval_zsl_knn(gen_feat.numpy(), gen_label.numpy(), dataset) result_zsl_knn.update(it, acc) log_print("{}nn Classifer: ".format(opt.Knn), log_dir) log_print( "Accuracy is {:.2f}%, Best_acc [{:.2f}% | Iter-{}]".format( acc, result_zsl_knn.best_acc, result_zsl_knn.best_iter), log_dir) """ GZSL""" # note test label need be shift with offset ntrain_class train_X = torch.cat((dataset.train_feature, gen_feat), 0) train_Y = torch.cat( (dataset.train_label, gen_label + dataset.ntrain_class), 0) cls = classifier.CLASSIFIER( train_X, train_Y, dataset, dataset.ntrain_class + dataset.ntest_class, True, opt.classifier_lr, 0.5, 25, opt.nSample, True) result_gzsl_soft.update_gzsl(it, cls.acc_unseen, cls.acc_seen, cls.H) log_print("GZSL Softmax:", log_dir) log_print( "U->T {:.2f}% S->T {:.2f}% H {:.2f}% Best_H [{:.2f}% {:.2f}% {:.2f}% | Iter-{}]" .format(cls.acc_unseen, cls.acc_seen, cls.H, result_gzsl_soft.best_acc_U_T, result_gzsl_soft.best_acc_S_T, result_gzsl_soft.best_acc, result_gzsl_soft.best_iter), log_dir) if result_zsl_knn.save_model: files2remove = glob.glob(out_dir + '/Best_model_ZSL_*') for _i in files2remove: os.remove(_i) save_model( it, netG, train_z, opt.manualSeed, log_text, out_dir + '/Best_model_ZSL_Acc_{:.2f}.tar'.format( result_zsl_knn.acc_list[-1])) if result_gzsl_soft.save_model: files2remove = glob.glob(out_dir + '/Best_model_GZSL_*') for _i in files2remove: os.remove(_i) save_model( it, netG, train_z, opt.manualSeed, log_text, out_dir + '/Best_model_GZSL_H_{:.2f}_S_{:.2f}_U_{:.2f}.tar'.format( result_gzsl_soft.best_acc, result_gzsl_soft.best_acc_S_T, result_gzsl_soft.best_acc_U_T)) netG.train() if it % opt.save_interval == 0 and it: save_model(it, netG, train_z, opt.manualSeed, log_text, out_dir + '/Iter_{:d}.tar'.format(it)) print('Save model to ' + out_dir + '/Iter_{:d}.tar'.format(it))
if opt.cuda: ones = ones.cuda() gradients = autograd.grad(outputs=disc_interpolates, inputs=interpolates, grad_outputs=ones, create_graph=True, retain_graph=True, only_inputs=True)[0] gradient_penalty = ((gradients.norm(2, dim=1) - 1)**2).mean() * opt.lambda1 return gradient_penalty pretrain_cls = classifier.CLASSIFIER(data.train_feature, data.train_label, opt.nclass_all, opt.resSize, opt.cuda, 0.001, 0.5, 50, 100, opt.pretrain_classifier) sim_vector = np.zeros((opt.nepoch, 10, 40)) idex_vector = np.zeros((opt.nepoch, 10, 40)) centroid_feat = np.zeros((opt.nepoch, 10, data.train_feature.shape[1])) counter = 0 main_seen_dic = {} main_unseen_dic = {} best_acc = 0 flag = False for epoch in range(opt.nepoch): if epoch > 14 and flag == False and opt.dataset == "AWA1": opt.cls_weight = float(opt.cls_weight / 2)
def cf_gzsl(test_x, test_l, split): preds = [] truths = [] test_l_np = test_l.cpu().numpy() test_l_binary = np.array( [y in data.unseenclasses for y in test_l]) if additional_train: gen_sx, gen_sl = generate_syn_feature( self.netG, self.data.seenclasses, self.data.attribute, 100, netF=self.netF, netDec=self.netDec, opt=opt) #gen_sx = self.conditional_sample(data.train_feature, data.attribute[data.train_label], deterministic=False) #gen_sx2 = self.conditional_sample(data.train_feature, data.attribute[data.train_label], deterministic=False) #gen_sx3 = self.conditional_sample(data.train_feature, data.attribute[data.train_label], deterministic=False) #gen_sx = torch.cat((gen_sx, gen_sx2, gen_sx3), 0) #gen_sl = torch.cat((data.train_label.cuda(), data.train_label.cuda(), data.train_label.cuda()), 0) for i in range(test_x.shape[0]): gen_x, gen_l = self.generate_syn_feature_cf( test_x[i], data.unseenclasses, deterministic=deterministic) if use_train: #if additional_train: # train_x = torch.cat((gen_sx, gen_x), 0) # train_y = torch.cat((gen_sl, gen_l), 0) #else: train_x = torch.cat((data.train_feature, gen_x), 0) train_y = torch.cat((data.train_label.cuda(), gen_l), 0) else: gen_s_x, gen_s_l = self.generate_syn_feature_cf( test_x[i], data.seenclasses, deterministic=deterministic) train_x = torch.cat((gen_s_x, gen_x), 0) train_y = torch.cat((gen_s_l, gen_l), 0) if additional_train: train_x = torch.cat((train_x, gen_sx), 0) train_y = torch.cat((train_y, gen_sl.cuda()), 0) if softmax_clf: if not binary: clf = classifier.CLASSIFIER(train_x, train_y, data, self.opt.nclass_all, opt.cuda, opt.classifier_lr, opt.beta1,\ self.epoch, opt.syn_num, generalized=True, netDec=self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, x=test_x[i], use_tde=use_tde, alpha=self.alpha) if self.test_logits is None: self.test_logits = clf.logits else: self.test_logits = np.concatenate( (self.test_logits, clf.logits), axis=0) else: clf = BINARY_CLASSIFIER(train_x, train_y, data, 2, True, opt.classifier_lr, 0.5, self.epoch, opt.syn_num, netDec=self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, use_tde=use_tde, alpha=self.alpha, x=test_x[i]) pred = clf.pred truths.append(test_l_np[i]) preds.append(pred.item()) else: clf = KNNClassifier(train_x, train_y, test_x[i].unsqueeze(0), self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, batch_size=100) pred = clf.fit()[0] preds.append(pred) truths.append(test_l_np[i]) if (i + 1) % 500 == 0: if not binary: binary_acc = self.get_binary_acc(truths, preds) print("%s-%dth acc: %.3f, binary acc: %.3f" % (split, i + 1, cal_macc(truth=truths, pred=preds), binary_acc)) else: test_l_binary_t = test_l_binary[:len(preds )].astype(int) preds_np = np.array(preds) acc = (preds_np == test_l_binary_t).mean() print("%s-%dth binary acc: %.3f" % (split, i + 1, acc)) if self.opt.sanity: break # Sanity check if not binary: acc = cal_macc(truth=truths, pred=preds) binary_acc = self.get_binary_acc(truths, preds) clf_results = {"truths": truths, "preds": preds} else: acc = (np.array(preds) == test_l_binary.astype(int)).mean() binary_acc = acc clf_results = {"truths": test_l_binary, "preds": preds} save_file = self.get_save_result_file(split) if self.log_to_file: with open(save_file, 'wb') as handle: pickle.dump(clf_results, handle) return acc, binary_acc
def gzsl(self, use_train, softmax_clf, cf, deterministic=False, additional_train=False, use_tde=False, binary=False): opt = self.opt data = self.data if self.siamese: clf = SiameseClassifier(data, opt, self.netE, self.netG, self.netF, self.cls_netDec, dec_size=opt.attSize, cf=cf, n_epochs=opt.clf_epoch, distance="l1") if self.netS is None: clf.train() else: clf.network = self.netS s_acc, u_acc = clf.validate(gzsl=True) if not cf: with torch.no_grad(): gen_x, gen_l = generate_syn_feature(self.netG, self.data.unseenclasses, self.data.attribute, opt.syn_num, netF=self.netF, netDec=self.netDec, opt=opt) if use_train: train_x = torch.cat((data.train_feature, gen_x), 0) train_y = torch.cat((data.train_label, gen_l), 0) else: with torch.no_grad(): gen_s_x, gen_s_l = generate_syn_feature( self.netG, self.data.seenclasses, self.data.attribute, opt.syn_num, netF=self.netF, netDec=self.netDec, opt=opt) train_x = torch.cat((gen_s_x, gen_x), 0) train_y = torch.cat((gen_s_l, gen_l), 0) if softmax_clf: if not binary: gzsl_cls = classifier.CLASSIFIER(train_x, train_y, \ data, data.allclasses.size(0), opt.cuda, opt.classifier_lr, 0.5, self.epoch, opt.syn_num, generalized=True, netDec=self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, use_tde=use_tde, alpha=self.alpha) self.test_logits = gzsl_cls.all_outputs else: gzsl_cls = BINARY_CLASSIFIER(train_x, train_y, data, 2, True, opt.classifier_lr, 0.5, self.epoch, opt.syn_num, netDec=self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, use_tde=use_tde, alpha=self.alpha) s_acc = gzsl_cls.acc_seen u_acc = gzsl_cls.acc_unseen h_acc = gzsl_cls.H self.s_bacc = gzsl_cls.s_bacc self.u_bacc = gzsl_cls.u_bacc if not binary: clf_results = {"preds": gzsl_cls.pred_s.cpu().numpy()} save_file = self.get_save_result_file("seen") if self.log_to_file and not binary: with open(save_file, 'wb') as handle: pickle.dump(clf_results, handle) if not binary: clf_results = {"preds": gzsl_cls.pred_u.cpu().numpy()} save_file = self.get_save_result_file("unseen") if self.log_to_file and not binary: with open(save_file, 'wb') as handle: pickle.dump(clf_results, handle) else: u_cls = KNNClassifier(train_x, train_y, data.test_unseen_feature, self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, batch_size=100) preds = u_cls.fit() truths = data.test_unseen_label.cpu().numpy() u_acc = cal_macc(truth=truths, pred=preds) s_cls = KNNClassifier(train_x, train_y, data.test_seen_feature, self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, batch_size=100) preds = s_cls.fit() truths = data.test_seen_label.cpu().numpy() s_acc = cal_macc(truth=truths, pred=preds) h_acc = 2 * u_acc * s_acc / (u_acc + s_acc) else: self.test_logits = None def cf_gzsl(test_x, test_l, split): preds = [] truths = [] test_l_np = test_l.cpu().numpy() test_l_binary = np.array( [y in data.unseenclasses for y in test_l]) if additional_train: gen_sx, gen_sl = generate_syn_feature( self.netG, self.data.seenclasses, self.data.attribute, 100, netF=self.netF, netDec=self.netDec, opt=opt) #gen_sx = self.conditional_sample(data.train_feature, data.attribute[data.train_label], deterministic=False) #gen_sx2 = self.conditional_sample(data.train_feature, data.attribute[data.train_label], deterministic=False) #gen_sx3 = self.conditional_sample(data.train_feature, data.attribute[data.train_label], deterministic=False) #gen_sx = torch.cat((gen_sx, gen_sx2, gen_sx3), 0) #gen_sl = torch.cat((data.train_label.cuda(), data.train_label.cuda(), data.train_label.cuda()), 0) for i in range(test_x.shape[0]): gen_x, gen_l = self.generate_syn_feature_cf( test_x[i], data.unseenclasses, deterministic=deterministic) if use_train: #if additional_train: # train_x = torch.cat((gen_sx, gen_x), 0) # train_y = torch.cat((gen_sl, gen_l), 0) #else: train_x = torch.cat((data.train_feature, gen_x), 0) train_y = torch.cat((data.train_label.cuda(), gen_l), 0) else: gen_s_x, gen_s_l = self.generate_syn_feature_cf( test_x[i], data.seenclasses, deterministic=deterministic) train_x = torch.cat((gen_s_x, gen_x), 0) train_y = torch.cat((gen_s_l, gen_l), 0) if additional_train: train_x = torch.cat((train_x, gen_sx), 0) train_y = torch.cat((train_y, gen_sl.cuda()), 0) if softmax_clf: if not binary: clf = classifier.CLASSIFIER(train_x, train_y, data, self.opt.nclass_all, opt.cuda, opt.classifier_lr, opt.beta1,\ self.epoch, opt.syn_num, generalized=True, netDec=self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, x=test_x[i], use_tde=use_tde, alpha=self.alpha) if self.test_logits is None: self.test_logits = clf.logits else: self.test_logits = np.concatenate( (self.test_logits, clf.logits), axis=0) else: clf = BINARY_CLASSIFIER(train_x, train_y, data, 2, True, opt.classifier_lr, 0.5, self.epoch, opt.syn_num, netDec=self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, use_tde=use_tde, alpha=self.alpha, x=test_x[i]) pred = clf.pred truths.append(test_l_np[i]) preds.append(pred.item()) else: clf = KNNClassifier(train_x, train_y, test_x[i].unsqueeze(0), self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, batch_size=100) pred = clf.fit()[0] preds.append(pred) truths.append(test_l_np[i]) if (i + 1) % 500 == 0: if not binary: binary_acc = self.get_binary_acc(truths, preds) print("%s-%dth acc: %.3f, binary acc: %.3f" % (split, i + 1, cal_macc(truth=truths, pred=preds), binary_acc)) else: test_l_binary_t = test_l_binary[:len(preds )].astype(int) preds_np = np.array(preds) acc = (preds_np == test_l_binary_t).mean() print("%s-%dth binary acc: %.3f" % (split, i + 1, acc)) if self.opt.sanity: break # Sanity check if not binary: acc = cal_macc(truth=truths, pred=preds) binary_acc = self.get_binary_acc(truths, preds) clf_results = {"truths": truths, "preds": preds} else: acc = (np.array(preds) == test_l_binary.astype(int)).mean() binary_acc = acc clf_results = {"truths": test_l_binary, "preds": preds} save_file = self.get_save_result_file(split) if self.log_to_file: with open(save_file, 'wb') as handle: pickle.dump(clf_results, handle) return acc, binary_acc s_acc, s_bacc = cf_gzsl(data.test_seen_feature, data.test_seen_label, "seen") u_acc, u_bacc = cf_gzsl(data.test_unseen_feature, data.test_unseen_label, "unseen") # s_acc = 0.3 if u_acc + s_acc == 0: h_acc = 0 else: h_acc = 2 * u_acc * s_acc / (u_acc + s_acc) self.s_bacc = s_bacc self.u_bacc = u_bacc return s_acc, u_acc, h_acc
def two_stage(self, use_mask, use_tde, seen_mask=None, unseen_mask=None, save_clf=False): opt = self.opt data = self.data # Unseen: if unseen_mask is None: save_file = "out/%s-unseen.pickle" % use_mask with open(save_file, 'rb') as handle: clf_results = pickle.load(handle) preds = clf_results["preds"] mask = [pred in self.data.unseenclasses for pred in preds] mask = torch.from_numpy(np.array(mask).astype(int)) else: mask = unseen_mask with torch.no_grad(): gen_x, gen_l = generate_syn_feature(self.netG, self.data.unseenclasses, self.data.attribute, opt.u_num, netF=self.netF, netDec=self.netDec, opt=opt) if not save_clf or self.zsl_cls is None: zsl_cls = classifier.CLASSIFIER(gen_x, util.map_label(gen_l, data.unseenclasses), \ data, data.unseenclasses.size(0), opt.cuda, opt.u_lr, opt.u_beta, opt.u_epoch, opt.u_batch_size, \ generalized=False, netDec=self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, mask=mask, use_tde=False, alpha=self.alpha) u_acc = zsl_cls.acc else: zsl_cls = self.zsl_cls u_acc = zsl_cls.val(zsl_cls.test_unseen_feature, zsl_cls.test_unseen_label, zsl_cls.unseenclasses, mask) if save_clf: self.zsl_cls = zsl_cls # Seen: if seen_mask is None: save_file = "out/%s-seen.pickle" % use_mask with open(save_file, 'rb') as handle: clf_results = pickle.load(handle) preds = clf_results["preds"] mask = [pred in self.data.seenclasses for pred in preds] mask = torch.from_numpy(np.array(mask).astype(int)) else: mask = seen_mask if not opt.adjust_s: zsl_cls = classifier.CLASSIFIER(data.train_feature, util.map_label(data.train_label, data.seenclasses), \ data, data.seenclasses.size(0), opt.cuda, opt.classifier_lr, 0.5, 15, opt.syn_num, \ generalized=False, netDec=self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, mask=mask, zsl_on_seen=True, use_tde=use_tde, alpha=self.alpha) else: zsl_cls = classifier.CLASSIFIER(data.train_feature, util.map_label(data.train_label, data.seenclasses), \ data, data.seenclasses.size(0), opt.cuda, opt.s_lr, opt.s_beta, opt.s_epoch, opt.s_batch_size, \ generalized=False, netDec=self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, mask=mask, zsl_on_seen=True, use_tde=use_tde, alpha=self.alpha) s_acc = zsl_cls.acc h_acc = 2 * u_acc * s_acc / (u_acc + s_acc) if opt.log_two_stage: out_dir = "results/two_stage/%s/%s" % (opt.dataset, self.exp_name) if not os.path.exists(out_dir): os.makedirs(out_dir) mask_list = use_mask.split('/') mask_name = "%s_%s" % (mask_list[0], mask_list[1]) out_file = os.path.join(out_dir, "%s.txt" % mask_name) config_msg = "u_num:%d u_lr:%.4f u_beta:%.2f u_epoch:%d u_batch_size:%d" % ( opt.u_num, opt.u_lr, opt.u_beta, opt.u_epoch, opt.u_batch_size) if self.opt.adjust_s: config_msg += " s_lr:%.4f s_beta:%.2f s_epoch:%d s_bs:%d" % ( opt.s_lr, opt.s_beta, opt.s_epoch, opt.s_batch_size) log_msg = "%s---S:%.3f U:%.3f H:%.3f\n" % (config_msg, s_acc, u_acc, h_acc) with open(out_file, "a") as f: f.write(log_msg) return s_acc, u_acc, h_acc
def zsl(self, softmax_clf, cf, deterministic=False): opt = self.opt data = self.data if not cf: with torch.no_grad(): gen_x, gen_l = generate_syn_feature(self.netG, self.data.unseenclasses, self.data.attribute, opt.syn_num, netF=self.netF, netDec=self.netDec, opt=opt) if softmax_clf: zsl_cls = classifier.CLASSIFIER(gen_x, util.map_label(gen_l, data.unseenclasses), \ data, data.unseenclasses.size(0), opt.cuda, opt.classifier_lr, 0.5, self.epoch, opt.syn_num, \ generalized=False, netDec=self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096) acc = zsl_cls.acc else: zsl_cls = KNNClassifier(gen_x, gen_l, data.test_unseen_feature, self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, batch_size=100) preds = zsl_cls.fit() truths = data.test_unseen_label.cpu().numpy() acc = cal_macc(truth=truths, pred=preds) else: preds = [] truths = [] test_x = data.test_unseen_feature mapped_unseen_l = util.map_label(data.test_unseen_label, data.unseenclasses) unseen_label_np = data.test_unseen_label.cpu().numpy() for i in range(test_x.shape[0]): gen_x, gen_l = self.generate_syn_feature_cf( test_x[i], data.unseenclasses, deterministic=deterministic) gen_l = util.map_label(gen_l, data.unseenclasses) if softmax_clf: clf = classifier.CLASSIFIER(gen_x, gen_l, data, data.unseenclasses.size(0), opt.cuda, opt.classifier_lr, 0.5, self.epoch, opt.syn_num, generalized=False, netDec=self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, x=test_x[i]) pred = clf.pred truths.append(mapped_unseen_l[i]) preds.append(pred) else: clf = KNNClassifier(gen_x, gen_l, test_x[i].unsqueeze(0), self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, batch_size=100) pred = clf.fit()[0] preds.append(pred) truths.append(unseen_label_np[i]) if (i + 1) % 500 == 0: print("%dth acc: %.3f" % (i + 1, cal_macc(truth=truths, pred=preds))) if self.opt.sanity: break # Sanity check acc = cal_macc(truth=truths, pred=preds) return acc