optimizerC = optim.Adam(netC.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999))
###### summary writer
writer = save_log(opt.log_path,mode)
# train a classifier on seen classes, obtain teacher
#print('train_feature', data.train_feature.shape)
if opt.need_teacher:
if opt.teacher_type =='seen_classes':
print('pretrain classifier for seen classes!')
pretrain_cls = classifier.CLASSIFIER(data.train_feature, util.map_label(data.train_label, data.seenclasses),
data.test_seen_feature,data.test_seen_label,data.seenclasses,data,data.seenclasses.size(0),
opt.resSize, opt.cuda, 0.001, 0.5, 100, 100, opt.pretrain_classifier,opt.teacher_type)
elif opt.teacher_type =='all_classes':
print('pretrain classifier for all classes!')
############# train/test split 4/1
feature_all = torch.cat((data.train_feature,data.test_seen_feature,data.test_unseen_feature),0)
label_all = torch.cat((data.train_label,data.test_seen_label,data.test_unseen_label),0)
data_num = label_all.shape[0]
print('all_data_num',data_num)
indices = list(range(data_num))
np.random.shuffle(indices)
train_indices, val_indices = indices[int(data_num*opt.val_split):], indices[:int(data_num*opt.val_split)]
#print('train_indices',train_indices)
#print('val_indeces',val_indeces)
netDec.eval()
netF.eval()
syn_feature, syn_label = generate_syn_feature(netG,
data.unseenclasses,
data.attribute,
opt.syn_num,
netF=netF,
netDec=netDec)
# Generalized zero-shot learning
if opt.gzsl:
# Concatenate real seen features with synthesized unseen features
train_X = torch.cat((data.train_feature, syn_feature), 0)
train_Y = torch.cat((data.train_label, syn_label), 0)
nclass = opt.nclass_all
# Train GZSL classifier
gzsl_cls = classifier.CLASSIFIER(train_X, train_Y, data, nclass, opt.cuda, opt.classifier_lr, 0.5, \
25, opt.syn_num, generalized=True, netDec=netDec, dec_size=opt.attSize, dec_hidden_size=4096)
if best_gzsl_acc < gzsl_cls.H:
best_acc_seen, best_acc_unseen, best_gzsl_acc = gzsl_cls.acc_seen, gzsl_cls.acc_unseen, gzsl_cls.H
print('GZSL: seen=%.4f, unseen=%.4f, h=%.4f' %
(gzsl_cls.acc_seen, gzsl_cls.acc_unseen, gzsl_cls.H),
end=" ")
# Zero-shot learning
# Train ZSL classifier
zsl_cls = classifier.CLASSIFIER(syn_feature, util.map_label(syn_label, data.unseenclasses), \
data, data.unseenclasses.size(0), opt.cuda, opt.classifier_lr, 0.5, 25, opt.syn_num, \
generalized=False, netDec=netDec, dec_size=opt.attSize, dec_hidden_size=4096)
acc = zsl_cls.acc
if best_zsl_acc < acc:
best_zsl_acc = acc
print('ZSL: unseen accuracy=%.4f' % (acc))
print("Generator {}th finished time taken {}".format(
epoch,
time.time() - tic))
netG.eval()
gzsl_syn_feature, gzsl_syn_label = generate_syn_feature(
netG, data.GZSL_fake_test_labels, opt.fake_batch_size)
if opt.gzsl:
nclass = opt.nclass_all
train_X = gzsl_syn_feature
train_Y = gzsl_syn_label
print(train_Y.shape)
tic = time.time()
gzsl_cls = classifier.CLASSIFIER(train_X, train_Y, data, nclass,
opt.cuda, opt, opt.classifier_lr, 0.5,
opt.classifier_epoch,
opt.classifier_batch_size, True)
sum_GZSL_F1_5 = gzsl_cls.sum_F1_scores_seen_unseen[
4] * 100 + gzsl_cls.sum_F1_scores_seen_unseen[0] * 100
if sum_f1_best_GZSL_F1 < sum_GZSL_F1_5:
gzsl_best_epoch = epoch
sum_f1_best_GZSL_F1 = sum_GZSL_F1_5
sum_f1_best_GZSL_AP = gzsl_cls.sum_F1_scores_seen_unseen[0]
sum_f1_best_GZSL_F1_3 = gzsl_cls.sum_F1_scores_seen_unseen[1]
sum_f1_best_GZSL_P_3 = gzsl_cls.sum_F1_scores_seen_unseen[2]
sum_f1_best_GZSL_R_3 = gzsl_cls.sum_F1_scores_seen_unseen[3]
sum_f1_best_GZSL_F1_5 = gzsl_cls.sum_F1_scores_seen_unseen[4]
sum_f1_best_GZSL_P_5 = gzsl_cls.sum_F1_scores_seen_unseen[5]
sum_f1_best_GZSL_R_5 = gzsl_cls.sum_F1_scores_seen_unseen[6]
def train():
dataset = DATA_LOADER(opt)
opt.C_dim = dataset.att_dim
opt.X_dim = dataset.feature_dim
opt.y_dim = dataset.ntrain_class
data_layer = FeatDataLayer(dataset.train_label.numpy(),
dataset.train_feature.numpy(), opt)
result_zsl_knn = Result()
result_gzsl_soft = Result()
netG = Glow(classes=opt.y_dim, condition_dim=opt.C_dim).cuda()
out_dir = 'out/{}/shuffle'.format(opt.dataset)
os.makedirs(out_dir, exist_ok=True)
print("The output dictionary is {}".format(out_dir))
log_dir = out_dir + '/log_{}.txt'.format(opt.dataset)
with open(log_dir, 'w') as f:
f.write('Training Start:')
f.write(strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()) + '\n')
start_step = 0
if opt.resume:
if os.pathls\
.isfile(opt.resume):
print("=> loading checkpoint '{}'".format(opt.resume))
checkpoint = torch.load(opt.resume)
netG.load_state_dict(checkpoint['state_dict_G'])
train_z = checkpoint['latent_z'].cuda()
start_step = checkpoint['it']
print(checkpoint['log'])
else:
print("=> no checkpoint found at '{}'".format(opt.resume))
initial = True
optimizerG = optim.Adam(netG.parameters(), lr=opt.lr)
for it in range(start_step, opt.niter + 1):
blobs = data_layer.forward()
feat_data = blobs['data'] # image data
labels = blobs['labels'].astype(int) # class labels
idx = blobs['idx'].astype(int)
C = np.array([dataset.train_att[i, :] for i in labels])
L = torch.from_numpy(labels).cuda()
C = torch.from_numpy(C.astype('float32')).cuda()
X = torch.from_numpy(feat_data).cuda()
X = X.view(*X.shape, 1, 1)
if initial is True:
netG(x=X, y_onehot=C, reverse=False)
initial = False
z, nll, vaeloss, y_logit = netG(x=X, y_onehot=C, reverse=False)
loss_generative = Glow.loss_generative(nll)
loss_classes = Glow.loss_class(y_logit, L)
loss = loss_generative + vaeloss + loss_classes * 0.01
netG.zero_grad()
optimizerG.zero_grad()
loss.backward()
optimizerG.step()
if it % opt.disp_interval == 0 and it:
log_text = 'Iter-[{}/{}]; epoch: {} Gloss: {:.3f} vaeloss: {:.3f} clsloss: {:.3f}'.format(
it, opt.niter, it // opt.evl_interval, float(loss_generative),
float(vaeloss), float(loss_classes))
log_print(log_text, log_dir)
if it % opt.evl_interval == 0 and it:
netG.eval()
gen_feat, gen_label = synthesize_feature_test(
netG, dataset, 300, 0.5, opt)
""" ZSL"""
acc = eval_zsl_knn(gen_feat.numpy(), gen_label.numpy(), dataset)
result_zsl_knn.update(it, acc)
log_print("{}nn Classifer: ".format(opt.Knn), log_dir)
log_print(
"Accuracy is {:.2f}%, Best_acc [{:.2f}% | Iter-{}]".format(
acc, result_zsl_knn.best_acc, result_zsl_knn.best_iter),
log_dir)
gen_feat, gen_label = synthesize_feature_test(
netG, dataset, opt.nSample, 1.0, opt)
""" GZSL"""
# note test label need be shift with offset ntrain_class
train_X = torch.cat((dataset.train_feature, gen_feat), 0)
train_Y = torch.cat(
(dataset.train_label, gen_label + dataset.ntrain_class), 0)
cls = classifier.CLASSIFIER(
train_X, train_Y, dataset,
dataset.ntrain_class + dataset.ntest_class, True,
opt.classifier_lr, 0.5, 25, opt.nSample, True)
result_gzsl_soft.update_gzsl(it, cls.acc_unseen, cls.acc_seen,
cls.H)
log_print("GZSL Softmax:", log_dir)
log_print(
"U->T {:.2f}% S->T {:.2f}% H {:.2f}% Best_H [{:.2f}% {:.2f}% {:.2f}% | Iter-{}]"
.format(cls.acc_unseen, cls.acc_seen, cls.H,
result_gzsl_soft.best_acc_U_T,
result_gzsl_soft.best_acc_S_T,
result_gzsl_soft.best_acc, result_gzsl_soft.best_iter),
log_dir)
if result_zsl_knn.save_model:
files2remove = glob.glob(out_dir + '/Best_model_ZSL_*')
for _i in files2remove:
os.remove(_i)
save_model(
it, netG, opt.manualSeed, log_text,
out_dir + '/Best_model_ZSL_Acc_{:.2f}.tar'.format(
result_zsl_knn.acc_list[-1]))
if result_gzsl_soft.save_model:
files2remove = glob.glob(out_dir + '/Best_model_GZSL_*')
for _i in files2remove:
os.remove(_i)
save_model(
it, netG, opt.manualSeed, log_text, out_dir +
'/Best_model_GZSL_H_{:.2f}_S_{:.2f}_U_{:.2f}.tar'.format(
result_gzsl_soft.best_acc,
result_gzsl_soft.best_acc_S_T,
result_gzsl_soft.best_acc_U_T))
netG.train()
if it % opt.save_interval == 0 and it:
save_model(it, netG, opt.manualSeed, log_text,
out_dir + '/Iter_{:d}.tar'.format(it))
print('Save model to ' + out_dir + '/Iter_{:d}.tar'.format(it))
def MI_loss(mus, sigmas, i_c, alpha=1e-8):
kl_divergence = (0.5 * torch.sum((mus ** 2) + (sigmas ** 2)
- torch.log((sigmas ** 2) + alpha) - 1, dim=1))
MI_loss = (torch.mean(kl_divergence) - i_c)
return MI_loss
def optimize_beta(beta, MI_loss,alpha2=1e-6):
beta_new = max(0, beta + (alpha2 * MI_loss))
# return the updated beta value:
return beta_new
# train a classifier on seen classes, obtain \theta of Equation (4)
pretrain_cls = classifier.CLASSIFIER(data.train_feature, util.map_label(data.train_label, data.seenclasses), data.seenclasses.size(0), opt.resSize, opt.cuda, 0.001, 0.5, 50, 100)
for p in pretrain_cls.model.parameters(): # set requires_grad to False
p.requires_grad = False
for epoch in range(opt.nepoch):
FP = 0
mean_lossD = 0
mean_lossG = 0
for i in range(0, data.ntrain, opt.batch_size):
for p in mapping.parameters(): # reset requires_grad
p.requires_grad = True # they are set to False below in netG update
for iter_d in range(opt.critic_iter):
sample()
(epoch, opt.nepoch, D_cost.data[0], G_cost.data[0], Wasserstein_D.data[0],vae_loss_seen.data[0]),end=" ")
# Evaluation
netG.eval()
netDec.eval()
netF.eval()
syn_feature, syn_label = generate_syn_feature(netG,
data.unseenclasses,
data.attribute,
opt.syn_num,
netF=netF,
netDec=netDec)
# Generalized zero-shot learning
if opt.gzsl_od:
# OD based GZSL
seen_class = data.seenclasses.size(0)
clsu = classifier.CLASSIFIER(syn_feature, util.map_label(syn_label, data.unseenclasses), data, data.unseenclasses.size(0), \
opt.cuda, _nepoch=25, _batch_size=opt.syn_num, netDec=netDec, dec_size=opt.attSize, dec_hidden_size=4096)
clss = classifier.CLASSIFIER(data.train_feature, util.map_label(data.train_label,data.seenclasses), data, seen_class, opt.cuda, \
_nepoch=25, _batch_size=opt.syn_num, netDec=netDec, dec_size=opt.attSize, dec_hidden_size=4096)
clsg = classifier_entropy.CLASSIFIER(data.train_feature, util.map_label(data.train_label,data.seenclasses), data, seen_class, \
syn_feature, syn_label, opt.cuda, clss, clsu, _batch_size=128, \
netDec=netDec, dec_size=opt.attSize, dec_hidden_size=4096)
if best_gzsl_acc < clsg.H:
best_acc_seen, best_acc_unseen, best_gzsl_acc = clsg.acc_seen, clsg.acc_unseen, clsg.H
print('GZSL-OD: Acc seen=%.4f, Acc unseen=%.4f, h=%.4f' %
(clsg.acc_seen, clsg.acc_unseen, clsg.H))
# Zero-shot learning
# Train ZSL classifier
zsl_cls = classifier.CLASSIFIER(syn_feature, util.map_label(syn_label, data.unseenclasses), data, data.unseenclasses.size(0), \
opt.cuda, opt.classifier_lr, 0.5, 25, opt.syn_num, generalized=False, netDec=netDec, \
dec_size=opt.attSize, dec_hidden_size=4096)
def train():
dataset = DATA_LOADER(opt)
opt.C_dim = dataset.att_dim
opt.X_dim = dataset.feature_dim
opt.Z_dim = opt.latent_dim
opt.y_dim = dataset.ntrain_class
opt.niter = int(dataset.ntrain / opt.batchsize) * opt.nepoch
data_layer = FeatDataLayer(dataset.train_label.numpy(),
dataset.train_feature.numpy(), opt)
result_zsl_knn = Result()
result_gzsl_soft = Result()
netG = Conditional_Generator(opt).cuda()
netG.apply(weights_init)
print(netG)
train_z = torch.FloatTensor(len(dataset.train_feature),
opt.Z_dim).normal_(0, opt.latent_var).cuda()
out_dir = 'out/{}/nSample-{}_nZ-{}_sigma-{}_langevin_s-{}_step-{}'.format(
opt.dataset, opt.nSample, opt.Z_dim, opt.sigma, opt.langevin_s,
opt.langevin_step)
os.makedirs(out_dir, exist_ok=True)
print("The output dictionary is {}".format(out_dir))
log_dir = out_dir + '/log_{}.txt'.format(opt.dataset)
with open(log_dir, 'w') as f:
f.write('Training Start:')
f.write(strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()) + '\n')
start_step = 0
if opt.resume:
if os.path.isfile(opt.resume):
print("=> loading checkpoint '{}'".format(opt.resume))
checkpoint = torch.load(opt.resume)
netG.load_state_dict(checkpoint['state_dict_G'])
train_z = checkpoint['latent_z'].cuda()
start_step = checkpoint['it']
print(checkpoint['log'])
else:
print("=> no checkpoint found at '{}'".format(opt.resume))
optimizerG = optim.Adam(netG.parameters(),
lr=opt.lr,
weight_decay=opt.weight_decay)
# range(start_step, opt.niter+1)
for it in range(start_step, opt.niter + 1):
blobs = data_layer.forward()
feat_data = blobs['data'] # image data
labels = blobs['labels'].astype(int) # class labels
idx = blobs['idx'].astype(int)
C = np.array([dataset.train_att[i, :] for i in labels])
C = torch.from_numpy(C.astype('float32')).cuda()
X = torch.from_numpy(feat_data).cuda()
Z = train_z[idx].cuda()
optimizer_z = torch.optim.Adam([Z],
lr=opt.lr,
weight_decay=opt.weight_decay)
# Alternatingly update weights w and infer latent_batch z
for em_step in range(2): # EM_STEP
# update w
for _ in range(1):
pred = netG(Z, C)
loss = getloss(pred, X, Z, opt)
loss.backward()
torch.nn.utils.clip_grad_norm_(netG.parameters(), 1)
optimizerG.step()
optimizerG.zero_grad()
# infer z
for _ in range(opt.langevin_step):
U_tau = torch.FloatTensor(Z.shape).normal_(0,
opt.sigma_U).cuda()
pred = netG(Z, C)
loss = getloss(pred, X, Z, opt)
loss = opt.langevin_s * 2 / 2 * loss
loss.backward()
torch.nn.utils.clip_grad_norm_([Z], 1)
optimizer_z.step()
optimizer_z.zero_grad()
if it < opt.niter / 3:
Z.data += opt.langevin_s * U_tau
# update Z
train_z[idx, ] = Z.data
if it % opt.disp_interval == 0 and it:
log_text = 'Iter-[{}/{}]; loss: {:.3f}'.format(
it, opt.niter, loss.item())
log_print(log_text, log_dir)
if it % opt.evl_interval == 0 and it:
netG.eval()
gen_feat, gen_label = synthesize_feature_test(netG, dataset, opt)
""" ZSL"""
acc = eval_zsl_knn(gen_feat.numpy(), gen_label.numpy(), dataset)
result_zsl_knn.update(it, acc)
log_print("{}nn Classifer: ".format(opt.Knn), log_dir)
log_print(
"Accuracy is {:.2f}%, Best_acc [{:.2f}% | Iter-{}]".format(
acc, result_zsl_knn.best_acc, result_zsl_knn.best_iter),
log_dir)
""" GZSL"""
# note test label need be shift with offset ntrain_class
train_X = torch.cat((dataset.train_feature, gen_feat), 0)
train_Y = torch.cat(
(dataset.train_label, gen_label + dataset.ntrain_class), 0)
cls = classifier.CLASSIFIER(
train_X, train_Y, dataset,
dataset.ntrain_class + dataset.ntest_class, True,
opt.classifier_lr, 0.5, 25, opt.nSample, True)
result_gzsl_soft.update_gzsl(it, cls.acc_unseen, cls.acc_seen,
cls.H)
log_print("GZSL Softmax:", log_dir)
log_print(
"U->T {:.2f}% S->T {:.2f}% H {:.2f}% Best_H [{:.2f}% {:.2f}% {:.2f}% | Iter-{}]"
.format(cls.acc_unseen, cls.acc_seen, cls.H,
result_gzsl_soft.best_acc_U_T,
result_gzsl_soft.best_acc_S_T,
result_gzsl_soft.best_acc, result_gzsl_soft.best_iter),
log_dir)
if result_zsl_knn.save_model:
files2remove = glob.glob(out_dir + '/Best_model_ZSL_*')
for _i in files2remove:
os.remove(_i)
save_model(
it, netG, train_z, opt.manualSeed, log_text,
out_dir + '/Best_model_ZSL_Acc_{:.2f}.tar'.format(
result_zsl_knn.acc_list[-1]))
if result_gzsl_soft.save_model:
files2remove = glob.glob(out_dir + '/Best_model_GZSL_*')
for _i in files2remove:
os.remove(_i)
save_model(
it, netG, train_z, opt.manualSeed, log_text, out_dir +
'/Best_model_GZSL_H_{:.2f}_S_{:.2f}_U_{:.2f}.tar'.format(
result_gzsl_soft.best_acc,
result_gzsl_soft.best_acc_S_T,
result_gzsl_soft.best_acc_U_T))
netG.train()
if it % opt.save_interval == 0 and it:
save_model(it, netG, train_z, opt.manualSeed, log_text,
out_dir + '/Iter_{:d}.tar'.format(it))
print('Save model to ' + out_dir + '/Iter_{:d}.tar'.format(it))
if opt.cuda:
ones = ones.cuda()
gradients = autograd.grad(outputs=disc_interpolates,
inputs=interpolates,
grad_outputs=ones,
create_graph=True,
retain_graph=True,
only_inputs=True)[0]
gradient_penalty = ((gradients.norm(2, dim=1) - 1)**2).mean() * opt.lambda1
return gradient_penalty
pretrain_cls = classifier.CLASSIFIER(data.train_feature, data.train_label,
opt.nclass_all, opt.resSize, opt.cuda,
0.001, 0.5, 50, 100,
opt.pretrain_classifier)
sim_vector = np.zeros((opt.nepoch, 10, 40))
idex_vector = np.zeros((opt.nepoch, 10, 40))
centroid_feat = np.zeros((opt.nepoch, 10, data.train_feature.shape[1]))
counter = 0
main_seen_dic = {}
main_unseen_dic = {}
best_acc = 0
flag = False
for epoch in range(opt.nepoch):
if epoch > 14 and flag == False and opt.dataset == "AWA1":
opt.cls_weight = float(opt.cls_weight / 2)
def cf_gzsl(test_x, test_l, split):
preds = []
truths = []
test_l_np = test_l.cpu().numpy()
test_l_binary = np.array(
[y in data.unseenclasses for y in test_l])
if additional_train:
gen_sx, gen_sl = generate_syn_feature(
self.netG,
self.data.seenclasses,
self.data.attribute,
100,
netF=self.netF,
netDec=self.netDec,
opt=opt)
#gen_sx = self.conditional_sample(data.train_feature, data.attribute[data.train_label], deterministic=False)
#gen_sx2 = self.conditional_sample(data.train_feature, data.attribute[data.train_label], deterministic=False)
#gen_sx3 = self.conditional_sample(data.train_feature, data.attribute[data.train_label], deterministic=False)
#gen_sx = torch.cat((gen_sx, gen_sx2, gen_sx3), 0)
#gen_sl = torch.cat((data.train_label.cuda(), data.train_label.cuda(), data.train_label.cuda()), 0)
for i in range(test_x.shape[0]):
gen_x, gen_l = self.generate_syn_feature_cf(
test_x[i],
data.unseenclasses,
deterministic=deterministic)
if use_train:
#if additional_train:
# train_x = torch.cat((gen_sx, gen_x), 0)
# train_y = torch.cat((gen_sl, gen_l), 0)
#else:
train_x = torch.cat((data.train_feature, gen_x), 0)
train_y = torch.cat((data.train_label.cuda(), gen_l),
0)
else:
gen_s_x, gen_s_l = self.generate_syn_feature_cf(
test_x[i],
data.seenclasses,
deterministic=deterministic)
train_x = torch.cat((gen_s_x, gen_x), 0)
train_y = torch.cat((gen_s_l, gen_l), 0)
if additional_train:
train_x = torch.cat((train_x, gen_sx), 0)
train_y = torch.cat((train_y, gen_sl.cuda()), 0)
if softmax_clf:
if not binary:
clf = classifier.CLASSIFIER(train_x, train_y, data, self.opt.nclass_all, opt.cuda, opt.classifier_lr, opt.beta1,\
self.epoch, opt.syn_num, generalized=True, netDec=self.cls_netDec, dec_size=opt.attSize,
dec_hidden_size=4096, x=test_x[i], use_tde=use_tde, alpha=self.alpha)
if self.test_logits is None:
self.test_logits = clf.logits
else:
self.test_logits = np.concatenate(
(self.test_logits, clf.logits), axis=0)
else:
clf = BINARY_CLASSIFIER(train_x,
train_y,
data,
2,
True,
opt.classifier_lr,
0.5,
self.epoch,
opt.syn_num,
netDec=self.cls_netDec,
dec_size=opt.attSize,
dec_hidden_size=4096,
use_tde=use_tde,
alpha=self.alpha,
x=test_x[i])
pred = clf.pred
truths.append(test_l_np[i])
preds.append(pred.item())
else:
clf = KNNClassifier(train_x,
train_y,
test_x[i].unsqueeze(0),
self.cls_netDec,
dec_size=opt.attSize,
dec_hidden_size=4096,
batch_size=100)
pred = clf.fit()[0]
preds.append(pred)
truths.append(test_l_np[i])
if (i + 1) % 500 == 0:
if not binary:
binary_acc = self.get_binary_acc(truths, preds)
print("%s-%dth acc: %.3f, binary acc: %.3f" %
(split, i + 1,
cal_macc(truth=truths,
pred=preds), binary_acc))
else:
test_l_binary_t = test_l_binary[:len(preds
)].astype(int)
preds_np = np.array(preds)
acc = (preds_np == test_l_binary_t).mean()
print("%s-%dth binary acc: %.3f" %
(split, i + 1, acc))
if self.opt.sanity:
break # Sanity check
if not binary:
acc = cal_macc(truth=truths, pred=preds)
binary_acc = self.get_binary_acc(truths, preds)
clf_results = {"truths": truths, "preds": preds}
else:
acc = (np.array(preds) == test_l_binary.astype(int)).mean()
binary_acc = acc
clf_results = {"truths": test_l_binary, "preds": preds}
save_file = self.get_save_result_file(split)
if self.log_to_file:
with open(save_file, 'wb') as handle:
pickle.dump(clf_results, handle)
return acc, binary_acc
def gzsl(self,
use_train,
softmax_clf,
cf,
deterministic=False,
additional_train=False,
use_tde=False,
binary=False):
opt = self.opt
data = self.data
if self.siamese:
clf = SiameseClassifier(data,
opt,
self.netE,
self.netG,
self.netF,
self.cls_netDec,
dec_size=opt.attSize,
cf=cf,
n_epochs=opt.clf_epoch,
distance="l1")
if self.netS is None:
clf.train()
else:
clf.network = self.netS
s_acc, u_acc = clf.validate(gzsl=True)
if not cf:
with torch.no_grad():
gen_x, gen_l = generate_syn_feature(self.netG,
self.data.unseenclasses,
self.data.attribute,
opt.syn_num,
netF=self.netF,
netDec=self.netDec,
opt=opt)
if use_train:
train_x = torch.cat((data.train_feature, gen_x), 0)
train_y = torch.cat((data.train_label, gen_l), 0)
else:
with torch.no_grad():
gen_s_x, gen_s_l = generate_syn_feature(
self.netG,
self.data.seenclasses,
self.data.attribute,
opt.syn_num,
netF=self.netF,
netDec=self.netDec,
opt=opt)
train_x = torch.cat((gen_s_x, gen_x), 0)
train_y = torch.cat((gen_s_l, gen_l), 0)
if softmax_clf:
if not binary:
gzsl_cls = classifier.CLASSIFIER(train_x, train_y, \
data, data.allclasses.size(0), opt.cuda, opt.classifier_lr, 0.5, self.epoch, opt.syn_num,
generalized=True, netDec=self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096,
use_tde=use_tde, alpha=self.alpha)
self.test_logits = gzsl_cls.all_outputs
else:
gzsl_cls = BINARY_CLASSIFIER(train_x,
train_y,
data,
2,
True,
opt.classifier_lr,
0.5,
self.epoch,
opt.syn_num,
netDec=self.cls_netDec,
dec_size=opt.attSize,
dec_hidden_size=4096,
use_tde=use_tde,
alpha=self.alpha)
s_acc = gzsl_cls.acc_seen
u_acc = gzsl_cls.acc_unseen
h_acc = gzsl_cls.H
self.s_bacc = gzsl_cls.s_bacc
self.u_bacc = gzsl_cls.u_bacc
if not binary:
clf_results = {"preds": gzsl_cls.pred_s.cpu().numpy()}
save_file = self.get_save_result_file("seen")
if self.log_to_file and not binary:
with open(save_file, 'wb') as handle:
pickle.dump(clf_results, handle)
if not binary:
clf_results = {"preds": gzsl_cls.pred_u.cpu().numpy()}
save_file = self.get_save_result_file("unseen")
if self.log_to_file and not binary:
with open(save_file, 'wb') as handle:
pickle.dump(clf_results, handle)
else:
u_cls = KNNClassifier(train_x,
train_y,
data.test_unseen_feature,
self.cls_netDec,
dec_size=opt.attSize,
dec_hidden_size=4096,
batch_size=100)
preds = u_cls.fit()
truths = data.test_unseen_label.cpu().numpy()
u_acc = cal_macc(truth=truths, pred=preds)
s_cls = KNNClassifier(train_x,
train_y,
data.test_seen_feature,
self.cls_netDec,
dec_size=opt.attSize,
dec_hidden_size=4096,
batch_size=100)
preds = s_cls.fit()
truths = data.test_seen_label.cpu().numpy()
s_acc = cal_macc(truth=truths, pred=preds)
h_acc = 2 * u_acc * s_acc / (u_acc + s_acc)
else:
self.test_logits = None
def cf_gzsl(test_x, test_l, split):
preds = []
truths = []
test_l_np = test_l.cpu().numpy()
test_l_binary = np.array(
[y in data.unseenclasses for y in test_l])
if additional_train:
gen_sx, gen_sl = generate_syn_feature(
self.netG,
self.data.seenclasses,
self.data.attribute,
100,
netF=self.netF,
netDec=self.netDec,
opt=opt)
#gen_sx = self.conditional_sample(data.train_feature, data.attribute[data.train_label], deterministic=False)
#gen_sx2 = self.conditional_sample(data.train_feature, data.attribute[data.train_label], deterministic=False)
#gen_sx3 = self.conditional_sample(data.train_feature, data.attribute[data.train_label], deterministic=False)
#gen_sx = torch.cat((gen_sx, gen_sx2, gen_sx3), 0)
#gen_sl = torch.cat((data.train_label.cuda(), data.train_label.cuda(), data.train_label.cuda()), 0)
for i in range(test_x.shape[0]):
gen_x, gen_l = self.generate_syn_feature_cf(
test_x[i],
data.unseenclasses,
deterministic=deterministic)
if use_train:
#if additional_train:
# train_x = torch.cat((gen_sx, gen_x), 0)
# train_y = torch.cat((gen_sl, gen_l), 0)
#else:
train_x = torch.cat((data.train_feature, gen_x), 0)
train_y = torch.cat((data.train_label.cuda(), gen_l),
0)
else:
gen_s_x, gen_s_l = self.generate_syn_feature_cf(
test_x[i],
data.seenclasses,
deterministic=deterministic)
train_x = torch.cat((gen_s_x, gen_x), 0)
train_y = torch.cat((gen_s_l, gen_l), 0)
if additional_train:
train_x = torch.cat((train_x, gen_sx), 0)
train_y = torch.cat((train_y, gen_sl.cuda()), 0)
if softmax_clf:
if not binary:
clf = classifier.CLASSIFIER(train_x, train_y, data, self.opt.nclass_all, opt.cuda, opt.classifier_lr, opt.beta1,\
self.epoch, opt.syn_num, generalized=True, netDec=self.cls_netDec, dec_size=opt.attSize,
dec_hidden_size=4096, x=test_x[i], use_tde=use_tde, alpha=self.alpha)
if self.test_logits is None:
self.test_logits = clf.logits
else:
self.test_logits = np.concatenate(
(self.test_logits, clf.logits), axis=0)
else:
clf = BINARY_CLASSIFIER(train_x,
train_y,
data,
2,
True,
opt.classifier_lr,
0.5,
self.epoch,
opt.syn_num,
netDec=self.cls_netDec,
dec_size=opt.attSize,
dec_hidden_size=4096,
use_tde=use_tde,
alpha=self.alpha,
x=test_x[i])
pred = clf.pred
truths.append(test_l_np[i])
preds.append(pred.item())
else:
clf = KNNClassifier(train_x,
train_y,
test_x[i].unsqueeze(0),
self.cls_netDec,
dec_size=opt.attSize,
dec_hidden_size=4096,
batch_size=100)
pred = clf.fit()[0]
preds.append(pred)
truths.append(test_l_np[i])
if (i + 1) % 500 == 0:
if not binary:
binary_acc = self.get_binary_acc(truths, preds)
print("%s-%dth acc: %.3f, binary acc: %.3f" %
(split, i + 1,
cal_macc(truth=truths,
pred=preds), binary_acc))
else:
test_l_binary_t = test_l_binary[:len(preds
)].astype(int)
preds_np = np.array(preds)
acc = (preds_np == test_l_binary_t).mean()
print("%s-%dth binary acc: %.3f" %
(split, i + 1, acc))
if self.opt.sanity:
break # Sanity check
if not binary:
acc = cal_macc(truth=truths, pred=preds)
binary_acc = self.get_binary_acc(truths, preds)
clf_results = {"truths": truths, "preds": preds}
else:
acc = (np.array(preds) == test_l_binary.astype(int)).mean()
binary_acc = acc
clf_results = {"truths": test_l_binary, "preds": preds}
save_file = self.get_save_result_file(split)
if self.log_to_file:
with open(save_file, 'wb') as handle:
pickle.dump(clf_results, handle)
return acc, binary_acc
s_acc, s_bacc = cf_gzsl(data.test_seen_feature,
data.test_seen_label, "seen")
u_acc, u_bacc = cf_gzsl(data.test_unseen_feature,
data.test_unseen_label, "unseen")
# s_acc = 0.3
if u_acc + s_acc == 0:
h_acc = 0
else:
h_acc = 2 * u_acc * s_acc / (u_acc + s_acc)
self.s_bacc = s_bacc
self.u_bacc = u_bacc
return s_acc, u_acc, h_acc
def two_stage(self,
use_mask,
use_tde,
seen_mask=None,
unseen_mask=None,
save_clf=False):
opt = self.opt
data = self.data
# Unseen:
if unseen_mask is None:
save_file = "out/%s-unseen.pickle" % use_mask
with open(save_file, 'rb') as handle:
clf_results = pickle.load(handle)
preds = clf_results["preds"]
mask = [pred in self.data.unseenclasses for pred in preds]
mask = torch.from_numpy(np.array(mask).astype(int))
else:
mask = unseen_mask
with torch.no_grad():
gen_x, gen_l = generate_syn_feature(self.netG,
self.data.unseenclasses,
self.data.attribute,
opt.u_num,
netF=self.netF,
netDec=self.netDec,
opt=opt)
if not save_clf or self.zsl_cls is None:
zsl_cls = classifier.CLASSIFIER(gen_x, util.map_label(gen_l, data.unseenclasses), \
data, data.unseenclasses.size(0), opt.cuda, opt.u_lr, opt.u_beta, opt.u_epoch, opt.u_batch_size, \
generalized=False, netDec=self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, mask=mask,
use_tde=False, alpha=self.alpha)
u_acc = zsl_cls.acc
else:
zsl_cls = self.zsl_cls
u_acc = zsl_cls.val(zsl_cls.test_unseen_feature,
zsl_cls.test_unseen_label,
zsl_cls.unseenclasses, mask)
if save_clf:
self.zsl_cls = zsl_cls
# Seen:
if seen_mask is None:
save_file = "out/%s-seen.pickle" % use_mask
with open(save_file, 'rb') as handle:
clf_results = pickle.load(handle)
preds = clf_results["preds"]
mask = [pred in self.data.seenclasses for pred in preds]
mask = torch.from_numpy(np.array(mask).astype(int))
else:
mask = seen_mask
if not opt.adjust_s:
zsl_cls = classifier.CLASSIFIER(data.train_feature, util.map_label(data.train_label, data.seenclasses), \
data, data.seenclasses.size(0), opt.cuda, opt.classifier_lr, 0.5, 15, opt.syn_num, \
generalized=False, netDec=self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, mask=mask, zsl_on_seen=True,
use_tde=use_tde, alpha=self.alpha)
else:
zsl_cls = classifier.CLASSIFIER(data.train_feature, util.map_label(data.train_label, data.seenclasses), \
data, data.seenclasses.size(0), opt.cuda, opt.s_lr, opt.s_beta, opt.s_epoch, opt.s_batch_size, \
generalized=False, netDec=self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096, mask=mask, zsl_on_seen=True,
use_tde=use_tde, alpha=self.alpha)
s_acc = zsl_cls.acc
h_acc = 2 * u_acc * s_acc / (u_acc + s_acc)
if opt.log_two_stage:
out_dir = "results/two_stage/%s/%s" % (opt.dataset, self.exp_name)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
mask_list = use_mask.split('/')
mask_name = "%s_%s" % (mask_list[0], mask_list[1])
out_file = os.path.join(out_dir, "%s.txt" % mask_name)
config_msg = "u_num:%d u_lr:%.4f u_beta:%.2f u_epoch:%d u_batch_size:%d" % (
opt.u_num, opt.u_lr, opt.u_beta, opt.u_epoch, opt.u_batch_size)
if self.opt.adjust_s:
config_msg += " s_lr:%.4f s_beta:%.2f s_epoch:%d s_bs:%d" % (
opt.s_lr, opt.s_beta, opt.s_epoch, opt.s_batch_size)
log_msg = "%s---S:%.3f U:%.3f H:%.3f\n" % (config_msg, s_acc,
u_acc, h_acc)
with open(out_file, "a") as f:
f.write(log_msg)
return s_acc, u_acc, h_acc
def zsl(self, softmax_clf, cf, deterministic=False):
opt = self.opt
data = self.data
if not cf:
with torch.no_grad():
gen_x, gen_l = generate_syn_feature(self.netG,
self.data.unseenclasses,
self.data.attribute,
opt.syn_num,
netF=self.netF,
netDec=self.netDec,
opt=opt)
if softmax_clf:
zsl_cls = classifier.CLASSIFIER(gen_x, util.map_label(gen_l, data.unseenclasses), \
data, data.unseenclasses.size(0), opt.cuda, opt.classifier_lr, 0.5, self.epoch, opt.syn_num, \
generalized=False, netDec=self.cls_netDec, dec_size=opt.attSize, dec_hidden_size=4096)
acc = zsl_cls.acc
else:
zsl_cls = KNNClassifier(gen_x,
gen_l,
data.test_unseen_feature,
self.cls_netDec,
dec_size=opt.attSize,
dec_hidden_size=4096,
batch_size=100)
preds = zsl_cls.fit()
truths = data.test_unseen_label.cpu().numpy()
acc = cal_macc(truth=truths, pred=preds)
else:
preds = []
truths = []
test_x = data.test_unseen_feature
mapped_unseen_l = util.map_label(data.test_unseen_label,
data.unseenclasses)
unseen_label_np = data.test_unseen_label.cpu().numpy()
for i in range(test_x.shape[0]):
gen_x, gen_l = self.generate_syn_feature_cf(
test_x[i], data.unseenclasses, deterministic=deterministic)
gen_l = util.map_label(gen_l, data.unseenclasses)
if softmax_clf:
clf = classifier.CLASSIFIER(gen_x,
gen_l,
data,
data.unseenclasses.size(0),
opt.cuda,
opt.classifier_lr,
0.5,
self.epoch,
opt.syn_num,
generalized=False,
netDec=self.cls_netDec,
dec_size=opt.attSize,
dec_hidden_size=4096,
x=test_x[i])
pred = clf.pred
truths.append(mapped_unseen_l[i])
preds.append(pred)
else:
clf = KNNClassifier(gen_x,
gen_l,
test_x[i].unsqueeze(0),
self.cls_netDec,
dec_size=opt.attSize,
dec_hidden_size=4096,
batch_size=100)
pred = clf.fit()[0]
preds.append(pred)
truths.append(unseen_label_np[i])
if (i + 1) % 500 == 0:
print("%dth acc: %.3f" %
(i + 1, cal_macc(truth=truths, pred=preds)))
if self.opt.sanity:
break # Sanity check
acc = cal_macc(truth=truths, pred=preds)
return acc
