def main():
test_dir = opt.test_dir
feature_param_file = opt.feat
class_param_file = opt.cls
bsize = opt.b
# models
if 'vgg' == opt.i:
feature = Vgg16()
elif 'resnet' == opt.i:
feature = resnet50()
elif 'densenet' == opt.i:
feature = densenet121()
feature.cuda()
# feature.load_state_dict(torch.load(feature_param_file))
feature.eval()
classifier = Classifier(opt.i)
classifier.cuda()
# classifier.load_state_dict(torch.load(class_param_file))
classifier.eval()
loader = torch.utils.data.DataLoader(MyClsTestData(test_dir,
transform=True),
batch_size=bsize,
shuffle=True,
num_workers=4,
pin_memory=True)
acc = eval_acc(feature, classifier, loader)
print acc
def test(config):
dataset = img_dataset("./dataset/test", "test")
dataloader = torch.utils.data.DataLoader(
dataset=dataset, batch_size=config.bs
)
net = Classifier(num_classes=13).cuda()
net.load_state_dict(
torch.load(
join(
f"{config.weight_path}", f"{config.test_epoch}_classifier.pth"
)
)
)
net.eval()
with open("output.csv", "w", newline="") as csvfile:
writer = csv.writer(csvfile, delimiter=",")
writer.writerow(["id", "label"])
for _, data in enumerate(dataloader):
with torch.no_grad():
inputs, img_name = data
inputs = inputs.cuda()
outputs = net(inputs)
_, predicted = torch.max(outputs.data, 1)
writer.writerow([img_name[0], class_name[predicted.data]])
def main():
args = docopt(__doc__)
enable_all_pools = args['--enable-all-pools']
hidden = int(args['--hidden'])
dropout = float(args['--dropout'])
device = torch.device(int(args['--device']))
print(f"{device} will be used")
ratio = 0.8
valid_dset = QueryDataset(split='valid',
ratio=ratio,
equally_handle_foreign_authors=False)
valid_loader = DataLoader(valid_dset,
batch_size=1,
num_workers=1,
shuffle=False)
embedding_mode, embedding = load_embedding(args['--embedding'], False,
device)
classifier = Classifier(embedding,
hidden,
dropout,
args['--deepset'],
equally_handle_foreign_authors=False,
enable_all_pools=enable_all_pools)
classifier.load_state_dict(torch.load(args['--classifier']))
classifier.eval()
if torch.cuda.is_available():
classifier.to(device)
thresholds = [0.05 * i for i in range(1, 20)]
for thres in thresholds:
test_classifier(valid_loader, classifier, device, thres)
def predict(self, img_path, model_path, visualize=False):
model = Classifier()
ckpt = torch.load(model_path)
model.load_state_dict(ckpt)
model = model.to(self.device)
model.eval()
images = [f for f in os.listdir(img_path) if f.endswith('.png')]
success = 0
y_true, y_pred = [], []
for imgname in images:
image = cv2.imread(os.path.join(img_path, imgname))
img = image.copy() / 255.0
img = np.transpose(img, (2, 0, 1))
img = np.expand_dims(img, axis=0)
img = torch.from_numpy(img).to(self.device).float()
pred_label = model(img).cpu().detach().numpy()
if ("Lateral" in imgname) and (pred_label < 0.5):
success += 1
elif ("AP" in imgname) and (pred_label > 0.5):
success += 1
true_label = "Lateral" if "Lateral" in imgname else "AP"
pred_label = "Lateral" if pred_label < 0.5 else "AP"
y_true.append(true_label)
y_pred.append(pred_label)
if visualize:
cv2.putText(image, "pred: {}".format(pred_label), (20, 20),
cv2.FONT_HERSHEY_PLAIN, 1, (0, 0, 255), 2)
cv2.putText(image, "true: {}".format(true_label), (20, 40),
cv2.FONT_HERSHEY_PLAIN, 1, (0, 255, 0), 2)
cv2.imshow('predicted', image)
cv2.waitKey(0)
print("[INFO] accuracy is : {:.2f} %".format(100 * success /
len(images)))
cm = confusion_matrix(y_true, y_pred)
print(
classification_report(y_true,
y_pred,
target_names=["AP", "Lateral"]))
#print("[INFO] confusion matrix:\n {}".format(cm))
disp = ConfusionMatrixDisplay(confusion_matrix=cm,
display_labels=("AP", "Lateral")).plot()
plt.show()
def train(trainloader, validloader):
criterion = nn.CrossEntropyLoss()
model = Classifier().cuda()
optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9) # SGD
# scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)
# for i in range(50):
# for para in optimizer.param_groups:
# scheduler.step()
# print(para['lr'])
# exit()
for epoch in range(80):
print('Epoch', epoch)
train_loss = 0
for batch, (x, label) in enumerate(trainloader, 1):
x, label = x.cuda(), label.cuda()
model.train()
model.zero_grad()
output = model(x)
loss = criterion(output, label)
train_loss += loss.item()
loss.backward()
optimizer.step()
if batch % 300 == 0:
print('Loss: {}'.format(train_loss / batch))
# scheduler.step()
with torch.no_grad():
total_loss = 0
model.eval()
correct = 0
total = 0
for batch, (x, label) in enumerate(validloader, 1):
x, label = x.cuda(), label.cuda()
output = model(x)
loss = criterion(output, label)
total_loss += loss.item()
pred = output.max(dim=1)[1]
correct += (pred == label).sum().item()
total += pred.size(0)
print('EPOCH: {}, validloss: {}, Acc: {}'.format(
epoch, total_loss / batch, correct / total))
def main():
args = docopt(__doc__)
device = torch.device(int(args['--device']))
print(f"{device} will be used")
threshold = float(args['--threshold'])
answer_path = args['--answer-path']
query_path = args['--query-path']
hidden = int(args['--hidden'])
dropout = float(args['--dropout'])
enable_all_pools = args['--enable-all-pools']
if os.path.exists(answer_path):
warnings.warn(
'Answer file already exists. Please delete it before run the code. Otherwise, lines will be appended.'
)
testset = QueryTestset(query_path)
testloader = DataLoader(testset,
batch_size=1,
num_workers=1,
shuffle=False)
embedding_mode, embedding = load_embedding(args['--embedding'], None,
device)
classifier = Classifier(embedding,
hidden,
dropout,
args['--deepset'],
False,
enable_all_pools=enable_all_pools).to(device)
classifier.load_state_dict(torch.load(args['--classifier']))
classifier.eval()
with torch.no_grad():
for collab in testloader:
score = classifier(collab.to(device))
if score >= threshold:
answer = True
else:
answer = False
with open(answer_path, 'a') as f:
f.write(str(answer) + '\n')
def train(self, epochs, batchsize, output_model):
# setup model
torch.manual_seed(42) # for reproducibility!
model = Classifier()
model = model.to(device=self.device)
# setup data loader
trn_dataset = XrayDataset(data_path="./Train", augment=True)
val_dataset = XrayDataset(data_path='./Test', augment=False)
trn_loader = DataLoader(trn_dataset,
batch_size=batchsize,
shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=1)
# setup loss
criterion = nn.BCELoss()
# setup optimizer
optimizer = optim.Adam(model.parameters(), lr=0.0001)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=8)
# setup trainer
best_loss = np.inf
trn_loss = []
val_loss = []
model.train()
for epoch in range(epochs):
pbar = tqdm(desc='Epoch {:02d}/{}'.format(epoch + 1, epochs),
total=len(trn_loader),
bar_format='{l_bar}{bar:10}{r_bar}{bar:-10b}')
losses = []
for idx, sample in enumerate(trn_loader):
image, label = sample
image, label = image.to(self.device).float(), label.to(
self.device).float()
optimizer.zero_grad()
pred_label = model(image)
loss = criterion(pred_label, label)
losses.append(loss.item())
loss.backward()
optimizer.step()
pbar.update(1)
trn_loss.append(np.mean(losses))
# evaluate model on val set
model.eval()
with torch.no_grad():
val_losses = []
for idx, sample in enumerate(val_loader):
image, label = sample
image, label = image.to(self.device).float(), label.to(
self.device).float()
pred_label = model(image)
loss = criterion(pred_label, label)
val_losses.append(loss.item())
val_loss.append(np.mean(val_losses))
pbar.set_postfix(trn_loss=np.mean(losses),
val_loss=np.mean(val_losses))
pbar.close()
# save model
torch.save(model.state_dict(), output_model)
model.train()
scheduler.step()
# plot the training curves
plt.plot(np.arange(epochs), trn_loss, 'b-', label='train')
plt.plot(np.arange(epochs), val_loss, 'r-', label='validation')
plt.legend(loc=1)
plt.show()
class ModelBuilder(object):
def __init__(self, use_cuda, conf):
self.cuda = use_cuda
self.conf = conf
self._pre_data()
self._build_model()
def _pre_data(self):
print('pre data...')
self.data = Data(self.cuda, self.conf)
def _build_model(self):
print('loading embedding...')
if self.conf.corpus_splitting == 1:
pre = './data/processed/lin/'
elif self.conf.corpus_splitting == 2:
pre = './data/processed/ji/'
elif self.conf.corpus_splitting == 3:
pre = './data/processed/l/'
we = torch.load(pre + 'we.pkl')
char_table = None
sub_table = None
if self.conf.need_char or self.conf.need_elmo:
char_table = torch.load(pre + 'char_table.pkl')
if self.conf.need_sub:
sub_table = torch.load(pre + 'sub_table.pkl')
print('building model...')
self.encoder = ArgEncoder(self.conf, we, char_table, sub_table,
self.cuda)
self.classifier = Classifier(self.conf.clf_class_num, self.conf)
if self.conf.is_mttrain:
self.conn_classifier = Classifier(self.conf.conn_num, self.conf)
if self.cuda:
self.encoder.cuda()
self.classifier.cuda()
if self.conf.is_mttrain:
self.conn_classifier.cuda()
self.criterion = torch.nn.CrossEntropyLoss()
para_filter = lambda model: filter(lambda p: p.requires_grad,
model.parameters())
self.e_optimizer = torch.optim.Adagrad(
para_filter(self.encoder),
self.conf.lr,
weight_decay=self.conf.l2_penalty)
self.c_optimizer = torch.optim.Adagrad(
para_filter(self.classifier),
self.conf.lr,
weight_decay=self.conf.l2_penalty)
if self.conf.is_mttrain:
self.con_optimizer = torch.optim.Adagrad(
para_filter(self.conn_classifier),
self.conf.lr,
weight_decay=self.conf.l2_penalty)
def _print_train(self, epoch, time, loss, acc):
print('-' * 80)
print(
'| end of epoch {:3d} | time: {:5.2f}s | loss: {:10.5f} | acc: {:5.2f}% |'
.format(epoch, time, loss, acc * 100))
print('-' * 80)
def _print_eval(self, task, loss, acc, f1):
print('| ' + task +
' loss {:10.5f} | acc {:5.2f}% | f1 {:5.2f}%'.format(
loss, acc * 100, f1 * 100))
print('-' * 80)
def _save_model(self, model, filename):
torch.save(model.state_dict(), './weights/' + filename)
def _load_model(self, model, filename):
model.load_state_dict(torch.load('./weights/' + filename))
def _train_one(self):
self.encoder.train()
self.classifier.train()
if self.conf.is_mttrain:
self.conn_classifier.train()
total_loss = 0
correct_n = 0
train_size = self.data.train_size
for a1, a2, sense, conn in self.data.train_loader:
if self.conf.four_or_eleven == 2:
mask1 = (sense == self.conf.binclass)
mask2 = (sense != self.conf.binclass)
sense[mask1] = 1
sense[mask2] = 0
if self.cuda:
a1, a2, sense, conn = a1.cuda(), a2.cuda(), sense.cuda(
), conn.cuda()
a1, a2, sense, conn = Variable(a1), Variable(a2), Variable(
sense), Variable(conn)
repr = self.encoder(a1, a2)
output = self.classifier(repr)
_, output_sense = torch.max(output, 1)
assert output_sense.size() == sense.size()
tmp = (output_sense == sense).long()
correct_n += torch.sum(tmp).data
loss = self.criterion(output, sense)
if self.conf.is_mttrain:
conn_output = self.conn_classifier(repr)
loss2 = self.criterion(conn_output, conn)
loss = loss + loss2 * self.conf.lambda1
self.e_optimizer.zero_grad()
self.c_optimizer.zero_grad()
if self.conf.is_mttrain:
self.con_optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm(self.encoder.parameters(),
self.conf.grad_clip)
torch.nn.utils.clip_grad_norm(self.classifier.parameters(),
self.conf.grad_clip)
if self.conf.is_mttrain:
torch.nn.utils.clip_grad_norm(
self.conn_classifier.parameters(), self.conf.grad_clip)
self.e_optimizer.step()
self.c_optimizer.step()
if self.conf.is_mttrain:
self.con_optimizer.step()
total_loss += loss.data * sense.size(0)
return total_loss[0] / train_size, correct_n[0] / train_size
def _train(self, pre):
for epoch in range(self.conf.epochs):
start_time = time.time()
loss, acc = self._train_one()
self._print_train(epoch, time.time() - start_time, loss, acc)
self.logwriter.add_scalar('loss/train_loss', loss, epoch)
self.logwriter.add_scalar('acc/train_acc', acc * 100, epoch)
dev_loss, dev_acc, dev_f1 = self._eval('dev')
self._print_eval('dev', dev_loss, dev_acc, dev_f1)
self.logwriter.add_scalar('loss/dev_loss', dev_loss, epoch)
self.logwriter.add_scalar('acc/dev_acc', dev_acc * 100, epoch)
self.logwriter.add_scalar('f1/dev_f1', dev_f1 * 100, epoch)
test_loss, test_acc, test_f1 = self._eval('test')
self._print_eval('test', test_loss, test_acc, test_f1)
self.logwriter.add_scalar('loss/test_loss', test_loss, epoch)
self.logwriter.add_scalar('acc/test_acc', test_acc * 100, epoch)
self.logwriter.add_scalar('f1/test_f1', test_f1 * 100, epoch)
def train(self, pre):
print('start training')
self.logwriter = SummaryWriter(self.conf.logdir)
self._train(pre)
print('training done')
def _eval(self, task):
self.encoder.eval()
self.classifier.eval()
total_loss = 0
correct_n = 0
if task == 'dev':
data = self.data.dev_loader
n = self.data.dev_size
elif task == 'test':
data = self.data.test_loader
n = self.data.test_size
else:
raise Exception('wrong eval task')
output_list = []
gold_list = []
for a1, a2, sense1, sense2 in data:
if self.conf.four_or_eleven == 2:
mask1 = (sense1 == self.conf.binclass)
mask2 = (sense1 != self.conf.binclass)
sense1[mask1] = 1
sense1[mask2] = 0
mask0 = (sense2 == -1)
mask1 = (sense2 == self.conf.binclass)
mask2 = (sense2 != self.conf.binclass)
sense2[mask1] = 1
sense2[mask2] = 0
sense2[mask0] = -1
if self.cuda:
a1, a2, sense1, sense2 = a1.cuda(), a2.cuda(), sense1.cuda(
), sense2.cuda()
a1 = Variable(a1, volatile=True)
a2 = Variable(a2, volatile=True)
sense1 = Variable(sense1, volatile=True)
sense2 = Variable(sense2, volatile=True)
output = self.classifier(self.encoder(a1, a2))
_, output_sense = torch.max(output, 1)
assert output_sense.size() == sense1.size()
gold_sense = sense1
mask = (output_sense == sense2)
gold_sense[mask] = sense2[mask]
tmp = (output_sense == gold_sense).long()
correct_n += torch.sum(tmp).data
output_list.append(output_sense)
gold_list.append(gold_sense)
loss = self.criterion(output, gold_sense)
total_loss += loss.data * gold_sense.size(0)
output_s = torch.cat(output_list)
gold_s = torch.cat(gold_list)
if self.conf.four_or_eleven == 2:
f1 = f1_score(gold_s.cpu().data.numpy(),
output_s.cpu().data.numpy(),
average='binary')
else:
f1 = f1_score(gold_s.cpu().data.numpy(),
output_s.cpu().data.numpy(),
average='macro')
return total_loss[0] / n, correct_n[0] / n, f1
def eval(self, pre):
print('evaluating...')
self._load_model(self.encoder, pre + '_eparams.pkl')
self._load_model(self.classifier, pre + '_cparams.pkl')
test_loss, test_acc, f1 = self._eval('test')
self._print_eval('test', test_loss, test_acc, f1)
for j in range(bSize):
conf[(predClass[j], labels[j])] += 1
bar.update(i)
bar.finish()
print("Epoch [%d] Training Loss: %.4f Training Acc: %.2f" %
(epoch + 1, running_loss / (i + 1), running_acc / (imgCnt)))
#ploter.plot("loss", "train", epoch+1, running_loss/(i+1))
running_loss = 0.0
running_acc = 0.0
imgCnt = 0
conf = torch.zeros(numClass, numClass).long()
modelConv.eval()
modelClass.eval()
modelHess.eval()
bar = progressbar.ProgressBar(0, len(valloader), redirect_stdout=False)
for i, (images, labels) in enumerate(valloader):
if torch.cuda.is_available():
images = images.float().cuda()
labels = labels.cuda()
if hessL or hessMC:
pred = torch.squeeze(modelHess(images))
else:
final = modelConv(images)[1]
pred = torch.squeeze(modelClass(final))
loss = criterion(pred, labels)
bSize = images.size()[0]
running_loss = 0.0
loss_train = []
print('Finished training for epoch ' + str(epoch) + ' time taken = ' +
str(time.time() - start_time))
file.write('Finished training for epoch ' + str(epoch) + ' time taken = ' +
str(time.time() - start_time) + '\n')
file.write(
'##################################evaluation##############################\n'
)
print(
'################################evaluation###########################\n'
)
with torch.no_grad():
val_loss = 0
net.eval()
for i, data in enumerate(test_loader, 0):
step += 1
inputs, labels = data[0].to(device), data[1].to(device)
outputs = net(inputs)
loss = criterion(outputs, labels)
loss_val.append(loss.item())
validation_loss_store.append([epoch, loss.item()])
val_loss += loss
val_loss = val_loss / float(i + 1)
if val_loss < min_loss:
min_loss = val_loss
no_impr_epoch = 0
class Gzsl_vae():
"""docstring for Gzsl_vae"""
def __init__(self, ):
args = parser.parse_args()
self.device = torch.device(args.device)
######################## LOAD DATA #############################
self.trainval_set = dataloader(root=args.dataset_path,
split='trainval',
device=self.device)
#train_set = dataloader(root=args.dataset_path,split='train', device=self.device)
self.test_set = dataloader(root=args.dataset_path,
split='test_unseen',
device=self.device)
self.test_set_seen = dataloader(root=args.dataset_path,
split='test_seen',
device=self.device)
#val_set = dataloader(root=args.dataset_path,split='val', device=self.device)
self.trainloader = data.DataLoader(self.trainval_set,
batch_size=args.batch_size,
shuffle=True)
#self.testloader = data.DataLoader(self.test_set, batch_size=args.batch_size, shuffle=False)
self.input_dim = self.trainval_set.__getlen__()
self.atts_dim = self.test_set.__get_attlen__()
self.num_classes = self.trainval_set.__totalClasses__()
print(20 * ('-'))
print("Input_dimension=%d" % self.input_dim)
print("Attribute_dimension=%d" % self.atts_dim)
print("z=%d" % args.latent_size)
print("num_classes=%d" % self.num_classes)
print(20 * ('-'))
####################### INITIALIZE THE MODEL AND OPTIMIZER #####################
self.model_encoder = encoder_cada(input_dim=self.input_dim,
atts_dim=self.atts_dim,
z=args.latent_size).to(self.device)
self.model_decoder = decoder_cada(input_dim=self.input_dim,
atts_dim=self.atts_dim,
z=args.latent_size).to(self.device)
learnable_params = list(self.model_encoder.parameters()) + list(
self.model_decoder.parameters())
self.optimizer = optim.Adam(learnable_params,
lr=0.00015,
betas=(0.9, 0.999),
eps=1e-08,
weight_decay=0,
amsgrad=True)
self.classifier = Classifier(input_dim=args.latent_size,
num_class=self.num_classes)
self.cls_optimizer = optim.Adam(self.classifier.parameters(),
lr=0.001,
betas=(0.5, 0.999))
print(self.model_encoder)
print(self.model_decoder)
print(self.classifier)
########## LOSS ############
self.l1_loss = nn.L1Loss(reduction='sum')
self.lossfunction_classifier = nn.NLLLoss()
self.gamma = torch.zeros(1, device=self.device).float()
self.beta = torch.zeros(1, device=self.device).float()
self.delta = torch.zeros(1, device=self.device).float()
def train(self, epoch):
'''
This function trains the cada_vae model
'''
if epoch > 5 and epoch < 21:
self.delta += 0.54
if epoch > 20 and epoch < 75:
self.gamma += 0.044
if epoch < 91:
self.beta += 0.0026
trainbar = tqdm(self.trainloader)
self.model_encoder.train()
self.model_decoder.train()
train_loss = 0
for batch_idx, (x, y, sig) in enumerate(trainbar):
z_img, z_sig, mu_x, logvar_x, mu_sig, logvar_sig = self.model_encoder(
x, sig)
recon_x, recon_sig, sigDecoder_x, xDecoder_sig = self.model_decoder(
z_img, z_sig)
#loss
vae_reconstruction_loss = self.l1_loss(recon_x, x) + self.l1_loss(
recon_sig, sig)
cross_reconstruction_loss = self.l1_loss(
xDecoder_sig, x) + self.l1_loss(sigDecoder_x, sig)
KLD_loss = (
0.5 * torch.sum(1 + logvar_x - mu_x.pow(2) - logvar_x.exp())
) + (0.5 *
torch.sum(1 + logvar_sig - mu_sig.pow(2) - logvar_sig.exp()))
distributed_loss = torch.sqrt(
torch.sum((mu_x - mu_sig)**2, dim=1) +
torch.sum((torch.sqrt(logvar_x.exp()) -
torch.sqrt(logvar_sig.exp()))**2,
dim=1))
distributed_loss = distributed_loss.sum()
self.optimizer.zero_grad()
loss = vae_reconstruction_loss - self.beta * KLD_loss
if self.delta > 0:
loss += distributed_loss * self.delta
if self.gamma > 0:
loss += cross_reconstruction_loss * self.gamma
loss.backward()
self.optimizer.step()
train_loss += loss.item()
trainbar.set_description('l:%.3f' % (train_loss / (batch_idx + 1)))
#print("vae %f, da %f, ca %f"%(vae,da,ca))
print(train_loss / (batch_idx + 1))
if epoch % 100 == 0:
name = "models/checkpoint_cada.pth"
torch.save(
{
'epoch': epoch,
'model_encoder_state_dict':
self.model_encoder.state_dict(),
'model_decoder_state_dict':
self.model_decoder.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'loss': loss,
}, name)
def extract_features(self, split):
'''
This function is used to extract the features from the encoder network
'''
features_img = []
features_att = []
gt = []
self.model_encoder.eval()
if split == 'trainval':
num_classes = self.trainval_set.__NumClasses__()
if split == 'test_unseen':
num_classes = self.test_set.__NumClasses__()
if split == 'test_seen':
num_classes = self.test_set_seen.__NumClasses__()
for n in num_classes:
# 50 latent features per seen classes and 100 latent features per unseen classes
if split == 'trainval':
num_features = 50
indexs = self.trainval_set.__getLabels__(n)
dataset = self.trainval_set
if split == 'test_unseen':
num_features = 100
indexs = self.test_set.__getLabels__(n)
dataset = self.test_set
if split == 'test_seen':
num_features = 100
indexs = self.test_set_seen.__getLabels__(n)
dataset = self.test_set_seen
index_of_classLabels = indexs.tolist()
if len(index_of_classLabels) < num_features:
j = num_features - len(index_of_classLabels)
index_of_classLabels = index_of_classLabels[:j] + index_of_classLabels
else:
index_of_classLabels = index_of_classLabels[:num_features]
#print(len(index_of_classLabels))
x_list = []
y_list = []
sig_list = []
with torch.no_grad():
for i in index_of_classLabels:
x, y, sig = dataset.__getitem__(i)
x_list.append(torch.unsqueeze(x, dim=0))
y_list.append(torch.unsqueeze(y, dim=0))
sig_list.append(torch.unsqueeze(sig, dim=0))
x_tensor = torch.cat(x_list)
y_tensor = torch.cat(y_list)
sig_tensor = torch.cat(sig_list)
z_imgs, z_atts, mu_imgs, logvar_imgs, mu_att, logvar_att = self.model_encoder(
x_tensor, sig_tensor)
features_img.extend(torch.unsqueeze(
z_imgs.detach(), dim=0)) #make it a torch tensor
features_att.extend(torch.unsqueeze(z_atts.detach(), dim=0))
gt.extend(y_tensor)
return features_img, features_att, gt, num_classes
##################### FEATURE EXTRCTION #######################
def prepare_data_classifier(self, batch_size):
print(20 * '-')
print("Preparing dataset for the classifier..")
trainval_features_img, trainval_features_att, trainval_labels, self.trainval_target_classes = self.extract_features(
'trainval')
print(">> Extraction of trainval features is complete!")
test_features_img_unseen, test_features_att_unseen, test_labels_unseen, self.test_unseen_target_classes = self.extract_features(
'test_unseen')
print(">> Extraction of test_unseen features is complete!")
test_features_img_seen, test_features_att_seen, test_labels_seen, self.test_seen_target_classes = self.extract_features(
'test_seen')
print(">> Extraction of test_seen features is complete!")
self.cls_data_trainval = classifier_dataloader(trainval_features_img,
trainval_features_att,
trainval_labels)
self.cls_data_test_unseen = classifier_dataloader(
test_features_img_unseen, test_features_att_unseen,
test_labels_unseen)
self.cls_data_test_seen = classifier_dataloader(
test_features_img_seen, test_features_att_seen, test_labels_seen)
self.cls_trainloader = data.DataLoader(self.cls_data_trainval,
batch_size=batch_size,
shuffle=True)
self.cls_testloader_unseen = data.DataLoader(self.cls_data_test_unseen,
batch_size=batch_size,
shuffle=False)
self.cls_testloader_seen = data.DataLoader(self.cls_data_test_seen,
batch_size=batch_size,
shuffle=False)
##################### TRAINING THE CLASSIFIER #######################
def train_classifier(self, epochs):
best_H = -1
best_seen = 0
best_unseen = 0
############## TRAINING ####################
for epoch in range(1, epochs + 1):
print("Training: Epoch - ", epoch)
self.classifier.train()
trainbar_cls = tqdm(self.cls_trainloader)
train_loss = 0
for batch_idx, (x, sig, y) in enumerate(trainbar_cls):
output = self.classifier(x)
loss = self.lossfunction_classifier(output, y)
self.cls_optimizer.zero_grad()
loss.backward()
self.cls_optimizer.step()
train_loss += loss.item()
trainbar_cls.set_description('l:%.3f' % (train_loss /
(batch_idx + 1)))
########## VALIDATION ##################
accu_unseen = 0
accu_seen = 0
def val_gzsl(testbar_cls):
with torch.no_grad():
self.classifier.eval()
print("**Validation**")
preds = []
target = []
for batch_idx, (x, sig, y) in enumerate(testbar_cls):
output = self.classifier(x)
output_data = torch.argmax(output.data, 1)
preds.append(output_data)
target.append(y)
predictions = torch.cat(preds)
targets = torch.cat(target)
return predictions, targets
testbar_cls_unseen = tqdm(self.cls_testloader_unseen)
testbar_cls_seen = tqdm(self.cls_testloader_seen)
preds_unseen, target_unseen = val_gzsl(testbar_cls_unseen)
preds_seen, target_seen = val_gzsl(testbar_cls_seen)
########## ACCURACY METRIC ##################
def compute_per_class_acc_gzsl(test_label, predicted_label,
target_classes):
per_class_accuracies = torch.zeros(
target_classes.shape[0]).float().to(self.device)
predicted_label = predicted_label.to(self.device)
for i in range(target_classes.shape[0]):
is_class = test_label == target_classes[i]
per_class_accuracies[i] = torch.div(
(predicted_label[is_class] == test_label[is_class]
).sum().float(),
is_class.sum().float())
return per_class_accuracies.mean()
accu_unseen = compute_per_class_acc_gzsl(
target_unseen, preds_unseen, self.test_unseen_target_classes)
accu_seen = compute_per_class_acc_gzsl(
target_seen, preds_seen, self.test_seen_target_classes)
if (accu_seen + accu_unseen) > 0:
H = (2 * acc_seen * acc_novel) / (acc_seen + acc_novel)
else:
H = 0
if H > best_H:
best_seen = accu_seen
best_unseen = accu_unseen
best_H = H
print(20 * '-')
print('Epoch:', epoch)
print('u, s, h =%.4f,%.4f,%.4f' % (best_unseen, best_seen, best_H))
print(20 * '-')
return best_seen, best_unseen, best_H
class ClassifierTrainer(object):
def __init__(self,
config,
train_attack_mode='vanilla',
eval_attack_mode_list=['vanilla']):
assert train_attack_mode in attack_mode_list
for eval_attack_mode in eval_attack_mode_list:
assert eval_attack_mode in attack_mode_list
# set up random seed and device
torch.manual_seed(config.random_seed)
self.device = torch.device(
'cuda:0' if torch.cuda.is_available() else 'cpu')
# set up data and loader
train_tfs = transforms.Compose([
transforms.Pad(4, padding_mode='reflect'),
transforms.RandomHorizontalFlip(),
transforms.RandomCrop(32),
transforms.ToTensor()
])
eval_tfs = transforms.ToTensor()
train_idx = np.load(config.train_idx_path)
train_data = Subset(
CIFAR10(config.data_dir, True, train_tfs, download=True),
train_idx)
self.train_loader = DataLoader(train_data,
config.batch_size['train'],
True,
num_workers=2)
val_idx = np.load(config.val_idx_path)
val_data = Subset(
CIFAR10(config.data_dir, True, eval_tfs, download=True), val_idx)
self.val_loader = DataLoader(val_data,
config.batch_size['val'],
False,
num_workers=2)
test_data = CIFAR10(config.data_dir, False, eval_tfs, download=True)
self.test_loader = DataLoader(test_data,
config.batch_size['test'],
False,
num_workers=2)
# set up Classifier
self.cla = Classifier(config.nr_block, config.widen_factor,
config.dropout, config.nr_label)
if torch.cuda.device_count() > 1:
self.cla = torch.nn.DataParallel(self.cla)
self.cla.to(self.device)
# set up Attacker
self.train_attack_mode = train_attack_mode
self.eval_attack_mode_list = eval_attack_mode_list
self.epsilon = config.epsilon
# set up criterion and optimizer
self.criterion = torch.nn.CrossEntropyLoss()
init_lr, momentum, weight_decay = config.init_lr, config.momentum, config.weight_decay
self.optimizer = torch.optim.SGD(self.cla.parameters(),
init_lr,
momentum,
weight_decay=weight_decay)
# set up other parameters
self.data_mean = torch.tensor(config.data_mean).view(1, 3, 1,
1).to(self.device)
self.data_std = torch.tensor(config.data_std).view(1, 3, 1,
1).to(self.device)
self.nr_epoch = config.nr_epoch
self.boundary, self.lr_decay_rate = config.boundary, config.lr_decay_rate
self.save_dir = config.save_dir
def train(self):
print('training start!')
# training process
opt_val_acc = 0
for epoch in range(1, self.nr_epoch + 1):
if epoch in self.boundary:
adjust_lr(self.optimizer, self.lr_decay_rate)
train_loss, train_acc = 0, 0
for i, data in enumerate(self.train_loader, 1):
imgs, labels = data[0].to(self.device), data[1].to(self.device)
# generate adversarial example
#if i%2==0:
imgs = self.attacker(imgs, labels,
self.train_attack_mode).detach()
# calc loss
logits = self.cla(self.normalize(imgs))
loss = self.criterion(logits, labels)
# update parameters of Classifer
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# update output metrics
train_loss = 0.5 * train_loss + 0.5 * loss.item()
_, labels_pre = logits.max(dim=1)
acc = (labels_pre == labels).float().mean()
train_acc = 0.5 * train_acc + 0.5 * acc.item()
print('_______training_______')
print('epoch: ', epoch, 'loss: ', '%.3f' % train_loss, 'acc: ',
'%.3f' % train_acc)
self.cla.eval()
val_acc = self.evaluation('val', self.train_attack_mode)
print('_______validation_______')
print('epoch: ', epoch, 'acc: ', '%.3f' % val_acc)
if opt_val_acc <= val_acc:
opt_val_acc = val_acc
torch.save(self.cla.state_dict(),
self.save_dir + '/opt' + self.train_attack_mode)
print('Current opt val acc is: ', '%.3f' % opt_val_acc)
self.cla.train()
print('Training finish! Optimal val acc is: ', '%.3f' % opt_val_acc)
self.cla.load_state_dict(
torch.load(self.save_dir + '/opt' + self.train_attack_mode))
self.cla.eval()
print('_______testing_______')
for eval_attack_mode in self.eval_attack_mode_list:
test_acc = self.evaluation('test', eval_attack_mode)
print('acc for ' + eval_attack_mode + ' is: ', '%.3f' % test_acc)
self.cla.train()
def evaluation(self, mode, attack_mode):
assert mode in ['val', 'test']
assert attack_mode in attack_mode_list
if mode == 'val':
loader = self.val_loader
else:
loader = self.test_loader
ret_acc = 0
for i, data in enumerate(loader, 1):
imgs, labels = data[0].to(self.device), data[1].to(self.device)
# generate adversarial example
imgs = self.attacker(imgs, labels, attack_mode).detach()
logits = self.cla(self.normalize(imgs))
_, labels_pre = logits.max(dim=1)
acc = (labels_pre == labels).float().mean()
ret_acc = (i - 1) / i * ret_acc + acc.item() / i
return ret_acc
def normalize(self, imgs):
return (imgs - self.data_mean) / self.data_std
def attacker(self, imgs, labels, attack_mode):
assert attack_mode in attack_mode_list
# gradient based attacker
if attack_mode == 'vanilla':
return imgs
if attack_mode == 'FGSM':
imgs.requires_grad = True
logits = self.cla(self.normalize(imgs))
loss = self.criterion(logits, labels)
loss.backward()
grads = imgs.grad.data
return torch.clamp(imgs + self.epsilon * grads.sign(), 0,
1).detach()
if attack_mode == 'PGD':
tot_step, lr = tot_step_pgd, lr_pgd
delta = torch.rand_like(imgs, requires_grad=True)
delta.data = (delta * 2 - 1) * self.epsilon
delta.data = torch.max(torch.min(delta, 1 - imgs), -imgs)
for _ in range(tot_step):
logits = self.cla(self.normalize(imgs + delta))
loss = self.criterion(logits, labels)
loss.backward()
grads = delta.grad.data
delta.data = torch.clamp(delta + lr * grads.sign().detach(),
-self.epsilon, self.epsilon)
delta.data = torch.max(torch.min(delta, 1 - imgs), -imgs)
delta.grad.zero_()
return imgs + delta.detach()
logger.info("Step [%d/%d] Epoch [%d/%d] lr: %f, s1_cls_loss: %.4f, s2_cls_loss: %.4f,s3_cls_loss: %.4f, s1_t_dis_loss: %.4f, " \
"s2_t_dis_loss: %.4f, s3_t_dis_loss: %.4f, s1_t_confusion_loss_s1: %.4f, s1_t_confusion_loss_t: %.4f, " \
"s2_t_confusion_loss_s2: %.4f, s2_t_confusion_loss_t: %.4f,s3_t_confusion_loss_s3: %.4f, s3_t_confusion_loss_t: %.4f, selected_source: %s" \
% (step, steps, epoch, epoches, lr, l1, l2, l3, l4, l5, l6, l7, l8,l9,l10,l11,l12, flag))
count = 0
max_correct = 0
max_step = 0
max_epoch = 0
ploter = LinePlotter(env_name="bvlc_A_W_2_D")
for step in range(steps):
# Part 1: assign psudo-labels to t-domain and update the label-dataset
logger.info("#################### Part1 ####################")
extractor.eval()
s1_classifier.eval()
s2_classifier.eval()
s3_classifier.eval()
fin = open(t_label)
fout = open(
os.path.join(data_root, args.t,
"pseudo/pse_label_" + str(step) + ".txt"), "w")
if step > 0:
s1_weight = s1_weight_loss / (s1_weight_loss + s2_weight_loss +
s3_weight_loss)
s2_weight = s2_weight_loss / (s1_weight_loss + s2_weight_loss +
s3_weight_loss)
s3_weight = s3_weight_loss / (s1_weight_loss + s2_weight_loss +
s3_weight_loss)
logger.info("s1_weight is:{}".format(s1_weight))
class ModelRunner:
def __init__(self):
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
print(f"Device is {self.device}")
model_name = 'bert-base-cased'
self.model_path = 'NER_CONLL2003.pt'
self.tokenizer = BertTokenizer.from_pretrained(model_name)
self.model = Classifier(model_name)
self.loader = NERDataLoader(tokenizer=self.tokenizer)
self.optimizer = optim.Adam(params=self.model.parameters(),
lr=LEARNING_RATE)
def train_and_save(self):
for epoch in range(EPOCHS):
self.train_epoch(epoch)
torch.save(self.model.state_dict(), self.model_path)
def load_model(self):
loaded_state = torch.load(
self.model_path,
map_location=torch.device(
"cuda" if torch.cuda.is_available() else "cpu"))
self.model.load_state_dict(loaded_state)
def train_epoch(self, epoch):
training_loader = self.loader.train()
self.model.train()
for _, data in enumerate(training_loader, 0):
input_data = data['ids'].to(self.device, dtype=torch.long)
mask_data = data['mask'].to(self.device, dtype=torch.long)
labels = data['tags'].to(self.device, dtype=torch.long)
loss = self.model(input_data, mask_data, labels=labels)[0]
if _ % 10 == 0:
print(f'Epoch: {epoch}, Loss: {loss.item()}')
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
def test(self):
testing_loader = self.loader.test()
self.model.eval()
eval_loss = 0
eval_accuracy = 0
predictions, true_labels = [], []
nb_eval_steps, nb_eval_examples = 0, 0
with torch.no_grad():
for _, data in enumerate(testing_loader, 0):
input_data = data['ids'].to(self.device, dtype=torch.long)
mask_data = data['mask'].to(self.device, dtype=torch.long)
labels = data['tags'].to(self.device, dtype=torch.long)
output = self.model(input_data, mask_data, labels=labels)
loss, logits = output[:2]
logits = logits.detach().cpu().numpy()
label_ids = labels.to('cpu').numpy()
predictions.extend(
[list(p) for p in np.argmax(logits, axis=2)])
true_labels.append(label_ids)
accuracy = self.flat_accuracy(logits, label_ids)
eval_loss += loss.mean().item()
eval_accuracy += accuracy
nb_eval_examples += input_data.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
print("Validation loss: {}".format(eval_loss))
print("Validation Accuracy: {}".format(eval_accuracy /
nb_eval_steps))
pred_tags = [TAGS[p_i] for p in predictions for p_i in p]
valid_tags = [
TAGS[l_ii] for l in true_labels for l_i in l for l_ii in l_i
]
clean_pred_tags, clean_valid_tags = self.remove_none_tags(
pred_tags, valid_tags)
print("Micro F1-Score: {}".format(
f1_score(clean_pred_tags, clean_valid_tags, average='micro')))
print("Macro F1-Score: {}".format(
f1_score(clean_pred_tags, clean_valid_tags, average='macro')))
def remove_none_tags(self, pred_tags, valid_tags):
return zip(*[(pred, valid)
for pred, valid in zip(pred_tags, valid_tags)
if "None" != valid])
def flat_accuracy(self, preds, labels):
flat_preds = np.argmax(preds, axis=2).flatten()
flat_labels = labels.flatten()
return np.sum(flat_preds == flat_labels) / len(flat_labels)
class Training(object):
def __init__(self, config, logger=None):
if logger is None:
logger = logging.getLogger('logger')
logger.setLevel(logging.DEBUG)
logging.basicConfig(format='%(message)s', level=logging.DEBUG)
self.logger = logger
self.config = config
self.classes = list(config.id2label.keys())
self.num_classes = config.num_classes
self.embedder = Embedder(self.config)
self.encoder = LSTMEncoder(self.config)
self.clf = Classifier(self.config)
self.clf_loss = SequenceCriteria(class_weight=None)
if self.config.lambda_ae > 0: self.ae = AEModel(self.config)
self.writer = SummaryWriter(log_dir="TFBoardSummary")
self.global_steps = 0
self.enc_clf_opt = Adam(self._get_trainabe_modules(),
lr=self.config.lr,
betas=(config.beta1, config.beta2),
weight_decay=config.weight_decay,
eps=config.eps)
if config.scheduler == "ReduceLROnPlateau":
self.scheduler = lr_scheduler.ReduceLROnPlateau(
self.enc_clf_opt,
mode='max',
factor=config.lr_decay,
patience=config.patience,
verbose=True)
elif config.scheduler == "ExponentialLR":
self.scheduler = lr_scheduler.ExponentialLR(self.enc_clf_opt,
gamma=config.gamma)
self._init_or_load_model()
if config.multi_gpu:
self.embedder.cuda()
self.encoder.cuda()
self.clf.cuda()
self.clf_loss.cuda()
if self.config.lambda_ae > 0: self.ae.cuda()
self.ema_embedder = ExponentialMovingAverage(decay=0.999)
self.ema_embedder.register(self.embedder.state_dict())
self.ema_encoder = ExponentialMovingAverage(decay=0.999)
self.ema_encoder.register(self.encoder.state_dict())
self.ema_clf = ExponentialMovingAverage(decay=0.999)
self.ema_clf.register(self.clf.state_dict())
self.time_s = time()
def _get_trainabe_modules(self):
param_list = list(self.embedder.parameters()) + \
list(self.encoder.parameters()) + \
list(self.clf.parameters())
if self.config.lambda_ae > 0:
param_list += list(self.ae.parameters())
return param_list
def _get_l2_norm_loss(self):
total_norm = 0.
for p in self._get_trainabe_modules():
param_norm = p.data.norm(p=2)
total_norm += param_norm # ** 2
return total_norm # / 2.
def _init_or_load_model(self):
# if not self._load_model():
ensure_directory(self.config.output_path)
self.epoch = 0
self.best_accuracy = -np.inf
def _compute_vocab_freq(self, train_, dev_):
counter = collections.Counter()
for _, ids_ in train_:
counter.update(ids_)
for _, ids_ in dev_:
counter.update(ids_)
word_freq = np.zeros(self.config.n_vocab)
for index_, freq_ in counter.items():
word_freq[index_] = freq_
return torch.from_numpy(word_freq).type(batch_utils.FLOAT_TYPE)
def _save_model(self):
state = {
'epoch': self.epoch,
'state_dict_encoder': self.ema_encoder.shadow_variable_dict,
# self.encoder.state_dict(),
'state_dict_embedder': self.ema_embedder.shadow_variable_dict,
# self.embedder.state_dict(),
'state_dict_clf': self.ema_clf.shadow_variable_dict,
# self.clf.state_dict(),
'best_accuracy': self.best_accuracy
}
torch.save(
state, os.path.join(self.config.output_path,
self.config.model_file))
def _load_model(self):
checkpoint_path = os.path.join(self.config.output_path,
self.config.model_file)
if self.config.load_checkpoint and os.path.isfile(checkpoint_path):
# Code taken from here: https://github.com/pytorch/examples/blob/master/imagenet/main.py
dict_ = torch.load(checkpoint_path)
self.epoch = dict_['epoch']
self.best_accuracy = dict_['best_accuracy']
self.embedder.load_state_dict(dict_['state_dict_embedder'])
self.encoder.load_state_dict(dict_['state_dict_encoder'])
self.clf.load_state_dict(dict_['state_dict_clf'])
self.logger.info("=> loaded checkpoint '{}' (epoch {})".format(
checkpoint_path, self.epoch))
return True
def __call__(self, train, dev, test, unlabel, addn, addn_un, addn_test):
self.logger.info('Start training')
self._train(train, dev, unlabel, addn, addn_un, addn_test)
self._evaluate(test, addn_test)
def _create_iter(self, data_, wbatchsize, random_shuffler=None):
iter_ = data.iterator.pool(data_,
wbatchsize,
key=lambda x: len(x[1]),
batch_size_fn=batch_size_fn,
random_shuffler=None)
return iter_
def _run_epoch(self, train_data, dev_data, unlabel_data, addn_data,
addn_data_unlab, addn_dev):
addn_dev.cuda()
report_stats = utils.Statistics()
cm = ConfusionMatrix(self.classes)
_, seq_data = list(zip(*train_data))
total_seq_words = len(list(itertools.chain.from_iterable(seq_data)))
iter_per_epoch = (1.5 * total_seq_words) // self.config.wbatchsize
self.encoder.train()
self.clf.train()
self.embedder.train()
# print(addn_data)
# print(addn_data.shape)
# print(train_data[:5])
train_iter = self._create_iter(train_data, self.config.wbatchsize)
# addn_iter = self._create_iter(addn_data, self.config.wbatchsize)
# train_iter = self._create_iter(zip(train_data, addn_data), self.config.wbatchsize)
unlabel_iter = self._create_iter(unlabel_data,
self.config.wbatchsize_unlabel)
sofar = 0
sofar_1 = 0
for batch_index, train_batch_raw in enumerate(train_iter):
seq_iter = list(zip(*train_batch_raw))[1]
seq_words = len(list(itertools.chain.from_iterable(seq_iter)))
report_stats.n_words += seq_words
self.global_steps += 1
# self.enc_clf_opt.zero_grad()
if self.config.add_noise:
train_batch_raw = add_noise(train_batch_raw,
self.config.noise_dropout,
self.config.random_permutation)
train_batch = batch_utils.seq_pad_concat(train_batch_raw, -1)
# print(train_batch.shape)
train_embedded = self.embedder(train_batch)
memory_bank_train, enc_final_train = self.encoder(
train_embedded, train_batch)
if self.config.lambda_vat > 0 or self.config.lambda_ae > 0 or self.config.lambda_entropy:
try:
unlabel_batch_raw = next(unlabel_iter)
except StopIteration:
unlabel_iter = self._create_iter(
unlabel_data, self.config.wbatchsize_unlabel)
unlabel_batch_raw = next(unlabel_iter)
if self.config.add_noise:
unlabel_batch_raw = add_noise(
unlabel_batch_raw, self.config.noise_dropout,
self.config.random_permutation)
unlabel_batch = batch_utils.seq_pad_concat(
unlabel_batch_raw, -1)
unlabel_embedded = self.embedder(unlabel_batch)
memory_bank_unlabel, enc_final_unlabel = self.encoder(
unlabel_embedded, unlabel_batch)
# print(memory_bank_unlabel.shape[0])
addn_batch_unlab = retAddnBatch(addn_data_unlab,
memory_bank_unlabel.shape[0],
sofar_1).cuda()
sofar_1 += addn_batch_unlab.shape[0]
# print(addn_batch_unlab.shape)
# print(memory_bank_train.shape[0])
addn_batch = retAddnBatch(addn_data, memory_bank_train.shape[0],
sofar).cuda()
sofar += addn_batch.shape[0]
# print(addn_batch.shape)
pred = self.clf(memory_bank_train, addn_batch, enc_final_train)
# print(pred)
accuracy = self.get_accuracy(cm, pred.data,
train_batch.labels.data)
lclf = self.clf_loss(pred, train_batch.labels)
lat = Variable(
torch.FloatTensor([-1.]).type(batch_utils.FLOAT_TYPE))
lvat = Variable(
torch.FloatTensor([-1.]).type(batch_utils.FLOAT_TYPE))
if self.config.lambda_at > 0:
lat = at_loss(
self.embedder,
self.encoder,
self.clf,
train_batch,
addn_batch,
perturb_norm_length=self.config.perturb_norm_length)
if self.config.lambda_vat > 0:
lvat_train = vat_loss(
self.embedder,
self.encoder,
self.clf,
train_batch,
addn_batch,
p_logit=pred,
perturb_norm_length=self.config.perturb_norm_length)
if self.config.inc_unlabeled_loss:
lvat_unlabel = vat_loss(
self.embedder,
self.encoder,
self.clf,
unlabel_batch,
addn_batch_unlab,
p_logit=self.clf(memory_bank_unlabel, addn_batch_unlab,
enc_final_unlabel),
perturb_norm_length=self.config.perturb_norm_length)
if self.config.unlabeled_loss_type == "AvgTrainUnlabel":
lvat = 0.5 * (lvat_train + lvat_unlabel)
elif self.config.unlabeled_loss_type == "Unlabel":
lvat = lvat_unlabel
else:
lvat = lvat_train
lentropy = Variable(
torch.FloatTensor([-1.]).type(batch_utils.FLOAT_TYPE))
if self.config.lambda_entropy > 0:
lentropy_train = entropy_loss(pred)
if self.config.inc_unlabeled_loss:
lentropy_unlabel = entropy_loss(
self.clf(memory_bank_unlabel, addn_batch_unlab,
enc_final_unlabel))
if self.config.unlabeled_loss_type == "AvgTrainUnlabel":
lentropy = 0.5 * (lentropy_train + lentropy_unlabel)
elif self.config.unlabeled_loss_type == "Unlabel":
lentropy = lentropy_unlabel
else:
lentropy = lentropy_train
lae = Variable(
torch.FloatTensor([-1.]).type(batch_utils.FLOAT_TYPE))
if self.config.lambda_ae > 0:
lae = self.ae(memory_bank_unlabel, enc_final_unlabel,
unlabel_batch.sent_len, unlabel_batch_raw)
ltotal = (self.config.lambda_clf * lclf) + \
(self.config.lambda_ae * lae) + \
(self.config.lambda_at * lat) + \
(self.config.lambda_vat * lvat) + \
(self.config.lambda_entropy * lentropy)
report_stats.clf_loss += lclf.data.cpu().numpy()
report_stats.at_loss += lat.data.cpu().numpy()
report_stats.vat_loss += lvat.data.cpu().numpy()
report_stats.ae_loss += lae.data.cpu().numpy()
report_stats.entropy_loss += lentropy.data.cpu().numpy()
report_stats.n_sent += len(pred)
report_stats.n_correct += accuracy
self.enc_clf_opt.zero_grad()
ltotal.backward()
params_list = self._get_trainabe_modules()
# Excluding embedder form norm constraint when AT or VAT
if not self.config.normalize_embedding:
params_list += list(self.embedder.parameters())
norm = torch.nn.utils.clip_grad_norm(params_list,
self.config.max_norm)
report_stats.grad_norm += norm
self.enc_clf_opt.step()
if self.config.scheduler == "ExponentialLR":
self.scheduler.step()
self.ema_embedder.apply(self.embedder.named_parameters())
self.ema_encoder.apply(self.encoder.named_parameters())
self.ema_clf.apply(self.clf.named_parameters())
report_func(self.epoch, batch_index, iter_per_epoch, self.time_s,
report_stats, self.config.report_every, self.logger)
if self.global_steps % self.config.eval_steps == 0:
cm_, accuracy, prc_dev = self._run_evaluate(dev_data, addn_dev)
self.logger.info(
"- dev accuracy {} | best dev accuracy {} ".format(
accuracy, self.best_accuracy))
self.writer.add_scalar("Dev_Accuracy", accuracy,
self.global_steps)
pred_, lab_ = zip(*prc_dev)
pred_ = torch.cat(pred_)
lab_ = torch.cat(lab_)
self.writer.add_pr_curve("Dev PR-Curve", lab_, pred_,
self.global_steps)
pprint.pprint(cm_)
pprint.pprint(cm_.get_all_metrics())
if accuracy > self.best_accuracy:
self.logger.info("- new best score!")
self.best_accuracy = accuracy
self._save_model()
if self.config.scheduler == "ReduceLROnPlateau":
self.scheduler.step(accuracy)
self.encoder.train()
self.embedder.train()
self.clf.train()
if self.config.weight_decay > 0:
print(">> Square Norm: %1.4f " % self._get_l2_norm_loss())
cm, train_accuracy, _ = self._run_evaluate(train_data, addn_data)
self.logger.info("- Train accuracy {}".format(train_accuracy))
pprint.pprint(cm.get_all_metrics())
cm, dev_accuracy, _ = self._run_evaluate(dev_data, addn_dev)
self.logger.info("- Dev accuracy {} | best dev accuracy {}".format(
dev_accuracy, self.best_accuracy))
pprint.pprint(cm.get_all_metrics())
self.writer.add_scalars("Overall_Accuracy", {
"Train_Accuracy": train_accuracy,
"Dev_Accuracy": dev_accuracy
}, self.global_steps)
return dev_accuracy
@staticmethod
def get_accuracy(cm, output, target):
batch_size = output.size(0)
predictions = output.max(-1)[1].type_as(target)
correct = predictions.eq(target)
correct = correct.float()
if not hasattr(correct, 'sum'):
correct = correct.cpu()
correct = correct.sum()
cm.add_batch(target.cpu().numpy(), predictions.cpu().numpy())
return correct
def _predict_batch(self, cm, batch, addn_batch):
self.embedder.eval()
self.encoder.eval()
self.clf.eval()
one, two = self.encoder(self.embedder(batch), batch)
pred = self.clf(one, addn_batch, two)
accuracy = self.get_accuracy(cm, pred.data, batch.labels.data)
return pred, accuracy
def chunks(self, l, n=15):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
def _run_evaluate(self, test_data, addn_test):
pr_curve_data = []
cm = ConfusionMatrix(self.classes)
accuracy_list = []
# test_iter = self._create_iter(test_data, self.config.wbatchsize,
# random_shuffler=utils.identity_fun)
test_iter = self.chunks(test_data)
for batch_index, test_batch in enumerate(test_iter):
addn_batch = addn_test[batch_index * 15:(batch_index + 1) * 15]
test_batch = batch_utils.seq_pad_concat(test_batch, -1)
# print(addn_batch.shape)
try:
pred, acc = self._predict_batch(cm, test_batch, addn_batch)
except:
continue
accuracy_list.append(acc)
pr_curve_data.append(
(F.softmax(pred, -1)[:, 1].data, test_batch.labels.data))
accuracy = 100 * (sum(accuracy_list) / len(test_data))
return cm, accuracy, pr_curve_data
def _train(self, train_data, dev_data, unlabel_data, addn_data,
addn_data_unlab, addn_dev):
addn_data = addn_data.cuda()
addn_data_unlab = addn_data_unlab.cuda()
addn_dev = addn_dev.cuda()
# for early stopping
nepoch_no_imprv = 0
epoch_start = self.epoch + 1
epoch_end = self.epoch + self.config.nepochs + 1
for self.epoch in range(epoch_start, epoch_end):
self.logger.info("Epoch {:} out of {:}".format(
self.epoch, self.config.nepochs))
# random.shuffle(train_data)
# random.shuffle(unlabel_data)
accuracy = self._run_epoch(train_data, dev_data, unlabel_data,
addn_data, addn_data_unlab, addn_dev)
# early stopping and saving best parameters
if accuracy > self.best_accuracy:
nepoch_no_imprv = 0
self.best_accuracy = accuracy
self.logger.info("- new best score!")
self._save_model()
else:
nepoch_no_imprv += 1
if nepoch_no_imprv >= self.config.nepoch_no_imprv:
self.logger.info(
"- early stopping {} epochs without improvement".
format(nepoch_no_imprv))
break
if self.config.scheduler == "ReduceLROnPlateau":
self.scheduler.step(accuracy)
def _evaluate(self, test_data, addn_test):
addn_test = addn_test.cuda()
self.logger.info("Evaluating model over test set")
self._load_model()
_, accuracy, prc_test = self._run_evaluate(test_data, addn_test)
pred_, lab_ = zip(*prc_test)
pred_ = torch.cat(pred_).cpu().tolist()
lab_ = torch.cat(lab_).cpu().tolist()
path_ = os.path.join(self.config.output_path,
"{}_pred_gt.tsv".format(self.config.exp_name))
with open(path_, 'w') as fp:
for p, l in zip(pred_, lab_):
fp.write(str(p) + '\t' + str(l) + '\n')
self.logger.info("- test accuracy {}".format(accuracy))
def calcStatAndGenPlotForModel(filter_size, filter_depth, layer):
print(
f'Calc stats for {filter_size}x{filter_size} {filter_depth}bits {layer} layer'
)
model = Classifier(layer)
model_filename = model_output_filename(filter_size, filter_depth, layer)
model.load_state_dict(torch.load(model_filename))
model.eval()
dataset = Dataset(filter_size, filter_depth)
correct_top1 = 0
correct_top5 = 0
rank_array = np.zeros((CLASS_NUMBER))
t_c = np.zeros((CLASS_NUMBER, 102))
f_c = np.zeros((CLASS_NUMBER, 102))
tn_c = np.zeros((CLASS_NUMBER, 102))
tp_c = np.zeros((CLASS_NUMBER, 102))
t_n = np.zeros((CLASS_NUMBER, 1))
f_n = np.zeros((CLASS_NUMBER, 1))
for i in range(0, dataset.__len__()):
features, labels = dataset.get(i)
features_2 = features.view(1, features.shape[0] * features.shape[1])
logps = model.forward(features_2)
chance_log, predicted = torch.max(logps, 1)
label_id = labels
predicted_chance = math.exp(chance_log[0])
chance_table = torch.exp(logps)
#rank
class_prob = logps.data[0][label_id]
rank = 0
for x in range(0, logps.shape[1]):
if logps.data[0][x] > class_prob:
rank += 1
#Top-1 Calculating
if rank == 0:
correct_top1 += 1
#Top-5 Calculalting
if rank < 5:
correct_top5 += 1
#CMC
for x in range(rank, CLASS_NUMBER):
rank_array[x] = rank_array[x] + 1
chance_idx = math.ceil(predicted_chance * 100)
#ROC
for y in range(0, CLASS_NUMBER):
chance_idx = math.ceil(chance_table[0, y] * 100) + 1
if y == label_id: #klasa pozytywna(P)
t_c[y, :chance_idx] += 1
if y == predicted[0]:
tp_c[y, 0] += 1
t_n[y, 0] += 1
else: #klasa negatywna
f_c[y, :chance_idx] += 1
f_n[y, 0] += 1
if y != predicted[0]:
tn_c[y, 0] += 1
print("Top 1 = " + str(correct_top1 / dataset.__len__()))
print("Top 5 = " + str(correct_top5 / dataset.__len__()))
filename_part = f'Result for {filter_size}x{filter_size} {filter_depth}bit {layer} layers'
with open(OUTPUT_STATS + filename_part + 'top.csv', 'w') as csvfile:
writer = csv.writer(csvfile, delimiter=",", quoting=csv.QUOTE_ALL)
writer.writerow(['TOP-1', str(correct_top1 / dataset.__len__())])
writer.writerow(['TOP-5', str(correct_top5 / dataset.__len__())])
writer.writerow(['Class', 'Accurancy'])
#Accuracy ACC = TP+TN/ALL
for y in range(0, CLASS_NUMBER):
acc = (tp_c[y, 0] + tn_c[y, 0]) / (t_n[y, 0] + f_n[y, 0])
writer.writerow([dataset.getLabelName(y), acc])
x = np.arange(CLASS_NUMBER)
y = rank_array[x] / dataset.__len__()
fig, ax = plt.subplots()
line1, = ax.plot(
x,
y,
label=
f'CMC for {filter_size}x{filter_size} {filter_depth}bit {layer} layers'
)
ax.legend()
plt.savefig(OUTPUT_STATS + filename_part + "_CMC.png", bbox_inches="tight")
tpr = t_c / t_n
fpr = f_c / f_n
y2 = tpr[:, :]
x2 = fpr[:, :]
fig2, ax2 = plt.subplots()
plt.subplots_adjust(right=0.7)
labels_legend = []
for y in range(0, CLASS_NUMBER):
_, = ax2.plot(x2[y, :], y2[y, :], label=dataset.getLabelName(y))
plt.title(f"{filter_size}x{filter_size} {filter_depth}bit {layer} layers")
ax2.legend(loc='center left', bbox_to_anchor=(1, 0.5), ncol=2)
plt.savefig(OUTPUT_STATS + filename_part + "_ROC.png", bbox_inches="tight")
print('... Finish calc stats')
# calcStatAndGenPlotForModel(15, 8, 5)
class ModelBuilder(object):
def __init__(self, use_cuda, conf, model_name):
self.cuda = use_cuda
self.conf = conf
self.model_name = model_name
self._init_log()
self._pre_data()
self._build_model()
def _pre_data(self):
print('pre data...')
self.data = Data(self.cuda, self.conf)
self.spacy = spacy.load('en')
# print('pre train SenImg pickle...')
# self.img_pickle_train = self._load_text_img_pickle_all('train')
# print('pre dev SenImg pickle...')
# self.img_pickle_dev = self._load_text_img_pickle_all('dev')
# print('pre test SenImg pickle...')
# self.img_pickle_test = self._load_text_img_pickle_all('test')
def _init_log(self):
if self.conf.four_or_eleven == 2:
filename = 'logs/train_' + datetime.now().strftime(
'%B%d-%H_%M_%S'
) + '_' + self.model_name + self.conf.type + '_' + self.conf.i2senseclass[
self.conf.binclass]
else:
filename = 'logs/train_' + datetime.now().strftime(
'%B%d-%H_%M_%S') + '_' + self.model_name + '_' + self.conf.type
if self.conf.need_elmo:
filename += '_ELMO'
logging.basicConfig(filename=filename + '.log',
filemode='a',
format='%(asctime)s - %(levelname)s: %(message)s',
level=logging.DEBUG)
def _build_model(self):
print('loading embedding...')
if self.conf.corpus_splitting == 1:
pre = './data/processed/lin/'
elif self.conf.corpus_splitting == 2:
pre = './data/processed/ji/'
elif self.conf.corpus_splitting == 3:
pre = './data/processed/l/'
we = torch.load(pre + 'we.pkl')
char_table = None
sub_table = None
if self.conf.need_char or self.conf.need_elmo:
char_table = torch.load(pre + 'char_table.pkl')
if self.conf.need_sub:
sub_table = torch.load(pre + 'sub_table.pkl')
print('building model...')
if self.model_name == 'ArgSenImg':
self.encoder = ArgEncoderSentImg2(self.conf, we, char_table,
sub_table, self.cuda, None,
self.spacy)
elif self.model_name == 'ArgPhrImg':
self.encoder = ArgEncoderPhrImg(self.conf, we, char_table,
sub_table, self.cuda, None,
self.spacy)
elif self.model_name == 'ArgImgSelf':
self.encoder = ArgEncoderImgSelf(self.conf, we, char_table,
sub_table, self.cuda, None,
self.spacy)
else:
self.encoder = ArgEncoder(self.conf, we, char_table, sub_table,
self.cuda)
self.classifier = Classifier(self.conf.clf_class_num, self.conf)
if self.conf.is_mttrain:
self.conn_classifier = Classifier(self.conf.conn_num, self.conf)
if self.cuda:
self.encoder.cuda()
self.classifier.cuda()
if self.conf.is_mttrain:
self.conn_classifier.cuda()
self.criterion = torch.nn.CrossEntropyLoss()
para_filter = lambda model: filter(lambda p: p.requires_grad,
model.parameters())
self.e_optimizer = torch.optim.Adagrad(
para_filter(self.encoder),
self.conf.lr,
weight_decay=self.conf.l2_penalty)
self.c_optimizer = torch.optim.Adagrad(
para_filter(self.classifier),
self.conf.lr,
weight_decay=self.conf.l2_penalty)
if self.conf.is_mttrain:
self.con_optimizer = torch.optim.Adagrad(
para_filter(self.conn_classifier),
self.conf.lr,
weight_decay=self.conf.l2_penalty)
def _print_train(self, epoch, time, loss, acc):
print('-' * 80)
print(
'| end of epoch {:3d} | time: {:5.2f}s | loss: {:10.5f} | acc: {:5.2f}% |'
.format(epoch, time, loss, acc * 100))
print('-' * 80)
logging.debug('-' * 80)
logging.debug(
'| end of epoch {:3d} | time: {:5.2f}s | loss: {:10.5f} | acc: {:5.2f}% |'
.format(epoch, time, loss, acc * 100))
logging.debug('-' * 80)
def _print_eval(self, task, loss, acc, f1):
print('| ' + task +
' loss {:10.5f} | acc {:5.2f}% | f1 {:5.2f}%'.format(
loss, acc * 100, f1 * 100))
print('-' * 80)
logging.debug('| ' + task +
' loss {:10.5f} | acc {:5.2f}% | f1 {:5.2f}%'.format(
loss, acc * 100, f1 * 100))
logging.debug('-' * 80)
def _save_model(self, model, filename):
torch.save(model.state_dict(), './weights/' + filename)
def _load_model(self, model, filename):
model.load_state_dict(torch.load('./weights/' + filename))
def _train_one(self):
self.encoder.train()
self.classifier.train()
if self.conf.is_mttrain:
self.conn_classifier.train()
total_loss = 0
correct_n = 0
train_size = self.data.train_size
for i, (a1, a2, sense, conn, arg1_sen,
arg2_sen) in enumerate(self.data.train_loader):
try:
start_time = time.time()
if self.conf.four_or_eleven == 2:
mask1 = (sense == self.conf.binclass)
mask2 = (sense != self.conf.binclass)
sense[mask1] = 1
sense[mask2] = 0
if self.cuda:
a1, a2, sense, conn = a1.cuda(), a2.cuda(), sense.cuda(
), conn.cuda()
a1, a2, sense, conn = Variable(a1), Variable(a2), Variable(
sense), Variable(conn)
if self.model_name in [
'ArgImg', 'ArgSenImg', 'ArgPhrImg', 'ArgImgSelf'
]:
self._load_text_img_pickle_index(i)
# img_pickle = self.img_pickle_train[i]
repr = self.encoder(a1, a2, arg1_sen, arg2_sen,
self.text_pkl, self.img_pkl,
self.phrase_text_pkl,
self.phrase_img_pkl, i, 'train')
else:
repr = self.encoder(a1, a2)
output = self.classifier(repr)
_, output_sense = torch.max(output, 1)
assert output_sense.size() == sense.size()
tmp = (output_sense == sense).long()
correct_n += torch.sum(tmp).data
loss = self.criterion(output, sense)
if self.conf.is_mttrain:
conn_output = self.conn_classifier(repr)
loss2 = self.criterion(conn_output, conn)
loss = loss + loss2 * self.conf.lambda1
self.e_optimizer.zero_grad()
self.c_optimizer.zero_grad()
if self.conf.is_mttrain:
self.con_optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm(self.encoder.parameters(),
self.conf.grad_clip)
torch.nn.utils.clip_grad_norm(self.classifier.parameters(),
self.conf.grad_clip)
if self.conf.is_mttrain:
torch.nn.utils.clip_grad_norm(
self.conn_classifier.parameters(), self.conf.grad_clip)
self.e_optimizer.step()
self.c_optimizer.step()
if self.conf.is_mttrain:
self.con_optimizer.step()
total_loss += loss.data * sense.size(0)
if self.model_name in ['ArgImg', 'ArgSenImg', 'ArgPhrImg']:
print('==================' + str(i) + '==================')
print('total_loss:' + str(total_loss[0] / (len(arg1_sen) *
(i + 1))) +
' acc:' + str(correct_n[0].float() /
(len(arg1_sen) * (i + 1))) + ' time:' +
str(time.time() - start_time))
logging.debug('==================' + str(i) +
'==================')
logging.debug('total_loss:' + str(total_loss[0] /
(len(arg1_sen) *
(i + 1))) + ' acc:' +
str(correct_n[0].float() /
(len(arg1_sen) * (i + 1))) + ' time:' +
str(time.time() - start_time))
except Exception as e:
print(e)
logging.debug(e)
continue
return total_loss[0] / train_size, correct_n[0].float() / train_size
def _train(self, pre):
for epoch in range(self.conf.epochs):
start_time = time.time()
loss, acc = self._train_one()
self._print_train(epoch, time.time() - start_time, loss, acc)
self.logwriter.add_scalar('loss/train_loss', loss, epoch)
self.logwriter.add_scalar('acc/train_acc', acc * 100, epoch)
dev_loss, dev_acc, dev_f1 = self._eval('dev')
self._print_eval('dev', dev_loss, dev_acc, dev_f1)
self.logwriter.add_scalar('loss/dev_loss', dev_loss, epoch)
self.logwriter.add_scalar('acc/dev_acc', dev_acc * 100, epoch)
self.logwriter.add_scalar('f1/dev_f1', dev_f1 * 100, epoch)
test_loss, test_acc, test_f1 = self._eval('test')
self._print_eval('test', test_loss, test_acc, test_f1)
self.logwriter.add_scalar('loss/test_loss', test_loss, epoch)
self.logwriter.add_scalar('acc/test_acc', test_acc * 100, epoch)
self.logwriter.add_scalar('f1/test_f1', test_f1 * 100, epoch)
def train(self, pre):
print('start training')
logging.debug('start training')
self.logwriter = SummaryWriter(self.conf.logdir)
self._train(pre)
self._save_model(self.encoder, pre + '_eparams.pkl')
self._save_model(self.classifier, pre + '_cparams.pkl')
print('training done')
logging.debug('training done')
def _eval(self, task):
self.encoder.eval()
self.classifier.eval()
total_loss = 0
correct_n = 0
if task == 'dev':
data = self.data.dev_loader
n = self.data.dev_size
elif task == 'test':
data = self.data.test_loader
n = self.data.test_size
else:
raise Exception('wrong eval task')
output_list = []
gold_list = []
for i, (a1, a2, sense1, sense2, arg1_sen, arg2_sen) in enumerate(data):
try:
if self.conf.four_or_eleven == 2:
mask1 = (sense1 == self.conf.binclass)
mask2 = (sense1 != self.conf.binclass)
sense1[mask1] = 1
sense1[mask2] = 0
mask0 = (sense2 == -1)
mask1 = (sense2 == self.conf.binclass)
mask2 = (sense2 != self.conf.binclass)
sense2[mask1] = 1
sense2[mask2] = 0
sense2[mask0] = -1
if self.cuda:
a1, a2, sense1, sense2 = a1.cuda(), a2.cuda(), sense1.cuda(
), sense2.cuda()
a1 = Variable(a1, volatile=True)
a2 = Variable(a2, volatile=True)
sense1 = Variable(sense1, volatile=True)
sense2 = Variable(sense2, volatile=True)
if self.model_name in [
'ArgImg', 'ArgSenImg', 'ArgPhrImg', 'ArgImgSelf'
]:
# self._load_text_img_pickle_index(i)
# if task == 'dev':
# img_pickle = self.img_pickle_dev[i]
# else:
# img_pickle = self.img_pickle_test[i]
self._load_text_img_pickle_index(i)
# img_pickle = self.img_pkl
output = self.classifier(
self.encoder(a1,
a2,
arg1_sen,
arg2_sen,
self.text_pkl,
self.img_pkl,
self.phrase_text_pkl,
self.phrase_img_pkl,
i,
task=task))
else:
output = self.classifier(self.encoder(a1, a2))
_, output_sense = torch.max(output, 1)
assert output_sense.size() == sense1.size()
gold_sense = sense1
mask = (output_sense == sense2)
gold_sense[mask] = sense2[mask]
tmp = (output_sense == gold_sense).long()
correct_n += torch.sum(tmp).data
output_list.append(output_sense)
gold_list.append(gold_sense)
loss = self.criterion(output, gold_sense)
total_loss += loss.data * gold_sense.size(0)
output_s = torch.cat(output_list)
gold_s = torch.cat(gold_list)
if self.conf.four_or_eleven == 2:
f1 = f1_score(gold_s.cpu().data.numpy(),
output_s.cpu().data.numpy(),
average='binary')
else:
f1 = f1_score(gold_s.cpu().data.numpy(),
output_s.cpu().data.numpy(),
average='macro')
except Exception as e:
print(e)
logging.debug(e)
continue
return total_loss[0] / n, correct_n[0].float() / n, f1
def eval(self, pre):
print('evaluating...')
logging.debug('evaluating...')
self._load_model(self.encoder, pre + '_eparams.pkl')
self._load_model(self.classifier, pre + '_cparams.pkl')
test_loss, test_acc, f1 = self._eval('test')
self._print_eval('test', test_loss, test_acc, f1)
def _load_text_img_pickle_all(self, task='train'):
img_pickle = []
if task == 'dev':
data = self.data.dev_loader
n = self.data.dev_size
elif task == 'test':
data = self.data.test_loader
n = self.data.test_size
else:
data = self.data.train_loader
n = self.data.train_loader
for i, (a1, a2, sense1, sense2, arg1_sen, arg2_sen) in enumerate(data):
self._load_text_img_pickle_index(i, task)
img_pickle.append(self.img_pkl)
return np.array(img_pickle)
def _load_text_img_pickle_index(self, index, task='train'):
root_dir = '/home/wangjian/projects/RNNImageIDRR/data/text_img'
if task != 'train':
text_pkl_path = root_dir + '/text_' + task + '_' + str(
index) + '.pkl'
img_pkl_path = root_dir + '/img_' + task + '_' + str(
index) + '.pkl'
phrase_text_pkl_path = root_dir + '/phrase_text_' + task + '_' + str(
index) + '.pkl'
phrase_img_pkl_path = root_dir + '/phrase_img_' + task + '_' + str(
index) + '.pkl'
else:
text_pkl_path = root_dir + '/text_' + str(index) + '.pkl'
img_pkl_path = root_dir + '/img_' + str(index) + '.pkl'
phrase_text_pkl_path = root_dir + '/phrase_text_' + str(
index) + '.pkl'
phrase_img_pkl_path = root_dir + '/phrase_img_' + str(
index) + '.pkl'
self.text_pkl = []
self.img_pkl = []
self.phrase_text_pkl = []
self.phrase_img_pkl = []
if self.model_name in [
'ArgImg', 'ArgSenImg', 'ArgPhrImg', 'ArgImgSelf'
]:
if self.model_name in ['ArgImg', 'ArgSenImg', 'ArgImgSelf']:
if os.path.exists(text_pkl_path) and os.path.exists(
img_pkl_path):
# print(img_pkl_path)
# logging.debug(img_pkl_path)
# with open(text_pkl_path, 'rb') as f:
# try:
# while True:
# self.text_pkl.append(pickle.load(f))
# except:
# pass
with h5py.File(img_pkl_path) as f:
img = f['img_features'][:]
img = img.reshape((len(img), 3, 256, 256))
self.img_pkl.extend(img)
if self.model_name in ['ArgImg', 'ArgPhrImg', 'ArgImgSelf']:
if os.path.exists(phrase_text_pkl_path) and os.path.exists(
phrase_img_pkl_path):
with open(phrase_text_pkl_path, 'rb') as f:
try:
while True:
self.phrase_text_pkl.append(pickle.load(f))
except:
pass
with h5py.File(phrase_img_pkl_path) as f:
img = f['img_features'][:]
img = img.reshape((len(img), 3, 256, 256))
self.phrase_img_pkl.extend(img)
class ModelBuilder(object):
def __init__(self, use_cuda):
self.cuda = use_cuda
self._pre_data()
self._build_model()
self.i_mb = 0
def _pre_data(self):
print('pre data...')
self.data = Data(self.cuda)
def _build_model(self):
print('building model...')
we = torch.load('./data/processed/we.pkl')
self.i_encoder = CNN_Args_encoder(we)
self.a_encoder = CNN_Args_encoder(we, need_kmaxavg=True)
self.classifier = Classifier()
self.discriminator = Discriminator()
if self.cuda:
self.i_encoder.cuda()
self.a_encoder.cuda()
self.classifier.cuda()
self.discriminator.cuda()
self.criterion_c = torch.nn.CrossEntropyLoss()
self.criterion_d = torch.nn.BCELoss()
para_filter = lambda model: filter(lambda p: p.requires_grad,
model.parameters())
self.i_optimizer = torch.optim.Adagrad(para_filter(self.i_encoder),
Config.lr,
weight_decay=Config.l2_penalty)
self.a_optimizer = torch.optim.Adagrad(para_filter(self.a_encoder),
Config.lr,
weight_decay=Config.l2_penalty)
self.c_optimizer = torch.optim.Adagrad(self.classifier.parameters(),
Config.lr,
weight_decay=Config.l2_penalty)
self.d_optimizer = torch.optim.Adam(self.discriminator.parameters(),
Config.lr_d,
weight_decay=Config.l2_penalty)
def _print_train(self, epoch, time, loss, acc):
print('-' * 80)
print(
'| end of epoch {:3d} | time: {:5.2f}s | loss: {:10.5f} | acc: {:5.2f}% |'
.format(epoch, time, loss, acc * 100))
print('-' * 80)
def _print_eval(self, task, loss, acc):
print('| ' + task +
' loss {:10.5f} | acc {:5.2f}% |'.format(loss, acc * 100))
print('-' * 80)
def _save_model(self, model, filename):
torch.save(model.state_dict(), './weights/' + filename)
def _load_model(self, model, filename):
model.load_state_dict(torch.load('./weights/' + filename))
def _pretrain_i_one(self):
self.i_encoder.train()
self.classifier.train()
total_loss = 0
correct_n = 0
for a1, a2i, a2a, sense in self.data.train_loader:
if self.cuda:
a1, a2i, a2a, sense = a1.cuda(), a2i.cuda(), a2a.cuda(
), sense.cuda()
a1, a2i, a2a, sense = Variable(a1), Variable(a2i), Variable(
a2a), Variable(sense)
output = self.classifier(self.i_encoder(a1, a2i))
_, output_sense = torch.max(output, 1)
assert output_sense.size() == sense.size()
tmp = (output_sense == sense).long()
correct_n += torch.sum(tmp).data
loss = self.criterion_c(output, sense)
self.i_optimizer.zero_grad()
self.c_optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm(self.i_encoder.parameters(),
Config.grad_clip)
torch.nn.utils.clip_grad_norm(self.classifier.parameters(),
Config.grad_clip)
self.i_optimizer.step()
self.c_optimizer.step()
total_loss += loss.data * sense.size(0)
return total_loss[0] / self.data.train_size, correct_n[
0] / self.data.train_size
def _pretrain_i_a_one(self):
self.i_encoder.train()
self.a_encoder.train()
self.classifier.train()
total_loss = 0
correct_n = 0
total_loss_a = 0
correct_n_a = 0
for a1, a2i, a2a, sense in self.data.train_loader:
if self.cuda:
a1, a2i, a2a, sense = a1.cuda(), a2i.cuda(), a2a.cuda(
), sense.cuda()
a1, a2i, a2a, sense = Variable(a1), Variable(a2i), Variable(
a2a), Variable(sense)
# train i
output = self.classifier(self.i_encoder(a1, a2i))
_, output_sense = torch.max(output, 1)
assert output_sense.size() == sense.size()
tmp = (output_sense == sense).long()
correct_n += torch.sum(tmp).data
loss = self.criterion_c(output, sense)
self.i_optimizer.zero_grad()
self.c_optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm(self.i_encoder.parameters(),
Config.grad_clip)
torch.nn.utils.clip_grad_norm(self.classifier.parameters(),
Config.grad_clip)
self.i_optimizer.step()
self.c_optimizer.step()
total_loss += loss.data * sense.size(0)
#train a
output = self.classifier(self.a_encoder(a1, a2a))
_, output_sense = torch.max(output, 1)
assert output_sense.size() == sense.size()
tmp = (output_sense == sense).long()
correct_n_a += torch.sum(tmp).data
loss = self.criterion_c(output, sense)
self.a_optimizer.zero_grad()
self.c_optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm(self.a_encoder.parameters(),
Config.grad_clip)
torch.nn.utils.clip_grad_norm(self.classifier.parameters(),
Config.grad_clip)
self.a_optimizer.step()
self.c_optimizer.step()
total_loss_a += loss.data * sense.size(0)
return total_loss[0] / self.data.train_size, correct_n[
0] / self.data.train_size, total_loss_a[
0] / self.data.train_size, correct_n_a[0] / self.data.train_size
def _adtrain_one(self, acc_d_for_train):
total_loss = 0
total_loss_2 = 0
correct_n = 0
correct_n_d = 0
correct_n_d_for_train = 0
for a1, a2i, a2a, sense in self.data.train_loader:
if self.cuda:
a1, a2i, a2a, sense = a1.cuda(), a2i.cuda(), a2a.cuda(
), sense.cuda()
a1, a2i, a2a, sense = Variable(a1), Variable(a2i), Variable(
a2a), Variable(sense)
# phase 1, train discriminator
flag = 0
for k in range(Config.kd):
# if self._test_d() != 1:
if True:
temp_d = 0
self.a_encoder.eval()
self.i_encoder.eval()
self.discriminator.train()
self.d_optimizer.zero_grad()
output_i = self.discriminator(self.i_encoder(a1, a2i))
temp_d += torch.sum((output_i < 0.5).long()).data
# zero_tensor = torch.zeros(output_i.size())
zero_tensor = torch.Tensor(output_i.size()).random_(
0, 100) * 0.003
if self.cuda:
zero_tensor = zero_tensor.cuda()
zero_tensor = Variable(zero_tensor)
d_loss_i = self.criterion_d(output_i, zero_tensor)
d_loss_i.backward()
output_a = self.discriminator(self.a_encoder(a1, a2a))
temp_d += torch.sum((output_a >= 0.5).long()).data
# one_tensor = torch.ones(output_a.size())
# one_tensor = torch.Tensor(output_a.size()).fill_(Config.alpha)
one_tensor = torch.Tensor(output_a.size()).random_(
0, 100) * 0.005 + 0.7
if self.cuda:
one_tensor = one_tensor.cuda()
one_tensor = Variable(one_tensor)
d_loss_a = self.criterion_d(output_a, one_tensor)
d_loss_a.backward()
correct_n_d_for_train += temp_d
temp_d = max(temp_d[0] / sense.size(0) / 2,
acc_d_for_train)
if temp_d < Config.thresh_high:
torch.nn.utils.clip_grad_norm(
self.discriminator.parameters(), Config.grad_clip)
self.d_optimizer.step()
# phase 2, train i/c
self.i_encoder.train()
self.classifier.train()
self.discriminator.eval()
self.i_optimizer.zero_grad()
self.c_optimizer.zero_grad()
sent_repr = self.i_encoder(a1, a2i)
output = self.classifier(sent_repr)
_, output_sense = torch.max(output, 1)
assert output_sense.size() == sense.size()
tmp = (output_sense == sense).long()
correct_n += torch.sum(tmp).data
loss_1 = self.criterion_c(output, sense)
output_d = self.discriminator(sent_repr)
correct_n_d += torch.sum((output_d < 0.5).long()).data
one_tensor = torch.ones(output_d.size())
# one_tensor = torch.Tensor(output_d.size()).fill_(Config.alpha)
# one_tensor = torch.Tensor(output_d.size()).random_(0,100) * 0.005 + 0.7
if self.cuda:
one_tensor = one_tensor.cuda()
one_tensor = Variable(one_tensor)
loss_2 = self.criterion_d(output_d, one_tensor)
loss = loss_1 + loss_2 * Config.lambda1
loss.backward()
torch.nn.utils.clip_grad_norm(self.i_encoder.parameters(),
Config.grad_clip)
torch.nn.utils.clip_grad_norm(self.classifier.parameters(),
Config.grad_clip)
self.i_optimizer.step()
self.c_optimizer.step()
total_loss += loss.data * sense.size(0)
total_loss_2 += loss_2.data * sense.size(0)
test_loss, test_acc = self._eval('test', 'i')
self.logwriter.add_scalar('acc/test_acc_t_mb', test_acc * 100,
self.i_mb)
self.i_mb += 1
return total_loss[0] / self.data.train_size, correct_n[
0] / self.data.train_size, correct_n_d[
0] / self.data.train_size, total_loss_2[
0] / self.data.train_size, correct_n_d_for_train[
0] / self.data.train_size / 2
def _pretrain_i(self):
best_test_acc = 0
for epoch in range(Config.pre_i_epochs):
start_time = time.time()
loss, acc = self._pretrain_i_one()
self._print_train(epoch, time.time() - start_time, loss, acc)
self.logwriter.add_scalar('loss/train_loss_i', loss, epoch)
self.logwriter.add_scalar('acc/train_acc_i', acc * 100, epoch)
dev_loss, dev_acc = self._eval('dev', 'i')
self._print_eval('dev', dev_loss, dev_acc)
self.logwriter.add_scalar('loss/dev_loss_i', dev_loss, epoch)
self.logwriter.add_scalar('acc/dev_acc_i', dev_acc * 100, epoch)
test_loss, test_acc = self._eval('test', 'i')
self._print_eval('test', test_loss, test_acc)
self.logwriter.add_scalar('loss/test_loss_i', test_loss, epoch)
self.logwriter.add_scalar('acc/test_acc_i', test_acc * 100, epoch)
if test_acc >= best_test_acc:
best_test_acc = test_acc
self._save_model(self.i_encoder, 'i.pkl')
self._save_model(self.classifier, 'c.pkl')
print('i_model saved at epoch {}'.format(epoch))
def _adjust_learning_rate(self, optimizer, lr):
for param_group in optimizer.param_groups:
param_group['lr'] = lr
def _train_together(self):
best_test_acc = 0
loss = acc = loss_a = acc_a = 0
lr_t = Config.lr_t
acc_d_for_train = 0
for epoch in range(Config.together_epochs):
start_time = time.time()
if epoch < Config.first_stage_epochs:
loss, acc, loss_a, acc_a = self._pretrain_i_a_one()
else:
if epoch == Config.first_stage_epochs:
self._adjust_learning_rate(self.i_optimizer, lr_t)
self._adjust_learning_rate(self.c_optimizer, lr_t / 2)
# elif (epoch - Config.first_stage_epochs) % 20 == 0:
# lr_t *= 0.8
# self._adjust_learning_rate(self.i_optimizer, lr_t)
# self._adjust_learning_rate(self.c_optimizer, lr_t)
loss, acc, acc_d, loss_2, acc_d_for_train = self._adtrain_one(
acc_d_for_train)
self._print_train(epoch, time.time() - start_time, loss, acc)
self.logwriter.add_scalar('loss/train_loss_t', loss, epoch)
self.logwriter.add_scalar('acc/train_acc_t', acc * 100, epoch)
self.logwriter.add_scalar('loss/train_loss_t_a', loss_a, epoch)
self.logwriter.add_scalar('acc/train_acc_t_a', acc_a * 100, epoch)
if epoch >= Config.first_stage_epochs:
self.logwriter.add_scalar('acc/train_acc_d', acc_d * 100,
epoch)
self.logwriter.add_scalar('loss/train_loss_2', loss_2, epoch)
self.logwriter.add_scalar('acc/acc_d_for_train',
acc_d_for_train * 100, epoch)
dev_loss, dev_acc = self._eval('dev', 'i')
dev_loss_a, dev_acc_a = self._eval('dev', 'a')
self._print_eval('dev', dev_loss, dev_acc)
self.logwriter.add_scalar('loss/dev_loss_t', dev_loss, epoch)
self.logwriter.add_scalar('acc/dev_acc_t', dev_acc * 100, epoch)
self.logwriter.add_scalar('loss/dev_loss_t_a', dev_loss_a, epoch)
self.logwriter.add_scalar('acc/dev_acc_t_a', dev_acc_a * 100,
epoch)
if epoch >= Config.first_stage_epochs:
dev_acc_d = self._eval_d('dev')
self.logwriter.add_scalar('acc/dev_acc_d', dev_acc_d * 100,
epoch)
test_loss, test_acc = self._eval('test', 'i')
test_loss_a, test_acc_a = self._eval('test', 'a')
self._print_eval('test', test_loss, test_acc)
self.logwriter.add_scalar('loss/test_loss_t', test_loss, epoch)
self.logwriter.add_scalar('acc/test_acc_t', test_acc * 100, epoch)
self.logwriter.add_scalar('loss/test_loss_t_a', test_loss_a, epoch)
self.logwriter.add_scalar('acc/test_acc_t_a', test_acc_a * 100,
epoch)
if epoch >= Config.first_stage_epochs:
test_acc_d = self._eval_d('test')
self.logwriter.add_scalar('acc/test_acc_d', test_acc_d * 100,
epoch)
if test_acc >= best_test_acc:
best_test_acc = test_acc
self._save_model(self.i_encoder, 't_i.pkl')
self._save_model(self.classifier, 't_c.pkl')
print('t_i t_c saved at epoch {}'.format(epoch))
def train(self, i_or_t):
print('start training')
self.logwriter = SummaryWriter(Config.logdir)
if i_or_t == 'i':
self._pretrain_i()
elif i_or_t == 't':
self._train_together()
else:
raise Exception('wrong i_or_t')
print('training done')
def _eval(self, task, i_or_a):
self.i_encoder.eval()
self.a_encoder.eval()
self.classifier.eval()
total_loss = 0
correct_n = 0
if task == 'dev':
data = self.data.dev_loader
n = self.data.dev_size
elif task == 'test':
data = self.data.test_loader
n = self.data.test_size
else:
raise Exception('wrong eval task')
for a1, a2i, a2a, sense1, sense2 in data:
if self.cuda:
a1, a2i, a2a, sense1, sense2 = a1.cuda(), a2i.cuda(), a2a.cuda(
), sense1.cuda(), sense2.cuda()
a1 = Variable(a1, volatile=True)
a2i = Variable(a2i, volatile=True)
a2a = Variable(a2a, volatile=True)
sense1 = Variable(sense1, volatile=True)
sense2 = Variable(sense2, volatile=True)
if i_or_a == 'i':
output = self.classifier(self.i_encoder(a1, a2i))
elif i_or_a == 'a':
output = self.classifier(self.a_encoder(a1, a2a))
else:
raise Exception('wrong i_or_a')
_, output_sense = torch.max(output, 1)
assert output_sense.size() == sense1.size()
gold_sense = sense1
mask = (output_sense == sense2)
gold_sense[mask] = sense2[mask]
tmp = (output_sense == gold_sense).long()
correct_n += torch.sum(tmp).data
loss = self.criterion_c(output, gold_sense)
total_loss += loss.data * gold_sense.size(0)
return total_loss[0] / n, correct_n[0] / n
def _eval_d(self, task):
self.i_encoder.eval()
self.a_encoder.eval()
self.classifier.eval()
correct_n = 0
if task == 'train':
n = self.data.train_size
for a1, a2i, a2a, sense in self.data.train_loader:
if self.cuda:
a1, a2i, a2a, sense = a1.cuda(), a2i.cuda(), a2a.cuda(
), sense.cuda()
a1 = Variable(a1, volatile=True)
a2i = Variable(a2i, volatile=True)
a2a = Variable(a2a, volatile=True)
sense = Variable(sense, volatile=True)
output_i = self.discriminator(self.i_encoder(a1, a2i))
correct_n += torch.sum((output_i < 0.5).long()).data
# output_a = self.discriminator(self.a_encoder(a1, a2a))
# correct_n += torch.sum((output_a >= 0.5).long()).data
else:
if task == 'dev':
data = self.data.dev_loader
n = self.data.dev_size
elif task == 'test':
data = self.data.test_loader
n = self.data.test_size
for a1, a2i, a2a, sense1, sense2 in data:
if self.cuda:
a1, a2i, a2a, sense1, sense2 = a1.cuda(), a2i.cuda(
), a2a.cuda(), sense1.cuda(), sense2.cuda()
a1 = Variable(a1, volatile=True)
a2i = Variable(a2i, volatile=True)
a2a = Variable(a2a, volatile=True)
sense1 = Variable(sense1, volatile=True)
sense2 = Variable(sense2, volatile=True)
output_i = self.discriminator(self.i_encoder(a1, a2i))
correct_n += torch.sum((output_i < 0.5).long()).data
# output_a = self.discriminator(self.a_encoder(a1, a2a))
# correct_n += torch.sum((output_a >= 0.5).long()).data
return correct_n[0] / n
def _test_d(self):
acc = self._eval_d('dev')
phase = -100
if acc >= Config.thresh_high:
phase = 1
elif acc > Config.thresh_low:
phase = 0
else:
phase = -1
return phase
def eval(self, stage):
if stage == 'i':
self._load_model(self.i_encoder, 'i.pkl')
self._load_model(self.classifier, 'c.pkl')
test_loss, test_acc = self._eval('test', 'i')
self._print_eval('test', test_loss, test_acc)
elif stage == 't':
self._load_model(self.i_encoder, 't_i.pkl')
self._load_model(self.classifier, 't_c.pkl')
test_loss, test_acc = self._eval('test', 'i')
self._print_eval('test', test_loss, test_acc)
else:
raise Exception('wrong eval stage')
print('[Test Accuracy] Classifier Model Classification loss: {} Regression loss: {} Total Loss: {} '.format(0, 0 ,0 ))
else:
print('[Test Accuracy] Classifier Model Classification loss: {} Regression loss: {} Total Loss: {} '.format(class_class_loss / count_class, regr_class_loss / count_class ,total_class_loss/ count_class ))
print('[Test Accuracy] RPN Model Classification loss: {} Regression loss: {} Total Loss: {} '.format(class_rpn_loss / count_rpn, regr_rpn_loss / count_rpn ,total_rpn_loss/ count_rpn ))
if len(rpn_accuracy_rpn_monitor) == 0 :
print('[Test Accuracy] RPN is not producing bounding boxes that overlap the ground truth boxes. Check RPN settings or keep training.')
else:
mean_overlapping_bboxes = float(sum(rpn_accuracy_rpn_monitor))/len(rpn_accuracy_rpn_monitor)
print('[Test Accuracy] Mean number of bounding boxes from RPN overlapping ground truth boxes: {}'.format(mean_overlapping_bboxes))
return total_class_loss/ count_class + total_rpn_loss/ count_rpn
model_rpn.eval()
model_classifier.eval()
total_loss = test(0)
print(os.environ['CUDA_VISIBLE_DEVICES'])
for i in range(start_epoch + 1 , args.max_epochs):
model_rpn.train()
model_classifier.train()
train(i)
scheduler_rpn.step()
scheduler_class.step()
model_rpn.eval()
model_classifier.eval()
total_loss = test(i)
if total_loss < best_error :
save_checkpoint(i, model_rpn, model_classifier, optimizer_model_rpn, optimizer_classifier , best_error, save_dir=args.save_dir)
print("=== {} === ".format(total_loss))
#train_pred.shape == [128, 11]
batch_loss = loss(
train_pred, data[1].cuda()
) # 計算 loss （注意 prediction 跟 label 必須同時在 CPU 或是 GPU 上）(argument [prediction, y'])
batch_loss.backward() # 利用 back propagation 算出每個參數的 gradient
optimizer.step() # 以 optimizer 用 gradient 更新參數值
train_acc += np.sum(
np.argmax(train_pred.cpu().data.numpy(),
axis=1) == data[1].numpy()
) #find max of 11 class and compare with the train label
#if i == 1:
# print(data[1])
train_loss += batch_loss.item() # type(batch_loss) == torch.Tensor
model.eval() #same as model.train(False)
with torch.no_grad():
for i, data in enumerate(val_loader):
val_pred = model(data[0].cuda())
batch_loss = loss(val_pred, data[1].cuda())
val_acc += np.sum(
np.argmax(val_pred.cpu().data.numpy(), axis=1) ==
data[1].numpy())
val_loss += batch_loss.item()
print('[%03d/%03d] %2.2f sec(s) Train Acc: %3.6f Loss: %3.6f | Val Acc: %3.6f loss: %3.6f' % \
(epoch + 1, num_epoch, time.time()-epoch_start_time, \
train_acc/train_set.__len__(), train_loss/train_set.__len__(), val_acc/val_set.__len__(), val_loss/val_set.__len__()))
savemodel(model, "./", 1)
'''
class Gzsl_vae():
"""docstring for Gzsl_vae"""
def __init__(self, param):
self.device = torch.device(param.device)
######################## LOAD DATA #############################
self.scalar = MinMaxScaler()
self.trainval_set = dataloader(
param, transform=self.scalar,
split='trainval') # AWA2: 23527 40类 CUB:7057 150类
self.test_set_unseen = dataloader(
param, transform=self.scalar,
split='test_unseen') # AWA2: 7913 10类 CUB:2967 50类
self.test_set_seen = dataloader(
param, transform=self.scalar,
split='test_seen') # AWA2: 5882 40类 CUB:1764 150类
self.trainloader = data.DataLoader(self.trainval_set,
batch_size=param.batch_size,
shuffle=True)
self.input_dim = self.trainval_set.__getlen__()
self.atts_dim = self.trainval_set.__get_attlen__()
self.num_classes = self.trainval_set.__totalClasses__()
print(30 * ('-'))
print("Input_dimension=%d" % self.input_dim)
print("Attribute_dimension=%d" % self.atts_dim)
print("z=%d" % param.latent_size)
print("num_classes=%d" % self.num_classes)
print(30 * ('-'))
####################### INITIALIZE THE MODEL AND OPTIMIZER #####################
self.model_encoder = encoder_cada(input_dim=self.input_dim,
atts_dim=self.atts_dim,
z=param.latent_size).to(self.device)
self.model_decoder = decoder_cada(input_dim=self.input_dim,
atts_dim=self.atts_dim,
z=param.latent_size).to(self.device)
learnable_params = list(self.model_encoder.parameters()) + list(
self.model_decoder.parameters())
self.optimizer = optim.Adam(learnable_params,
lr=0.00015,
betas=(0.9, 0.999),
eps=1e-08,
weight_decay=0,
amsgrad=True)
self.classifier = Classifier(input_dim=param.latent_size,
num_class=self.num_classes).to(
self.device)
self.cls_optimizer = optim.Adam(self.classifier.parameters(),
lr=0.001,
betas=(0.5, 0.999))
print(self.model_encoder)
print(self.model_decoder)
print(self.classifier)
################### LOAD PRETRAINED MODEL ########################
if param.pretrained:
if param.model_path == '':
print("Please provide the path of the pretrained model.")
else:
checkpoint = torch.load(param.model_path)
self.model_encoder.load_state_dict(
checkpoint['model_encoder_state_dict'])
self.model_decoder.load_state_dict(
checkpoint['model_decoder_state_dict'])
print(">> Pretrained model loaded!")
########## LOSS ############
self.l1_loss = nn.L1Loss(reduction='sum')
self.lossfunction_classifier = nn.NLLLoss()
######### Hyper-params #######
self.gamma = torch.zeros(1, device=self.device).float()
self.beta = torch.zeros(1, device=self.device).float()
self.delta = torch.zeros(1, device=self.device).float()
def train(self, epoch):
'''
This function trains the cada_vae model
'''
if epoch > 5 and epoch < 21:
self.delta += 0.54
if epoch > 20 and epoch < 76:
self.gamma += 0.044
if epoch < 93:
self.beta += 0.0026
trainbar = tqdm(self.trainloader)
self.model_encoder.train()
self.model_decoder.train()
train_loss = 0
for batch_idx, (x, y, sig) in enumerate(trainbar):
x.requires_grad = False
sig.requires_grad = False
z_img, z_sig, mu_x, logvar_x, mu_sig, logvar_sig = self.model_encoder(
x, sig)
recon_x, recon_sig, sigDecoder_x, xDecoder_sig = self.model_decoder(
z_img, z_sig)
# loss
vae_reconstruction_loss = self.l1_loss(recon_x, x) + self.l1_loss(
recon_sig, sig)
cross_reconstruction_loss = self.l1_loss(
xDecoder_sig, x) + self.l1_loss(sigDecoder_x, sig)
KLD_loss = (
0.5 * torch.sum(1 + logvar_x - mu_x.pow(2) - logvar_x.exp())
) + (0.5 *
torch.sum(1 + logvar_sig - mu_sig.pow(2) - logvar_sig.exp()))
distributed_loss = torch.sqrt(
torch.sum((mu_x - mu_sig)**2, dim=1) +
torch.sum((torch.sqrt(logvar_x.exp()) -
torch.sqrt(logvar_sig.exp()))**2,
dim=1))
distributed_loss = distributed_loss.sum()
self.optimizer.zero_grad()
loss = vae_reconstruction_loss - self.beta * KLD_loss
if self.delta > 0:
loss += distributed_loss * self.delta
if self.gamma > 0:
loss += cross_reconstruction_loss * self.gamma
loss.backward()
self.optimizer.step()
train_loss += loss.item()
trainbar.set_description('l:%.3f' % (train_loss / (batch_idx + 1)))
#print("vae %f, da %f, ca %f"%(vae,da,ca))
print(train_loss / (batch_idx + 1))
if epoch % 100 == 0:
if not os.path.exists('./checkpoints'):
os.makedirs('./checkpoints')
name = "./checkpoints/" + "checkpoint_cada_" + opt.dataset_name + ".pth"
torch.save(
{
'epoch': epoch,
'model_encoder_state_dict':
self.model_encoder.state_dict(),
'model_decoder_state_dict':
self.model_decoder.state_dict(),
'optimizer_state_dict': self.optimizer.state_dict(),
'loss': loss,
}, name)
##################### FEATURE EXTRCTION #######################
def extract_features(self, params):
print(30 * '-')
print("Preparing dataset for the classifier..")
self.model_encoder.eval()
self.model_decoder.eval()
img_seen_feats = params['img_seen'] # 200
img_unseen_feats = params['img_unseen'] # 0
att_seen_feats = params['att_seen'] # 0
att_unseen_feats = params['att_unseen'] # 400
seen_classes = self.trainval_set.__NumClasses__() # 可见类150类
unseen_classes = self.test_set_unseen.__NumClasses__() # 不可见类50类
#atts for unseen classes
attribute_vector_unseen, labels_unseen = self.test_set_unseen.__attributeVector__(
) # shape:50*312 50
#for trainval features:
features_seen = []
labels_seen = []
k = 0
for n in seen_classes: # 150类，每一类200张
perclass_feats = self.trainval_set.__get_perclass_feats__(
n) # n=1, perclass_feats:45*2048 n=2,perclass_feats:38*2048
k += perclass_feats.shape[0]
repeat_factor = math.ceil(
img_seen_feats / perclass_feats.shape[0]
) # 向上取整 n=1, repeat_factor=5 n=2,repeat_factor=6
perclass_X = np.repeat(
perclass_feats.cpu().numpy(), repeat_factor, axis=0
) # n=1, 225*2048 n=2 228*2048 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!为什么复制重复的
perclass_labels = torch.from_numpy(
np.repeat(n.cpu().numpy(), img_seen_feats,
axis=0)).long() # 200
seen_feats = perclass_X[:img_seen_feats].astype(np.float32)
# if seen_feats.shape[0] < 200:
# print(n,"-------", seen_feats.shape)
features_seen.append(torch.from_numpy(seen_feats))
labels_seen.append(perclass_labels)
print("Number of seen features:", k)
tensor_seen_features = torch.cat(features_seen).to(
self.device) # 30000*2048
tensor_seen_feats_labels = torch.cat(labels_seen)
tensor_unseen_attributes = torch.from_numpy( # 20000*312
np.repeat(attribute_vector_unseen, att_unseen_feats,
axis=0)).float().to(self.device) # 复制400次：50*400
tensor_unseen_labels = torch.from_numpy(
np.repeat(labels_unseen, att_unseen_feats, axis=0)).long() # 20000
test_unseen_X, test_unseen_Y = self.test_set_unseen.__Test_Features_Labels__(
) # 2967*2048 2967*1
test_seen_X, test_seen_Y = self.test_set_seen.__Test_Features_Labels__(
) # 1764*2048 1764*1
with torch.no_grad():
z_img, z_att, mu_x, logvar_x, mu_att, logvar_att = self.model_encoder(
tensor_seen_features, tensor_unseen_attributes
) # 括号内的30000*2048,20000*312！！！！！！！！！！！！！！！！！！！！！！！
z_unseen_test_img, z_unseen_test_att, mu_x_unseen, logvar_x, mu_att, logvar_att = self.model_encoder(
test_unseen_X,
tensor_unseen_attributes) # 括号内的：2967*2048,20000*312
z_seen_test_img, z_unseen_test_att, mu_x_seen, logvar_x, mu_att, logvar_att = self.model_encoder(
test_seen_X, tensor_unseen_attributes) # 1764*2048， 20000*312
train_features = torch.cat((z_att, z_img)) # 50000*64
train_labels = torch.cat(
(tensor_unseen_labels, tensor_seen_feats_labels)) # 50000
test_unseen_Y = torch.squeeze(test_unseen_Y) # 2967
test_seen_Y = torch.squeeze(test_seen_Y) # 1764
print(
">> Extraction of trainval, test seen, and test unseen features are complete!"
)
print(train_features.shape, train_labels.shape)
#return train_features, train_labels, z_unseen_test_img, test_unseen_Y, z_seen_test_img, test_seen_Y
return train_features, train_labels, mu_x_unseen, test_unseen_Y, mu_x_seen, test_seen_Y
##################### TRAINING THE CLASSIFIER #######################
def train_classifier(self, epochs):
train_features, train_labels, test_unseen_features, test_unseen_labels, test_seen_features, test_seen_labels = \
self.extract_features(params)
if not os.path.exists('./datas/' + opt.dataset_name + '_features'):
os.makedirs('./datas/' + opt.dataset_name + '_features')
np.save(
'./datas/' + opt.dataset_name + '_features/' + 'test_novel_Y.npy',
test_unseen_labels.cpu().numpy())
self.cls_trainData = classifier_dataloader(
features_img=train_features,
labels=train_labels,
device=self.device) # 50000*64
self.cls_trainloader = data.DataLoader(self.cls_trainData,
batch_size=32,
shuffle=True)
self.cls_test_unseen = classifier_dataloader(
features_img=test_unseen_features,
labels=test_unseen_labels,
device=self.device) # test_unseen_features为VAE的均值
self.cls_test_unseenLoader = data.DataLoader(self.cls_test_unseen,
batch_size=32,
shuffle=False)
self.test_unseen_target_classes = self.cls_test_unseen.__targetClasses__(
)
self.cls_test_seen = classifier_dataloader(
features_img=test_seen_features,
labels=test_seen_labels,
device=self.device) # test_seen_features为VAE的均值
self.cls_test_seenLoader = data.DataLoader(self.cls_test_seen,
batch_size=32,
shuffle=False)
self.test_seen_target_classes = self.cls_test_seen.__targetClasses__()
def val_gzsl(testbar_cls):
with torch.no_grad():
self.classifier.eval()
print("**Validation**")
preds = []
target = []
for batch_idx, (x, y) in enumerate(testbar_cls):
output = self.classifier(x)
output_data = torch.argmax(output.data, 1)
preds.append(output_data)
target.append(y)
predictions = torch.cat(preds)
targets = torch.cat(target)
return predictions, targets
best_H = -1
best_seen = 0
best_unseen = 0
############## TRAINING ####################
for epoch in range(1, epochs + 1): # 1-41
print("Training: Epoch - ", epoch)
self.classifier.train()
trainbar_cls = tqdm(
self.cls_trainloader) # cls_trainloader由50000*64的数据构成
train_loss = 0
for batch_idx, (x, y) in enumerate(trainbar_cls): # y从0开始
output = self.classifier(x.to(opt.device))
loss = self.lossfunction_classifier(output, y)
self.cls_optimizer.zero_grad()
loss.backward()
self.cls_optimizer.step()
train_loss += loss.item()
trainbar_cls.set_description('l:%.3f' % (train_loss /
(batch_idx + 1)))
########## VALIDATION ##################
accu_unseen = 0
accu_seen = 0
testbar_cls_unseen = tqdm(
self.cls_test_unseenLoader) # 由2967*64的数据构成
testbar_cls_seen = tqdm(self.cls_test_seenLoader) # 由1764*64的数据构成
preds_unseen, target_unseen = val_gzsl(testbar_cls_unseen)
preds_seen, target_seen = val_gzsl(testbar_cls_seen)
########## ACCURACY METRIC ##################
def compute_per_class_acc_gzsl(test_label, predicted_label,
target_classes):
per_class_accuracies = torch.zeros(
target_classes.shape[0]).float()
predicted_label = predicted_label
for i in range(target_classes.shape[0]):
is_class = test_label == target_classes[i] # 找出是该类的
per_class_accuracies[i] = torch.div(
(predicted_label[is_class] == test_label[is_class]
).sum().float(),
is_class.sum().float())
return per_class_accuracies.mean()
##################################
'''
For NLLL loss the labels are
mapped from 0-n, map them back to 1-n
for calculating accuracies.
'''
target_unseen = target_unseen + 1
preds_unseen = preds_unseen + 1
target_seen = target_seen + 1
preds_seen = preds_seen + 1
##################################
accu_unseen = compute_per_class_acc_gzsl(
target_unseen, preds_unseen, self.test_unseen_target_classes)
accu_seen = compute_per_class_acc_gzsl(
target_seen, preds_seen, self.test_seen_target_classes)
if (accu_seen + accu_unseen) > 0:
H = (2 * accu_seen * accu_unseen) / (accu_seen + accu_unseen)
else:
H = 0
if H > best_H:
best_seen = accu_seen
best_unseen = accu_unseen
best_H = H
print(30 * '-')
print('Epoch:', epoch)
print('Best: u, s, h =%.4f,%.4f,%.4f' %
(best_unseen, best_seen, best_H))
print('u, s, h =%.4f,%.4f,%.4f' % (accu_unseen, accu_seen, H))
print(30 * '-')
return best_seen, best_unseen, best_H
class Model(TransformerMixin, BaseEstimator):
def __init__(self,
use_slower_better_model: bool,
block_name: str,
number_of_iterations: int,
optimisation_step_size: float = 1.5,
n_octaves: int = 4,
octave_scale: float = 1.4,
filter_index: int = 8,
use_gpu: bool = False,
verbose: bool = False,
seed: Optional[int] = None):
if use_slower_better_model:
self._params = constants.AUDIO_PARAMS_SLOWER_BETTER_MODEL
else:
self._params = constants.AUDIO_PARAMS_FASTER_WORSE_MODEL
self._model_path = self._params.model_path.as_posix()
self._use_gpu = use_gpu
self._block_name = block_name
self._n_octaves = n_octaves
self._octave_scale = octave_scale
self._verbose = verbose
self._number_of_iterations = number_of_iterations
self._optimisation_step_size = optimisation_step_size
self._filter_index = filter_index
self._np_rng = np.random.RandomState(seed)
self._available_layers = {}
self._available_layers_names = []
self._classifier = Classifier(constants.NUMBER_OF_CLASSES)
self._net = NeuralNet(
self._classifier, nn.CrossEntropyLoss
)
self._net.initialize()
self._net.load_params(f_params=self._model_path)
self._classifier = self._classifier.eval()
for layer_name, layer in self._classifier.layers_blocks.items():
if "residual" in layer_name:
current_register = partial(self._register_layer_output,
layer_name=layer_name)
layer.register_forward_hook(current_register)
self._available_layers_names.append(layer_name)
if self._verbose:
print(f"Available layer names: \n{self._available_layers_names}")
def _register_layer_output(self, module, input_, output, layer_name):
self._available_layers[layer_name] = output
def fit(self, x, y, **fit_params):
return self
def transform(self, x: Iterable[Tuple[np.ndarray, int, float, np.ndarray]]) \
-> Iterable[Tuple[np.ndarray, int, float, np.ndarray]]:
output = []
for stft, sample_rate, mag_max_value, phase in x:
prediction = self._transform_single_normal_deep_dream(stft)
output.append((prediction, sample_rate, mag_max_value, phase))
return output
def _transform_single_normal_deep_dream(self, stft: np.ndarray) -> np.ndarray:
octaves = []
for i in range(self._n_octaves - 1):
hw = stft.shape[:2]
lo = \
cv2.resize(stft,
tuple(np.int32(np.float32(hw[::-1]) / self._octave_scale)))[
..., None]
hi = stft - cv2.resize(lo, tuple(np.int32(hw[::-1])))[..., None]
stft = lo
octaves.append(hi)
for octave in tqdm.trange(self._n_octaves, desc="Image optimisation"):
if octave > 0:
hi = octaves[-octave]
stft = cv2.resize(stft, tuple(np.int32(hi.shape[:2][::-1])))[
..., None] + hi
stft = torch.from_numpy(stft).float()
if self._use_gpu:
stft = stft.cuda()
stft = stft.permute((2, 0, 1))
for i in tqdm.trange(self._number_of_iterations, desc="Octave optimisation"):
g = self.calc_grad_tiled(stft)
g /= (g.abs().mean() + 1e-8)
g *= self._optimisation_step_size
stft += g
if self._use_gpu:
stft = stft.cpu()
stft = stft.detach().numpy().transpose((1, 2, 0))
return stft
def calc_grad_tiled(self, stft: torch.Tensor, tile_size: int = 128) -> torch.Tensor:
h, w = stft.shape[1:]
sx, sy = self._np_rng.randint(tile_size, size=2)
stft_shift = roll(roll(stft, sx, axis=2), sy, axis=1)
grads = torch.zeros_like(stft)
for y in range(0, max(h - tile_size // 2, tile_size), tile_size):
for x in range(0, max(w - tile_size // 2, tile_size), tile_size):
frame = stft_shift[:, y:y + tile_size, x:x + tile_size]
frame.requires_grad = True
self._classifier(frame[None])
layer_output = self._available_layers[self._block_name][
0, self._filter_index]
objective_output = layer_output.mean()
objective_output.backward()
frame.requires_grad = False
grad = frame.grad.detach().clone()
grads[:, y:y + tile_size, x:x + tile_size] = grad
result = roll(roll(grads, -sx, axis=2), -sy, axis=1)
return result
f.write('epoch {}:\n'.format(epoch))
log_entry = ' | time {:.3f} | loss {:.5f} | loss_z {:.5f} | loss_recon {:.5f} | loss_classify {:.5f} \n'.format(
time.time() - epoch_start_time, loss.avg, loss_z.avg,
loss_recon.avg, loss_classify.avg)
f.write(log_entry)
#logger.log_scalar('train_loss',train_loss, epoch)
with torch.no_grad():
loss_z_val = AverageMeter()
loss_recon_val = AverageMeter()
loss_classify_val = AverageMeter()
loss_val = AverageMeter()
psnr_val = AverageMeter()
encoder.eval()
decoder.eval()
classifier.eval()
for batch_idx, data in enumerate(val_loader):
batch_start_time = time.time()
img_1 = data[0].cuda()
img_2 = data[1].cuda()
img_1_atts = data[2].cuda()
img_2_atts = data[3].cuda()
z_1 = encoder(img_1)
z_2 = encoder(img_2)
img_2_trans = decoder(z_1, img_2_atts)
img_1_trans = decoder(z_2, img_1_atts)
img_1_recon = decoder(z_1, img_1_atts)
img_2_recon = decoder(z_2, img_2_atts)
img_1_atts_pre = classifier(img_1_trans)
img_2_atts_pre = classifier(img_2_trans)
import os
from flask import Flask, request, make_response, send_from_directory
import torch
import PIL
from model import Classifier, image_transform
app = Flask(__name__, static_folder="static")
# initialize model from path
net = Classifier()
net.load_state_dict(torch.load("model.th", map_location=torch.device("cpu")))
net.eval() # IMPORTANT
# map of model outputs to ones we can understand
class_map = {
0: "cat",
1: "dog"
}
# outline routes
@app.route("/", methods=["GET"])
def hello_world(path):
return "Hello World!"
@app.route("/classify", methods=["POST"])
def classify():
output = "idk"
probability = -1
if "image" in request.files:
f = request.files["image"]
batch_loss = F.cross_entropy(p_batch, y_batch)
losses.append(batch_loss.item())
optimizer.zero_grad()
batch_loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), 3)
optimizer.step()
eval_at -= len(x_batch)
if eval_at <= 0:
avg_loss = np.mean(losses)
dev_acc = 0
dev_rmse = 0
with torch.no_grad():
for j in tqdm(range(0, len(x_dev), batch_size)):
model.eval()
x_batch, m_batch = utils.pad(x_dev[j:j + batch_size])
x_batch = to_tensor(x_batch)
m_batch = to_tensor(m_batch).float()
c_batch = to_tensor(c_dev[j:j + batch_size])
y_batch = to_tensor(y_dev[j:j + batch_size])
p_batch = model(x_batch, m_batch, c_batch).max(1)[1]
dev_acc += p_batch.eq(y_batch).long().sum().item()
p_batch = p_batch.cpu().detach().numpy()
p_batch = np.array(
[int(rev_label_dict[p]) for p in p_batch])
y_batch = y_batch.cpu().detach().numpy()
y_batch = np.array(
def main():
x_train, val_data, x_label, val_label = LoadData(sys.argv[1])
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(device)
data_transform = transforms.Compose([
transforms.ToPILImage(),
transforms.RandomResizedCrop(44),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=[0], std=[255])
])
val_transform = transforms.Compose([
transforms.ToPILImage(),
transforms.CenterCrop(44),
transforms.ToTensor(),
transforms.Normalize(mean=[0], std=[255])
])
train_set = MyDataset(x_train, x_label, data_transform)
val_set = MyDataset(val_data, val_label, val_transform)
batch_size = 128
train_loader = DataLoader(train_set,
batch_size=batch_size,
shuffle=True,
num_workers=8)
val_loader = DataLoader(val_set,
batch_size=batch_size,
shuffle=False,
num_workers=8)
model = Classifier().to(device)
model.initialize_weights()
print(model.eval())
loss = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
scheduler = ReduceLROnPlateau(optimizer, 'min')
best_acc = 0.0
num_epoch = 1000
for epoch in range(num_epoch):
adjust_learning_rate(optimizer, epoch)
epoch_start_time = time.time()
train_acc = 0.0
train_loss = 0.0
val_acc = 0.0
val_loss = 0.0
model.train()
for i, data in enumerate(train_loader):
optimizer.zero_grad()
train_pred = model(data[0].to(device))
batch_loss = loss(train_pred, data[1].to(device))
batch_loss.backward()
optimizer.step()
train_acc += np.sum(
np.argmax(train_pred.cpu().data.numpy(), axis=1) ==
data[1].numpy())
train_loss += batch_loss.item()
progress = ('#' * int(float(i) / len(train_loader) * 40)).ljust(40)
print ('[%03d/%03d] %2.2f sec(s) | %s |' % (epoch+1, num_epoch, \
(time.time() - epoch_start_time), progress), end='\r', flush=True)
model.eval()
for i, data in enumerate(val_loader):
val_pred = model(data[0].cuda())
batch_loss = loss(val_pred, data[1].to(device))
val_acc += np.sum(
np.argmax(val_pred.cpu().data.numpy(), axis=1) ==
data[1].numpy())
val_loss += batch_loss.item()
progress = ('#' * int(float(i) / len(val_loader) * 40)).ljust(40)
print ('[%03d/%03d] %2.2f sec(s) | %s |' % (epoch+1, num_epoch, \
(time.time() - epoch_start_time), progress), end='\r', flush=True)
val_acc = val_acc / val_set.__len__()
print('[%03d/%03d] %2.2f sec(s) Train Acc: %3.6f Loss: %3.6f | Val Acc: %3.6f loss: %3.6f' % \
(epoch + 1, num_epoch, time.time()-epoch_start_time, \
train_acc/train_set.__len__(), train_loss, val_acc, val_loss))
# scheduler.step(val_loss)
if (val_acc > best_acc):
best_acc = val_acc
if not os.path.exists("static_dict/"):
os.system('mkdir static_dict/')
saving_compression(model)
os.system("du static_dict/ --apparent-size --bytes --max-depth=0")
with open('acc.txt', 'w') as f:
f.write(str(epoch) + '\t' + str(val_acc) + '\t')
# torch.save(model.state_dict(), 'save/model.pth')
print('Model Saved!')
