def __init__(self, folder="/", load=True):
# Datasets
if load:
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True,
folder=folder)
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=128,
shuffle=False,
drop_last=False,
folder=folder,
nops=args.nops)
else:
self.valid_tuple = None
# Model
# self.model = VQAModel(self.train_tuple.dataset.num_answers)
is_cp = False
if "vqacpv2" in folder:
is_cp = True
if not is_cp:
self.model = VQAModel(3129)
else:
self.model = VQAModel(16039)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if load:
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def __init__(self):
# Datasets
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True,
dataset_type=args.dataset_type)
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=128,
shuffle=False,
drop_last=False,
dataset_type=args.dataset_type)
else:
self.valid_tuple = None
# Model
self.model = VQAModel(
self.train_tuple.dataset.num_answers
if not args.transfer_learning else VQADataset.get_answers_number(),
encoder_type=args.encoder_type)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
self.prepare_model()
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def __init__(self):
# Datasets
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
if args.bert_type == 'ft':
bs_infer = 256
else:
bs_infer = 1024
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=bs_infer,
shuffle=False,
drop_last=False)
else:
self.valid_tuple = None
print("args.lr is {0}".format(args.lr))
# Model
self.model = VQAModel(self.train_tuple.dataset.num_answers)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
# if type(args.lr) == type("sdfg"):
# args.lr = float(args.lr)
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total,
schedule=args.lr_schedule,
args=args)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def __init__(self):
# Datasets
self.train_tuple = get_data_tuple(
args.train, bs=args.batch_size, shuffle=True, drop_last=True
)
if args.valid != "":
self.valid_tuple = get_data_tuple(
args.valid, bs=1024,
shuffle=False, drop_last=False
)
else:
self.valid_tuple = None
# Model
self.momentum = 0.99995
self.model = VQAModel(self.train_tuple.dataset.num_answers)
self.siam_model = copy.deepcopy(self.model)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
self.siam_model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa, self.model,
label2ans=self.train_tuple.dataset.label2ans)
load_lxmert_qa(args.load_lxmert_qa, self.siam_model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
self.siam_model = self.siam_model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
self.siam_model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def __init__(self,folder="/",load=True):
# Datasets
if load:
self.train_tuple = get_data_tuple(
args.train, bs=args.batch_size, shuffle=True, drop_last=True,folder=folder
)
if args.valid != "":
self.valid_tuple = get_data_tuple(
args.valid, bs=128,
shuffle=False, drop_last=False, folder=folder,nops=args.nops
)
else:
self.valid_tuple = None
get_bias(self.train_tuple.dataset,self.valid_tuple.dataset)
# Model
# self.model = VQAModel(self.train_tuple.dataset.num_answers)
label2ans = json.load(open("/data/datasets/vqa_mutant/data/vqa/mutant_l2a/mutant_label2ans.json"))
self.model = VQAModel(len(label2ans))
self.debias = LearnedMixin(w=0.36,hid_dim=self.model.lxrt_encoder.dim)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa, self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
self.debias = self.debias.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if load :
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
class VQA:
def __init__(self):
# Datasets
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=1024,
shuffle=False,
drop_last=False)
else:
self.valid_tuple = None
# Model
self.model = VQAModel(self.train_tuple.dataset.num_answers)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def train(self, train_tuple, eval_tuple):
dset, loader, evaluator = train_tuple
iter_wrapper = (lambda x: tqdm(x, total=len(loader))
) if args.tqdm else (lambda x: x)
best_valid = 0.
for epoch in range(args.epochs):
quesid2ans = {}
for i, (ques_id, feats, boxes, sent,
target) in iter_wrapper(enumerate(loader)):
self.model.train()
self.optim.zero_grad()
feats, boxes, target = feats.cuda(), boxes.cuda(), target.cuda(
)
logit = self.model(feats, boxes, sent)
assert logit.dim() == target.dim() == 2
loss = self.bce_loss(logit, target)
loss = loss * logit.size(1)
loss.backward()
nn.utils.clip_grad_norm_(self.model.parameters(), 5.)
self.optim.step()
score, label = logit.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
log_str = "\nEpoch %d: Train %0.2f\n" % (
epoch, evaluator.evaluate(quesid2ans) * 100.)
if self.valid_tuple is not None: # Do Validation
valid_score = self.evaluate(eval_tuple)
if valid_score > best_valid:
best_valid = valid_score
self.save("BEST")
log_str += "Epoch %d: Valid %0.2f\n" % (epoch, valid_score * 100.) + \
"Epoch %d: Best %0.2f\n" % (epoch, best_valid * 100.)
print(log_str, end='')
with open(self.output + "/log.log", 'a') as f:
f.write(log_str)
f.flush()
self.save("LAST")
def predict(self, eval_tuple: DataTuple, dump=None):
"""
Predict the answers to questions in a data split.
:param eval_tuple: The data tuple to be evaluated.
:param dump: The path of saved file to dump results.
:return: A dict of question_id to answer.
"""
self.model.eval()
dset, loader, evaluator = eval_tuple
quesid2ans = {}
for i, datum_tuple in enumerate(loader):
ques_id, feats, boxes, sent = datum_tuple[:
4] # Avoid seeing ground truth
with torch.no_grad():
feats, boxes = feats.cuda(), boxes.cuda()
logit = self.model(feats, boxes, sent)
score, label = logit.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
if dump is not None:
evaluator.dump_result(quesid2ans, dump)
return quesid2ans
def evaluate(self, eval_tuple: DataTuple, dump=None):
"""Evaluate all data in data_tuple."""
quesid2ans = self.predict(eval_tuple, dump)
return eval_tuple.evaluator.evaluate(quesid2ans)
@staticmethod
def oracle_score(data_tuple):
dset, loader, evaluator = data_tuple
quesid2ans = {}
for i, (ques_id, feats, boxes, sent, target) in enumerate(loader):
_, label = target.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
return evaluator.evaluate(quesid2ans)
def save(self, name):
torch.save(self.model.state_dict(),
os.path.join(self.output, "%s.pth" % name))
def load(self, path):
print("Load model from %s" % path)
state_dict = torch.load("%s.pth" % path)
self.model.load_state_dict(state_dict)
class VQA:
def __init__(self,folder="/",load=True):
# Datasets
if load:
self.train_tuple = get_data_tuple(
args.train, bs=args.batch_size, shuffle=True, drop_last=True,folder=folder
)
if args.valid != "":
self.valid_tuple = get_data_tuple(
args.valid, bs=128,
shuffle=False, drop_last=False, folder=folder
)
else:
self.valid_tuple = None
# Model
# self.model = VQAModel(self.train_tuple.dataset.num_answers)
self.model = VQAModel(3129)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa, self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
self.yes_index=425
self.no_index=1403
self.mask_yes = torch.zeros(len(self.indexlist)).cuda()
self.mask_yes[self.yes_index]=1.0
self.mask_yes[self.no_index]=1.0
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if load :
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def train(self, train_tuple, eval_tuple):
dset, loader, evaluator = train_tuple
iter_wrapper = (lambda x: tqdm(x, total=len(loader),ascii=True)) if args.tqdm else (lambda x: x)
best_valid = 0.
for epoch in range(args.epochs):
quesid2ans = {}
for i, (ques_id, feats, boxes, ques, op, q1, q2, typetarget, q1typetarget, q2typetarget, yesnotypetargets, q1yntypetargets, q2yntypetargets, target, q1_target, q2_target) in iter_wrapper(enumerate(loader)):
self.model.train()
self.optim.zero_grad()
feats, boxes, target, yntypetarget, typetarget = feats.cuda(), boxes.cuda(), target.cuda(), yesnotypetargets.cuda(), typetarget.cuda()
op, q1typetarget, q2typetarget, q1yntypetargets, q2yntypetargets , q1_target, q2_target = op.cuda(), q1typetarget.cuda(), q2typetarget.cuda(), q1yntypetargets.cuda(), q2yntypetargets.cuda() , q1_target.cuda(), q2_target.cuda()
logit = self.model(feats, boxes, sent)
assert logit.dim() == target.dim() == 2
loss = self.bce_loss(logit, target)
loss = loss * logit.size(1)
q1logit = self.model(feats, boxes, q1)
q2logit = self.model(feats, boxes, q2)
constraint_loss = self.constraint_loss(logit,q1logit,q2logit,op)
loss = 0.5*loss + 0.5*constraint_loss
loss.backward()
nn.utils.clip_grad_norm_(self.model.parameters(), 5.)
self.optim.step()
score, label = logit.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
log_str = "\nEpoch %d: Train %0.2f\n" % (epoch, evaluator.evaluate(quesid2ans) * 100.)
if self.valid_tuple is not None: # Do Validation
valid_score = self.evaluate(eval_tuple)
if valid_score > best_valid:
best_valid = valid_score
self.save("BEST")
log_str += "Epoch %d: Valid %0.2f\n" % (epoch, valid_score * 100.) + \
"Epoch %d: Best %0.2f\n" % (epoch, best_valid * 100.)
print(log_str, end='')
with open(self.output + "/log.log", 'a') as f:
f.write(log_str)
f.flush()
self.save("LAST")
return best_valid
def rangeloss(self,x,lower,upper,lamb=4):
mean = (lower+upper)/2
sigma = (upper-lower+0.00001)/lamb
loss = 1 - torch.exp(-0.5*torch.pow(torch.div(x-mean,sigma),2))
return loss.sum()
def select_yesnoprobs(self,logit,x,op):
op_mask = torch.eq(op,x)
logit = logit[op_mask].view(-1,3129)
logit_m = logit * self.mask_yes
m = logit_m == 0
logit_m = logit_m[~m].view(-1,2)
logit_m = torch.softmax(logit_m,1)
return logit_m.select(dim=1,index=0).view(-1,1)
def constraintloss(self,logit,q1_logit,q2_logit,op):
total_loss=torch.zeros([1]).cuda()
for x in range(1,11):
logit_m= self.select_yesnoprobs(logit,x,op)
q1_logit_m= self.select_yesnoprobs(q1_logit,x,op)
q2_logit_m= self.select_yesnoprobs(q2_logit,x,op)
if logit_m.nelement()==0:
continue
ideal_logit_m = op_map[x](q1_logit_m,q2_logit_m)
rangeloss = self.mseloss(logit_m,ideal_logit_m)
total_loss+=rangeloss
return total_loss
def predict(self, eval_tuple: DataTuple, dump=None):
"""
Predict the answers to questions in a data split.
:param eval_tuple: The data tuple to be evaluated.
:param dump: The path of saved file to dump results.
:return: A dict of question_id to answer.
"""
self.model.eval()
dset, loader, evaluator = eval_tuple
quesid2ans = {}
for i, datum_tuple in tqdm(enumerate(loader),ascii=True,desc="Evaluating"):
# ques_id, feats, boxes, sent = datum_tuple[:4] # Avoid seeing ground truth
ques_id, feats, boxes, ques, op, q1, q2, typetarget, q1typetarget, q2typetarget, yesnotypetargets, q1yntypetargets, q2yntypetargets, target, q1_target, q2_target = datum_tuple
with torch.no_grad():
feats, boxes = feats.cuda(), boxes.cuda()
logit = self.model(feats, boxes, sent)
score, label = logit.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
if dump is not None:
evaluator.dump_result(quesid2ans, dump)
return quesid2ans
def evaluate(self, eval_tuple: DataTuple, dump=None):
"""Evaluate all data in data_tuple."""
quesid2ans = self.predict(eval_tuple, dump)
return eval_tuple.evaluator.evaluate(quesid2ans)
@staticmethod
def oracle_score(data_tuple):
dset, loader, evaluator = data_tuple
quesid2ans = {}
for i, datum_tuple in enumerate(loader):
ques_id, feats, boxes, ques, op, q1, q2, typetarget, q1typetarget, q2typetarget, yesnotypetargets, q1yntypetargets, q2yntypetargets, target, q1_target, q2_target = datum_tuple
_, label = target.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
return evaluator.evaluate(quesid2ans)
class VQA:
def __init__(self):
# Datasets
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=1024,
shuffle=False,
drop_last=False)
else:
self.valid_tuple = None
# Model
self.model = VQAModel(self.train_tuple.dataset.num_answers)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def train(self, train_tuple, eval_tuple):
dset, loader, evaluator = train_tuple
iter_wrapper = (lambda x: tqdm(x, total=len(loader))
) if args.tqdm else (lambda x: x)
best_valid = 0.
for epoch in range(args.epochs):
quesid2ans = {}
for i, (ques_id, feats, boxes, sent,
target) in iter_wrapper(enumerate(loader)):
self.model.train()
self.optim.zero_grad()
feats, boxes, target = feats.cuda(), boxes.cuda(), target.cuda(
)
logit = self.model(feats, boxes, sent)
assert logit.dim() == target.dim() == 2
loss = self.bce_loss(logit, target)
loss = loss * logit.size(1)
loss.backward()
nn.utils.clip_grad_norm_(self.model.parameters(), 5.)
self.optim.step()
score, label = logit.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
log_str = "\nEpoch %d: Train %0.2f\n" % (
epoch, evaluator.evaluate(quesid2ans) * 100.)
if self.valid_tuple is not None: # Do Validation
valid_score = self.evaluate(eval_tuple)
if valid_score > best_valid:
best_valid = valid_score
self.save("BEST")
log_str += "Epoch %d: Valid %0.2f\n" % (epoch, valid_score * 100.) + \
"Epoch %d: Best %0.2f\n" % (epoch, best_valid * 100.)
print(log_str, end='')
with open(self.output + "/log.log", 'a') as f:
f.write(log_str)
f.flush()
self.save("LAST")
def predict(self, eval_tuple: DataTuple, dump=None):
"""
Predict the answers to questions in a data split.
:param eval_tuple: The data tuple to be evaluated.
:param dump: The path of saved file to dump results.
:return: A dict of question_id to answer.
"""
self.model.eval()
dset, loader, evaluator = eval_tuple
question_id2img_id = {x["question_id"]: x["img_id"] for x in dset.data}
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased",
do_lower_case=True)
plt.rcParams['figure.figsize'] = (12, 10)
num_regions = 36
count = 0
quesid2ans = {}
for i, datum_tuple in enumerate(loader):
ques_id, feats, boxes, sent = datum_tuple[:
4] # Avoid seeing ground truth
with torch.no_grad():
feats, boxes = feats.cuda(), boxes.cuda()
logit = self.model(feats, boxes, sent)
for layer in [0, 4]:
for head in [0, 1]:
for datapoint in range(len(sent)):
print(count, len(sent))
count += 1
lang2vis_attention_probs = self.model.lxrt_encoder.model.bert.encoder.x_layers[
layer].lang_att_map[datapoint][head].detach(
).cpu().numpy()
vis2lang_attention_probs = self.model.lxrt_encoder.model.bert.encoder.x_layers[
layer].visn_att_map[datapoint][head].detach(
).cpu().numpy()
plt.clf()
plt.subplot(2, 3, 1)
plt.gca().set_axis_off()
plt.title("Image (regions 0-7)")
im = cv2.imread(
os.path.join(
"/mnt/8tera/claudio.greco/mscoco_trainval_2014",
question_id2img_id[
ques_id[datapoint].item()]) + ".jpg")
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
plt.imshow(im)
plt.subplot(2, 3, 2)
plt.gca().set_axis_off()
plt.title("Image (regions 8-15)")
im = cv2.imread(
os.path.join(
"/mnt/8tera/claudio.greco/mscoco_trainval_2014",
question_id2img_id[
ques_id[datapoint].item()]) + ".jpg")
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
plt.imshow(im)
plt.subplot(2, 3, 3)
plt.gca().set_axis_off()
plt.title("Image (regions 16-35)")
im = cv2.imread(
os.path.join(
"/mnt/8tera/claudio.greco/mscoco_trainval_2014",
question_id2img_id[
ques_id[datapoint].item()]) + ".jpg")
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
plt.imshow(im)
img_info = loader.dataset.imgid2img[
question_id2img_id[ques_id[datapoint].item()]]
img_h, img_w = img_info['img_h'], img_info['img_w']
unnormalized_boxes = boxes[datapoint].clone()
unnormalized_boxes[:, (0, 2)] *= img_w
unnormalized_boxes[:, (1, 3)] *= img_h
for i, bbox in enumerate(unnormalized_boxes):
if i < 8:
plt.subplot(2, 3, 1)
elif i < 16:
plt.subplot(2, 3, 2)
else:
plt.subplot(2, 3, 3)
bbox = [
bbox[0].item(), bbox[1].item(),
bbox[2].item(), bbox[3].item()
]
if bbox[0] == 0:
bbox[0] = 2
if bbox[1] == 0:
bbox[1] = 2
plt.gca().add_patch(
plt.Rectangle((bbox[0], bbox[1]),
bbox[2] - bbox[0] - 4,
bbox[3] - bbox[1] - 4,
fill=False,
edgecolor='red',
linewidth=1))
plt.gca().text(bbox[0],
bbox[1] - 2,
'%s' % i,
bbox=dict(facecolor='blue'),
fontsize=9,
color='white')
ax = plt.subplot(2, 1, 2)
plt.title("Cross-modal attention lang2vis")
tokenized_question = tokenizer.tokenize(
sent[datapoint])
tokenized_question = [
"<CLS>"
] + tokenized_question + ["<SEP>"]
transposed_attention_map = lang2vis_attention_probs[:len(
tokenized_question), :num_regions]
im = plt.imshow(transposed_attention_map,
vmin=0,
vmax=1)
for i in range(len(tokenized_question)):
for j in range(num_regions):
att_value = round(
transposed_attention_map[i, j], 1)
text = ax.text(
j,
i,
att_value,
ha="center",
va="center",
color="w" if att_value <= 0.5 else "b",
fontsize=6)
ax.set_xticks(np.arange(num_regions))
ax.set_xticklabels(list(range(num_regions)))
ax.set_yticks(np.arange(len(tokenized_question)))
ax.set_yticklabels(tokenized_question)
plt.tight_layout()
# plt.gca().set_axis_off()
plt.savefig(
"/mnt/8tera/claudio.greco/guesswhat_lxmert/guesswhat/visualization_vqa/lang2vis_question_{}_layer_{}_head_{}.png"
.format(ques_id[datapoint].item(), layer,
head),
bbox_inches='tight',
pad_inches=0.5)
plt.close()
## vis2lang
plt.clf()
plt.subplot(2, 3, 1)
plt.gca().set_axis_off()
plt.title("Image (regions 0-7)")
im = cv2.imread(
os.path.join(
"/mnt/8tera/claudio.greco/mscoco_trainval_2014",
question_id2img_id[
ques_id[datapoint].item()]) + ".jpg")
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
plt.imshow(im)
plt.subplot(2, 3, 2)
plt.gca().set_axis_off()
plt.title("Image (regions 8-15)")
im = cv2.imread(
os.path.join(
"/mnt/8tera/claudio.greco/mscoco_trainval_2014",
question_id2img_id[
ques_id[datapoint].item()]) + ".jpg")
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
plt.imshow(im)
plt.subplot(2, 3, 3)
plt.gca().set_axis_off()
plt.title("Image (regions 16-35)")
im = cv2.imread(
os.path.join(
"/mnt/8tera/claudio.greco/mscoco_trainval_2014",
question_id2img_id[
ques_id[datapoint].item()]) + ".jpg")
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
plt.imshow(im)
img_info = loader.dataset.imgid2img[
question_id2img_id[ques_id[datapoint].item()]]
img_h, img_w = img_info['img_h'], img_info['img_w']
unnormalized_boxes = boxes[datapoint].clone()
unnormalized_boxes[:, (0, 2)] *= img_w
unnormalized_boxes[:, (1, 3)] *= img_h
for i, bbox in enumerate(unnormalized_boxes):
if i < 8:
plt.subplot(2, 3, 1)
elif i < 16:
plt.subplot(2, 3, 2)
else:
plt.subplot(2, 3, 3)
bbox = [
bbox[0].item(), bbox[1].item(),
bbox[2].item(), bbox[3].item()
]
if bbox[0] == 0:
bbox[0] = 2
if bbox[1] == 0:
bbox[1] = 2
plt.gca().add_patch(
plt.Rectangle((bbox[0], bbox[1]),
bbox[2] - bbox[0] - 4,
bbox[3] - bbox[1] - 4,
fill=False,
edgecolor='red',
linewidth=1))
plt.gca().text(bbox[0],
bbox[1] - 2,
'%s' % i,
bbox=dict(facecolor='blue'),
fontsize=9,
color='white')
ax = plt.subplot(2, 1, 2)
plt.title("Cross-modal attention vis2lang")
tokenized_question = tokenizer.tokenize(
sent[datapoint])
tokenized_question = [
"<CLS>"
] + tokenized_question + ["<SEP>"]
transposed_attention_map = vis2lang_attention_probs.transpose(
)[:len(tokenized_question), :num_regions]
im = plt.imshow(transposed_attention_map,
vmin=0,
vmax=1)
for i in range(len(tokenized_question)):
for j in range(num_regions):
att_value = round(
transposed_attention_map[i, j], 1)
text = ax.text(
j,
i,
att_value,
ha="center",
va="center",
color="w" if att_value <= 0.5 else "b",
fontsize=6)
ax.set_xticks(np.arange(num_regions))
ax.set_xticklabels(list(range(num_regions)))
ax.set_yticks(np.arange(len(tokenized_question)))
ax.set_yticklabels(tokenized_question)
plt.tight_layout()
# plt.gca().set_axis_off()
plt.savefig(
"/mnt/8tera/claudio.greco/guesswhat_lxmert/guesswhat/visualization_vqa/vis2lang_question_{}_layer_{}_head_{}.png"
.format(ques_id[datapoint].item(), layer,
head),
bbox_inches='tight',
pad_inches=0.5)
plt.close()
# print(datapoint, len(sent))
#
# print(datapoint)
# if datapoint > 20:
# break
# if datapoint > 20:
# break
# if datapoint > 20:
# break
score, label = logit.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
if dump is not None:
evaluator.dump_result(quesid2ans, dump)
return quesid2ans
def evaluate(self, eval_tuple: DataTuple, dump=None):
"""Evaluate all data in data_tuple."""
quesid2ans = self.predict(eval_tuple, dump)
return eval_tuple.evaluator.evaluate(quesid2ans)
@staticmethod
def oracle_score(data_tuple):
dset, loader, evaluator = data_tuple
quesid2ans = {}
for i, (ques_id, feats, boxes, sent, target) in enumerate(loader):
_, label = target.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
return evaluator.evaluate(quesid2ans)
def save(self, name):
torch.save(self.model.state_dict(),
os.path.join(self.output, "%s.pth" % name))
def load(self, path):
print("Load model from %s" % path)
state_dict = torch.load("%s.pth" % path)
self.model.load_state_dict(state_dict)
class VQA:
def __init__(self):
# Datasets
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
if args.valid != "":
valid_bsize = args.get("valid_batch_size", 16)
self.valid_tuple = get_data_tuple(args.valid,
bs=valid_bsize,
shuffle=False,
drop_last=False)
else:
self.valid_tuple = None
# Model
self.model = VQAModel(self.train_tuple.dataset.num_answers)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.get("load_lxmert_pretrain", None) is not None:
load_lxmert_from_pretrain_noqa(args.load_lxmert_pretrain,
self.model)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
self.model.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def train(self, train_tuple, eval_tuple):
dset, loader, evaluator = train_tuple
iter_wrapper = (lambda x: tqdm(x, total=len(loader))
) if args.tqdm else (lambda x: x)
best_valid = 0.
train_results = []
report_every = args.get("report_every", 100)
for epoch in range(args.epochs):
quesid2ans = {}
for i, batch in iter_wrapper(enumerate(loader)):
ques_id, feats, boxes, sent, tags, target = zip(*batch)
self.model.train()
self.optim.zero_grad()
target = torch.stack(target).cuda()
logit = self.model(feats, boxes, sent, tags)
assert logit.dim() == target.dim() == 2
loss = self.bce_loss(logit, target)
loss = loss * logit.size(1)
loss.backward()
nn.utils.clip_grad_norm_(self.model.parameters(), 5.)
self.optim.step()
train_results.append(
pd.Series({"loss": loss.detach().mean().item()}))
score, label = logit.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid] = ans
if i % report_every == 0 and i > 0:
print("Epoch: {}, Iter: {}/{}".format(
epoch, i, len(loader)))
print(" {}\n~~~~~~~~~~~~~~~~~~\n".format(
pd.DataFrame(train_results[-report_every:]).mean()))
log_str = "\nEpoch %d: Train %0.2f\n" % (
epoch, evaluator.evaluate(quesid2ans) * 100.)
if self.valid_tuple is not None: # Do Validation
valid_score = self.evaluate(eval_tuple)
if valid_score > best_valid and not args.get(
"special_test", False):
best_valid = valid_score
self.save("BEST")
log_str += "Epoch %d: Valid %0.2f\n" % (epoch, valid_score * 100.) + \
"Epoch %d: Best %0.2f\n" % (epoch, best_valid * 100.)
if epoch >= 5:
self.save("Epoch{}".format(epoch))
print(log_str, end='')
print(args.output)
self.save("LAST")
def predict(self, eval_tuple: DataTuple, dump=None):
"""
Predict the answers to questions in a data split.
:param eval_tuple: The data tuple to be evaluated.
:param dump: The path of saved file to dump results.
:return: A dict of question_id to answer.
"""
self.model.eval()
dset, loader, evaluator = eval_tuple
quesid2ans = {}
for i, batch in enumerate(tqdm(loader)):
_ = list(zip(*batch))
ques_id, feats, boxes, sent, tags = _[:5] #, target = zip(*batch)
with torch.no_grad():
#target = torch.stack(target).cuda()
logit = self.model(feats, boxes, sent, tags)
score, label = logit.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid] = ans
if dump is not None:
evaluator.dump_result(quesid2ans, dump)
return quesid2ans
def evaluate(self, eval_tuple: DataTuple, dump=None):
"""Evaluate all data in data_tuple."""
quesid2ans = self.predict(eval_tuple, dump)
return eval_tuple.evaluator.evaluate(quesid2ans)
@staticmethod
def oracle_score(data_tuple):
dset, loader, evaluator = data_tuple
quesid2ans = {}
for i, (ques_id, feats, boxes, sent, target) in enumerate(loader):
_, label = target.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
return evaluator.evaluate(quesid2ans)
def save(self, name):
torch.save(self.model.state_dict(),
os.path.join(self.output, "%s.pth" % name))
def load(self, path):
print("Load model from %s" % path)
state_dict = torch.load("%s.pth" % path)
self.model.load_state_dict(state_dict)
class VQA:
MAX_SIZE = 1333
MIN_SIZE = 800
def __init__(self):
# Datasets
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=args.batch_size,
shuffle=False,
drop_last=False)
else:
self.valid_tuple = None
# Model
self.model = VQAModel(self.train_tuple.dataset.num_answers)
self.args = self.model.args
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def _image_transform(self, path):
img = Image.open(path)
im = np.array(img).astype(np.float32)
# IndexError: too many indices for array, grayscale images
if len(im.shape) < 3:
im = np.repeat(im[:, :, np.newaxis], 3, axis=2)
im = im[:, :, ::-1]
im -= np.array([102.9801, 115.9465, 122.7717])
im_shape = im.shape
im_height = im_shape[0]
im_width = im_shape[1]
im_size_min = np.min(im_shape[0:2])
im_size_max = np.max(im_shape[0:2])
# Scale based on minimum size
im_scale = self.MIN_SIZE / im_size_min
# Prevent the biggest axis from being more than max_size
# If bigger, scale it down
if np.round(im_scale * im_size_max) > self.MAX_SIZE:
im_scale = self.MAX_SIZE / im_size_max
im = cv2.resize(im,
None,
None,
fx=im_scale,
fy=im_scale,
interpolation=cv2.INTER_LINEAR)
img = torch.from_numpy(im).permute(2, 0, 1)
im_info = {"width": im_width, "height": im_height}
return img, im_scale, im_info
def _process_feature_extraction(self,
output,
im_scales,
im_infos,
feature_name="fc6",
conf_thresh=0):
batch_size = len(output[0]["proposals"])
n_boxes_per_image = [len(boxes) for boxes in output[0]["proposals"]]
score_list = output[0]["scores"].split(n_boxes_per_image)
score_list = [torch.nn.functional.softmax(x, -1) for x in score_list]
feats = output[0][feature_name].split(n_boxes_per_image)
cur_device = score_list[0].device
feat_list = []
info_list = []
for i in range(batch_size):
dets = output[0]["proposals"][i].bbox / im_scales[i]
scores = score_list[i]
max_conf = torch.zeros((scores.shape[0])).to(cur_device)
conf_thresh_tensor = torch.full_like(max_conf, conf_thresh)
start_index = 1
# Column 0 of the scores matrix is for the background class
if self.args.background:
start_index = 0
for cls_ind in range(start_index, scores.shape[1]):
cls_scores = scores[:, cls_ind]
keep = nms(dets, cls_scores, 0.5)
max_conf[keep] = torch.where(
# Better than max one till now and minimally greater than conf_thresh
(cls_scores[keep] > max_conf[keep])
& (cls_scores[keep] > conf_thresh_tensor[keep]),
cls_scores[keep],
max_conf[keep],
)
sorted_scores, sorted_indices = torch.sort(max_conf,
descending=True)
num_boxes = (sorted_scores[:self.args.num_features] != 0).sum()
keep_boxes = sorted_indices[:self.args.num_features]
feat = feats[i][keep_boxes]
feat_list.append(feat)
bbox = output[0]["proposals"][i][keep_boxes].bbox / im_scales[i]
# Normalize the boxes (to 0 ~ 1)
img_h, img_w = im_infos[i]['height'], im_infos[i]['width']
# boxes = boxes.copy()
bbox[:, (0, 2)] /= img_w
bbox[:, (1, 3)] /= img_h
info_list.append(bbox)
# print('size:', bbox.size(), feat.size())
return feat_list, info_list
def predict(self, eval_tuple: DataTuple, dump=None):
"""
Predict the answers to questions in a data split.
:param eval_tuple: The data tuple to be evaluated.
:param dump: The path of saved file to dump results.
:return: A dict of question_id to answer.
"""
self.model.eval()
dset, loader, evaluator = eval_tuple
quesid2ans = {}
import time
from tqdm import tqdm
import torchprof
# import torch.autograd.profiler as profiler
start = time.time()
print('model set up, starting warming up prediction...')
count = 0
batches = 0
# with torch.no_grad(), profiler.profile(record_shapes=True) as prof:
with torch.no_grad():
for i, datum_tuple in tqdm(enumerate(loader)):
ques_id, img_paths, sent = datum_tuple[:
3] # Avoid seeing ground truth
img_tensor, im_scales, im_infos = [], [], []
for img_path in img_paths:
im, im_scale, im_info = self._image_transform(img_path)
# im, im_scale, im_info = img_item
img_tensor.append(im)
im_scales.append(im_scale)
im_infos.append(im_info)
current_img_list = to_image_list(img_tensor, size_divisible=32)
# print('current_img_list.device', current_img_list.tensors.size())
current_img_list = current_img_list.to("cuda")
output = self.model.detection_model(current_img_list)
# get bbox and features
feat_list, info_list = self._process_feature_extraction(
output,
im_scales,
im_infos,
self.args.feature_name,
self.args.confidence_threshold,
)
feats = torch.stack(feat_list)
boxes = torch.stack(info_list)
# feats, boxes = feats.cuda(), boxes.cuda()
logit = self.model(feats, boxes, sent)
score, label = logit.max(1)
batches += 1
if batches >= 2:
break
batches = 0
count = 0
print('model warmed up, starting predicting...')
with torch.no_grad(), torchprof.Profile(self.model,
use_cuda=True) as prof:
for i, datum_tuple in tqdm(enumerate(loader)):
ques_id, img_paths, sent = datum_tuple[:
3] # Avoid seeing ground truth
img_tensor, im_scales, im_infos = [], [], []
for img_path in img_paths:
im, im_scale, im_info = self._image_transform(img_path)
# im, im_scale, im_info = img_item
img_tensor.append(im)
im_scales.append(im_scale)
im_infos.append(im_info)
current_img_list = to_image_list(img_tensor, size_divisible=32)
# print('current_img_list.device', current_img_list.tensors.size())
current_img_list = current_img_list.to("cuda")
output = self.model.detection_model(current_img_list)
# get bbox and features
feat_list, info_list = self._process_feature_extraction(
output,
im_scales,
im_infos,
self.args.feature_name,
self.args.confidence_threshold,
)
feats = torch.stack(feat_list)
boxes = torch.stack(info_list)
# feats, boxes = feats.cuda(), boxes.cuda()
logit = self.model(feats, boxes, sent)
score, label = logit.max(1)
batches += 1
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
count += 1
print(prof.display(show_events=False))
end = time.time()
trace, event_lists_dict = prof.raw()
import pickle
with open(args.profile_save or 'profile.pk', 'wb') as f:
pickle.dump(event_lists_dict, f)
print('prediction finished!', end - start, batches, count)
if dump is not None:
evaluator.dump_result(quesid2ans, dump)
return quesid2ans
def evaluate(self, eval_tuple: DataTuple, dump=None):
"""Evaluate all data in data_tuple."""
quesid2ans = self.predict(eval_tuple, dump)
return eval_tuple.evaluator.evaluate(quesid2ans)
@staticmethod
def oracle_score(data_tuple):
dset, loader, evaluator = data_tuple
quesid2ans = {}
for i, (ques_id, feats, boxes, sent, target) in enumerate(loader):
_, label = target.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
return evaluator.evaluate(quesid2ans)
def save(self, name):
torch.save(self.model.state_dict(),
os.path.join(self.output, "%s.pth" % name))
def load(self, path):
# FIXME: load correct checkpoints
print("Load model from %s" % path)
state_dict = torch.load("%s.pth" % path)
print(self.args.model_file)
checkpoint = torch.load(self.args.model_file,
map_location=torch.device("cpu"))
detection_stat_dict = checkpoint.pop("model")
state_dict.update(detection_stat_dict)
# print(checkpoint)
# load_state_dict(model, checkpoint.pop("model"))
self.model.load_state_dict(state_dict)
class VQA:
def __init__(self):
# Datasets
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=1024,
shuffle=False,
drop_last=False)
else:
self.valid_tuple = None
# Model
self.model = VQAModel(self.train_tuple.dataset.num_answers,
finetune_strategy=args.finetune_strategy)
# if finetune strategy is spottune
if args.finetune_strategy in PolicyStrategies:
self.policy_model = PolicyLXRT(
PolicyStrategies[args.finetune_strategy])
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.finetune_strategy in PolicyStrategies:
self.policy_model = self.policy_model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
self.policy_model.policy_lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Optimizer for policy net
if args.finetune_strategy in PolicyStrategies:
self.policy_optim = args.policy_optimizer(
self.policy_model.parameters(), args.policy_lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def train(self, train_tuple, eval_tuple, visualizer=None):
dset, loader, evaluator = train_tuple
iter_wrapper = (lambda x: tqdm(x, total=len(loader))
) if args.tqdm else (lambda x: x)
wandb.watch(self.model, log='all')
if args.finetune_strategy in PolicyStrategies:
wandb.watch(self.policy_model, log='all')
best_valid = 0.
for epoch in range(args.epochs):
# for policy vec plotting
if args.finetune_strategy in PolicyStrategies:
policy_save = torch.zeros(
PolicyStrategies[args.finetune_strategy] // 2).cpu()
policy_max = 0
quesid2ans = {}
for i, (ques_id, feats, boxes, sent,
target) in iter_wrapper(enumerate(loader)):
self.model.train()
self.optim.zero_grad()
if args.finetune_strategy in PolicyStrategies:
self.policy_model.train()
self.policy_optim.zero_grad()
feats, boxes, target = feats.cuda(), boxes.cuda(), target.cuda(
)
if args.finetune_strategy in PolicyStrategies:
# calculate the policy vector here
policy_vec = self.policy_model(feats, boxes, sent)
policy_action = gumbel_softmax(
policy_vec.view(policy_vec.size(0), -1, 2))
policy = policy_action[:, :, 1]
policy_save = policy_save + policy.clone().detach().cpu(
).sum(0)
policy_max += policy.size(0)
logit = self.model(feats, boxes, sent, policy)
else:
logit = self.model(feats, boxes, sent)
assert logit.dim() == target.dim() == 2
loss = self.bce_loss(logit, target)
loss = loss * logit.size(1)
loss.backward()
nn.utils.clip_grad_norm_(self.model.parameters(), 5.)
self.optim.step()
if args.finetune_strategy in PolicyStrategies:
self.policy_optim.step()
score, label = logit.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
# check if visualizer is not none
if visualizer is not None:
print(f'Creating training visualizations for epoch {epoch}')
visualizer.plot(policy_save,
policy_max,
epoch=epoch,
mode='train')
train_acc = evaluator.evaluate(quesid2ans) * 100.
log_str = "\nEpoch %d: Train %0.2f\n" % (epoch, train_acc)
wandb.log({'Training Accuracy': train_acc})
if self.valid_tuple is not None: # Do Validation
valid_score = self.evaluate(eval_tuple,
epoch=epoch,
visualizer=visualizer)
if valid_score > best_valid:
best_valid = valid_score
self.save("BEST")
log_str += "Epoch %d: Valid %0.2f\n" % (epoch, valid_score * 100.) + \
"Epoch %d: Best %0.2f\n" % (epoch, best_valid * 100.)
wandb.log({'Validation Accuracy': valid_score * 100.})
print(log_str, end='')
with open(self.output + "/log.log", 'a') as f:
f.write(log_str)
f.flush()
self.save("LAST")
def predict(self,
eval_tuple: DataTuple,
dump=None,
epoch=0,
visualizer=None):
"""
Predict the answers to questions in a data split.
:param eval_tuple: The data tuple to be evaluated.
:param dump: The path of saved file to dump results.
:return: A dict of question_id to answer.
"""
self.model.eval()
if args.finetune_strategy in PolicyStrategies:
self.policy_model.eval()
policy_save = torch.zeros(
PolicyStrategies[args.finetune_strategy] // 2)
policy_max = 0
dset, loader, evaluator = eval_tuple
quesid2ans = {}
for i, datum_tuple in enumerate(loader):
ques_id, feats, boxes, sent = datum_tuple[:
4] # Avoid seeing ground truth
with torch.no_grad():
feats, boxes = feats.cuda(), boxes.cuda()
if args.finetune_strategy in PolicyStrategies:
# calculate the policy vector here
policy_vec = self.policy_model(feats, boxes, sent)
policy_action = gumbel_softmax(
policy_vec.view(policy_vec.size(0), -1, 2))
policy = policy_action[:, :, 1]
policy_save = policy_save + policy.clone().detach().cpu(
).sum(0)
policy_max += policy.size(0)
logit = self.model(feats, boxes, sent, policy)
else:
logit = self.model(feats, boxes, sent)
score, label = logit.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
if visualizer is not None:
print(f'Creating validation visualization for epoch {epoch}...')
visualizer.plot(policy_save, policy_max, epoch=epoch, mode='val')
if dump is not None:
evaluator.dump_result(quesid2ans, dump)
return quesid2ans
def evaluate(self,
eval_tuple: DataTuple,
dump=None,
epoch=0,
visualizer=None):
"""Evaluate all data in data_tuple."""
quesid2ans = self.predict(eval_tuple,
dump,
epoch=epoch,
visualizer=visualizer)
return eval_tuple.evaluator.evaluate(quesid2ans)
@staticmethod
def oracle_score(data_tuple):
dset, loader, evaluator = data_tuple
quesid2ans = {}
for i, (ques_id, feats, boxes, sent, target) in enumerate(loader):
_, label = target.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
return evaluator.evaluate(quesid2ans)
def save(self, name):
torch.save(self.model.state_dict(),
os.path.join(self.output, "%s.pth" % name))
def load(self, path):
print("Load model from %s" % path)
state_dict = torch.load("%s.pth" % path)
self.model.load_state_dict(state_dict)
class VQA:
def __init__(self):
# Datasets
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=1024,
shuffle=False,
drop_last=False)
else:
self.valid_tuple = None
# Model
self.model = VQAModel(self.train_tuple.dataset.num_answers)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def train(self,
train_tuple,
eval_tuple,
adversarial=False,
adv_batch_prob=0.0,
attack_name=None,
attack_params={}):
dset, loader, evaluator = train_tuple
iter_wrapper = (lambda x: tqdm(x, total=len(loader))
) if args.tqdm else (lambda x: x)
use_adv_batch = False
best_valid = 0.
for epoch in range(args.epochs):
quesid2ans = {}
# Count the number of batches that were adversarially perturbed
n_adv_batches = 0
for i, (ques_id, feats, boxes, sent,
target) in iter_wrapper(enumerate(loader)):
self.model.train()
self.optim.zero_grad()
feats, boxes, target = feats.cuda(), boxes.cuda(), target.cuda(
)
# If doing adversarial training, perturb input features
# with probability adv_batch_prob
if adversarial:
rand = random.uniform(0, 1)
use_adv_batch = rand <= adv_batch_prob
if use_adv_batch:
# Create adversary from given class name and parameters
n_adv_batches += 1
AdversaryClass_ = getattr(advertorch_module, attack_name)
adversary = AdversaryClass_(
lambda x: self.model(x, boxes, sent),
loss_fn=self.bce_loss,
**attack_params)
# Perturb feats using adversary
feats = adversary.perturb(feats, target)
logit = self.model(feats, boxes, sent)
assert logit.dim() == target.dim() == 2
loss = self.bce_loss(logit, target)
loss = loss * logit.size(1)
loss.backward()
nn.utils.clip_grad_norm_(self.model.parameters(), 5.)
self.optim.step()
score, label = logit.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
log_str = "\nEpoch %d: Train %0.2f\n" % (epoch, evaluator.evaluate(quesid2ans) * 100.) + \
"Epoch %d: Num adversarial batches %d / %d\n" % (epoch, n_adv_batches, i+1)
if self.valid_tuple is not None: # Do Validation
valid_score = self.evaluate(eval_tuple)
if valid_score > best_valid:
best_valid = valid_score
self.save("BEST")
log_str += "Epoch %d: Valid %0.2f\n" % (epoch, valid_score * 100.) + \
"Epoch %d: Best %0.2f\n" % (epoch, best_valid * 100.)
print(log_str, end='')
with open(self.output + "/log.log", 'a') as f:
f.write(log_str)
f.flush()
self.save("LAST")
def predict(self, eval_tuple: DataTuple, dump=None):
"""
Predict the answers to questions in a data split.
:param eval_tuple: The data tuple to be evaluated.
:param dump: The path of saved file to dump results.
:return: A dict of question_id to answer.
"""
self.model.eval()
dset, loader, evaluator = eval_tuple
quesid2ans = {}
for i, datum_tuple in enumerate(loader):
ques_id, feats, boxes, sent = datum_tuple[:
4] # Avoid seeing ground truth
with torch.no_grad():
feats, boxes = feats.cuda(), boxes.cuda()
logit = self.model(feats, boxes, sent)
score, label = logit.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
if dump is not None:
evaluator.dump_result(quesid2ans, dump)
return quesid2ans
def adversarial_predict(self,
eval_tuple: DataTuple,
dump=None,
attack_name='GradientAttack',
attack_params={}):
"""
Predict the answers to questions in a data split, but
using a specified adversarial attack on the inputs.
:param eval_tuple: The data tuple to be evaluated.
:param dump: The path of saved file to dump results.
:return: A dict of question_id to answer.
"""
self.model.eval()
dset, loader, evaluator = eval_tuple
quesid2ans = {}
sim_trace = [] # Track avg cos similarity across batches
for i, datum_tuple in enumerate(tqdm(loader)):
ques_id, feats, boxes, sent, target = datum_tuple
feats, boxes, target = feats.cuda(), boxes.cuda(), target.cuda()
# Create adversary from given class name and parameters
AdversaryClass_ = getattr(advertorch_module, attack_name)
adversary = AdversaryClass_(lambda x: self.model(x, boxes, sent),
loss_fn=self.bce_loss,
**attack_params)
# Perturb feats using adversary
feats_adv = adversary.perturb(feats, target)
# Compute average cosine similarity between true
# and perturbed features
sim_trace.append(self.avg_cosine_sim(feats, feats_adv))
# Compute prediction on adversarial examples
with torch.no_grad():
feats_adv = feats_adv.cuda()
logit = self.model(feats_adv, boxes, sent)
score, label = logit.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
if dump is not None:
evaluator.dump_result(quesid2ans, dump)
print(
f"Average cosine similarity across batches: {torch.mean(torch.Tensor(sim_trace))}"
)
return quesid2ans
def evaluate(self, eval_tuple: DataTuple, dump=None):
"""Evaluate all data in data_tuple."""
quesid2ans = self.predict(eval_tuple, dump)
return eval_tuple.evaluator.evaluate(quesid2ans)
def adversarial_evaluate(self,
eval_tuple: DataTuple,
dump=None,
attack_name='GradientAttack',
attack_params={}):
"""Evaluate model on adversarial inputs"""
quesid2ans = self.adversarial_predict(eval_tuple, dump, attack_name,
attack_params)
return eval_tuple.evaluator.evaluate(quesid2ans)
def avg_cosine_sim(self, feats: torch.Tensor, feats_adv: torch.Tensor):
"""Computes the average cosine similarity between true and adversarial examples"""
return nn.functional.cosine_similarity(feats, feats_adv, dim=-1).mean()
@staticmethod
def oracle_score(data_tuple):
dset, loader, evaluator = data_tuple
quesid2ans = {}
for i, (ques_id, feats, boxes, sent, target) in enumerate(loader):
_, label = target.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
return evaluator.evaluate(quesid2ans)
def save(self, name):
torch.save(self.model.state_dict(),
os.path.join(self.output, "%s.pth" % name))
def load(self, path):
print("Load model from %s" % path)
state_dict = torch.load("%s.pth" % path)
self.model.load_state_dict(state_dict)
def __init__(self, attention=False):
# Datasets
print("Fetching data")
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True,
dataset_name="test")
print("Got data")
print("fetching val data")
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=args.batch_size,
shuffle=False,
drop_last=False,
dataset_name="test")
print("got data")
else:
self.valid_tuple = None
print("Got data")
# Model
print("Making model")
self.model = VQAModel(self.train_tuple.dataset.num_answers, attention)
print("Ready model")
# Print model info:
print("Num of answers:")
print(self.train_tuple.dataset.num_answers)
# print("Model info:")
# print(self.model)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
class VQA:
def __init__(self, attention=False):
# Datasets
print("Fetching data")
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True,
dataset_name="test")
print("Got data")
print("fetching val data")
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=args.batch_size,
shuffle=False,
drop_last=False,
dataset_name="test")
print("got data")
else:
self.valid_tuple = None
print("Got data")
# Model
print("Making model")
self.model = VQAModel(self.train_tuple.dataset.num_answers, attention)
print("Ready model")
# Print model info:
print("Num of answers:")
print(self.train_tuple.dataset.num_answers)
# print("Model info:")
# print(self.model)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def train(self, train_tuple, eval_tuple):
log_freq = 810
dset, loader, evaluator = train_tuple
iter_wrapper = (lambda x: tqdm(x, total=len(loader))
) if args.tqdm else (lambda x: x)
best_valid = 0.
flag = True
for epoch in range(args.epochs):
quesid2ans = {}
correct = 0
total_loss = 0
total = 0
print("Len of the dataloader: ", len(loader))
# Our new TGIFQA-Dataset returns:
# return gif_tensor, self.questions[i], self.ans2id[self.answer[i]]
for i, (feats1, feats2, sent,
target) in iter_wrapper(enumerate(loader)):
ques_id, boxes = -1, None
self.model.train()
self.optim.zero_grad()
feats1, feats2, target = feats1.cuda(), feats2.cuda(
), target.cuda()
feats = [feats1, feats2]
logit = self.model(feats, boxes, sent)
assert logit.dim() == target.dim() == 2
loss = self.bce_loss(logit, target)
loss = loss * logit.size(1)
total_loss += loss.item()
loss.backward()
nn.utils.clip_grad_norm_(self.model.parameters(), 5.)
self.optim.step()
score, label = logit.max(1)
score_t, target = target.max(1)
correct += (label == target).sum().cpu().numpy()
total += len(label)
#if epoch > -1:
#for l,s,t in zip(label, sent, target):
# print(l)
# print(s)
# print("Prediction", loader.dataset.label2ans[int(l.cpu().numpy())])
# print("Answer", loader.dataset.label2ans[int(t.cpu().numpy())])
if i % log_freq == 1 and i > 1:
results = []
for l, s, t in zip(label, sent, target):
result = []
result.append(s)
result.append("Prediction: {}".format(
loader.dataset.label2ans[int(l.cpu().numpy())]))
result.append("Answer: {}".format(
loader.dataset.label2ans[int(t.cpu().numpy())]))
results.append(result)
torch.cuda.empty_cache()
val_loss, val_acc, val_results = self.val(eval_tuple)
logger.log(total_loss / total, correct / total * 100,
val_loss, val_acc, epoch, results, val_results)
print("==" * 30)
print("Accuracy = ", correct / total * 100)
print("Loss =", total_loss / total)
print("==" * 30)
# log_str = "\nEpoch %d: Train %0.2f\n" % (epoch, evaluator.evaluate(quesid2ans) * 100.)
# if self.valid_tuple is not None: # Do Validation
# valid_score = self.evaluate(eval_tuple)
# if valid_score > best_valid:
# best_valid = valid_score
# self.save("BEST")
# log_str += "Epoch %d: Valid %0.2f\n" % (epoch, valid_score * 100.) + \
# "Epoch %d: Best %0.2f\n" % (epoch, best_valid * 100.)
# print(log_str, end='')
# with open(self.output + "/log.log", 'a') as f:
# f.write(log_str)
# f.flush()
self.save("Check" + str(epoch))
def val(self, eval_tuple):
dset, loader, evaluator = eval_tuple
iter_wrapper = (lambda x: tqdm(x, total=len(loader))
) if args.tqdm else (lambda x: x)
self.model.eval()
best_valid = 0.
flag = True
quesid2ans = {}
correct = 0
total_loss = 0
total = 0
results = []
print("Len of the dataloader: ", len(loader))
# Our new TGIFQA-Dataset returns:
# return gif_tensor, self.questions[i], self.ans2id[self.answer[i]]
with torch.no_grad():
for i, (feats1, feats2, sent,
target) in iter_wrapper(enumerate(loader)):
ques_id, boxes = -1, None
feats1, feats2, target = feats1.cuda(), feats2.cuda(
), target.cuda()
feats = [feats1, feats2]
logit = self.model(feats, boxes, sent)
assert logit.dim() == target.dim() == 2
loss = self.bce_loss(logit, target)
loss = loss * logit.size(1)
total_loss += loss.item()
score, label = logit.max(1)
score_t, target = target.max(1)
correct += (label == target).sum().cpu().numpy()
total += len(label)
for l, s, t in zip(label, sent, target):
result = []
result.append(s)
result.append("Prediction: {}".format(
loader.dataset.label2ans[int(l.cpu().numpy())]))
result.append("Answer: {}".format(
loader.dataset.label2ans[int(t.cpu().numpy())]))
results.append(result)
return total_loss / total, correct / total * 100, results
def predict(self, eval_tuple: DataTuple, dump=None):
"""
Predict the answers to questions in a data split.
:param eval_tuple: The data tuple to be evaluated.
:param dump: The path of saved file to dump results.
:return: A dict of question_id to answer.
"""
self.model.eval()
dset, loader, evaluator = eval_tuple
quesid2ans = {}
for i, datum_tuple in enumerate(loader):
ques_id, feats, boxes, sent = datum_tuple[:
4] # Avoid seeing ground truth
with torch.no_grad():
feats, boxes = feats.cuda(), boxes.cuda()
logit = self.model(feats, boxes, sent)
score, label = logit.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
if dump is not None:
evaluator.dump_result(quesid2ans, dump)
return quesid2ans
def evaluate(self, eval_tuple: DataTuple, dump=None):
"""Evaluate all data in data_tuple."""
quesid2ans = self.predict(eval_tuple, dump)
return eval_tuple.evaluator.evaluate(quesid2ans)
@staticmethod
def oracle_score(data_tuple):
dset, loader, evaluator = data_tuple
quesid2ans = {}
for i, (ques_id, feats, boxes, sent, target) in enumerate(loader):
_, label = target.max(1)
for qid, l in zip(ques_id, label.cpu().numpy()):
ans = dset.label2ans[l]
quesid2ans[qid.item()] = ans
return evaluator.evaluate(quesid2ans)
def save(self, name):
torch.save(self.model.state_dict(),
os.path.join(self.output, "%s.pth" % name))
def load(self, path):
print("Load model from %s" % path)
state_dict = torch.load("%s.pth" % path)
self.model.load_state_dict(state_dict)
def __init__(self,
args,
train_loader=None,
val_loader=None,
logger=None,
num_answers=0,
train=True):
self.args = args
self.max_text_length = args.max_text_length
self.train_loader = train_loader
self.val_loader = val_loader
self.num_answers = num_answers
self.logger = logger
# Model
self.model = VQAModel.from_pretrained("bert-base-uncased",
args=args,
num_answers=self.num_answers)
self.verbose = True
if self.args.distributed:
if self.args.gpu != 0:
self.verbose = False
# Load Checkpoint
self.start_epoch = None
if args.load is not None:
path = args.load + '.pth'
self.load(path, verbose=self.verbose)
elif args.load_lxmert_qa is not None:
path = args.load_lxmert_qa + '_LXRT.pth'
load_lxmert_qa(
args,
path,
self.model,
label2ans=self.train_loader.dataset.raw_dataset.label2ans,
verbose=self.verbose)
# GPU Options
print(f'Model Launching at GPU {self.args.gpu}')
from time import time
start = time()
self.model.cuda(args.gpu)
# Optimizer
if train:
self.optim, self.lr_scheduler = self.create_optimizer_and_scheduler(
)
self.bce_loss = nn.BCEWithLogitsLoss()
if args.multiGPU:
assert args.distributed
self.model = DDP(self.model,
device_ids=[args.gpu],
find_unused_parameters=True)
if args.gpu == 0:
print(f'It took {time() - start:.1f}s')
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)