def __init__(self):
self.train_tuple = get_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
if args.valid != "":
valid_bsize = 2048 if args.multiGPU else 512
self.valid_tuple = get_tuple(args.valid,
bs=valid_bsize,
shuffle=False,
drop_last=False,
skip_semantics=True)
else:
self.valid_tuple = None
self.model = GQAModel(self.train_tuple.dataset.num_answers)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Losses and optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
self.mce_loss = nn.CrossEntropyLoss(ignore_index=-1)
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
if args.task_nsp_qfpm or args.task_mlm_qfpm:
self.task_optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(list(self.model.parameters()), args.lr)
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def __init__(self, folder="/", load=True):
# Datasets
if load:
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True,
folder=folder)
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=128,
shuffle=False,
drop_last=False,
folder=folder,
nops=args.nops)
else:
self.valid_tuple = None
# Model
# self.model = VQAModel(self.train_tuple.dataset.num_answers)
is_cp = False
if "vqacpv2" in folder:
is_cp = True
if not is_cp:
self.model = VQAModel(3129)
else:
self.model = VQAModel(16039)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if load:
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def __init__(self):
# Datasets
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
if args.bert_type == 'ft':
bs_infer = 256
else:
bs_infer = 1024
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=bs_infer,
shuffle=False,
drop_last=False)
else:
self.valid_tuple = None
print("args.lr is {0}".format(args.lr))
# Model
self.model = VQAModel(self.train_tuple.dataset.num_answers)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
# if type(args.lr) == type("sdfg"):
# args.lr = float(args.lr)
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total,
schedule=args.lr_schedule,
args=args)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def train(self, train_tuple: DataTuple, eval_tuple: DataTuple):
train_ld = train_tuple.loader
# Optimizer
from lxrt.optimization import BertAdam
batch_per_epoch = len(train_ld)
t_total = int(batch_per_epoch * args.epochs)
warmup_ratio = 0.05
warmup_iters = int(t_total * warmup_ratio)
print("Batch per epoch: %d" % batch_per_epoch)
print("Total Iters: %d" % t_total)
print("Warm up Iters: %d" % warmup_iters)
optim = BertAdam(
self.model.parameters(), lr=args.lr, warmup=warmup_ratio, t_total=t_total
)
# Train
best_eval_loss = 9595.0
for epoch in range(args.epochs):
# Train
self.model.train()
total_loss = 0.0
total_losses = 0.0
uid2ans = {}
for batch in tqdm(train_ld, total=len(train_ld)):
loss, losses, logit = self.train_batch(optim, batch)
total_loss += loss
total_losses += losses
if args.task_qa:
score, label = logit.max(1)
for datum, l in zip(batch, label.cpu().numpy()):
uid = datum.uid
ans = train_tuple.dataset.answer_table.id2ans(l)
uid2ans[uid] = ans
print(epoch, total_loss, batch_per_epoch)
print(
"The training loss for Epoch %d is %0.4f"
% (epoch, total_loss / batch_per_epoch)
)
losses_str = str(total_losses)
# for name, loss in zip(LOSSES_NAME, total_losses):
# losses_str += "%s: %0.4f " % (name, loss / batch_per_epoch)
print(losses_str)
if args.task_qa:
train_tuple.evaluator.evaluate(uid2ans, pprint=True)
# Eval
avg_eval_loss = self.evaluate_epoch(eval_tuple, iters=-1)
# Save
if avg_eval_loss < best_eval_loss:
best_eval_loss = avg_eval_loss
self.save("BEST_EVAL_LOSS")
self.save("Epoch%02d" % (epoch + 1))
def setup_optimizers(self):
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(list(self.model.parameters()), args.lr)
def __init__(self):
# Datasets
self.train_tuple = get_data_tuple(
args.train, bs=args.batch_size, shuffle=True, drop_last=True
)
if args.valid != "":
self.valid_tuple = get_data_tuple(
args.valid, bs=1024,
shuffle=False, drop_last=False
)
else:
self.valid_tuple = None
# Model
self.momentum = 0.99995
self.model = VQAModel(self.train_tuple.dataset.num_answers)
self.siam_model = copy.deepcopy(self.model)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
self.siam_model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa, self.model,
label2ans=self.train_tuple.dataset.label2ans)
load_lxmert_qa(args.load_lxmert_qa, self.siam_model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
self.siam_model = self.siam_model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
self.siam_model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def __init__(self,folder="/",load=True):
# Datasets
if load:
self.train_tuple = get_data_tuple(
args.train, bs=args.batch_size, shuffle=True, drop_last=True,folder=folder
)
if args.valid != "":
self.valid_tuple = get_data_tuple(
args.valid, bs=128,
shuffle=False, drop_last=False, folder=folder,nops=args.nops
)
else:
self.valid_tuple = None
get_bias(self.train_tuple.dataset,self.valid_tuple.dataset)
# Model
# self.model = VQAModel(self.train_tuple.dataset.num_answers)
label2ans = json.load(open("/data/datasets/vqa_mutant/data/vqa/mutant_l2a/mutant_label2ans.json"))
self.model = VQAModel(len(label2ans))
self.debias = LearnedMixin(w=0.36,hid_dim=self.model.lxrt_encoder.dim)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa, self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
self.debias = self.debias.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if load :
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def __init__(self):
if args.train_json != "-1":
self.train_tuple = get_tuple(
args.train_json, bs=args.batch_size, shuffle=True, drop_last=True
)
else:
self.train_tuple = None
if args.valid_json != "-1":
valid_bsize = 2048 if args.multiGPU else 512
self.valid_tuple = get_tuple(
args.valid_json, bs=valid_bsize,
shuffle=False, drop_last=False
)
else:
self.valid_tuple = None
n_answers = len(json.load(open(args.ans2label)))
self.model = ClassifierModel(n_answers, model_type=args.model_type)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Losses and optimizer, only if training
if args.train_json != "-1":
self.bce_loss = nn.BCEWithLogitsLoss()
self.mce_loss = nn.CrossEntropyLoss(ignore_index=-1)
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(list(self.model.parameters()), args.lr)
self.output = args.output_dir
self.best_name = None
os.makedirs(self.output, exist_ok=True)
def prepare_model(self):
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
def __init__(self):
self.train_tuple = get_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
if args.valid != "":
valid_bsize = 2048 if args.multiGPU else 512
self.valid_tuple = get_tuple(args.valid,
bs=valid_bsize,
shuffle=False,
drop_last=False)
else:
self.valid_tuple = None
self.model = NLVR2Model()
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
# GPU options
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
self.model = self.model.cuda()
# Losses and optimizer
self.mce_loss = nn.CrossEntropyLoss(ignore_index=-1)
if "bert" in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(list(self.model.parameters()), args.lr)
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def __init__(self, folder="/", load=True):
# Datasets
if load:
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True,
folder=folder)
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=512,
shuffle=False,
drop_last=False,
folder=folder)
else:
self.valid_tuple = None
# Model
# self.model = VQAModel(self.train_tuple.dataset.num_answers)
self.model = VQAModel(len(self.train_tuple.dataset.label2ans),
fn_type=args.fn_type)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Load IndexList of Answer to Type Map
self.indexlist = json.load(
open(
"/data/datasets/vqa_mutant/data/vqa/mutant_l2a/mutant_merge_indexlist.json"
))
print("Length of Masks", len(self.indexlist), flush=True)
indextensor = torch.cuda.LongTensor(self.indexlist)
self.mask0 = torch.eq(indextensor, 0).float()
self.mask1 = torch.eq(indextensor, 1).float()
self.mask2 = torch.eq(indextensor, 2).float()
self.mask3 = torch.eq(indextensor, 3).float()
self.mask_cache = {}
# Loss and Optimizer
self.logsoftmax = nn.LogSoftmax()
self.sigmoid = nn.Sigmoid()
self.softmax = nn.Softmax()
self.bceloss = nn.BCELoss()
self.nllloss = nn.NLLLoss()
self.bce_loss = nn.BCEWithLogitsLoss()
self.ce_loss = nn.CrossEntropyLoss()
if load:
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def train(self, train_tuple: DataTuple, eval_tuple: DataTuple):
train_ld = train_tuple.loader
# Optimizer
from lxrt.optimization import BertAdam
batch_per_epoch = len(train_ld)
t_total = int(batch_per_epoch * args.epochs)
warmup_ratio = 0.05
warmup_iters = int(t_total * warmup_ratio)
print("Batch per epoch: %d" % batch_per_epoch)
print("Total Iters: %d" % t_total)
print("Warm up Iters: %d" % warmup_iters)
optim = BertAdam(self.model.parameters(),
lr=args.lr,
warmup=warmup_ratio,
t_total=t_total)
start_epoch = 0
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
self.model, optim = amp.initialize(self.model,
optim,
opt_level='O1')
# GPU Options
if args.multiGPU:
self.model = nn.DataParallel(self.model)
if args.start_from > 0 and args.pretraining_index == 0:
start_path = os.path.join(
args.output,
"Epoch%s_LXRT.pth" % format(int(args.start_from), '02'))
print('start training from {0}'.format(start_path))
state = torch.load(start_path)
self.model.load_state_dict(state['state_dict'])
optim.load_state_dict(state['optimizer'], strict=False)
start_epoch = args.start_from
del state
torch.cuda.empty_cache()
elif args.start_from > 0 and args.pretraining_index == 1:
start_path = os.path.join(
args.output,
"Epoch%s_LXRT.pth" % format(int(args.start_from), '02'))
print('start training from {0}'.format(start_path))
state = torch.load(start_path)
self.model.load_state_dict(state['state_dict'], strict=False)
del state
torch.cuda.empty_cache()
# Train
best_eval_loss = 9595.
for epoch in range(start_epoch, args.epochs):
# Train
self.model.train()
total_loss = 0.
total_losses = 0.
uid2ans = {}
for batch in tqdm(train_ld, total=len(train_ld)):
loss, losses, logit = self.train_batch(optim, batch)
total_loss += loss
total_losses += losses
if args.task_qa:
score, label = logit.max(1)
for datum, l in zip(batch, label.cpu().numpy()):
uid = datum.uid
ans = train_tuple.dataset.answer_table.id2ans(l)
uid2ans[uid] = ans
print("The training loss for Epoch %d is %0.4f" %
(epoch, total_loss / batch_per_epoch))
log_str = "\nThe training loss for Epoch %d is %0.4f" % (
epoch, total_loss / batch_per_epoch)
losses_str = "\nThe losses are "
log_str += "\nThe losses are "
for name, loss in zip(LOSSES_NAME, total_losses):
losses_str += "%s: %0.4f " % (name, loss / batch_per_epoch)
log_str += "\n %s: %0.4f " % (name, loss / batch_per_epoch)
print(losses_str)
with open(self.output + "/log.log", 'a') as f:
f.write(log_str)
f.flush()
if args.task_qa:
train_tuple.evaluator.evaluate(uid2ans, pprint=True)
# Eval
avg_eval_loss = self.evaluate_epoch(eval_tuple, iters=-1)
state = {
'state_dict': self.model.state_dict(),
'optimizer': optim.state_dict(),
}
# Save
if avg_eval_loss < best_eval_loss:
best_eval_loss = avg_eval_loss
self.save("BEST_EVAL_LOSS", state)
if args.pretraining_index == 0:
self.save("Epoch%02d" % (epoch + 1), state)
elif args.pretraining_index == 1:
self.save("Epoch%02d" % (epoch + 1 + args.start_from), state)
def __init__(self):
self.train_tuple = get_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
if args.valid != "":
valid_bsize = 2048 if args.multiGPU else 512
self.valid_tuple = get_tuple(args.valid,
bs=valid_bsize,
shuffle=False,
drop_last=False)
else:
self.valid_tuple = None
self.model = GQAModel(self.train_tuple.dataset.num_answers)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
self.new_ans_label = load_lxmert_qa(
args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Losses and optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
self.mce_loss = nn.CrossEntropyLoss(ignore_index=-1)
# self.KL_loss = nn.KLDivLoss(reduction='none')
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(list(self.model.parameters()), args.lr)
self.output = args.output
os.makedirs(self.output, exist_ok=True)
# Tensorboard
self.boards_dir = os.path.join('boards', self.output)
if not os.path.exists(self.boards_dir):
os.makedirs(self.boards_dir)
self.writerTbrd = SummaryWriter(self.boards_dir)
# get Glove projection for all answers
if args.answer_loss == 'glove':
path_glove = './data/GloVe/GloVeDict.pkl'
with open(path_glove, 'rb') as f:
glove_dic = pickle.load(f)
glove_dim = glove_dic['the'].shape[-1]
print("Loading Glove%d answer's vector" % glove_dim)
self.labelans2glove = []
self.valid_ans_embed = [1] * len(
self.train_tuple.dataset.label2ans)
for label, ans in enumerate(self.train_tuple.dataset.label2ans):
ans = ans.split(' ')
glove_ans = []
for w in ans:
#print(w)
try:
glove_ans.append(glove_dic[w])
except KeyError:
#print('Full ans: %s' % ans)
#input(' ')
self.valid_ans_embed[label] = 0
glove_ans.append(np.zeros(glove_dim))
#print(glove_ans)
glove_ans = torch.tensor(glove_ans).mean(-2)
self.labelans2glove.append(torch.tensor(glove_ans))
#print(self.labelans2glove)
print(
'Ratio of valid ans embedding: %f' %
(float(sum(self.valid_ans_embed)) / len(self.valid_ans_embed)))
self.labelans2glove = torch.stack(
self.labelans2glove).float().cuda()
self.cosineSim = torch.nn.CosineSimilarity(dim=1, eps=1e-08)
def finetune(self, train_tuple, eval_tuple):
# log
output_1 = os.path.join(self.output, 'finetune_1')
os.makedirs(output_1, exist_ok=True)
output_2 = os.path.join(self.output, 'finetune_2')
os.makedirs(output_2, exist_ok=True)
# Tensorboard
boards_dir_1 = os.path.join(self.boards_dir, 'finetune_1')
if not os.path.exists(boards_dir_1):
os.makedirs(boards_dir_1)
boards_dir_2 = os.path.join(self.boards_dir, 'finetune_2')
if not os.path.exists(boards_dir_2):
os.makedirs(boards_dir_2)
# Params
lr_1 = args.lr
lr_2 = args.lr / 10
epochs_1 = 4 #int(args.epochs / 3)
epochs_2 = args.epochs - epochs_1
# Step 0: evaluate pretraining
if self.valid_tuple is not None: # Do Validation
valid_score = self.evaluate(eval_tuple)
print("Before finetune: Valid %0.2f\n" % (valid_score * 100.))
# Step 0.1: finetune new ans only
# new_ans_params = []
# for name, p in self.model.named_parameters():
# if "logit_fc.3" in name:
# for idx in range(p.size(0)):
# if idx in self.new_ans_label:
# new_ans_params.append({'params': p[idx]})
# args.epochs = epochs_0
# from lxrt.optimization import BertAdam
# self.optim = BertAdam(new_ans_params,
# lr=lr_1,
# warmup=0.0,
# t_total=-1)
# print('### Start finetuning new ans...')
# self.train(train_tuple, eval_tuple)
# First step, only updates answer head
#self.optim = torch.optim.Adamax(list(self.model.parameters()), lr_1)
#self.optim = torch.optim.SGD(list(self.model.parameters()), lr_1)
args.epochs = epochs_1
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * epochs_1)
print("Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(
list(self.model.parameters()),
lr=lr_1,
warmup=0.0, #!0.034
t_total=-1)
# loaded_optim = torch.load("%s_LXRT.pth" % args.load_lxmert_qa)['optimizer']
# self.optim.load_state_dict(loaded_optim)
# for group in loaded_optim.param_groups:
# for p in group['params']:
# if p in loaded_optim['state']:
# self.optim.state[p] = loaded_optim.state[p]
self.writerTbrd = SummaryWriter(boards_dir_1)
self.output = output_1
for name, p in self.model.named_parameters():
if "logit_fc" in name:
p.requires_grad = True
else:
p.requires_grad = False
print('### Start finetuning step 1...')
self.train(train_tuple, eval_tuple)
# Second step, finetune all
for name, p in self.model.named_parameters():
p.requires_grad = True
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * epochs_2)
print("Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=lr_2,
warmup=0.1,
t_total=t_total,
lr_min=1e-7)
else:
self.optim = args.optimizer(list(self.model.parameters()), lr_2)
args.epochs = epochs_2
self.writerTbrd = SummaryWriter(boards_dir_2)
self.output = output_2
print('### Start finetuning step 2...')
self.train(train_tuple, eval_tuple)
def __init__(self, attention=False):
# Datasets
print("Fetching data")
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True,
dataset_name="test")
print("Got data")
print("fetching val data")
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=args.batch_size,
shuffle=False,
drop_last=False,
dataset_name="test")
print("got data")
else:
self.valid_tuple = None
print("Got data")
# Model
print("Making model")
self.model = VQAModel(self.train_tuple.dataset.num_answers, attention)
print("Ready model")
# Print model info:
print("Num of answers:")
print(self.train_tuple.dataset.num_answers)
# print("Model info:")
# print(self.model)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def train(self): #, eval_tuple: DataTuple):
# train_ld = train_tuple.loader
# Optimizer
from lxrt.optimization import BertAdam
batch_per_epoch = len(train_dset) // args.batch_size + bool(
len(train_dset) % args.batch_size)
t_total = int(batch_per_epoch * args.epochs)
warmup_ratio = 0.05
warmup_iters = int(t_total * warmup_ratio)
print("Batch per epoch: %d" % batch_per_epoch)
print("Total Iters: %d" % t_total)
print("Warm up Iters: %d" % warmup_iters)
optim = BertAdam(self.model.parameters(),
lr=args.lr,
warmup=warmup_ratio,
t_total=t_total)
# Train
best_eval_loss = 9595.
for epoch in range(args.epochs):
# Train
self.model.train()
total_loss = 0.
total_losses = 0.
uid2ans = {}
# for training subset (shuffle order across epochs)
for i in tqdm(range(NUM_MINISETS), total=NUM_MINISETS):
# create dataloader train_ld for this subset
dtuple = get_tuple(args.train + '_%d' % i,
args.batch_size,
shuffle=True,
drop_last=True)
dl = dtuple.loader
for batch in tqdm(dl, total=len(dl)):
loss, losses, logit = self.train_batch(
optim, batch, dtuple.torchdset)
total_loss += loss
total_losses += losses
if args.task_qa:
score, label = logit.max(1)
for datum, l in zip(batch, label.cpu().numpy()):
uid = datum.uid
ans = dtuple.dataset.answer_table.id2ans(l)
uid2ans[uid] = ans
print("The training loss for Epoch %d is %0.4f" %
(epoch, total_loss / batch_per_epoch))
losses_str = "The losses are "
for name, loss in zip(LOSSES_NAME, total_losses):
losses_str += "%s: %0.4f " % (name, loss / batch_per_epoch)
print(losses_str)
if args.task_qa:
dtuple.evaluator.evaluate(uid2ans, pprint=True)
# Eval
dtuple = get_tuple(args.valid,
valid_batch_size,
shuffle=False,
drop_last=False) #, topk=5000)
avg_eval_loss = self.evaluate_epoch(dtuple, iters=-1)
# Save
if avg_eval_loss < best_eval_loss:
best_eval_loss = avg_eval_loss
self.save("BEST_EVAL_LOSS")
self.save("Epoch%02d" % (epoch + 1))
def __init__(self, folder="/", load=True):
# Datasets
if load:
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True,
folder=folder)
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=128,
shuffle=False,
drop_last=False,
folder=folder,
nops=args.nops)
else:
self.valid_tuple = None
# Model
self.model = VQAModel(self.train_tuple.dataset.num_answers)
# is_cp=False
# if "vqacpv2" in folder:
# is_cp=True
# if not is_cp:
# self.model = VQAModel(3129)
# else:
# self.model = VQAModel(16039)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
ans_embed = np.load(
"/data/datasets/vqa_mutant/data/vqa/mutant_l2a/answer_embs.npy"
) + 1e-8
ans_embed = torch.tensor(ans_embed).cuda()
self.ans_embed = torch.nn.functional.normalize(ans_embed, dim=1)
self.embed_cache = {}
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if load:
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
self.cos = nn.CosineSimilarity()
def train(self, train_tuple: DataTuple, eval_tuple: DataTuple):
train_ld = train_tuple.loader
# Optimizer
from lxrt.optimization import BertAdam
batch_per_epoch = len(train_ld)
t_total = int(batch_per_epoch * args.epochs)
warmup_ratio = args.get("warmup_ratio", 0.05)
print("Total Iters: %d" % t_total)
if args.get("t_total", None):
t_total = args.t_total
print("!! Changing to specified t_toal in args: {}".format(t_total))
self.t_total = t_total
warmup_iters = int(t_total * warmup_ratio)
print("Batch per epoch: %d" % batch_per_epoch)
print("Warm up Iters: %d" % warmup_iters)
self.optim = BertAdam(self.model.parameters(), lr=args.lr, warmup=warmup_ratio, t_total=t_total)
if args.load is not None:
self.load(args.load, t_total = t_total)
gradient_accumulation_steps = args.get("gradient_accumulation_steps", 1)
# Train
best_eval_loss = 9595.
report_every = args.get("report_every", 100)
custom_train_meter = TrainingMeter()
for epoch in range(args.epochs):
# Train
self.model.train()
total_loss = 0.
total_losses = 0.
uid2ans = {}
for batch_id, batch in enumerate(tqdm(train_ld, total=len(train_ld))):
if args.get("skip_training", False):
break
loss, losses, logit, losses_dict = self.train_batch(self.optim, batch)
total_loss += loss
try:
total_losses += losses
except:
pass
if args.task_qa and batch[0].sent is not None:
assert(0) # Not used in our experiment
score, label = logit.max(1)
for datum, l in zip(batch, label.cpu().numpy()):
uid = datum.uid
ans = train_tuple.dataset.answer_table.id2ans(l)
uid2ans[uid] = ans
for key, value in losses_dict.items():
losses_dict[key] = value.mean().item() # make the losses scalar
if "Masked LM" in losses_dict and losses_dict["Masked LM"] == 0:
del losses_dict["Masked LM"]
custom_train_meter.update(losses_dict)
if batch_id % report_every == 0 and batch_id > 0:
print("Folder: {} \n Epoch {} Iter: {}/{}".format(args.output, epoch, batch_id, len(train_ld)))
#print(pd.DataFrame(train_results[-report_every:]).mean())
custom_train_meter.report()
custom_train_meter.clean()
print()
if args.get("save_step", -1) != -1 and self.global_step != 0 and (self.global_step // gradient_accumulation_steps) % args.save_step == 0:
self.save("Step{}".format(self.global_step))
self.global_step += 1
print("The training loss for Epoch %d is %0.4f" % (epoch, total_loss / batch_per_epoch))
if args.task_qa:
train_tuple.evaluator.evaluate(uid2ans, pprint=True)
# Eval
avg_eval_loss = self.evaluate_epoch(eval_tuple, iters=-1)
if args.get("eval_on_train", False):
print("On train set")
self.evaluate_epoch(train_tuple, iters=-1)
if avg_eval_loss < best_eval_loss:
best_eval_loss = avg_eval_loss
self.save("BEST_EVAL_LOSS")
self.save("Epoch%02d" % (epoch+1))